@hasna/skills 0.0.2

package/skills/skill-convert/src/converters/ai.ts

@@ -0,0 +1,375 @@
+/**
+ * AI-powered conversion utilities
+ * Uses Claude/GPT-4o for OCR, extraction, and intelligent conversions
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+import OpenAI from 'openai';
+import { readFile, writeFile, stat } from 'fs/promises';
+import { extname, basename, dirname, join } from 'path';
+import type { AIModel, ConvertOptions, ConvertResult } from '../types';
+import { MAX_SINGLE_PASS_SIZE, DEFAULT_CHUNK_SIZE } from '../types';
+import { extractPdfText, splitPdfIntoChunks, getPdfPageCount } from './pdf';
+
+// Initialize clients lazily
+let anthropicClient: Anthropic | null = null;
+let openaiClient: OpenAI | null = null;
+
+function getAnthropicClient(): Anthropic {
+  if (!anthropicClient) {
+    anthropicClient = new Anthropic();
+  }
+  return anthropicClient;
+}
+
+function getOpenAIClient(): OpenAI {
+  if (!openaiClient) {
+    openaiClient = new OpenAI();
+  }
+  return openaiClient;
+}
+
+/**
+ * Convert image to markdown using AI vision
+ */
+export async function imageToMarkdownAI(
+  imagePath: string,
+  model: AIModel = 'claude',
+  clean: boolean = false
+): Promise<string> {
+  const imageBuffer = await readFile(imagePath);
+  const base64Image = imageBuffer.toString('base64');
+  const ext = extname(imagePath).slice(1).toLowerCase();
+  const mediaType = ext === 'png' ? 'image/png' :
+                    ext === 'gif' ? 'image/gif' :
+                    ext === 'webp' ? 'image/webp' : 'image/jpeg';
+
+  const cleanPrompt = clean
+    ? ' Clean up and format the content properly, fixing any OCR errors or formatting issues.'
+    : '';
+
+  if (model === 'claude') {
+    const response = await getAnthropicClient().messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 8192,
+      messages: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'image',
+              source: {
+                type: 'base64',
+                media_type: mediaType,
+                data: base64Image,
+              },
+            },
+            {
+              type: 'text',
+              text: `Extract all text and content from this image and convert it to well-formatted Markdown.${cleanPrompt} Preserve the structure, headings, lists, tables, and any other formatting visible in the image. Return only the Markdown content, no explanations.`,
+            },
+          ],
+        },
+      ],
+    });
+
+    const textContent = response.content.find((c) => c.type === 'text');
+    return textContent ? textContent.text : '';
+  } else {
+    // OpenAI GPT-4o
+    const response = await getOpenAIClient().chat.completions.create({
+      model: model === 'gpt-4o-mini' ? 'gpt-4o-mini' : 'gpt-4o',
+      max_tokens: 8192,
+      messages: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'image_url',
+              image_url: {
+                url: `data:${mediaType};base64,${base64Image}`,
+              },
+            },
+            {
+              type: 'text',
+              text: `Extract all text and content from this image and convert it to well-formatted Markdown.${cleanPrompt} Preserve the structure, headings, lists, tables, and any other formatting visible in the image. Return only the Markdown content, no explanations.`,
+            },
+          ],
+        },
+      ],
+    });
+
+    return response.choices[0]?.message?.content || '';
+  }
+}
+
+/**
+ * Convert image to JSON (structured data extraction) using AI
+ */
+export async function imageToJsonAI(
+  imagePath: string,
+  model: AIModel = 'claude'
+): Promise<string> {
+  const imageBuffer = await readFile(imagePath);
+  const base64Image = imageBuffer.toString('base64');
+  const ext = extname(imagePath).slice(1).toLowerCase();
+  const mediaType = ext === 'png' ? 'image/png' :
+                    ext === 'gif' ? 'image/gif' :
+                    ext === 'webp' ? 'image/webp' : 'image/jpeg';
+
+  if (model === 'claude') {
+    const response = await getAnthropicClient().messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 8192,
+      messages: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'image',
+              source: {
+                type: 'base64',
+                media_type: mediaType,
+                data: base64Image,
+              },
+            },
+            {
+              type: 'text',
+              text: 'Extract all structured data from this image and return it as a well-organized JSON object. Include any tables, lists, key-value pairs, or structured information visible in the image. Return only valid JSON, no explanations.',
+            },
+          ],
+        },
+      ],
+    });
+
+    const textContent = response.content.find((c) => c.type === 'text');
+    return textContent ? textContent.text : '{}';
+  } else {
+    const response = await getOpenAIClient().chat.completions.create({
+      model: model === 'gpt-4o-mini' ? 'gpt-4o-mini' : 'gpt-4o',
+      max_tokens: 8192,
+      response_format: { type: 'json_object' },
+      messages: [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'image_url',
+              image_url: {
+                url: `data:${mediaType};base64,${base64Image}`,
+              },
+            },
+            {
+              type: 'text',
+              text: 'Extract all structured data from this image and return it as a well-organized JSON object. Include any tables, lists, key-value pairs, or structured information visible in the image.',
+            },
+          ],
+        },
+      ],
+    });
+
+    return response.choices[0]?.message?.content || '{}';
+  }
+}
+
+/**
+ * Convert PDF to Markdown using AI (with chunking for large files)
+ */
+export async function pdfToMarkdownAI(
+  pdfPath: string,
+  model: AIModel = 'claude',
+  clean: boolean = false,
+  chunkSize: number = DEFAULT_CHUNK_SIZE
+): Promise<{ content: string; chunksProcessed: number }> {
+  const fileStat = await stat(pdfPath);
+  const pageCount = await getPdfPageCount(pdfPath);
+
+  const cleanPrompt = clean
+    ? ' Clean up and format the content properly, fixing any OCR errors, formatting issues, or extraction artifacts. Remove page numbers, headers/footers if they repeat, and organize the content logically.'
+    : '';
+
+  // For small files, process directly
+  if (fileStat.size <= MAX_SINGLE_PASS_SIZE) {
+    const text = await extractPdfText(pdfPath);
+
+    if (model === 'claude') {
+      const response = await getAnthropicClient().messages.create({
+        model: 'claude-sonnet-4-20250514',
+        max_tokens: 8192,
+        messages: [
+          {
+            role: 'user',
+            content: `Convert the following PDF text content to well-formatted Markdown.${cleanPrompt} Preserve the structure including headings, lists, tables, and paragraphs.\n\n${text}`,
+          },
+        ],
+      });
+
+      const textContent = response.content.find((c) => c.type === 'text');
+      return { content: textContent?.text || '', chunksProcessed: 1 };
+    } else {
+      const response = await getOpenAIClient().chat.completions.create({
+        model: model === 'gpt-4o-mini' ? 'gpt-4o-mini' : 'gpt-4o',
+        max_tokens: 8192,
+        messages: [
+          {
+            role: 'user',
+            content: `Convert the following PDF text content to well-formatted Markdown.${cleanPrompt} Preserve the structure including headings, lists, tables, and paragraphs.\n\n${text}`,
+          },
+        ],
+      });
+
+      return { content: response.choices[0]?.message?.content || '', chunksProcessed: 1 };
+    }
+  }
+
+  // For large files, split into chunks
+  const pagesPerChunk = Math.ceil(chunkSize / (fileStat.size / pageCount));
+  const chunks = await splitPdfIntoChunks(pdfPath, pagesPerChunk);
+  const results: string[] = [];
+
+  for (let i = 0; i < chunks.length; i++) {
+    // Extract text from chunk
+    const pdfParse = await import('pdf-parse');
+    const data = await pdfParse.default(chunks[i]);
+    const text = data.text;
+
+    const chunkPrompt = `Convert the following PDF text content (part ${i + 1} of ${chunks.length}) to well-formatted Markdown.${cleanPrompt} Preserve the structure including headings, lists, tables, and paragraphs.\n\n${text}`;
+
+    if (model === 'claude') {
+      const response = await getAnthropicClient().messages.create({
+        model: 'claude-sonnet-4-20250514',
+        max_tokens: 8192,
+        messages: [{ role: 'user', content: chunkPrompt }],
+      });
+
+      const textContent = response.content.find((c) => c.type === 'text');
+      results.push(textContent?.text || '');
+    } else {
+      const response = await getOpenAIClient().chat.completions.create({
+        model: model === 'gpt-4o-mini' ? 'gpt-4o-mini' : 'gpt-4o',
+        max_tokens: 8192,
+        messages: [{ role: 'user', content: chunkPrompt }],
+      });
+
+      results.push(response.choices[0]?.message?.content || '');
+    }
+  }
+
+  return { content: results.join('\n\n---\n\n'), chunksProcessed: chunks.length };
+}
+
+/**
+ * Clean/sanitize text content using AI
+ */
+export async function cleanTextAI(
+  text: string,
+  model: AIModel = 'claude'
+): Promise<string> {
+  if (model === 'claude') {
+    const response = await getAnthropicClient().messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 8192,
+      messages: [
+        {
+          role: 'user',
+          content: `Clean and sanitize the following text. Fix any typos, formatting issues, and improve readability while preserving the original meaning and structure. Return only the cleaned text.\n\n${text}`,
+        },
+      ],
+    });
+
+    const textContent = response.content.find((c) => c.type === 'text');
+    return textContent?.text || text;
+  } else {
+    const response = await getOpenAIClient().chat.completions.create({
+      model: model === 'gpt-4o-mini' ? 'gpt-4o-mini' : 'gpt-4o',
+      max_tokens: 8192,
+      messages: [
+        {
+          role: 'user',
+          content: `Clean and sanitize the following text. Fix any typos, formatting issues, and improve readability while preserving the original meaning and structure. Return only the cleaned text.\n\n${text}`,
+        },
+      ],
+    });
+
+    return response.choices[0]?.message?.content || text;
+  }
+}
+
+/**
+ * AI-powered conversion
+ */
+export async function convertWithAI(options: ConvertOptions): Promise<ConvertResult> {
+  const startTime = Date.now();
+  const inputStat = await stat(options.input);
+  const inputExt = extname(options.input).slice(1).toLowerCase();
+  const model = options.model || 'claude';
+
+  const outputPath = options.output || join(
+    dirname(options.input),
+    `${basename(options.input, extname(options.input))}.${options.format}`
+  );
+
+  try {
+    let content: string;
+    let chunksProcessed = 1;
+
+    // Image to Markdown/Text
+    if (['png', 'jpg', 'jpeg', 'webp', 'gif', 'tiff', 'bmp'].includes(inputExt)) {
+      if (options.format === 'json') {
+        content = await imageToJsonAI(options.input, model);
+      } else {
+        content = await imageToMarkdownAI(options.input, model, options.clean);
+      }
+    }
+    // PDF to Markdown
+    else if (inputExt === 'pdf' && (options.format === 'md' || options.format === 'markdown')) {
+      const result = await pdfToMarkdownAI(
+        options.input,
+        model,
+        options.clean,
+        options.chunkSize
+      );
+      content = result.content;
+      chunksProcessed = result.chunksProcessed;
+    }
+    // Document to Markdown with cleaning
+    else if (['docx', 'doc'].includes(inputExt) && options.clean) {
+      const mammoth = await import('mammoth');
+      const buffer = await readFile(options.input);
+      const result = await mammoth.extractRawText({ buffer });
+      content = await cleanTextAI(result.value, model);
+    }
+    else {
+      throw new Error(`AI conversion not supported for ${inputExt} to ${options.format}`);
+    }
+
+    await writeFile(outputPath, content);
+    const outputStat = await stat(outputPath);
+
+    return {
+      success: true,
+      input: options.input,
+      output: outputPath,
+      inputFormat: inputExt,
+      outputFormat: options.format,
+      inputSize: inputStat.size,
+      outputSize: outputStat.size,
+      chunksProcessed,
+      aiProcessed: true,
+      duration: Date.now() - startTime,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      input: options.input,
+      output: outputPath,
+      inputFormat: inputExt,
+      outputFormat: options.format,
+      inputSize: inputStat.size,
+      outputSize: 0,
+      aiProcessed: true,
+      error: error instanceof Error ? error.message : String(error),
+      duration: Date.now() - startTime,
+    };
+  }
+}

package/skills/skill-convert/src/converters/data.ts

@@ -0,0 +1,339 @@
+/**
+ * Data format conversion utilities (CSV, Excel, JSON, YAML)
+ */
+
+import ExcelJS from 'exceljs';
+import { parse as csvParse } from 'csv-parse/sync';
+import { stringify as csvStringify } from 'csv-stringify/sync';
+import { readFile, writeFile, stat } from 'fs/promises';
+import { extname, basename, dirname, join } from 'path';
+import * as yaml from 'yaml';
+import type { ConvertOptions, ConvertResult } from '../types';
+
+/**
+ * Parse CSV file to array of objects
+ */
+export async function parseCsv(filePath: string): Promise<Record<string, string>[]> {
+  const content = await readFile(filePath, 'utf-8');
+  return csvParse(content, {
+    columns: true,
+    skip_empty_lines: true,
+    trim: true,
+  });
+}
+
+/**
+ * Parse TSV file to array of objects
+ */
+export async function parseTsv(filePath: string): Promise<Record<string, string>[]> {
+  const content = await readFile(filePath, 'utf-8');
+  return csvParse(content, {
+    columns: true,
+    skip_empty_lines: true,
+    trim: true,
+    delimiter: '\t',
+  });
+}
+
+/**
+ * Convert CSV to JSON
+ */
+export async function csvToJson(csvPath: string): Promise<string> {
+  const data = await parseCsv(csvPath);
+  return JSON.stringify(data, null, 2);
+}
+
+/**
+ * Convert JSON to CSV
+ */
+export async function jsonToCsv(jsonPath: string): Promise<string> {
+  const content = await readFile(jsonPath, 'utf-8');
+  const data = JSON.parse(content);
+
+  if (!Array.isArray(data)) {
+    throw new Error('JSON must be an array of objects to convert to CSV');
+  }
+
+  return csvStringify(data, {
+    header: true,
+  });
+}
+
+/**
+ * Convert CSV to Excel
+ */
+export async function csvToExcel(csvPath: string, outputPath: string): Promise<void> {
+  const data = await parseCsv(csvPath);
+  const workbook = new ExcelJS.Workbook();
+  const worksheet = workbook.addWorksheet('Sheet1');
+
+  if (data.length > 0) {
+    // Add headers
+    const headers = Object.keys(data[0]);
+    worksheet.addRow(headers);
+
+    // Style headers
+    const headerRow = worksheet.getRow(1);
+    headerRow.font = { bold: true };
+    headerRow.fill = {
+      type: 'pattern',
+      pattern: 'solid',
+      fgColor: { argb: 'FFE0E0E0' },
+    };
+
+    // Add data rows
+    data.forEach((row) => {
+      worksheet.addRow(headers.map((h) => row[h]));
+    });
+
+    // Auto-fit columns
+    worksheet.columns.forEach((column) => {
+      let maxLength = 0;
+      column.eachCell?.({ includeEmpty: true }, (cell) => {
+        const cellLength = cell.value ? String(cell.value).length : 10;
+        maxLength = Math.max(maxLength, cellLength);
+      });
+      column.width = Math.min(maxLength + 2, 50);
+    });
+  }
+
+  await workbook.xlsx.writeFile(outputPath);
+}
+
+/**
+ * Convert Excel to CSV
+ */
+export async function excelToCsv(excelPath: string): Promise<string> {
+  const workbook = new ExcelJS.Workbook();
+  await workbook.xlsx.readFile(excelPath);
+
+  const worksheet = workbook.worksheets[0];
+  if (!worksheet) {
+    throw new Error('No worksheets found in Excel file');
+  }
+
+  const rows: string[][] = [];
+  worksheet.eachRow((row, rowNumber) => {
+    const rowData: string[] = [];
+    row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
+      rowData[colNumber - 1] = cell.text || '';
+    });
+    rows.push(rowData);
+  });
+
+  return csvStringify(rows);
+}
+
+/**
+ * Convert Excel to JSON
+ */
+export async function excelToJson(excelPath: string): Promise<string> {
+  const workbook = new ExcelJS.Workbook();
+  await workbook.xlsx.readFile(excelPath);
+
+  const worksheet = workbook.worksheets[0];
+  if (!worksheet) {
+    throw new Error('No worksheets found in Excel file');
+  }
+
+  const rows: Record<string, string>[] = [];
+  let headers: string[] = [];
+
+  worksheet.eachRow((row, rowNumber) => {
+    if (rowNumber === 1) {
+      // First row is headers
+      row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
+        headers[colNumber - 1] = cell.text || `Column${colNumber}`;
+      });
+    } else {
+      const rowData: Record<string, string> = {};
+      row.eachCell({ includeEmpty: true }, (cell, colNumber) => {
+        const header = headers[colNumber - 1] || `Column${colNumber}`;
+        rowData[header] = cell.text || '';
+      });
+      rows.push(rowData);
+    }
+  });
+
+  return JSON.stringify(rows, null, 2);
+}
+
+/**
+ * Convert JSON to YAML
+ */
+export async function jsonToYaml(jsonPath: string): Promise<string> {
+  const content = await readFile(jsonPath, 'utf-8');
+  const data = JSON.parse(content);
+  return yaml.stringify(data);
+}
+
+/**
+ * Convert YAML to JSON
+ */
+export async function yamlToJson(yamlPath: string): Promise<string> {
+  const content = await readFile(yamlPath, 'utf-8');
+  const data = yaml.parse(content);
+  return JSON.stringify(data, null, 2);
+}
+
+/**
+ * Convert data format
+ */
+export async function convertData(options: ConvertOptions): Promise<ConvertResult> {
+  const startTime = Date.now();
+  const inputStat = await stat(options.input);
+  const inputExt = extname(options.input).slice(1).toLowerCase();
+
+  const outputPath = options.output || join(
+    dirname(options.input),
+    `${basename(options.input, extname(options.input))}.${options.format}`
+  );
+
+  try {
+    let content: string | undefined;
+
+    // CSV conversions
+    if (inputExt === 'csv') {
+      switch (options.format) {
+        case 'json':
+          content = await csvToJson(options.input);
+          break;
+        case 'xlsx':
+          await csvToExcel(options.input, outputPath);
+          break;
+        case 'yaml':
+          const csvData = await parseCsv(options.input);
+          content = yaml.stringify(csvData);
+          break;
+        case 'tsv':
+          const csvContent = await parseCsv(options.input);
+          content = csvStringify(csvContent, { header: true, delimiter: '\t' });
+          break;
+        default:
+          throw new Error(`Cannot convert CSV to ${options.format}`);
+      }
+    }
+    // TSV conversions
+    else if (inputExt === 'tsv') {
+      switch (options.format) {
+        case 'csv':
+          const tsvData = await parseTsv(options.input);
+          content = csvStringify(tsvData, { header: true });
+          break;
+        case 'json':
+          const tsvJson = await parseTsv(options.input);
+          content = JSON.stringify(tsvJson, null, 2);
+          break;
+        case 'xlsx':
+          // Convert TSV to CSV first, then to Excel
+          const tsvCsv = await parseTsv(options.input);
+          const tempCsv = csvStringify(tsvCsv, { header: true });
+          const tempPath = options.input + '.temp.csv';
+          await writeFile(tempPath, tempCsv);
+          await csvToExcel(tempPath, outputPath);
+          await Bun.file(tempPath).text().then(() => {}).catch(() => {});
+          break;
+        default:
+          throw new Error(`Cannot convert TSV to ${options.format}`);
+      }
+    }
+    // Excel conversions
+    else if (inputExt === 'xlsx' || inputExt === 'xls') {
+      switch (options.format) {
+        case 'csv':
+          content = await excelToCsv(options.input);
+          break;
+        case 'json':
+          content = await excelToJson(options.input);
+          break;
+        case 'yaml':
+          const xlsJson = await excelToJson(options.input);
+          content = yaml.stringify(JSON.parse(xlsJson));
+          break;
+        case 'tsv':
+          const xlsCsv = await excelToCsv(options.input);
+          // Replace commas with tabs (simple conversion)
+          const rows = csvParse(xlsCsv);
+          content = rows.map((row: string[]) => row.join('\t')).join('\n');
+          break;
+        default:
+          throw new Error(`Cannot convert Excel to ${options.format}`);
+      }
+    }
+    // JSON conversions
+    else if (inputExt === 'json') {
+      switch (options.format) {
+        case 'csv':
+          content = await jsonToCsv(options.input);
+          break;
+        case 'yaml':
+          content = await jsonToYaml(options.input);
+          break;
+        case 'xlsx':
+          // JSON to CSV to Excel
+          const jsonCsv = await jsonToCsv(options.input);
+          const tempCsvPath = options.input + '.temp.csv';
+          await writeFile(tempCsvPath, jsonCsv);
+          await csvToExcel(tempCsvPath, outputPath);
+          break;
+        default:
+          throw new Error(`Cannot convert JSON to ${options.format}`);
+      }
+    }
+    // YAML conversions
+    else if (inputExt === 'yaml' || inputExt === 'yml') {
+      switch (options.format) {
+        case 'json':
+          content = await yamlToJson(options.input);
+          break;
+        case 'csv':
+          const yamlJson = await yamlToJson(options.input);
+          const yamlData = JSON.parse(yamlJson);
+          if (!Array.isArray(yamlData)) {
+            throw new Error('YAML must contain an array to convert to CSV');
+          }
+          content = csvStringify(yamlData, { header: true });
+          break;
+        default:
+          throw new Error(`Cannot convert YAML to ${options.format}`);
+      }
+    }
+    // XML conversions (basic)
+    else if (inputExt === 'xml') {
+      throw new Error('XML conversion not yet implemented');
+    }
+    else {
+      throw new Error(`Unsupported input format: ${inputExt}`);
+    }
+
+    // Write content if not already written (Excel writes directly)
+    if (content !== undefined) {
+      await writeFile(outputPath, content);
+    }
+
+    const outputStat = await stat(outputPath);
+
+    return {
+      success: true,
+      input: options.input,
+      output: outputPath,
+      inputFormat: inputExt,
+      outputFormat: options.format,
+      inputSize: inputStat.size,
+      outputSize: outputStat.size,
+      duration: Date.now() - startTime,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      input: options.input,
+      output: outputPath,
+      inputFormat: inputExt,
+      outputFormat: options.format,
+      inputSize: inputStat.size,
+      outputSize: 0,
+      error: error instanceof Error ? error.message : String(error),
+      duration: Date.now() - startTime,
+    };
+  }
+}