@meyverick/omnicode 0.1.3 → 0.2.0

package/src/installer/mineru-client.js

@@ -0,0 +1,164 @@
+/**
+ * MinerU API Client Integration
+ * 
+ * Provides concurrent, non-blocking interaction with the MinerU API
+ * for complex document structural extraction.
+ */
+
+const API_BASE = 'https://mineru.net/api/v4';
+const MAX_RETRIES = 3;
+
+/**
+ * Helper to sleep for a given number of milliseconds
+ */
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Fetch wrapper with built-in retry logic for 5xx errors and timeouts.
+ */
+async function fetchWithRetry(url, options, retries = MAX_RETRIES) {
+    for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+            // We set an AbortController for timeout
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), 30000); // 30s timeout
+
+            const response = await fetch(url, {
+                ...options,
+                signal: controller.signal
+            });
+
+            clearTimeout(timeoutId);
+
+            // Handle quota exhaustion or invalid key explicitly (do not retry)
+            if (response.status === 401 || response.status === 402) {
+                const error = new Error(`MinerU API rejected request: HTTP ${response.status}`);
+                error.status = response.status;
+                throw error;
+            }
+
+            // If 5xx error, throw so it can be caught and retried
+            if (response.status >= 500 && response.status < 600) {
+                throw new Error(`MinerU API returned 5xx error: HTTP ${response.status}`);
+            }
+
+            if (!response.ok) {
+                throw new Error(`MinerU API error: HTTP ${response.status}`);
+            }
+
+            return response;
+        } catch (error) {
+            // If it's a 401/402, bubble it up immediately
+            if (error.status === 401 || error.status === 402) {
+                throw error;
+            }
+
+            if (attempt === retries) {
+                throw new Error(`MinerU API request failed after ${retries} attempts: ${error.message}`);
+            }
+            
+            // Exponential backoff
+            await sleep(attempt * 2000);
+        }
+    }
+}
+
+/**
+ * Submit a document to the MinerU extraction API.
+ * @param {Buffer} fileBuffer - The binary content of the file.
+ * @param {string} fileName - The name of the file (e.g. for extension extraction).
+ * @param {string} apiKey - The MINERU_API_KEY.
+ * @returns {Promise<string>} The MinerU task ID.
+ */
+export async function submitExtractionTask(fileBuffer, fileName, apiKey) {
+    const formData = new FormData();
+    // Wrap buffer in a Blob for FormData
+    const blob = new Blob([fileBuffer]);
+    formData.append('file', blob, fileName);
+    formData.append('language', 'en');
+    formData.append('is_ocr', 'true');
+
+    const response = await fetchWithRetry(`${API_BASE}/extract/task`, {
+        method: 'POST',
+        headers: {
+            'Authorization': `Bearer ${apiKey}`
+            // Note: browser/node fetch automatically sets the proper multipart boundary 
+            // when passing a FormData object.
+        },
+        body: formData
+    });
+
+    const data = await response.json();
+    if (!data || !data.taskId) {
+        throw new Error('Invalid response from MinerU API: missing taskId');
+    }
+
+    return data.taskId;
+}
+
+/**
+ * Poll the MinerU API for task completion and download the extraction result.
+ * @param {string} taskId - The task ID returned from submission.
+ * @param {string} apiKey - The MINERU_API_KEY.
+ * @returns {Promise<string>} The extracted markdown content.
+ */
+export async function pollAndDownloadExtraction(taskId, apiKey) {
+    const pollInterval = 5000; // 5 seconds
+    const maxWaitTime = 300000; // 5 minutes max wait
+    const startTime = Date.now();
+
+    while (Date.now() - startTime < maxWaitTime) {
+        const response = await fetchWithRetry(`${API_BASE}/extract/task/${taskId}`, {
+            method: 'GET',
+            headers: {
+                'Authorization': `Bearer ${apiKey}`
+            }
+        });
+
+        const data = await response.json();
+        
+        // Assuming status 'done' or 'completed'
+        if (data.status === 'completed' || data.status === 'done') {
+            if (!data.downloadUrl) {
+                throw new Error('MinerU API reported completion but no downloadUrl was provided.');
+            }
+            
+            // Download the extracted result
+            const downloadResponse = await fetchWithRetry(data.downloadUrl, {
+                method: 'GET'
+            });
+            
+            // Assuming the result is directly markdown text or a JSON payload containing the markdown
+            // We try to parse it as JSON first, if it fails, treat as text
+            const textResponse = await downloadResponse.text();
+            try {
+                const jsonPayload = JSON.parse(textResponse);
+                return jsonPayload.markdown || jsonPayload.content || textResponse;
+            } catch (e) {
+                return textResponse; // Was just plain text/markdown
+            }
+        }
+
+        if (data.status === 'failed' || data.status === 'error') {
+            throw new Error(`MinerU API extraction failed: ${data.errorMessage || 'Unknown error'}`);
+        }
+
+        // Wait before polling again
+        await sleep(pollInterval);
+    }
+
+    throw new Error('MinerU API polling timed out');
+}
+
+/**
+ * High-level orchestration function to process a complex document.
+ * @param {Buffer} fileBuffer 
+ * @param {string} fileName 
+ * @param {string} apiKey 
+ * @returns {Promise<string>} The markdown result.
+ */
+export async function processComplexDocument(fileBuffer, fileName, apiKey) {
+    const taskId = await submitExtractionTask(fileBuffer, fileName, apiKey);
+    const markdown = await pollAndDownloadExtraction(taskId, apiKey);
+    return markdown;
+}

package/src/installer/tree-sitter.js

@@ -0,0 +1,270 @@
+import fs from "fs";
+import fsPromises from "fs/promises";
+import path from "path";
+import os from "os";
+import Parser from "web-tree-sitter";
+
+// Configure paths for grammar cache
+const CONFIG_DIR = path.join(os.homedir(), ".config", "omnicode");
+const GRAMMARS_CACHE_DIR = path.join(CONFIG_DIR, "grammars");
+
+// Ensure cache directory exists
+if (!fs.existsSync(GRAMMARS_CACHE_DIR)) {
+  fs.mkdirSync(GRAMMARS_CACHE_DIR, { recursive: true });
+}
+
+const EXTENSION_MAP = {
+  js: "javascript",
+  ts: "typescript",
+  py: "python",
+  go: "go",
+  rs: "rust",
+  cpp: "cpp",
+  c: "c",
+  cs: "c_sharp",
+  java: "java",
+  rb: "ruby",
+  php: "php"
+};
+
+// Tree-sitter must be initialized before use
+let parserInitialized = false;
+const LOADED_LANGUAGES_CACHE = new Map();
+const parserPool = [];
+async function ensureParserInitialized() {
+  if (!parserInitialized) {
+    await Parser.init();
+    parserInitialized = true;
+  }
+}
+
+/**
+ * Downloads a tree-sitter language parser from CDN if not cached
+ * @param {string} extension The file extension (e.g. 'js', 'go')
+ * @returns {Promise<Parser.Language|null>}
+ */
+export async function getOrDownloadLanguage(extension) {
+  await ensureParserInitialized();
+  
+  // Clean extension (remove leading dot)
+  const ext = extension.startsWith(".") ? extension.substring(1) : extension;
+  const grammarName = EXTENSION_MAP[ext];
+  
+  if (!grammarName) {
+    return null; // Unsupported language
+  }
+
+  // Check memory cache first
+  if (LOADED_LANGUAGES_CACHE.has(grammarName)) {
+    return LOADED_LANGUAGES_CACHE.get(grammarName);
+  }
+
+  const localWasmPath = path.join(GRAMMARS_CACHE_DIR, `tree-sitter-${grammarName}.wasm`);
+
+  // 1. Check local cache
+  if (fs.existsSync(localWasmPath)) {
+    try {
+      const lang = await Parser.Language.load(localWasmPath);
+      LOADED_LANGUAGES_CACHE.set(grammarName, lang);
+      return lang;
+    } catch (err) {
+      console.warn(`[omnicode] Error loading cached parser ${grammarName}: ${err.message}. Removing cache and retrying.`);
+      fs.unlinkSync(localWasmPath);
+    }
+  }
+
+  // 2. Download from CDN
+  try {
+    // Pin to latest 0.26 to match web-tree-sitter ABI (adjust if needed)
+    const cdnUrl = `https://unpkg.com/tree-sitter-wasms@0.1.11/out/tree-sitter-${grammarName}.wasm`;
+    console.log(`[omnicode] Downloading Tree-sitter parser for ${grammarName}...`);
+    
+    const response = await fetch(cdnUrl);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+    
+    const arrayBuffer = await response.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    
+    await fsPromises.writeFile(localWasmPath, buffer);
+    const lang = await Parser.Language.load(localWasmPath);
+    LOADED_LANGUAGES_CACHE.set(grammarName, lang);
+    return lang;
+  } catch (error) {
+    console.warn(`[omnicode] Failed to download parser for ${grammarName}: ${error.message}. Falling back to linear chunking.`);
+    return null;
+  }
+}
+
+/**
+ * Manually download a grammar via CLI
+ */
+export async function downloadLanguageCmd(language) {
+  // Allow user to pass "js" or "javascript"
+  const grammarName = Object.values(EXTENSION_MAP).includes(language) 
+    ? language 
+    : EXTENSION_MAP[language];
+    
+  if (!grammarName) {
+    console.error(`[omnicode] Error: Unsupported language '${language}'. Supported: ${Object.keys(EXTENSION_MAP).join(", ")}`);
+    return false;
+  }
+
+  // Force extension representation to reuse download logic
+  const mockExtension = Object.keys(EXTENSION_MAP).find(key => EXTENSION_MAP[key] === grammarName);
+  
+  const localWasmPath = path.join(GRAMMARS_CACHE_DIR, `tree-sitter-${grammarName}.wasm`);
+  if (fs.existsSync(localWasmPath)) {
+    fs.unlinkSync(localWasmPath); // Force re-download
+  }
+
+  const lang = await getOrDownloadLanguage(mockExtension);
+  if (lang) {
+    console.log(`[omnicode] Successfully downloaded and cached parser for ${grammarName} at ${localWasmPath}`);
+    return true;
+  }
+  return false;
+}
+
+const MAX_CHUNK_SIZE = 4000;
+
+function getLines(sourceCode, startLine, endLine) {
+  const lines = sourceCode.split("\n");
+  // tree-sitter lines are 0-indexed
+  return lines.slice(startLine, endLine + 1).join("\n").trim();
+}
+
+/**
+ * Recursively splits large AST nodes
+ */
+function extractChunksFromNode(node, sourceCode, chunks, coveredLines) {
+  const nodeText = sourceCode.substring(node.startIndex, node.endIndex);
+  
+  if (nodeText.length <= MAX_CHUNK_SIZE) {
+    // Fits perfectly!
+    chunks.push({
+      text: nodeText,
+      type: node.type,
+      startLine: node.startPosition.row,
+      endLine: node.endPosition.row
+    });
+    
+    // Mark lines as covered
+    for (let i = node.startPosition.row; i <= node.endPosition.row; i++) {
+      coveredLines[i] = true;
+    }
+    return;
+  }
+
+  // Node too large, attempt to split by structural children
+  let hasStructuralChildren = false;
+  for (let i = 0; i < node.childCount; i++) {
+    const child = node.child(i);
+    // Common block definitions across languages
+    if (["method_definition", "function_declaration", "class_declaration", "declaration", "function_definition", "statement_block", "block"].includes(child.type)) {
+      hasStructuralChildren = true;
+      extractChunksFromNode(child, sourceCode, chunks, coveredLines);
+    }
+  }
+
+  if (!hasStructuralChildren) {
+    // No structural children to split by, slice linearly
+    let index = 0;
+    while (index < nodeText.length) {
+      chunks.push({
+        text: nodeText.substring(index, index + MAX_CHUNK_SIZE),
+        type: `${node.type}_slice`,
+        startLine: node.startPosition.row,
+        endLine: node.endPosition.row
+      });
+      index += MAX_CHUNK_SIZE;
+    }
+    // Mark lines as covered
+    for (let i = node.startPosition.row; i <= node.endPosition.row; i++) {
+      coveredLines[i] = true;
+    }
+  } else {
+    // If we extracted children, we don't mark the whole parent as covered yet.
+    // The "orphaned lines" logic will catch the unextracted boilerplate (like class signatures)
+  }
+}
+
+/**
+ * Chunks a file structurally using Tree-sitter
+ * @returns {Promise<string[]|null>} Array of text chunks, or null if parser fails/unsupported
+ */
+export async function chunkWithTreeSitter(content, filePath) {
+  const ext = path.extname(filePath);
+  if (!ext) return null;
+
+  const language = await getOrDownloadLanguage(ext);
+  if (!language) return null; // Fall back to linear chunker
+
+  await ensureParserInitialized();
+  const parser = parserPool.pop() || new Parser();
+  parser.setLanguage(language);
+
+  try {
+    const tree = parser.parse(content);
+    const chunks = [];
+    
+    // Track covered lines for orphaned lines capture
+    const totalLines = content.split("\n").length;
+    const coveredLines = new Array(totalLines).fill(false);
+
+    // Walk the AST
+    const cursor = tree.walk();
+    
+    function traverse(node) {
+      if (["function_declaration", "class_declaration", "method_definition", "function_definition", "type_declaration", "interface_declaration"].includes(node.type)) {
+        extractChunksFromNode(node, content, chunks, coveredLines);
+        // Don't traverse inside, extractChunksFromNode handles it if needed
+        return;
+      }
+      for (let i = 0; i < node.childCount; i++) {
+        traverse(node.child(i));
+      }
+    }
+    
+    traverse(tree.rootNode);
+
+    // Collect Orphaned Lines
+    const sourceLines = content.split("\n");
+    let currentOrphanBlock = [];
+    
+    for (let i = 0; i < totalLines; i++) {
+      if (!coveredLines[i]) {
+        currentOrphanBlock.push(sourceLines[i]);
+      } else {
+        if (currentOrphanBlock.length > 0) {
+          const orphanText = currentOrphanBlock.join("\n").trim();
+          if (orphanText.length > 0) {
+            chunks.push({
+              text: `// Global/Module Scope Fragment from: ${filePath}\n${orphanText}`,
+              type: "module_scope"
+            });
+          }
+          currentOrphanBlock = [];
+        }
+      }
+    }
+    if (currentOrphanBlock.length > 0) {
+      const orphanText = currentOrphanBlock.join("\n").trim();
+      if (orphanText.length > 0) {
+        chunks.push({
+          text: `// Global/Module Scope Fragment from: ${filePath}\n${orphanText}`,
+          type: "module_scope"
+        });
+      }
+    }
+
+    // Filter and return plain text array matching `chunkFile` signature
+    return chunks.map(c => c.text).filter(t => t.length > 0);
+  } catch (err) {
+    console.warn(`[omnicode] Tree-sitter parsing failed for ${filePath}: ${err.message}. Falling back to linear chunker.`);
+    return null;
+  } finally {
+    parserPool.push(parser);
+  }
+}