legacyver 2.1.0 → 2.1.2

package/src/llm/queue.js

@@ -1,99 +1,70 @@
-/**
- * LLM request queue with concurrency control and retry logic.
- * Uses p-limit for concurrency and p-retry for retries.
- */
+'use strict';
 
-import pLimit from 'p-limit';
-import pRetry from 'p-retry';
-import { RateLimitError } from '../utils/errors.js';
-import { logger } from '../utils/logger.js';
-
-/**
- * @typedef {object} DocFragment
- * @property {string} relativePath
- * @property {string} content - Generated markdown documentation
- * @property {object} tokensUsed - { input, output }
- */
+const pLimit = require('p-limit');
+const pRetry = require('p-retry');
+const logger = require('../utils/logger');
 
 /**
- * Create a processing queue for LLM requests.
- * @param {object} engine - LLM engine with `complete(request)` method
- * @param {object} options
- * @param {number} [options.concurrency=5] - Max concurrent requests
- * @param {function} [options.onProgress] - Called after each request completes
- * @param {function} [options.onError] - Called when a request fails after retries
- * @returns {object} Queue with processAll method
+ * Process chunks through the LLM provider with concurrency and retry.
+ * @param {Array} chunks
+ * @param {Object} provider
+ * @param {Object} config
+ * @param {{ onProgress, onError }} callbacks
+ * @returns {Promise<Array>} DocFragment[]
  */
-export function createQueue(engine, options = {}) {
-  const {
-    concurrency = 5,
-    onProgress = () => {},
-    onError = () => {},
-  } = options;
-
-  const limit = pLimit(Math.max(1, parseInt(concurrency, 10) || 5));
-  const failures = [];
+async function createQueue(chunks, provider, config, callbacks = {}) {
+  const concurrency = config.concurrency || 3;
+  const limit = pLimit(concurrency);
+  const fragments = [];
 
-  return {
-    /**
-     * Process all chunks through the LLM engine.
-     * @param {Array} chunks - LLM request chunks
-     * @returns {Promise<DocFragment[]>}
-     */
-    async processAll(chunks) {
-      const results = await Promise.allSettled(
-        chunks.map((chunk) =>
-          limit(async () => {
+  const tasks = chunks.map((chunk) =>
+    limit(async () => {
+      try {
+        const result = await pRetry(
+          async () => {
             try {
-              const result = await pRetry(
-                async () => {
-                  const response = await engine.complete({
-                    systemPrompt: chunk.systemPrompt,
-                    userMessage: chunk.userMessage,
-                  });
-                  return {
-                    relativePath: chunk.relativePath,
-                    content: response.content,
-                    tokensUsed: response.tokensUsed,
-                  };
-                },
-                {
-                  retries: 3,
-                  minTimeout: 1000,
-                  factor: 2,
-                  onFailedAttempt: (error) => {
-                    if (error instanceof RateLimitError) {
-                      logger.debug(
-                        `Rate limited for ${chunk.relativePath}, retrying (attempt ${error.attemptNumber}/4)...`
-                      );
-                    } else {
-                      logger.debug(
-                        `Request failed for ${chunk.relativePath}: ${error.message} (attempt ${error.attemptNumber}/4)`
-                      );
-                    }
-                  },
-                }
-              );
-
-              onProgress();
-              return result;
+              return await provider.complete(chunk);
             } catch (err) {
-              onError(err, chunk);
-              failures.push({ file: chunk.relativePath, error: err.message });
-              onProgress();
-              return null;
+              // If it's a rate-limit error, wait for the retry-after period
+              // before letting p-retry schedule the next attempt.
+              if (err.code === 'RATE_LIMIT' && err.retryAfter) {
+                await new Promise((r) => setTimeout(r, err.retryAfter));
+              }
+              throw err;
             }
-          })
-        )
-      );
+          },
+          {
+            retries: 3,
+            minTimeout: config.isFreeModel ? 8000 : 1000,
+            factor: 3,
+            onFailedAttempt: (error) => {
+              logger.warn(`Retry ${error.attemptNumber}/3 for ${chunk.relativePath}: ${error.message}`);
+            },
+          }
+        );
+        fragments.push({
+          relativePath: chunk.relativePath,
+          content: result.content,
+          tokensUsed: result.tokensUsed,
+          _qualityWarnings: [],
+        });
+      } catch (e) {
+        logger.error(`Failed to generate docs for ${chunk.relativePath}: ${e.message}`);
+        if (callbacks.onError) callbacks.onError(e, chunk);
+        fragments.push({
+          relativePath: chunk.relativePath,
+          content: `<!-- Documentation generation failed: ${e.message} -->`,
+          tokensUsed: { input: 0, output: 0 },
+          _qualityWarnings: [`Generation failed: ${e.message}`],
+        });
+      }
 
-      return results
-        .map((r) => (r.status === 'fulfilled' ? r.value : null))
-        .filter(Boolean);
-    },
+      if (callbacks.onProgress) callbacks.onProgress();
+    })
+  );
 
-    getFailures() {
-      return failures;
-    },
-  };
+  await Promise.all(tasks);
+  return fragments;
 }
+
+module.exports = { createQueue };

package/src/llm/re-prompter.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const { SYSTEM_PROMPT } = require('./prompts');
+const logger = require('../utils/logger');
+
+/**
+ * Re-prompt the LLM for missing exports.
+ * @param {Object} fragment
+ * @param {Object} fileFacts
+ * @param {Object} provider
+ * @param {Object} config
+ * @returns {Promise<Object|null>} updated fragment or null
+ */
+async function reprompt(fragment, fileFacts, provider, config) {
+  const missing = (fileFacts.exports || []).filter(sym => sym && !fragment.content.includes(sym));
+
+  if (missing.length === 0) return null;
+
+  const followUpMessage =
+    `The following exported symbols were not documented in your previous response. ` +
+    `Please document each one based only on the FileFacts provided:\n\n` +
+    missing.map(s => `- ${s}`).join('\n');
+
+  try {
+    const result = await provider.complete({
+      systemPrompt: SYSTEM_PROMPT,
+      userMessage: followUpMessage,
+      relativePath: fragment.relativePath,
+    });
+    logger.info(`Re-prompted for ${fragment.relativePath}: ${missing.length} missing exports covered.`);
+    return {
+      ...fragment,
+      content: fragment.content + '\n\n' + result.content,
+    };
+  } catch (e) {
+    logger.warn(`Re-prompt failed for ${fragment.relativePath}: ${e.message}`);
+    return null;
+  }
+}
+
+module.exports = { reprompt };

package/src/llm/validator.js

@@ -0,0 +1,72 @@
+'use strict';
+
+const logger = require('../utils/logger');
+
+/**
+ * Post-generation quality validator.
+ *
+ * Two checks:
+ *  1. Hallucination check: identifiers in LLM output not found in FileFacts or raw source
+ *  2. Completeness check: exported symbols not mentioned in LLM output
+ */
+
+/**
+ * @param {Object} fragment  { relativePath, content }
+ * @param {Object} fileFacts
+ * @returns {{ hallucinations: string[], missingExports: string[], passed: boolean }}
+ */
+function validateFragment(fragment, fileFacts) {
+  const hallucinations = [];
+  const missingExports = [];
+
+  if (!fileFacts) return { hallucinations, missingExports, passed: true };
+
+  const outputText = fragment.content || '';
+
+  // ── Hallucination check ──────────────────────────────────────────────────
+  // Collect all known identifiers from FileFacts
+  const knownIdentifiers = new Set();
+  for (const fn of (fileFacts.functions || [])) {
+    knownIdentifiers.add(fn.name);
+    for (const p of (fn.params || [])) knownIdentifiers.add(p.name && p.name.replace(/^\$/, ''));
+  }
+  for (const cls of (fileFacts.classes || [])) {
+    knownIdentifiers.add(cls.name);
+    for (const m of (cls.methods || [])) knownIdentifiers.add(m);
+  }
+  for (const imp of (fileFacts.imports || [])) {
+    for (const s of (imp.specifiers || [])) knownIdentifiers.add(s);
+  }
+  for (const exp of (fileFacts.exports || [])) knownIdentifiers.add(exp);
+
+  // Common words to skip (not identifiers)
+  const stopWords = new Set(['The', 'This', 'File', 'Function', 'Class', 'Method', 'Returns', 'Return', 'Parameter', 'Param', 'Import', 'Export', 'Overview', 'Usage', 'Example', 'Dependencies', 'Async', 'Static', 'Public', 'Private', 'Protected', 'Boolean', 'String', 'Number', 'Object', 'Array', 'Void', 'Null', 'Undefined', 'True', 'False', 'Error', 'Promise', 'Request', 'Response', 'Node', 'JavaScript', 'TypeScript', 'PHP', 'Python', 'Laravel', 'Express', 'Route', 'Controller', 'Model', 'Service', 'Repository', 'Middleware', 'Provider', 'Summary']);
+
+  const capitalizedIdentifiers = outputText.match(/\b([A-Z][a-zA-Z]{2,})\b/g) || [];
+  for (const identifier of capitalizedIdentifiers) {
+    if (stopWords.has(identifier)) continue;
+    if (!knownIdentifiers.has(identifier)) {
+      // Check raw source — it's embedded in the user message so we approximate
+      // by checking against function/class names only
+      const isInFacts = [...knownIdentifiers].some(k => k && k.toLowerCase() === identifier.toLowerCase());
+      if (!isInFacts) {
+        logger.warn(`Possible hallucination in ${fragment.relativePath}: identifier '${identifier}' not found in source facts`);
+        hallucinations.push(`Suspected hallucination: '${identifier}'`);
+      }
+    }
+  }
+
+  // ── Completeness check ───────────────────────────────────────────────────
+  const exportedSymbols = fileFacts.exports || [];
+  for (const symbol of exportedSymbols) {
+    if (symbol && !outputText.includes(symbol)) {
+      logger.warn(`Incomplete doc in ${fragment.relativePath}: exported symbol '${symbol}' not documented`);
+      missingExports.push(symbol);
+    }
+  }
+
+  const passed = hallucinations.length === 0 && missingExports.length === 0;
+  return { hallucinations, missingExports, passed };
+}
+
+module.exports = { validateFragment };

package/src/parser/ast/generic.js

@@ -1,249 +1,72 @@
-/**
- * Generic regex-based fallback parser.
- * Extracts function-like patterns, import-like lines.
- * Returns partial FileFacts with parserType: "generic".
- */
+'use strict';
 
 /**
- * @param {string} source - Raw file content
- * @param {string} relativePath - Relative file path
- * @param {string} language - Detected language
- * @returns {import('../index.js').FileFacts}
+ * Generic regex-based fallback parser for unsupported languages.
  */
-export function parseGeneric(source, relativePath, language) {
-  const lines = source.split('\n');
-  const linesOfCode = lines.filter((l) => l.trim().length > 0).length;
-
-  const functions = extractFunctions(source, language);
-  const classes = extractClasses(source, language);
-  const imports = extractImports(source, language);
-  const exports = extractExports(source, language);
+const logger = require('../../utils/logger');
 
-  return {
-    relativePath,
-    language,
-    linesOfCode,
-    parserType: 'generic',
-    functions,
-    classes,
-    imports,
-    exports,
-    callsTo: [],
-    calledBy: [],
-    hash: '',
-  };
-}
-
-function extractFunctions(source, language) {
+function parse(sourceText, relativePath, language) {
+  const lines = sourceText.split('\n');
   const functions = [];
-  const lines = source.split('\n');
-
-  // Patterns for different languages
-  const patterns = [
-    // JS/TS: function name(params)
-    /(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\(([^)]*)\)/,
-    // JS/TS: const name = (params) =>
-    /(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\(([^)]*)\)\s*=>/,
-    // JS/TS: name(params) { (class method)
-    /^\s+(?:async\s+)?(\w+)\s*\(([^)]*)\)\s*\{/,
-    // Python: def name(params):
-    /def\s+(\w+)\s*\(([^)]*)\)\s*(?:->.*)?:/,
-    // Java/Go: type name(params)
-    /(?:public|private|protected|static|\s)*\s+\w+\s+(\w+)\s*\(([^)]*)\)/,
-    // Go: func name(params)
-    /func\s+(?:\([^)]*\)\s+)?(\w+)\s*\(([^)]*)\)/,
-  ];
+  const imports = [];
+  const classes = [];
 
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
-    for (const pattern of patterns) {
-      const match = line.match(pattern);
-      if (match) {
-        const name = match[1];
-        // Skip common false positives
-        if (['if', 'for', 'while', 'switch', 'catch', 'return'].includes(name)) continue;
-
-        const params = parseParamString(match[2]);
-        const isExported = /^export\s/.test(line.trim()) || /^module\.exports/.test(line.trim());
-        const isAsync = /async\s/.test(line);
 
+    // Function-like patterns (very rough)
+    let m = line.match(/(?:function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\(?.*?\)?\s*=>|\bdef\s+(\w+)|(?:public|private|protected|static)?\s+function\s+(\w+))/);
+    if (m) {
+      const name = m[1] || m[2] || m[3] || m[4];
+      if (name) {
         functions.push({
           name,
-          params,
+          params: [],
           returnType: null,
-          isExported,
-          isAsync,
+          isExported: /\bexport\b/.test(line),
+          isAsync: /\basync\b/.test(line),
           lineStart: i + 1,
-          lineEnd: findBlockEnd(lines, i),
+          lineEnd: i + 1,
           calls: [],
+          complexityScore: null,
+          complexityClass: null,
+          detectedPatterns: [],
+          bodySnippet: null,
+          bodySnippetTruncated: false,
         });
-        break;
       }
     }
-  }
-
-  return functions;
-}
-
-function extractClasses(source, language) {
-  const classes = [];
-  const classPattern = /class\s+(\w+)(?:\s+extends\s+(\w+))?/g;
 
-  let match;
-  while ((match = classPattern.exec(source)) !== null) {
-    const name = match[1];
-    const extendsName = match[2] || null;
-
-    // Extract method names from the class body (simplified)
-    const methods = [];
-    const lines = source.substring(match.index).split('\n');
-    const methodPattern = /^\s+(?:async\s+)?(\w+)\s*\(/;
-    for (const line of lines.slice(1)) {
-      const m = line.match(methodPattern);
-      if (m && !['constructor', 'if', 'for', 'while'].includes(m[1])) {
-        methods.push(m[1]);
-      }
-      if (/^}/.test(line.trim())) break;
+    // Import-like lines
+    m = line.match(/(?:import|require|from|#include|using)\s+['"]?([^'";\s]+)['"]?/);
+    if (m) {
+      imports.push({ module: m[1], specifiers: [] });
     }
 
-    classes.push({ name, methods, extends: extendsName });
-  }
-
-  return classes;
-}
-
-function extractImports(source, language) {
-  const imports = [];
-  const lines = source.split('\n');
-
-  for (const line of lines) {
-    // ES imports
-    const esMatch = line.match(/import\s+(?:{([^}]*)}\s+from\s+)?(?:(\w+)\s+from\s+)?['"]([^'"]+)['"]/);
-    if (esMatch) {
-      const specifiers = [];
-      if (esMatch[1]) {
-        specifiers.push(...esMatch[1].split(',').map((s) => s.trim().split(' as ')[0].trim()).filter(Boolean));
-      }
-      if (esMatch[2]) {
-        specifiers.push(esMatch[2]);
-      }
-      imports.push({ module: esMatch[3], specifiers });
-      continue;
-    }
-
-    // CommonJS require
-    const cjsMatch = line.match(/(?:const|let|var)\s+(?:{([^}]*)}\s*=|(\w+)\s*=)\s*require\(['"]([^'"]+)['"]\)/);
-    if (cjsMatch) {
-      const specifiers = [];
-      if (cjsMatch[1]) {
-        specifiers.push(...cjsMatch[1].split(',').map((s) => s.trim()).filter(Boolean));
-      }
-      if (cjsMatch[2]) {
-        specifiers.push(cjsMatch[2]);
-      }
-      imports.push({ module: cjsMatch[3], specifiers });
-      continue;
-    }
-
-    // Python imports
-    const pyFromMatch = line.match(/from\s+(\S+)\s+import\s+(.+)/);
-    if (pyFromMatch) {
-      const specifiers = pyFromMatch[2].split(',').map((s) => s.trim().split(' as ')[0].trim()).filter(Boolean);
-      imports.push({ module: pyFromMatch[1], specifiers });
-      continue;
-    }
-
-    const pyImportMatch = line.match(/^import\s+(\S+)(?:\s+as\s+\w+)?$/);
-    if (pyImportMatch) {
-      imports.push({ module: pyImportMatch[1], specifiers: [pyImportMatch[1]] });
-      continue;
-    }
-
-    // Go imports
-    const goMatch = line.match(/^\s*"([^"]+)"/);
-    if (language === 'go' && goMatch) {
-      imports.push({ module: goMatch[1], specifiers: [] });
-      continue;
-    }
-
-    // Java imports
-    const javaMatch = line.match(/import\s+([\w.]+(?:\.\*)?)\s*;/);
-    if (javaMatch) {
-      const parts = javaMatch[1].split('.');
-      imports.push({ module: javaMatch[1], specifiers: [parts[parts.length - 1]] });
+    // Class definitions
+    m = line.match(/\bclass\s+(\w+)/);
+    if (m) {
+      classes.push({ name: m[1], methods: [], extends: null });
     }
   }
 
-  return imports;
-}
+  const exports = functions.filter(f => f.isExported).map(f => f.name);
 
-function extractExports(source, language) {
-  const exports = [];
-  const lines = source.split('\n');
+  logger.warn(`No AST parser for ${language || 'unknown'}, using generic fallback: ${relativePath}`);
 
-  for (const line of lines) {
-    // ES named export
-    const esMatch = line.match(/export\s+(?:const|let|var|function|class|async function)\s+(\w+)/);
-    if (esMatch) {
-      exports.push(esMatch[1]);
-      continue;
-    }
-
-    // ES default export
-    if (/export\s+default\s/.test(line)) {
-      const defMatch = line.match(/export\s+default\s+(?:class|function)?\s*(\w+)/);
-      if (defMatch) exports.push(defMatch[1]);
-      continue;
-    }
-
-    // CommonJS
-    const cjsMatch = line.match(/module\.exports\s*=\s*(?:{([^}]*)}\s*|(\w+))/);
-    if (cjsMatch) {
-      if (cjsMatch[1]) {
-        exports.push(...cjsMatch[1].split(',').map((s) => s.trim().split(':')[0].trim()).filter(Boolean));
-      }
-      if (cjsMatch[2]) {
-        exports.push(cjsMatch[2]);
-      }
-    }
-  }
-
-  return exports;
-}
-
-function parseParamString(paramStr) {
-  if (!paramStr || !paramStr.trim()) return [];
-  return paramStr.split(',').map((p) => {
-    const parts = p.trim().split(/[:\s]+/);
-    return {
-      name: parts[0].replace(/[=?].*/, ''),
-      type: parts.length > 1 ? parts.slice(1).join(' ') : null,
-    };
-  }).filter((p) => p.name && p.name !== '');
+  return {
+    relativePath,
+    language: language || 'unknown',
+    linesOfCode: lines.length,
+    functions,
+    classes,
+    imports,
+    exports,
+    callsTo: [],
+    calledBy: [],
+    hash: null,
+    parserType: 'generic',
+  };
 }
 
-function findBlockEnd(lines, startLine) {
-  let depth = 0;
-  let started = false;
-  for (let i = startLine; i < lines.length; i++) {
-    for (const ch of lines[i]) {
-      if (ch === '{' || ch === ':') {
-        if (ch === '{') { depth++; started = true; }
-        else if (ch === ':' && !started && i === startLine) { started = true; depth = 1; }
-      }
-      if (ch === '}' && started) {
-        depth--;
-        if (depth <= 0) return i + 1;
-      }
-    }
-    // Python: detect end by indentation
-    if (started && depth === 0 && i > startLine) {
-      const indent = lines[i].match(/^\s*/)[0].length;
-      const startIndent = lines[startLine].match(/^\s*/)[0].length;
-      if (indent <= startIndent && lines[i].trim() !== '') {
-        return i;
-      }
-    }
-  }
-  return Math.min(startLine + 50, lines.length);
-}
+module.exports = { parse };