legacyver 2.1.0 → 2.1.2

package/src/crawler/index.js

@@ -1,49 +1,65 @@
-/**
- * Crawler main module.
- * Composes walk.js and manifest.js.
- * Applies size filter, logs warnings for skipped files.
- */
+'use strict';
 
-import { walkFiles } from './walk.js';
-import { buildManifest } from './manifest.js';
-import { logger } from '../utils/logger.js';
+const { existsSync } = require('fs');
+const path = require('path');
+const { walk } = require('./walk');
+const { buildManifest } = require('./manifest');
+const { detectPrimaryLanguage } = require('./filters');
+const logger = require('../utils/logger');
 
 /**
- * Crawl a target directory and return file manifests.
- * @param {string} targetDir - Absolute path to scan
- * @param {object} options
- * @param {number} [options.maxFileSizeKb=500] - Skip files larger than this (KB)
- * @param {string[]} [options.ignore] - Additional ignore patterns
- * @returns {Promise<import('./manifest.js').FileManifest[]>}
+ * @typedef {Object} FileManifest
+ * @property {string} relativePath
+ * @property {string} absolutePath
+ * @property {string} language
+ * @property {number} sizeBytes
+ * @property {string} hash
  */
-export async function crawl(targetDir, options = {}) {
-  const { maxFileSizeKb = 500, ignore: customIgnore = [] } = options;
-  const maxBytes = maxFileSizeKb * 1024;
 
-  // Discover files
-  const relativePaths = await walkFiles(targetDir, { customIgnore });
+/**
+ * Main crawler entrypoint.
+ * @param {string} targetDir
+ * @param {Object} config
+ * @returns {{ files: FileManifest[], skipped: Object[], meta: Object }}
+ */
+async function crawl(targetDir, config) {
+  const maxSizeBytes = (config.maxFileSizeKb || 500) * 1024;
 
-  // Build manifest
-  const manifest = buildManifest(relativePaths, targetDir);
+  const absPaths = await walk(targetDir, config);
+  let allFiles = buildManifest(absPaths, targetDir);
 
-  // Filter by size
-  const kept = [];
+  // Size filter
   const skipped = [];
-
-  for (const entry of manifest) {
-    if (entry.sizeBytes > maxBytes) {
-      skipped.push(entry);
-    } else {
-      kept.push(entry);
+  const files = allFiles.filter((f) => {
+    if (f.sizeBytes > maxSizeBytes) {
+      logger.warn(`Skipping large file (${Math.round(f.sizeBytes / 1024)}KB): ${f.relativePath}`);
+      skipped.push({ ...f, reason: 'too large' });
+      return false;
     }
-  }
+    return true;
+  });
 
-  if (skipped.length > 0) {
-    logger.warn(`Skipped ${skipped.length} file(s) exceeding ${maxFileSizeKb}KB:`);
-    for (const s of skipped) {
-      logger.debug(`  ${s.relativePath} (${(s.sizeBytes / 1024).toFixed(1)}KB)`);
-    }
+  const primaryLanguage = detectPrimaryLanguage(files);
+
+  // Laravel detection
+  let framework = null;
+  const artisanPath = path.join(targetDir, 'artisan');
+  const appDir = path.join(targetDir, 'app');
+  if (existsSync(artisanPath) && existsSync(appDir)) {
+    framework = 'laravel';
+    logger.info('Detected Laravel project.');
   }
 
-  return kept;
+  const meta = {
+    name: path.basename(targetDir),
+    primaryLanguage,
+    framework,
+    targetDir,
+    totalFiles: files.length,
+    analyzedAt: new Date().toISOString(),
+  };
+
+  return { files, skipped, meta };
 }
+
+module.exports = { crawl };

package/src/crawler/manifest.js

@@ -1,51 +1,39 @@
-/**
- * File manifest builder.
- * For each discovered file: compute sizeBytes, detect language, compute SHA-256 hash.
- */
+'use strict';
 
-import { readFileSync, statSync } from 'node:fs';
-import { join, extname } from 'node:path';
-import { createHash } from 'node:crypto';
-import { EXTENSION_TO_LANGUAGE } from './filters.js';
+const { createHash } = require('crypto');
+const { statSync, readFileSync } = require('fs');
+const path = require('path');
+const { detectLanguage } = require('./filters');
 
 /**
- * @typedef {object} FileManifest
- * @property {string} relativePath - Relative path from target dir
- * @property {string} absolutePath - Absolute path on disk
- * @property {string} language - Detected language
- * @property {number} sizeBytes - File size in bytes
- * @property {string} hash - SHA-256 hash of file content
+ * Build a FileManifest[] from a list of absolute paths.
+ * @param {string[]} absPaths
+ * @param {string} targetDir
+ * @returns {import('./index').FileManifest[]}
  */
+function buildManifest(absPaths, targetDir) {
+  return absPaths.map((absPath) => {
+    const ext = path.extname(absPath).toLowerCase();
+    const language = detectLanguage(ext);
+    const stat = statSync(absPath);
+    const sizeBytes = stat.size;
 
-/**
- * Build manifest entry for a single file.
- * @param {string} relativePath - Relative path
- * @param {string} targetDir - Base directory
- * @returns {FileManifest}
- */
-export function buildManifestEntry(relativePath, targetDir) {
-  const absolutePath = join(targetDir, relativePath);
-  const ext = extname(relativePath).toLowerCase();
-  const language = EXTENSION_TO_LANGUAGE[ext] || 'unknown';
-  const stat = statSync(absolutePath);
-  const content = readFileSync(absolutePath);
-  const hash = 'sha256:' + createHash('sha256').update(content).digest('hex');
+    let hash = null;
+    try {
+      const content = readFileSync(absPath);
+      hash = 'sha256:' + createHash('sha256').update(content).digest('hex');
+    } catch (e) {
+      // unreadable file
+    }
 
-  return {
-    relativePath,
-    absolutePath,
-    language,
-    sizeBytes: stat.size,
-    hash,
-  };
+    return {
+      relativePath: path.relative(targetDir, absPath).replace(/\\/g, '/'),
+      absolutePath: absPath,
+      language,
+      sizeBytes,
+      hash,
+    };
+  });
 }
 
-/**
- * Build manifest entries for all discovered files.
- * @param {string[]} relativePaths - List of relative file paths
- * @param {string} targetDir - Base directory
- * @returns {FileManifest[]}
- */
-export function buildManifest(relativePaths, targetDir) {
-  return relativePaths.map((rp) => buildManifestEntry(rp, targetDir));
-}
+module.exports = { buildManifest };

package/src/crawler/walk.js

@@ -1,57 +1,51 @@
-/**
- * File walker using fast-glob with ignore package.
- * Loads .legacyverignore if present.
- * Applies default ignores + custom ignores.
- */
+'use strict';
 
-import fg from 'fast-glob';
-import ignore from 'ignore';
-import { readFileSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
-import { SUPPORTED_EXTENSIONS, DEFAULT_IGNORE_PATTERNS } from './filters.js';
+const fg = require('fast-glob');
+const ignore = require('ignore');
+const { readFileSync, existsSync } = require('fs');
+const path = require('path');
+const { DEFAULT_IGNORE_PATTERNS, ALL_EXTENSIONS } = require('./filters');
 
 /**
- * Discover all source files in the target directory.
- * @param {string} targetDir - Absolute path to scan
- * @param {object} options
- * @param {string[]} [options.customIgnore] - Additional ignore patterns
- * @returns {Promise<string[]>} List of relative file paths
+ * Walk a directory and return all matching source file paths.
+ * @param {string} targetDir
+ * @param {Object} config
+ * @returns {string[]} absolute paths
  */
-export async function walkFiles(targetDir, options = {}) {
-  const { customIgnore = [] } = options;
-
-  // Build glob patterns for supported extensions
-  const patterns = SUPPORTED_EXTENSIONS.map((ext) => `**/*${ext}`);
+async function walk(targetDir, config) {
+  // Build ignore manager
+  const ig = ignore();
 
   // Load .legacyverignore
-  const ig = ignore();
-  const legacyverignorePath = join(targetDir, '.legacyverignore');
-  if (existsSync(legacyverignorePath)) {
-    const content = readFileSync(legacyverignorePath, 'utf-8');
+  const legacyverIgnorePath = path.join(targetDir, '.legacyverignore');
+  if (existsSync(legacyverIgnorePath)) {
+    const content = readFileSync(legacyverIgnorePath, 'utf8');
     ig.add(content);
   }
 
-  // Also load .gitignore patterns
-  const gitignorePath = join(targetDir, '.gitignore');
+  // Load .gitignore
+  const gitignorePath = path.join(targetDir, '.gitignore');
   if (existsSync(gitignorePath)) {
-    const content = readFileSync(gitignorePath, 'utf-8');
+    const content = readFileSync(gitignorePath, 'utf8');
     ig.add(content);
   }
 
-  // Combine all ignore patterns
-  const allIgnore = [...DEFAULT_IGNORE_PATTERNS, ...customIgnore];
+  // Build extension glob patterns
+  const extGlobs = ALL_EXTENSIONS.map(ext => `**/*${ext}`);
 
-  // Use fast-glob to discover files
-  const files = await fg(patterns, {
+  const rawPaths = await fg(extGlobs, {
     cwd: targetDir,
-    ignore: allIgnore,
+    absolute: true,
+    ignore: DEFAULT_IGNORE_PATTERNS,
     dot: false,
-    absolute: false,
-    onlyFiles: true,
+    followSymbolicLinks: false,
   });
 
-  // Apply .legacyverignore / .gitignore filtering
-  const filtered = ig.filter(files);
-
-  return filtered.sort();
+  // Apply .legacyverignore and .gitignore on relative paths
+  return rawPaths.filter((absPath) => {
+    const rel = path.relative(targetDir, absPath);
+    return !ig.ignores(rel);
+  });
 }
+
+module.exports = { walk };

package/src/llm/chunker.js

@@ -1,77 +1,55 @@
-/**
- * LLM request chunker.
- * Builds user messages from FileFacts + raw source.
- * Checks token count vs provider limit.
- * Truncates raw source if over limit (never truncates FileFacts).
- */
+'use strict';
 
-import { readFileSync } from 'node:fs';
-import { join } from 'node:path';
-import { buildUserMessage, SYSTEM_PROMPT } from './prompts.js';
-import { countTokens } from './cost-estimator.js';
+const { readFileSync } = require('fs');
+const { SYSTEM_PROMPT, buildUserMessage } = require('./prompts');
+const { countTokens } = require('./cost-estimator');
+const logger = require('../utils/logger');
 
-/**
- * Provider token limits (input context window).
- */
-const TOKEN_LIMITS = {
-  anthropic: 200000,
-  openai: 128000,
-  google: 1000000,
-  groq: 128000,
-  ollama: 8000,
-};
+const DEFAULT_MAX_TOKENS = 120000;
 
 /**
- * Build LLM request chunks for all files in the PKG.
- * @param {import('../parser/index.js').FileFacts[]} allFacts
- * @param {object} config - App configuration
- * @param {string} targetDir - Base directory for reading source files
- * @returns {Array<{relativePath: string, userMessage: string, systemPrompt: string}>}
+ * Build LLM request chunks from the PKG.
+ * @param {Object} pkg
+ * @param {Object} config
+ * @returns {Array<{relativePath, systemPrompt, userMessage, tokenCount}>}
  */
-export function buildChunks(allFacts, config, targetDir) {
-  const maxTokens = TOKEN_LIMITS[config.provider] || TOKEN_LIMITS.anthropic;
-  // Reserve tokens for system prompt and output
-  const systemTokens = countTokens(SYSTEM_PROMPT);
-  const reservedForOutput = Math.min(4000, maxTokens * 0.1);
-  const availableTokens = maxTokens - systemTokens - reservedForOutput;
-
+function buildChunks(pkg, config) {
   const chunks = [];
+  const maxTokens = DEFAULT_MAX_TOKENS;
 
-  for (const facts of allFacts) {
-    let rawSource;
+  for (const [relativePath, fileFacts] of Object.entries(pkg.files || {})) {
+    let rawSource = '';
     try {
-      rawSource = readFileSync(join(targetDir, facts.relativePath), 'utf-8');
-    } catch {
-      rawSource = '// Source file not readable';
+      rawSource = readFileSync(fileFacts.relativePath && fileFacts.absolutePath ? fileFacts.absolutePath : (pkg.meta.targetDir + '/' + relativePath), 'utf8');
+    } catch (e) {
+      // file might not be available; use empty source
+      rawSource = '';
     }
 
-    // Build the full user message
-    let userMessage = buildUserMessage(facts, rawSource);
-    let messageTokens = countTokens(userMessage);
+    let userMessage = buildUserMessage(fileFacts, rawSource);
+    const combined = SYSTEM_PROMPT + '\n' + userMessage;
+    let tokenCount = countTokens(combined);
 
     // If over limit, truncate raw source (never truncate FileFacts)
-    if (messageTokens > availableTokens) {
-      const factsJson = JSON.stringify(facts, null, 2);
-      const factsTokens = countTokens(factsJson);
-      const remainingTokens = availableTokens - factsTokens - 200; // 200 token buffer for template text
-
-      if (remainingTokens > 0) {
-        // Truncate source to fit
-        const maxChars = remainingTokens * 4; // inverse of our token estimation
-        const truncatedSource = rawSource.substring(0, maxChars) + '\n\n[... source truncated due to token limit ...]';
-        userMessage = buildUserMessage(facts, truncatedSource);
-      } else {
-        // FileFacts alone exceeds limit — skip source entirely
-        userMessage = buildUserMessage(facts, '[Source omitted — file too large for context window]');
-      }
+    if (tokenCount > maxTokens) {
+      const factsPart = buildUserMessage(fileFacts, '');
+      const factsTokens = countTokens(SYSTEM_PROMPT + '\n' + factsPart);
+      const budgetChars = Math.max(0, (maxTokens - factsTokens) * 4 - 200);
+      const truncatedSource = rawSource.slice(0, budgetChars) + '\n...[truncated]';
+      userMessage = buildUserMessage(fileFacts, truncatedSource);
+      tokenCount = countTokens(SYSTEM_PROMPT + '\n' + userMessage);
+      logger.warn(`Truncated source for ${relativePath} to fit context window.`);
     }
 
     chunks.push({
-      relativePath: facts.relativePath,
-      userMessage,
+      relativePath,
       systemPrompt: SYSTEM_PROMPT,
+      userMessage,
+      tokenCount,
     });
   }
 
   return chunks;
 }
+
+module.exports = { buildChunks };

package/src/llm/cost-estimator.js

@@ -1,67 +1,84 @@
-/**
- * Cost estimator using tiktoken for token counting.
- * Calculates total cost per provider based on per-token pricing.
- */
+'use strict';
 
-import { PROVIDERS } from '../cli/commands/providers.js';
+const { readFileSync } = require('fs');
+const path = require('path');
+const logger = require('../utils/logger');
 
-/**
- * Pricing table per provider (per token, not per 1k).
- */
-const PRICING = {};
-for (const p of PROVIDERS) {
-  PRICING[p.name] = {
-    inputPerToken: p.costPer1kInput / 1000,
-    outputPerToken: p.costPer1kOutput / 1000,
-  };
+// Simple token estimator (tiktoken may have WASM issues in some envs)
+let encoder = null;
+function countTokens(text) {
+  try {
+    if (!encoder) {
+      const { get_encoding } = require('tiktoken');
+      encoder = get_encoding('cl100k_base');
+    }
+    return encoder.encode(text).length;
+  } catch (e) {
+    // Fallback: rough approximation (1 token ≈ 4 chars)
+    return Math.ceil(text.length / 4);
+  }
 }
 
-/**
- * Estimate token count for a string.
- * Uses a simple heuristic (~4 chars per token for English code).
- * For more accuracy, tiktoken can be used but is heavy to initialize.
- * @param {string} text
- * @returns {number} Estimated token count
- */
-export function countTokens(text) {
-  // Rough estimate: ~4 chars per token for code
-  return Math.ceil(text.length / 4);
+// Cached model pricing (populated lazily)
+let modelPricingCache = null;
+let modelPricingCachedAt = 0;
+const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+
+const FALLBACK_PRICING = {
+  'meta-llama/llama-3.3-70b-instruct:free': { input: 0, output: 0 },
+  'anthropic/claude-haiku-3-5': { input: 0.8 / 1e6, output: 4.0 / 1e6 },
+  'anthropic/claude-sonnet-4-5': { input: 3.0 / 1e6, output: 15.0 / 1e6 },
+  'openai/gpt-4o-mini': { input: 0.15 / 1e6, output: 0.6 / 1e6 },
+  'openai/gpt-4o': { input: 5.0 / 1e6, output: 15.0 / 1e6 },
+  'google/gemini-flash-1.5': { input: 0.075 / 1e6, output: 0.3 / 1e6 },
+};
+
+async function fetchModelPricing() {
+  if (modelPricingCache && Date.now() - modelPricingCachedAt < CACHE_TTL_MS) {
+    return modelPricingCache;
+  }
+  try {
+    const resp = await fetch('https://openrouter.ai/api/v1/models');
+    if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+    const data = await resp.json();
+    const pricing = {};
+    for (const m of (data.data || [])) {
+      pricing[m.id] = {
+        input: parseFloat(m.pricing && m.pricing.prompt) || 0,
+        output: parseFloat(m.pricing && m.pricing.completion) || 0,
+      };
+    }
+    modelPricingCache = pricing;
+    modelPricingCachedAt = Date.now();
+    return pricing;
+  } catch (e) {
+    logger.warn(`Could not fetch model pricing from OpenRouter: ${e.message}. Using fallback.`);
+    return FALLBACK_PRICING;
+  }
 }
 
 /**
- * Estimate cost for a set of LLM requests.
- * @param {Array<{userMessage: string, relativePath: string}>} chunks - LLM request chunks
- * @param {string} providerName - Provider name
- * @returns {object} Cost estimate
+ * Estimate cost for a list of LLM request chunks.
+ * @param {Array} chunks
+ * @param {Object} config
+ * @returns {Promise<{totalInputTokens, totalOutputTokens, estimatedCostUSD, modelId}>}
  */
-export function estimateCost(chunks, providerName) {
-  const pricing = PRICING[providerName] || PRICING.anthropic;
+async function estimateCost(chunks, config) {
+  const modelId = config.model || 'meta-llama/llama-3.3-70b-instruct:free';
+  const pricing = await fetchModelPricing();
+  const modelPricing = pricing[modelId] || { input: 0, output: 0 };
 
   let totalInputTokens = 0;
-  const breakdown = [];
-
   for (const chunk of chunks) {
-    const inputTokens = countTokens(chunk.userMessage);
-    totalInputTokens += inputTokens;
-
-    breakdown.push({
-      file: chunk.relativePath,
-      inputTokens,
-    });
+    totalInputTokens += countTokens(chunk.systemPrompt + '\n' + chunk.userMessage);
   }
-
-  // Estimate output tokens as ~40% of input (documentation is shorter than code + facts)
-  const totalOutputTokens = Math.ceil(totalInputTokens * 0.4);
+  const avgOutputTokens = 400;
+  const totalOutputTokens = chunks.length * avgOutputTokens;
 
   const estimatedCostUSD =
-    totalInputTokens * pricing.inputPerToken +
-    totalOutputTokens * pricing.outputPerToken;
+    totalInputTokens * modelPricing.input + totalOutputTokens * modelPricing.output;
 
-  return {
-    provider: providerName,
-    totalInputTokens,
-    totalOutputTokens,
-    estimatedCostUSD,
-    breakdown,
-  };
+  return { totalInputTokens, totalOutputTokens, estimatedCostUSD, modelId };
 }
+
+module.exports = { estimateCost, countTokens };

package/src/llm/free-model.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const logger = require('../utils/logger');
+const pc = require('picocolors');
+
+/**
+ * Apply free model policies when model ID ends with `:free`.
+ * @param {Object} config
+ * @returns {Object} mutated config
+ */
+function applyFreeModelPolicy(config) {
+  const provider = (config.provider || 'openrouter').toLowerCase();
+
+  // Ollama, Groq, and Gemini are always free — skip openrouter-specific logic
+  if (provider === 'ollama' || provider === 'groq' || provider === 'gemini') {
+    config.isFreeModel = true;
+    config.skipCostEstimation = true;
+    if (provider === 'groq') {
+      console.log(pc.cyan('[info] Using Groq — free tier. Rate limit: 30 req/min, 14,400 req/day'));
+      // Cap concurrency to 1 for Groq to avoid rate limits (30 req/min)
+      const requested = parseInt(config.concurrency) || 1;
+      if (requested > 1) {
+        logger.warn(`Groq free tier: capping concurrency from ${requested} to 1.`);
+        config.concurrency = 1;
+      }
+    }
+    if (provider === 'gemini') {
+      console.log(pc.cyan('[info] Using Gemini — free tier. Rate limit: 15 req/min, 1,500 req/day'));
+      // Cap concurrency to 2 for Gemini (15 req/min is more generous)
+      const requested = parseInt(config.concurrency) || 1;
+      if (requested > 2) {
+        logger.warn(`Gemini free tier: capping concurrency from ${requested} to 2.`);
+        config.concurrency = 2;
+      }
+    }
+    return config;
+  }
+
+  const model = config.model || 'meta-llama/llama-3.3-70b-instruct:free';
+  if (!model.endsWith(':free')) {
+    config.isFreeModel = false;
+    return config;
+  }
+
+  config.isFreeModel = true;
+
+  // Display notice
+  console.log(pc.cyan('[info] Using free model — no cost estimate available. Rate limit: 200 req/day'));
+
+  // Cap concurrency
+  const requestedConcurrency = parseInt(config.concurrency) || 1;
+  if (requestedConcurrency > 2) {
+    logger.warn(`Free model: capping concurrency from ${requestedConcurrency} to 2.`);
+    config.concurrency = 2;
+  } else if (requestedConcurrency > 1) {
+    config.concurrency = requestedConcurrency;
+  } else {
+    config.concurrency = 1;
+  }
+
+  // Skip cost estimation
+  config.skipCostEstimation = true;
+
+  return config;
+}
+
+module.exports = { applyFreeModelPolicy };