contextforge-cli-ai-prompt-pirates 0.4.0

package/src/scanner/config-files.js

@@ -0,0 +1,52 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+
+const CONFIG_PATTERNS = [
+  { key: 'packageJson', match: (p) => p === 'package.json' },
+  { key: 'tsconfig', match: (p) => /^tsconfig(\..*)?\.json$/i.test(path.basename(p)) },
+  { key: 'vite', match: (p) => /vite\.config\.(js|ts|mjs|cjs)$/i.test(p) },
+  { key: 'next', match: (p) => /next\.config\.(js|ts|mjs|cjs)$/i.test(p) },
+  { key: 'dockerfile', match: (p) => /dockerfile$/i.test(p) || p.toLowerCase() === 'dockerfile' },
+  { key: 'dockerCompose', match: (p) => /docker-compose\.(yml|yaml)$/i.test(p) },
+  { key: 'tailwind', match: (p) => /tailwind\.config\.(js|ts|cjs|mjs)$/i.test(p) },
+  { key: 'prisma', match: (p) => p.replace(/\\/g, '/').endsWith('schema.prisma') },
+  { key: 'webpack', match: (p) => /webpack\.config\.(js|ts)$/i.test(p) },
+  { key: 'nginx', match: (p) => /nginx\.conf$/i.test(p) },
+  { key: 'envExample', match: (p) => /^\.env\.example$/i.test(path.basename(p)) },
+  { key: 'eslint', match: (p) => /^\.eslintrc/i.test(path.basename(p)) || p.endsWith('eslint.config.js') },
+  { key: 'prettier', match: (p) => /^\.prettierrc/i.test(path.basename(p)) || p === 'prettier.config.js' },
+];
+
+/**
+ * @param {string} projectRoot
+ * @param {string[]} relativePaths
+ */
+export async function findConfigFiles(projectRoot, relativePaths) {
+  const found = {
+    paths: [],
+    packageJson: null,
+    parsed: {},
+  };
+
+  for (const rel of relativePaths) {
+    for (const { key, match } of CONFIG_PATTERNS) {
+      if (match(rel)) {
+        found.paths.push(rel);
+        if (!found.parsed[key]) {
+          found.parsed[key] = rel;
+        }
+      }
+    }
+  }
+
+  const pkgPath = path.join(projectRoot, 'package.json');
+  if (await fs.pathExists(pkgPath)) {
+    try {
+      found.packageJson = await fs.readJson(pkgPath);
+    } catch {
+      found.packageJson = null;
+    }
+  }
+
+  return found;
+}

package/src/scanner/file-index.js

@@ -0,0 +1,55 @@
+import fs from 'fs-extra';
+import crypto from 'node:crypto';
+import { getFileIndexPath } from '../core/paths.js';
+
+export async function loadFileIndex(projectRoot) {
+  const indexPath = getFileIndexPath(projectRoot);
+  if (await fs.pathExists(indexPath)) {
+    return fs.readJson(indexPath);
+  }
+  return { version: 1, files: {} };
+}
+
+export async function saveFileIndex(projectRoot, index) {
+  await fs.writeJson(getFileIndexPath(projectRoot), index, { spaces: 2 });
+}
+
+export function hashContent(content) {
+  return crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
+}
+
+/**
+ * @param {{ files: Record<string, { mtime: number, size: number, hash: string }> }} previous
+ * @param {Array<{ relativePath: string, mtime: number, size: number, hash: string }>} scannedFiles
+ */
+export function updateFileIndex(previous, scannedFiles) {
+  const files = { ...previous.files };
+
+  for (const file of scannedFiles) {
+    files[file.relativePath] = {
+      mtime: file.mtime,
+      size: file.size,
+      hash: file.hash,
+    };
+  }
+
+  const scannedSet = new Set(scannedFiles.map((f) => f.relativePath));
+  for (const key of Object.keys(files)) {
+    if (!scannedSet.has(key)) {
+      delete files[key];
+    }
+  }
+
+  return { version: 1, updatedAt: new Date().toISOString(), files };
+}
+
+export function getChangedFiles(previous, current) {
+  const changed = [];
+  for (const file of current) {
+    const prev = previous.files?.[file.relativePath];
+    if (!prev || prev.hash !== file.hash) {
+      changed.push(file.relativePath);
+    }
+  }
+  return changed;
+}

package/src/scanner/package-deps.js

@@ -0,0 +1,32 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+
+/**
+ * Merge dependencies from root + nested package.json (monorepos / client+server).
+ * @param {string} projectRoot
+ * @param {string[]} relativePaths
+ * @param {object | null} rootPackageJson
+ */
+export async function mergePackageDependencies(projectRoot, relativePaths, rootPackageJson) {
+  const merged = {
+    ...(rootPackageJson?.dependencies || {}),
+    ...(rootPackageJson?.devDependencies || {}),
+    ...(rootPackageJson?.peerDependencies || {}),
+  };
+
+  const pkgPaths = relativePaths.filter((p) => {
+    const n = p.replace(/\\/g, '/');
+    return n === 'package.json' || /\/package\.json$/.test(n);
+  });
+
+  for (const rel of pkgPaths) {
+    try {
+      const pkg = await fs.readJson(path.join(projectRoot, rel));
+      Object.assign(merged, pkg.dependencies || {}, pkg.devDependencies || {}, pkg.peerDependencies || {});
+    } catch {
+      /* skip invalid package.json */
+    }
+  }
+
+  return merged;
+}

package/src/scanner/repo-scanner.js

@@ -0,0 +1,143 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+import ignore from 'ignore';
+import { isSensitivePath } from '../core/config.js';
+import { hashContent } from './file-index.js';
+
+const SCAN_EXTENSIONS = new Set([
+  '.js', '.jsx', '.ts', '.tsx', '.mjs', '.cjs',
+  '.json', '.md', '.yml', '.yaml', '.sql',
+  '.prisma', '.graphql', '.vue', '.svelte',
+  '.py', '.go', '.rs', '.java', '.php',
+  '.css', '.scss', '.html', '.env.example',
+]);
+
+const ALWAYS_INCLUDE_FILES = new Set([
+  'package.json', 'tsconfig.json', 'dockerfile',
+  'docker-compose.yml', 'readme.md', 'schema.prisma',
+]);
+
+const PROJECT_DIRS = [
+  'src', 'app', 'pages', 'components', 'controllers',
+  'routes', 'services', 'models', 'middleware', 'hooks',
+  'utils', 'config', 'api', 'lib', 'server', 'client',
+  'features', 'modules', 'prisma',
+];
+
+async function walkDir(dir, projectRoot, ig, config, results, depth = 0) {
+  if (depth > 12) return;
+
+  let entries;
+  try {
+    entries = await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    const relativePath = path.relative(projectRoot, fullPath).replace(/\\/g, '/');
+
+    if (ig.ignores(relativePath) || ig.ignores(`${relativePath}/`)) {
+      continue;
+    }
+
+    if (isSensitivePath(relativePath)) {
+      continue;
+    }
+
+    if (entry.isDirectory()) {
+      await walkDir(fullPath, projectRoot, ig, config, results, depth + 1);
+      continue;
+    }
+
+    if (!entry.isFile()) continue;
+
+    const ext = path.extname(entry.name).toLowerCase();
+    const lowerName = entry.name.toLowerCase();
+
+    const shouldInclude =
+      SCAN_EXTENSIONS.has(ext) ||
+      ALWAYS_INCLUDE_FILES.has(lowerName) ||
+      lowerName.startsWith('vite.config') ||
+      lowerName.startsWith('next.config') ||
+      lowerName.startsWith('tailwind.config') ||
+      lowerName.startsWith('webpack.config') ||
+      lowerName === 'nginx.conf' ||
+      lowerName === 'dockerfile';
+
+    if (!shouldInclude) continue;
+
+    let stat;
+    try {
+      stat = await fs.stat(fullPath);
+    } catch {
+      continue;
+    }
+
+    if (stat.size > config.maxFileSizeBytes) continue;
+
+    let hash = 'skipped-large';
+    if (stat.size < config.maxFileSizeBytes) {
+      try {
+        const content = await fs.readFile(fullPath, 'utf8');
+        hash = hashContent(content);
+      } catch {
+        hash = 'unreadable';
+      }
+    }
+
+    results.files.push({
+      relativePath,
+      mtime: stat.mtimeMs,
+      size: stat.size,
+      hash,
+    });
+    results.relativePaths.push(relativePath);
+  }
+}
+
+/**
+ * @param {string} projectRoot
+ * @param {import('../core/config.js').DEFAULT_CONFIG} config
+ */
+export async function scanRepository(projectRoot, config) {
+  const ig = ignore().add(config.ignore);
+
+  const results = {
+    files: [],
+    relativePaths: [],
+    projectDirs: [],
+  };
+
+  for (const dir of PROJECT_DIRS) {
+    const full = path.join(projectRoot, dir);
+    if (await fs.pathExists(full)) {
+      const stat = await fs.stat(full);
+      if (stat.isDirectory()) {
+        results.projectDirs.push(dir);
+      }
+    }
+  }
+
+  await walkDir(projectRoot, projectRoot, ig, config, results);
+
+  const rootPackage = path.join(projectRoot, 'package.json');
+  if (await fs.pathExists(rootPackage)) {
+    const rel = 'package.json';
+    if (!results.relativePaths.includes(rel)) {
+      const stat = await fs.stat(rootPackage);
+      const content = await fs.readFile(rootPackage, 'utf8');
+      results.files.push({
+        relativePath: rel,
+        mtime: stat.mtimeMs,
+        size: stat.size,
+        hash: hashContent(content),
+      });
+      results.relativePaths.push(rel);
+    }
+  }
+
+  results.relativePaths.sort();
+  return results;
+}

package/src/services/ai/background.js

@@ -0,0 +1,87 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+import { getContextForgeRoot, getCacheDir } from '../../core/paths.js';
+import { enrichWithAI } from './enrich.js';
+import { writeArtifacts } from '../../generators/artifacts.js';
+import { renderContextMarkdown } from '../../generators/context-markdown.js';
+import { saveScanHash } from '../../core/snapshot-hash.js';
+import { installCursorRules } from '../../core/cursor-rules.js';
+import { loadConfig } from '../../core/config.js';
+
+function getAiCachePath(projectRoot) {
+  return path.join(getCacheDir(projectRoot), 'ai-enrichment.json');
+}
+
+function getAiStatusPath(projectRoot) {
+  return path.join(getCacheDir(projectRoot), 'ai-status.json');
+}
+
+/** Prevent duplicate background jobs */
+let activeJobs = new Set();
+
+/**
+ * Run AI enrichment in background; updates context.md when complete.
+ * @param {string} projectRoot
+ * @param {object} snapshot
+ * @param {object} config
+ */
+export function scheduleBackgroundEnrichment(projectRoot, snapshot, config, scanHash = null) {
+  const key = path.resolve(projectRoot);
+  if (activeJobs.has(key)) {
+    return;
+  }
+  activeJobs.add(key);
+
+  setImmediate(async () => {
+    try {
+      await fs.writeJson(getAiStatusPath(projectRoot), {
+        status: 'running',
+        startedAt: new Date().toISOString(),
+      });
+
+      console.log('[contextforge] AI enrichment running in background...');
+      const enrichment = await enrichWithAI(snapshot, config);
+
+      snapshot.aiEnrichment = enrichment;
+      await fs.writeJson(getAiCachePath(projectRoot), enrichment, { spaces: 2 });
+
+      await writeArtifacts(projectRoot, snapshot);
+      await renderContextMarkdown(projectRoot, snapshot);
+
+      const cfg = await loadConfig(projectRoot);
+      if (cfg.installCursorRules !== false) {
+        await installCursorRules(projectRoot, snapshot);
+      }
+
+      if (scanHash) await saveScanHash(projectRoot, scanHash);
+
+      await fs.writeJson(getAiStatusPath(projectRoot), {
+        status: 'completed',
+        provider: enrichment.provider,
+        model: enrichment.model,
+        completedAt: new Date().toISOString(),
+      });
+
+      console.log(
+        `[contextforge] AI enrichment done (${enrichment.provider}/${enrichment.model}) — context.md updated`
+      );
+    } catch (err) {
+      await fs.writeJson(getAiStatusPath(projectRoot), {
+        status: 'failed',
+        error: err.message,
+        failedAt: new Date().toISOString(),
+      });
+      console.error('[contextforge] AI enrichment failed:', err.message);
+    } finally {
+      activeJobs.delete(key);
+    }
+  });
+}
+
+export async function loadCachedEnrichment(projectRoot) {
+  const cachePath = getAiCachePath(projectRoot);
+  if (await fs.pathExists(cachePath)) {
+    return fs.readJson(cachePath);
+  }
+  return null;
+}

package/src/services/ai/config.js

@@ -0,0 +1,48 @@
+export const DEFAULT_AI_CONFIG = {
+  enabled: false,
+  /** Primary provider: "openai" | "groq" */
+  provider: 'openai',
+  /** Try this if primary fails */
+  fallbackProvider: 'groq',
+  enrichContext: true,
+  /** Run AI after scan without blocking CLI (updates context.md when done) */
+  background: true,
+  openai: {
+    model: 'gpt-4o-mini',
+  },
+  groq: {
+    model: 'llama-3.3-70b-versatile',
+  },
+  maxTokens: 6144,
+  timeoutMs: 120000,
+  /** Folder for editable prompts (relative to project root) */
+  promptsDir: '.contextforge/prompts',
+};
+
+import { isAiEnabledViaEnv, getEnvModel } from '../../core/env.js';
+
+export function resolveAiConfig(config) {
+  const ai = { ...DEFAULT_AI_CONFIG, ...(config.ai || {}) };
+  ai.openai = { ...DEFAULT_AI_CONFIG.openai, ...(config.ai?.openai || {}) };
+  ai.groq = { ...DEFAULT_AI_CONFIG.groq, ...(config.ai?.groq || {}) };
+
+  const envProvider = process.env.CONTEXTFORGE_AI_PROVIDER;
+  if (envProvider === 'openai' || envProvider === 'groq') {
+    ai.provider = envProvider;
+  }
+
+  const envEnabled = isAiEnabledViaEnv();
+  if (envEnabled !== null) {
+    ai.enabled = envEnabled;
+  }
+
+  ai.openai.model = getEnvModel('openai', ai.openai.model);
+  ai.groq.model = getEnvModel('groq', ai.groq.model);
+
+  return ai;
+}
+
+export function getModelForProvider(aiConfig, provider) {
+  if (provider === 'groq') return aiConfig.groq.model;
+  return aiConfig.openai.model;
+}

package/src/services/ai/enrich.js

@@ -0,0 +1,87 @@
+import { getApiKey } from '../../core/env.js';
+import { resolveAiConfig, getModelForProvider } from './config.js';
+import { buildEnrichmentMessages, parseAiResponse } from './prompt.js';
+import { callOpenAI } from './providers/openai.js';
+import { callGroq } from './providers/groq.js';
+import { validateAiSections } from './validate.js';
+
+const PROVIDERS = {
+  openai: callOpenAI,
+  groq: callGroq,
+};
+
+const MAX_RETRIES = 2;
+
+/**
+ * @param {object} snapshot
+ * @param {object} config
+ */
+export async function enrichWithAI(snapshot, config) {
+  const aiConfig = resolveAiConfig(config);
+
+  if (!aiConfig.enabled || !aiConfig.enrichContext) {
+    return null;
+  }
+
+  const providersToTry = [aiConfig.provider];
+  if (aiConfig.fallbackProvider && aiConfig.fallbackProvider !== aiConfig.provider) {
+    providersToTry.push(aiConfig.fallbackProvider);
+  }
+
+  let lastError = null;
+
+  for (const providerName of providersToTry) {
+    if (!PROVIDERS[providerName]) continue;
+    if (!getApiKey(providerName)) {
+      lastError = new Error(`${providerName.toUpperCase()} API key missing`);
+      continue;
+    }
+
+      const model = getModelForProvider(aiConfig, providerName);
+
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        const messages = await buildEnrichmentMessages(snapshot, config, attempt > 0);
+
+        const result = await PROVIDERS[providerName](messages, {
+          model,
+          maxTokens: aiConfig.maxTokens,
+          timeoutMs: aiConfig.timeoutMs,
+        });
+
+        const parsed = parseAiResponse(result.content);
+        const validation = validateAiSections(parsed, snapshot);
+
+        if (!validation.valid && attempt < MAX_RETRIES) {
+          lastError = new Error(validation.errors.join('; '));
+          continue;
+        }
+
+        if (!validation.valid) {
+          throw new Error(`AI validation failed: ${validation.errors.join('; ')}`);
+        }
+
+        return {
+          provider: result.provider,
+          model: result.model,
+          generatedAt: new Date().toISOString(),
+          usage: result.usage,
+          sections: validation.parsed,
+          validated: true,
+          attempt: attempt + 1,
+        };
+      } catch (err) {
+        lastError = err;
+        if (attempt >= MAX_RETRIES) break;
+      }
+    }
+  }
+
+  throw lastError || new Error('No AI provider available. Set OPENAI_API_KEY or GROQ_API_KEY in .env');
+}
+
+export function isAiAvailable(config) {
+  const aiConfig = resolveAiConfig(config);
+  if (!aiConfig.enabled) return false;
+  return !!(getApiKey('openai') || getApiKey('groq'));
+}

package/src/services/ai/index.js

@@ -0,0 +1,3 @@
+export { enrichWithAI, isAiAvailable } from './enrich.js';
+export { scheduleBackgroundEnrichment, loadCachedEnrichment } from './background.js';
+export { resolveAiConfig, DEFAULT_AI_CONFIG } from './config.js';

package/src/services/ai/prompt-loader.js

@@ -0,0 +1,104 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const PACKAGE_PROMPTS = path.join(__dirname, '../../..', 'prompts');
+
+const PROMPT_FILES = {
+  system: 'system.md',
+  userTemplate: 'user-template.md',
+  responseSchema: 'response-schema.md',
+  retryAddon: 'retry-addon.md',
+};
+
+/**
+ * Resolve prompts directory: project .contextforge/prompts or config override.
+ * @param {string} projectRoot
+ * @param {object} aiConfig
+ */
+export function resolvePromptsDir(projectRoot, aiConfig) {
+  const rel = aiConfig?.promptsDir || '.contextforge/prompts';
+  const projectPrompts = path.isAbsolute(rel)
+    ? rel
+    : path.join(projectRoot, rel);
+  return projectPrompts;
+}
+
+async function readPromptFile(dir, filename, fallbackDir) {
+  const primary = path.join(dir, filename);
+  if (await fs.pathExists(primary)) {
+    return fs.readFile(primary, 'utf8');
+  }
+  const fallback = path.join(fallbackDir, filename);
+  if (await fs.pathExists(fallback)) {
+    return fs.readFile(fallback, 'utf8');
+  }
+  return null;
+}
+
+/**
+ * Load all prompt templates from disk.
+ * @param {string} projectRoot
+ * @param {object} aiConfig
+ */
+export async function loadPrompts(projectRoot, aiConfig) {
+  const promptsDir = resolvePromptsDir(projectRoot, aiConfig);
+  const fallback = PACKAGE_PROMPTS;
+
+  const [system, userTemplate, responseSchema, retryAddon] = await Promise.all([
+    readPromptFile(promptsDir, PROMPT_FILES.system, fallback),
+    readPromptFile(promptsDir, PROMPT_FILES.userTemplate, fallback),
+    readPromptFile(promptsDir, PROMPT_FILES.responseSchema, fallback),
+    readPromptFile(promptsDir, PROMPT_FILES.retryAddon, fallback),
+  ]);
+
+  if (!system || !userTemplate) {
+    throw new Error(
+      `Prompt files missing in ${promptsDir}. Run: npx contextforge init`
+    );
+  }
+
+  return {
+    promptsDir,
+    system: system.trim(),
+    userTemplate: userTemplate.trim(),
+    responseSchema: (responseSchema || '').trim(),
+    retryAddon: (retryAddon || '').trim(),
+  };
+}
+
+/**
+ * Build user prompt from template + placeholders.
+ */
+export function renderUserPrompt(template, { scanData, projectName, retryNote }) {
+  return template
+    .replace(/\{\{SCAN_DATA\}\}/g, scanData)
+    .replace(/\{\{PROJECT_NAME\}\}/g, projectName || 'project')
+    .replace(/\{\{RETRY_NOTE\}\}/g, retryNote || '')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+/**
+ * Full messages array for chat completion.
+ */
+export function buildMessages(prompts, { scanData, projectName, isRetry }) {
+  const retryNote = isRetry ? prompts.retryAddon : '';
+  let userContent = renderUserPrompt(prompts.userTemplate, {
+    scanData,
+    projectName,
+    retryNote,
+  });
+
+  if (prompts.responseSchema) {
+    userContent += `\n\n## Required response schema\n\n${prompts.responseSchema}`;
+  }
+
+  return [
+    { role: 'system', content: prompts.system },
+    { role: 'user', content: userContent },
+  ];
+}
+
+export { PACKAGE_PROMPTS, PROMPT_FILES };

package/src/services/ai/prompt.js

@@ -0,0 +1,42 @@
+import { sanitizeSnapshotForAi } from './redact.js';
+import { loadPrompts, buildMessages } from './prompt-loader.js';
+
+/**
+ * Build chat messages from external prompt files.
+ * @param {object} snapshot
+ * @param {object} config - full project config
+ * @param {boolean} isRetry
+ */
+export async function buildEnrichmentMessages(snapshot, config, isRetry = false) {
+  const aiConfig = config.ai || {};
+  const prompts = await loadPrompts(snapshot.projectRoot, aiConfig);
+  const scanData = sanitizeSnapshotForAi(snapshot);
+
+  return buildMessages(prompts, {
+    scanData,
+    projectName: snapshot.projectName,
+    isRetry,
+  });
+}
+
+/** @deprecated Use buildEnrichmentMessages — kept for tests */
+export function buildEnrichmentPrompt(snapshot, isRetry = false) {
+  const scanData = sanitizeSnapshotForAi(snapshot);
+  const payload = JSON.parse(scanData);
+  Object.assign(payload, {
+    confidence: snapshot.confidence,
+    changelog: snapshot.changelog?.summary,
+    astRoutes: snapshot.ast?.routes?.slice(0, 20),
+  });
+  const retryNote = isRetry ? '\nRETRY: Output complete valid JSON.\n' : '';
+  return `Analyze ${snapshot.projectName}.\n${retryNote}\n${JSON.stringify(payload, null, 2)}`;
+}
+
+export function parseAiResponse(text) {
+  const trimmed = text.trim();
+  const jsonMatch = trimmed.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) {
+    throw new Error('AI response did not contain JSON');
+  }
+  return JSON.parse(jsonMatch[0]);
+}

package/src/services/ai/providers/groq.js

@@ -0,0 +1,36 @@
+import OpenAI from 'openai';
+import { getApiKey } from '../../../core/env.js';
+
+const GROQ_BASE_URL = 'https://api.groq.com/openai/v1';
+
+/**
+ * Groq uses OpenAI-compatible API.
+ * @param {{ model: string, maxTokens: number, timeoutMs: number }} options
+ */
+export async function callGroq(messages, options) {
+  const apiKey = getApiKey('groq');
+  if (!apiKey) {
+    throw new Error('GROQ_API_KEY is not set in .env');
+  }
+
+  const client = new OpenAI({
+    apiKey,
+    baseURL: GROQ_BASE_URL,
+    timeout: options.timeoutMs,
+  });
+
+  const response = await client.chat.completions.create({
+    model: options.model,
+    messages,
+    max_tokens: options.maxTokens,
+    temperature: 0.3,
+    response_format: { type: 'json_object' },
+  });
+
+  return {
+    content: response.choices[0]?.message?.content || '',
+    model: response.model,
+    provider: 'groq',
+    usage: response.usage,
+  };
+}