@pithyjs/codex 0.1.0-beta.0

package/dist/sync.js

@@ -0,0 +1,318 @@
+#!/usr/bin/env node
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.sync",
+ *   "title": "Codex Sync",
+ *   "category": "feature"
+ * }
+ *
+ * Orchestrates full sync workflow (scan, generate, write entries).
+ */
+import fs from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { scan } from './scan.js';
+import { codexSchema } from './schema.js';
+import { askLLM } from './llm.js';
+import { CHANGED_ONLY_PATHSPEC } from './changed-scope.js';
+import { simpleGit } from 'simple-git';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const nowIso = () => new Date().toISOString();
+async function loadIndex(root) {
+    try {
+        const raw = await fs.readFile(join(root, 'codex.index.json'), 'utf8');
+        const j = JSON.parse(raw);
+        return (j.byId || {});
+    }
+    catch {
+        return {};
+    }
+}
+async function loadProposalHashes(root) {
+    const proposalsDir = join(root, 'codex', '.proposals');
+    const hashes = {};
+    try {
+        const files = await fs.readdir(proposalsDir);
+        const jsonFiles = files.filter(f => f.endsWith('.json'));
+        for (const file of jsonFiles) {
+            try {
+                const content = await fs.readFile(join(proposalsDir, file), 'utf8');
+                const proposal = JSON.parse(content);
+                if (proposal.id && proposal.meta?.hash) {
+                    hashes[proposal.id] = proposal.meta.hash;
+                }
+            }
+            catch {
+                // Skip malformed proposals
+            }
+        }
+    }
+    catch {
+        // Proposals directory doesn't exist yet
+    }
+    return hashes;
+}
+function determineChangeStatus(currentHash, existingHash, proposalHash) {
+    // If no previous hash exists, it's new
+    if (!existingHash && !proposalHash) {
+        return 'NEW';
+    }
+    // Compare with most recent hash (proposal takes precedence over applied)
+    const prevHash = proposalHash || existingHash;
+    if (currentHash !== prevHash) {
+        return 'CHANGED';
+    }
+    return 'UNCHANGED';
+}
+async function lastModifiedISO(root, files) {
+    const git = simpleGit(root);
+    try {
+        const rel = files.map(f => {
+            const r = f.startsWith(root) ? f.slice(root.length + 1) : f;
+            return r.replace(/\\/g, '/');
+        });
+        const res = await git.raw(['log', '-1', '--format=%cI', '--', ...rel]);
+        const iso = res.trim();
+        return iso || null;
+    }
+    catch {
+        return null;
+    }
+}
+function needsUpdate(entryUpdatedISO, codeISO) {
+    if (!entryUpdatedISO && codeISO)
+        return true;
+    if (!codeISO)
+        return false;
+    return new Date(codeISO) > new Date(entryUpdatedISO || 0);
+}
+function buildPrompt(existing, id, category, tags, codeContext) {
+    const rules = `
+Return a single JSON object for this schema:
+{ id, title, category, tags[], updated, related[] }
+
+Rules:
+- id must be exactly "${id}".
+- category must be "${category}".
+- Merge these tags first: [${tags.join(', ')}].
+- updated must be current ISO datetime.
+- related: propose 0–3 likely related ids (or [] if unsure).
+- If an existing entry is provided, keep fields consistent and improve only where helpful (title/tags/related). Do not invent unrelated info.
+Return ONLY JSON.
+`;
+    const existingBlock = existing
+        ? `\nExisting entry:\n${JSON.stringify(existing, null, 2)}\n`
+        : '';
+    return [
+        {
+            role: 'system',
+            content: 'You are CodexSync. Keep entries concise and accurate.',
+        },
+        {
+            role: 'user',
+            content: `${rules}\n${existingBlock}\nCode context (truncated):\n${codeContext}`,
+        },
+    ];
+}
+async function extractContext(files) {
+    const chunks = [];
+    for (const f of files.slice(0, 20)) {
+        const src = await fs.readFile(f, 'utf8').catch(() => '');
+        if (!src)
+            continue;
+        const head = src.split('\n').slice(0, 200).join('\n');
+        chunks.push(`// ${f}\n${head}`);
+        if (chunks.join('\n\n').length > 120_000)
+            break;
+    }
+    return chunks.join('\n\n');
+}
+/**
+ * Deduplicates APIs by name+signature, keeping the first occurrence.
+ */
+function deduplicateApis(apis) {
+    const seen = new Set();
+    return apis.filter(api => {
+        const key = `${api.name}::${api.signature || ''}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/**
+ * Builds a codex entry directly from scan data without LLM enrichment.
+ * All required fields (id, title, category) come from @codex annotations;
+ * APIs come from @codexApi annotations. No AI needed.
+ */
+function buildEntryFromScan(c, existing) {
+    return {
+        id: c.id,
+        title: c.title || existing?.title || c.id.split('.').pop() || c.id,
+        category: c.category,
+        tags: Array.from(new Set([...(c.tags || []), ...(existing?.tags || [])])),
+        updated: nowIso(),
+        public: existing?.public ?? true,
+        status: existing?.status ?? 'draft',
+        version: existing?.version ?? '1.0',
+        related: existing?.related ?? [],
+        dependsOn: existing?.dependsOn ?? [],
+        apis: deduplicateApis(c.apis && c.apis.length > 0 ? c.apis : (existing?.apis ?? [])),
+        rids: existing?.rids ?? [],
+        acceptance: existing?.acceptance ?? [],
+        tests_expected: existing?.tests_expected ?? {
+            unit: [],
+            integration: [],
+            e2e: [],
+            property: [],
+            contract: [],
+        },
+        meta: c.hash ? { hash: c.hash } : (existing?.meta ?? {}),
+    };
+}
+async function main() {
+    const rootArg = process.argv.includes('--root')
+        ? process.argv[process.argv.indexOf('--root') + 1]
+        : '../../..';
+    // Resolve relative to __dirname (packages/codex/dist/)
+    const root = join(__dirname, rootArg);
+    const mode = process.argv.includes('--mode')
+        ? process.argv[process.argv.indexOf('--mode') + 1]
+        : 'batch'; // "batch" | "one"
+    const targetId = process.argv.includes('--id')
+        ? process.argv[process.argv.indexOf('--id') + 1]
+        : null;
+    const noLlm = process.argv.includes('--no-llm');
+    const direct = process.argv.includes('--direct');
+    const changedOnly = process.argv.includes('--changed-only');
+    const baseBranch = process.argv.includes('--base')
+        ? process.argv[process.argv.indexOf('--base') + 1]
+        : 'origin/master';
+    // --direct writes to codex/ directly (skips proposals/review/apply cycle)
+    // --no-llm without --direct still uses the proposals workflow
+    const outputDir = direct
+        ? join(root, 'codex')
+        : join(root, 'codex', '.proposals');
+    await fs.mkdir(outputDir, { recursive: true });
+    // When --changed-only, use git diff to find changed source files
+    let changedFiles;
+    if (changedOnly) {
+        const git = simpleGit(root);
+        try {
+            const diff = await git.raw([
+                'diff',
+                '--name-only',
+                baseBranch + '...HEAD',
+                '--',
+                ...CHANGED_ONLY_PATHSPEC,
+            ]);
+            changedFiles = diff
+                .trim()
+                .split('\n')
+                .filter(Boolean)
+                .map(f => join(root, f));
+            console.log(`[Codex] Changed files: ${changedFiles.length} (vs ${baseBranch})`);
+            if (changedFiles.length === 0) {
+                console.log('No source files changed. Nothing to sync.');
+                return;
+            }
+        }
+        catch {
+            console.warn(`[Codex] Failed to detect changed files, falling back to full scan`);
+            changedFiles = undefined;
+        }
+    }
+    const indexById = await loadIndex(root);
+    const proposalHashes = await loadProposalHashes(root);
+    const candidates = (await scan(root, changedFiles)).filter(c => !targetId || c.id === targetId);
+    let generated = 0;
+    let skipped = 0;
+    for (const c of candidates) {
+        const existing = indexById[c.id] ?? null;
+        // Enhanced change detection using content hashes
+        if (c.hash) {
+            const existingHash = existing?.meta?.hash || null;
+            const proposalHash = proposalHashes[c.id] || null;
+            const changeStatus = determineChangeStatus(c.hash, existingHash, proposalHash);
+            if (changeStatus === 'UNCHANGED') {
+                skipped++;
+                if (process.env.NODE_ENV === 'development') {
+                    console.log(`[Codex] Skipping unchanged: ${c.id}`);
+                }
+                continue;
+            }
+            if (process.env.NODE_ENV === 'development') {
+                console.log(`[Codex] Processing ${changeStatus}: ${c.id}`);
+            }
+        }
+        else {
+            // Fallback to legacy timestamp-based detection for glob-only candidates
+            const lastCodeISO = await lastModifiedISO(root, c.files);
+            const entryUpdated = existing?.updated ?? null;
+            if (existing && !needsUpdate(entryUpdated, lastCodeISO)) {
+                skipped++;
+                continue;
+            }
+        }
+        let json;
+        if (noLlm) {
+            // Build entry directly from annotations — no AI required
+            json = buildEntryFromScan(c, existing);
+        }
+        else {
+            const ctx = await extractContext(c.files);
+            const messages = buildPrompt(existing, c.id, c.category, c.tags, ctx);
+            const text = await askLLM(messages);
+            try {
+                json = JSON.parse(text);
+            }
+            catch {
+                continue;
+            }
+            json.id = c.id;
+            json.category = c.category;
+            json.updated = nowIso();
+            json.tags = Array.from(new Set([...(c.tags || []), ...(json.tags || [])]));
+            // Add discovered APIs if available
+            if (c.apis && c.apis.length > 0) {
+                json.apis = c.apis;
+            }
+            // Add content hash for change detection
+            if (c.hash) {
+                json.meta = { hash: c.hash };
+            }
+        }
+        const parsed = codexSchema.safeParse(json);
+        if (!parsed.success) {
+            if (process.env.NODE_ENV === 'development') {
+                console.warn(`[Codex] Schema validation failed for ${c.id}:`, parsed.error);
+            }
+            continue;
+        }
+        await fs.writeFile(join(outputDir, `${c.id}.json`), JSON.stringify(parsed.data, null, 2));
+        generated++;
+        if (mode === 'one')
+            break;
+    }
+    if (direct) {
+        console.log(`Codex entries written directly (generated ${generated}, skipped ${skipped} unchanged).`);
+    }
+    else {
+        // Regenerate review.md for whatever exists in .proposals
+        const { execa } = await import('execa');
+        try {
+            await execa('node', [join(__dirname, 'review.js'), '--root', root], {
+                stdio: 'inherit',
+            });
+        }
+        catch {
+            void 0;
+        }
+        console.log(`Proposals ready in codex/.proposals (generated ${generated}, skipped ${skipped} unchanged).`);
+    }
+}
+main().catch(e => {
+    console.error(e);
+    process.exit(1);
+});

package/dist/validate-cli.d.ts

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.validate-cli",
+ *   "title": "Validate CLI",
+ *   "category": "plugin"
+ * }
+ *
+ * CLI for validating extracted documentation examples.
+ * Checks that TypeScript/JavaScript examples are syntactically valid.
+ *
+ * Usage:
+ *   pnpm codex validate [options]
+ *
+ * Options:
+ *   --root <path>       Project root directory (default: cwd)
+ *   --verbose           Show detailed progress
+ */
+export {};

package/dist/validate-cli.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.validate-cli",
+ *   "title": "Validate CLI",
+ *   "category": "plugin"
+ * }
+ *
+ * CLI for validating extracted documentation examples.
+ * Checks that TypeScript/JavaScript examples are syntactically valid.
+ *
+ * Usage:
+ *   pnpm codex validate [options]
+ *
+ * Options:
+ *   --root <path>       Project root directory (default: cwd)
+ *   --verbose           Show detailed progress
+ */
+import { resolve } from 'node:path';
+import fs from 'node:fs/promises';
+import { runExtractionPipeline } from './extraction/pipeline.js';
+import { validateExtractionResult, formatValidationReport, } from './validate.js';
+function parseArgs(args) {
+    const options = {
+        root: process.cwd(),
+        verbose: false,
+    };
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg.startsWith('--root=')) {
+            options.root = arg.slice('--root='.length);
+            continue;
+        }
+        switch (arg) {
+            case '--root': {
+                const next = args[i + 1];
+                if (!next || next.startsWith('-')) {
+                    console.error('Error: --root option requires a path argument.');
+                    process.exit(1);
+                }
+                options.root = next;
+                i++;
+                break;
+            }
+            case '--verbose':
+            case '-v':
+                options.verbose = true;
+                break;
+            case '--help':
+            case '-h':
+                printHelp();
+                process.exit(0);
+                break;
+            default:
+                if (arg.startsWith('-')) {
+                    console.warn(`Warning: unknown option "${arg}" (ignored)`);
+                }
+        }
+    }
+    return options;
+}
+function printHelp() {
+    console.log(`
+Codex Validate
+
+Validates that extracted documentation examples are syntactically correct.
+TypeScript and JavaScript examples are parsed for syntax errors.
+
+Usage:
+  pithy-codex-validate [options]
+  pnpm --filter @pithyjs/codex validate [-- options]
+
+Options:
+  --root <path>       Project root directory (default: current directory)
+  --verbose, -v       Show detailed progress
+  --help, -h          Show this help message
+
+Examples:
+  # Validate examples in project
+  pithy-codex-validate --root ../..
+
+  # Validate with verbose output
+  pithy-codex-validate --root ../.. --verbose
+
+Exit Codes:
+  0 - All examples are valid
+  1 - One or more examples have syntax errors
+`);
+}
+async function main() {
+    const args = process.argv.slice(2);
+    const options = parseArgs(args);
+    options.root = resolve(options.root);
+    // Verify root exists
+    try {
+        const stat = await fs.stat(options.root);
+        if (!stat.isDirectory()) {
+            console.error(`❌ Error: "${options.root}" is not a directory`);
+            process.exit(1);
+        }
+    }
+    catch {
+        console.error(`❌ Error: Root directory "${options.root}" does not exist`);
+        process.exit(1);
+    }
+    if (options.verbose) {
+        console.log('🔍 Codex Validate');
+        console.log(`   Root: ${options.root}`);
+        console.log('');
+    }
+    try {
+        // Run extraction first to get examples
+        if (options.verbose) {
+            console.log('📂 Running extraction pipeline...');
+        }
+        const extraction = await runExtractionPipeline({
+            root: options.root,
+            analyzeTests: true,
+            linkReadmes: true,
+            extractRunnableExamples: true,
+        });
+        if (options.verbose) {
+            console.log(`   Found ${extraction.stats.componentsFound} components with ${extraction.stats.apisExtracted} APIs`);
+            console.log('');
+            console.log('📋 Validating examples...');
+        }
+        // Validate
+        const result = validateExtractionResult(extraction);
+        // Report
+        console.log(formatValidationReport(result));
+        if (result.invalidCount > 0) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        console.error('❌ Validation failed:', error instanceof Error ? error.message : error);
+        process.exit(1);
+    }
+}
+main().catch(err => {
+    console.error('Fatal error:', err);
+    process.exit(1);
+});

package/dist/validate.d.ts

@@ -0,0 +1,76 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.validate",
+ *   "title": "Documentation Validator",
+ *   "category": "feature"
+ * }
+ *
+ * Validates extracted documentation examples by checking they are
+ * syntactically valid TypeScript. Used by `pnpm codex validate` and CI drift checks.
+ */
+import type { ExtendedExtractionResult } from './extraction/types.js';
+/**
+ * Result of validating a single example
+ */
+export interface ExampleValidationResult {
+    /** The example code that was validated */
+    code: string;
+    /** Component the example belongs to */
+    componentId: string;
+    /** API name (if applicable) */
+    apiName?: string;
+    /** Whether the example is syntactically valid */
+    valid: boolean;
+    /** Diagnostic messages if invalid */
+    diagnostics: string[];
+}
+/**
+ * Aggregate result of validating all examples
+ */
+export interface ValidationResult {
+    /** Total examples checked */
+    totalExamples: number;
+    /** Number of valid examples */
+    validCount: number;
+    /** Number of invalid examples */
+    invalidCount: number;
+    /** Number of examples skipped (non-TypeScript/JavaScript) */
+    skippedCount: number;
+    /** Individual results for invalid examples only */
+    invalidExamples: ExampleValidationResult[];
+    /** Total validation time in ms */
+    totalTimeMs: number;
+}
+/**
+ * @codexApi {"parent":"pithy.codex.validate","name":"validateExampleSyntax","stability":"stable","signature":"(code: string, componentId?: string, apiName?: string, language?: string) => ExampleValidationResult"}
+ *
+ * Validates a single code example for syntactic correctness using TypeScript compiler.
+ * Does NOT type-check — only parses for syntax errors.
+ *
+ * @param code - The example code to validate
+ * @param componentId - The ID of the component the example belongs to (default: "")
+ * @param apiName - The name of the API the example belongs to
+ * @param language - The language of the example (default: "typescript")
+ * @returns Validation result with diagnostics if invalid
+ */
+export declare function validateExampleSyntax(code: string, componentId?: string, apiName?: string, language?: string): ExampleValidationResult;
+/**
+ * @codexApi {"parent":"pithy.codex.validate","name":"validateExtractionResult","stability":"stable","signature":"(result: ExtendedExtractionResult) => ValidationResult"}
+ *
+ * Validates all extracted examples from an extraction pipeline result.
+ * Checks that TypeScript/JavaScript examples are syntactically valid.
+ *
+ * @param result - The extraction result to validate
+ * @returns Aggregate validation result
+ */
+export declare function validateExtractionResult(result: ExtendedExtractionResult): ValidationResult;
+/**
+ * @codexApi {"parent":"pithy.codex.validate","name":"formatValidationReport","stability":"stable","signature":"(result: ValidationResult) => string"}
+ *
+ * Formats a human-readable validation report.
+ *
+ * @param result - Validation result to format
+ * @returns Formatted report string
+ */
+export declare function formatValidationReport(result: ValidationResult): string;