@pithyjs/codex 0.1.0-beta.0

package/dist/extraction/types.d.ts

@@ -0,0 +1,401 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction.types",
+ *   "title": "Extraction Types",
+ *   "category": "types"
+ * }
+ */
+/**
+ * Source location information for linking docs to code
+ */
+export interface SourceLocation {
+    file: string;
+    line: number;
+    column?: number;
+    endLine?: number;
+    endColumn?: number;
+}
+/**
+ * Parsed JSDoc/TSDoc parameter documentation
+ */
+export interface ParamDoc {
+    name: string;
+    type?: string;
+    description: string;
+    optional?: boolean;
+    defaultValue?: string;
+}
+/**
+ * Parsed JSDoc/TSDoc return documentation
+ */
+export interface ReturnDoc {
+    type?: string;
+    description: string;
+}
+/**
+ * Extracted example block from documentation
+ */
+export interface ExtractedExample {
+    code: string;
+    language: string;
+    description?: string;
+    runnable: boolean;
+    isDoctest: boolean;
+    location: SourceLocation;
+}
+/**
+ * Parsed JSDoc/TSDoc block with all metadata
+ */
+export interface ParsedJSDoc {
+    description: string;
+    params: ParamDoc[];
+    returns?: ReturnDoc;
+    examples: ExtractedExample[];
+    throws?: string[];
+    see?: string[];
+    since?: string;
+    deprecated?: string | boolean;
+    tags: Record<string, string[]>;
+    location: SourceLocation;
+}
+/**
+ * Test case extracted from test files
+ */
+export interface ExtractedTestCase {
+    name: string;
+    description?: string;
+    type: 'describe' | 'it' | 'test';
+    location: SourceLocation;
+    parentDescribe?: string;
+    code: string;
+    assertions: string[];
+}
+/**
+ * Test file analysis result
+ */
+export interface TestFileAnalysis {
+    file: string;
+    testCases: ExtractedTestCase[];
+    imports: string[];
+    coveredApis: string[];
+}
+/**
+ * Enriched API entry with extracted documentation
+ */
+export interface EnrichedApiEntry {
+    id: string;
+    name: string;
+    parent: string;
+    signature?: string;
+    stability: 'internal' | 'experimental' | 'stable' | 'deprecated';
+    jsdoc?: ParsedJSDoc;
+    examples: ExtractedExample[];
+    testRefs: TestReference[];
+    sourceLocation: SourceLocation;
+}
+/**
+ * Reference to a test that covers an API
+ */
+export interface TestReference {
+    file: string;
+    line: number;
+    name: string;
+    type: 'unit' | 'integration' | 'e2e';
+}
+/**
+ * Enriched component with all extracted data
+ */
+export interface EnrichedComponent {
+    id: string;
+    title: string;
+    category: string;
+    risk?: RiskLevel;
+    testing?: TestingRequirements;
+    description?: string;
+    apis: EnrichedApiEntry[];
+    examples: ExtractedExample[];
+    testRefs: TestReference[];
+    sourceFiles: SourceLocation[];
+    readmeSections?: ReadmeSection[];
+}
+/**
+ * Section extracted from README files
+ */
+export interface ReadmeSection {
+    heading: string;
+    level: number;
+    content: string;
+    codeBlocks: ExtractedExample[];
+    linkedEntryId?: string;
+}
+/**
+ * Configuration for the extraction pipeline
+ */
+export interface ExtractionConfig {
+    /** Root directory to scan */
+    root: string;
+    /** Glob patterns for source files */
+    sourcePatterns: string[];
+    /** Glob patterns for test files */
+    testPatterns: string[];
+    /** Glob patterns for README files */
+    readmePatterns: string[];
+    /** Whether to extract examples as runnable */
+    extractRunnableExamples: boolean;
+    /** Whether to parse test files for coverage */
+    analyzeTests: boolean;
+    /** Whether to link README sections */
+    linkReadmes: boolean;
+    /** Optional path to vitest JSON output for test status matching */
+    vitestOutputPath?: string;
+}
+/**
+ * Result of the extraction pipeline
+ */
+export interface ExtractionResult {
+    components: EnrichedComponent[];
+    testAnalysis: TestFileAnalysis[];
+    readmeSections: ReadmeSection[];
+    stats: ExtractionStats;
+}
+/**
+ * Statistics from the extraction process
+ */
+export interface ExtractionStats {
+    filesScanned: number;
+    componentsFound: number;
+    apisExtracted: number;
+    examplesExtracted: number;
+    testsAnalyzed: number;
+    readmeSectionsLinked: number;
+    processingTimeMs: number;
+}
+/**
+ * Default extraction configuration (all fields except root)
+ */
+export declare const DEFAULT_EXTRACTION_CONFIG: Omit<ExtractionConfig, 'root'>;
+/**
+ * Extracted TypeScript type/interface definition
+ */
+export interface ExtractedTypeDefinition {
+    name: string;
+    kind: 'interface' | 'type' | 'enum' | 'class';
+    exported: boolean;
+    generics?: string[];
+    extends?: string[];
+    implements?: string[];
+    members: TypeMember[];
+    jsdoc?: ParsedJSDoc;
+    location: SourceLocation;
+}
+/**
+ * Member of a type/interface/class
+ */
+export interface TypeMember {
+    name: string;
+    type: string;
+    optional: boolean;
+    readonly: boolean;
+    description?: string;
+    signature?: string;
+}
+/**
+ * Test example extracted via @codex:example marker
+ */
+export interface TestExample {
+    /** The codex entry ID this example belongs to */
+    entryId: string;
+    /** Name/description of the example */
+    name: string;
+    /** The actual test code */
+    code: string;
+    /** Source location in the test file */
+    location: SourceLocation;
+    /** Test file path */
+    testFile: string;
+    /** Whether this test is passing */
+    status?: 'passing' | 'failing' | 'skipped' | 'unknown';
+    /** Error message if failing */
+    error?: string;
+}
+/**
+ * Test status from vitest/jest output
+ */
+export interface TestStatus {
+    file: string;
+    testName: string;
+    status: 'passed' | 'failed' | 'skipped';
+    duration?: number;
+    error?: string;
+}
+/**
+ * Extended extraction result with type definitions and test examples
+ */
+export interface ExtendedExtractionResult extends ExtractionResult {
+    typeDefinitions: ExtractedTypeDefinition[];
+    testExamples: TestExample[];
+    testStatuses: TestStatus[];
+}
+/**
+ * Marker type for auto-populated README sections
+ */
+export type ReadmeMarkerType = 'install' | 'examples' | 'api' | 'bundle' | 'testing';
+/**
+ * Parsed README marker with its attributes and position
+ */
+export interface ReadmeMarker {
+    /** Marker type: install, examples, api, bundle, testing */
+    type: ReadmeMarkerType;
+    /** Raw attributes from the marker tag */
+    attributes: Record<string, string>;
+    /** The original opening tag string (preserved for idempotent rewrites) */
+    openTag: string;
+    /** Start position (character index) in the README content */
+    startIndex: number;
+    /** End position (character index) after <!-- @codex:end --> */
+    endIndex: number;
+    /** The full original content between markers (including markers themselves) */
+    originalContent: string;
+    /** Just the content between markers (excluding marker tags) */
+    innerContent: string;
+}
+/**
+ * Result of syncing a single README file
+ */
+export interface ReadmeSyncResult {
+    /** Path to the README file */
+    filePath: string;
+    /** Whether any content was changed */
+    changed: boolean;
+    /** Number of markers processed */
+    markersProcessed: number;
+    /** Number of markers that generated new content */
+    markersUpdated: number;
+    /** Markers that could not be processed (with reasons) */
+    warnings: string[];
+    /** The updated content (only if changed) */
+    updatedContent?: string;
+}
+/**
+ * Generic logger interface used by codex CLI tools and pipelines.
+ */
+export interface CodexLogger {
+    log(message: string): void;
+    warn(message: string): void;
+    error(message: string): void;
+}
+/**
+ * Configuration for README sync.
+ * All fields are optional — `syncAllReadmes` provides sensible defaults.
+ */
+export interface ReadmeSyncConfig {
+    /** Project root directory (default: cwd) */
+    root?: string;
+    /** Glob patterns for README files to sync (default: `packages/＊/README.md`, `README.md`) */
+    readmePatterns?: string[];
+    /** Whether to write changes to disk or just report (default: false) */
+    dryRun?: boolean;
+    /** Whether to show verbose output (default: false) */
+    verbose?: boolean;
+}
+/**
+ * Individual API entry within a snapshot
+ */
+export interface SnapshotEntry {
+    /** API name */
+    name: string;
+    /** Parent component/module id */
+    parent: string;
+    /** Qualified name: parent.name */
+    qualifiedName: string;
+    /** Function/type signature */
+    signature?: string;
+    /** Stability level */
+    stability: 'internal' | 'experimental' | 'stable' | 'deprecated';
+    /** Kind for type definitions (interface, type, enum, class) */
+    kind?: 'interface' | 'type' | 'enum' | 'class';
+    /** Members for type definitions */
+    members?: TypeMember[];
+}
+/**
+ * A point-in-time snapshot of all public APIs at a given version
+ */
+export interface ApiSnapshot {
+    /** Semantic version string */
+    version: string;
+    /** ISO timestamp when the snapshot was created */
+    timestamp: string;
+    /** All API entries captured in this snapshot */
+    entries: SnapshotEntry[];
+}
+/** Kind of change detected between two API versions */
+export type ChangeKind = 'added' | 'removed' | 'signature-changed' | 'stability-changed' | 'members-changed';
+/**
+ * A single detected change between two API snapshots
+ */
+export interface ApiChange {
+    /** API name */
+    name: string;
+    /** Parent component/module id */
+    parent: string;
+    /** Qualified name: parent.name */
+    qualifiedName: string;
+    /** Type of change */
+    changeKind: ChangeKind;
+    /** Entry from the old snapshot (absent for additions) */
+    before?: SnapshotEntry;
+    /** Entry from the new snapshot (absent for removals) */
+    after?: SnapshotEntry;
+}
+/**
+ * Result of diffing two API snapshots
+ */
+export interface SnapshotDiff {
+    /** Version of the older snapshot */
+    fromVersion: string;
+    /** Version of the newer snapshot */
+    toVersion: string;
+    /** APIs that were added */
+    added: ApiChange[];
+    /** APIs that were removed */
+    removed: ApiChange[];
+    /** APIs whose signature, members, or stability changed */
+    changed: ApiChange[];
+    /** Count of APIs that remain unchanged */
+    unchanged: number;
+}
+/** Semver bump classification for a change */
+export type ChangeClassification = 'major' | 'minor' | 'patch';
+/** Risk level for a codex entry, affects testing requirements */
+export type RiskLevel = 'critical' | 'high' | 'medium' | 'low';
+/**
+ * Testing level requirements and status
+ */
+export interface TestLevel {
+    /** Whether this test level is required for this feature */
+    required: boolean;
+    /** Whether tests currently exist */
+    covered?: boolean;
+    /** Minimum percentage coverage target */
+    coverage?: number;
+    /** Path to the test file */
+    file?: string;
+    /** Specific aspects to test */
+    focus?: string[];
+    /** Required scenarios (for e2e) */
+    scenarios?: string[];
+    /** Dependencies to integrate with (for integration tests) */
+    dependencies?: string[];
+}
+/**
+ * Testing requirements for a codex entry across all pyramid levels
+ */
+export interface TestingRequirements {
+    /** Unit test requirements */
+    unit?: TestLevel;
+    /** Integration test requirements */
+    integration?: TestLevel;
+    /** End-to-end test requirements */
+    e2e?: TestLevel;
+}

package/dist/extraction/types.js

@@ -0,0 +1,34 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction.types",
+ *   "title": "Extraction Types",
+ *   "category": "types"
+ * }
+ */
+/**
+ * Default extraction configuration (all fields except root)
+ */
+export const DEFAULT_EXTRACTION_CONFIG = {
+    sourcePatterns: [
+        'packages/*/src/**/*.ts',
+        'packages/*/src/**/*.js',
+        'packages/*/src/**/*.mjs',
+        'apps/*/src/**/*.ts',
+        'apps/*/src/**/*.js',
+        'apps/*/src/**/*.mjs',
+    ],
+    testPatterns: [
+        'packages/*/src/**/*.test.ts',
+        'packages/*/src/**/*.spec.ts',
+        'packages/*/tests/**/*.ts',
+        'apps/*/src/**/*.test.ts',
+        'apps/*/src/**/*.spec.ts',
+        'apps/*/tests/**/*.ts',
+        'e2e/tests/**/*.ts',
+    ],
+    readmePatterns: ['packages/*/README.md', 'README.md'],
+    extractRunnableExamples: true,
+    analyzeTests: true,
+    linkReadmes: true,
+};

package/dist/indexer.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/indexer.js

@@ -0,0 +1,107 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.indexer",
+ *   "title": "Index Generator",
+ *   "category": "feature"
+ * }
+ *
+ * Regenerates codex index files from entries.
+ */
+import fs from 'node:fs/promises';
+import { watch as fsWatch } from 'node:fs';
+import { join, dirname, relative } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { codexSchema } from './schema.js';
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repoRoot = join(__dirname, '../../..');
+const codexDir = join(repoRoot, 'codex');
+const indexPath = join(repoRoot, 'codex.index.json');
+async function ensureDir(dir) {
+    try {
+        await fs.mkdir(dir, { recursive: true });
+    }
+    catch {
+        // ignore
+    }
+}
+async function getJsonFiles(dir) {
+    const dirents = await fs
+        .readdir(dir, { withFileTypes: true })
+        .catch(() => []);
+    const files = [];
+    for (const dirent of dirents) {
+        const res = join(dir, dirent.name);
+        if (dirent.isDirectory()) {
+            // Skip proposals directory — only index applied entries
+            if (dirent.name === '.proposals')
+                continue;
+            files.push(...(await getJsonFiles(res)));
+        }
+        else if (dirent.isFile() && res.endsWith('.json')) {
+            files.push(res);
+        }
+    }
+    return files;
+}
+async function buildIndex() {
+    await ensureDir(codexDir);
+    const files = await getJsonFiles(codexDir);
+    const entries = [];
+    for (const file of files) {
+        try {
+            const raw = await fs.readFile(file, 'utf8');
+            const data = JSON.parse(raw);
+            const parsed = codexSchema.safeParse(data);
+            if (!parsed.success) {
+                const issues = parsed.error.issues.map(i => i.message).join('; ');
+                console.warn(`Skipping ${file}: schema errors: ${issues}`);
+                continue;
+            }
+            entries.push({ ...parsed.data, path: relative(repoRoot, file) });
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`Skipping ${file}: ${msg}`);
+        }
+    }
+    // deterministic sort
+    entries.sort((a, b) => (a.category || '').localeCompare(b.category || '') ||
+        (a.title || '').localeCompare(b.title || '') ||
+        a.id.localeCompare(b.id));
+    // quick lookup map
+    const byId = Object.fromEntries(entries.map(e => [e.id, e]));
+    const payload = {
+        updatedAt: new Date().toISOString(),
+        count: entries.length,
+        entries,
+        byId,
+    };
+    const tmp = `${indexPath}.tmp`;
+    await fs.writeFile(tmp, JSON.stringify(payload, null, 2));
+    await fs.rename(tmp, indexPath);
+    console.log(`Wrote ${indexPath} (${entries.length} entries)`);
+}
+function watch() {
+    let timer = null;
+    const debouncedBuild = () => {
+        if (timer)
+            clearTimeout(timer);
+        timer = setTimeout(() => {
+            buildIndex().catch(console.error);
+        }, 100);
+    };
+    fsWatch(codexDir, { recursive: false }, debouncedBuild);
+    // fs.watch won’t see new subdirs on some platforms — periodic rescan is a cheap fix
+    setInterval(debouncedBuild, 2000);
+    void buildIndex().catch(console.error);
+}
+if (process.argv.includes('--watch')) {
+    watch();
+}
+else {
+    buildIndex().catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/llm.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Sends a prompt to the OpenAI API and returns the response text.
+ * @codexApi {"parent":"pithy.codex.llm","name":"askLLM","stability":"stable","signature":"(messages: { role: 'system' | 'user'; content: string }[]) => Promise<string>"}
+ */
+export declare function askLLM(messages: {
+    role: 'system' | 'user';
+    content: string;
+}[]): Promise<string>;

package/dist/llm.js

@@ -0,0 +1,75 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.llm",
+ *   "title": "LLM Integration",
+ *   "category": "feature"
+ * }
+ *
+ * Sends prompts to OpenAI for codex generation.
+ */
+import OpenAI from 'openai';
+import { loadEnv } from './env.js';
+loadEnv();
+const baseURL = process.env.OPENAI_BASE_URL || undefined;
+const apiKey = process.env.OPENAI_API_KEY || 'EMPTY';
+const model = process.env.CODEX_MODEL || 'gpt-5-mini';
+// Optional temperature (omitted by default). We also retry if a model rejects it.
+const tempEnv = process.env.CODEX_TEMPERATURE;
+const parsedTemp = tempEnv !== undefined && tempEnv !== '' ? Number(tempEnv) : undefined;
+const temperature = Number.isFinite(parsedTemp) ? parsedTemp : undefined;
+const client = new OpenAI({ apiKey, baseURL });
+function errStatus(e) {
+    if (e && typeof e === 'object') {
+        const obj = e;
+        return obj.status ?? obj.response?.status;
+    }
+    return undefined;
+}
+function errBody(e) {
+    if (e && typeof e === 'object') {
+        const obj = e;
+        return obj.response?.data ?? obj.message ?? String(e);
+    }
+    return String(e);
+}
+/**
+ * Sends a prompt to the OpenAI API and returns the response text.
+ * @codexApi {"parent":"pithy.codex.llm","name":"askLLM","stability":"stable","signature":"(messages: { role: 'system' | 'user'; content: string }[]) => Promise<string>"}
+ */
+export async function askLLM(messages) {
+    // Build params and include temperature only if provided
+    const params = {
+        model,
+        messages,
+    };
+    if (typeof temperature === 'number')
+        params.temperature = temperature;
+    try {
+        const res = await client.chat.completions.create(params);
+        return res.choices[0]?.message?.content ?? '';
+    }
+    catch (e) {
+        const status = errStatus(e);
+        const body = errBody(e);
+        console.error(`[codex:llm] request failed (model="${model}", status=${status})`);
+        console.error(body);
+        // Auto-retry if the error complains about temperature
+        const bodyText = typeof body === 'string' ? body : JSON.stringify(body);
+        if (status === 400 && /temperature/i.test(bodyText)) {
+            try {
+                delete params.temperature;
+                console.warn('[codex:llm] retrying without temperature…');
+                const res = await client.chat.completions.create(params);
+                return res.choices[0]?.message?.content ?? '';
+            }
+            catch (e2) {
+                const s2 = errStatus(e2);
+                const b2 = errBody(e2);
+                console.error(`[codex:llm] retry failed (model="${model}", status=${s2})`);
+                console.error(b2);
+            }
+        }
+        return '';
+    }
+}

package/dist/readme-sync-cli.d.ts

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction.readme-sync-cli",
+ *   "title": "README Sync CLI",
+ *   "category": "plugin"
+ * }
+ *
+ * CLI for syncing README files with codex extraction data.
+ *
+ * Usage:
+ *   pnpm --filter @pithyjs/codex readme-sync [-- options]
+ *
+ * Options:
+ *   --root <path>       Project root directory (default: cwd)
+ *   --dry-run           Report changes without writing files
+ *   --verbose           Show detailed progress
+ */
+export {};