@pithyjs/codex 0.1.0-beta.0

package/dist/extraction/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction",
+ *   "title": "Code Extraction Pipeline",
+ *   "category": "feature"
+ * }
+ *
+ * Multi-source extraction pipeline for documentation generation.
+ * Extracts JSDoc, examples, test patterns, and links to source code.
+ */
+export type { SourceLocation, ParamDoc, ReturnDoc, ExtractedExample, ParsedJSDoc, ExtractedTestCase, TestFileAnalysis, EnrichedApiEntry, TestReference, EnrichedComponent, ReadmeSection, ExtractionConfig, ExtractionResult, ExtractionStats, ExtractedTypeDefinition, TypeMember, TestExample, TestStatus, ExtendedExtractionResult, TestLevel, TestingRequirements, RiskLevel, } from './types.js';
+export { DEFAULT_EXTRACTION_CONFIG } from './types.js';
+export { parseJSDocComment, extractJSDocBlocks, linkJSDocToDeclarations, extractSignature, } from './jsdoc-parser.js';
+export { extractExamples, validateExample, wrapDoctestInHarness, extractExamplesFromSource, generateRunnableExampleFile, } from './example-extractor.js';
+export { extractTestCases, extractTestImports, identifyCoveredApis, analyzeTestFile, testCasesToReferences, matchTestToEntry, generateCoverageSummary, } from './test-pattern-extractor.js';
+export type { DeclarationLocation } from './source-linker.js';
+export { findDeclarationLocation, extractAllDeclarations, parseReadmeSections, linkReadmeToEntries, generateSourceLink, toRelativePath, enrichSourceLocation, } from './source-linker.js';
+export { runExtractionPipeline, extractSingleFile, generateExtractionReport, exportToCodexFormat, } from './pipeline.js';
+export { extractTypesFromFile, getTypeSignature, generateMethodTable, } from './type-extractor.js';
+export { extractTestExamples, parseVitestOutput, matchExamplesToStatuses, generateExampleReport, validateTestExamples, } from './test-example-extractor.js';
+export type { ReadmeSyncResult, ReadmeSyncConfig } from './types.js';
+export { syncReadmeContent, syncAllReadmes } from './readme-sync.js';
+export type { SnapshotEntry, ApiSnapshot, ChangeKind, ApiChange, SnapshotDiff, ChangeClassification, } from './types.js';
+export { createApiSnapshot, snapshotFromTypeDefinitions, diffSnapshots, classifyChange, detectBreakingChanges, generateMigrationGuide, generateChangelog, } from './breaking-changes.js';
+export { saveSnapshot, loadSnapshot, listSnapshots, getSnapshotPath, } from './snapshot-store.js';
+export { getTestingDefaults, applyRiskModifiers, resolveTestingRequirements, computeTestingStatus, } from './testing-pyramid.js';

package/dist/extraction/index.js

@@ -0,0 +1,31 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction",
+ *   "title": "Code Extraction Pipeline",
+ *   "category": "feature"
+ * }
+ *
+ * Multi-source extraction pipeline for documentation generation.
+ * Extracts JSDoc, examples, test patterns, and links to source code.
+ */
+export { DEFAULT_EXTRACTION_CONFIG } from './types.js';
+// JSDoc Parser
+export { parseJSDocComment, extractJSDocBlocks, linkJSDocToDeclarations, extractSignature, } from './jsdoc-parser.js';
+// Example Extractor
+export { extractExamples, validateExample, wrapDoctestInHarness, extractExamplesFromSource, generateRunnableExampleFile, } from './example-extractor.js';
+// Test Pattern Extractor
+export { extractTestCases, extractTestImports, identifyCoveredApis, analyzeTestFile, testCasesToReferences, matchTestToEntry, generateCoverageSummary, } from './test-pattern-extractor.js';
+export { findDeclarationLocation, extractAllDeclarations, parseReadmeSections, linkReadmeToEntries, generateSourceLink, toRelativePath, enrichSourceLocation, } from './source-linker.js';
+// Pipeline
+export { runExtractionPipeline, extractSingleFile, generateExtractionReport, exportToCodexFormat, } from './pipeline.js';
+// Type Extractor (TypeScript Compiler API)
+export { extractTypesFromFile, getTypeSignature, generateMethodTable, } from './type-extractor.js';
+// Test Example Extractor (@codex:example markers)
+export { extractTestExamples, parseVitestOutput, matchExamplesToStatuses, generateExampleReport, validateTestExamples, } from './test-example-extractor.js';
+export { syncReadmeContent, syncAllReadmes } from './readme-sync.js';
+export { createApiSnapshot, snapshotFromTypeDefinitions, diffSnapshots, classifyChange, detectBreakingChanges, generateMigrationGuide, generateChangelog, } from './breaking-changes.js';
+// Snapshot Store (Version Persistence)
+export { saveSnapshot, loadSnapshot, listSnapshots, getSnapshotPath, } from './snapshot-store.js';
+// Testing Pyramid
+export { getTestingDefaults, applyRiskModifiers, resolveTestingRequirements, computeTestingStatus, } from './testing-pyramid.js';

package/dist/extraction/jsdoc-parser.d.ts

@@ -0,0 +1,43 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction.jsdoc-parser",
+ *   "title": "JSDoc/TSDoc Parser",
+ *   "category": "compiler"
+ * }
+ */
+import type { ParsedJSDoc, SourceLocation } from './types.js';
+/**
+ * Parses a single JSDoc comment block
+ * @param comment - The JSDoc comment content (without delimiters)
+ * @param location - Source location of the comment
+ * @returns Parsed JSDoc structure
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"parseJSDocComment","stability":"stable","signature":"(comment: string, location: SourceLocation) => ParsedJSDoc"}
+ */
+export declare function parseJSDocComment(comment: string, location: SourceLocation): ParsedJSDoc;
+/**
+ * Extracts all JSDoc blocks from a source file
+ * @param content - File content
+ * @param filePath - Path to the file for location tracking
+ * @returns Array of parsed JSDoc blocks with locations
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"extractJSDocBlocks","stability":"stable","signature":"(content: string, filePath: string) => { jsdoc: ParsedJSDoc; followingCode: string }[]"}
+ */
+export declare function extractJSDocBlocks(content: string, filePath: string): {
+    jsdoc: ParsedJSDoc;
+    followingCode: string;
+}[];
+/**
+ * Links JSDoc blocks to their associated function/class declarations
+ * @param content - File content
+ * @param filePath - Path to the file
+ * @returns Map of declaration names to their JSDoc
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"linkJSDocToDeclarations","stability":"stable","signature":"(content: string, filePath: string) => Map<string, ParsedJSDoc>"}
+ */
+export declare function linkJSDocToDeclarations(content: string, filePath: string): Map<string, ParsedJSDoc>;
+/**
+ * Extracts function signature from source code
+ * @param code - Code following the JSDoc block
+ * @returns Extracted signature or undefined
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"extractSignature","stability":"stable","signature":"(code: string) => string | undefined"}
+ */
+export declare function extractSignature(code: string): string | undefined;

package/dist/extraction/jsdoc-parser.js

@@ -0,0 +1,274 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction.jsdoc-parser",
+ *   "title": "JSDoc/TSDoc Parser",
+ *   "category": "compiler"
+ * }
+ */
+/**
+ * Regex patterns for JSDoc parsing
+ * Note: Tag regex uses atomic grouping simulation to prevent ReDoS
+ */
+// Require /** at line start (optional whitespace) to avoid matching
+// /** inside line comments like: // delimiters (*, */, /**)
+const JSDOC_BLOCK_REGEX = /(?:^|(?<=\n))\s*\/\*\*\s*([\s\S]*?)\s*\*\//g;
+const EXAMPLE_CODE_BLOCK_REGEX = /```(\w*)\s*\n([\s\S]*?)\n\s*```/g;
+/**
+ * Parses JSDoc tags using a safer line-by-line approach to avoid ReDoS
+ * @param tagContent - Content containing JSDoc tags
+ * @returns Array of parsed tag objects
+ */
+function parseJSDocTags(tagContent) {
+    const tags = [];
+    const lines = tagContent.split('\n');
+    let currentTag = null;
+    for (const line of lines) {
+        const trimmed = line.replace(/^\s*\*?\s*/, '');
+        // Check if this line starts a new tag
+        const tagMatch = trimmed.match(/^@(\w+)(?:\s+\{([^}]*)\})?\s*(\[?\w+(?:\.\w+)*\]?)?\s*(?:-\s*)?(.*)$/);
+        if (tagMatch) {
+            // Save previous tag if exists
+            if (currentTag) {
+                tags.push(currentTag);
+            }
+            currentTag = {
+                tagName: tagMatch[1],
+                type: tagMatch[2] || undefined,
+                name: tagMatch[3] || undefined,
+                description: tagMatch[4] || '',
+            };
+        }
+        else if (currentTag && trimmed && !trimmed.startsWith('@')) {
+            // Continue previous tag's description
+            currentTag.description += '\n' + trimmed;
+        }
+    }
+    // Don't forget the last tag
+    if (currentTag) {
+        tags.push(currentTag);
+    }
+    return tags;
+}
+/**
+ * Parses a single JSDoc comment block
+ * @param comment - The JSDoc comment content (without delimiters)
+ * @param location - Source location of the comment
+ * @returns Parsed JSDoc structure
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"parseJSDocComment","stability":"stable","signature":"(comment: string, location: SourceLocation) => ParsedJSDoc"}
+ */
+export function parseJSDocComment(comment, location) {
+    // Clean up the comment: remove leading asterisks and normalize whitespace
+    const cleanedComment = comment
+        .split('\n')
+        .map(line => line.replace(/^\s*\*\s?/, ''))
+        .join('\n')
+        .trim();
+    const result = {
+        description: '',
+        params: [],
+        examples: [],
+        tags: {},
+        location,
+    };
+    // Extract the main description (text before the first tag at line start).
+    // Mid-sentence @mentions (e.g. "parses @codex annotations") are not tags.
+    const firstTagIndex = cleanedComment.search(/^@\w+/m);
+    if (firstTagIndex === -1) {
+        result.description = cleanedComment;
+    }
+    else if (firstTagIndex > 0) {
+        result.description = cleanedComment.slice(0, firstTagIndex).trim();
+    }
+    // Parse all tags using safer line-by-line approach
+    const tagContent = firstTagIndex >= 0 ? cleanedComment.slice(firstTagIndex) : '';
+    const parsedTags = parseJSDocTags(tagContent);
+    for (const { tagName, type, name, description } of parsedTags) {
+        const cleanDesc = (description || '').trim();
+        switch (tagName.toLowerCase()) {
+            case 'param':
+            case 'parameter':
+            case 'arg':
+            case 'argument':
+                result.params.push(parseParam(type, name, cleanDesc));
+                break;
+            case 'returns':
+            case 'return':
+                result.returns = parseReturn(type, cleanDesc);
+                break;
+            case 'example':
+                result.examples.push(...parseExamples(cleanDesc, location));
+                break;
+            case 'throws':
+            case 'exception':
+                result.throws = result.throws || [];
+                result.throws.push(cleanDesc);
+                break;
+            case 'see':
+                result.see = result.see || [];
+                result.see.push(cleanDesc);
+                break;
+            case 'since':
+                result.since = cleanDesc;
+                break;
+            case 'deprecated':
+                result.deprecated = cleanDesc || true;
+                break;
+            default:
+                // Store unknown tags for extensibility
+                if (!result.tags[tagName]) {
+                    result.tags[tagName] = [];
+                }
+                result.tags[tagName].push(cleanDesc);
+        }
+    }
+    return result;
+}
+/**
+ * Parses a @param tag into structured data
+ */
+function parseParam(type, name, description) {
+    const isOptional = name?.startsWith('[') || false;
+    let cleanName = name?.replace(/^\[|\]$/g, '') || '';
+    let defaultValue;
+    // Handle default values: [param=default]
+    if (cleanName.includes('=')) {
+        const [paramName, defVal] = cleanName.split('=');
+        cleanName = paramName;
+        defaultValue = defVal;
+    }
+    return {
+        name: cleanName,
+        type: type || undefined,
+        description,
+        optional: isOptional,
+        defaultValue,
+    };
+}
+/**
+ * Parses a @returns tag into structured data
+ */
+function parseReturn(type, description) {
+    return {
+        type: type || undefined,
+        description,
+    };
+}
+/**
+ * Parses @example content into ExtractedExample objects
+ */
+function parseExamples(content, baseLocation) {
+    const examples = [];
+    // Reset regex state
+    EXAMPLE_CODE_BLOCK_REGEX.lastIndex = 0;
+    let match;
+    while ((match = EXAMPLE_CODE_BLOCK_REGEX.exec(content)) !== null) {
+        const [, language, code] = match;
+        const isDoctest = language?.includes('doctest') || false;
+        const lang = language?.replace('doctest', '').trim() || 'typescript';
+        examples.push({
+            code: code.trim(),
+            language: lang || 'typescript',
+            runnable: isDoctest || lang === 'ts' || lang === 'typescript',
+            isDoctest,
+            location: baseLocation,
+        });
+    }
+    // If no code blocks found, treat the whole content as an example
+    if (examples.length === 0 && content.trim()) {
+        // Strip JSDoc comment delimiter artifacts that leak in when
+        // two adjacent /** */ blocks are captured as one.
+        // Only remove lines that are exactly delimiters (*, */, /**), not blank lines.
+        const cleanedCode = content
+            .split('\n')
+            .filter(l => !/^\s*\*\/$/.test(l) &&
+            !/^\s*\/\*\*?\s*$/.test(l) &&
+            !/^\s*\*\s*$/.test(l))
+            .join('\n')
+            .trim();
+        if (cleanedCode) {
+            examples.push({
+                code: cleanedCode,
+                language: 'typescript',
+                runnable: false,
+                isDoctest: false,
+                location: baseLocation,
+            });
+        }
+    }
+    return examples;
+}
+/**
+ * Extracts all JSDoc blocks from a source file
+ * @param content - File content
+ * @param filePath - Path to the file for location tracking
+ * @returns Array of parsed JSDoc blocks with locations
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"extractJSDocBlocks","stability":"stable","signature":"(content: string, filePath: string) => { jsdoc: ParsedJSDoc; followingCode: string }[]"}
+ */
+export function extractJSDocBlocks(content, filePath) {
+    const results = [];
+    // Reset regex state
+    JSDOC_BLOCK_REGEX.lastIndex = 0;
+    let match;
+    while ((match = JSDOC_BLOCK_REGEX.exec(content)) !== null) {
+        const [fullMatch, commentContent] = match;
+        const startIndex = match.index;
+        // Calculate line number
+        const textBefore = content.slice(0, startIndex);
+        const lineNumber = textBefore.split('\n').length;
+        // Find the code following this JSDoc block
+        const afterMatch = content.slice(startIndex + fullMatch.length);
+        const nextLineMatch = afterMatch.match(/^\s*\n?\s*(.+)/);
+        const followingCode = nextLineMatch ? nextLineMatch[1] : '';
+        const location = {
+            file: filePath,
+            line: lineNumber,
+        };
+        const jsdoc = parseJSDocComment(commentContent, location);
+        results.push({ jsdoc, followingCode });
+    }
+    return results;
+}
+/**
+ * Links JSDoc blocks to their associated function/class declarations
+ * @param content - File content
+ * @param filePath - Path to the file
+ * @returns Map of declaration names to their JSDoc
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"linkJSDocToDeclarations","stability":"stable","signature":"(content: string, filePath: string) => Map<string, ParsedJSDoc>"}
+ */
+export function linkJSDocToDeclarations(content, filePath) {
+    const linked = new Map();
+    const blocks = extractJSDocBlocks(content, filePath);
+    for (const { jsdoc, followingCode } of blocks) {
+        // Try to extract declaration name from following code
+        const declarationMatch = followingCode.match(/(?:export\s+)?(?:async\s+)?(?:function|class|const|let|var|interface|type)\s+(\w+)|(\w+)\s*[=:]\s*(?:function|\(|async)/);
+        if (declarationMatch) {
+            const name = declarationMatch[1] || declarationMatch[2];
+            if (name) {
+                linked.set(name, jsdoc);
+            }
+        }
+    }
+    return linked;
+}
+/**
+ * Extracts function signature from source code
+ * @param code - Code following the JSDoc block
+ * @returns Extracted signature or undefined
+ * @codexApi {"parent":"pithy.codex.extraction.jsdoc-parser","name":"extractSignature","stability":"stable","signature":"(code: string) => string | undefined"}
+ */
+export function extractSignature(code) {
+    // Match function declarations
+    const funcMatch = code.match(/(?:export\s+)?(?:async\s+)?function\s+\w+\s*(<[^>]+>)?\s*\(([^)]*)\)\s*(?::\s*([^{]+))?/);
+    if (funcMatch) {
+        const [, generics, params, returnType] = funcMatch;
+        return `${generics || ''}(${params})${returnType ? ` => ${returnType.trim()}` : ''}`;
+    }
+    // Match arrow functions
+    const arrowMatch = code.match(/(?:const|let|var)\s+\w+\s*(?::\s*[^=]+)?\s*=\s*(?:async\s+)?(?:<[^>]+>)?\s*\(([^)]*)\)\s*(?::\s*([^=]+))?\s*=>/);
+    if (arrowMatch) {
+        const [, params, returnType] = arrowMatch;
+        return `(${params})${returnType ? ` => ${returnType.trim()}` : ''}`;
+    }
+    return undefined;
+}

package/dist/extraction/pipeline.d.ts

@@ -0,0 +1,54 @@
+/**
+ * @codex
+ * {
+ *   "id": "pithy.codex.extraction.pipeline",
+ *   "title": "Extraction Pipeline",
+ *   "category": "feature"
+ * }
+ */
+import { type ExtractionConfig, type ExtractionResult, type EnrichedComponent, type ExtendedExtractionResult } from './types.js';
+/**
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"runExtractionPipeline","stability":"stable","signature":"(config?: Partial<ExtractionConfig>) => Promise<ExtendedExtractionResult>"}
+ *
+ * Runs the complete multi-source extraction pipeline
+ * @param config - Configuration options
+ * @returns Extraction result with all enriched data including types and test examples
+ *
+ * @example
+ * ```typescript
+ * const result = await runExtractionPipeline({
+ *   root: '/path/to/project',
+ *   extractRunnableExamples: true,
+ * });
+ *
+ * console.log(`Found ${result.stats.componentsFound} components`);
+ * console.log(`Extracted ${result.typeDefinitions.length} types`);
+ * console.log(`Found ${result.testExamples.length} test examples`);
+ * ```
+ */
+export declare function runExtractionPipeline(config?: Partial<ExtractionConfig>): Promise<ExtendedExtractionResult>;
+/**
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"extractSingleFile","stability":"stable","signature":"(filePath: string, content?: string) => Promise<EnrichedComponent | null>"}
+ *
+ * Extracts documentation from a single source file
+ * @param filePath - Path to the source file
+ * @param content - Optional file content (will read if not provided)
+ * @returns Enriched component or null if no codex annotation found
+ */
+export declare function extractSingleFile(filePath: string, content?: string): Promise<EnrichedComponent | null>;
+/**
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"generateExtractionReport","stability":"stable","signature":"(result: ExtractionResult) => string"}
+ *
+ * Generates a human-readable report from extraction results
+ * @param result - Extraction result
+ * @returns Formatted report string
+ */
+export declare function generateExtractionReport(result: ExtractionResult): string;
+/**
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"exportToCodexFormat","stability":"stable","signature":"(result: ExtractionResult) => object"}
+ *
+ * Exports extraction results to codex-compatible JSON format
+ * @param result - Extraction result
+ * @returns Object suitable for codex.index.json
+ */
+export declare function exportToCodexFormat(result: ExtractionResult): object;