@pithyjs/codex 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +4401 -0
  3. package/dist/annotations.d.ts +106 -0
  4. package/dist/annotations.js +306 -0
  5. package/dist/apply.d.ts +2 -0
  6. package/dist/apply.js +80 -0
  7. package/dist/changed-scope.d.ts +23 -0
  8. package/dist/changed-scope.js +31 -0
  9. package/dist/check.d.ts +2 -0
  10. package/dist/check.js +117 -0
  11. package/dist/cli.d.ts +2 -0
  12. package/dist/cli.js +67 -0
  13. package/dist/env.d.ts +5 -0
  14. package/dist/env.js +35 -0
  15. package/dist/extract-cli.d.ts +23 -0
  16. package/dist/extract-cli.js +192 -0
  17. package/dist/extraction/breaking-changes.d.ts +66 -0
  18. package/dist/extraction/breaking-changes.js +352 -0
  19. package/dist/extraction/example-extractor.d.ts +55 -0
  20. package/dist/extraction/example-extractor.js +272 -0
  21. package/dist/extraction/index.d.ts +27 -0
  22. package/dist/extraction/index.js +31 -0
  23. package/dist/extraction/jsdoc-parser.d.ts +43 -0
  24. package/dist/extraction/jsdoc-parser.js +274 -0
  25. package/dist/extraction/pipeline.d.ts +54 -0
  26. package/dist/extraction/pipeline.js +526 -0
  27. package/dist/extraction/readme-sync.d.ts +108 -0
  28. package/dist/extraction/readme-sync.js +592 -0
  29. package/dist/extraction/snapshot-store.d.ts +40 -0
  30. package/dist/extraction/snapshot-store.js +153 -0
  31. package/dist/extraction/source-linker.d.ts +80 -0
  32. package/dist/extraction/source-linker.js +316 -0
  33. package/dist/extraction/test-example-extractor.d.ts +69 -0
  34. package/dist/extraction/test-example-extractor.js +400 -0
  35. package/dist/extraction/test-pattern-extractor.d.ts +68 -0
  36. package/dist/extraction/test-pattern-extractor.js +261 -0
  37. package/dist/extraction/testing-pyramid.d.ts +44 -0
  38. package/dist/extraction/testing-pyramid.js +163 -0
  39. package/dist/extraction/type-extractor.d.ts +34 -0
  40. package/dist/extraction/type-extractor.js +494 -0
  41. package/dist/extraction/types.d.ts +401 -0
  42. package/dist/extraction/types.js +34 -0
  43. package/dist/indexer.d.ts +1 -0
  44. package/dist/indexer.js +107 -0
  45. package/dist/llm.d.ts +8 -0
  46. package/dist/llm.js +75 -0
  47. package/dist/readme-sync-cli.d.ts +20 -0
  48. package/dist/readme-sync-cli.js +167 -0
  49. package/dist/review.d.ts +2 -0
  50. package/dist/review.js +93 -0
  51. package/dist/scan.d.ts +29 -0
  52. package/dist/scan.js +221 -0
  53. package/dist/schema.d.ts +169 -0
  54. package/dist/schema.js +70 -0
  55. package/dist/snapshot-cli.d.ts +45 -0
  56. package/dist/snapshot-cli.js +217 -0
  57. package/dist/sync-cli.d.ts +22 -0
  58. package/dist/sync-cli.js +154 -0
  59. package/dist/sync-pipeline.d.ts +58 -0
  60. package/dist/sync-pipeline.js +104 -0
  61. package/dist/sync.d.ts +2 -0
  62. package/dist/sync.js +318 -0
  63. package/dist/validate-cli.d.ts +20 -0
  64. package/dist/validate-cli.js +144 -0
  65. package/dist/validate.d.ts +76 -0
  66. package/dist/validate.js +183 -0
  67. package/dist/watch-cli.d.ts +21 -0
  68. package/dist/watch-cli.js +220 -0
  69. package/package.json +62 -0
@@ -0,0 +1,526 @@
1
+ /**
2
+ * @codex
3
+ * {
4
+ * "id": "pithy.codex.extraction.pipeline",
5
+ * "title": "Extraction Pipeline",
6
+ * "category": "feature"
7
+ * }
8
+ */
9
+ import { globby } from 'globby';
10
+ import fs from 'node:fs/promises';
11
+ import { basename, isAbsolute, join, relative } from 'node:path';
12
+ import { DEFAULT_EXTRACTION_CONFIG, } from './types.js';
13
+ import { linkJSDocToDeclarations } from './jsdoc-parser.js';
14
+ import { extractExamplesFromSource } from './example-extractor.js';
15
+ import { analyzeTestFile, matchTestToEntry, testCasesToReferences, } from './test-pattern-extractor.js';
16
+ import { extractAllDeclarations, parseReadmeSections, linkReadmeToEntries, } from './source-linker.js';
17
+ import { discoverComponentsFromFile } from '../annotations.js';
18
+ import { extractTypesFromFile } from './type-extractor.js';
19
+ import { extractTestExamples, parseVitestOutput, matchExamplesToStatuses, } from './test-example-extractor.js';
20
+ /**
21
+ * Concurrency limit for parallel file processing
22
+ */
23
+ const CONCURRENCY_LIMIT = 10;
24
+ /**
25
+ * Processes items in parallel with a concurrency limit
26
+ * @param items - Items to process
27
+ * @param processor - Async function to process each item
28
+ * @param limit - Maximum concurrent operations
29
+ */
30
+ async function processInParallel(items, processor, limit = CONCURRENCY_LIMIT) {
31
+ const results = [];
32
+ for (let i = 0; i < items.length; i += limit) {
33
+ const batch = items.slice(i, i + limit);
34
+ const batchResults = await Promise.all(batch.map(processor));
35
+ results.push(...batchResults);
36
+ }
37
+ return results;
38
+ }
39
+ /**
40
+ * Gets the first description from a JSDoc map, if any exists
41
+ * @param jsdocMap - Map of declaration names to JSDoc
42
+ * @returns First description found, or undefined
43
+ */
44
+ function getFirstDescription(jsdocMap) {
45
+ for (const jsdoc of jsdocMap.values()) {
46
+ if (jsdoc.description) {
47
+ return jsdoc.description;
48
+ }
49
+ }
50
+ return undefined;
51
+ }
52
+ /**
53
+ * Builds full config by merging defaults with provided options
54
+ */
55
+ function buildFullConfig(config) {
56
+ return {
57
+ root: config?.root ?? process.cwd(),
58
+ sourcePatterns: config?.sourcePatterns ?? DEFAULT_EXTRACTION_CONFIG.sourcePatterns,
59
+ testPatterns: config?.testPatterns ?? DEFAULT_EXTRACTION_CONFIG.testPatterns,
60
+ readmePatterns: config?.readmePatterns ?? DEFAULT_EXTRACTION_CONFIG.readmePatterns,
61
+ extractRunnableExamples: config?.extractRunnableExamples ??
62
+ DEFAULT_EXTRACTION_CONFIG.extractRunnableExamples,
63
+ analyzeTests: config?.analyzeTests ?? DEFAULT_EXTRACTION_CONFIG.analyzeTests,
64
+ linkReadmes: config?.linkReadmes ?? DEFAULT_EXTRACTION_CONFIG.linkReadmes,
65
+ vitestOutputPath: config?.vitestOutputPath,
66
+ };
67
+ }
68
+ /**
69
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"runExtractionPipeline","stability":"stable","signature":"(config?: Partial<ExtractionConfig>) => Promise<ExtendedExtractionResult>"}
70
+ *
71
+ * Runs the complete multi-source extraction pipeline
72
+ * @param config - Configuration options
73
+ * @returns Extraction result with all enriched data including types and test examples
74
+ *
75
+ * @example
76
+ * ```typescript
77
+ * const result = await runExtractionPipeline({
78
+ * root: '/path/to/project',
79
+ * extractRunnableExamples: true,
80
+ * });
81
+ *
82
+ * console.log(`Found ${result.stats.componentsFound} components`);
83
+ * console.log(`Extracted ${result.typeDefinitions.length} types`);
84
+ * console.log(`Found ${result.testExamples.length} test examples`);
85
+ * ```
86
+ */
87
+ export async function runExtractionPipeline(config) {
88
+ const startTime = performance.now();
89
+ const fullConfig = buildFullConfig(config);
90
+ const stats = {
91
+ filesScanned: 0,
92
+ componentsFound: 0,
93
+ apisExtracted: 0,
94
+ examplesExtracted: 0,
95
+ testsAnalyzed: 0,
96
+ readmeSectionsLinked: 0,
97
+ processingTimeMs: 0,
98
+ };
99
+ // Phase 1: Discover source files
100
+ const sourceFiles = await globby(fullConfig.sourcePatterns, {
101
+ cwd: fullConfig.root,
102
+ absolute: true,
103
+ gitignore: true,
104
+ });
105
+ stats.filesScanned = sourceFiles.length;
106
+ // Phase 2: Extract components and APIs from source files (parallelized)
107
+ const components = [];
108
+ const allApiNames = [];
109
+ const extractionResults = await processInParallel(sourceFiles, async (filePath) => {
110
+ const content = await fs.readFile(filePath, 'utf8').catch(() => '');
111
+ if (!content)
112
+ return null;
113
+ // Get base component info from codex annotations
114
+ const discoveredComponents = await discoverComponentsFromFile(filePath, content);
115
+ if (discoveredComponents.length === 0)
116
+ return null;
117
+ const fileComponents = [];
118
+ const fileApiNames = [];
119
+ let fileExamplesCount = 0;
120
+ for (const discovered of discoveredComponents) {
121
+ // Enrich with JSDoc parsing
122
+ const jsdocMap = linkJSDocToDeclarations(content, filePath);
123
+ const declarations = extractAllDeclarations(content, filePath);
124
+ const sourceExamples = extractExamplesFromSource(content, filePath);
125
+ // Enrich APIs with JSDoc
126
+ const enrichedApis = [];
127
+ for (const api of discovered.apis) {
128
+ const jsdoc = jsdocMap.get(api.name);
129
+ const declaration = declarations.find(d => d.name === api.name);
130
+ // Collect examples for this API
131
+ const apiExamples = sourceExamples
132
+ .filter(se => se.functionName === api.name)
133
+ .flatMap(se => se.examples);
134
+ if (jsdoc) {
135
+ apiExamples.push(...jsdoc.examples);
136
+ }
137
+ enrichedApis.push({
138
+ id: `${discovered.id}.${api.name}`,
139
+ name: api.name,
140
+ parent: discovered.id,
141
+ signature: api.signature || declaration?.signature,
142
+ stability: api.stability || 'stable',
143
+ jsdoc,
144
+ examples: apiExamples,
145
+ testRefs: [], // Will be populated in Phase 3
146
+ sourceLocation: declaration?.location || {
147
+ file: filePath,
148
+ line: 1,
149
+ },
150
+ });
151
+ fileApiNames.push(api.name);
152
+ fileExamplesCount += apiExamples.length;
153
+ }
154
+ // Collect component-level examples
155
+ const componentExamples = sourceExamples
156
+ .filter(se => !enrichedApis.some(a => a.name === se.functionName))
157
+ .flatMap(se => se.examples);
158
+ fileComponents.push({
159
+ id: discovered.id,
160
+ title: discovered.title,
161
+ category: discovered.category,
162
+ risk: discovered.risk,
163
+ testing: discovered.testing,
164
+ description: getFirstDescription(jsdocMap),
165
+ apis: enrichedApis,
166
+ examples: componentExamples,
167
+ testRefs: [], // Will be populated in Phase 3
168
+ sourceFiles: [{ file: filePath, line: 1 }],
169
+ });
170
+ }
171
+ return {
172
+ components: fileComponents,
173
+ apiNames: fileApiNames,
174
+ examplesCount: fileExamplesCount,
175
+ };
176
+ });
177
+ // Aggregate results from parallel processing
178
+ for (const result of extractionResults) {
179
+ if (result) {
180
+ components.push(...result.components);
181
+ allApiNames.push(...result.apiNames);
182
+ stats.examplesExtracted += result.examplesCount;
183
+ stats.apisExtracted += result.components.reduce((sum, c) => sum + c.apis.length, 0);
184
+ stats.componentsFound += result.components.length;
185
+ }
186
+ }
187
+ // Phase 3: Analyze test files and link to components (parallelized)
188
+ const testAnalyses = [];
189
+ // Discover test files once, reused by Phase 6
190
+ const testFiles = fullConfig.analyzeTests
191
+ ? await globby(fullConfig.testPatterns, {
192
+ cwd: fullConfig.root,
193
+ absolute: true,
194
+ gitignore: true,
195
+ })
196
+ : [];
197
+ if (fullConfig.analyzeTests) {
198
+ const entryIds = components.map(c => c.id);
199
+ // Build a reverse map from test filenames to entry IDs using
200
+ // the testing.unit.file / testing.integration.file hints in source
201
+ // annotations. Multiple entries may reference the same test file,
202
+ // so the map stores arrays of entry IDs.
203
+ //
204
+ // Keys are scoped by package directory (e.g. "packages/vitePlugins:scanner.test.ts")
205
+ // to prevent collisions when different packages share test filenames.
206
+ const testFileHints = new Map();
207
+ // Extract the package scope from an absolute path.
208
+ // Detects monorepo segments like "packages/<name>" or "apps/<name>" from
209
+ // the absolute path so hints work regardless of what `root` is set to.
210
+ // Falls back to the first two segments relative to root when no monorepo
211
+ // segment is found (single-package repos or non-standard layouts).
212
+ function packageDirOf(absPath) {
213
+ const normalized = absPath.replace(/\\/g, '/');
214
+ const monoMatch = normalized.match(/\/(packages|apps)\/([^/]+)\//);
215
+ if (monoMatch)
216
+ return `${monoMatch[1]}/${monoMatch[2]}`;
217
+ const rel = relative(fullConfig.root, absPath).replace(/\\/g, '/');
218
+ const parts = rel.split('/');
219
+ return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : parts[0];
220
+ }
221
+ for (const comp of components) {
222
+ const srcFile = comp.sourceFiles[0]?.file;
223
+ if (!srcFile)
224
+ continue;
225
+ const pkgDir = packageDirOf(srcFile);
226
+ for (const level of ['unit', 'integration', 'e2e']) {
227
+ const hint = comp.testing?.[level]?.file;
228
+ if (hint) {
229
+ const key = `${pkgDir}:${basename(hint).toLowerCase()}`;
230
+ const existing = testFileHints.get(key) ?? [];
231
+ if (!existing.includes(comp.id)) {
232
+ existing.push(comp.id);
233
+ }
234
+ testFileHints.set(key, existing);
235
+ }
236
+ }
237
+ }
238
+ const analysisResults = await processInParallel(testFiles, async (testPath) => {
239
+ const content = await fs.readFile(testPath, 'utf8').catch(() => '');
240
+ if (!content)
241
+ return null;
242
+ const analysis = analyzeTestFile(content, testPath, allApiNames);
243
+ const hintKey = `${packageDirOf(testPath)}:${basename(testPath).toLowerCase()}`;
244
+ // Prefer explicit testing.*.file hints from source annotations
245
+ // (avoids heuristic mismatches when multiple entries share names).
246
+ // Keys are scoped by package directory so identically-named test
247
+ // files in different packages don't collide.
248
+ const hintEntryIds = testFileHints.get(hintKey);
249
+ const matchedEntryId = hintEntryIds?.[0] ?? matchTestToEntry(testPath, entryIds);
250
+ return { analysis, matchedEntryId, testPath, hintEntryIds };
251
+ });
252
+ // Helper: link test refs to a component and its APIs
253
+ function linkTestRefsToComponent(component, refs, analysis) {
254
+ component.testRefs.push(...refs);
255
+ for (const api of component.apis) {
256
+ const apiRefs = refs.filter(r => r.name.toLowerCase().includes(api.name.toLowerCase()) ||
257
+ analysis.coveredApis.includes(api.name));
258
+ api.testRefs.push(...apiRefs);
259
+ }
260
+ }
261
+ // Build a lookup map for constant-time component access by ID
262
+ const componentById = new Map();
263
+ for (const comp of components) {
264
+ componentById.set(comp.id, comp);
265
+ }
266
+ // Apply results (must be sequential for component mutations)
267
+ for (const result of analysisResults) {
268
+ if (!result)
269
+ continue;
270
+ const { analysis, matchedEntryId, testPath, hintEntryIds } = result;
271
+ testAnalyses.push(analysis);
272
+ stats.testsAnalyzed += analysis.testCases.length;
273
+ const refs = testCasesToReferences(analysis.testCases, testPath);
274
+ // Link tests to ALL components that declare this test file
275
+ if (hintEntryIds && hintEntryIds.length > 0) {
276
+ for (const entryId of hintEntryIds) {
277
+ const component = componentById.get(entryId);
278
+ if (component) {
279
+ linkTestRefsToComponent(component, refs, analysis);
280
+ }
281
+ }
282
+ }
283
+ else if (matchedEntryId) {
284
+ const component = componentById.get(matchedEntryId);
285
+ if (component) {
286
+ linkTestRefsToComponent(component, refs, analysis);
287
+ }
288
+ }
289
+ }
290
+ }
291
+ // Phase 4: Parse and link README files (parallelized)
292
+ const readmeSections = [];
293
+ if (fullConfig.linkReadmes) {
294
+ const readmeFiles = await globby(fullConfig.readmePatterns, {
295
+ cwd: fullConfig.root,
296
+ absolute: true,
297
+ gitignore: true,
298
+ });
299
+ const entryIds = components.map(c => c.id);
300
+ const readmeResults = await processInParallel(readmeFiles, async (readmePath) => {
301
+ const content = await fs.readFile(readmePath, 'utf8').catch(() => '');
302
+ if (!content)
303
+ return null;
304
+ const sections = parseReadmeSections(content, readmePath);
305
+ const linkedSections = linkReadmeToEntries(sections, entryIds);
306
+ return linkedSections;
307
+ });
308
+ // Apply results (must be sequential for component mutations)
309
+ for (const linkedSections of readmeResults) {
310
+ if (!linkedSections)
311
+ continue;
312
+ for (const section of linkedSections) {
313
+ if (section.linkedEntryId) {
314
+ stats.readmeSectionsLinked++;
315
+ // Add README examples to component
316
+ const component = components.find(c => c.id === section.linkedEntryId);
317
+ if (component) {
318
+ component.readmeSections = component.readmeSections || [];
319
+ component.readmeSections.push(section);
320
+ component.examples.push(...section.codeBlocks);
321
+ stats.examplesExtracted += section.codeBlocks.length;
322
+ }
323
+ }
324
+ }
325
+ readmeSections.push(...linkedSections);
326
+ }
327
+ }
328
+ // Phase 5: Extract TypeScript type definitions
329
+ const typeDefinitions = [];
330
+ const typeResults = await processInParallel(sourceFiles.filter(f => f.endsWith('.ts')), async (filePath) => {
331
+ const content = await fs.readFile(filePath, 'utf8').catch(() => '');
332
+ if (!content)
333
+ return null;
334
+ return extractTypesFromFile(filePath, content);
335
+ });
336
+ for (const types of typeResults) {
337
+ if (types) {
338
+ typeDefinitions.push(...types);
339
+ }
340
+ }
341
+ // Phase 6: Extract test examples via @codex:example markers
342
+ // Reuses testFiles discovered in Phase 3
343
+ let testExamples = [];
344
+ if (fullConfig.analyzeTests && testFiles.length > 0) {
345
+ const exampleResults = await processInParallel(testFiles, async (testPath) => {
346
+ const content = await fs.readFile(testPath, 'utf8').catch(() => '');
347
+ if (!content)
348
+ return null;
349
+ return extractTestExamples(content, testPath);
350
+ });
351
+ for (const examples of exampleResults) {
352
+ if (examples) {
353
+ testExamples.push(...examples);
354
+ }
355
+ }
356
+ }
357
+ // Phase 7: Optionally parse vitest output and match to examples
358
+ let testStatuses = parseVitestOutput('{}'); // empty default
359
+ if (fullConfig.vitestOutputPath) {
360
+ const resolvedPath = isAbsolute(fullConfig.vitestOutputPath)
361
+ ? fullConfig.vitestOutputPath
362
+ : join(fullConfig.root, fullConfig.vitestOutputPath);
363
+ const vitestJson = await fs.readFile(resolvedPath, 'utf8').catch(err => {
364
+ console.warn(`[codex] Failed to read vitest output from ${resolvedPath}:`, err instanceof Error ? err.message : String(err));
365
+ return '';
366
+ });
367
+ if (vitestJson) {
368
+ testStatuses = parseVitestOutput(vitestJson);
369
+ testExamples = matchExamplesToStatuses(testExamples, testStatuses);
370
+ }
371
+ }
372
+ stats.processingTimeMs = performance.now() - startTime;
373
+ const result = {
374
+ components,
375
+ testAnalysis: testAnalyses,
376
+ readmeSections,
377
+ stats,
378
+ typeDefinitions,
379
+ testExamples,
380
+ testStatuses,
381
+ };
382
+ return result;
383
+ }
384
+ /**
385
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"extractSingleFile","stability":"stable","signature":"(filePath: string, content?: string) => Promise<EnrichedComponent | null>"}
386
+ *
387
+ * Extracts documentation from a single source file
388
+ * @param filePath - Path to the source file
389
+ * @param content - Optional file content (will read if not provided)
390
+ * @returns Enriched component or null if no codex annotation found
391
+ */
392
+ export async function extractSingleFile(filePath, content) {
393
+ const fileContent = content || (await fs.readFile(filePath, 'utf8').catch(() => ''));
394
+ if (!fileContent)
395
+ return null;
396
+ const discoveredComponents = await discoverComponentsFromFile(filePath, fileContent);
397
+ if (discoveredComponents.length === 0)
398
+ return null;
399
+ const discovered = discoveredComponents[0];
400
+ const jsdocMap = linkJSDocToDeclarations(fileContent, filePath);
401
+ const declarations = extractAllDeclarations(fileContent, filePath);
402
+ const sourceExamples = extractExamplesFromSource(fileContent, filePath);
403
+ const enrichedApis = [];
404
+ for (const api of discovered.apis) {
405
+ const jsdoc = jsdocMap.get(api.name);
406
+ const declaration = declarations.find(d => d.name === api.name);
407
+ const apiExamples = sourceExamples
408
+ .filter(se => se.functionName === api.name)
409
+ .flatMap(se => se.examples);
410
+ if (jsdoc) {
411
+ apiExamples.push(...jsdoc.examples);
412
+ }
413
+ enrichedApis.push({
414
+ id: `${discovered.id}.${api.name}`,
415
+ name: api.name,
416
+ parent: discovered.id,
417
+ signature: api.signature || declaration?.signature,
418
+ stability: api.stability || 'stable',
419
+ jsdoc,
420
+ examples: apiExamples,
421
+ testRefs: [],
422
+ sourceLocation: declaration?.location || { file: filePath, line: 1 },
423
+ });
424
+ }
425
+ return {
426
+ id: discovered.id,
427
+ title: discovered.title,
428
+ category: discovered.category,
429
+ description: getFirstDescription(jsdocMap),
430
+ apis: enrichedApis,
431
+ examples: sourceExamples.flatMap(se => se.examples),
432
+ testRefs: [],
433
+ sourceFiles: [{ file: filePath, line: 1 }],
434
+ };
435
+ }
436
+ /**
437
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"generateExtractionReport","stability":"stable","signature":"(result: ExtractionResult) => string"}
438
+ *
439
+ * Generates a human-readable report from extraction results
440
+ * @param result - Extraction result
441
+ * @returns Formatted report string
442
+ */
443
+ export function generateExtractionReport(result) {
444
+ const { stats, components, testAnalysis } = result;
445
+ const lines = [
446
+ '# Extraction Pipeline Report',
447
+ '',
448
+ '## Statistics',
449
+ `- Files scanned: ${stats.filesScanned}`,
450
+ `- Components found: ${stats.componentsFound}`,
451
+ `- APIs extracted: ${stats.apisExtracted}`,
452
+ `- Examples extracted: ${stats.examplesExtracted}`,
453
+ `- Tests analyzed: ${stats.testsAnalyzed}`,
454
+ `- README sections linked: ${stats.readmeSectionsLinked}`,
455
+ `- Processing time: ${stats.processingTimeMs.toFixed(2)}ms`,
456
+ '',
457
+ '## Components',
458
+ ];
459
+ for (const component of components) {
460
+ lines.push(`\n### ${component.title} (\`${component.id}\`)`);
461
+ lines.push(`Category: ${component.category}`);
462
+ if (component.description) {
463
+ lines.push(`\n${component.description}`);
464
+ }
465
+ lines.push(`\n**APIs (${component.apis.length}):**`);
466
+ for (const api of component.apis) {
467
+ const stability = api.stability === 'stable' ? '' : ` [${api.stability}]`;
468
+ lines.push(`- \`${api.name}\`${stability} - ${api.examples.length} examples, ${api.testRefs.length} tests`);
469
+ }
470
+ lines.push(`\n**Test coverage:** ${component.testRefs.length} test references`);
471
+ }
472
+ lines.push('\n## Test Analysis Summary');
473
+ const totalTests = testAnalysis.reduce((sum, a) => sum + a.testCases.length, 0);
474
+ lines.push(`Total test cases: ${totalTests}`);
475
+ return lines.join('\n');
476
+ }
477
+ /**
478
+ * @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"exportToCodexFormat","stability":"stable","signature":"(result: ExtractionResult) => object"}
479
+ *
480
+ * Exports extraction results to codex-compatible JSON format
481
+ * @param result - Extraction result
482
+ * @returns Object suitable for codex.index.json
483
+ */
484
+ export function exportToCodexFormat(result) {
485
+ return {
486
+ entries: result.components.map(c => ({
487
+ id: c.id,
488
+ title: c.title,
489
+ category: c.category,
490
+ description: c.description,
491
+ sourceFiles: c.sourceFiles.map(sf => ({
492
+ file: sf.file,
493
+ line: sf.line,
494
+ })),
495
+ apis: c.apis.map(api => ({
496
+ name: api.name,
497
+ signature: api.signature,
498
+ stability: api.stability,
499
+ examples: api.examples.map(ex => ({
500
+ code: ex.code,
501
+ language: ex.language,
502
+ runnable: ex.runnable,
503
+ })),
504
+ testRefs: api.testRefs.map(tr => ({
505
+ file: tr.file,
506
+ line: tr.line,
507
+ name: tr.name,
508
+ type: tr.type,
509
+ })),
510
+ })),
511
+ examples: c.examples.map(ex => ({
512
+ code: ex.code,
513
+ language: ex.language,
514
+ runnable: ex.runnable,
515
+ })),
516
+ testRefs: c.testRefs.map(tr => ({
517
+ file: tr.file,
518
+ line: tr.line,
519
+ name: tr.name,
520
+ type: tr.type,
521
+ })),
522
+ })),
523
+ stats: result.stats,
524
+ generatedAt: new Date().toISOString(),
525
+ };
526
+ }
@@ -0,0 +1,108 @@
1
+ /**
2
+ * @codex
3
+ * {
4
+ * "id": "pithy.codex.extraction.readme-sync",
5
+ * "title": "README Marker Sync",
6
+ * "category": "feature"
7
+ * }
8
+ *
9
+ * Parses README files for @codex:auto markers and replaces their content
10
+ * with generated documentation from extraction pipeline data.
11
+ */
12
+ import type { ReadmeMarker, ReadmeMarkerType, ReadmeSyncResult, ReadmeSyncConfig, EnrichedComponent, ExtendedExtractionResult, TestExample } from './types.js';
13
+ /**
14
+ * Parses HTML-style attributes from a marker opening tag string.
15
+ * Only quoted attribute values are supported (double or single quotes).
16
+ * Unquoted values like `install=pkg` are ignored by design.
17
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"parseMarkerAttributes","stability":"stable","signature":"(attrString: string) => Record<string, string>"}
18
+ */
19
+ export declare function parseMarkerAttributes(attrString: string): Record<string, string>;
20
+ /**
21
+ * Determines marker type from its attributes.
22
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"resolveMarkerType","stability":"stable","signature":"(attributes: Record<string, string>) => ReadmeMarkerType | null"}
23
+ */
24
+ export declare function resolveMarkerType(attributes: Record<string, string>): ReadmeMarkerType | null;
25
+ /**
26
+ * Parses all @codex:auto markers from README content.
27
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"parseReadmeMarkers","stability":"stable","signature":"(content: string) => ReadmeMarker[]"}
28
+ */
29
+ export declare function parseReadmeMarkers(content: string): ReadmeMarker[];
30
+ /**
31
+ * Matches codex entry IDs against a glob-style pattern.
32
+ * Supports: exact match, trailing `*` (one segment), trailing `**` (any depth).
33
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"matchIdPattern","stability":"stable","signature":"(pattern: string, entryId: string) => boolean"}
34
+ */
35
+ export declare function matchIdPattern(pattern: string, entryId: string): boolean;
36
+ /**
37
+ * Filters components by an ID pattern.
38
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"filterByPattern","stability":"stable","signature":"(components: EnrichedComponent[], pattern: string) => EnrichedComponent[]"}
39
+ */
40
+ export declare function filterByPattern(components: EnrichedComponent[], pattern: string): EnrichedComponent[];
41
+ /**
42
+ * Strips test framework boilerplate from example code extracted from tests.
43
+ * Removes it()/test() wrapper, dedents, converts expect() to value comments.
44
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"stripTestBoilerplate","stability":"stable","signature":"(code: string) => string"}
45
+ */
46
+ export declare function stripTestBoilerplate(code: string): string;
47
+ /**
48
+ * Generates install command content for a package.
49
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateInstallContent","stability":"stable","signature":"(packageName: string) => string"}
50
+ */
51
+ export declare function generateInstallContent(packageName: string): string;
52
+ /**
53
+ * Generates examples content from extraction data.
54
+ * Prefers API-level JSDoc examples; falls back to @codex:example test examples
55
+ * when no API examples are found (test examples are safe from feedback loops).
56
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateExamplesContent","stability":"stable","signature":"(components: EnrichedComponent[], limit?: number, testExamples?: TestExample[], pattern?: string) => string"}
57
+ */
58
+ export declare function generateExamplesContent(components: EnrichedComponent[], limit?: number, testExamples?: TestExample[], pattern?: string): string;
59
+ /**
60
+ * Escapes pipe characters and collapses newlines for use in markdown table cells.
61
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"escapeTableCell","stability":"stable","signature":"(str: string) => string"}
62
+ */
63
+ export declare function escapeTableCell(str: string): string;
64
+ /**
65
+ * Escapes markdown special characters in inline text (bold markers, brackets, pipes, backticks).
66
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"escapeMd","stability":"stable","signature":"(str: string) => string"}
67
+ */
68
+ export declare function escapeMd(str: string): string;
69
+ /**
70
+ * Strips backticks from a string so it can safely be wrapped in inline code.
71
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"sanitizeForInlineCode","stability":"stable","signature":"(str: string) => string"}
72
+ */
73
+ export declare function sanitizeForInlineCode(str: string): string;
74
+ /** Returns the length of the longest consecutive run of backticks in a string.
75
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"longestBacktickRun","stability":"stable","signature":"(str: string) => number"} */
76
+ export declare function longestBacktickRun(str: string): number;
77
+ /** Sanitizes a language identifier for use in fenced code blocks.
78
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"sanitizeLanguage","stability":"stable","signature":"(lang: string) => string"} */
79
+ export declare function sanitizeLanguage(lang: string): string;
80
+ /**
81
+ * Generates API reference table from extraction data.
82
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateApiContent","stability":"stable","signature":"(components: EnrichedComponent[], format?: string) => string"}
83
+ */
84
+ export declare function generateApiContent(components: EnrichedComponent[], format?: string): string;
85
+ /**
86
+ * Generates testing pyramid table from extraction data.
87
+ * Includes a Status column showing coverage completeness based on
88
+ * test references and category-aware testing pyramid requirements.
89
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateTestingContent","stability":"stable","signature":"(components: EnrichedComponent[]) => string"}
90
+ */
91
+ export declare function generateTestingContent(components: EnrichedComponent[]): string;
92
+ /**
93
+ * Placeholder for bundle content generation (not yet implemented).
94
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateBundleContent","stability":"stable","signature":"(_packageName: string) => string"}
95
+ */
96
+ export declare function generateBundleContent(_packageName: string): string;
97
+ /**
98
+ * Syncs a single README's content: parses markers, generates content, replaces.
99
+ *
100
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"syncReadmeContent","stability":"stable","signature":"(content: string, filePath: string, components: EnrichedComponent[], testExamples?: TestExample[]) => ReadmeSyncResult"}
101
+ */
102
+ export declare function syncReadmeContent(content: string, filePath: string, components: EnrichedComponent[], testExamples?: TestExample[]): ReadmeSyncResult;
103
+ /**
104
+ * Syncs all README files matching configured patterns.
105
+ *
106
+ * @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"syncAllReadmes","stability":"stable","signature":"(extractionResult: ExtendedExtractionResult, config?: ReadmeSyncConfig) => Promise<ReadmeSyncResult[]>"}
107
+ */
108
+ export declare function syncAllReadmes(extractionResult: ExtendedExtractionResult, config?: ReadmeSyncConfig): Promise<ReadmeSyncResult[]>;