@pithyjs/codex 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +4401 -0
- package/dist/annotations.d.ts +106 -0
- package/dist/annotations.js +306 -0
- package/dist/apply.d.ts +2 -0
- package/dist/apply.js +80 -0
- package/dist/changed-scope.d.ts +23 -0
- package/dist/changed-scope.js +31 -0
- package/dist/check.d.ts +2 -0
- package/dist/check.js +117 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +67 -0
- package/dist/env.d.ts +5 -0
- package/dist/env.js +35 -0
- package/dist/extract-cli.d.ts +23 -0
- package/dist/extract-cli.js +192 -0
- package/dist/extraction/breaking-changes.d.ts +66 -0
- package/dist/extraction/breaking-changes.js +352 -0
- package/dist/extraction/example-extractor.d.ts +55 -0
- package/dist/extraction/example-extractor.js +272 -0
- package/dist/extraction/index.d.ts +27 -0
- package/dist/extraction/index.js +31 -0
- package/dist/extraction/jsdoc-parser.d.ts +43 -0
- package/dist/extraction/jsdoc-parser.js +274 -0
- package/dist/extraction/pipeline.d.ts +54 -0
- package/dist/extraction/pipeline.js +526 -0
- package/dist/extraction/readme-sync.d.ts +108 -0
- package/dist/extraction/readme-sync.js +592 -0
- package/dist/extraction/snapshot-store.d.ts +40 -0
- package/dist/extraction/snapshot-store.js +153 -0
- package/dist/extraction/source-linker.d.ts +80 -0
- package/dist/extraction/source-linker.js +316 -0
- package/dist/extraction/test-example-extractor.d.ts +69 -0
- package/dist/extraction/test-example-extractor.js +400 -0
- package/dist/extraction/test-pattern-extractor.d.ts +68 -0
- package/dist/extraction/test-pattern-extractor.js +261 -0
- package/dist/extraction/testing-pyramid.d.ts +44 -0
- package/dist/extraction/testing-pyramid.js +163 -0
- package/dist/extraction/type-extractor.d.ts +34 -0
- package/dist/extraction/type-extractor.js +494 -0
- package/dist/extraction/types.d.ts +401 -0
- package/dist/extraction/types.js +34 -0
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +107 -0
- package/dist/llm.d.ts +8 -0
- package/dist/llm.js +75 -0
- package/dist/readme-sync-cli.d.ts +20 -0
- package/dist/readme-sync-cli.js +167 -0
- package/dist/review.d.ts +2 -0
- package/dist/review.js +93 -0
- package/dist/scan.d.ts +29 -0
- package/dist/scan.js +221 -0
- package/dist/schema.d.ts +169 -0
- package/dist/schema.js +70 -0
- package/dist/snapshot-cli.d.ts +45 -0
- package/dist/snapshot-cli.js +217 -0
- package/dist/sync-cli.d.ts +22 -0
- package/dist/sync-cli.js +154 -0
- package/dist/sync-pipeline.d.ts +58 -0
- package/dist/sync-pipeline.js +104 -0
- package/dist/sync.d.ts +2 -0
- package/dist/sync.js +318 -0
- package/dist/validate-cli.d.ts +20 -0
- package/dist/validate-cli.js +144 -0
- package/dist/validate.d.ts +76 -0
- package/dist/validate.js +183 -0
- package/dist/watch-cli.d.ts +21 -0
- package/dist/watch-cli.js +220 -0
- package/package.json +62 -0
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.pipeline",
|
|
5
|
+
* "title": "Extraction Pipeline",
|
|
6
|
+
* "category": "feature"
|
|
7
|
+
* }
|
|
8
|
+
*/
|
|
9
|
+
import { globby } from 'globby';
|
|
10
|
+
import fs from 'node:fs/promises';
|
|
11
|
+
import { basename, isAbsolute, join, relative } from 'node:path';
|
|
12
|
+
import { DEFAULT_EXTRACTION_CONFIG, } from './types.js';
|
|
13
|
+
import { linkJSDocToDeclarations } from './jsdoc-parser.js';
|
|
14
|
+
import { extractExamplesFromSource } from './example-extractor.js';
|
|
15
|
+
import { analyzeTestFile, matchTestToEntry, testCasesToReferences, } from './test-pattern-extractor.js';
|
|
16
|
+
import { extractAllDeclarations, parseReadmeSections, linkReadmeToEntries, } from './source-linker.js';
|
|
17
|
+
import { discoverComponentsFromFile } from '../annotations.js';
|
|
18
|
+
import { extractTypesFromFile } from './type-extractor.js';
|
|
19
|
+
import { extractTestExamples, parseVitestOutput, matchExamplesToStatuses, } from './test-example-extractor.js';
|
|
20
|
+
/**
|
|
21
|
+
* Concurrency limit for parallel file processing
|
|
22
|
+
*/
|
|
23
|
+
const CONCURRENCY_LIMIT = 10;
|
|
24
|
+
/**
|
|
25
|
+
* Processes items in parallel with a concurrency limit
|
|
26
|
+
* @param items - Items to process
|
|
27
|
+
* @param processor - Async function to process each item
|
|
28
|
+
* @param limit - Maximum concurrent operations
|
|
29
|
+
*/
|
|
30
|
+
async function processInParallel(items, processor, limit = CONCURRENCY_LIMIT) {
|
|
31
|
+
const results = [];
|
|
32
|
+
for (let i = 0; i < items.length; i += limit) {
|
|
33
|
+
const batch = items.slice(i, i + limit);
|
|
34
|
+
const batchResults = await Promise.all(batch.map(processor));
|
|
35
|
+
results.push(...batchResults);
|
|
36
|
+
}
|
|
37
|
+
return results;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Gets the first description from a JSDoc map, if any exists
|
|
41
|
+
* @param jsdocMap - Map of declaration names to JSDoc
|
|
42
|
+
* @returns First description found, or undefined
|
|
43
|
+
*/
|
|
44
|
+
function getFirstDescription(jsdocMap) {
|
|
45
|
+
for (const jsdoc of jsdocMap.values()) {
|
|
46
|
+
if (jsdoc.description) {
|
|
47
|
+
return jsdoc.description;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Builds full config by merging defaults with provided options
|
|
54
|
+
*/
|
|
55
|
+
function buildFullConfig(config) {
|
|
56
|
+
return {
|
|
57
|
+
root: config?.root ?? process.cwd(),
|
|
58
|
+
sourcePatterns: config?.sourcePatterns ?? DEFAULT_EXTRACTION_CONFIG.sourcePatterns,
|
|
59
|
+
testPatterns: config?.testPatterns ?? DEFAULT_EXTRACTION_CONFIG.testPatterns,
|
|
60
|
+
readmePatterns: config?.readmePatterns ?? DEFAULT_EXTRACTION_CONFIG.readmePatterns,
|
|
61
|
+
extractRunnableExamples: config?.extractRunnableExamples ??
|
|
62
|
+
DEFAULT_EXTRACTION_CONFIG.extractRunnableExamples,
|
|
63
|
+
analyzeTests: config?.analyzeTests ?? DEFAULT_EXTRACTION_CONFIG.analyzeTests,
|
|
64
|
+
linkReadmes: config?.linkReadmes ?? DEFAULT_EXTRACTION_CONFIG.linkReadmes,
|
|
65
|
+
vitestOutputPath: config?.vitestOutputPath,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"runExtractionPipeline","stability":"stable","signature":"(config?: Partial<ExtractionConfig>) => Promise<ExtendedExtractionResult>"}
|
|
70
|
+
*
|
|
71
|
+
* Runs the complete multi-source extraction pipeline
|
|
72
|
+
* @param config - Configuration options
|
|
73
|
+
* @returns Extraction result with all enriched data including types and test examples
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* ```typescript
|
|
77
|
+
* const result = await runExtractionPipeline({
|
|
78
|
+
* root: '/path/to/project',
|
|
79
|
+
* extractRunnableExamples: true,
|
|
80
|
+
* });
|
|
81
|
+
*
|
|
82
|
+
* console.log(`Found ${result.stats.componentsFound} components`);
|
|
83
|
+
* console.log(`Extracted ${result.typeDefinitions.length} types`);
|
|
84
|
+
* console.log(`Found ${result.testExamples.length} test examples`);
|
|
85
|
+
* ```
|
|
86
|
+
*/
|
|
87
|
+
export async function runExtractionPipeline(config) {
|
|
88
|
+
const startTime = performance.now();
|
|
89
|
+
const fullConfig = buildFullConfig(config);
|
|
90
|
+
const stats = {
|
|
91
|
+
filesScanned: 0,
|
|
92
|
+
componentsFound: 0,
|
|
93
|
+
apisExtracted: 0,
|
|
94
|
+
examplesExtracted: 0,
|
|
95
|
+
testsAnalyzed: 0,
|
|
96
|
+
readmeSectionsLinked: 0,
|
|
97
|
+
processingTimeMs: 0,
|
|
98
|
+
};
|
|
99
|
+
// Phase 1: Discover source files
|
|
100
|
+
const sourceFiles = await globby(fullConfig.sourcePatterns, {
|
|
101
|
+
cwd: fullConfig.root,
|
|
102
|
+
absolute: true,
|
|
103
|
+
gitignore: true,
|
|
104
|
+
});
|
|
105
|
+
stats.filesScanned = sourceFiles.length;
|
|
106
|
+
// Phase 2: Extract components and APIs from source files (parallelized)
|
|
107
|
+
const components = [];
|
|
108
|
+
const allApiNames = [];
|
|
109
|
+
const extractionResults = await processInParallel(sourceFiles, async (filePath) => {
|
|
110
|
+
const content = await fs.readFile(filePath, 'utf8').catch(() => '');
|
|
111
|
+
if (!content)
|
|
112
|
+
return null;
|
|
113
|
+
// Get base component info from codex annotations
|
|
114
|
+
const discoveredComponents = await discoverComponentsFromFile(filePath, content);
|
|
115
|
+
if (discoveredComponents.length === 0)
|
|
116
|
+
return null;
|
|
117
|
+
const fileComponents = [];
|
|
118
|
+
const fileApiNames = [];
|
|
119
|
+
let fileExamplesCount = 0;
|
|
120
|
+
for (const discovered of discoveredComponents) {
|
|
121
|
+
// Enrich with JSDoc parsing
|
|
122
|
+
const jsdocMap = linkJSDocToDeclarations(content, filePath);
|
|
123
|
+
const declarations = extractAllDeclarations(content, filePath);
|
|
124
|
+
const sourceExamples = extractExamplesFromSource(content, filePath);
|
|
125
|
+
// Enrich APIs with JSDoc
|
|
126
|
+
const enrichedApis = [];
|
|
127
|
+
for (const api of discovered.apis) {
|
|
128
|
+
const jsdoc = jsdocMap.get(api.name);
|
|
129
|
+
const declaration = declarations.find(d => d.name === api.name);
|
|
130
|
+
// Collect examples for this API
|
|
131
|
+
const apiExamples = sourceExamples
|
|
132
|
+
.filter(se => se.functionName === api.name)
|
|
133
|
+
.flatMap(se => se.examples);
|
|
134
|
+
if (jsdoc) {
|
|
135
|
+
apiExamples.push(...jsdoc.examples);
|
|
136
|
+
}
|
|
137
|
+
enrichedApis.push({
|
|
138
|
+
id: `${discovered.id}.${api.name}`,
|
|
139
|
+
name: api.name,
|
|
140
|
+
parent: discovered.id,
|
|
141
|
+
signature: api.signature || declaration?.signature,
|
|
142
|
+
stability: api.stability || 'stable',
|
|
143
|
+
jsdoc,
|
|
144
|
+
examples: apiExamples,
|
|
145
|
+
testRefs: [], // Will be populated in Phase 3
|
|
146
|
+
sourceLocation: declaration?.location || {
|
|
147
|
+
file: filePath,
|
|
148
|
+
line: 1,
|
|
149
|
+
},
|
|
150
|
+
});
|
|
151
|
+
fileApiNames.push(api.name);
|
|
152
|
+
fileExamplesCount += apiExamples.length;
|
|
153
|
+
}
|
|
154
|
+
// Collect component-level examples
|
|
155
|
+
const componentExamples = sourceExamples
|
|
156
|
+
.filter(se => !enrichedApis.some(a => a.name === se.functionName))
|
|
157
|
+
.flatMap(se => se.examples);
|
|
158
|
+
fileComponents.push({
|
|
159
|
+
id: discovered.id,
|
|
160
|
+
title: discovered.title,
|
|
161
|
+
category: discovered.category,
|
|
162
|
+
risk: discovered.risk,
|
|
163
|
+
testing: discovered.testing,
|
|
164
|
+
description: getFirstDescription(jsdocMap),
|
|
165
|
+
apis: enrichedApis,
|
|
166
|
+
examples: componentExamples,
|
|
167
|
+
testRefs: [], // Will be populated in Phase 3
|
|
168
|
+
sourceFiles: [{ file: filePath, line: 1 }],
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
return {
|
|
172
|
+
components: fileComponents,
|
|
173
|
+
apiNames: fileApiNames,
|
|
174
|
+
examplesCount: fileExamplesCount,
|
|
175
|
+
};
|
|
176
|
+
});
|
|
177
|
+
// Aggregate results from parallel processing
|
|
178
|
+
for (const result of extractionResults) {
|
|
179
|
+
if (result) {
|
|
180
|
+
components.push(...result.components);
|
|
181
|
+
allApiNames.push(...result.apiNames);
|
|
182
|
+
stats.examplesExtracted += result.examplesCount;
|
|
183
|
+
stats.apisExtracted += result.components.reduce((sum, c) => sum + c.apis.length, 0);
|
|
184
|
+
stats.componentsFound += result.components.length;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
// Phase 3: Analyze test files and link to components (parallelized)
|
|
188
|
+
const testAnalyses = [];
|
|
189
|
+
// Discover test files once, reused by Phase 6
|
|
190
|
+
const testFiles = fullConfig.analyzeTests
|
|
191
|
+
? await globby(fullConfig.testPatterns, {
|
|
192
|
+
cwd: fullConfig.root,
|
|
193
|
+
absolute: true,
|
|
194
|
+
gitignore: true,
|
|
195
|
+
})
|
|
196
|
+
: [];
|
|
197
|
+
if (fullConfig.analyzeTests) {
|
|
198
|
+
const entryIds = components.map(c => c.id);
|
|
199
|
+
// Build a reverse map from test filenames to entry IDs using
|
|
200
|
+
// the testing.unit.file / testing.integration.file hints in source
|
|
201
|
+
// annotations. Multiple entries may reference the same test file,
|
|
202
|
+
// so the map stores arrays of entry IDs.
|
|
203
|
+
//
|
|
204
|
+
// Keys are scoped by package directory (e.g. "packages/vitePlugins:scanner.test.ts")
|
|
205
|
+
// to prevent collisions when different packages share test filenames.
|
|
206
|
+
const testFileHints = new Map();
|
|
207
|
+
// Extract the package scope from an absolute path.
|
|
208
|
+
// Detects monorepo segments like "packages/<name>" or "apps/<name>" from
|
|
209
|
+
// the absolute path so hints work regardless of what `root` is set to.
|
|
210
|
+
// Falls back to the first two segments relative to root when no monorepo
|
|
211
|
+
// segment is found (single-package repos or non-standard layouts).
|
|
212
|
+
function packageDirOf(absPath) {
|
|
213
|
+
const normalized = absPath.replace(/\\/g, '/');
|
|
214
|
+
const monoMatch = normalized.match(/\/(packages|apps)\/([^/]+)\//);
|
|
215
|
+
if (monoMatch)
|
|
216
|
+
return `${monoMatch[1]}/${monoMatch[2]}`;
|
|
217
|
+
const rel = relative(fullConfig.root, absPath).replace(/\\/g, '/');
|
|
218
|
+
const parts = rel.split('/');
|
|
219
|
+
return parts.length >= 2 ? `${parts[0]}/${parts[1]}` : parts[0];
|
|
220
|
+
}
|
|
221
|
+
for (const comp of components) {
|
|
222
|
+
const srcFile = comp.sourceFiles[0]?.file;
|
|
223
|
+
if (!srcFile)
|
|
224
|
+
continue;
|
|
225
|
+
const pkgDir = packageDirOf(srcFile);
|
|
226
|
+
for (const level of ['unit', 'integration', 'e2e']) {
|
|
227
|
+
const hint = comp.testing?.[level]?.file;
|
|
228
|
+
if (hint) {
|
|
229
|
+
const key = `${pkgDir}:${basename(hint).toLowerCase()}`;
|
|
230
|
+
const existing = testFileHints.get(key) ?? [];
|
|
231
|
+
if (!existing.includes(comp.id)) {
|
|
232
|
+
existing.push(comp.id);
|
|
233
|
+
}
|
|
234
|
+
testFileHints.set(key, existing);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const analysisResults = await processInParallel(testFiles, async (testPath) => {
|
|
239
|
+
const content = await fs.readFile(testPath, 'utf8').catch(() => '');
|
|
240
|
+
if (!content)
|
|
241
|
+
return null;
|
|
242
|
+
const analysis = analyzeTestFile(content, testPath, allApiNames);
|
|
243
|
+
const hintKey = `${packageDirOf(testPath)}:${basename(testPath).toLowerCase()}`;
|
|
244
|
+
// Prefer explicit testing.*.file hints from source annotations
|
|
245
|
+
// (avoids heuristic mismatches when multiple entries share names).
|
|
246
|
+
// Keys are scoped by package directory so identically-named test
|
|
247
|
+
// files in different packages don't collide.
|
|
248
|
+
const hintEntryIds = testFileHints.get(hintKey);
|
|
249
|
+
const matchedEntryId = hintEntryIds?.[0] ?? matchTestToEntry(testPath, entryIds);
|
|
250
|
+
return { analysis, matchedEntryId, testPath, hintEntryIds };
|
|
251
|
+
});
|
|
252
|
+
// Helper: link test refs to a component and its APIs
|
|
253
|
+
function linkTestRefsToComponent(component, refs, analysis) {
|
|
254
|
+
component.testRefs.push(...refs);
|
|
255
|
+
for (const api of component.apis) {
|
|
256
|
+
const apiRefs = refs.filter(r => r.name.toLowerCase().includes(api.name.toLowerCase()) ||
|
|
257
|
+
analysis.coveredApis.includes(api.name));
|
|
258
|
+
api.testRefs.push(...apiRefs);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
// Build a lookup map for constant-time component access by ID
|
|
262
|
+
const componentById = new Map();
|
|
263
|
+
for (const comp of components) {
|
|
264
|
+
componentById.set(comp.id, comp);
|
|
265
|
+
}
|
|
266
|
+
// Apply results (must be sequential for component mutations)
|
|
267
|
+
for (const result of analysisResults) {
|
|
268
|
+
if (!result)
|
|
269
|
+
continue;
|
|
270
|
+
const { analysis, matchedEntryId, testPath, hintEntryIds } = result;
|
|
271
|
+
testAnalyses.push(analysis);
|
|
272
|
+
stats.testsAnalyzed += analysis.testCases.length;
|
|
273
|
+
const refs = testCasesToReferences(analysis.testCases, testPath);
|
|
274
|
+
// Link tests to ALL components that declare this test file
|
|
275
|
+
if (hintEntryIds && hintEntryIds.length > 0) {
|
|
276
|
+
for (const entryId of hintEntryIds) {
|
|
277
|
+
const component = componentById.get(entryId);
|
|
278
|
+
if (component) {
|
|
279
|
+
linkTestRefsToComponent(component, refs, analysis);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
else if (matchedEntryId) {
|
|
284
|
+
const component = componentById.get(matchedEntryId);
|
|
285
|
+
if (component) {
|
|
286
|
+
linkTestRefsToComponent(component, refs, analysis);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
// Phase 4: Parse and link README files (parallelized)
|
|
292
|
+
const readmeSections = [];
|
|
293
|
+
if (fullConfig.linkReadmes) {
|
|
294
|
+
const readmeFiles = await globby(fullConfig.readmePatterns, {
|
|
295
|
+
cwd: fullConfig.root,
|
|
296
|
+
absolute: true,
|
|
297
|
+
gitignore: true,
|
|
298
|
+
});
|
|
299
|
+
const entryIds = components.map(c => c.id);
|
|
300
|
+
const readmeResults = await processInParallel(readmeFiles, async (readmePath) => {
|
|
301
|
+
const content = await fs.readFile(readmePath, 'utf8').catch(() => '');
|
|
302
|
+
if (!content)
|
|
303
|
+
return null;
|
|
304
|
+
const sections = parseReadmeSections(content, readmePath);
|
|
305
|
+
const linkedSections = linkReadmeToEntries(sections, entryIds);
|
|
306
|
+
return linkedSections;
|
|
307
|
+
});
|
|
308
|
+
// Apply results (must be sequential for component mutations)
|
|
309
|
+
for (const linkedSections of readmeResults) {
|
|
310
|
+
if (!linkedSections)
|
|
311
|
+
continue;
|
|
312
|
+
for (const section of linkedSections) {
|
|
313
|
+
if (section.linkedEntryId) {
|
|
314
|
+
stats.readmeSectionsLinked++;
|
|
315
|
+
// Add README examples to component
|
|
316
|
+
const component = components.find(c => c.id === section.linkedEntryId);
|
|
317
|
+
if (component) {
|
|
318
|
+
component.readmeSections = component.readmeSections || [];
|
|
319
|
+
component.readmeSections.push(section);
|
|
320
|
+
component.examples.push(...section.codeBlocks);
|
|
321
|
+
stats.examplesExtracted += section.codeBlocks.length;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
readmeSections.push(...linkedSections);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// Phase 5: Extract TypeScript type definitions
|
|
329
|
+
const typeDefinitions = [];
|
|
330
|
+
const typeResults = await processInParallel(sourceFiles.filter(f => f.endsWith('.ts')), async (filePath) => {
|
|
331
|
+
const content = await fs.readFile(filePath, 'utf8').catch(() => '');
|
|
332
|
+
if (!content)
|
|
333
|
+
return null;
|
|
334
|
+
return extractTypesFromFile(filePath, content);
|
|
335
|
+
});
|
|
336
|
+
for (const types of typeResults) {
|
|
337
|
+
if (types) {
|
|
338
|
+
typeDefinitions.push(...types);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
// Phase 6: Extract test examples via @codex:example markers
|
|
342
|
+
// Reuses testFiles discovered in Phase 3
|
|
343
|
+
let testExamples = [];
|
|
344
|
+
if (fullConfig.analyzeTests && testFiles.length > 0) {
|
|
345
|
+
const exampleResults = await processInParallel(testFiles, async (testPath) => {
|
|
346
|
+
const content = await fs.readFile(testPath, 'utf8').catch(() => '');
|
|
347
|
+
if (!content)
|
|
348
|
+
return null;
|
|
349
|
+
return extractTestExamples(content, testPath);
|
|
350
|
+
});
|
|
351
|
+
for (const examples of exampleResults) {
|
|
352
|
+
if (examples) {
|
|
353
|
+
testExamples.push(...examples);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
// Phase 7: Optionally parse vitest output and match to examples
|
|
358
|
+
let testStatuses = parseVitestOutput('{}'); // empty default
|
|
359
|
+
if (fullConfig.vitestOutputPath) {
|
|
360
|
+
const resolvedPath = isAbsolute(fullConfig.vitestOutputPath)
|
|
361
|
+
? fullConfig.vitestOutputPath
|
|
362
|
+
: join(fullConfig.root, fullConfig.vitestOutputPath);
|
|
363
|
+
const vitestJson = await fs.readFile(resolvedPath, 'utf8').catch(err => {
|
|
364
|
+
console.warn(`[codex] Failed to read vitest output from ${resolvedPath}:`, err instanceof Error ? err.message : String(err));
|
|
365
|
+
return '';
|
|
366
|
+
});
|
|
367
|
+
if (vitestJson) {
|
|
368
|
+
testStatuses = parseVitestOutput(vitestJson);
|
|
369
|
+
testExamples = matchExamplesToStatuses(testExamples, testStatuses);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
stats.processingTimeMs = performance.now() - startTime;
|
|
373
|
+
const result = {
|
|
374
|
+
components,
|
|
375
|
+
testAnalysis: testAnalyses,
|
|
376
|
+
readmeSections,
|
|
377
|
+
stats,
|
|
378
|
+
typeDefinitions,
|
|
379
|
+
testExamples,
|
|
380
|
+
testStatuses,
|
|
381
|
+
};
|
|
382
|
+
return result;
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"extractSingleFile","stability":"stable","signature":"(filePath: string, content?: string) => Promise<EnrichedComponent | null>"}
|
|
386
|
+
*
|
|
387
|
+
* Extracts documentation from a single source file
|
|
388
|
+
* @param filePath - Path to the source file
|
|
389
|
+
* @param content - Optional file content (will read if not provided)
|
|
390
|
+
* @returns Enriched component or null if no codex annotation found
|
|
391
|
+
*/
|
|
392
|
+
export async function extractSingleFile(filePath, content) {
|
|
393
|
+
const fileContent = content || (await fs.readFile(filePath, 'utf8').catch(() => ''));
|
|
394
|
+
if (!fileContent)
|
|
395
|
+
return null;
|
|
396
|
+
const discoveredComponents = await discoverComponentsFromFile(filePath, fileContent);
|
|
397
|
+
if (discoveredComponents.length === 0)
|
|
398
|
+
return null;
|
|
399
|
+
const discovered = discoveredComponents[0];
|
|
400
|
+
const jsdocMap = linkJSDocToDeclarations(fileContent, filePath);
|
|
401
|
+
const declarations = extractAllDeclarations(fileContent, filePath);
|
|
402
|
+
const sourceExamples = extractExamplesFromSource(fileContent, filePath);
|
|
403
|
+
const enrichedApis = [];
|
|
404
|
+
for (const api of discovered.apis) {
|
|
405
|
+
const jsdoc = jsdocMap.get(api.name);
|
|
406
|
+
const declaration = declarations.find(d => d.name === api.name);
|
|
407
|
+
const apiExamples = sourceExamples
|
|
408
|
+
.filter(se => se.functionName === api.name)
|
|
409
|
+
.flatMap(se => se.examples);
|
|
410
|
+
if (jsdoc) {
|
|
411
|
+
apiExamples.push(...jsdoc.examples);
|
|
412
|
+
}
|
|
413
|
+
enrichedApis.push({
|
|
414
|
+
id: `${discovered.id}.${api.name}`,
|
|
415
|
+
name: api.name,
|
|
416
|
+
parent: discovered.id,
|
|
417
|
+
signature: api.signature || declaration?.signature,
|
|
418
|
+
stability: api.stability || 'stable',
|
|
419
|
+
jsdoc,
|
|
420
|
+
examples: apiExamples,
|
|
421
|
+
testRefs: [],
|
|
422
|
+
sourceLocation: declaration?.location || { file: filePath, line: 1 },
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
return {
|
|
426
|
+
id: discovered.id,
|
|
427
|
+
title: discovered.title,
|
|
428
|
+
category: discovered.category,
|
|
429
|
+
description: getFirstDescription(jsdocMap),
|
|
430
|
+
apis: enrichedApis,
|
|
431
|
+
examples: sourceExamples.flatMap(se => se.examples),
|
|
432
|
+
testRefs: [],
|
|
433
|
+
sourceFiles: [{ file: filePath, line: 1 }],
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"generateExtractionReport","stability":"stable","signature":"(result: ExtractionResult) => string"}
|
|
438
|
+
*
|
|
439
|
+
* Generates a human-readable report from extraction results
|
|
440
|
+
* @param result - Extraction result
|
|
441
|
+
* @returns Formatted report string
|
|
442
|
+
*/
|
|
443
|
+
export function generateExtractionReport(result) {
|
|
444
|
+
const { stats, components, testAnalysis } = result;
|
|
445
|
+
const lines = [
|
|
446
|
+
'# Extraction Pipeline Report',
|
|
447
|
+
'',
|
|
448
|
+
'## Statistics',
|
|
449
|
+
`- Files scanned: ${stats.filesScanned}`,
|
|
450
|
+
`- Components found: ${stats.componentsFound}`,
|
|
451
|
+
`- APIs extracted: ${stats.apisExtracted}`,
|
|
452
|
+
`- Examples extracted: ${stats.examplesExtracted}`,
|
|
453
|
+
`- Tests analyzed: ${stats.testsAnalyzed}`,
|
|
454
|
+
`- README sections linked: ${stats.readmeSectionsLinked}`,
|
|
455
|
+
`- Processing time: ${stats.processingTimeMs.toFixed(2)}ms`,
|
|
456
|
+
'',
|
|
457
|
+
'## Components',
|
|
458
|
+
];
|
|
459
|
+
for (const component of components) {
|
|
460
|
+
lines.push(`\n### ${component.title} (\`${component.id}\`)`);
|
|
461
|
+
lines.push(`Category: ${component.category}`);
|
|
462
|
+
if (component.description) {
|
|
463
|
+
lines.push(`\n${component.description}`);
|
|
464
|
+
}
|
|
465
|
+
lines.push(`\n**APIs (${component.apis.length}):**`);
|
|
466
|
+
for (const api of component.apis) {
|
|
467
|
+
const stability = api.stability === 'stable' ? '' : ` [${api.stability}]`;
|
|
468
|
+
lines.push(`- \`${api.name}\`${stability} - ${api.examples.length} examples, ${api.testRefs.length} tests`);
|
|
469
|
+
}
|
|
470
|
+
lines.push(`\n**Test coverage:** ${component.testRefs.length} test references`);
|
|
471
|
+
}
|
|
472
|
+
lines.push('\n## Test Analysis Summary');
|
|
473
|
+
const totalTests = testAnalysis.reduce((sum, a) => sum + a.testCases.length, 0);
|
|
474
|
+
lines.push(`Total test cases: ${totalTests}`);
|
|
475
|
+
return lines.join('\n');
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* @codexApi {"parent":"pithy.codex.extraction.pipeline","name":"exportToCodexFormat","stability":"stable","signature":"(result: ExtractionResult) => object"}
|
|
479
|
+
*
|
|
480
|
+
* Exports extraction results to codex-compatible JSON format
|
|
481
|
+
* @param result - Extraction result
|
|
482
|
+
* @returns Object suitable for codex.index.json
|
|
483
|
+
*/
|
|
484
|
+
export function exportToCodexFormat(result) {
|
|
485
|
+
return {
|
|
486
|
+
entries: result.components.map(c => ({
|
|
487
|
+
id: c.id,
|
|
488
|
+
title: c.title,
|
|
489
|
+
category: c.category,
|
|
490
|
+
description: c.description,
|
|
491
|
+
sourceFiles: c.sourceFiles.map(sf => ({
|
|
492
|
+
file: sf.file,
|
|
493
|
+
line: sf.line,
|
|
494
|
+
})),
|
|
495
|
+
apis: c.apis.map(api => ({
|
|
496
|
+
name: api.name,
|
|
497
|
+
signature: api.signature,
|
|
498
|
+
stability: api.stability,
|
|
499
|
+
examples: api.examples.map(ex => ({
|
|
500
|
+
code: ex.code,
|
|
501
|
+
language: ex.language,
|
|
502
|
+
runnable: ex.runnable,
|
|
503
|
+
})),
|
|
504
|
+
testRefs: api.testRefs.map(tr => ({
|
|
505
|
+
file: tr.file,
|
|
506
|
+
line: tr.line,
|
|
507
|
+
name: tr.name,
|
|
508
|
+
type: tr.type,
|
|
509
|
+
})),
|
|
510
|
+
})),
|
|
511
|
+
examples: c.examples.map(ex => ({
|
|
512
|
+
code: ex.code,
|
|
513
|
+
language: ex.language,
|
|
514
|
+
runnable: ex.runnable,
|
|
515
|
+
})),
|
|
516
|
+
testRefs: c.testRefs.map(tr => ({
|
|
517
|
+
file: tr.file,
|
|
518
|
+
line: tr.line,
|
|
519
|
+
name: tr.name,
|
|
520
|
+
type: tr.type,
|
|
521
|
+
})),
|
|
522
|
+
})),
|
|
523
|
+
stats: result.stats,
|
|
524
|
+
generatedAt: new Date().toISOString(),
|
|
525
|
+
};
|
|
526
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.readme-sync",
|
|
5
|
+
* "title": "README Marker Sync",
|
|
6
|
+
* "category": "feature"
|
|
7
|
+
* }
|
|
8
|
+
*
|
|
9
|
+
* Parses README files for @codex:auto markers and replaces their content
|
|
10
|
+
* with generated documentation from extraction pipeline data.
|
|
11
|
+
*/
|
|
12
|
+
import type { ReadmeMarker, ReadmeMarkerType, ReadmeSyncResult, ReadmeSyncConfig, EnrichedComponent, ExtendedExtractionResult, TestExample } from './types.js';
|
|
13
|
+
/**
|
|
14
|
+
* Parses HTML-style attributes from a marker opening tag string.
|
|
15
|
+
* Only quoted attribute values are supported (double or single quotes).
|
|
16
|
+
* Unquoted values like `install=pkg` are ignored by design.
|
|
17
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"parseMarkerAttributes","stability":"stable","signature":"(attrString: string) => Record<string, string>"}
|
|
18
|
+
*/
|
|
19
|
+
export declare function parseMarkerAttributes(attrString: string): Record<string, string>;
|
|
20
|
+
/**
|
|
21
|
+
* Determines marker type from its attributes.
|
|
22
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"resolveMarkerType","stability":"stable","signature":"(attributes: Record<string, string>) => ReadmeMarkerType | null"}
|
|
23
|
+
*/
|
|
24
|
+
export declare function resolveMarkerType(attributes: Record<string, string>): ReadmeMarkerType | null;
|
|
25
|
+
/**
|
|
26
|
+
* Parses all @codex:auto markers from README content.
|
|
27
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"parseReadmeMarkers","stability":"stable","signature":"(content: string) => ReadmeMarker[]"}
|
|
28
|
+
*/
|
|
29
|
+
export declare function parseReadmeMarkers(content: string): ReadmeMarker[];
|
|
30
|
+
/**
|
|
31
|
+
* Matches codex entry IDs against a glob-style pattern.
|
|
32
|
+
* Supports: exact match, trailing `*` (one segment), trailing `**` (any depth).
|
|
33
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"matchIdPattern","stability":"stable","signature":"(pattern: string, entryId: string) => boolean"}
|
|
34
|
+
*/
|
|
35
|
+
export declare function matchIdPattern(pattern: string, entryId: string): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* Filters components by an ID pattern.
|
|
38
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"filterByPattern","stability":"stable","signature":"(components: EnrichedComponent[], pattern: string) => EnrichedComponent[]"}
|
|
39
|
+
*/
|
|
40
|
+
export declare function filterByPattern(components: EnrichedComponent[], pattern: string): EnrichedComponent[];
|
|
41
|
+
/**
|
|
42
|
+
* Strips test framework boilerplate from example code extracted from tests.
|
|
43
|
+
* Removes it()/test() wrapper, dedents, converts expect() to value comments.
|
|
44
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"stripTestBoilerplate","stability":"stable","signature":"(code: string) => string"}
|
|
45
|
+
*/
|
|
46
|
+
export declare function stripTestBoilerplate(code: string): string;
|
|
47
|
+
/**
|
|
48
|
+
* Generates install command content for a package.
|
|
49
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateInstallContent","stability":"stable","signature":"(packageName: string) => string"}
|
|
50
|
+
*/
|
|
51
|
+
export declare function generateInstallContent(packageName: string): string;
|
|
52
|
+
/**
|
|
53
|
+
* Generates examples content from extraction data.
|
|
54
|
+
* Prefers API-level JSDoc examples; falls back to @codex:example test examples
|
|
55
|
+
* when no API examples are found (test examples are safe from feedback loops).
|
|
56
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateExamplesContent","stability":"stable","signature":"(components: EnrichedComponent[], limit?: number, testExamples?: TestExample[], pattern?: string) => string"}
|
|
57
|
+
*/
|
|
58
|
+
export declare function generateExamplesContent(components: EnrichedComponent[], limit?: number, testExamples?: TestExample[], pattern?: string): string;
|
|
59
|
+
/**
|
|
60
|
+
* Escapes pipe characters and collapses newlines for use in markdown table cells.
|
|
61
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"escapeTableCell","stability":"stable","signature":"(str: string) => string"}
|
|
62
|
+
*/
|
|
63
|
+
export declare function escapeTableCell(str: string): string;
|
|
64
|
+
/**
|
|
65
|
+
* Escapes markdown special characters in inline text (bold markers, brackets, pipes, backticks).
|
|
66
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"escapeMd","stability":"stable","signature":"(str: string) => string"}
|
|
67
|
+
*/
|
|
68
|
+
export declare function escapeMd(str: string): string;
|
|
69
|
+
/**
|
|
70
|
+
* Strips backticks from a string so it can safely be wrapped in inline code.
|
|
71
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"sanitizeForInlineCode","stability":"stable","signature":"(str: string) => string"}
|
|
72
|
+
*/
|
|
73
|
+
export declare function sanitizeForInlineCode(str: string): string;
|
|
74
|
+
/** Returns the length of the longest consecutive run of backticks in a string.
|
|
75
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"longestBacktickRun","stability":"stable","signature":"(str: string) => number"} */
|
|
76
|
+
export declare function longestBacktickRun(str: string): number;
|
|
77
|
+
/** Sanitizes a language identifier for use in fenced code blocks.
|
|
78
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"sanitizeLanguage","stability":"stable","signature":"(lang: string) => string"} */
|
|
79
|
+
export declare function sanitizeLanguage(lang: string): string;
|
|
80
|
+
/**
|
|
81
|
+
* Generates API reference table from extraction data.
|
|
82
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateApiContent","stability":"stable","signature":"(components: EnrichedComponent[], format?: string) => string"}
|
|
83
|
+
*/
|
|
84
|
+
export declare function generateApiContent(components: EnrichedComponent[], format?: string): string;
|
|
85
|
+
/**
|
|
86
|
+
* Generates testing pyramid table from extraction data.
|
|
87
|
+
* Includes a Status column showing coverage completeness based on
|
|
88
|
+
* test references and category-aware testing pyramid requirements.
|
|
89
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateTestingContent","stability":"stable","signature":"(components: EnrichedComponent[]) => string"}
|
|
90
|
+
*/
|
|
91
|
+
export declare function generateTestingContent(components: EnrichedComponent[]): string;
|
|
92
|
+
/**
|
|
93
|
+
* Placeholder for bundle content generation (not yet implemented).
|
|
94
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"generateBundleContent","stability":"stable","signature":"(_packageName: string) => string"}
|
|
95
|
+
*/
|
|
96
|
+
export declare function generateBundleContent(_packageName: string): string;
|
|
97
|
+
/**
|
|
98
|
+
* Syncs a single README's content: parses markers, generates content, replaces.
|
|
99
|
+
*
|
|
100
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"syncReadmeContent","stability":"stable","signature":"(content: string, filePath: string, components: EnrichedComponent[], testExamples?: TestExample[]) => ReadmeSyncResult"}
|
|
101
|
+
*/
|
|
102
|
+
export declare function syncReadmeContent(content: string, filePath: string, components: EnrichedComponent[], testExamples?: TestExample[]): ReadmeSyncResult;
|
|
103
|
+
/**
|
|
104
|
+
* Syncs all README files matching configured patterns.
|
|
105
|
+
*
|
|
106
|
+
* @codexApi {"parent":"pithy.codex.extraction.readme-sync","name":"syncAllReadmes","stability":"stable","signature":"(extractionResult: ExtendedExtractionResult, config?: ReadmeSyncConfig) => Promise<ReadmeSyncResult[]>"}
|
|
107
|
+
*/
|
|
108
|
+
export declare function syncAllReadmes(extractionResult: ExtendedExtractionResult, config?: ReadmeSyncConfig): Promise<ReadmeSyncResult[]>;
|