@pithyjs/codex 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +4401 -0
- package/dist/annotations.d.ts +106 -0
- package/dist/annotations.js +306 -0
- package/dist/apply.d.ts +2 -0
- package/dist/apply.js +80 -0
- package/dist/changed-scope.d.ts +23 -0
- package/dist/changed-scope.js +31 -0
- package/dist/check.d.ts +2 -0
- package/dist/check.js +117 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +67 -0
- package/dist/env.d.ts +5 -0
- package/dist/env.js +35 -0
- package/dist/extract-cli.d.ts +23 -0
- package/dist/extract-cli.js +192 -0
- package/dist/extraction/breaking-changes.d.ts +66 -0
- package/dist/extraction/breaking-changes.js +352 -0
- package/dist/extraction/example-extractor.d.ts +55 -0
- package/dist/extraction/example-extractor.js +272 -0
- package/dist/extraction/index.d.ts +27 -0
- package/dist/extraction/index.js +31 -0
- package/dist/extraction/jsdoc-parser.d.ts +43 -0
- package/dist/extraction/jsdoc-parser.js +274 -0
- package/dist/extraction/pipeline.d.ts +54 -0
- package/dist/extraction/pipeline.js +526 -0
- package/dist/extraction/readme-sync.d.ts +108 -0
- package/dist/extraction/readme-sync.js +592 -0
- package/dist/extraction/snapshot-store.d.ts +40 -0
- package/dist/extraction/snapshot-store.js +153 -0
- package/dist/extraction/source-linker.d.ts +80 -0
- package/dist/extraction/source-linker.js +316 -0
- package/dist/extraction/test-example-extractor.d.ts +69 -0
- package/dist/extraction/test-example-extractor.js +400 -0
- package/dist/extraction/test-pattern-extractor.d.ts +68 -0
- package/dist/extraction/test-pattern-extractor.js +261 -0
- package/dist/extraction/testing-pyramid.d.ts +44 -0
- package/dist/extraction/testing-pyramid.js +163 -0
- package/dist/extraction/type-extractor.d.ts +34 -0
- package/dist/extraction/type-extractor.js +494 -0
- package/dist/extraction/types.d.ts +401 -0
- package/dist/extraction/types.js +34 -0
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +107 -0
- package/dist/llm.d.ts +8 -0
- package/dist/llm.js +75 -0
- package/dist/readme-sync-cli.d.ts +20 -0
- package/dist/readme-sync-cli.js +167 -0
- package/dist/review.d.ts +2 -0
- package/dist/review.js +93 -0
- package/dist/scan.d.ts +29 -0
- package/dist/scan.js +221 -0
- package/dist/schema.d.ts +169 -0
- package/dist/schema.js +70 -0
- package/dist/snapshot-cli.d.ts +45 -0
- package/dist/snapshot-cli.js +217 -0
- package/dist/sync-cli.d.ts +22 -0
- package/dist/sync-cli.js +154 -0
- package/dist/sync-pipeline.d.ts +58 -0
- package/dist/sync-pipeline.js +104 -0
- package/dist/sync.d.ts +2 -0
- package/dist/sync.js +318 -0
- package/dist/validate-cli.d.ts +20 -0
- package/dist/validate-cli.js +144 -0
- package/dist/validate.d.ts +76 -0
- package/dist/validate.js +183 -0
- package/dist/watch-cli.d.ts +21 -0
- package/dist/watch-cli.js +220 -0
- package/package.json +62 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.snapshot-store",
|
|
5
|
+
* "title": "API Snapshot Store",
|
|
6
|
+
* "category": "feature"
|
|
7
|
+
* }
|
|
8
|
+
*
|
|
9
|
+
* Persists and retrieves API snapshots per version in `codex.versions/`.
|
|
10
|
+
* Enables historical API tracking and versioned breaking-change detection.
|
|
11
|
+
*/
|
|
12
|
+
import { readFile, writeFile, readdir, mkdir } from 'node:fs/promises';
|
|
13
|
+
import { join, extname, basename } from 'node:path';
|
|
14
|
+
/**
|
|
15
|
+
* Validate that a version string is safe for use in filenames.
|
|
16
|
+
* Rejects strings containing path separators or traversal patterns.
|
|
17
|
+
*/
|
|
18
|
+
function validateVersionString(version) {
|
|
19
|
+
if (!/^[a-zA-Z0-9._-]+$/.test(version)) {
|
|
20
|
+
throw new Error(`Invalid version string "${version}": must contain only alphanumeric characters, dots, hyphens, and underscores`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Get the file path for a snapshot of a given version.
|
|
25
|
+
*
|
|
26
|
+
* @codexApi {"parent":"pithy.codex.snapshot-store","name":"getSnapshotPath","stability":"stable","signature":"(dir: string, version: string) => string"}
|
|
27
|
+
*/
|
|
28
|
+
export function getSnapshotPath(dir, version) {
|
|
29
|
+
validateVersionString(version);
|
|
30
|
+
return join(dir, `${version}.json`);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Save an API snapshot to disk as a JSON file named `{version}.json`.
|
|
34
|
+
* Creates the target directory if it doesn't exist.
|
|
35
|
+
*
|
|
36
|
+
* @codexApi {"parent":"pithy.codex.snapshot-store","name":"saveSnapshot","stability":"stable","signature":"(snapshot: ApiSnapshot, dir: string) => Promise<void>"}
|
|
37
|
+
*/
|
|
38
|
+
export async function saveSnapshot(snapshot, dir) {
|
|
39
|
+
await mkdir(dir, { recursive: true });
|
|
40
|
+
const filePath = getSnapshotPath(dir, snapshot.version);
|
|
41
|
+
const content = JSON.stringify(snapshot, null, 2) + '\n';
|
|
42
|
+
await writeFile(filePath, content, 'utf8');
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Load an API snapshot from disk by version.
|
|
46
|
+
* Returns `null` if the snapshot file does not exist.
|
|
47
|
+
* Throws if the file exists but cannot be parsed.
|
|
48
|
+
*
|
|
49
|
+
* @codexApi {"parent":"pithy.codex.snapshot-store","name":"loadSnapshot","stability":"stable","signature":"(version: string, dir: string) => Promise<ApiSnapshot | null>"}
|
|
50
|
+
*/
|
|
51
|
+
export async function loadSnapshot(version, dir) {
|
|
52
|
+
const filePath = getSnapshotPath(dir, version);
|
|
53
|
+
let content;
|
|
54
|
+
try {
|
|
55
|
+
content = await readFile(filePath, 'utf8');
|
|
56
|
+
}
|
|
57
|
+
catch (err) {
|
|
58
|
+
if (err instanceof Error &&
|
|
59
|
+
'code' in err &&
|
|
60
|
+
err.code === 'ENOENT') {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
throw err;
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(content);
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
throw new Error(`Failed to parse snapshot for version "${version}" at ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* List all stored snapshot versions, sorted by semver (with lexicographic fallback).
|
|
74
|
+
* Returns an empty array if the directory does not exist.
|
|
75
|
+
*
|
|
76
|
+
* @codexApi {"parent":"pithy.codex.snapshot-store","name":"listSnapshots","stability":"stable","signature":"(dir: string) => Promise<string[]>"}
|
|
77
|
+
*/
|
|
78
|
+
export async function listSnapshots(dir) {
|
|
79
|
+
try {
|
|
80
|
+
const files = await readdir(dir);
|
|
81
|
+
return files
|
|
82
|
+
.filter(f => extname(f) === '.json')
|
|
83
|
+
.map(f => basename(f, '.json'))
|
|
84
|
+
.sort(compareSemver);
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
if (err instanceof Error &&
|
|
88
|
+
'code' in err &&
|
|
89
|
+
err.code === 'ENOENT') {
|
|
90
|
+
return [];
|
|
91
|
+
}
|
|
92
|
+
throw err;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Compare two version strings by semver numeric parts, falling back to
|
|
97
|
+
* lexicographic comparison for non-numeric segments (e.g. prerelease tags).
|
|
98
|
+
*/
|
|
99
|
+
function compareSemver(a, b) {
|
|
100
|
+
// Split into numeric core (e.g. ["1","0","0"]) and prerelease parts
|
|
101
|
+
const [coreA, preA] = splitSemver(a);
|
|
102
|
+
const [coreB, preB] = splitSemver(b);
|
|
103
|
+
// Compare numeric core segments first
|
|
104
|
+
const coreLen = Math.max(coreA.length, coreB.length);
|
|
105
|
+
for (let i = 0; i < coreLen; i++) {
|
|
106
|
+
const na = Number(coreA[i] ?? '0');
|
|
107
|
+
const nb = Number(coreB[i] ?? '0');
|
|
108
|
+
if (na !== nb)
|
|
109
|
+
return na - nb;
|
|
110
|
+
}
|
|
111
|
+
// Cores are equal — apply prerelease precedence:
|
|
112
|
+
// A version WITHOUT prerelease has higher precedence than one WITH prerelease
|
|
113
|
+
if (preA.length === 0 && preB.length > 0)
|
|
114
|
+
return 1;
|
|
115
|
+
if (preA.length > 0 && preB.length === 0)
|
|
116
|
+
return -1;
|
|
117
|
+
// Both have prerelease — compare segments
|
|
118
|
+
const preLen = Math.max(preA.length, preB.length);
|
|
119
|
+
for (let i = 0; i < preLen; i++) {
|
|
120
|
+
const sa = preA[i] ?? '';
|
|
121
|
+
const sb = preB[i] ?? '';
|
|
122
|
+
if (sa === '' && sb !== '')
|
|
123
|
+
return -1;
|
|
124
|
+
if (sa !== '' && sb === '')
|
|
125
|
+
return 1;
|
|
126
|
+
const na = Number(sa);
|
|
127
|
+
const nb = Number(sb);
|
|
128
|
+
if (!Number.isNaN(na) && !Number.isNaN(nb)) {
|
|
129
|
+
if (na !== nb)
|
|
130
|
+
return na - nb;
|
|
131
|
+
}
|
|
132
|
+
else {
|
|
133
|
+
const cmp = sa.localeCompare(sb);
|
|
134
|
+
if (cmp !== 0)
|
|
135
|
+
return cmp;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* Splits a version string into core numeric parts and prerelease parts.
|
|
142
|
+
* "1.2.3-beta.1" → [["1","2","3"], ["beta","1"]]
|
|
143
|
+
* "1.2.3" → [["1","2","3"], []]
|
|
144
|
+
*/
|
|
145
|
+
function splitSemver(version) {
|
|
146
|
+
const dashIndex = version.indexOf('-');
|
|
147
|
+
if (dashIndex === -1) {
|
|
148
|
+
return [version.split('.'), []];
|
|
149
|
+
}
|
|
150
|
+
const core = version.slice(0, dashIndex).split('.');
|
|
151
|
+
const pre = version.slice(dashIndex + 1).split('.');
|
|
152
|
+
return [core, pre];
|
|
153
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.source-linker",
|
|
5
|
+
* "title": "Source Linker",
|
|
6
|
+
* "category": "compiler"
|
|
7
|
+
* }
|
|
8
|
+
*/
|
|
9
|
+
import type { SourceLocation, ReadmeSection } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Result of finding a declaration in source code
|
|
12
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"DeclarationLocation","stability":"stable","signature":"interface DeclarationLocation { name: string; type: string; location: SourceLocation; signature?: string; exported: boolean }"}
|
|
13
|
+
*/
|
|
14
|
+
export interface DeclarationLocation {
|
|
15
|
+
name: string;
|
|
16
|
+
type: 'function' | 'class' | 'interface' | 'type' | 'const';
|
|
17
|
+
location: SourceLocation;
|
|
18
|
+
signature?: string;
|
|
19
|
+
exported: boolean;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Finds the source location of a declaration by name
|
|
23
|
+
* @param content - Source file content
|
|
24
|
+
* @param filePath - Path to the source file
|
|
25
|
+
* @param declarationName - Name of the declaration to find
|
|
26
|
+
* @returns Source location or undefined if not found
|
|
27
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"findDeclarationLocation","stability":"stable","signature":"(content: string, filePath: string, declarationName: string) => DeclarationLocation | undefined"}
|
|
28
|
+
*/
|
|
29
|
+
export declare function findDeclarationLocation(content: string, filePath: string, declarationName: string): DeclarationLocation | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* Extracts all declarations from a source file
|
|
32
|
+
* @param content - Source file content
|
|
33
|
+
* @param filePath - Path to the source file
|
|
34
|
+
* @returns Array of declaration locations
|
|
35
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"extractAllDeclarations","stability":"stable","signature":"(content: string, filePath: string) => DeclarationLocation[]"}
|
|
36
|
+
*/
|
|
37
|
+
export declare function extractAllDeclarations(content: string, filePath: string): DeclarationLocation[];
|
|
38
|
+
/**
|
|
39
|
+
* Parses README content into sections
|
|
40
|
+
* @param content - README file content
|
|
41
|
+
* @param filePath - Path to the README file
|
|
42
|
+
* @returns Array of parsed sections
|
|
43
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"parseReadmeSections","stability":"stable","signature":"(content: string, filePath: string) => ReadmeSection[]"}
|
|
44
|
+
*/
|
|
45
|
+
export declare function parseReadmeSections(content: string, filePath: string): ReadmeSection[];
|
|
46
|
+
/**
|
|
47
|
+
* Links README sections to codex entry IDs based on heading matching
|
|
48
|
+
* @param sections - Parsed README sections
|
|
49
|
+
* @param entryIds - List of codex entry IDs
|
|
50
|
+
* @returns Sections with linked entry IDs
|
|
51
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"linkReadmeToEntries","stability":"stable","signature":"(sections: ReadmeSection[], entryIds: string[]) => ReadmeSection[]"}
|
|
52
|
+
*/
|
|
53
|
+
export declare function linkReadmeToEntries(sections: ReadmeSection[], entryIds: string[]): ReadmeSection[];
|
|
54
|
+
/**
|
|
55
|
+
* Generates source links for display (GitHub-style URLs)
|
|
56
|
+
* @param location - Source location
|
|
57
|
+
* @param repoUrl - Base repository URL
|
|
58
|
+
* @param branch - Branch name (default: main)
|
|
59
|
+
* @returns URL string for the source location
|
|
60
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"generateSourceLink","stability":"stable","signature":"(location: SourceLocation, repoUrl: string, branch?: string) => string"}
|
|
61
|
+
*/
|
|
62
|
+
export declare function generateSourceLink(location: SourceLocation, repoUrl: string, branch?: string): string;
|
|
63
|
+
/**
|
|
64
|
+
* Creates a relative path from project root
|
|
65
|
+
* @param absolutePath - Absolute file path
|
|
66
|
+
* @param projectRoot - Project root path
|
|
67
|
+
* @returns Relative path
|
|
68
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"toRelativePath","stability":"stable","signature":"(absolutePath: string, projectRoot: string) => string"}
|
|
69
|
+
*/
|
|
70
|
+
export declare function toRelativePath(absolutePath: string, projectRoot: string): string;
|
|
71
|
+
/**
|
|
72
|
+
* Enriches source locations with additional context
|
|
73
|
+
* @param location - Basic source location
|
|
74
|
+
* @param content - File content
|
|
75
|
+
* @returns Enriched location with context
|
|
76
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"enrichSourceLocation","stability":"stable","signature":"(location: SourceLocation, content: string) => SourceLocation & { context?: string }"}
|
|
77
|
+
*/
|
|
78
|
+
export declare function enrichSourceLocation(location: SourceLocation, content: string): SourceLocation & {
|
|
79
|
+
context?: string;
|
|
80
|
+
};
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.source-linker",
|
|
5
|
+
* "title": "Source Linker",
|
|
6
|
+
* "category": "compiler"
|
|
7
|
+
* }
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Patterns for declaration extraction
|
|
11
|
+
*
|
|
12
|
+
* Note: These patterns are designed to reduce ReDoS risk by:
|
|
13
|
+
* - Bounding matches and avoiding unbounded wildcards where practical
|
|
14
|
+
* - Avoiding nested quantifiers with overlapping character classes
|
|
15
|
+
* - Using (?:=>|[^=])+ to match type annotations including arrow types while stopping at assignment =
|
|
16
|
+
*/
|
|
17
|
+
const FUNCTION_DECLARATION_PATTERN = /(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*(?:<[^>]*>)?\s*\([^)]*\)/g;
|
|
18
|
+
// Pattern that matches const/let/var arrow functions with complex type annotations
|
|
19
|
+
// Uses (?:=>|[^=])+ to allow arrow types like (a: T) => U and object types like {a: string}
|
|
20
|
+
// while stopping at the assignment = (since standalone = is not matched, only =>)
|
|
21
|
+
const CONST_FUNCTION_PATTERN = /(?:export\s+)?(?:const|let|var)\s+(\w+)\s*(?::\s*(?:=>|[^=])+)?\s*=\s*(?:async\s+)?(?:\([^)]*\)|\w+)\s*=>/g;
|
|
22
|
+
const CLASS_DECLARATION_PATTERN = /(?:export\s+)?(?:abstract\s+)?class\s+(\w+)(?:\s+extends\s+\w+)?(?:\s+implements\s+[\w,\s]+)?/g;
|
|
23
|
+
const INTERFACE_PATTERN = /(?:export\s+)?interface\s+(\w+)(?:\s+extends\s+[\w,\s]+)?/g;
|
|
24
|
+
const TYPE_ALIAS_PATTERN = /(?:export\s+)?type\s+(\w+)\s*(?:<[^>]*>)?\s*=/g;
|
|
25
|
+
/**
|
|
26
|
+
* Finds the source location of a declaration by name
|
|
27
|
+
* @param content - Source file content
|
|
28
|
+
* @param filePath - Path to the source file
|
|
29
|
+
* @param declarationName - Name of the declaration to find
|
|
30
|
+
* @returns Source location or undefined if not found
|
|
31
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"findDeclarationLocation","stability":"stable","signature":"(content: string, filePath: string, declarationName: string) => DeclarationLocation | undefined"}
|
|
32
|
+
*/
|
|
33
|
+
export function findDeclarationLocation(content, filePath, declarationName) {
|
|
34
|
+
const declarations = extractAllDeclarations(content, filePath);
|
|
35
|
+
return declarations.find(d => d.name === declarationName);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Extracts all declarations from a source file
|
|
39
|
+
* @param content - Source file content
|
|
40
|
+
* @param filePath - Path to the source file
|
|
41
|
+
* @returns Array of declaration locations
|
|
42
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"extractAllDeclarations","stability":"stable","signature":"(content: string, filePath: string) => DeclarationLocation[]"}
|
|
43
|
+
*/
|
|
44
|
+
export function extractAllDeclarations(content, filePath) {
|
|
45
|
+
const declarations = [];
|
|
46
|
+
// Extract functions
|
|
47
|
+
FUNCTION_DECLARATION_PATTERN.lastIndex = 0;
|
|
48
|
+
let match;
|
|
49
|
+
while ((match = FUNCTION_DECLARATION_PATTERN.exec(content)) !== null) {
|
|
50
|
+
const lineNumber = getLineNumber(content, match.index);
|
|
51
|
+
const isExported = content
|
|
52
|
+
.slice(Math.max(0, match.index - 10), match.index)
|
|
53
|
+
.includes('export');
|
|
54
|
+
declarations.push({
|
|
55
|
+
name: match[1],
|
|
56
|
+
type: 'function',
|
|
57
|
+
location: { file: filePath, line: lineNumber },
|
|
58
|
+
signature: match[0],
|
|
59
|
+
exported: isExported,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
// Extract const/arrow functions
|
|
63
|
+
CONST_FUNCTION_PATTERN.lastIndex = 0;
|
|
64
|
+
while ((match = CONST_FUNCTION_PATTERN.exec(content)) !== null) {
|
|
65
|
+
const lineNumber = getLineNumber(content, match.index);
|
|
66
|
+
const isExported = content
|
|
67
|
+
.slice(Math.max(0, match.index - 10), match.index)
|
|
68
|
+
.includes('export');
|
|
69
|
+
declarations.push({
|
|
70
|
+
name: match[1],
|
|
71
|
+
type: 'const',
|
|
72
|
+
location: { file: filePath, line: lineNumber },
|
|
73
|
+
signature: match[0],
|
|
74
|
+
exported: isExported,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
// Extract classes
|
|
78
|
+
CLASS_DECLARATION_PATTERN.lastIndex = 0;
|
|
79
|
+
while ((match = CLASS_DECLARATION_PATTERN.exec(content)) !== null) {
|
|
80
|
+
const lineNumber = getLineNumber(content, match.index);
|
|
81
|
+
const isExported = content
|
|
82
|
+
.slice(Math.max(0, match.index - 10), match.index)
|
|
83
|
+
.includes('export');
|
|
84
|
+
declarations.push({
|
|
85
|
+
name: match[1],
|
|
86
|
+
type: 'class',
|
|
87
|
+
location: { file: filePath, line: lineNumber },
|
|
88
|
+
signature: match[0],
|
|
89
|
+
exported: isExported,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
// Extract interfaces
|
|
93
|
+
INTERFACE_PATTERN.lastIndex = 0;
|
|
94
|
+
while ((match = INTERFACE_PATTERN.exec(content)) !== null) {
|
|
95
|
+
const lineNumber = getLineNumber(content, match.index);
|
|
96
|
+
const isExported = content
|
|
97
|
+
.slice(Math.max(0, match.index - 10), match.index)
|
|
98
|
+
.includes('export');
|
|
99
|
+
declarations.push({
|
|
100
|
+
name: match[1],
|
|
101
|
+
type: 'interface',
|
|
102
|
+
location: { file: filePath, line: lineNumber },
|
|
103
|
+
signature: match[0],
|
|
104
|
+
exported: isExported,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
// Extract type aliases
|
|
108
|
+
TYPE_ALIAS_PATTERN.lastIndex = 0;
|
|
109
|
+
while ((match = TYPE_ALIAS_PATTERN.exec(content)) !== null) {
|
|
110
|
+
const lineNumber = getLineNumber(content, match.index);
|
|
111
|
+
const isExported = content
|
|
112
|
+
.slice(Math.max(0, match.index - 10), match.index)
|
|
113
|
+
.includes('export');
|
|
114
|
+
declarations.push({
|
|
115
|
+
name: match[1],
|
|
116
|
+
type: 'type',
|
|
117
|
+
location: { file: filePath, line: lineNumber },
|
|
118
|
+
signature: match[0],
|
|
119
|
+
exported: isExported,
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return declarations;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Gets the line number for a character index
|
|
126
|
+
*/
|
|
127
|
+
function getLineNumber(content, index) {
|
|
128
|
+
return content.slice(0, index).split('\n').length;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Parses README content into sections
|
|
132
|
+
* @param content - README file content
|
|
133
|
+
* @param filePath - Path to the README file
|
|
134
|
+
* @returns Array of parsed sections
|
|
135
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"parseReadmeSections","stability":"stable","signature":"(content: string, filePath: string) => ReadmeSection[]"}
|
|
136
|
+
*/
|
|
137
|
+
export function parseReadmeSections(content, filePath) {
|
|
138
|
+
const sections = [];
|
|
139
|
+
const lines = content.split('\n');
|
|
140
|
+
let currentSection = null;
|
|
141
|
+
let contentBuffer = [];
|
|
142
|
+
let lineNumber = 0;
|
|
143
|
+
let sectionContentStart = 0;
|
|
144
|
+
let fenceChar = null;
|
|
145
|
+
let fenceLen = 0;
|
|
146
|
+
for (const line of lines) {
|
|
147
|
+
lineNumber++;
|
|
148
|
+
// Track fenced code blocks (``` or ~~~, 3+ chars) so we skip
|
|
149
|
+
// heading-like lines inside them (e.g. "## More" in a template literal).
|
|
150
|
+
// CommonMark allows up to 3 leading spaces before a fence.
|
|
151
|
+
// Closing fences must have only trailing whitespace (no info string).
|
|
152
|
+
const fenceMatch = line.match(/^\s{0,3}(`{3,}|~{3,})(.*)/);
|
|
153
|
+
if (fenceMatch) {
|
|
154
|
+
const ch = fenceMatch[1][0];
|
|
155
|
+
const len = fenceMatch[1].length;
|
|
156
|
+
const trailing = fenceMatch[2];
|
|
157
|
+
if (!fenceChar) {
|
|
158
|
+
// Opening fence (info string allowed)
|
|
159
|
+
fenceChar = ch;
|
|
160
|
+
fenceLen = len;
|
|
161
|
+
}
|
|
162
|
+
else if (ch === fenceChar &&
|
|
163
|
+
len >= fenceLen &&
|
|
164
|
+
/^\s*$/.test(trailing)) {
|
|
165
|
+
// Closing fence (same char, at least as many, no info string)
|
|
166
|
+
fenceChar = null;
|
|
167
|
+
fenceLen = 0;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// Check for heading (only outside code fences)
|
|
171
|
+
const headingMatch = !fenceChar && line.match(/^(#{1,6})\s+(.+)/);
|
|
172
|
+
if (headingMatch) {
|
|
173
|
+
// Save previous section
|
|
174
|
+
if (currentSection) {
|
|
175
|
+
currentSection.content = contentBuffer.join('\n').trim();
|
|
176
|
+
currentSection.codeBlocks = extractCodeBlocks(currentSection.content, {
|
|
177
|
+
file: filePath,
|
|
178
|
+
line: sectionContentStart,
|
|
179
|
+
});
|
|
180
|
+
sections.push(currentSection);
|
|
181
|
+
}
|
|
182
|
+
// Start new section; content begins on the line after the heading
|
|
183
|
+
currentSection = {
|
|
184
|
+
heading: headingMatch[2],
|
|
185
|
+
level: headingMatch[1].length,
|
|
186
|
+
content: '',
|
|
187
|
+
codeBlocks: [],
|
|
188
|
+
};
|
|
189
|
+
contentBuffer = [];
|
|
190
|
+
sectionContentStart = lineNumber + 1;
|
|
191
|
+
}
|
|
192
|
+
else if (currentSection) {
|
|
193
|
+
contentBuffer.push(line);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Save final section
|
|
197
|
+
if (currentSection) {
|
|
198
|
+
currentSection.content = contentBuffer.join('\n').trim();
|
|
199
|
+
currentSection.codeBlocks = extractCodeBlocks(currentSection.content, {
|
|
200
|
+
file: filePath,
|
|
201
|
+
line: sectionContentStart,
|
|
202
|
+
});
|
|
203
|
+
sections.push(currentSection);
|
|
204
|
+
}
|
|
205
|
+
return sections;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Extracts code blocks from markdown content
|
|
209
|
+
*/
|
|
210
|
+
function extractCodeBlocks(content, baseLocation) {
|
|
211
|
+
const examples = [];
|
|
212
|
+
// Anchored to line start to prevent matching inline sequences like foo```.
|
|
213
|
+
// Backreference \1 ensures closing fence uses the same char as opening.
|
|
214
|
+
// [^\S\n] = horizontal whitespace only (prevents consuming newlines between blocks).
|
|
215
|
+
// Allows up to 3 leading spaces per CommonMark.
|
|
216
|
+
const codeBlockPattern = /(?:^|\n)[^\S\n]{0,3}(`{3,}|~{3,})(\w*)[^\S\n]*\n([\s\S]*?)\n[^\S\n]{0,3}\1[^\S\n]*(?:\n|$)/g;
|
|
217
|
+
let match;
|
|
218
|
+
while ((match = codeBlockPattern.exec(content)) !== null) {
|
|
219
|
+
const language = match[2] || 'text';
|
|
220
|
+
const code = match[3];
|
|
221
|
+
const textBefore = content.slice(0, match.index);
|
|
222
|
+
const relativeLine = textBefore.split('\n').length;
|
|
223
|
+
examples.push({
|
|
224
|
+
code: code.trim(),
|
|
225
|
+
language,
|
|
226
|
+
runnable: ['typescript', 'ts', 'javascript', 'js'].includes(language.toLowerCase()),
|
|
227
|
+
isDoctest: false,
|
|
228
|
+
location: {
|
|
229
|
+
file: baseLocation.file,
|
|
230
|
+
line: baseLocation.line + relativeLine - 1,
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
return examples;
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Links README sections to codex entry IDs based on heading matching
|
|
238
|
+
* @param sections - Parsed README sections
|
|
239
|
+
* @param entryIds - List of codex entry IDs
|
|
240
|
+
* @returns Sections with linked entry IDs
|
|
241
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"linkReadmeToEntries","stability":"stable","signature":"(sections: ReadmeSection[], entryIds: string[]) => ReadmeSection[]"}
|
|
242
|
+
*/
|
|
243
|
+
export function linkReadmeToEntries(sections, entryIds) {
|
|
244
|
+
return sections.map(section => {
|
|
245
|
+
// Try to match section heading to an entry ID.
|
|
246
|
+
// We intentionally only match by heading — content-based matching
|
|
247
|
+
// is too loose for large sections (e.g. "## Examples" spanning thousands
|
|
248
|
+
// of lines) and causes false-positive attribution. Explicit linking
|
|
249
|
+
// should use @codex:auto markers instead.
|
|
250
|
+
const normalizedHeading = section.heading
|
|
251
|
+
.toLowerCase()
|
|
252
|
+
.replace(/\s+/g, '-');
|
|
253
|
+
for (const entryId of entryIds) {
|
|
254
|
+
const entryParts = entryId.toLowerCase().split('.');
|
|
255
|
+
const lastPart = entryParts[entryParts.length - 1];
|
|
256
|
+
// Check if heading matches entry name
|
|
257
|
+
if (normalizedHeading.includes(lastPart) ||
|
|
258
|
+
lastPart.includes(normalizedHeading)) {
|
|
259
|
+
return { ...section, linkedEntryId: entryId };
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
return section;
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Generates source links for display (GitHub-style URLs)
|
|
267
|
+
* @param location - Source location
|
|
268
|
+
* @param repoUrl - Base repository URL
|
|
269
|
+
* @param branch - Branch name (default: main)
|
|
270
|
+
* @returns URL string for the source location
|
|
271
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"generateSourceLink","stability":"stable","signature":"(location: SourceLocation, repoUrl: string, branch?: string) => string"}
|
|
272
|
+
*/
|
|
273
|
+
export function generateSourceLink(location, repoUrl, branch = 'main') {
|
|
274
|
+
// Normalize the file path (remove leading slashes, project root)
|
|
275
|
+
const normalizedPath = location.file.replace(/^.*?(packages|apps|src)/, '$1');
|
|
276
|
+
const lineFragment = location.endLine
|
|
277
|
+
? `#L${location.line}-L${location.endLine}`
|
|
278
|
+
: `#L${location.line}`;
|
|
279
|
+
return `${repoUrl}/blob/${branch}/${normalizedPath}${lineFragment}`;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Creates a relative path from project root
|
|
283
|
+
* @param absolutePath - Absolute file path
|
|
284
|
+
* @param projectRoot - Project root path
|
|
285
|
+
* @returns Relative path
|
|
286
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"toRelativePath","stability":"stable","signature":"(absolutePath: string, projectRoot: string) => string"}
|
|
287
|
+
*/
|
|
288
|
+
export function toRelativePath(absolutePath, projectRoot) {
|
|
289
|
+
if (absolutePath.startsWith(projectRoot)) {
|
|
290
|
+
return absolutePath.slice(projectRoot.length).replace(/^\//, '');
|
|
291
|
+
}
|
|
292
|
+
return absolutePath;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Enriches source locations with additional context
|
|
296
|
+
* @param location - Basic source location
|
|
297
|
+
* @param content - File content
|
|
298
|
+
* @returns Enriched location with context
|
|
299
|
+
* @codexApi {"parent":"pithy.codex.extraction.source-linker","name":"enrichSourceLocation","stability":"stable","signature":"(location: SourceLocation, content: string) => SourceLocation & { context?: string }"}
|
|
300
|
+
*/
|
|
301
|
+
export function enrichSourceLocation(location, content) {
|
|
302
|
+
const lines = content.split('\n');
|
|
303
|
+
const lineIndex = location.line - 1;
|
|
304
|
+
if (lineIndex >= 0 && lineIndex < lines.length) {
|
|
305
|
+
// Get 2 lines of context before and after
|
|
306
|
+
const start = Math.max(0, lineIndex - 2);
|
|
307
|
+
const end = Math.min(lines.length, lineIndex + 3);
|
|
308
|
+
const context = lines.slice(start, end).join('\n');
|
|
309
|
+
return {
|
|
310
|
+
...location,
|
|
311
|
+
endLine: location.endLine || location.line,
|
|
312
|
+
context,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
return location;
|
|
316
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.test-example-extractor",
|
|
5
|
+
* "title": "Test Example Extractor",
|
|
6
|
+
* "category": "compiler"
|
|
7
|
+
* }
|
|
8
|
+
*
|
|
9
|
+
* Extracts examples from test files marked with @codex:example
|
|
10
|
+
* and tracks test status from vitest/jest output.
|
|
11
|
+
*/
|
|
12
|
+
import type { TestExample, TestStatus } from './types.js';
|
|
13
|
+
/**
|
|
14
|
+
* @codexApi {"parent":"pithy.codex.extraction.test-example-extractor","name":"extractTestExamples","stability":"stable","signature":"(content: string, filePath: string) => TestExample[]"}
|
|
15
|
+
*
|
|
16
|
+
* Extracts examples marked with @codex:example from a test file
|
|
17
|
+
* @param content - Test file content
|
|
18
|
+
* @param filePath - Path to the test file
|
|
19
|
+
* @returns Array of extracted test examples
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```typescript
|
|
23
|
+
* // In a test file:
|
|
24
|
+
* // @codex:example pithy.signals.effect
|
|
25
|
+
* it("creates an effect that tracks dependencies", () => {
|
|
26
|
+
* const count = signal(0);
|
|
27
|
+
* let runCount = 0;
|
|
28
|
+
* effect(() => { count(); runCount++; });
|
|
29
|
+
* expect(runCount).toBe(1);
|
|
30
|
+
* });
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
export declare function extractTestExamples(content: string, filePath: string): TestExample[];
|
|
34
|
+
/**
|
|
35
|
+
* @codexApi {"parent":"pithy.codex.extraction.test-example-extractor","name":"parseVitestOutput","stability":"stable","signature":"(output: string) => TestStatus[]"}
|
|
36
|
+
*
|
|
37
|
+
* Parses vitest JSON output to extract test statuses
|
|
38
|
+
* @param output - JSON string from vitest --reporter=json
|
|
39
|
+
* @returns Array of test statuses
|
|
40
|
+
*/
|
|
41
|
+
export declare function parseVitestOutput(output: string): TestStatus[];
|
|
42
|
+
/**
|
|
43
|
+
* @codexApi {"parent":"pithy.codex.extraction.test-example-extractor","name":"matchExamplesToStatuses","stability":"stable","signature":"(examples: TestExample[], statuses: TestStatus[]) => TestExample[]"}
|
|
44
|
+
*
|
|
45
|
+
* Matches extracted examples to test statuses by test name
|
|
46
|
+
* @param examples - Extracted test examples
|
|
47
|
+
* @param statuses - Test statuses from vitest output
|
|
48
|
+
* @returns Examples with updated status information
|
|
49
|
+
*/
|
|
50
|
+
export declare function matchExamplesToStatuses(examples: TestExample[], statuses: TestStatus[]): TestExample[];
|
|
51
|
+
/**
|
|
52
|
+
* @codexApi {"parent":"pithy.codex.extraction.test-example-extractor","name":"generateExampleReport","stability":"stable","signature":"(examples: TestExample[]) => string"}
|
|
53
|
+
*
|
|
54
|
+
* Generates a markdown report of test examples with their statuses
|
|
55
|
+
* @param examples - Test examples to report
|
|
56
|
+
* @returns Markdown formatted report
|
|
57
|
+
*/
|
|
58
|
+
export declare function generateExampleReport(examples: TestExample[]): string;
|
|
59
|
+
/**
|
|
60
|
+
* @codexApi {"parent":"pithy.codex.extraction.test-example-extractor","name":"validateTestExamples","stability":"experimental","signature":"(examples: TestExample[]) => { valid: TestExample[]; invalid: TestExample[] }"}
|
|
61
|
+
*
|
|
62
|
+
* Validates that test examples are working by checking their status
|
|
63
|
+
* @param examples - Test examples to validate
|
|
64
|
+
* @returns Object with valid and invalid examples
|
|
65
|
+
*/
|
|
66
|
+
export declare function validateTestExamples(examples: TestExample[]): {
|
|
67
|
+
valid: TestExample[];
|
|
68
|
+
invalid: TestExample[];
|
|
69
|
+
};
|