@pithyjs/codex 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +4401 -0
- package/dist/annotations.d.ts +106 -0
- package/dist/annotations.js +306 -0
- package/dist/apply.d.ts +2 -0
- package/dist/apply.js +80 -0
- package/dist/changed-scope.d.ts +23 -0
- package/dist/changed-scope.js +31 -0
- package/dist/check.d.ts +2 -0
- package/dist/check.js +117 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +67 -0
- package/dist/env.d.ts +5 -0
- package/dist/env.js +35 -0
- package/dist/extract-cli.d.ts +23 -0
- package/dist/extract-cli.js +192 -0
- package/dist/extraction/breaking-changes.d.ts +66 -0
- package/dist/extraction/breaking-changes.js +352 -0
- package/dist/extraction/example-extractor.d.ts +55 -0
- package/dist/extraction/example-extractor.js +272 -0
- package/dist/extraction/index.d.ts +27 -0
- package/dist/extraction/index.js +31 -0
- package/dist/extraction/jsdoc-parser.d.ts +43 -0
- package/dist/extraction/jsdoc-parser.js +274 -0
- package/dist/extraction/pipeline.d.ts +54 -0
- package/dist/extraction/pipeline.js +526 -0
- package/dist/extraction/readme-sync.d.ts +108 -0
- package/dist/extraction/readme-sync.js +592 -0
- package/dist/extraction/snapshot-store.d.ts +40 -0
- package/dist/extraction/snapshot-store.js +153 -0
- package/dist/extraction/source-linker.d.ts +80 -0
- package/dist/extraction/source-linker.js +316 -0
- package/dist/extraction/test-example-extractor.d.ts +69 -0
- package/dist/extraction/test-example-extractor.js +400 -0
- package/dist/extraction/test-pattern-extractor.d.ts +68 -0
- package/dist/extraction/test-pattern-extractor.js +261 -0
- package/dist/extraction/testing-pyramid.d.ts +44 -0
- package/dist/extraction/testing-pyramid.js +163 -0
- package/dist/extraction/type-extractor.d.ts +34 -0
- package/dist/extraction/type-extractor.js +494 -0
- package/dist/extraction/types.d.ts +401 -0
- package/dist/extraction/types.js +34 -0
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +107 -0
- package/dist/llm.d.ts +8 -0
- package/dist/llm.js +75 -0
- package/dist/readme-sync-cli.d.ts +20 -0
- package/dist/readme-sync-cli.js +167 -0
- package/dist/review.d.ts +2 -0
- package/dist/review.js +93 -0
- package/dist/scan.d.ts +29 -0
- package/dist/scan.js +221 -0
- package/dist/schema.d.ts +169 -0
- package/dist/schema.js +70 -0
- package/dist/snapshot-cli.d.ts +45 -0
- package/dist/snapshot-cli.js +217 -0
- package/dist/sync-cli.d.ts +22 -0
- package/dist/sync-cli.js +154 -0
- package/dist/sync-pipeline.d.ts +58 -0
- package/dist/sync-pipeline.js +104 -0
- package/dist/sync.d.ts +2 -0
- package/dist/sync.js +318 -0
- package/dist/validate-cli.d.ts +20 -0
- package/dist/validate-cli.js +144 -0
- package/dist/validate.d.ts +76 -0
- package/dist/validate.js +183 -0
- package/dist/watch-cli.d.ts +21 -0
- package/dist/watch-cli.js +220 -0
- package/package.json +62 -0
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.breaking-changes",
|
|
5
|
+
* "title": "Breaking Change Detection",
|
|
6
|
+
* "category": "feature"
|
|
7
|
+
* }
|
|
8
|
+
*
|
|
9
|
+
* Detects breaking changes between API snapshots by comparing
|
|
10
|
+
* extracted API signatures, stability levels, and type definitions.
|
|
11
|
+
* Generates migration guides and changelogs for version transitions.
|
|
12
|
+
*/
|
|
13
|
+
import { getTypeSignature } from './type-extractor.js';
|
|
14
|
+
import { basename } from 'node:path';
|
|
15
|
+
// ── Helpers ──────────────────────────────────────────────────────
|
|
16
|
+
/** Normalize a signature string for comparison (collapse whitespace, trim). */
|
|
17
|
+
function normalizeSignature(sig) {
|
|
18
|
+
if (sig === undefined)
|
|
19
|
+
return undefined;
|
|
20
|
+
return sig.replace(/\s+/g, ' ').trim();
|
|
21
|
+
}
|
|
22
|
+
/** Serialize members for deep comparison (order-independent). */
|
|
23
|
+
function membersKey(members) {
|
|
24
|
+
if (!members || members.length === 0)
|
|
25
|
+
return '';
|
|
26
|
+
return members
|
|
27
|
+
.map(m => {
|
|
28
|
+
const sig = normalizeSignature(m.signature) ?? m.type;
|
|
29
|
+
const ro = m.readonly ? 'readonly ' : '';
|
|
30
|
+
const opt = m.optional ? '?' : '';
|
|
31
|
+
return `${m.name}|${ro}${opt}|${sig}`;
|
|
32
|
+
})
|
|
33
|
+
.sort()
|
|
34
|
+
.join(';');
|
|
35
|
+
}
|
|
36
|
+
// ── createApiSnapshot ────────────────────────────────────────────
|
|
37
|
+
/**
|
|
38
|
+
* Create an API snapshot from extraction results at a given version.
|
|
39
|
+
* Captures all enriched API entries across all components.
|
|
40
|
+
*
|
|
41
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"createApiSnapshot","stability":"stable","signature":"(result: ExtendedExtractionResult, version: string, timestamp?: string) => ApiSnapshot"}
|
|
42
|
+
*/
|
|
43
|
+
export function createApiSnapshot(result, version, timestamp) {
|
|
44
|
+
const entries = [];
|
|
45
|
+
for (const component of result.components) {
|
|
46
|
+
for (const api of component.apis) {
|
|
47
|
+
entries.push({
|
|
48
|
+
name: api.name,
|
|
49
|
+
parent: component.id,
|
|
50
|
+
qualifiedName: `${component.id}.${api.name}`,
|
|
51
|
+
signature: api.signature,
|
|
52
|
+
stability: api.stability,
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
version,
|
|
58
|
+
timestamp: timestamp ?? new Date().toISOString(),
|
|
59
|
+
entries,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
// ── snapshotFromTypeDefinitions ──────────────────────────────────
|
|
63
|
+
/**
|
|
64
|
+
* Create an API snapshot from extracted type definitions.
|
|
65
|
+
* Only includes exported types (public API surface).
|
|
66
|
+
*
|
|
67
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"snapshotFromTypeDefinitions","stability":"stable","signature":"(types: ExtractedTypeDefinition[], version: string, timestamp?: string) => ApiSnapshot"}
|
|
68
|
+
*/
|
|
69
|
+
export function snapshotFromTypeDefinitions(types, version, timestamp) {
|
|
70
|
+
const entries = [];
|
|
71
|
+
for (const type of types) {
|
|
72
|
+
if (!type.exported)
|
|
73
|
+
continue;
|
|
74
|
+
const stability = type.jsdoc?.deprecated
|
|
75
|
+
? 'deprecated'
|
|
76
|
+
: 'stable';
|
|
77
|
+
const parent = basename(type.location.file);
|
|
78
|
+
entries.push({
|
|
79
|
+
name: type.name,
|
|
80
|
+
parent,
|
|
81
|
+
qualifiedName: `${parent}.${type.name}`,
|
|
82
|
+
stability,
|
|
83
|
+
signature: getTypeSignature(type),
|
|
84
|
+
kind: type.kind,
|
|
85
|
+
members: type.members,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
version,
|
|
90
|
+
timestamp: timestamp ?? new Date().toISOString(),
|
|
91
|
+
entries,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
// ── diffSnapshots ────────────────────────────────────────────────
|
|
95
|
+
/**
|
|
96
|
+
* Compare two API snapshots and produce a structured diff.
|
|
97
|
+
* Matches entries by qualifiedName (parent.name).
|
|
98
|
+
*
|
|
99
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"diffSnapshots","stability":"stable","signature":"(before: ApiSnapshot, after: ApiSnapshot) => SnapshotDiff"}
|
|
100
|
+
*/
|
|
101
|
+
export function diffSnapshots(before, after) {
|
|
102
|
+
const beforeMap = new Map();
|
|
103
|
+
for (const entry of before.entries) {
|
|
104
|
+
beforeMap.set(entry.qualifiedName, entry);
|
|
105
|
+
}
|
|
106
|
+
const afterMap = new Map();
|
|
107
|
+
for (const entry of after.entries) {
|
|
108
|
+
afterMap.set(entry.qualifiedName, entry);
|
|
109
|
+
}
|
|
110
|
+
const added = [];
|
|
111
|
+
const removed = [];
|
|
112
|
+
const changed = [];
|
|
113
|
+
let unchanged = 0;
|
|
114
|
+
// Find removed and changed entries
|
|
115
|
+
for (const [qn, beforeEntry] of beforeMap) {
|
|
116
|
+
const afterEntry = afterMap.get(qn);
|
|
117
|
+
if (!afterEntry) {
|
|
118
|
+
removed.push({
|
|
119
|
+
name: beforeEntry.name,
|
|
120
|
+
parent: beforeEntry.parent,
|
|
121
|
+
qualifiedName: qn,
|
|
122
|
+
changeKind: 'removed',
|
|
123
|
+
before: beforeEntry,
|
|
124
|
+
});
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
// Check for changes — priority: signature > members > stability
|
|
128
|
+
// When multiple aspects change, the highest-priority changeKind is used.
|
|
129
|
+
// The full before/after entries are always preserved so consumers can
|
|
130
|
+
// detect secondary changes (e.g. stability shift alongside signature change).
|
|
131
|
+
const beforeSig = normalizeSignature(beforeEntry.signature);
|
|
132
|
+
const afterSig = normalizeSignature(afterEntry.signature);
|
|
133
|
+
if (beforeSig !== afterSig) {
|
|
134
|
+
changed.push({
|
|
135
|
+
name: beforeEntry.name,
|
|
136
|
+
parent: beforeEntry.parent,
|
|
137
|
+
qualifiedName: qn,
|
|
138
|
+
changeKind: 'signature-changed',
|
|
139
|
+
before: beforeEntry,
|
|
140
|
+
after: afterEntry,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
else if (membersKey(beforeEntry.members) !== membersKey(afterEntry.members)) {
|
|
144
|
+
changed.push({
|
|
145
|
+
name: beforeEntry.name,
|
|
146
|
+
parent: beforeEntry.parent,
|
|
147
|
+
qualifiedName: qn,
|
|
148
|
+
changeKind: 'members-changed',
|
|
149
|
+
before: beforeEntry,
|
|
150
|
+
after: afterEntry,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
else if (beforeEntry.stability !== afterEntry.stability) {
|
|
154
|
+
changed.push({
|
|
155
|
+
name: beforeEntry.name,
|
|
156
|
+
parent: beforeEntry.parent,
|
|
157
|
+
qualifiedName: qn,
|
|
158
|
+
changeKind: 'stability-changed',
|
|
159
|
+
before: beforeEntry,
|
|
160
|
+
after: afterEntry,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
unchanged++;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
// Find added entries
|
|
168
|
+
for (const [qn, afterEntry] of afterMap) {
|
|
169
|
+
if (!beforeMap.has(qn)) {
|
|
170
|
+
added.push({
|
|
171
|
+
name: afterEntry.name,
|
|
172
|
+
parent: afterEntry.parent,
|
|
173
|
+
qualifiedName: qn,
|
|
174
|
+
changeKind: 'added',
|
|
175
|
+
after: afterEntry,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return {
|
|
180
|
+
fromVersion: before.version,
|
|
181
|
+
toVersion: after.version,
|
|
182
|
+
added,
|
|
183
|
+
removed,
|
|
184
|
+
changed,
|
|
185
|
+
unchanged,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
// ── classifyChange ───────────────────────────────────────────────
|
|
189
|
+
/**
|
|
190
|
+
* Classify a single API change as major, minor, or patch
|
|
191
|
+
* according to semver conventions.
|
|
192
|
+
*
|
|
193
|
+
* - **major**: removal or signature/member change of a stable API
|
|
194
|
+
* - **minor**: addition, removal of experimental/deprecated, or changes to experimental APIs
|
|
195
|
+
* - **patch**: stability level changes
|
|
196
|
+
*
|
|
197
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"classifyChange","stability":"stable","signature":"(change: ApiChange) => ChangeClassification"}
|
|
198
|
+
*/
|
|
199
|
+
export function classifyChange(change) {
|
|
200
|
+
switch (change.changeKind) {
|
|
201
|
+
case 'added':
|
|
202
|
+
return 'minor';
|
|
203
|
+
case 'removed': {
|
|
204
|
+
const stability = change.before?.stability;
|
|
205
|
+
// Removing internal (non-public), experimental, or deprecated APIs is
|
|
206
|
+
// non-breaking — only stable public APIs count as breaking removals.
|
|
207
|
+
if (stability === 'internal' ||
|
|
208
|
+
stability === 'experimental' ||
|
|
209
|
+
stability === 'deprecated') {
|
|
210
|
+
return 'minor';
|
|
211
|
+
}
|
|
212
|
+
return 'major';
|
|
213
|
+
}
|
|
214
|
+
case 'signature-changed':
|
|
215
|
+
case 'members-changed': {
|
|
216
|
+
// Changing internal (non-public), experimental, or deprecated APIs is
|
|
217
|
+
// non-breaking — only stable public APIs count as breaking changes.
|
|
218
|
+
const stability = change.before?.stability;
|
|
219
|
+
if (stability === 'internal' ||
|
|
220
|
+
stability === 'experimental' ||
|
|
221
|
+
stability === 'deprecated') {
|
|
222
|
+
return 'minor';
|
|
223
|
+
}
|
|
224
|
+
return 'major';
|
|
225
|
+
}
|
|
226
|
+
case 'stability-changed':
|
|
227
|
+
return 'patch';
|
|
228
|
+
default: {
|
|
229
|
+
const _exhaustive = change.changeKind;
|
|
230
|
+
throw new Error(`Unknown change kind: ${_exhaustive}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
// ── detectBreakingChanges ────────────────────────────────────────
|
|
235
|
+
/**
|
|
236
|
+
* Filter a snapshot diff down to only breaking changes
|
|
237
|
+
* (those classified as "major").
|
|
238
|
+
*
|
|
239
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"detectBreakingChanges","stability":"stable","signature":"(diff: SnapshotDiff) => ApiChange[]"}
|
|
240
|
+
*/
|
|
241
|
+
export function detectBreakingChanges(diff) {
|
|
242
|
+
const allChanges = [...diff.removed, ...diff.changed];
|
|
243
|
+
return allChanges.filter(c => classifyChange(c) === 'major');
|
|
244
|
+
}
|
|
245
|
+
// ── generateMigrationGuide ───────────────────────────────────────
|
|
246
|
+
/**
|
|
247
|
+
* Generate a human-readable migration guide for a set of breaking changes.
|
|
248
|
+
* Groups changes by parent component and provides before/after comparisons.
|
|
249
|
+
*
|
|
250
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"generateMigrationGuide","stability":"stable","signature":"(changes: ApiChange[]) => string"}
|
|
251
|
+
*/
|
|
252
|
+
export function generateMigrationGuide(changes) {
|
|
253
|
+
if (changes.length === 0)
|
|
254
|
+
return '';
|
|
255
|
+
const lines = ['# Migration Guide', ''];
|
|
256
|
+
// Group changes by parent
|
|
257
|
+
const grouped = new Map();
|
|
258
|
+
for (const change of changes) {
|
|
259
|
+
const group = grouped.get(change.parent) ?? [];
|
|
260
|
+
group.push(change);
|
|
261
|
+
grouped.set(change.parent, group);
|
|
262
|
+
}
|
|
263
|
+
for (const [parent, groupChanges] of grouped) {
|
|
264
|
+
lines.push(`## ${parent}`, '');
|
|
265
|
+
for (const change of groupChanges) {
|
|
266
|
+
switch (change.changeKind) {
|
|
267
|
+
case 'removed':
|
|
268
|
+
lines.push(`### \`${change.name}\` — removed`, '', `The \`${change.name}\` API has been removed.`);
|
|
269
|
+
if (change.before?.signature) {
|
|
270
|
+
lines.push('', '**Previous signature:**', '```typescript', change.before.signature, '```');
|
|
271
|
+
}
|
|
272
|
+
lines.push('');
|
|
273
|
+
break;
|
|
274
|
+
case 'signature-changed':
|
|
275
|
+
lines.push(`### \`${change.name}\` — signature changed`, '', `The signature of \`${change.name}\` has changed.`, '', '**Before:**', '```typescript', change.before?.signature ?? '(unknown)', '```', '', '**After:**', '```typescript', change.after?.signature ?? '(unknown)', '```', '');
|
|
276
|
+
break;
|
|
277
|
+
case 'members-changed':
|
|
278
|
+
lines.push(`### \`${change.name}\` — members changed`, '', `The members of the \`${change.name}\` type have changed.`, '');
|
|
279
|
+
break;
|
|
280
|
+
case 'stability-changed':
|
|
281
|
+
lines.push(`### \`${change.name}\` — stability changed`, '', `Stability changed from \`${change.before?.stability}\` to \`${change.after?.stability}\`.`, '');
|
|
282
|
+
break;
|
|
283
|
+
// "added" is not typically in a migration guide, skip
|
|
284
|
+
default:
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return lines.join('\n');
|
|
290
|
+
}
|
|
291
|
+
// ── generateChangelog ────────────────────────────────────────────
|
|
292
|
+
/**
|
|
293
|
+
* Generate a changelog in Markdown from a snapshot diff.
|
|
294
|
+
* Sections: Breaking Changes, Added, Changed. Omits empty sections.
|
|
295
|
+
*
|
|
296
|
+
* @codexApi {"parent":"pithy.codex.breaking-changes","name":"generateChangelog","stability":"stable","signature":"(diff: SnapshotDiff) => string"}
|
|
297
|
+
*/
|
|
298
|
+
export function generateChangelog(diff) {
|
|
299
|
+
const breaking = detectBreakingChanges(diff);
|
|
300
|
+
const hasBreaking = breaking.length > 0;
|
|
301
|
+
const hasAdded = diff.added.length > 0;
|
|
302
|
+
const hasChanged = diff.changed.length > 0;
|
|
303
|
+
const hasRemoved = diff.removed.length > 0;
|
|
304
|
+
if (!hasBreaking && !hasAdded && !hasChanged && !hasRemoved) {
|
|
305
|
+
return '';
|
|
306
|
+
}
|
|
307
|
+
const lines = [`## ${diff.fromVersion} → ${diff.toVersion}`, ''];
|
|
308
|
+
if (hasBreaking) {
|
|
309
|
+
lines.push('### ⚠️ Breaking Changes', '');
|
|
310
|
+
for (const change of breaking) {
|
|
311
|
+
if (change.changeKind === 'removed') {
|
|
312
|
+
lines.push(`- **Removed** \`${change.qualifiedName}\``);
|
|
313
|
+
}
|
|
314
|
+
else if (change.changeKind === 'signature-changed') {
|
|
315
|
+
lines.push(`- **Changed** \`${change.qualifiedName}\` signature`);
|
|
316
|
+
}
|
|
317
|
+
else if (change.changeKind === 'members-changed') {
|
|
318
|
+
lines.push(`- **Changed** \`${change.qualifiedName}\` members`);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
lines.push('');
|
|
322
|
+
}
|
|
323
|
+
if (hasAdded) {
|
|
324
|
+
lines.push('### Added', '');
|
|
325
|
+
for (const change of diff.added) {
|
|
326
|
+
lines.push(`- \`${change.qualifiedName}\``);
|
|
327
|
+
}
|
|
328
|
+
lines.push('');
|
|
329
|
+
}
|
|
330
|
+
// Non-breaking changes (stability, non-breaking signature changes)
|
|
331
|
+
const nonBreakingChanges = diff.changed.filter(c => classifyChange(c) !== 'major');
|
|
332
|
+
const nonBreakingRemovals = diff.removed.filter(c => classifyChange(c) !== 'major');
|
|
333
|
+
if (nonBreakingChanges.length > 0 || nonBreakingRemovals.length > 0) {
|
|
334
|
+
lines.push('### Changed', '');
|
|
335
|
+
for (const change of nonBreakingChanges) {
|
|
336
|
+
if (change.changeKind === 'stability-changed') {
|
|
337
|
+
lines.push(`- \`${change.qualifiedName}\` stability: ${change.before?.stability} → ${change.after?.stability}`);
|
|
338
|
+
}
|
|
339
|
+
else if (change.changeKind === 'signature-changed') {
|
|
340
|
+
lines.push(`- \`${change.qualifiedName}\` signature changed (non-breaking)`);
|
|
341
|
+
}
|
|
342
|
+
else if (change.changeKind === 'members-changed') {
|
|
343
|
+
lines.push(`- \`${change.qualifiedName}\` members changed (non-breaking)`);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
for (const change of nonBreakingRemovals) {
|
|
347
|
+
lines.push(`- **Removed** \`${change.qualifiedName}\` (was ${change.before?.stability})`);
|
|
348
|
+
}
|
|
349
|
+
lines.push('');
|
|
350
|
+
}
|
|
351
|
+
return lines.join('\n');
|
|
352
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.example-extractor",
|
|
5
|
+
* "title": "Example Extractor",
|
|
6
|
+
* "category": "compiler"
|
|
7
|
+
* }
|
|
8
|
+
*/
|
|
9
|
+
import type { ExtractedExample, SourceLocation } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Extracts all code examples from a markdown/documentation string
|
|
12
|
+
* @param content - Content to extract examples from
|
|
13
|
+
* @param baseLocation - Base source location for the content
|
|
14
|
+
* @returns Array of extracted examples
|
|
15
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"extractExamples","stability":"stable","signature":"(content: string, baseLocation: SourceLocation) => ExtractedExample[]"}
|
|
16
|
+
*/
|
|
17
|
+
export declare function extractExamples(content: string, baseLocation: SourceLocation): ExtractedExample[];
|
|
18
|
+
/**
|
|
19
|
+
* Validates that a code example is syntactically valid
|
|
20
|
+
* @param code - Code to validate
|
|
21
|
+
* @param language - Language of the code
|
|
22
|
+
* @returns Validation result with any errors
|
|
23
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"validateExample","stability":"stable","signature":"(code: string, language: string) => { valid: boolean; errors: string[] }"}
|
|
24
|
+
*/
|
|
25
|
+
export declare function validateExample(code: string, _language: string): {
|
|
26
|
+
valid: boolean;
|
|
27
|
+
errors: string[];
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Wraps a doctest example in a test harness
|
|
31
|
+
* @param example - The extracted example
|
|
32
|
+
* @param entryId - ID of the codex entry this example belongs to
|
|
33
|
+
* @returns Test code ready to run with vitest
|
|
34
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"wrapDoctestInHarness","stability":"stable","signature":"(example: ExtractedExample, entryId: string) => string"}
|
|
35
|
+
*/
|
|
36
|
+
export declare function wrapDoctestInHarness(example: ExtractedExample, entryId: string): string;
|
|
37
|
+
/**
|
|
38
|
+
* Extracts examples from JSDoc @example tags in source code
|
|
39
|
+
* @param content - Source file content
|
|
40
|
+
* @param filePath - Path to the source file
|
|
41
|
+
* @returns Array of extracted examples with their context
|
|
42
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"extractExamplesFromSource","stability":"stable","signature":"(content: string, filePath: string) => { functionName: string; examples: ExtractedExample[] }[]"}
|
|
43
|
+
*/
|
|
44
|
+
export declare function extractExamplesFromSource(content: string, filePath: string): {
|
|
45
|
+
functionName: string;
|
|
46
|
+
examples: ExtractedExample[];
|
|
47
|
+
}[];
|
|
48
|
+
/**
|
|
49
|
+
* Generates a runnable example file from extracted examples
|
|
50
|
+
* @param examples - Examples to include
|
|
51
|
+
* @param imports - Required imports
|
|
52
|
+
* @returns Complete TypeScript file content
|
|
53
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"generateRunnableExampleFile","stability":"stable","signature":"(examples: ExtractedExample[], imports: string[]) => string"}
|
|
54
|
+
*/
|
|
55
|
+
export declare function generateRunnableExampleFile(examples: ExtractedExample[], imports: string[]): string;
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.extraction.example-extractor",
|
|
5
|
+
* "title": "Example Extractor",
|
|
6
|
+
* "category": "compiler"
|
|
7
|
+
* }
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Regex patterns for code block extraction
|
|
11
|
+
*/
|
|
12
|
+
const CODE_BLOCK_REGEX = /```(\w*(?:\s+doctest)?)\s*\n([\s\S]*?)\n\s*```/g;
|
|
13
|
+
/**
|
|
14
|
+
* Escapes a string for use in a JavaScript single-quoted string.
|
|
15
|
+
* Prevents code injection by escaping backslashes, quotes, and newlines.
|
|
16
|
+
*/
|
|
17
|
+
function escapeForSingleQuotedString(str) {
|
|
18
|
+
return str
|
|
19
|
+
.replace(/\\/g, '\\\\')
|
|
20
|
+
.replace(/'/g, "\\'")
|
|
21
|
+
.replace(/\n/g, '\\n')
|
|
22
|
+
.replace(/\r/g, '\\r')
|
|
23
|
+
.replace(/\u2028/g, '\\u2028')
|
|
24
|
+
.replace(/\u2029/g, '\\u2029');
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Languages that are considered runnable
|
|
28
|
+
*/
|
|
29
|
+
const RUNNABLE_LANGUAGES = new Set([
|
|
30
|
+
'typescript',
|
|
31
|
+
'ts',
|
|
32
|
+
'javascript',
|
|
33
|
+
'js',
|
|
34
|
+
'tsx',
|
|
35
|
+
'jsx',
|
|
36
|
+
]);
|
|
37
|
+
/**
|
|
38
|
+
* Extracts all code examples from a markdown/documentation string
|
|
39
|
+
* @param content - Content to extract examples from
|
|
40
|
+
* @param baseLocation - Base source location for the content
|
|
41
|
+
* @returns Array of extracted examples
|
|
42
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"extractExamples","stability":"stable","signature":"(content: string, baseLocation: SourceLocation) => ExtractedExample[]"}
|
|
43
|
+
*/
|
|
44
|
+
export function extractExamples(content, baseLocation) {
|
|
45
|
+
const examples = [];
|
|
46
|
+
// Reset regex state
|
|
47
|
+
CODE_BLOCK_REGEX.lastIndex = 0;
|
|
48
|
+
let match;
|
|
49
|
+
while ((match = CODE_BLOCK_REGEX.exec(content)) !== null) {
|
|
50
|
+
const [, languageSpec, code] = match;
|
|
51
|
+
const startIndex = match.index;
|
|
52
|
+
// Calculate relative line number
|
|
53
|
+
const textBefore = content.slice(0, startIndex);
|
|
54
|
+
const relativeLine = textBefore.split('\n').length;
|
|
55
|
+
// Parse language and doctest flag
|
|
56
|
+
const isDoctest = languageSpec.includes('doctest');
|
|
57
|
+
const language = languageSpec.replace(/\s*doctest\s*/, '').trim() || 'typescript';
|
|
58
|
+
// Determine if runnable
|
|
59
|
+
const isRunnable = isDoctest || RUNNABLE_LANGUAGES.has(language.toLowerCase());
|
|
60
|
+
// Extract description from preceding text
|
|
61
|
+
const precedingLines = textBefore.split('\n').slice(-3);
|
|
62
|
+
const description = extractExampleDescription(precedingLines);
|
|
63
|
+
examples.push({
|
|
64
|
+
code: code.trim(),
|
|
65
|
+
language,
|
|
66
|
+
description,
|
|
67
|
+
runnable: isRunnable,
|
|
68
|
+
isDoctest,
|
|
69
|
+
location: {
|
|
70
|
+
...baseLocation,
|
|
71
|
+
line: baseLocation.line + relativeLine - 1,
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return examples;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Extracts description from lines preceding a code block
|
|
79
|
+
*/
|
|
80
|
+
function extractExampleDescription(precedingLines) {
|
|
81
|
+
// Look for a description in the last non-empty line before the code block
|
|
82
|
+
for (let i = precedingLines.length - 1; i >= 0; i--) {
|
|
83
|
+
const line = precedingLines[i].trim();
|
|
84
|
+
// Skip empty lines and headers
|
|
85
|
+
if (!line || line.startsWith('#') || line.startsWith('```')) {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
// Return the description if it's meaningful
|
|
89
|
+
if (line.length > 5 && !line.startsWith('@')) {
|
|
90
|
+
return line;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Validates that a code example is syntactically valid
|
|
97
|
+
* @param code - Code to validate
|
|
98
|
+
* @param language - Language of the code
|
|
99
|
+
* @returns Validation result with any errors
|
|
100
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"validateExample","stability":"stable","signature":"(code: string, language: string) => { valid: boolean; errors: string[] }"}
|
|
101
|
+
*/
|
|
102
|
+
export function validateExample(code, _language) {
|
|
103
|
+
const errors = [];
|
|
104
|
+
// Basic syntax checks
|
|
105
|
+
if (!code.trim()) {
|
|
106
|
+
errors.push('Empty code block');
|
|
107
|
+
return { valid: false, errors };
|
|
108
|
+
}
|
|
109
|
+
// Check for balanced brackets
|
|
110
|
+
const brackets = {
|
|
111
|
+
'(': ')',
|
|
112
|
+
'[': ']',
|
|
113
|
+
'{': '}',
|
|
114
|
+
};
|
|
115
|
+
const stack = [];
|
|
116
|
+
let inString = false;
|
|
117
|
+
let stringChar = '';
|
|
118
|
+
for (let i = 0; i < code.length; i++) {
|
|
119
|
+
const char = code[i];
|
|
120
|
+
const prevChar = i > 0 ? code[i - 1] : '';
|
|
121
|
+
// Handle strings
|
|
122
|
+
if ((char === '"' || char === "'" || char === '`') && prevChar !== '\\') {
|
|
123
|
+
if (!inString) {
|
|
124
|
+
inString = true;
|
|
125
|
+
stringChar = char;
|
|
126
|
+
}
|
|
127
|
+
else if (char === stringChar) {
|
|
128
|
+
inString = false;
|
|
129
|
+
stringChar = '';
|
|
130
|
+
}
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
if (inString)
|
|
134
|
+
continue;
|
|
135
|
+
// Handle brackets
|
|
136
|
+
if (brackets[char]) {
|
|
137
|
+
stack.push(brackets[char]);
|
|
138
|
+
}
|
|
139
|
+
else if (Object.values(brackets).includes(char)) {
|
|
140
|
+
if (stack.length === 0 || stack.pop() !== char) {
|
|
141
|
+
errors.push(`Unbalanced brackets: unexpected '${char}'`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (stack.length > 0) {
|
|
146
|
+
errors.push(`Unbalanced brackets: missing ${stack.join(', ')}`);
|
|
147
|
+
}
|
|
148
|
+
if (inString) {
|
|
149
|
+
errors.push(`Unterminated string starting with ${stringChar}`);
|
|
150
|
+
}
|
|
151
|
+
return { valid: errors.length === 0, errors };
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Wraps a doctest example in a test harness
|
|
155
|
+
* @param example - The extracted example
|
|
156
|
+
* @param entryId - ID of the codex entry this example belongs to
|
|
157
|
+
* @returns Test code ready to run with vitest
|
|
158
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"wrapDoctestInHarness","stability":"stable","signature":"(example: ExtractedExample, entryId: string) => string"}
|
|
159
|
+
*/
|
|
160
|
+
export function wrapDoctestInHarness(example, entryId) {
|
|
161
|
+
const testName = example.description || `doctest from ${entryId}`;
|
|
162
|
+
// Properly escape for single-quoted strings in generated code
|
|
163
|
+
const safeEntryId = escapeForSingleQuotedString(entryId);
|
|
164
|
+
const safeTestName = escapeForSingleQuotedString(testName);
|
|
165
|
+
// Indent the example code
|
|
166
|
+
const indentedCode = example.code
|
|
167
|
+
.split('\n')
|
|
168
|
+
.map(line => ` ${line}`)
|
|
169
|
+
.join('\n');
|
|
170
|
+
return `import { describe, it, expect } from 'vitest';
|
|
171
|
+
|
|
172
|
+
describe('${safeEntryId}', () => {
|
|
173
|
+
it('${safeTestName}', async () => {
|
|
174
|
+
${indentedCode}
|
|
175
|
+
});
|
|
176
|
+
});`;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Extracts examples from JSDoc @example tags in source code
|
|
180
|
+
* @param content - Source file content
|
|
181
|
+
* @param filePath - Path to the source file
|
|
182
|
+
* @returns Array of extracted examples with their context
|
|
183
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"extractExamplesFromSource","stability":"stable","signature":"(content: string, filePath: string) => { functionName: string; examples: ExtractedExample[] }[]"}
|
|
184
|
+
*/
|
|
185
|
+
export function extractExamplesFromSource(content, filePath) {
|
|
186
|
+
const results = [];
|
|
187
|
+
// Match a single JSDoc block followed by a function declaration.
|
|
188
|
+
// Uses ((?:[^*]|\*(?!\/))*) instead of ([\s\S]*?) to prevent the regex
|
|
189
|
+
// from backtracking across two adjacent /** */ blocks (e.g., a JSDoc
|
|
190
|
+
// comment immediately followed by a @codexApi comment).
|
|
191
|
+
const jsdocFunctionPattern = /\/\*\*((?:[^*]|\*(?!\/))*)\*\/\s*(?:\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)*(?:export\s+)?(?:async\s+)?(?:function|const|let|var)\s+(\w+)/g;
|
|
192
|
+
let match;
|
|
193
|
+
while ((match = jsdocFunctionPattern.exec(content)) !== null) {
|
|
194
|
+
const [, jsdocContent, functionName] = match;
|
|
195
|
+
const startIndex = match.index;
|
|
196
|
+
// Calculate line number
|
|
197
|
+
const textBefore = content.slice(0, startIndex);
|
|
198
|
+
const lineNumber = textBefore.split('\n').length;
|
|
199
|
+
// Clean JSDoc asterisk prefixes from the captured content so that:
|
|
200
|
+
// 1. Code block regexes (``` ... ```) can match correctly
|
|
201
|
+
// 2. @example end-detection doesn't false-positive on @word inside code
|
|
202
|
+
// (e.g., '@pithyjs/shared' in an import statement)
|
|
203
|
+
const cleanedJsdoc = jsdocContent
|
|
204
|
+
.split('\n')
|
|
205
|
+
.map(l => l.replace(/^\s*\*\s?/, ''))
|
|
206
|
+
.join('\n');
|
|
207
|
+
// Match @example at the start of a line only — not mid-sentence
|
|
208
|
+
// mentions like "from JSDoc @example tags". Terminates at the next
|
|
209
|
+
// @tag at the start of a line, or end of string.
|
|
210
|
+
const examplePattern = /(?:^|\n)@example[ \t]*\n?([\s\S]*?)(?=\n@\w+|$)/g;
|
|
211
|
+
const examples = [];
|
|
212
|
+
let exampleMatch;
|
|
213
|
+
while ((exampleMatch = examplePattern.exec(cleanedJsdoc)) !== null) {
|
|
214
|
+
const exampleContent = exampleMatch[1].trim();
|
|
215
|
+
const baseLocation = {
|
|
216
|
+
file: filePath,
|
|
217
|
+
line: lineNumber,
|
|
218
|
+
};
|
|
219
|
+
// Extract code blocks from the example content
|
|
220
|
+
const extractedExamples = extractExamples(exampleContent, baseLocation);
|
|
221
|
+
// If no code blocks found, treat the content as inline code
|
|
222
|
+
if (extractedExamples.length === 0 && exampleContent) {
|
|
223
|
+
// Strip JSDoc comment delimiter artifacts that leak in when
|
|
224
|
+
// two adjacent /** */ blocks are captured as one
|
|
225
|
+
const cleanedCode = exampleContent
|
|
226
|
+
.split('\n')
|
|
227
|
+
.filter(l => !/^\s*\*?\/?$/.test(l) && !/^\s*\/\*\*?\s*$/.test(l))
|
|
228
|
+
.join('\n')
|
|
229
|
+
.trim();
|
|
230
|
+
if (cleanedCode) {
|
|
231
|
+
examples.push({
|
|
232
|
+
code: cleanedCode,
|
|
233
|
+
language: 'typescript',
|
|
234
|
+
runnable: true,
|
|
235
|
+
isDoctest: false,
|
|
236
|
+
location: baseLocation,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
examples.push(...extractedExamples);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (examples.length > 0) {
|
|
245
|
+
results.push({ functionName, examples });
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
return results;
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Generates a runnable example file from extracted examples
|
|
252
|
+
* @param examples - Examples to include
|
|
253
|
+
* @param imports - Required imports
|
|
254
|
+
* @returns Complete TypeScript file content
|
|
255
|
+
* @codexApi {"parent":"pithy.codex.extraction.example-extractor","name":"generateRunnableExampleFile","stability":"stable","signature":"(examples: ExtractedExample[], imports: string[]) => string"}
|
|
256
|
+
*/
|
|
257
|
+
export function generateRunnableExampleFile(examples, imports) {
|
|
258
|
+
const importStatements = imports.map(imp => `import ${imp};`).join('\n');
|
|
259
|
+
const exampleBlocks = examples
|
|
260
|
+
.filter(ex => ex.runnable)
|
|
261
|
+
.map((ex, i) => `
|
|
262
|
+
// Example ${i + 1}
|
|
263
|
+
// From: ${ex.location.file}:${ex.location.line}
|
|
264
|
+
${ex.description ? `// ${ex.description}` : ''}
|
|
265
|
+
${ex.code}
|
|
266
|
+
`)
|
|
267
|
+
.join('\n');
|
|
268
|
+
return `${importStatements}
|
|
269
|
+
|
|
270
|
+
${exampleBlocks}
|
|
271
|
+
`.trim();
|
|
272
|
+
}
|