@pithyjs/codex 0.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +4401 -0
- package/dist/annotations.d.ts +106 -0
- package/dist/annotations.js +306 -0
- package/dist/apply.d.ts +2 -0
- package/dist/apply.js +80 -0
- package/dist/changed-scope.d.ts +23 -0
- package/dist/changed-scope.js +31 -0
- package/dist/check.d.ts +2 -0
- package/dist/check.js +117 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +67 -0
- package/dist/env.d.ts +5 -0
- package/dist/env.js +35 -0
- package/dist/extract-cli.d.ts +23 -0
- package/dist/extract-cli.js +192 -0
- package/dist/extraction/breaking-changes.d.ts +66 -0
- package/dist/extraction/breaking-changes.js +352 -0
- package/dist/extraction/example-extractor.d.ts +55 -0
- package/dist/extraction/example-extractor.js +272 -0
- package/dist/extraction/index.d.ts +27 -0
- package/dist/extraction/index.js +31 -0
- package/dist/extraction/jsdoc-parser.d.ts +43 -0
- package/dist/extraction/jsdoc-parser.js +274 -0
- package/dist/extraction/pipeline.d.ts +54 -0
- package/dist/extraction/pipeline.js +526 -0
- package/dist/extraction/readme-sync.d.ts +108 -0
- package/dist/extraction/readme-sync.js +592 -0
- package/dist/extraction/snapshot-store.d.ts +40 -0
- package/dist/extraction/snapshot-store.js +153 -0
- package/dist/extraction/source-linker.d.ts +80 -0
- package/dist/extraction/source-linker.js +316 -0
- package/dist/extraction/test-example-extractor.d.ts +69 -0
- package/dist/extraction/test-example-extractor.js +400 -0
- package/dist/extraction/test-pattern-extractor.d.ts +68 -0
- package/dist/extraction/test-pattern-extractor.js +261 -0
- package/dist/extraction/testing-pyramid.d.ts +44 -0
- package/dist/extraction/testing-pyramid.js +163 -0
- package/dist/extraction/type-extractor.d.ts +34 -0
- package/dist/extraction/type-extractor.js +494 -0
- package/dist/extraction/types.d.ts +401 -0
- package/dist/extraction/types.js +34 -0
- package/dist/indexer.d.ts +1 -0
- package/dist/indexer.js +107 -0
- package/dist/llm.d.ts +8 -0
- package/dist/llm.js +75 -0
- package/dist/readme-sync-cli.d.ts +20 -0
- package/dist/readme-sync-cli.js +167 -0
- package/dist/review.d.ts +2 -0
- package/dist/review.js +93 -0
- package/dist/scan.d.ts +29 -0
- package/dist/scan.js +221 -0
- package/dist/schema.d.ts +169 -0
- package/dist/schema.js +70 -0
- package/dist/snapshot-cli.d.ts +45 -0
- package/dist/snapshot-cli.js +217 -0
- package/dist/sync-cli.d.ts +22 -0
- package/dist/sync-cli.js +154 -0
- package/dist/sync-pipeline.d.ts +58 -0
- package/dist/sync-pipeline.js +104 -0
- package/dist/sync.d.ts +2 -0
- package/dist/sync.js +318 -0
- package/dist/validate-cli.d.ts +20 -0
- package/dist/validate-cli.js +144 -0
- package/dist/validate.d.ts +76 -0
- package/dist/validate.js +183 -0
- package/dist/watch-cli.d.ts +21 -0
- package/dist/watch-cli.js +220 -0
- package/package.json +62 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @codex
|
|
4
|
+
* {
|
|
5
|
+
* "id": "pithy.codex.extraction.readme-sync-cli",
|
|
6
|
+
* "title": "README Sync CLI",
|
|
7
|
+
* "category": "plugin"
|
|
8
|
+
* }
|
|
9
|
+
*
|
|
10
|
+
* CLI for syncing README files with codex extraction data.
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* pnpm --filter @pithyjs/codex readme-sync [-- options]
|
|
14
|
+
*
|
|
15
|
+
* Options:
|
|
16
|
+
* --root <path> Project root directory (default: cwd)
|
|
17
|
+
* --dry-run Report changes without writing files
|
|
18
|
+
* --verbose Show detailed progress
|
|
19
|
+
*/
|
|
20
|
+
import { resolve } from 'node:path';
|
|
21
|
+
import fs from 'node:fs/promises';
|
|
22
|
+
import { syncAllReadmes } from './extraction/readme-sync.js';
|
|
23
|
+
import { runExtractionPipeline } from './extraction/pipeline.js';
|
|
24
|
+
function parseArgs(args) {
|
|
25
|
+
const options = {
|
|
26
|
+
root: process.cwd(),
|
|
27
|
+
dryRun: false,
|
|
28
|
+
verbose: false,
|
|
29
|
+
};
|
|
30
|
+
for (let i = 0; i < args.length; i++) {
|
|
31
|
+
const arg = args[i];
|
|
32
|
+
// Handle --root=value syntax
|
|
33
|
+
if (arg.startsWith('--root=')) {
|
|
34
|
+
options.root = arg.slice('--root='.length);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
switch (arg) {
|
|
38
|
+
case '--root': {
|
|
39
|
+
const next = args[i + 1];
|
|
40
|
+
if (!next || next.startsWith('-')) {
|
|
41
|
+
console.error('Error: --root option requires a path argument.');
|
|
42
|
+
process.exit(1);
|
|
43
|
+
}
|
|
44
|
+
options.root = next;
|
|
45
|
+
i++;
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
case '--dry-run':
|
|
49
|
+
options.dryRun = true;
|
|
50
|
+
break;
|
|
51
|
+
case '--verbose':
|
|
52
|
+
case '-v':
|
|
53
|
+
options.verbose = true;
|
|
54
|
+
break;
|
|
55
|
+
case '--help':
|
|
56
|
+
case '-h':
|
|
57
|
+
printHelp();
|
|
58
|
+
process.exit(0);
|
|
59
|
+
break; // safety: unreachable but prevents fallthrough if exit is refactored
|
|
60
|
+
default:
|
|
61
|
+
if (arg.startsWith('-')) {
|
|
62
|
+
console.warn(`Warning: unknown option "${arg}" (ignored)`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return options;
|
|
67
|
+
}
|
|
68
|
+
function printHelp() {
|
|
69
|
+
console.log(`
|
|
70
|
+
Codex README Sync
|
|
71
|
+
|
|
72
|
+
Syncs README files by replacing <!-- @codex:auto --> markers with
|
|
73
|
+
generated content from the extraction pipeline.
|
|
74
|
+
|
|
75
|
+
Usage:
|
|
76
|
+
pithy-codex-readme-sync [options]
|
|
77
|
+
pnpm --filter @pithyjs/codex readme-sync [-- options]
|
|
78
|
+
|
|
79
|
+
Options:
|
|
80
|
+
--root <path> Project root directory (default: current directory)
|
|
81
|
+
--dry-run Report changes without writing files
|
|
82
|
+
--verbose, -v Show detailed progress
|
|
83
|
+
--help, -h Show this help message
|
|
84
|
+
|
|
85
|
+
Marker Syntax:
|
|
86
|
+
<!-- @codex:auto install="@pithyjs/pkg" -->...<!-- @codex:end -->
|
|
87
|
+
<!-- @codex:auto api="pithy.pkg.*" format="table" -->...<!-- @codex:end -->
|
|
88
|
+
<!-- @codex:auto examples="pithy.pkg.*" limit="3" -->...<!-- @codex:end -->
|
|
89
|
+
<!-- @codex:auto testing="pithy.pkg.*" -->...<!-- @codex:end -->
|
|
90
|
+
|
|
91
|
+
Examples:
|
|
92
|
+
# Sync READMEs from project root
|
|
93
|
+
pithy-codex-readme-sync --root ../..
|
|
94
|
+
|
|
95
|
+
# Preview changes without writing
|
|
96
|
+
pithy-codex-readme-sync --root ../.. --dry-run --verbose
|
|
97
|
+
`);
|
|
98
|
+
}
|
|
99
|
+
async function main() {
|
|
100
|
+
const args = process.argv.slice(2);
|
|
101
|
+
const options = parseArgs(args);
|
|
102
|
+
options.root = resolve(options.root);
|
|
103
|
+
// Verify root exists
|
|
104
|
+
try {
|
|
105
|
+
const stat = await fs.stat(options.root);
|
|
106
|
+
if (!stat.isDirectory()) {
|
|
107
|
+
console.error(`Error: "${options.root}" is not a directory`);
|
|
108
|
+
process.exit(1);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
console.error(`Error: Root directory "${options.root}" does not exist`);
|
|
113
|
+
process.exit(1);
|
|
114
|
+
}
|
|
115
|
+
if (options.verbose) {
|
|
116
|
+
console.log('README Sync');
|
|
117
|
+
console.log(` Root: ${options.root}`);
|
|
118
|
+
console.log(` Mode: ${options.dryRun ? 'dry-run' : 'write'}`);
|
|
119
|
+
console.log('');
|
|
120
|
+
}
|
|
121
|
+
// Always run extraction pipeline — it's the only reliable source of EnrichedComponent data
|
|
122
|
+
if (options.verbose)
|
|
123
|
+
console.log('Running extraction pipeline...');
|
|
124
|
+
const extractionResult = await runExtractionPipeline({
|
|
125
|
+
root: options.root,
|
|
126
|
+
analyzeTests: true,
|
|
127
|
+
linkReadmes: true,
|
|
128
|
+
extractRunnableExamples: true,
|
|
129
|
+
});
|
|
130
|
+
if (options.verbose) {
|
|
131
|
+
console.log(` Found ${extractionResult.components.length} components`);
|
|
132
|
+
console.log('');
|
|
133
|
+
}
|
|
134
|
+
// Sync READMEs
|
|
135
|
+
if (options.verbose)
|
|
136
|
+
console.log('Syncing README files...');
|
|
137
|
+
const results = await syncAllReadmes(extractionResult, {
|
|
138
|
+
root: options.root,
|
|
139
|
+
dryRun: options.dryRun,
|
|
140
|
+
verbose: options.verbose,
|
|
141
|
+
});
|
|
142
|
+
// Summary
|
|
143
|
+
const changed = results.filter(r => r.changed);
|
|
144
|
+
const totalMarkers = results.reduce((sum, r) => sum + r.markersProcessed, 0);
|
|
145
|
+
const totalUpdated = results.reduce((sum, r) => sum + r.markersUpdated, 0);
|
|
146
|
+
const allWarnings = results.flatMap(r => r.warnings);
|
|
147
|
+
console.log('');
|
|
148
|
+
console.log(`README Sync complete:`);
|
|
149
|
+
console.log(` Files scanned: ${results.length}`);
|
|
150
|
+
console.log(` Files changed: ${changed.length}`);
|
|
151
|
+
console.log(` Markers processed: ${totalMarkers}`);
|
|
152
|
+
console.log(` Markers updated: ${totalUpdated}`);
|
|
153
|
+
if (allWarnings.length > 0) {
|
|
154
|
+
console.log(` Warnings: ${allWarnings.length}`);
|
|
155
|
+
for (const w of allWarnings) {
|
|
156
|
+
console.log(` - ${w}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (options.dryRun && changed.length > 0) {
|
|
160
|
+
console.log('');
|
|
161
|
+
console.log('(dry-run mode — no files were written)');
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
main().catch(err => {
|
|
165
|
+
console.error('Fatal error:', err);
|
|
166
|
+
process.exit(1);
|
|
167
|
+
});
|
package/dist/review.d.ts
ADDED
package/dist/review.js
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @codex
|
|
4
|
+
* {
|
|
5
|
+
* "id": "pithy.codex.review",
|
|
6
|
+
* "title": "Entry Reviewer",
|
|
7
|
+
* "category": "feature"
|
|
8
|
+
* }
|
|
9
|
+
*
|
|
10
|
+
* Generates review.md from pending codex entries using LLM.
|
|
11
|
+
*/
|
|
12
|
+
import fs from 'node:fs/promises';
|
|
13
|
+
import { join, dirname, relative } from 'node:path';
|
|
14
|
+
import { fileURLToPath } from 'node:url';
|
|
15
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
16
|
+
async function loadIndex(root) {
|
|
17
|
+
try {
|
|
18
|
+
const raw = await fs.readFile(join(root, 'codex.index.json'), 'utf8');
|
|
19
|
+
const j = JSON.parse(raw);
|
|
20
|
+
return (j.byId || {});
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return {};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
async function main() {
|
|
27
|
+
const root = process.argv.includes('--root')
|
|
28
|
+
? process.argv[process.argv.indexOf('--root') + 1]
|
|
29
|
+
: join(__dirname, '../../..');
|
|
30
|
+
const proposalsDir = join(root, 'codex', '.proposals');
|
|
31
|
+
const files = await fs.readdir(proposalsDir).catch(() => []);
|
|
32
|
+
const items = files.filter(f => f.endsWith('.json'));
|
|
33
|
+
// Load existing index to determine change status
|
|
34
|
+
const indexById = await loadIndex(root);
|
|
35
|
+
let md = `# Codex Proposals Review\n\n`;
|
|
36
|
+
md += `Edit this file: mark accepted items with \`[x]\` then run \`pnpm -w --filter @pithyjs/codex apply\`.\n\n`;
|
|
37
|
+
if (items.length === 0) {
|
|
38
|
+
md += `*No proposals found. All components are up to date.*\n\n`;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
md += `## Summary\n\n`;
|
|
42
|
+
md += `Found **${items.length}** items requiring review:\n\n`;
|
|
43
|
+
// Categorize proposals
|
|
44
|
+
const newItems = [];
|
|
45
|
+
const changedItems = [];
|
|
46
|
+
for (const f of items) {
|
|
47
|
+
const p = join(proposalsDir, f);
|
|
48
|
+
const json = JSON.parse(await fs.readFile(p, 'utf8'));
|
|
49
|
+
const existing = indexById[json.id];
|
|
50
|
+
if (!existing) {
|
|
51
|
+
newItems.push(json.id);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
changedItems.push(json.id);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (newItems.length > 0) {
|
|
58
|
+
md += `- **${newItems.length} NEW** components: ${newItems.join(', ')}\n`;
|
|
59
|
+
}
|
|
60
|
+
if (changedItems.length > 0) {
|
|
61
|
+
md += `- **${changedItems.length} CHANGED** components: ${changedItems.join(', ')}\n`;
|
|
62
|
+
}
|
|
63
|
+
md += `\n---\n\n`;
|
|
64
|
+
md += `## Proposals\n\n`;
|
|
65
|
+
for (const f of items) {
|
|
66
|
+
const p = join(proposalsDir, f);
|
|
67
|
+
const json = JSON.parse(await fs.readFile(p, 'utf8'));
|
|
68
|
+
const existing = indexById[json.id];
|
|
69
|
+
const status = existing ? 'CHANGED' : 'NEW';
|
|
70
|
+
const statusEmoji = status === 'NEW' ? '🆕' : '🔄';
|
|
71
|
+
md += `- [ ] ${statusEmoji} **${json.id}** — *${json.title || '(no title)'}* (${status})\n`;
|
|
72
|
+
md += ` - category: \`${json.category}\`\n`;
|
|
73
|
+
md += ` - tags: \`${(json.tags || []).join(', ')}\`\n`;
|
|
74
|
+
md += ` - related: \`${(json.related || []).join(', ')}\`\n`;
|
|
75
|
+
if (json.apis && json.apis.length > 0) {
|
|
76
|
+
md += ` - apis: \`${json.apis.map((api) => api.name).join(', ')}\`\n`;
|
|
77
|
+
}
|
|
78
|
+
if (json.meta?.hash) {
|
|
79
|
+
md += ` - hash: \`${json.meta.hash.substring(0, 8)}...\`\n`;
|
|
80
|
+
}
|
|
81
|
+
md += ` - file: \`${relative(root, p)}\`\n\n`;
|
|
82
|
+
md += '```json\n' + JSON.stringify(json, null, 2) + '\n```\n\n';
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
await fs.mkdir(proposalsDir, { recursive: true });
|
|
86
|
+
const reviewPath = join(proposalsDir, 'review.md');
|
|
87
|
+
await fs.writeFile(reviewPath, md);
|
|
88
|
+
console.log(`Review file: ${relative(root, reviewPath)}`);
|
|
89
|
+
}
|
|
90
|
+
main().catch(e => {
|
|
91
|
+
console.error(e);
|
|
92
|
+
process.exit(1);
|
|
93
|
+
});
|
package/dist/scan.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { CodexApiAnnotation } from './annotations.js';
|
|
3
|
+
export interface Candidate {
|
|
4
|
+
id: string;
|
|
5
|
+
title: string;
|
|
6
|
+
category: string;
|
|
7
|
+
tags: string[];
|
|
8
|
+
files: string[];
|
|
9
|
+
hash?: string;
|
|
10
|
+
apis?: CodexApiAnnotation[];
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Scans source files for @codex annotations and returns discovered components/APIs.
|
|
14
|
+
* @codexApi {"parent":"pithy.codex.scan","name":"scan","stability":"stable","signature":"(root: string, files?: string[]) => Promise<Candidate[]>"}
|
|
15
|
+
*/
|
|
16
|
+
export declare function scan(root: string, files?: string[]): Promise<Candidate[]>;
|
|
17
|
+
/**
|
|
18
|
+
* Fold the per-file fingerprints of one `@codex` id into a single stable one.
|
|
19
|
+
*
|
|
20
|
+
* Sorts the COMPLETE list before hashing, which is what makes the result
|
|
21
|
+
* independent of scan order. Folding pairwise — `hash(sort([acc, next]))` —
|
|
22
|
+
* looks equivalent and is not: SHA-256 is not associative, so sorting at each
|
|
23
|
+
* step only commutes for two inputs. With three or more (and `pithy.doctor.rules`
|
|
24
|
+
* is declared by 22 files) a different scan order yields a different fingerprint
|
|
25
|
+
* for identical sources, which reports every entry as changed at random.
|
|
26
|
+
*
|
|
27
|
+
* @codexApi {"parent":"pithy.codex.scan","name":"combineFingerprints","stability":"internal","signature":"(fingerprints: string[]) => string"}
|
|
28
|
+
*/
|
|
29
|
+
export declare function combineFingerprints(fingerprints: string[]): string;
|
package/dist/scan.js
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* @codex
|
|
4
|
+
* {
|
|
5
|
+
* "id": "pithy.codex.scan",
|
|
6
|
+
* "title": "Source Scanner",
|
|
7
|
+
* "category": "feature"
|
|
8
|
+
* }
|
|
9
|
+
*
|
|
10
|
+
* Scans source files for codex annotation candidates.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { globby } from 'globby';
|
|
14
|
+
import fs from 'node:fs/promises';
|
|
15
|
+
import { join, dirname } from 'node:path';
|
|
16
|
+
import { fileURLToPath } from 'node:url';
|
|
17
|
+
import { discoverComponentsFromFile, createCanonicalFingerprint, } from './annotations.js';
|
|
18
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
/**
|
|
20
|
+
* Scans source files for @codex annotations and returns discovered components/APIs.
|
|
21
|
+
* @codexApi {"parent":"pithy.codex.scan","name":"scan","stability":"stable","signature":"(root: string, files?: string[]) => Promise<Candidate[]>"}
|
|
22
|
+
*/
|
|
23
|
+
export async function scan(root, files) {
|
|
24
|
+
// Pure annotation-based discovery
|
|
25
|
+
return await scanWithAnnotations(root, files);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Enhanced scanner that discovers components via annotations
|
|
29
|
+
* @param root - Repository root path
|
|
30
|
+
* @param files - Optional list of absolute file paths to scan (if omitted, scans all source files)
|
|
31
|
+
* @returns Array of candidates with hash and API metadata
|
|
32
|
+
*/
|
|
33
|
+
async function scanWithAnnotations(root, files) {
|
|
34
|
+
let sourceFiles;
|
|
35
|
+
const partial = files !== undefined && files.length > 0;
|
|
36
|
+
if (partial) {
|
|
37
|
+
// Scan only the provided files (filtered to .ts/.js/.mjs)
|
|
38
|
+
sourceFiles = files.filter(f => f.endsWith('.ts') || f.endsWith('.js') || f.endsWith('.mjs'));
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
sourceFiles = await allSourceFiles(root);
|
|
42
|
+
}
|
|
43
|
+
// A partial scan — `sync --changed-only` — sees only the files git reports as
|
|
44
|
+
// changed. For an id declared by several files that is not enough: the
|
|
45
|
+
// fingerprint would cover only the changed declarations, and
|
|
46
|
+
// `buildEntryFromScan` replaces an entry's whole API list with whatever this
|
|
47
|
+
// scan produced, so editing one of `pithy.doctor.rules`' 22 files would both
|
|
48
|
+
// mis-fingerprint the entry and delete the other 21 files' APIs from it.
|
|
49
|
+
//
|
|
50
|
+
// So: find which ids the changed files touch, then pull in EVERY file in the
|
|
51
|
+
// repository that declares one of them before aggregating.
|
|
52
|
+
if (partial && sourceFiles.length > 0) {
|
|
53
|
+
sourceFiles = await withAllDeclarationsOf(root, sourceFiles);
|
|
54
|
+
}
|
|
55
|
+
if (process.env.NODE_ENV === 'development') {
|
|
56
|
+
console.log(`[Codex] Scanning ${sourceFiles.length} source files for annotations`);
|
|
57
|
+
}
|
|
58
|
+
const discoveredComponents = [];
|
|
59
|
+
// Process files in parallel for better performance
|
|
60
|
+
const discoveries = await Promise.all(sourceFiles.map(async (filePath) => {
|
|
61
|
+
try {
|
|
62
|
+
return await discoverComponentsFromFile(filePath);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
if (process.env.NODE_ENV === 'development') {
|
|
66
|
+
console.warn(`[Codex] Failed to scan file ${filePath}:`, error);
|
|
67
|
+
}
|
|
68
|
+
return [];
|
|
69
|
+
}
|
|
70
|
+
}));
|
|
71
|
+
// Flatten and merge discoveries
|
|
72
|
+
for (const fileDiscoveries of discoveries) {
|
|
73
|
+
discoveredComponents.push(...fileDiscoveries);
|
|
74
|
+
}
|
|
75
|
+
// Build candidates map with hash metadata
|
|
76
|
+
const candidatesMap = new Map();
|
|
77
|
+
/**
|
|
78
|
+
* Every per-file fingerprint contributing to each id, kept whole so the fold
|
|
79
|
+
* below can sort the COMPLETE list before hashing.
|
|
80
|
+
*
|
|
81
|
+
* Several files can declare one `@codex` id — `pithy.runtime.components` is
|
|
82
|
+
* declared by both `bindComponentsTags.ts` and `forwardedAttrs.ts`, and
|
|
83
|
+
* `pithy.doctor.rules` by 22 files. This used to do `existing.hash = hash`,
|
|
84
|
+
* so the file scanned LAST decided the fingerprint and a material change to
|
|
85
|
+
* any other was invisible to change detection: `sync` skipped the entry as
|
|
86
|
+
* unchanged and its documented digest silently drifted from the source.
|
|
87
|
+
*
|
|
88
|
+
* Folding pairwise as `hash(sort([acc, next]))` fixes two files but not three:
|
|
89
|
+
* SHA-256 is not associative, so sorting at each step cannot make the
|
|
90
|
+
* reduction commutative, and a different scan order would yield a different
|
|
91
|
+
* fingerprint for identical sources — spurious "changed" for the 22-file case.
|
|
92
|
+
* Sorting the whole list once is what actually makes it order-independent.
|
|
93
|
+
*/
|
|
94
|
+
const fingerprintsById = new Map();
|
|
95
|
+
for (const component of discoveredComponents) {
|
|
96
|
+
const hash = createCanonicalFingerprint(component);
|
|
97
|
+
const existing = candidatesMap.get(component.id);
|
|
98
|
+
if (!existing) {
|
|
99
|
+
candidatesMap.set(component.id, {
|
|
100
|
+
id: component.id,
|
|
101
|
+
title: component.title,
|
|
102
|
+
category: component.category,
|
|
103
|
+
tags: [], // Will be populated from annotations or merged with glob tags
|
|
104
|
+
files: component.files,
|
|
105
|
+
hash,
|
|
106
|
+
apis: component.apis,
|
|
107
|
+
});
|
|
108
|
+
fingerprintsById.set(component.id, [hash]);
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
// Merge files from multiple sources
|
|
112
|
+
existing.files = Array.from(new Set([...existing.files, ...component.files]));
|
|
113
|
+
// Collect every contributing fingerprint; they are folded into one after
|
|
114
|
+
// the loop. See `fingerprintsById`.
|
|
115
|
+
const fingerprints = fingerprintsById.get(component.id);
|
|
116
|
+
if (fingerprints)
|
|
117
|
+
fingerprints.push(hash);
|
|
118
|
+
// Merge APIs (deduplicate by name+signature)
|
|
119
|
+
const merged = [...(existing.apis || []), ...component.apis];
|
|
120
|
+
const seen = new Set();
|
|
121
|
+
existing.apis = merged.filter(api => {
|
|
122
|
+
const key = `${api.name}::${api.signature || ''}`;
|
|
123
|
+
if (seen.has(key))
|
|
124
|
+
return false;
|
|
125
|
+
seen.add(key);
|
|
126
|
+
return true;
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
// Fold each id's contributing fingerprints into one, order-independently.
|
|
131
|
+
for (const [id, fingerprints] of fingerprintsById) {
|
|
132
|
+
const candidate = candidatesMap.get(id);
|
|
133
|
+
if (!candidate || fingerprints.length < 2)
|
|
134
|
+
continue;
|
|
135
|
+
candidate.hash = combineFingerprints(fingerprints);
|
|
136
|
+
}
|
|
137
|
+
if (process.env.NODE_ENV === 'development') {
|
|
138
|
+
console.log(`[Codex] Discovered ${candidatesMap.size} annotated components`);
|
|
139
|
+
}
|
|
140
|
+
return Array.from(candidatesMap.values());
|
|
141
|
+
}
|
|
142
|
+
// Debug run: node dist/scan.js
|
|
143
|
+
if (process.argv[1]?.endsWith('scan.js')) {
|
|
144
|
+
const repoRoot = join(__dirname, '../../..');
|
|
145
|
+
scan(repoRoot).then(cs => console.log(JSON.stringify(cs, null, 2)));
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Fold the per-file fingerprints of one `@codex` id into a single stable one.
|
|
149
|
+
*
|
|
150
|
+
* Sorts the COMPLETE list before hashing, which is what makes the result
|
|
151
|
+
* independent of scan order. Folding pairwise — `hash(sort([acc, next]))` —
|
|
152
|
+
* looks equivalent and is not: SHA-256 is not associative, so sorting at each
|
|
153
|
+
* step only commutes for two inputs. With three or more (and `pithy.doctor.rules`
|
|
154
|
+
* is declared by 22 files) a different scan order yields a different fingerprint
|
|
155
|
+
* for identical sources, which reports every entry as changed at random.
|
|
156
|
+
*
|
|
157
|
+
* @codexApi {"parent":"pithy.codex.scan","name":"combineFingerprints","stability":"internal","signature":"(fingerprints: string[]) => string"}
|
|
158
|
+
*/
|
|
159
|
+
export function combineFingerprints(fingerprints) {
|
|
160
|
+
return createHash('sha256')
|
|
161
|
+
.update([...fingerprints].sort().join(':'), 'utf8')
|
|
162
|
+
.digest('hex');
|
|
163
|
+
}
|
|
164
|
+
/** Every source file the scanner considers, for a full scan. */
|
|
165
|
+
async function allSourceFiles(root) {
|
|
166
|
+
const sourceGlobs = [
|
|
167
|
+
'packages/*/src/**/*.ts',
|
|
168
|
+
'packages/*/src/**/*.js',
|
|
169
|
+
'packages/*/src/**/*.mjs',
|
|
170
|
+
'apps/*/src/**/*.ts',
|
|
171
|
+
'apps/*/src/**/*.js',
|
|
172
|
+
'apps/*/src/**/*.mjs',
|
|
173
|
+
'src/**/*.ts',
|
|
174
|
+
'src/**/*.js',
|
|
175
|
+
'src/**/*.mjs',
|
|
176
|
+
];
|
|
177
|
+
return await globby(sourceGlobs, {
|
|
178
|
+
cwd: root,
|
|
179
|
+
absolute: true,
|
|
180
|
+
gitignore: true,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Widen a partial file set to include every other file declaring the same
|
|
185
|
+
* component ids, so a multi-file id is aggregated from all of its parts.
|
|
186
|
+
*
|
|
187
|
+
* Ids are read from the changed files first, then matched against the rest of
|
|
188
|
+
* the repository by a plain substring test on the id — cheap, and the id is a
|
|
189
|
+
* literal inside the annotation JSON, so anything declaring it contains it.
|
|
190
|
+
*/
|
|
191
|
+
async function withAllDeclarationsOf(root, changed) {
|
|
192
|
+
const ids = new Set();
|
|
193
|
+
for (const file of changed) {
|
|
194
|
+
try {
|
|
195
|
+
for (const c of await discoverComponentsFromFile(file))
|
|
196
|
+
ids.add(c.id);
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
// Unreadable or unparseable: the main scan below reports it.
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (ids.size === 0)
|
|
203
|
+
return changed;
|
|
204
|
+
const result = new Set(changed);
|
|
205
|
+
const candidates = (await allSourceFiles(root)).filter(f => !result.has(f));
|
|
206
|
+
await Promise.all(candidates.map(async (file) => {
|
|
207
|
+
try {
|
|
208
|
+
const content = await fs.readFile(file, 'utf8');
|
|
209
|
+
for (const id of ids) {
|
|
210
|
+
if (content.includes(id)) {
|
|
211
|
+
result.add(file);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
/* ignore unreadable files */
|
|
218
|
+
}
|
|
219
|
+
}));
|
|
220
|
+
return [...result];
|
|
221
|
+
}
|
package/dist/schema.d.ts
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @codex
|
|
3
|
+
* {
|
|
4
|
+
* "id": "pithy.codex.schema",
|
|
5
|
+
* "title": "Entry Schema",
|
|
6
|
+
* "category": "feature"
|
|
7
|
+
* }
|
|
8
|
+
*
|
|
9
|
+
* Zod validation schemas for codex entries.
|
|
10
|
+
*/
|
|
11
|
+
import { z } from 'zod';
|
|
12
|
+
/** Zod schema for an API signature entry with name, stability, and optional signature.
|
|
13
|
+
* @codexApi {"parent":"pithy.codex.schema","name":"ApiSig","stability":"stable","signature":"z.ZodObject<{ name: string; stability: string; signature?: string }>"} */
|
|
14
|
+
export declare const ApiSig: z.ZodObject<{
|
|
15
|
+
name: z.ZodString;
|
|
16
|
+
stability: z.ZodDefault<z.ZodEnum<["internal", "experimental", "stable", "deprecated"]>>;
|
|
17
|
+
signature: z.ZodOptional<z.ZodString>;
|
|
18
|
+
}, "strip", z.ZodTypeAny, {
|
|
19
|
+
name: string;
|
|
20
|
+
stability: "internal" | "experimental" | "stable" | "deprecated";
|
|
21
|
+
signature?: string | undefined;
|
|
22
|
+
}, {
|
|
23
|
+
name: string;
|
|
24
|
+
stability?: "internal" | "experimental" | "stable" | "deprecated" | undefined;
|
|
25
|
+
signature?: string | undefined;
|
|
26
|
+
}>;
|
|
27
|
+
/** Zod schema defining the full structure of a codex entry.
|
|
28
|
+
* @codexApi {"parent":"pithy.codex.schema","name":"codexSchema","stability":"stable","signature":"z.ZodObject<CodexEntry>"} */
|
|
29
|
+
export declare const codexSchema: z.ZodObject<{
|
|
30
|
+
id: z.ZodString;
|
|
31
|
+
title: z.ZodString;
|
|
32
|
+
category: z.ZodString;
|
|
33
|
+
tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
34
|
+
updated: z.ZodString;
|
|
35
|
+
public: z.ZodDefault<z.ZodBoolean>;
|
|
36
|
+
status: z.ZodDefault<z.ZodEnum<["draft", "experimental", "stable", "deprecated"]>>;
|
|
37
|
+
version: z.ZodDefault<z.ZodString>;
|
|
38
|
+
owner: z.ZodOptional<z.ZodString>;
|
|
39
|
+
related: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
40
|
+
dependsOn: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
41
|
+
apis: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
42
|
+
name: z.ZodString;
|
|
43
|
+
stability: z.ZodDefault<z.ZodEnum<["internal", "experimental", "stable", "deprecated"]>>;
|
|
44
|
+
signature: z.ZodOptional<z.ZodString>;
|
|
45
|
+
}, "strip", z.ZodTypeAny, {
|
|
46
|
+
name: string;
|
|
47
|
+
stability: "internal" | "experimental" | "stable" | "deprecated";
|
|
48
|
+
signature?: string | undefined;
|
|
49
|
+
}, {
|
|
50
|
+
name: string;
|
|
51
|
+
stability?: "internal" | "experimental" | "stable" | "deprecated" | undefined;
|
|
52
|
+
signature?: string | undefined;
|
|
53
|
+
}>, "many">>;
|
|
54
|
+
rids: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
55
|
+
id: z.ZodString;
|
|
56
|
+
text: z.ZodString;
|
|
57
|
+
critical: z.ZodDefault<z.ZodBoolean>;
|
|
58
|
+
}, "strip", z.ZodTypeAny, {
|
|
59
|
+
critical: boolean;
|
|
60
|
+
id: string;
|
|
61
|
+
text: string;
|
|
62
|
+
}, {
|
|
63
|
+
id: string;
|
|
64
|
+
text: string;
|
|
65
|
+
critical?: boolean | undefined;
|
|
66
|
+
}>, "many">>;
|
|
67
|
+
acceptance: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
68
|
+
tests_expected: z.ZodDefault<z.ZodObject<{
|
|
69
|
+
unit: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
70
|
+
integration: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
71
|
+
e2e: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
72
|
+
property: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
73
|
+
contract: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
74
|
+
}, "strip", z.ZodTypeAny, {
|
|
75
|
+
unit: string[];
|
|
76
|
+
integration: string[];
|
|
77
|
+
e2e: string[];
|
|
78
|
+
property: string[];
|
|
79
|
+
contract: string[];
|
|
80
|
+
}, {
|
|
81
|
+
unit?: string[] | undefined;
|
|
82
|
+
integration?: string[] | undefined;
|
|
83
|
+
e2e?: string[] | undefined;
|
|
84
|
+
property?: string[] | undefined;
|
|
85
|
+
contract?: string[] | undefined;
|
|
86
|
+
}>>;
|
|
87
|
+
summary: z.ZodOptional<z.ZodString>;
|
|
88
|
+
bodyPath: z.ZodOptional<z.ZodString>;
|
|
89
|
+
meta: z.ZodOptional<z.ZodObject<{
|
|
90
|
+
hash: z.ZodOptional<z.ZodString>;
|
|
91
|
+
}, "strip", z.ZodTypeAny, {
|
|
92
|
+
hash?: string | undefined;
|
|
93
|
+
}, {
|
|
94
|
+
hash?: string | undefined;
|
|
95
|
+
}>>;
|
|
96
|
+
}, "strip", z.ZodTypeAny, {
|
|
97
|
+
id: string;
|
|
98
|
+
title: string;
|
|
99
|
+
category: string;
|
|
100
|
+
status: "experimental" | "stable" | "deprecated" | "draft";
|
|
101
|
+
tags: string[];
|
|
102
|
+
updated: string;
|
|
103
|
+
public: boolean;
|
|
104
|
+
version: string;
|
|
105
|
+
related: string[];
|
|
106
|
+
dependsOn: string[];
|
|
107
|
+
apis: {
|
|
108
|
+
name: string;
|
|
109
|
+
stability: "internal" | "experimental" | "stable" | "deprecated";
|
|
110
|
+
signature?: string | undefined;
|
|
111
|
+
}[];
|
|
112
|
+
rids: {
|
|
113
|
+
critical: boolean;
|
|
114
|
+
id: string;
|
|
115
|
+
text: string;
|
|
116
|
+
}[];
|
|
117
|
+
acceptance: string[];
|
|
118
|
+
tests_expected: {
|
|
119
|
+
unit: string[];
|
|
120
|
+
integration: string[];
|
|
121
|
+
e2e: string[];
|
|
122
|
+
property: string[];
|
|
123
|
+
contract: string[];
|
|
124
|
+
};
|
|
125
|
+
owner?: string | undefined;
|
|
126
|
+
summary?: string | undefined;
|
|
127
|
+
bodyPath?: string | undefined;
|
|
128
|
+
meta?: {
|
|
129
|
+
hash?: string | undefined;
|
|
130
|
+
} | undefined;
|
|
131
|
+
}, {
|
|
132
|
+
id: string;
|
|
133
|
+
title: string;
|
|
134
|
+
category: string;
|
|
135
|
+
updated: string;
|
|
136
|
+
status?: "experimental" | "stable" | "deprecated" | "draft" | undefined;
|
|
137
|
+
tags?: string[] | undefined;
|
|
138
|
+
public?: boolean | undefined;
|
|
139
|
+
version?: string | undefined;
|
|
140
|
+
owner?: string | undefined;
|
|
141
|
+
related?: string[] | undefined;
|
|
142
|
+
dependsOn?: string[] | undefined;
|
|
143
|
+
apis?: {
|
|
144
|
+
name: string;
|
|
145
|
+
stability?: "internal" | "experimental" | "stable" | "deprecated" | undefined;
|
|
146
|
+
signature?: string | undefined;
|
|
147
|
+
}[] | undefined;
|
|
148
|
+
rids?: {
|
|
149
|
+
id: string;
|
|
150
|
+
text: string;
|
|
151
|
+
critical?: boolean | undefined;
|
|
152
|
+
}[] | undefined;
|
|
153
|
+
acceptance?: string[] | undefined;
|
|
154
|
+
tests_expected?: {
|
|
155
|
+
unit?: string[] | undefined;
|
|
156
|
+
integration?: string[] | undefined;
|
|
157
|
+
e2e?: string[] | undefined;
|
|
158
|
+
property?: string[] | undefined;
|
|
159
|
+
contract?: string[] | undefined;
|
|
160
|
+
} | undefined;
|
|
161
|
+
summary?: string | undefined;
|
|
162
|
+
bodyPath?: string | undefined;
|
|
163
|
+
meta?: {
|
|
164
|
+
hash?: string | undefined;
|
|
165
|
+
} | undefined;
|
|
166
|
+
}>;
|
|
167
|
+
/** Inferred TypeScript type from the codex schema.
|
|
168
|
+
* @codexApi {"parent":"pithy.codex.schema","name":"CodexEntry","stability":"stable","signature":"z.infer<typeof codexSchema>"} */
|
|
169
|
+
export type CodexEntry = z.infer<typeof codexSchema>;
|