@dailephd/my-dev-kit-lab 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/README.md +6 -4
  3. package/benchmarks/projects/README.md +3 -1
  4. package/dist/src/cli/help.js +20 -3
  5. package/dist/src/commands/runExperimentDescribeCommand.js +10 -1
  6. package/dist/src/commands/runExperimentRunCommand.js +169 -19
  7. package/dist/src/core/fileGlobs.js +44 -0
  8. package/dist/src/evaluation/localRepositorySubject/config.js +296 -0
  9. package/dist/src/evaluation/localRepositorySubject/evaluationCase.js +36 -0
  10. package/dist/src/evaluation/localRepositorySubject/expectedFiles.js +51 -0
  11. package/dist/src/evaluation/localRepositorySubject/gitRepository.js +128 -0
  12. package/dist/src/evaluation/localRepositorySubject/index.js +8 -0
  13. package/dist/src/evaluation/localRepositorySubject/inventory.js +127 -0
  14. package/dist/src/evaluation/localRepositorySubject/loadLocalRepositorySubject.js +48 -0
  15. package/dist/src/evaluation/localRepositorySubject/manifest.js +52 -0
  16. package/dist/src/evaluation/localRepositorySubject/types.js +55 -0
  17. package/dist/src/evaluation/projectFileTree.js +45 -1
  18. package/dist/src/evaluation/runMyDevKitRetrieval.js +3 -1
  19. package/dist/src/evaluation/runRawFullFileBaseline.js +18 -4
  20. package/dist/src/evaluation/syntheticRepository/config.js +304 -0
  21. package/dist/src/evaluation/syntheticRepository/evaluationCase.js +43 -0
  22. package/dist/src/evaluation/syntheticRepository/identity.js +41 -0
  23. package/dist/src/evaluation/syntheticRepository/index.js +16 -0
  24. package/dist/src/evaluation/syntheticRepository/inspection.js +92 -0
  25. package/dist/src/evaluation/syntheticRepository/layout.js +50 -0
  26. package/dist/src/evaluation/syntheticRepository/manifest.js +281 -0
  27. package/dist/src/evaluation/syntheticRepository/manifestVerification.js +112 -0
  28. package/dist/src/evaluation/syntheticRepository/materialize.js +170 -0
  29. package/dist/src/evaluation/syntheticRepository/planning.js +479 -0
  30. package/dist/src/evaluation/syntheticRepository/prng.js +47 -0
  31. package/dist/src/evaluation/syntheticRepository/renderPython.js +68 -0
  32. package/dist/src/evaluation/syntheticRepository/renderRepository.js +39 -0
  33. package/dist/src/evaluation/syntheticRepository/renderShared.js +93 -0
  34. package/dist/src/evaluation/syntheticRepository/renderTypeScript.js +82 -0
  35. package/dist/src/evaluation/syntheticRepository/topology.js +96 -0
  36. package/dist/src/evaluation/syntheticRepository/types.js +26 -0
  37. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +29 -7
  38. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +4 -0
  39. package/dist/src/experiments/plugins/contextWindowScaling/index.js +1 -0
  40. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectErrors.js +35 -0
  41. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExclusions.js +75 -0
  42. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js +136 -0
  43. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +128 -0
  44. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js +85 -0
  45. package/dist/src/experiments/plugins/contextWindowScaling/metadata.js +2 -2
  46. package/dist/src/experiments/plugins/contextWindowScaling/plugin.js +93 -0
  47. package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js +4 -9
  48. package/dist/src/experiments/plugins/contextWindowScaling/syntheticInputs.js +56 -0
  49. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +4 -3
  50. package/dist/src/report/experiments/writePluginExperimentReports.js +8 -2
  51. package/docs/METRICS.md +5 -1
  52. package/package.json +1 -1
@@ -0,0 +1,16 @@
1
+ export * from "./types.js";
2
+ export { SYNTHETIC_REPOSITORY_CASE_FIELDS, SYNTHETIC_REPOSITORY_LIMITS, compareCodeUnits, maxImportEdges, normalizeSyntheticRepositoryConfig, validateSyntheticRepositoryCaseSpec, validateSyntheticRepositoryConfig, } from "./config.js";
3
+ export { canonicalSyntheticRepositoryCaseText, canonicalSyntheticRepositoryConfigText, computeSyntheticRepositoryCaseIdentity, computeSyntheticRepositoryConfigIdentity, } from "./identity.js";
4
+ export { createSyntheticRepositoryPrng, syntheticRepositoryPrngInitialState } from "./prng.js";
5
+ export { planTopology } from "./topology.js";
6
+ export { planSyntheticRepositories, planSyntheticRepositoryCase, verifySyntheticRepositoryPlan } from "./planning.js";
7
+ export { SyntheticRepositoryRenderError } from "./renderShared.js";
8
+ export { renderTypeScriptRepository } from "./renderTypeScript.js";
9
+ export { pythonModuleName, renderPythonRepository } from "./renderPython.js";
10
+ export { renderSyntheticRepository } from "./renderRepository.js";
11
+ export { inspectRepositoryFiles } from "./inspection.js";
12
+ export { deriveSyntheticRepositoryLayout, describeSyntheticTask } from "./layout.js";
13
+ export { SYNTHETIC_REPOSITORY_MANIFEST_FILE_NAME, SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_ID, SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_VERSION, SYNTHETIC_REPOSITORY_REPOSITORY_DIRECTORY, buildSyntheticRepositoryManifest, computeRepositoryContentIdentity, serializeSyntheticRepositoryManifest, validateSyntheticRepositoryManifest, } from "./manifest.js";
14
+ export { verifySyntheticRepositoryMaterialization } from "./manifestVerification.js";
15
+ export { buildSyntheticAnswerKey, buildSyntheticEvaluationCase } from "./evaluationCase.js";
16
+ export { SyntheticRepositoryMaterializationError, materializeSyntheticRepository } from "./materialize.js";
@@ -0,0 +1,92 @@
1
+ import path from "node:path";
2
+ import { REPEATED_PATTERN_MARKER } from "./renderShared.js";
3
+ import { pythonModuleName } from "./renderPython.js";
4
+ const TS_SYMBOL = /^export function (sym_\d+)\(/gm;
5
+ const PY_SYMBOL = /^def (sym_\d+)\(/gm;
6
+ const TS_IMPORT = /^import \{ (sym_\d+) \} from "([^"]+)";$/gm;
7
+ const PY_IMPORT = /^from ([A-Za-z0-9_.]+) import (sym_\d+)$/gm;
8
+ const PATTERN = new RegExp(`${REPEATED_PATTERN_MARKER} (pattern_\\d+) `, "g");
9
+ function matchAll(pattern, text) {
10
+ return [...text.matchAll(new RegExp(pattern.source, pattern.flags))];
11
+ }
12
+ /**
13
+ * Measures a generated repository from its file bytes alone (no plan): file roles, `sym_` definitions,
14
+ * resolved internal imports, repeated-pattern markers and the longest dependency chain. The verifier and the
15
+ * renderer self-check use it so requested dimensions are compared with what the files really contain.
16
+ */
17
+ export function inspectRepositoryFiles(language, files) {
18
+ const issues = [];
19
+ const sources = files.filter((file) => file.role === "source");
20
+ const tests = files.filter((file) => file.role === "test");
21
+ const sourcePaths = new Set(sources.map((file) => file.path));
22
+ const pathByModuleName = new Map();
23
+ if (language === "python") {
24
+ for (const source of sources)
25
+ pathByModuleName.set(pythonModuleName(source.path), source.path);
26
+ }
27
+ const symbolNames = [];
28
+ let patterns = 0;
29
+ const edges = new Set();
30
+ const outgoing = new Map(sources.map((file) => [file.path, []]));
31
+ for (const source of sources) {
32
+ for (const match of matchAll(language === "typescript" ? TS_SYMBOL : PY_SYMBOL, source.content))
33
+ symbolNames.push(match[1]);
34
+ patterns += matchAll(PATTERN, source.content).length;
35
+ for (const match of matchAll(language === "typescript" ? TS_IMPORT : PY_IMPORT, source.content)) {
36
+ const target = language === "typescript"
37
+ ? path.posix.join(path.posix.dirname(source.path), match[2]).replace(/\.js$/, ".ts")
38
+ : pathByModuleName.get(match[1]);
39
+ if (target === undefined || !sourcePaths.has(target)) {
40
+ issues.push(`${source.path}: import ${JSON.stringify(match[0])} does not resolve to a source module.`);
41
+ continue;
42
+ }
43
+ if (target === source.path)
44
+ issues.push(`${source.path}: imports itself.`);
45
+ const key = `${source.path} -> ${target}`;
46
+ if (edges.has(key))
47
+ issues.push(`${source.path}: duplicate import of ${target}.`);
48
+ edges.add(key);
49
+ outgoing.get(source.path)?.push(target);
50
+ }
51
+ }
52
+ for (const other of [...tests, ...files.filter((file) => file.role === "support")]) {
53
+ if (matchAll(PATTERN, other.content).length > 0)
54
+ issues.push(`${other.path}: ${other.role} file contains a repeated-pattern marker.`);
55
+ }
56
+ if (new Set(symbolNames).size !== symbolNames.length)
57
+ issues.push("duplicate sym_ definition names.");
58
+ const longest = new Map();
59
+ const visiting = new Set();
60
+ const visit = (node) => {
61
+ const known = longest.get(node);
62
+ if (known !== undefined)
63
+ return known;
64
+ if (visiting.has(node)) {
65
+ issues.push("import graph contains a cycle.");
66
+ return 0;
67
+ }
68
+ visiting.add(node);
69
+ let best = 1;
70
+ for (const next of outgoing.get(node) ?? [])
71
+ best = Math.max(best, 1 + visit(next));
72
+ visiting.delete(node);
73
+ longest.set(node, best);
74
+ return best;
75
+ };
76
+ let moduleDepth = 0;
77
+ for (const source of sources)
78
+ moduleDepth = Math.max(moduleDepth, visit(source.path));
79
+ return {
80
+ dimensions: {
81
+ sourceFileCount: sources.length,
82
+ moduleDepth,
83
+ internalImportCount: edges.size,
84
+ symbolCount: symbolNames.length,
85
+ testFileCount: tests.length,
86
+ repeatedPatternCount: patterns,
87
+ },
88
+ importEdges: [...edges].sort(),
89
+ symbolNames,
90
+ issues,
91
+ };
92
+ }
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Pure derivation of the generated repository layout from a plan. Globs list only roots that exist: the
3
+ * existing glob collector fails when a glob base directory is missing, so `tests/` appears only when the
4
+ * plan has test files. Support files (package.json, tsconfig.json, pyproject.toml) match no glob.
5
+ */
6
+ export function deriveSyntheticRepositoryLayout(plan) {
7
+ const hasTests = plan.requested.testFileCount > 0;
8
+ const sourceGlob = plan.language === "typescript" ? "src/**/*.ts" : "src/**/*.py";
9
+ return {
10
+ logicalTargetRoot: `synthetic/${plan.caseId}/repository`,
11
+ sourceRoots: hasTests ? ["src", "tests"] : ["src"],
12
+ testRoots: hasTests ? ["tests"] : [],
13
+ rawIncludeGlobs: hasTests ? [sourceGlob, "tests/**/*"] : [sourceGlob],
14
+ };
15
+ }
16
+ function describeFact(fact, moduleById, symbolById, testPathById) {
17
+ if (fact.role === "symbol-definition") {
18
+ const symbol = symbolById.get(fact.symbolIds[0]);
19
+ return `${symbol.name} is defined in ${moduleById.get(symbol.moduleId).path}.`;
20
+ }
21
+ if (fact.role === "import-relation") {
22
+ const edge = fact.edge;
23
+ return `${moduleById.get(edge.from).path} imports ${moduleById.get(edge.to).path}.`;
24
+ }
25
+ return `${testPathById.get(fact.testId)} exercises code in the task module.`;
26
+ }
27
+ /** Deterministic task title, query and fact prose from the plan's task and answer-key plan. */
28
+ export function describeSyntheticTask(plan) {
29
+ const moduleById = new Map(plan.modules.map((module) => [module.moduleId, module]));
30
+ const symbolById = new Map(plan.symbols.map((symbol) => [symbol.symbolId, symbol]));
31
+ const testPathById = new Map(plan.testFiles.map((test) => [test.testId, test.path]));
32
+ const names = plan.answerKey.expectedSymbols.join(", ");
33
+ const pathOf = (moduleId) => moduleById.get(moduleId).path;
34
+ let query;
35
+ if (plan.task.queryPlan.kind === "locate-symbols") {
36
+ query = `Locate the source file that defines the symbols ${names}.`;
37
+ }
38
+ else if (plan.task.queryPlan.kind === "trace-import") {
39
+ const edge = plan.task.relationEdge;
40
+ query = `Trace how ${pathOf(edge.from)} depends on ${pathOf(edge.to)} and identify the symbols ${names}.`;
41
+ }
42
+ else {
43
+ query = `Survey the repository and list the files and symbols (${names}) involved in a change that spans ${plan.task.ownerModuleIds.length} separate modules.`;
44
+ }
45
+ return {
46
+ title: `Synthetic ${plan.task.locality} task for ${plan.caseId}`,
47
+ query,
48
+ factTextById: new Map(plan.answerKey.facts.map((fact) => [fact.factId, describeFact(fact, moduleById, symbolById, testPathById)])),
49
+ };
50
+ }
@@ -0,0 +1,281 @@
1
+ import { createHash } from "node:crypto";
2
+ import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../../core/countTokens.js";
3
+ import { deriveSyntheticRepositoryLayout } from "./layout.js";
4
+ export const SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_ID = "my-dev-kit-lab-synthetic-repository-manifest-v1";
5
+ export const SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_VERSION = "1.0.0";
6
+ export const SYNTHETIC_REPOSITORY_MANIFEST_FILE_NAME = "synthetic-repository-manifest.json";
7
+ export const SYNTHETIC_REPOSITORY_REPOSITORY_DIRECTORY = "repository";
8
+ export function sha256Hex(bytes) {
9
+ return createHash("sha256").update(bytes).digest("hex");
10
+ }
11
+ export function describeContent(content) {
12
+ const bytes = Buffer.from(content, "utf8");
13
+ return {
14
+ byteLength: bytes.length,
15
+ charCount: countTextChars(content),
16
+ estimatedContentTokens: countEstimatedTokens(content),
17
+ sha256: sha256Hex(bytes),
18
+ };
19
+ }
20
+ export function describeRenderedFile(file) {
21
+ const metrics = describeContent(file.content);
22
+ return {
23
+ path: file.path,
24
+ role: file.role,
25
+ ...(file.language ? { language: file.language } : {}),
26
+ ...metrics,
27
+ };
28
+ }
29
+ /** SHA-256 over canonical JSON of the path-ordered {path, role, byteLength, sha256} records. */
30
+ export function computeRepositoryContentIdentity(records) {
31
+ const canonical = [...records]
32
+ .sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0))
33
+ .map((record) => ({ path: record.path, role: record.role, byteLength: record.byteLength, sha256: record.sha256 }));
34
+ return sha256Hex(Buffer.from(JSON.stringify(canonical), "utf8"));
35
+ }
36
+ /** Aggregate over files in the given order; the token estimate uses the existing estimator on the canonical concatenation. */
37
+ export function computeAggregate(records, concatenatedContent) {
38
+ return {
39
+ fileCount: records.length,
40
+ sourceFileCount: records.filter((record) => record.role === "source").length,
41
+ testFileCount: records.filter((record) => record.role === "test").length,
42
+ supportFileCount: records.filter((record) => record.role === "support").length,
43
+ totalBytes: records.reduce((total, record) => total + record.byteLength, 0),
44
+ totalChars: records.reduce((total, record) => total + record.charCount, 0),
45
+ estimatedContentTokens: countEstimatedTokens(concatenatedContent),
46
+ tokenCountMethod,
47
+ };
48
+ }
49
+ function dimensionsOf(source) {
50
+ return {
51
+ sourceFileCount: source.sourceFileCount,
52
+ moduleDepth: source.moduleDepth,
53
+ internalImportCount: source.internalImportCount,
54
+ symbolCount: source.symbolCount,
55
+ testFileCount: source.testFileCount,
56
+ repeatedPatternCount: source.repeatedPatternCount,
57
+ };
58
+ }
59
+ /** Pure manifest construction from a frozen plan and its rendered files (already in path order). No host data. */
60
+ export function buildSyntheticRepositoryManifest(plan, files) {
61
+ const records = files.map(describeRenderedFile);
62
+ const layout = deriveSyntheticRepositoryLayout(plan);
63
+ return {
64
+ schemaId: SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_ID,
65
+ schemaVersion: SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_VERSION,
66
+ generationIdentity: plan.generationIdentity,
67
+ caseId: plan.caseId,
68
+ language: plan.language,
69
+ taskLocality: plan.task.locality,
70
+ logicalTargetRoot: layout.logicalTargetRoot,
71
+ requestedDimensions: dimensionsOf(plan.requested),
72
+ realizedDimensions: dimensionsOf(plan.realized),
73
+ repositoryContentIdentity: computeRepositoryContentIdentity(records),
74
+ sourceRoots: layout.sourceRoots,
75
+ testRoots: layout.testRoots,
76
+ rawIncludeGlobs: layout.rawIncludeGlobs,
77
+ files: records,
78
+ aggregate: computeAggregate(records, files.map((file) => file.content).join("")),
79
+ task: {
80
+ taskId: plan.task.taskId,
81
+ expectedFiles: [...plan.answerKey.expectedFiles],
82
+ expectedSymbols: [...plan.answerKey.expectedSymbols],
83
+ expectedFactIds: plan.answerKey.facts.map((fact) => fact.factId),
84
+ },
85
+ };
86
+ }
87
+ /** Canonical manifest bytes: two-space JSON, LF, one trailing newline. */
88
+ export function serializeSyntheticRepositoryManifest(manifest) {
89
+ return `${JSON.stringify(manifest, null, 2)}\n`;
90
+ }
91
+ const HEX64 = /^[0-9a-f]{64}$/;
92
+ const FILE_KEYS = ["path", "role", "language", "byteLength", "charCount", "estimatedContentTokens", "sha256"];
93
+ const TOP_KEYS = [
94
+ "schemaId",
95
+ "schemaVersion",
96
+ "generationIdentity",
97
+ "caseId",
98
+ "language",
99
+ "taskLocality",
100
+ "logicalTargetRoot",
101
+ "requestedDimensions",
102
+ "realizedDimensions",
103
+ "repositoryContentIdentity",
104
+ "sourceRoots",
105
+ "testRoots",
106
+ "rawIncludeGlobs",
107
+ "files",
108
+ "aggregate",
109
+ "task",
110
+ ];
111
+ const DIMENSION_KEYS = ["sourceFileCount", "moduleDepth", "internalImportCount", "symbolCount", "testFileCount", "repeatedPatternCount"];
112
+ const AGGREGATE_KEYS = [
113
+ "fileCount",
114
+ "sourceFileCount",
115
+ "testFileCount",
116
+ "supportFileCount",
117
+ "totalBytes",
118
+ "totalChars",
119
+ "estimatedContentTokens",
120
+ "tokenCountMethod",
121
+ ];
122
+ function isRecord(value) {
123
+ return typeof value === "object" && value !== null && !Array.isArray(value);
124
+ }
125
+ function isCount(value) {
126
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
127
+ }
128
+ /** True for a plain relative POSIX logical path with no "." / ".." / empty segment, drive, UNC or backslash form. */
129
+ export function isSafeManifestPath(value) {
130
+ if (typeof value !== "string" || value.length === 0)
131
+ return false;
132
+ if (value.startsWith("/") || value.includes("\\") || value.includes(":") || value.includes("\0"))
133
+ return false;
134
+ return value.split("/").every((segment) => segment.length > 0 && segment !== "." && segment !== "..");
135
+ }
136
+ function checkKeys(label, value, allowed, required, issues) {
137
+ for (const key of Object.keys(value).sort())
138
+ if (!allowed.includes(key))
139
+ issues.push(`${label}: unknown field ${JSON.stringify(key)}.`);
140
+ for (const key of required)
141
+ if (!(key in value))
142
+ issues.push(`${label}: missing field ${key}.`);
143
+ }
144
+ function checkDimensions(label, value, issues) {
145
+ if (!isRecord(value)) {
146
+ issues.push(`${label}: must be an object.`);
147
+ return;
148
+ }
149
+ checkKeys(label, value, DIMENSION_KEYS, DIMENSION_KEYS, issues);
150
+ for (const key of DIMENSION_KEYS)
151
+ if (key in value && !isCount(value[key]))
152
+ issues.push(`${label}.${key}: must be a nonnegative integer.`);
153
+ }
154
+ /**
155
+ * Pure structural validation of an untrusted manifest value. Closed schema (unknown fields rejected),
156
+ * supported major version only, safe unique path-ordered file records, and aggregates that agree with the
157
+ * file records. Content-derived checks (hashes, tokens, identity of bytes) belong to the verifier.
158
+ */
159
+ export function validateSyntheticRepositoryManifest(input) {
160
+ const issues = [];
161
+ if (!isRecord(input))
162
+ return { ok: false, issues: ["manifest: must be an object."] };
163
+ checkKeys("manifest", input, TOP_KEYS, TOP_KEYS, issues);
164
+ if (input.schemaId !== SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_ID)
165
+ issues.push("manifest.schemaId: unsupported manifest schema.");
166
+ if (typeof input.schemaVersion !== "string" || !/^1\.\d+\.\d+$/.test(input.schemaVersion)) {
167
+ issues.push(`manifest.schemaVersion: unsupported schema major (received ${JSON.stringify(input.schemaVersion)}).`);
168
+ }
169
+ else if (input.schemaVersion !== SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_VERSION) {
170
+ issues.push(`manifest.schemaVersion: only ${SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_VERSION} is understood (received ${input.schemaVersion}).`);
171
+ }
172
+ for (const key of ["generationIdentity", "repositoryContentIdentity"]) {
173
+ if (typeof input[key] !== "string" || !HEX64.test(input[key]))
174
+ issues.push(`manifest.${key}: must be lowercase 64-hex SHA-256.`);
175
+ }
176
+ if (typeof input.caseId !== "string" || input.caseId.length === 0)
177
+ issues.push("manifest.caseId: must be a nonempty string.");
178
+ if (input.language !== "typescript" && input.language !== "python")
179
+ issues.push("manifest.language: must be typescript or python.");
180
+ if (!isSafeManifestPath(input.logicalTargetRoot))
181
+ issues.push("manifest.logicalTargetRoot: must be a safe relative logical path.");
182
+ checkDimensions("manifest.requestedDimensions", input.requestedDimensions, issues);
183
+ checkDimensions("manifest.realizedDimensions", input.realizedDimensions, issues);
184
+ for (const key of ["sourceRoots", "testRoots", "rawIncludeGlobs"]) {
185
+ const list = input[key];
186
+ if (!Array.isArray(list) || list.some((entry) => typeof entry !== "string" || entry.includes("\\") || entry.startsWith("/") || entry.includes(".."))) {
187
+ issues.push(`manifest.${key}: must be an array of safe relative strings.`);
188
+ }
189
+ }
190
+ const files = input.files;
191
+ if (!Array.isArray(files) || files.length === 0) {
192
+ issues.push("manifest.files: must be a nonempty array.");
193
+ }
194
+ else {
195
+ const seen = new Set();
196
+ let previous;
197
+ files.forEach((record, index) => {
198
+ const label = `manifest.files[${index}]`;
199
+ if (!isRecord(record)) {
200
+ issues.push(`${label}: must be an object.`);
201
+ return;
202
+ }
203
+ checkKeys(label, record, FILE_KEYS, FILE_KEYS.filter((key) => key !== "language"), issues);
204
+ if (!isSafeManifestPath(record.path)) {
205
+ issues.push(`${label}.path: must be a safe relative logical path.`);
206
+ }
207
+ else {
208
+ if (seen.has(record.path))
209
+ issues.push(`${label}.path: duplicate path ${record.path}.`);
210
+ seen.add(record.path);
211
+ if (previous !== undefined && record.path < previous)
212
+ issues.push(`${label}.path: files are not in ascending path order.`);
213
+ previous = record.path;
214
+ }
215
+ if (record.role !== "source" && record.role !== "test" && record.role !== "support")
216
+ issues.push(`${label}.role: must be source, test or support.`);
217
+ if ("language" in record && record.language !== "typescript" && record.language !== "python")
218
+ issues.push(`${label}.language: must be typescript or python.`);
219
+ for (const key of ["byteLength", "charCount", "estimatedContentTokens"])
220
+ if (!isCount(record[key]))
221
+ issues.push(`${label}.${key}: must be a nonnegative integer.`);
222
+ if (typeof record.sha256 !== "string" || !HEX64.test(record.sha256))
223
+ issues.push(`${label}.sha256: must be lowercase 64-hex SHA-256.`);
224
+ });
225
+ }
226
+ const aggregate = input.aggregate;
227
+ if (!isRecord(aggregate)) {
228
+ issues.push("manifest.aggregate: must be an object.");
229
+ }
230
+ else {
231
+ checkKeys("manifest.aggregate", aggregate, AGGREGATE_KEYS, AGGREGATE_KEYS, issues);
232
+ if (aggregate.tokenCountMethod !== tokenCountMethod)
233
+ issues.push(`manifest.aggregate.tokenCountMethod: must be ${tokenCountMethod}.`);
234
+ }
235
+ const task = input.task;
236
+ if (!isRecord(task)) {
237
+ issues.push("manifest.task: must be an object.");
238
+ }
239
+ else {
240
+ checkKeys("manifest.task", task, ["taskId", "expectedFiles", "expectedSymbols", "expectedFactIds"], ["taskId", "expectedFiles", "expectedSymbols", "expectedFactIds"], issues);
241
+ for (const key of ["expectedFiles", "expectedSymbols", "expectedFactIds"]) {
242
+ if (!Array.isArray(task[key]) || task[key].some((entry) => typeof entry !== "string"))
243
+ issues.push(`manifest.task.${key}: must be an array of strings.`);
244
+ }
245
+ if (Array.isArray(task.expectedFiles)) {
246
+ for (const expected of task.expectedFiles)
247
+ if (!isSafeManifestPath(expected))
248
+ issues.push(`manifest.task.expectedFiles: unsafe path ${JSON.stringify(expected)}.`);
249
+ }
250
+ }
251
+ if (issues.length > 0)
252
+ return { ok: false, issues };
253
+ const manifest = input;
254
+ const records = manifest.files;
255
+ const consistency = [];
256
+ const expectedAggregate = {
257
+ fileCount: records.length,
258
+ sourceFileCount: records.filter((record) => record.role === "source").length,
259
+ testFileCount: records.filter((record) => record.role === "test").length,
260
+ supportFileCount: records.filter((record) => record.role === "support").length,
261
+ totalBytes: records.reduce((total, record) => total + record.byteLength, 0),
262
+ totalChars: records.reduce((total, record) => total + record.charCount, 0),
263
+ };
264
+ for (const key of Object.keys(expectedAggregate)) {
265
+ if (manifest.aggregate[key] !== expectedAggregate[key])
266
+ consistency.push(`manifest.aggregate.${key}: ${manifest.aggregate[key]} does not match the file records (${expectedAggregate[key]}).`);
267
+ }
268
+ if (manifest.realizedDimensions.sourceFileCount !== expectedAggregate.sourceFileCount)
269
+ consistency.push("manifest.realizedDimensions.sourceFileCount does not match source file records.");
270
+ if (manifest.realizedDimensions.testFileCount !== expectedAggregate.testFileCount)
271
+ consistency.push("manifest.realizedDimensions.testFileCount does not match test file records.");
272
+ if (manifest.repositoryContentIdentity !== computeRepositoryContentIdentity(records))
273
+ consistency.push("manifest.repositoryContentIdentity does not match the file records.");
274
+ const paths = new Set(records.map((record) => record.path));
275
+ for (const expected of manifest.task.expectedFiles)
276
+ if (!paths.has(expected))
277
+ consistency.push(`manifest.task.expectedFiles: ${expected} is not a listed file.`);
278
+ if (consistency.length > 0)
279
+ return { ok: false, issues: consistency };
280
+ return { ok: true, issues: [], manifest };
281
+ }
@@ -0,0 +1,112 @@
1
+ import { lstatSync, readFileSync, readdirSync } from "node:fs";
2
+ import path from "node:path";
3
+ import { relativeWithinRoot, resolveWithinRoot } from "../../core/pathSafety.js";
4
+ import { inspectRepositoryFiles } from "./inspection.js";
5
+ import { computeAggregate, computeRepositoryContentIdentity, describeContent, validateSyntheticRepositoryManifest } from "./manifest.js";
6
+ const MAX_REPORTED_ISSUES = 50;
7
+ function walk(root, directory, found, issues) {
8
+ for (const entry of readdirSync(directory, { withFileTypes: true }).sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0))) {
9
+ const full = path.join(directory, entry.name);
10
+ if (entry.isSymbolicLink()) {
11
+ issues.push(`unexpected symbolic link ${relativeWithinRoot(root, full)}.`);
12
+ }
13
+ else if (entry.isDirectory()) {
14
+ walk(root, full, found, issues);
15
+ }
16
+ else {
17
+ found.push(relativeWithinRoot(root, full));
18
+ }
19
+ }
20
+ }
21
+ function bounded(issues) {
22
+ if (issues.length <= MAX_REPORTED_ISSUES)
23
+ return issues;
24
+ return [...issues.slice(0, MAX_REPORTED_ISSUES), `... ${issues.length - MAX_REPORTED_ISSUES} more issue(s) omitted.`];
25
+ }
26
+ /**
27
+ * Read-only verification of an existing materialization against its manifest. Re-measures every listed file
28
+ * (bytes, chars, estimated tokens, SHA-256), the repository content identity, the aggregate metrics and the
29
+ * realized dimensions (from the files' own content), checks that no unlisted file exists and that the answer-key
30
+ * files exist. Never writes, repairs or deletes. Issues are reported in manifest (path) order.
31
+ */
32
+ export function verifySyntheticRepositoryMaterialization(options) {
33
+ let parsed;
34
+ try {
35
+ parsed = JSON.parse(readFileSync(options.manifestPath, "utf8"));
36
+ }
37
+ catch (error) {
38
+ return { ok: false, issues: [`manifest cannot be read as JSON: ${error instanceof Error ? error.message : String(error)}`] };
39
+ }
40
+ const validation = validateSyntheticRepositoryManifest(parsed);
41
+ if (!validation.ok)
42
+ return { ok: false, issues: bounded(validation.issues) };
43
+ const manifest = validation.manifest;
44
+ const issues = [];
45
+ const inspectable = [];
46
+ const measured = [];
47
+ const contents = [];
48
+ for (const record of manifest.files) {
49
+ let absolute;
50
+ try {
51
+ absolute = resolveWithinRoot(options.repositoryRoot, record.path);
52
+ }
53
+ catch (error) {
54
+ issues.push(`${record.path}: ${error instanceof Error ? error.message : String(error)}`);
55
+ continue;
56
+ }
57
+ let content;
58
+ try {
59
+ const stats = lstatSync(absolute);
60
+ if (!stats.isFile()) {
61
+ issues.push(`${record.path}: is not a regular file.`);
62
+ continue;
63
+ }
64
+ content = readFileSync(absolute, "utf8");
65
+ }
66
+ catch {
67
+ issues.push(`${record.path}: listed file is missing or unreadable.`);
68
+ continue;
69
+ }
70
+ if (content.includes("\r") || content.startsWith(""))
71
+ issues.push(`${record.path}: contains a carriage return or byte-order mark.`);
72
+ const metrics = describeContent(content);
73
+ for (const key of ["byteLength", "charCount", "estimatedContentTokens", "sha256"]) {
74
+ if (metrics[key] !== record[key])
75
+ issues.push(`${record.path}: ${key} ${String(metrics[key])} does not match the manifest (${String(record[key])}).`);
76
+ }
77
+ measured.push({ ...record, ...metrics });
78
+ contents.push(content);
79
+ inspectable.push({ path: record.path, role: record.role, content });
80
+ }
81
+ const found = [];
82
+ try {
83
+ walk(options.repositoryRoot, options.repositoryRoot, found, issues);
84
+ }
85
+ catch {
86
+ issues.push("repository root cannot be enumerated.");
87
+ }
88
+ const listed = new Set(manifest.files.map((record) => record.path));
89
+ for (const file of found.sort())
90
+ if (!listed.has(file))
91
+ issues.push(`${file}: file exists but is not listed in the manifest.`);
92
+ if (issues.length === 0) {
93
+ if (computeRepositoryContentIdentity(measured) !== manifest.repositoryContentIdentity)
94
+ issues.push("repositoryContentIdentity does not match the files.");
95
+ const aggregate = computeAggregate(measured, contents.join(""));
96
+ for (const key of Object.keys(aggregate)) {
97
+ if (aggregate[key] !== manifest.aggregate[key])
98
+ issues.push(`aggregate.${key} ${String(aggregate[key])} does not match the manifest (${String(manifest.aggregate[key])}).`);
99
+ }
100
+ const inspection = inspectRepositoryFiles(manifest.language, inspectable);
101
+ issues.push(...inspection.issues);
102
+ for (const key of Object.keys(inspection.dimensions)) {
103
+ if (inspection.dimensions[key] !== manifest.realizedDimensions[key])
104
+ issues.push(`realized ${key} ${inspection.dimensions[key]} measured from files does not match the manifest (${manifest.realizedDimensions[key]}).`);
105
+ if (manifest.requestedDimensions[key] !== manifest.realizedDimensions[key])
106
+ issues.push(`requested ${key} ${manifest.requestedDimensions[key]} does not equal realized ${manifest.realizedDimensions[key]}.`);
107
+ }
108
+ }
109
+ if (issues.length > 0)
110
+ return { ok: false, issues: bounded(issues), manifest };
111
+ return { ok: true, issues: [], manifest };
112
+ }