@dailephd/my-dev-kit-lab 0.7.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +10 -5
  3. package/benchmarks/contracts/warm-index-benchmark-cases.json +330 -0
  4. package/dist/src/cli/help.js +27 -4
  5. package/dist/src/commands/runExperimentDescribeCommand.js +13 -2
  6. package/dist/src/commands/runExperimentRunCommand.js +210 -20
  7. package/dist/src/core/fileGlobs.js +44 -0
  8. package/dist/src/evaluation/benchmarkMetadata.js +13 -0
  9. package/dist/src/evaluation/localRepositorySubject/config.js +296 -0
  10. package/dist/src/evaluation/localRepositorySubject/evaluationCase.js +36 -0
  11. package/dist/src/evaluation/localRepositorySubject/expectedFiles.js +51 -0
  12. package/dist/src/evaluation/localRepositorySubject/gitRepository.js +128 -0
  13. package/dist/src/evaluation/localRepositorySubject/index.js +8 -0
  14. package/dist/src/evaluation/localRepositorySubject/inventory.js +127 -0
  15. package/dist/src/evaluation/localRepositorySubject/loadLocalRepositorySubject.js +48 -0
  16. package/dist/src/evaluation/localRepositorySubject/manifest.js +52 -0
  17. package/dist/src/evaluation/localRepositorySubject/types.js +55 -0
  18. package/dist/src/evaluation/projectFileTree.js +45 -1
  19. package/dist/src/evaluation/retrievalQuality/buildRetrievalEvidence.js +292 -0
  20. package/dist/src/evaluation/retrievalQuality/corpusCompleteness.js +122 -0
  21. package/dist/src/evaluation/retrievalQuality/factContextTargets.js +44 -0
  22. package/dist/src/evaluation/retrievalQuality/index.js +5 -0
  23. package/dist/src/evaluation/retrievalQuality/metrics.js +233 -0
  24. package/dist/src/evaluation/retrievalQuality/types.js +9 -0
  25. package/dist/src/evaluation/runMyDevKitRetrieval.js +6 -1
  26. package/dist/src/evaluation/runRawFullFileBaseline.js +18 -4
  27. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +29 -7
  28. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +4 -0
  29. package/dist/src/experiments/defaultRegistry.js +2 -0
  30. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectErrors.js +35 -0
  31. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExclusions.js +75 -0
  32. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js +136 -0
  33. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +128 -0
  34. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js +85 -0
  35. package/dist/src/experiments/plugins/contextWindowScaling/metadata.js +2 -2
  36. package/dist/src/experiments/plugins/contextWindowScaling/plugin.js +93 -0
  37. package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js +4 -9
  38. package/dist/src/experiments/plugins/index.js +1 -0
  39. package/dist/src/experiments/plugins/retrievalPrecisionRecall/config.js +39 -0
  40. package/dist/src/experiments/plugins/retrievalPrecisionRecall/execution.js +152 -0
  41. package/dist/src/experiments/plugins/retrievalPrecisionRecall/executionArtifact.js +15 -0
  42. package/dist/src/experiments/plugins/retrievalPrecisionRecall/index.js +9 -0
  43. package/dist/src/experiments/plugins/retrievalPrecisionRecall/localSubjectExecution.js +139 -0
  44. package/dist/src/experiments/plugins/retrievalPrecisionRecall/localSubjectPrivacy.js +119 -0
  45. package/dist/src/experiments/plugins/retrievalPrecisionRecall/metadata.js +21 -0
  46. package/dist/src/experiments/plugins/retrievalPrecisionRecall/metrics.js +158 -0
  47. package/dist/src/experiments/plugins/retrievalPrecisionRecall/plugin.js +253 -0
  48. package/dist/src/experiments/plugins/retrievalPrecisionRecall/types.js +8 -0
  49. package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -2
  50. package/dist/src/report/experiments/buildRetrievalPrecisionRecallReport.js +83 -0
  51. package/dist/src/report/experiments/index.js +4 -0
  52. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +7 -3
  53. package/dist/src/report/experiments/renderPluginExperimentReportText.js +9 -0
  54. package/dist/src/report/experiments/renderRetrievalPrecisionRecallHtml.js +96 -0
  55. package/dist/src/report/experiments/renderRetrievalPrecisionRecallText.js +121 -0
  56. package/dist/src/report/experiments/retrievalPrecisionRecallReportModel.js +3 -0
  57. package/dist/src/report/experiments/writePluginExperimentReports.js +8 -2
  58. package/docs/METRICS.md +34 -2
  59. package/package.json +1 -1
@@ -0,0 +1,5 @@
1
+ export * from "./types.js";
2
+ export * from "./buildRetrievalEvidence.js";
3
+ export * from "./factContextTargets.js";
4
+ export * from "./metrics.js";
5
+ export * from "./corpusCompleteness.js";
@@ -0,0 +1,233 @@
1
+ import { compareCodeUnits } from "../indexSnapshot.js";
2
+ import { normalizeRetrievedRepositoryPath } from "./buildRetrievalEvidence.js";
3
+ import { hasFactMapping, interpretFactContextTarget } from "./factContextTargets.js";
4
+ const sortedUnique = (values) => [...new Set(values)].sort(compareCodeUnits);
5
+ function ratio(numerator, denominator, notApplicableReason) {
6
+ if (denominator === 0) {
7
+ return { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: notApplicableReason };
8
+ }
9
+ return { availability: "available", numerator, denominator, value: numerator / denominator, reason: null };
10
+ }
11
+ function unavailableRatio(reason) {
12
+ return { availability: "unavailable", numerator: null, denominator: null, value: null, reason };
13
+ }
14
+ function normalizeExpectedFiles(values) {
15
+ if (!Array.isArray(values))
16
+ return { ok: false, reason: "expected-files-missing" };
17
+ const normalized = [];
18
+ for (const value of values) {
19
+ const path = normalizeRetrievedRepositoryPath(value);
20
+ if (path === null)
21
+ return { ok: false, reason: "unsafe-expected-file-identity" };
22
+ normalized.push(path);
23
+ }
24
+ return { ok: true, values: sortedUnique(normalized) };
25
+ }
26
+ function normalizeExpectedSymbols(values) {
27
+ if (!Array.isArray(values))
28
+ return { ok: false, reason: "expected-symbols-missing" };
29
+ if (values.some((value) => typeof value !== "string" || value.length === 0))
30
+ return { ok: false, reason: "invalid-expected-symbol" };
31
+ return { ok: true, values: sortedUnique(values) };
32
+ }
33
+ function sameSet(left, right) {
34
+ return left.length === right.length && left.every((value, index) => value === right[index]);
35
+ }
36
+ /**
37
+ * The answer key is the semantic source of relevance. A top-level expectation that is also present must agree
38
+ * with it after normalization; disagreement is never silently reconciled.
39
+ */
40
+ function resolveExpectation(topLevel, answerKeyValues, hasAnswerKey, normalize, mismatchReason) {
41
+ if (!hasAnswerKey)
42
+ return normalize(topLevel);
43
+ const fromKey = normalize(answerKeyValues);
44
+ if (!fromKey.ok)
45
+ return fromKey;
46
+ if (Array.isArray(topLevel)) {
47
+ const fromTop = normalize(topLevel);
48
+ if (!fromTop.ok)
49
+ return fromTop;
50
+ if (!sameSet(fromTop.values, fromKey.values))
51
+ return { ok: false, reason: mismatchReason };
52
+ }
53
+ return fromKey;
54
+ }
55
+ const expectationStatus = (resolved) => resolved.ok ? { availability: "available", reason: null } : { availability: "unavailable", reason: resolved.reason };
56
+ /** Symbol name -> normalized files it was retrieved from. Symbols without file identity cannot satisfy a file-specific target. */
57
+ function symbolFilesByName(evidence) {
58
+ const byName = new Map();
59
+ for (const symbol of evidence.symbols) {
60
+ if (symbol.file === undefined)
61
+ continue;
62
+ const files = byName.get(symbol.name) ?? new Set();
63
+ files.add(symbol.file);
64
+ byName.set(symbol.name, files);
65
+ }
66
+ return byName;
67
+ }
68
+ function calculateFactCoverage(answerKey, evidence, retrievedFiles) {
69
+ if (!answerKey || typeof answerKey !== "object")
70
+ return { kind: "unavailable", reason: "answer-key-missing" };
71
+ const facts = answerKey.expectedFacts;
72
+ if (!Array.isArray(facts))
73
+ return { kind: "unavailable", reason: "expected-facts-invalid" };
74
+ const factIds = [];
75
+ for (const fact of facts) {
76
+ const id = typeof fact === "object" && fact !== null ? fact.id : undefined;
77
+ if (typeof id !== "string" || id.length === 0 || factIds.includes(id)) {
78
+ return { kind: "unavailable", reason: "expected-facts-invalid" };
79
+ }
80
+ factIds.push(id);
81
+ }
82
+ if (factIds.length === 0)
83
+ return { kind: "not-applicable", reason: "no-expected-facts" };
84
+ const targets = answerKey.expectedContextTargets;
85
+ if (!Array.isArray(targets))
86
+ return { kind: "unavailable", reason: "fact-context-mapping-unavailable" };
87
+ const known = new Set(factIds);
88
+ const targetsByFact = new Map(factIds.map((id) => [id, []]));
89
+ for (const target of targets) {
90
+ if (!hasFactMapping(target))
91
+ continue;
92
+ const required = target.required !== false;
93
+ const interpretation = interpretFactContextTarget(target, known);
94
+ if (!interpretation.ok) {
95
+ // An invalid optional target cannot gate coverage; an invalid required one makes the mapping untrustworthy.
96
+ if (required)
97
+ return { kind: "unavailable", reason: "fact-context-mapping-invalid" };
98
+ continue;
99
+ }
100
+ if (!interpretation.target.required)
101
+ continue;
102
+ for (const id of interpretation.target.factIds)
103
+ targetsByFact.get(id)?.push(interpretation.target);
104
+ }
105
+ if (factIds.some((id) => (targetsByFact.get(id) ?? []).length === 0)) {
106
+ return { kind: "unavailable", reason: "fact-context-mapping-unavailable" };
107
+ }
108
+ const symbolFiles = symbolFilesByName(evidence);
109
+ const isCovered = (target) => retrievedFiles.has(target.file) && target.symbols.every((symbol) => symbolFiles.get(symbol)?.has(target.file) === true);
110
+ const covered = factIds.filter((id) => (targetsByFact.get(id) ?? []).every(isCovered));
111
+ const coveredSet = new Set(covered);
112
+ return {
113
+ kind: "computed",
114
+ covered: sortedUnique(covered),
115
+ uncovered: sortedUnique(factIds.filter((id) => !coveredSet.has(id))),
116
+ total: factIds.length
117
+ };
118
+ }
119
+ function evidenceReason(evidence) {
120
+ if (evidence === undefined)
121
+ return "retrieval-evidence-missing";
122
+ if (evidence.availability === "partial")
123
+ return "retrieval-evidence-partial";
124
+ if (evidence.availability === "unavailable")
125
+ return "retrieval-evidence-unavailable";
126
+ return null;
127
+ }
128
+ /**
129
+ * Pure per-case retrieval-quality calculation. Formulas are planner-frozen:
130
+ * file precision = |R_file ∩ E_file| / |R_file|; file recall = |R_file ∩ E_file| / |E_file|;
131
+ * irrelevant context ratio = |R_file - E_file| / |R_file|; symbol precision/recall use the same set formulas
132
+ * over exact unique symbol names; fact coverage = covered facts / expected facts, unweighted and derived only
133
+ * from explicit `factIds` target mappings. No input is mutated and nothing is read from disk or text.
134
+ * Unavailable or not-applicable results are never converted to zero.
135
+ */
136
+ export function calculateRetrievalQualityMetrics(input) {
137
+ const { evaluationCase, retrieval } = input;
138
+ const evidence = retrieval.retrievalEvidence;
139
+ const evidenceBlocked = evidenceReason(evidence);
140
+ const answerKey = evaluationCase.answerKey;
141
+ const hasAnswerKey = answerKey !== undefined && answerKey !== null && typeof answerKey === "object";
142
+ const expectedFiles = resolveExpectation(evaluationCase.expectedFiles, hasAnswerKey ? answerKey.expectedFiles : undefined, hasAnswerKey, normalizeExpectedFiles, "expected-files-answer-key-mismatch");
143
+ const expectedSymbols = resolveExpectation(evaluationCase.expectedSymbols, hasAnswerKey ? answerKey.expectedSymbols : undefined, hasAnswerKey, normalizeExpectedSymbols, "expected-symbols-answer-key-mismatch");
144
+ const usable = evidenceBlocked === null && evidence !== undefined ? evidence : undefined;
145
+ const retrievedFiles = usable ? sortedUnique(usable.files.map((file) => file.path)) : undefined;
146
+ const retrievedSymbols = usable ? sortedUnique(usable.symbols.map((symbol) => symbol.name)) : undefined;
147
+ // ---- files
148
+ let file = {
149
+ relevantRetrievedFiles: null,
150
+ irrelevantRetrievedFiles: null,
151
+ missedFiles: null,
152
+ missedFileCount: null,
153
+ precision: unavailableRatio(evidenceBlocked ?? (expectedFiles.ok ? "unavailable" : expectedFiles.reason)),
154
+ recall: unavailableRatio(evidenceBlocked ?? (expectedFiles.ok ? "unavailable" : expectedFiles.reason))
155
+ };
156
+ let irrelevantContextRatio = unavailableRatio(evidenceBlocked ?? (expectedFiles.ok ? "unavailable" : expectedFiles.reason));
157
+ if (retrievedFiles !== undefined && expectedFiles.ok) {
158
+ const expected = new Set(expectedFiles.values);
159
+ const retrieved = new Set(retrievedFiles);
160
+ const relevant = retrievedFiles.filter((path) => expected.has(path));
161
+ const irrelevant = retrievedFiles.filter((path) => !expected.has(path));
162
+ const missed = expectedFiles.values.filter((path) => !retrieved.has(path));
163
+ file = {
164
+ relevantRetrievedFiles: relevant,
165
+ irrelevantRetrievedFiles: irrelevant,
166
+ missedFiles: missed,
167
+ missedFileCount: missed.length,
168
+ precision: ratio(relevant.length, retrievedFiles.length, "no-retrieved-files"),
169
+ recall: ratio(relevant.length, expectedFiles.values.length, "no-expected-files")
170
+ };
171
+ irrelevantContextRatio = ratio(irrelevant.length, retrievedFiles.length, "no-retrieved-files");
172
+ }
173
+ // ---- symbols (exact unique names)
174
+ let symbol = {
175
+ relevantRetrievedSymbols: null,
176
+ irrelevantRetrievedSymbols: null,
177
+ missedSymbols: null,
178
+ missedSymbolCount: null,
179
+ precision: unavailableRatio(evidenceBlocked ?? (expectedSymbols.ok ? "unavailable" : expectedSymbols.reason)),
180
+ recall: unavailableRatio(evidenceBlocked ?? (expectedSymbols.ok ? "unavailable" : expectedSymbols.reason))
181
+ };
182
+ if (retrievedSymbols !== undefined && expectedSymbols.ok) {
183
+ const expected = new Set(expectedSymbols.values);
184
+ const retrieved = new Set(retrievedSymbols);
185
+ const relevant = retrievedSymbols.filter((name) => expected.has(name));
186
+ const missed = expectedSymbols.values.filter((name) => !retrieved.has(name));
187
+ symbol = {
188
+ relevantRetrievedSymbols: relevant,
189
+ irrelevantRetrievedSymbols: retrievedSymbols.filter((name) => !expected.has(name)),
190
+ missedSymbols: missed,
191
+ missedSymbolCount: missed.length,
192
+ precision: ratio(relevant.length, retrievedSymbols.length, "no-retrieved-symbols"),
193
+ recall: ratio(relevant.length, expectedSymbols.values.length, "no-expected-symbols")
194
+ };
195
+ }
196
+ // ---- facts
197
+ let fact = {
198
+ coveredFactIds: null,
199
+ uncoveredFactIds: null,
200
+ uncoveredFactCount: null,
201
+ coverage: unavailableRatio(evidenceBlocked ?? "unavailable")
202
+ };
203
+ if (usable !== undefined && retrievedFiles !== undefined) {
204
+ const outcome = calculateFactCoverage(answerKey, usable, new Set(retrievedFiles));
205
+ if (outcome.kind === "computed") {
206
+ fact = {
207
+ coveredFactIds: outcome.covered,
208
+ uncoveredFactIds: outcome.uncovered,
209
+ uncoveredFactCount: outcome.uncovered.length,
210
+ coverage: ratio(outcome.covered.length, outcome.total, "no-expected-facts")
211
+ };
212
+ }
213
+ else {
214
+ fact = { ...fact, coverage: outcome.kind === "unavailable" ? unavailableRatio(outcome.reason) : { availability: "not-applicable", numerator: null, denominator: null, value: null, reason: outcome.reason } };
215
+ }
216
+ }
217
+ const tokensValid = typeof retrieval.totalEstimatedTokens === "number" && Number.isFinite(retrieval.totalEstimatedTokens) && retrieval.totalEstimatedTokens >= 0;
218
+ return {
219
+ schemaVersion: "retrieval-quality-metrics-v1",
220
+ caseId: evaluationCase.id,
221
+ evidence: {
222
+ availability: evidence === undefined ? "missing" : evidence.availability,
223
+ reason: evidence === undefined ? "retrieval-evidence-missing" : (evidence.availabilityReason ?? null)
224
+ },
225
+ expectations: { files: expectationStatus(expectedFiles), symbols: expectationStatus(expectedSymbols) },
226
+ file,
227
+ symbol,
228
+ fact,
229
+ irrelevantContextRatio,
230
+ retrievedTokenCount: tokensValid ? retrieval.totalEstimatedTokens : null,
231
+ tokenCountMethod: tokensValid && typeof retrieval.tokenCountMethod === "string" ? retrieval.tokenCountMethod : null
232
+ };
233
+ }
@@ -0,0 +1,9 @@
1
+ /**
2
+ * Normalized retrieval-evidence contract (V1).
3
+ *
4
+ * Describes which files and symbols the existing my-dev-kit search -> lookup -> slice -> source
5
+ * lifecycle actually surfaced. It is observation only: it carries no relevance judgement and no
6
+ * score. It deliberately contains no source text, stdout/stderr bodies, or machine-local paths.
7
+ */
8
+ /** Retrieval command families in fixed lifecycle order. */
9
+ export const RETRIEVAL_COMMAND_FAMILIES = ["search", "lookup", "slice", "source"];
@@ -2,6 +2,7 @@ import path from "node:path";
2
2
  import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
3
3
  import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
4
4
  import { interpretToolVersionOutput } from "./indexSnapshot.js";
5
+ import { buildRetrievalEvidenceFromCommands } from "./retrievalQuality/buildRetrievalEvidence.js";
5
6
  const VERSION_PROBE_TIMEOUT_MS = 30_000;
6
7
  function parseJsonIfPossible(text) {
7
8
  try {
@@ -52,6 +53,7 @@ function skippedRetrieval(caseId, warnings, commands, durationMs) {
52
53
  contextText: "",
53
54
  filesRead: [],
54
55
  commands,
56
+ retrievalEvidence: buildRetrievalEvidenceFromCommands(commands),
55
57
  durationMs
56
58
  };
57
59
  }
@@ -177,6 +179,7 @@ export async function buildMyDevKitIndex(options) {
177
179
  "--root",
178
180
  options.target.absoluteTargetRoot,
179
181
  ...options.target.sourceRoots.flatMap((sourceRoot) => ["--src", sourceRoot]),
182
+ ...(options.excludePaths ?? []).flatMap((excludePath) => ["--exclude", excludePath]),
180
183
  "--out",
181
184
  options.indexDir,
182
185
  ...(mode.kind === "incremental" ? ["--incremental", "--refresh-scope", mode.refreshScope] : []),
@@ -336,6 +339,7 @@ export async function runMyDevKitRetrievalFromIndex(options) {
336
339
  selectedNodeId,
337
340
  selectedFile,
338
341
  selectedSymbol,
342
+ retrievalEvidence: buildRetrievalEvidenceFromCommands(commands, { nodeId: selectedNodeId, file: selectedFile }),
339
343
  durationMs: Date.now() - started
340
344
  };
341
345
  }
@@ -352,7 +356,8 @@ export async function runMyDevKitRetrieval(options) {
352
356
  kitCommand: options.kitCommand,
353
357
  indexDir,
354
358
  commandsDir,
355
- requireKit: options.requireKit
359
+ requireKit: options.requireKit,
360
+ excludePaths: options.excludePaths
356
361
  });
357
362
  if (!index.ok) {
358
363
  return skippedRetrieval(options.evaluationCase.id, [...index.warnings], [index.command], Date.now() - started);
@@ -1,7 +1,8 @@
1
- import { readFileSync, statSync } from "node:fs";
2
- import { collectFilesForGlobs } from "../core/fileGlobs.js";
1
+ import { lstatSync, readFileSync, statSync } from "node:fs";
2
+ import { collectFilesForGlobs, selectRelativePathsForGlobs } from "../core/fileGlobs.js";
3
+ import { resolveWithinRoot } from "../core/pathSafety.js";
3
4
  import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
4
- export async function runRawFullFileBaseline(evaluationCase) {
5
+ export async function runRawFullFileBaseline(evaluationCase, options = {}) {
5
6
  const started = Date.now();
6
7
  let stats;
7
8
  try {
@@ -13,7 +14,9 @@ export async function runRawFullFileBaseline(evaluationCase) {
13
14
  if (!stats.isDirectory()) {
14
15
  throw new Error(`Target root is not a directory: ${evaluationCase.targetRoot}`);
15
16
  }
16
- const files = collectFilesForGlobs(evaluationCase.absoluteTargetRoot, evaluationCase.rawIncludeGlobs);
17
+ const files = options.eligibleFiles === undefined
18
+ ? collectFilesForGlobs(evaluationCase.absoluteTargetRoot, evaluationCase.rawIncludeGlobs)
19
+ : selectEligibleFiles(evaluationCase, options.eligibleFiles);
17
20
  const contextText = files
18
21
  .map(({ absolutePath, relativePath }) => `=== FILE: ${relativePath} ===\n${readFileSync(absolutePath, "utf8")}\n`)
19
22
  .join("\n");
@@ -29,3 +32,14 @@ export async function runRawFullFileBaseline(evaluationCase) {
29
32
  durationMs: Date.now() - started
30
33
  };
31
34
  }
35
+ function selectEligibleFiles(evaluationCase, eligibleFiles) {
36
+ return selectRelativePathsForGlobs(eligibleFiles, evaluationCase.rawIncludeGlobs).map((relativePath) => {
37
+ const absolutePath = resolveWithinRoot(evaluationCase.absoluteTargetRoot, relativePath);
38
+ // The eligible list was built earlier; refuse a file that has since been swapped for a link or removed.
39
+ const stats = lstatSync(absolutePath, { throwIfNoEntry: false });
40
+ if (!stats || stats.isSymbolicLink() || !stats.isFile()) {
41
+ throw new Error(`Eligible file is no longer a regular file: ${relativePath}`);
42
+ }
43
+ return { absolutePath, relativePath };
44
+ });
45
+ }
@@ -18,7 +18,7 @@ function isInsideRoot(resolvedRoot, candidate) {
18
18
  const relative = path.relative(resolvedRoot, candidate);
19
19
  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
20
20
  }
21
- async function snapshotPathEntry(resolvedPath) {
21
+ async function snapshotPathEntry(resolvedPath, maxHashedFileBytes) {
22
22
  let stats;
23
23
  try {
24
24
  stats = await lstat(resolvedPath);
@@ -38,6 +38,15 @@ async function snapshotPathEntry(resolvedPath) {
38
38
  return { state: "directory", sha256: null, symbolicLinkTarget: null };
39
39
  }
40
40
  if (stats.isFile()) {
41
+ if (maxHashedFileBytes !== undefined && stats.size > maxHashedFileBytes) {
42
+ // Safe mode: a file above the bound is fingerprinted by metadata and its contents are never read.
43
+ return {
44
+ state: "file",
45
+ sha256: null,
46
+ symbolicLinkTarget: null,
47
+ contentFingerprint: { sizeBytes: stats.size, mtimeMs: stats.mtimeMs }
48
+ };
49
+ }
41
50
  const contents = await readFile(resolvedPath);
42
51
  return { state: "file", sha256: sha256Hex(contents), symbolicLinkTarget: null };
43
52
  }
@@ -69,8 +78,9 @@ async function runRequiredGit(resolvedRoot, args, targetRootPath) {
69
78
  : `git ${args.join(" ")} failed: ${result.message}`;
70
79
  return { ok: false, code, targetRootPath, message };
71
80
  }
72
- export async function captureTargetSnapshot(config) {
81
+ export async function captureTargetSnapshot(config, options = {}) {
73
82
  const resolvedTargetRootPath = path.resolve(config.targetRootPath);
83
+ const maxHashedFileBytes = options.externalLocalSafe?.maxHashedFileBytes;
74
84
  let rootStats;
75
85
  try {
76
86
  rootStats = await lstat(resolvedTargetRootPath);
@@ -113,13 +123,14 @@ export async function captureTargetSnapshot(config) {
113
123
  };
114
124
  }
115
125
  try {
116
- const entry = await snapshotPathEntry(resolvedPath);
126
+ const entry = await snapshotPathEntry(resolvedPath, maxHashedFileBytes);
117
127
  configuredFiles.push({
118
128
  relativePath,
119
129
  resolvedPath,
120
130
  state: entry.state,
121
131
  sha256: entry.sha256,
122
- symbolicLinkTarget: entry.symbolicLinkTarget
132
+ symbolicLinkTarget: entry.symbolicLinkTarget,
133
+ ...(entry.contentFingerprint ? { contentFingerprint: entry.contentFingerprint } : {})
123
134
  });
124
135
  }
125
136
  catch (error) {
@@ -183,15 +194,25 @@ export async function captureTargetSnapshot(config) {
183
194
  const untrackedFiles = [];
184
195
  for (const untrackedRelativePath of untrackedPaths) {
185
196
  const resolvedPath = path.resolve(resolvedTargetRootPath, untrackedRelativePath);
186
- const entry = await snapshotPathEntry(resolvedPath);
197
+ const entry = await snapshotPathEntry(resolvedPath, maxHashedFileBytes);
187
198
  const state = entry.state === "directory" ? "other" : entry.state;
188
199
  untrackedFiles.push({
189
200
  path: untrackedRelativePath,
190
201
  state,
191
202
  sha256: entry.sha256,
192
- symbolicLinkTarget: entry.symbolicLinkTarget
203
+ symbolicLinkTarget: entry.symbolicLinkTarget,
204
+ ...(entry.contentFingerprint ? { contentFingerprint: entry.contentFingerprint } : {})
193
205
  });
194
206
  }
207
+ let ignoredPaths;
208
+ if (maxHashedFileBytes !== undefined) {
209
+ // Safe mode: Lab-created files in ignored locations (for example a stray index directory) are invisible to
210
+ // `git status`, so record the ignored path set by name. Contents are never read.
211
+ const ignoredOutcome = await runRequiredGit(resolvedTargetRootPath, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], config.targetRootPath);
212
+ if (!ignoredOutcome.ok)
213
+ return ignoredOutcome;
214
+ ignoredPaths = ignoredOutcome.stdout.split("\0").filter((entry) => entry.length > 0).sort();
215
+ }
195
216
  const git = {
196
217
  availability: "available",
197
218
  branch,
@@ -199,7 +220,8 @@ export async function captureTargetSnapshot(config) {
199
220
  statusEntries,
200
221
  worktreeDiffSha256,
201
222
  stagedDiffSha256,
202
- untrackedFiles
223
+ untrackedFiles,
224
+ ...(ignoredPaths !== undefined ? { ignoredPaths } : {})
203
225
  };
204
226
  return {
205
227
  ok: true,
@@ -38,6 +38,10 @@ export function compareTargetSnapshots(before, after) {
38
38
  mutations.push(mutation(`git.untracked:${afterEntry.path}`, "git-untracked-file", `git.untrackedFiles.${afterEntry.path}`, null, afterEntry));
39
39
  }
40
40
  }
41
+ if ((before.git.ignoredPaths !== undefined || after.git.ignoredPaths !== undefined) &&
42
+ !isDeepStrictEqual(before.git.ignoredPaths ?? null, after.git.ignoredPaths ?? null)) {
43
+ mutations.push(mutation("git.ignored-paths", "git-ignored-paths", "git.ignoredPaths", (before.git.ignoredPaths ?? null), (after.git.ignoredPaths ?? null)));
44
+ }
41
45
  const afterConfiguredByPath = new Map(after.configuredFiles.map((entry) => [entry.relativePath, entry]));
42
46
  for (const beforeEntry of before.configuredFiles) {
43
47
  const afterEntry = afterConfiguredByPath.get(beforeEntry.relativePath);
@@ -2,6 +2,7 @@ import { ExperimentPluginRegistry } from "./registry.js";
2
2
  import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
3
3
  import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
4
4
  import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
5
+ import { retrievalPrecisionRecallPlugin } from "./plugins/retrievalPrecisionRecall/index.js";
5
6
  import { warmIndexReusePlugin } from "./plugins/warmIndexReuse/index.js";
6
7
  export function createDefaultExperimentPluginRegistry() {
7
8
  const registry = new ExperimentPluginRegistry();
@@ -9,5 +10,6 @@ export function createDefaultExperimentPluginRegistry() {
9
10
  registry.register(warmIndexReusePlugin);
10
11
  registry.register(incrementalChangeStalenessPlugin);
11
12
  registry.register(contextWindowScalingPlugin);
13
+ registry.register(retrievalPrecisionRecallPlugin);
12
14
  return registry;
13
15
  }
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Thrown when a local-subject execution did not complete cleanly. The primary code is the first issue in the
3
+ * ordered list (execution, mutation, snapshot, cleanup); every issue is retained so a secondary failure never
4
+ * masks the primary one. `caseEvidence` and `immutability` are runtime-only partial evidence for forensics and
5
+ * must not be persisted without the later privacy projection.
6
+ */
7
+ export class LocalSubjectExecutionError extends Error {
8
+ code;
9
+ issues;
10
+ caseEvidence;
11
+ immutability;
12
+ constructor(issues, partial = {}) {
13
+ if (issues.length === 0)
14
+ throw new Error("LocalSubjectExecutionError requires at least one issue.");
15
+ super(`Local subject execution failed (${issues[0].code}): ${issues.map((issue) => issue.message).join("; ")}`);
16
+ this.name = "LocalSubjectExecutionError";
17
+ this.code = issues[0].code;
18
+ this.issues = [...issues];
19
+ this.caseEvidence = partial.caseEvidence ?? null;
20
+ this.immutability = partial.immutability ?? null;
21
+ }
22
+ }
23
+ /** Replaces absolute private roots in a message so errors never carry the target or scratch location. */
24
+ export function redactPrivatePaths(message, privatePaths) {
25
+ let redacted = message;
26
+ for (const privatePath of privatePaths) {
27
+ if (!privatePath)
28
+ continue;
29
+ const variants = new Set([privatePath, privatePath.replace(/\\/g, "/"), privatePath.replace(/\//g, "\\")]);
30
+ for (const variant of variants) {
31
+ redacted = redacted.split(variant).join("<private-path>");
32
+ }
33
+ }
34
+ return redacted;
35
+ }
@@ -0,0 +1,75 @@
1
+ import { compareCodeUnits } from "../../../evaluation/localRepositorySubject/index.js";
2
+ import { LocalSubjectExecutionError } from "./localSubjectErrors.js";
3
+ /** Fail-closed ceilings so the exclusion arguments stay well inside Windows command-line limits. */
4
+ export const GUIDED_EXCLUSION_MAX_ENTRIES = 500;
5
+ export const GUIDED_EXCLUSION_MAX_CHARACTERS = 24_000;
6
+ /**
7
+ * Pure policy: turns Batch 1 runtime exclusion evidence into the exact `--exclude` paths for the guided index.
8
+ *
9
+ * Each excluded file is covered by the shallowest ancestor directory that contains no eligible file (so a fully
10
+ * ignored directory becomes one entry), otherwise by the file path itself. Bare names are never produced because
11
+ * my-dev-kit treats a bare name as "every directory with this name", which would drop eligible material. If the
12
+ * result is too large the call throws instead of truncating: an incomplete exclusion list would be unsafe.
13
+ */
14
+ export function deriveGuidedIndexExclusions(input) {
15
+ const eligibleDirectories = new Set();
16
+ const eligibleSegments = new Set();
17
+ for (const eligible of input.eligibleFiles) {
18
+ const segments = eligible.split("/");
19
+ segments.forEach((segment) => eligibleSegments.add(segment));
20
+ for (let depth = 1; depth < segments.length; depth += 1) {
21
+ eligibleDirectories.add(segments.slice(0, depth).join("/"));
22
+ }
23
+ }
24
+ const candidates = new Set();
25
+ for (const excluded of input.excludedFiles) {
26
+ const segments = excluded.split("/");
27
+ let covering = excluded;
28
+ for (let depth = 1; depth < segments.length; depth += 1) {
29
+ const directory = segments.slice(0, depth).join("/");
30
+ if (!eligibleDirectories.has(directory)) {
31
+ covering = directory;
32
+ break;
33
+ }
34
+ }
35
+ // A single-segment entry is a name match in my-dev-kit (any depth), and "./name" is silently ignored, so it is
36
+ // only safe when no eligible path uses that name. Otherwise descend to a multi-segment path, which is exact.
37
+ if (!covering.includes("/") && eligibleSegments.has(covering)) {
38
+ if (covering !== excluded && segments.length > 2) {
39
+ covering = segments.slice(0, 2).join("/");
40
+ }
41
+ else if (covering !== excluded) {
42
+ covering = excluded;
43
+ }
44
+ else {
45
+ throw new LocalSubjectExecutionError([
46
+ {
47
+ code: "GUIDED_EXCLUSION_UNREPRESENTABLE",
48
+ message: `an excluded top-level entry shares its name with eligible files and cannot be excluded exactly (${excluded}).`,
49
+ },
50
+ ]);
51
+ }
52
+ }
53
+ candidates.add(covering);
54
+ }
55
+ const exclusions = [...candidates]
56
+ .filter((candidate) => {
57
+ const segments = candidate.split("/");
58
+ for (let depth = 1; depth < segments.length; depth += 1) {
59
+ if (candidates.has(segments.slice(0, depth).join("/")))
60
+ return false;
61
+ }
62
+ return true;
63
+ })
64
+ .sort(compareCodeUnits);
65
+ const characters = exclusions.reduce((total, entry) => total + entry.length + 1, 0);
66
+ if (exclusions.length > GUIDED_EXCLUSION_MAX_ENTRIES || characters > GUIDED_EXCLUSION_MAX_CHARACTERS) {
67
+ throw new LocalSubjectExecutionError([
68
+ {
69
+ code: "GUIDED_EXCLUSION_LIMIT",
70
+ message: `guided index exclusions exceed the safe limit (${exclusions.length} entries, ${characters} characters).`,
71
+ },
72
+ ]);
73
+ }
74
+ return exclusions;
75
+ }