@dailephd/my-dev-kit-lab 0.8.1 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +10 -1
  2. package/README.md +5 -4
  3. package/dist/src/cli/help.js +16 -1
  4. package/dist/src/commands/runExperimentDescribeCommand.js +8 -1
  5. package/dist/src/commands/runExperimentRunCommand.js +90 -3
  6. package/dist/src/evaluation/runMyDevKitRetrieval.js +1 -0
  7. package/dist/src/experiments/defaultRegistry.js +2 -0
  8. package/dist/src/experiments/plugins/contextPackGeneration/analysis.js +157 -0
  9. package/dist/src/experiments/plugins/contextPackGeneration/analysisArtifact.js +22 -0
  10. package/dist/src/experiments/plugins/contextPackGeneration/analysisTypes.js +1 -0
  11. package/dist/src/experiments/plugins/contextPackGeneration/buildContextPack.js +84 -0
  12. package/dist/src/experiments/plugins/contextPackGeneration/config.js +39 -0
  13. package/dist/src/experiments/plugins/contextPackGeneration/execution.js +361 -0
  14. package/dist/src/experiments/plugins/contextPackGeneration/executionArtifact.js +17 -0
  15. package/dist/src/experiments/plugins/contextPackGeneration/executionTypes.js +11 -0
  16. package/dist/src/experiments/plugins/contextPackGeneration/identityEvidence.js +71 -0
  17. package/dist/src/experiments/plugins/contextPackGeneration/index.js +19 -0
  18. package/dist/src/experiments/plugins/contextPackGeneration/localSubjectExecution.js +146 -0
  19. package/dist/src/experiments/plugins/contextPackGeneration/localSubjectPrivacy.js +276 -0
  20. package/dist/src/experiments/plugins/contextPackGeneration/metadata.js +24 -0
  21. package/dist/src/experiments/plugins/contextPackGeneration/metrics.js +46 -0
  22. package/dist/src/experiments/plugins/contextPackGeneration/packArtifact.js +17 -0
  23. package/dist/src/experiments/plugins/contextPackGeneration/packEvidence.js +152 -0
  24. package/dist/src/experiments/plugins/contextPackGeneration/packSelectionPolicy.js +305 -0
  25. package/dist/src/experiments/plugins/contextPackGeneration/plugin.js +355 -0
  26. package/dist/src/experiments/plugins/contextPackGeneration/renderContextPack.js +61 -0
  27. package/dist/src/experiments/plugins/contextPackGeneration/types.js +29 -0
  28. package/dist/src/report/experiments/buildContextPackGenerationReport.js +171 -0
  29. package/dist/src/report/experiments/buildPluginExperimentReport.js +24 -2
  30. package/dist/src/report/experiments/contextPackGenerationReportModel.js +9 -0
  31. package/dist/src/report/experiments/index.js +5 -0
  32. package/dist/src/report/experiments/loadContextPackArtifacts.js +43 -0
  33. package/dist/src/report/experiments/renderContextPackGenerationHtml.js +122 -0
  34. package/dist/src/report/experiments/renderContextPackGenerationText.js +101 -0
  35. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
  36. package/dist/src/report/experiments/renderPluginExperimentReportText.js +7 -0
  37. package/dist/src/report/experiments/renderRetrievalQueryStrategyComparisonHtml.js +15 -15
  38. package/dist/src/report/experiments/writePluginExperimentReports.js +2 -0
  39. package/docs/METRICS.md +28 -1
  40. package/package.json +1 -1
@@ -0,0 +1,276 @@
1
+ import { compareCodeUnits } from "../../../evaluation/indexSnapshot.js";
2
+ import { redactedFileList } from "../contextWindowScaling/localSubjectPrivacy.js";
3
+ import { CONTEXT_PACK_GENERATION_ERROR_TEXT, CONTEXT_PACK_IDENTITY_REDACTION_SCHEMA_VERSION } from "./executionTypes.js";
4
+ import { collectContextPackFileIdentities } from "./localSubjectExecution.js";
5
+ /** Single fixed placeholder for a withheld case title. */
6
+ export const CONTEXT_PACK_REDACTED_CASE_TITLE = "<redacted case title>";
7
+ /** The one V1 marker stamped on every externally projected case. Every class it names is absent from durable output. */
8
+ export const CONTEXT_PACK_IDENTITY_REDACTION = {
9
+ schemaVersion: CONTEXT_PACK_IDENTITY_REDACTION_SCHEMA_VERSION,
10
+ fileIdentities: "redacted",
11
+ symbolIdentities: "redacted",
12
+ sourceText: "redacted",
13
+ callRelationships: "redacted",
14
+ testIdentities: "redacted",
15
+ factIdentities: "redacted",
16
+ taskText: "redacted",
17
+ warningText: "redacted",
18
+ semanticNodeIds: "redacted",
19
+ caseTitle: "redacted"
20
+ };
21
+ export const CONTEXT_PACK_PRIVACY_FAILURE_MESSAGE = "External-local context-pack-generation privacy projection failed; no durable output was written.";
22
+ export const CONTEXT_PACK_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE = "Local subject context-pack-generation failed (EXECUTION_FAILED): execution failed before completing (details withheld).";
23
+ export const CONTEXT_PACK_EXTERNAL_PERSISTENCE_FAILURE_MESSAGE = "External-local context-pack-generation artifact persistence failed; attempted files were removed.";
24
+ /** Fragments shorter than this (trimmed) are never searched: they collide with ordinary words and fixed text. */
25
+ export const CONTEXT_PACK_SOURCE_FRAGMENT_MIN_LENGTH = 8;
26
+ /** Upper bound on distinct source fragments retained for the leak assertion. */
27
+ export const CONTEXT_PACK_SOURCE_FRAGMENT_MAX_COUNT = 5000;
28
+ /** Identity names shorter than this are matched only as whole string values, never as substrings. */
29
+ export const CONTEXT_PACK_NAME_SUBSTRING_MIN_LENGTH = 8;
30
+ const sectionTruncatedCount = (pack, sectionId) => {
31
+ if (pack === null)
32
+ return undefined;
33
+ if (sectionId === "sourceSlices")
34
+ return pack.sourceSlices.filter((slice) => slice.truncated).length;
35
+ return undefined;
36
+ };
37
+ function projectTreatment(treatment, pack) {
38
+ return {
39
+ treatmentId: treatment.treatmentId,
40
+ status: treatment.status,
41
+ availability: treatment.availability,
42
+ availabilityReason: treatment.availabilityReason,
43
+ size: treatment.size === null ? null : { ...treatment.size },
44
+ // Identity evidence is the calculator input only; the durable artifact carries counts, never identities.
45
+ identityEvidence: null,
46
+ includedFiles: [],
47
+ identityCounts: treatment.identityEvidence === null
48
+ ? null
49
+ : {
50
+ files: treatment.identityEvidence.files.length,
51
+ symbols: treatment.identityEvidence.symbols.length,
52
+ includedFiles: treatment.includedFiles.length
53
+ },
54
+ // Node IDs are semantic identities; kind, outcome and fixed reason survive.
55
+ steps: treatment.steps.map((step) => ({ kind: step.kind, nodeId: null, sourceMode: step.sourceMode, succeeded: step.succeeded, reason: step.reason })),
56
+ sections: treatment.sections === null
57
+ ? null
58
+ : treatment.sections.map((section) => {
59
+ const truncatedCount = sectionTruncatedCount(pack, section.id);
60
+ return { ...section, ...(truncatedCount === undefined ? {} : { truncatedCount }) };
61
+ }),
62
+ evidenceNotes: [...treatment.evidenceNotes],
63
+ packArtifactPath: null,
64
+ errors: treatment.errors.map((error) => ({ code: error.code, message: CONTEXT_PACK_GENERATION_ERROR_TEXT[error.code] }))
65
+ };
66
+ }
67
+ /**
68
+ * Projects identity-bearing execution evidence into privacy-safe durable evidence. It performs no metric calculation and
69
+ * changes no count, size, availability or status. The in-memory pack is read only to count truncated slices.
70
+ */
71
+ export function projectContextPackExecutionForExternalLocalPersistence(results) {
72
+ return results.map((result) => {
73
+ const evidence = result.evidence;
74
+ return {
75
+ caseId: evidence.caseId,
76
+ caseName: CONTEXT_PACK_REDACTED_CASE_TITLE,
77
+ benchmarkProject: evidence.benchmarkProject,
78
+ taskLocality: evidence.taskLocality,
79
+ treatments: evidence.treatments.map((treatment) => projectTreatment(treatment, treatment.treatmentId === "context-pack" ? result.pack : null)),
80
+ identityRedaction: { ...CONTEXT_PACK_IDENTITY_REDACTION }
81
+ };
82
+ });
83
+ }
84
+ const placeholders = (kind, count) => Array.from({ length: count }, (_, index) => `<redacted ${kind} ${index + 1}>`);
85
+ const filePlaceholders = (list) => (list === null ? null : redactedFileList(list.length));
86
+ const symbolPlaceholders = (list) => (list === null ? null : placeholders("symbol", list.length));
87
+ const factPlaceholders = (list) => (list === null ? null : placeholders("fact", list.length));
88
+ /** Only identity lists change; every number, availability and fixed-vocabulary reason is carried through untouched. */
89
+ function projectQuality(quality) {
90
+ return {
91
+ ...quality,
92
+ file: {
93
+ ...quality.file,
94
+ relevantRetrievedFiles: filePlaceholders(quality.file.relevantRetrievedFiles),
95
+ irrelevantRetrievedFiles: filePlaceholders(quality.file.irrelevantRetrievedFiles),
96
+ missedFiles: filePlaceholders(quality.file.missedFiles)
97
+ },
98
+ symbol: {
99
+ ...quality.symbol,
100
+ relevantRetrievedSymbols: symbolPlaceholders(quality.symbol.relevantRetrievedSymbols),
101
+ irrelevantRetrievedSymbols: symbolPlaceholders(quality.symbol.irrelevantRetrievedSymbols),
102
+ missedSymbols: symbolPlaceholders(quality.symbol.missedSymbols)
103
+ },
104
+ fact: {
105
+ ...quality.fact,
106
+ coveredFactIds: factPlaceholders(quality.fact.coveredFactIds),
107
+ uncoveredFactIds: factPlaceholders(quality.fact.uncoveredFactIds)
108
+ }
109
+ };
110
+ }
111
+ /**
112
+ * Preserves every numeric scientific value; replaces only the private identity lists inside per-treatment quality.
113
+ * F1, deltas, savings, scope means and matched-case membership are never recalculated.
114
+ */
115
+ export function projectContextPackAnalysisForExternalLocalPersistence(analysis) {
116
+ const cloned = structuredClone(analysis);
117
+ return {
118
+ cases: cloned.cases.map((entry) => ({
119
+ ...entry,
120
+ treatments: entry.treatments.map((treatment) => ({
121
+ ...treatment,
122
+ quality: treatment.quality === null ? null : projectQuality(treatment.quality)
123
+ }))
124
+ })),
125
+ scopes: cloned.scopes
126
+ };
127
+ }
128
+ /** Distinct non-trivial source lines of a pack, in code-unit order and bounded. Never returns short tokens. */
129
+ export function collectContextPackSourceFragments(packs) {
130
+ const fragments = new Set();
131
+ for (const pack of packs) {
132
+ if (pack === null)
133
+ continue;
134
+ for (const slice of pack.sourceSlices) {
135
+ for (const line of slice.text.split(/\r?\n/)) {
136
+ const trimmed = line.trim();
137
+ if (trimmed.length >= CONTEXT_PACK_SOURCE_FRAGMENT_MIN_LENGTH)
138
+ fragments.add(trimmed);
139
+ }
140
+ }
141
+ }
142
+ return [...fragments].sort(compareCodeUnits).slice(0, CONTEXT_PACK_SOURCE_FRAGMENT_MAX_COUNT);
143
+ }
144
+ const variantsOf = (value) => {
145
+ const forward = value.replace(/\\/g, "/");
146
+ const backward = value.replace(/\//g, "\\");
147
+ return [value, forward, backward].flatMap((entry) => [entry, JSON.stringify(entry).slice(1, -1)]);
148
+ };
149
+ const baseName = (file) => file.slice(file.lastIndexOf("/") + 1);
150
+ /**
151
+ * Fail-closed defense in depth over a complete would-be durable payload. Roots, file paths, node IDs, task text,
152
+ * warnings and non-trivial source fragments are searched as substrings. Symbol names, fact IDs and file base names are
153
+ * searched only when at least CONTEXT_PACK_NAME_SUBSTRING_MIN_LENGTH characters long: shorter names collide with fixed
154
+ * schema vocabulary (for example a symbol called `search`) and would make ordinary repositories unrunnable. The structural
155
+ * projection, not this search, is what removes short identities.
156
+ */
157
+ export function assertContextPackExternalProjectionIsPrivate(projected, privateValues) {
158
+ const serialized = typeof projected === "string" ? projected : JSON.stringify(projected);
159
+ const substringCandidates = [
160
+ ...privateValues.privateRoots,
161
+ ...privateValues.filePaths,
162
+ ...privateValues.nodeIds,
163
+ ...privateValues.taskTexts,
164
+ ...privateValues.warnings,
165
+ ...privateValues.sourceFragments.filter((fragment) => fragment.trim().length >= CONTEXT_PACK_SOURCE_FRAGMENT_MIN_LENGTH),
166
+ ...[...privateValues.symbolNames, ...privateValues.factIds, ...privateValues.filePaths.map(baseName)].filter((name) => name.length >= CONTEXT_PACK_NAME_SUBSTRING_MIN_LENGTH)
167
+ ].filter((value) => value.length > 0);
168
+ if (substringCandidates.some((value) => variantsOf(value).some((variant) => serialized.includes(variant)))) {
169
+ throw new Error(CONTEXT_PACK_PRIVACY_FAILURE_MESSAGE);
170
+ }
171
+ }
172
+ const GENERIC_ISSUE_TEXT = {
173
+ WORK_ROOT_INSIDE_TARGET: "the output directory must be outside the local subject repository",
174
+ BEFORE_SNAPSHOT_FAILED: "the target snapshot before execution could not be captured",
175
+ AFTER_SNAPSHOT_FAILED: "the target snapshot after execution could not be captured",
176
+ SCRATCH_CLEANUP_FAILED: "the private scratch could not be removed",
177
+ GUIDED_EXCLUSION_LIMIT: "the guided-index exclusions exceeded the safe limit",
178
+ GUIDED_EXCLUSION_UNREPRESENTABLE: "the guided-index exclusions could not be expressed exactly"
179
+ };
180
+ /**
181
+ * Safe, persistable description of an external-local failure: issue codes, counts and mutation kinds only. Execution
182
+ * issue messages are fixed text by construction; nothing else (no path, identity, title, or caught text) can appear.
183
+ */
184
+ export function describeContextPackLocalSubjectFailureForPersistence(error) {
185
+ const parts = [];
186
+ const seen = new Set();
187
+ for (const issue of error.issues) {
188
+ if (issue.code === "EXECUTION_FAILED" || issue.code === "GROUND_TRUTH_INVALID" || issue.code === "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE") {
189
+ if (!seen.has(issue.message)) {
190
+ seen.add(issue.message);
191
+ parts.push(issue.message.replace(/\.$/, ""));
192
+ }
193
+ continue;
194
+ }
195
+ if (seen.has(issue.code))
196
+ continue;
197
+ seen.add(issue.code);
198
+ if (issue.code === "TARGET_MUTATED") {
199
+ const kinds = [...new Set((error.immutability?.mutations ?? []).map((mutation) => mutation.kind))].sort();
200
+ parts.push(`the target changed during execution${kinds.length > 0 ? ` (${kinds.join(", ")})` : ""}; it was not restored`);
201
+ }
202
+ else {
203
+ parts.push(GENERIC_ISSUE_TEXT[issue.code] ?? `issue ${issue.code}`);
204
+ }
205
+ }
206
+ return `Local subject context-pack-generation failed (${error.code}): ${parts.join("; ")}.`;
207
+ }
208
+ function collectNodeIds(value, into) {
209
+ if (Array.isArray(value))
210
+ for (const entry of value)
211
+ collectNodeIds(entry, into);
212
+ else if (value && typeof value === "object") {
213
+ for (const [key, nested] of Object.entries(value)) {
214
+ if ((key === "nodeId" || key === "fromNodeId" || key === "toNodeId") && typeof nested === "string")
215
+ into.add(nested);
216
+ else
217
+ collectNodeIds(nested, into);
218
+ }
219
+ }
220
+ }
221
+ /**
222
+ * Gathers the real private identities of one external-local run, before projection, for the fail-closed assertion.
223
+ * Everything is read from in-memory data already produced; nothing is read from disk or the repository.
224
+ */
225
+ export function collectContextPackExternalPrivateValues(args) {
226
+ const { subject, results } = args;
227
+ const filePaths = new Set([
228
+ ...subject.eligibleFiles,
229
+ ...subject.runtimeSafetyExclusions.gitIgnoredFiles,
230
+ ...subject.runtimeSafetyExclusions.oversizedFiles
231
+ ]);
232
+ const symbolNames = new Set();
233
+ const nodeIds = new Set();
234
+ const taskTexts = new Set();
235
+ const factIds = new Set();
236
+ for (const evaluationCase of subject.evaluationCases) {
237
+ taskTexts.add(evaluationCase.title);
238
+ taskTexts.add(evaluationCase.query);
239
+ for (const name of evaluationCase.expectedSymbols)
240
+ symbolNames.add(name);
241
+ for (const file of evaluationCase.expectedFiles)
242
+ filePaths.add(file);
243
+ for (const fact of evaluationCase.answerKey?.expectedFacts ?? evaluationCase.expectedFacts ?? [])
244
+ factIds.add(fact.id);
245
+ }
246
+ for (const result of results) {
247
+ for (const file of collectContextPackFileIdentities(result))
248
+ filePaths.add(file);
249
+ for (const treatment of result.evidence.treatments) {
250
+ for (const symbol of treatment.identityEvidence?.symbols ?? [])
251
+ symbolNames.add(symbol.name);
252
+ collectNodeIds(treatment.steps, nodeIds);
253
+ }
254
+ if (result.pack !== null) {
255
+ taskTexts.add(result.pack.task.title);
256
+ taskTexts.add(result.pack.task.summary);
257
+ for (const symbol of result.pack.symbols)
258
+ symbolNames.add(symbol.name);
259
+ for (const slice of result.pack.sourceSlices) {
260
+ if (slice.symbolName !== null)
261
+ symbolNames.add(slice.symbolName);
262
+ }
263
+ collectNodeIds(result.pack, nodeIds);
264
+ }
265
+ }
266
+ return {
267
+ privateRoots: [subject.repositoryRoot, args.outputRoot],
268
+ filePaths: [...filePaths].sort(compareCodeUnits),
269
+ symbolNames: [...symbolNames].sort(compareCodeUnits),
270
+ nodeIds: [...nodeIds].sort(compareCodeUnits),
271
+ taskTexts: [...taskTexts].sort(compareCodeUnits),
272
+ factIds: [...factIds].sort(compareCodeUnits),
273
+ warnings: [],
274
+ sourceFragments: collectContextPackSourceFragments(results.map((result) => result.pack))
275
+ };
276
+ }
@@ -0,0 +1,24 @@
1
+ export const CONTEXT_PACK_GENERATION_PLUGIN_ID = "context-pack-generation";
2
+ export const CONTEXT_PACK_GENERATION_TREATMENT_IDS = ["raw-full-file", "context-pack"];
3
+ export const contextPackGenerationMetadata = {
4
+ id: CONTEXT_PACK_GENERATION_PLUGIN_ID,
5
+ name: "Context Pack Generation",
6
+ description: "Compare a deterministic Lab-owned experimental context pack against raw full-file context on matched bundled benchmark cases without coding agents.",
7
+ schemaVersion: "1.0.0",
8
+ status: "experimental",
9
+ supportedTargets: ["self", "external-local"],
10
+ supportedOutputs: ["json", "html", "text", "artifact"]
11
+ };
12
+ /** Exactly two mandatory treatments, in fixed order. There is no treatment selector. */
13
+ export const CONTEXT_PACK_GENERATION_VARIANTS = [
14
+ {
15
+ id: "raw-full-file",
16
+ name: "Raw full-file context",
17
+ description: "Include every eligible file matched by the case raw include globs as raw text."
18
+ },
19
+ {
20
+ id: "context-pack",
21
+ name: "Context pack",
22
+ description: "Compose a bounded, deterministic pack of task, relevant files and symbols, source slices, call relationships, tests, and evidence notes from my-dev-kit evidence."
23
+ }
24
+ ];
@@ -0,0 +1,46 @@
1
+ import { CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "./metadata.js";
2
+ /** Projects already-calculated scientific values onto generic metrics. Owns no formula. */
3
+ export function toContextPackGenerationOutcomeMetrics(treatment, caseId) {
4
+ const base = { variantId: treatment.treatmentId, caseId };
5
+ const quality = treatment.quality;
6
+ const ratio = (id, name, metric) => ({ ...base, id, name, value: metric?.value ?? null, unit: "ratio" });
7
+ const count = (id, name, value) => ({ ...base, id, name, value: value ?? null, unit: "count" });
8
+ return [
9
+ ratio("file-precision", "File precision", quality?.file.precision),
10
+ ratio("file-recall", "File recall", quality?.file.recall),
11
+ ratio("file-f1", "File F1", treatment.fileF1),
12
+ ratio("symbol-precision", "Symbol precision", quality?.symbol.precision),
13
+ ratio("symbol-recall", "Symbol recall", quality?.symbol.recall),
14
+ ratio("symbol-f1", "Symbol F1", treatment.symbolF1),
15
+ ratio("fact-coverage", "Fact coverage", quality?.fact.coverage),
16
+ ratio("irrelevant-context-ratio", "Irrelevant context ratio", quality?.irrelevantContextRatio),
17
+ { ...base, id: "estimated-token-count", name: "Estimated token count", value: treatment.estimatedTokens, unit: "tokens" },
18
+ count("missed-file-count", "Missed file count", quality?.file.missedFileCount),
19
+ count("missed-symbol-count", "Missed symbol count", quality?.symbol.missedSymbolCount),
20
+ count("uncovered-fact-count", "Uncovered fact count", quality?.fact.uncoveredFactCount)
21
+ ];
22
+ }
23
+ export function toContextPackGenerationCaseComparisonMetrics(caseAnalysis) {
24
+ const { comparison, caseId } = caseAnalysis;
25
+ return [
26
+ { id: "file-f1-delta", name: "File F1 delta (pack minus raw)", value: comparison.fileF1Delta, unit: "ratio", caseId },
27
+ { id: "symbol-f1-delta", name: "Symbol F1 delta (pack minus raw)", value: comparison.symbolF1Delta, unit: "ratio", caseId },
28
+ { id: "fact-coverage-delta", name: "Fact coverage delta (pack minus raw)", value: comparison.factCoverageDelta, unit: "ratio", caseId },
29
+ { id: "estimated-token-delta", name: "Estimated token delta (pack minus raw)", value: comparison.estimatedTokenDelta, unit: "tokens", caseId },
30
+ { id: "tokens-saved", name: "Tokens saved (raw minus pack)", value: comparison.tokensSaved, unit: "tokens", caseId },
31
+ { id: "percent-saved", name: "Percent saved", value: comparison.percentSaved, unit: "percent", caseId }
32
+ ];
33
+ }
34
+ /** Per scope: matched case count, then four objective means per treatment in fixed order. No best/rank/score/front metric. */
35
+ export function toContextPackGenerationRunMetrics(analysis) {
36
+ const metrics = [];
37
+ for (const scope of analysis.scopes) {
38
+ const { scopeId } = scope;
39
+ metrics.push({ id: `${scopeId}-comparison-case-count`, name: `${scopeId} matched comparison cases`, value: scope.includedCaseCount, unit: "count" });
40
+ for (const treatmentId of CONTEXT_PACK_GENERATION_TREATMENT_IDS) {
41
+ const objectives = scope.treatmentSummaries.find((summary) => summary.treatmentId === treatmentId)?.objectives ?? null;
42
+ metrics.push({ id: `${scopeId}-mean-file-f1`, name: `${scopeId} mean file F1`, value: objectives?.meanFileF1 ?? null, unit: "ratio", variantId: treatmentId }, { id: `${scopeId}-mean-symbol-f1`, name: `${scopeId} mean symbol F1`, value: objectives?.meanSymbolF1 ?? null, unit: "ratio", variantId: treatmentId }, { id: `${scopeId}-mean-fact-coverage`, name: `${scopeId} mean fact coverage`, value: objectives?.meanFactCoverage ?? null, unit: "ratio", variantId: treatmentId }, { id: `${scopeId}-mean-estimated-token-count`, name: `${scopeId} mean estimated token count`, value: objectives?.meanEstimatedTokenCount ?? null, unit: "tokens", variantId: treatmentId });
43
+ }
44
+ }
45
+ return metrics;
46
+ }
@@ -0,0 +1,17 @@
1
+ import { taskOutputSegment } from "../warmIndexReuse/selection.js";
2
+ export const CONTEXT_PACK_ARTIFACT_DIRECTORY = "packs";
3
+ export const CONTEXT_PACK_ARTIFACT_SUFFIX = ".context-pack.json";
4
+ /**
5
+ * Relative path of a case's bundled pack artifact. The case segment comes from the established safe output-segment helper;
6
+ * a raw case id is never used as a path.
7
+ */
8
+ export function contextPackArtifactRelativePath(caseId) {
9
+ return `${CONTEXT_PACK_ARTIFACT_DIRECTORY}/${taskOutputSegment(caseId)}${CONTEXT_PACK_ARTIFACT_SUFFIX}`;
10
+ }
11
+ /** The per-case artifact is the complete bundled pack (schema my-dev-kit-lab-context-pack-experiment-v1); no envelope, no clock. */
12
+ export function buildContextPackArtifact(pack) {
13
+ return structuredClone(pack);
14
+ }
15
+ export function serializeContextPackArtifact(pack) {
16
+ return `${JSON.stringify(buildContextPackArtifact(pack), null, 2)}\n`;
17
+ }
@@ -0,0 +1,152 @@
1
+ /**
2
+ * Pure interpretation of already-captured my-dev-kit JSON output (search, lookup, slice, source) and of the validated
3
+ * symbol index. No I/O, no process execution. Anything not matching the expected shape is reported as malformed rather
4
+ * than guessed.
5
+ */
6
+ import { interpretSymbolIndexFiles } from "../../../evaluation/indexSnapshot.js";
7
+ import { MAX_SOURCE_LINES_PER_SLICE } from "./packSelectionPolicy.js";
8
+ const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
9
+ const asString = (value) => (typeof value === "string" && value.length > 0 ? value : null);
10
+ const asLine = (value) => (typeof value === "number" && Number.isInteger(value) && value >= 1 ? value : null);
11
+ function parseJson(stdout) {
12
+ try {
13
+ return JSON.parse(stdout);
14
+ }
15
+ catch {
16
+ return undefined;
17
+ }
18
+ }
19
+ function parseNode(value) {
20
+ if (!isRecord(value))
21
+ return null;
22
+ const id = asString(value.id);
23
+ const kind = asString(value.kind);
24
+ if (id === null || kind === null)
25
+ return null;
26
+ return { id, kind, path: asString(value.path), symbolName: asString(value.symbolName) ?? asString(value.label), line: asLine(value.line) };
27
+ }
28
+ function parseEdges(value) {
29
+ if (value === undefined)
30
+ return [];
31
+ if (!Array.isArray(value))
32
+ return null;
33
+ const edges = [];
34
+ for (const entry of value) {
35
+ if (!isRecord(entry))
36
+ return null;
37
+ const source = asString(entry.source);
38
+ const target = asString(entry.target);
39
+ const kind = asString(entry.kind);
40
+ if (source === null || target === null || kind === null)
41
+ return null;
42
+ edges.push({ source, target, kind });
43
+ }
44
+ return edges;
45
+ }
46
+ /** Preserves upstream order: rank is the 1-based position in `results`. Only file and symbol hits carry a seed node. */
47
+ export function parseSearchHits(stdout, limit) {
48
+ const payload = parseJson(stdout);
49
+ if (!isRecord(payload) || !Array.isArray(payload.results))
50
+ return { ok: false };
51
+ const hits = [];
52
+ for (const [index, entry] of payload.results.slice(0, limit).entries()) {
53
+ if (!isRecord(entry))
54
+ return { ok: false };
55
+ const kind = asString(entry.kind);
56
+ const nodeId = asString(entry.nodeId) ?? asString(entry.id);
57
+ if (kind === null)
58
+ return { ok: false };
59
+ if ((kind !== "file" && kind !== "symbol") || nodeId === null)
60
+ continue;
61
+ hits.push({ rank: index + 1, kind, nodeId, path: asString(entry.path), label: asString(entry.label) });
62
+ }
63
+ return { ok: true, value: hits };
64
+ }
65
+ export function parseLookup(stdout) {
66
+ const payload = parseJson(stdout);
67
+ if (!isRecord(payload))
68
+ return { ok: false };
69
+ const node = parseNode(payload.node);
70
+ return node === null ? { ok: false } : { ok: true, value: { node } };
71
+ }
72
+ export function parseSlice(stdout) {
73
+ const payload = parseJson(stdout);
74
+ if (!isRecord(payload) || !Array.isArray(payload.nodes))
75
+ return { ok: false };
76
+ const nodes = [];
77
+ for (const entry of payload.nodes) {
78
+ const node = parseNode(entry);
79
+ if (node === null)
80
+ return { ok: false };
81
+ nodes.push(node);
82
+ }
83
+ const edges = parseEdges(payload.edges);
84
+ return edges === null ? { ok: false } : { ok: true, value: { nodes, edges } };
85
+ }
86
+ export function parseSource(stdout) {
87
+ const payload = parseJson(stdout);
88
+ if (!isRecord(payload) || payload.status !== "ok" || typeof payload.content !== "string")
89
+ return { ok: false };
90
+ const startLine = asLine(payload.startLine);
91
+ const endLine = asLine(payload.endLine);
92
+ if (startLine === null || endLine === null || endLine < startLine)
93
+ return { ok: false };
94
+ const cursor = isRecord(payload.continuationCursor) ? payload.continuationCursor : null;
95
+ return {
96
+ ok: true,
97
+ value: {
98
+ startLine,
99
+ endLine,
100
+ content: payload.content,
101
+ eof: cursor?.eof === true,
102
+ boundaryKnown: !(cursor !== null && (cursor.symbolBoundaryKnown === false || cursor.reason === "symbol-end-unknown"))
103
+ }
104
+ };
105
+ }
106
+ /** Returns null when the symbol-index contract cannot be interpreted; symbol evidence is then unavailable. */
107
+ export function interpretSymbolIndex(symbolIndex) {
108
+ if (!interpretSymbolIndexFiles(symbolIndex, null).ok)
109
+ return null;
110
+ const files = [];
111
+ const lineCountByFile = new Map();
112
+ for (const entry of symbolIndex.files) {
113
+ if (!isRecord(entry) || typeof entry.path !== "string" || !Array.isArray(entry.symbols))
114
+ return null;
115
+ const symbols = [];
116
+ for (const symbol of entry.symbols) {
117
+ if (!isRecord(symbol) || typeof symbol.name !== "string" || !isRecord(symbol.location))
118
+ return null;
119
+ const line = asLine(symbol.location.line);
120
+ if (line === null)
121
+ return null;
122
+ symbols.push({ name: symbol.name, location: { line } });
123
+ }
124
+ files.push({ path: entry.path, symbols });
125
+ if (typeof entry.lineCount === "number" && Number.isInteger(entry.lineCount) && entry.lineCount >= 1)
126
+ lineCountByFile.set(entry.path, entry.lineCount);
127
+ }
128
+ return { files, lineCountByFile };
129
+ }
130
+ /**
131
+ * Index-derived bounded range for a symbol: it starts at the definition line and ends one line before the next indexed
132
+ * symbol of the same file (or at the file end), then is capped at MAX_SOURCE_LINES_PER_SLICE lines. Returns null when the
133
+ * index cannot establish a reliable boundary, in which case the caller falls back to bounded source by node.
134
+ */
135
+ export function deriveSymbolSourceRange(index, file, startLine) {
136
+ const entry = index.files.find((candidate) => candidate.path === file);
137
+ if (!entry || !entry.symbols.some((symbol) => symbol.location.line === startLine))
138
+ return null;
139
+ const next = entry.symbols.map((symbol) => symbol.location.line).filter((line) => line > startLine).sort((a, b) => a - b)[0];
140
+ const fileEnd = index.lineCountByFile.get(file);
141
+ const naturalEndLine = next !== undefined ? next - 1 : fileEnd;
142
+ if (naturalEndLine === undefined || naturalEndLine < startLine)
143
+ return null;
144
+ const endLine = Math.min(naturalEndLine, startLine + MAX_SOURCE_LINES_PER_SLICE - 1);
145
+ return { startLine, endLine, naturalEndLine, capped: naturalEndLine > endLine };
146
+ }
147
+ /** Finds a symbol definition line in the index by file and exact name (first definition in line order). */
148
+ export function findIndexedSymbolLine(index, file, name) {
149
+ const entry = index.files.find((candidate) => candidate.path === file);
150
+ const lines = entry?.symbols.filter((symbol) => symbol.name === name).map((symbol) => symbol.location.line).sort((a, b) => a - b);
151
+ return lines && lines.length > 0 ? lines[0] : null;
152
+ }