@dailephd/my-dev-kit-lab 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +6 -5
- package/dist/src/cli/help.js +15 -4
- package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
- package/dist/src/commands/runExperimentRunCommand.js +117 -18
- package/dist/src/core/fileGlobs.js +44 -0
- package/dist/src/evaluation/localRepositorySubject/config.js +296 -0
- package/dist/src/evaluation/localRepositorySubject/evaluationCase.js +36 -0
- package/dist/src/evaluation/localRepositorySubject/expectedFiles.js +51 -0
- package/dist/src/evaluation/localRepositorySubject/gitRepository.js +128 -0
- package/dist/src/evaluation/localRepositorySubject/index.js +8 -0
- package/dist/src/evaluation/localRepositorySubject/inventory.js +127 -0
- package/dist/src/evaluation/localRepositorySubject/loadLocalRepositorySubject.js +48 -0
- package/dist/src/evaluation/localRepositorySubject/manifest.js +52 -0
- package/dist/src/evaluation/localRepositorySubject/types.js +55 -0
- package/dist/src/evaluation/projectFileTree.js +45 -1
- package/dist/src/evaluation/runMyDevKitRetrieval.js +3 -1
- package/dist/src/evaluation/runRawFullFileBaseline.js +18 -4
- package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +29 -7
- package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +4 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectErrors.js +35 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExclusions.js +75 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js +136 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +128 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js +85 -0
- package/dist/src/experiments/plugins/contextWindowScaling/metadata.js +2 -2
- package/dist/src/experiments/plugins/contextWindowScaling/plugin.js +93 -0
- package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js +4 -9
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +4 -3
- package/dist/src/report/experiments/writePluginExperimentReports.js +8 -2
- package/docs/METRICS.md +2 -0
- package/package.json +1 -1
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { redactPrivatePaths } from "./localSubjectErrors.js";
|
|
3
|
+
/** Value used wherever a durable external-local artifact would otherwise show a machine-local path. */
|
|
4
|
+
export const EXTERNAL_LOCAL_REDACTED_VALUE = "[redacted]";
|
|
5
|
+
const REDACTED_FILE_TEXT = "<redacted file>";
|
|
6
|
+
/** One opaque placeholder per file: counts stay exact and an empty list still means "no files". */
|
|
7
|
+
export function redactedFileList(count) {
|
|
8
|
+
return Array.from({ length: count }, (_, index) => `<redacted file ${index + 1}>`);
|
|
9
|
+
}
|
|
10
|
+
/** Replaces known repository-relative paths (longest first) and private roots inside free text. */
|
|
11
|
+
export function redactKnownPaths(text, knownFiles, privateRoots = []) {
|
|
12
|
+
let redacted = redactPrivatePaths(text, privateRoots);
|
|
13
|
+
for (const file of [...knownFiles].sort((left, right) => right.length - left.length)) {
|
|
14
|
+
if (file.length === 0)
|
|
15
|
+
continue;
|
|
16
|
+
// Native tool errors on Windows may spell a repository-relative path with backslashes.
|
|
17
|
+
for (const variant of new Set([file, file.replace(/\//g, "\\")]))
|
|
18
|
+
redacted = redacted.split(variant).join(REDACTED_FILE_TEXT);
|
|
19
|
+
}
|
|
20
|
+
return redacted;
|
|
21
|
+
}
|
|
22
|
+
function redactStrings(value, redact) {
|
|
23
|
+
if (typeof value === "string")
|
|
24
|
+
return redact(value);
|
|
25
|
+
if (Array.isArray(value))
|
|
26
|
+
return value.map((entry) => redactStrings(entry, redact));
|
|
27
|
+
if (value && typeof value === "object") {
|
|
28
|
+
return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, redactStrings(nested, redact)]));
|
|
29
|
+
}
|
|
30
|
+
return value;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Projects execution evidence for durable persistence of an external-local run. File identity lists become
|
|
34
|
+
* placeholder lists of the same length (with an explicit per-treatment flag) and every free-text field has known
|
|
35
|
+
* repository paths and private roots replaced. Scientific numbers, statuses and identifiers are untouched.
|
|
36
|
+
*/
|
|
37
|
+
export function projectEvidenceForExternalLocalPersistence(evidence, privacy) {
|
|
38
|
+
const redact = (text) => redactKnownPaths(text, privacy.knownFiles, privacy.privateRoots);
|
|
39
|
+
return evidence.map((caseEvidence) => {
|
|
40
|
+
const cloned = structuredClone(caseEvidence);
|
|
41
|
+
return {
|
|
42
|
+
...redactStrings(cloned, redact),
|
|
43
|
+
treatments: cloned.treatments.map((treatment) => {
|
|
44
|
+
const safe = redactStrings(treatment, redact);
|
|
45
|
+
return {
|
|
46
|
+
...safe,
|
|
47
|
+
context: {
|
|
48
|
+
...safe.context,
|
|
49
|
+
observedFiles: treatment.context.observedFiles === null ? null : redactedFileList(treatment.context.observedFiles.length),
|
|
50
|
+
},
|
|
51
|
+
relevantFileEvidence: {
|
|
52
|
+
...safe.relevantFileEvidence,
|
|
53
|
+
expectedRelevantFiles: redactedFileList(treatment.relevantFileEvidence.expectedRelevantFiles.length),
|
|
54
|
+
omittedRelevantFiles: redactedFileList(treatment.relevantFileEvidence.omittedRelevantFiles.length),
|
|
55
|
+
},
|
|
56
|
+
fileIdentityRedaction: "redacted",
|
|
57
|
+
};
|
|
58
|
+
}),
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
/** Durable target identity for an external-local run: logical identity and Git identity, no filesystem facts. */
|
|
63
|
+
export function projectExternalLocalTarget(manifest) {
|
|
64
|
+
return {
|
|
65
|
+
kind: "external-local",
|
|
66
|
+
targetRoot: manifest.logicalTargetRoot,
|
|
67
|
+
toolRoot: EXTERNAL_LOCAL_REDACTED_VALUE,
|
|
68
|
+
packageName: null,
|
|
69
|
+
packageVersion: null,
|
|
70
|
+
hasPackageJson: false,
|
|
71
|
+
hasLockfile: false,
|
|
72
|
+
branch: manifest.repository.branch,
|
|
73
|
+
commit: manifest.repository.commit,
|
|
74
|
+
hasGit: true,
|
|
75
|
+
isSelf: false,
|
|
76
|
+
privacyProjection: "external-local-redacted",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/** Replaces target and machine-local metadata of a run before any report is serialized. */
|
|
80
|
+
export function projectRunForExternalLocalPersistence(run, target) {
|
|
81
|
+
const metadata = { ...(run.metadata ?? {}) };
|
|
82
|
+
if ("outputRoot" in metadata)
|
|
83
|
+
metadata.outputRoot = EXTERNAL_LOCAL_REDACTED_VALUE;
|
|
84
|
+
if (typeof metadata.executionArtifactPath === "string")
|
|
85
|
+
metadata.executionArtifactPath = path.basename(metadata.executionArtifactPath);
|
|
86
|
+
return { ...run, target, metadata: metadata };
|
|
87
|
+
}
|
|
88
|
+
const GENERIC_ISSUE_TEXT = {
|
|
89
|
+
WORK_ROOT_INSIDE_TARGET: "the output directory must be outside the local subject repository",
|
|
90
|
+
BEFORE_SNAPSHOT_FAILED: "the target snapshot before execution could not be captured",
|
|
91
|
+
AFTER_SNAPSHOT_FAILED: "the target snapshot after execution could not be captured",
|
|
92
|
+
SCRATCH_CLEANUP_FAILED: "the private scratch could not be removed",
|
|
93
|
+
GUIDED_EXCLUSION_LIMIT: "the guided-index exclusions exceeded the safe limit",
|
|
94
|
+
GUIDED_EXCLUSION_UNREPRESENTABLE: "the guided-index exclusions could not be expressed exactly",
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Safe, persistable description of a local-subject execution failure: issue codes, case and treatment identifiers,
|
|
98
|
+
* error codes and mutation kinds only. Repository file names, mutation identifiers and the full comparison are
|
|
99
|
+
* never included.
|
|
100
|
+
*/
|
|
101
|
+
export function describeLocalSubjectFailureForPersistence(error) {
|
|
102
|
+
const parts = [];
|
|
103
|
+
const seen = new Set();
|
|
104
|
+
for (const issue of error.issues) {
|
|
105
|
+
if (seen.has(issue.code))
|
|
106
|
+
continue;
|
|
107
|
+
seen.add(issue.code);
|
|
108
|
+
if (issue.code === "EXECUTION_FAILED") {
|
|
109
|
+
const details = [];
|
|
110
|
+
for (const caseEvidence of error.caseEvidence ?? []) {
|
|
111
|
+
for (const treatment of caseEvidence.treatments) {
|
|
112
|
+
if (treatment.status === "failed" || treatment.errors.length > 0) {
|
|
113
|
+
details.push(`case ${caseEvidence.caseId} treatment ${treatment.variantId}: ${treatment.errors.map((entry) => entry.code).join(", ") || "failed"}`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
parts.push(details.length > 0 ? `execution failed (${details.join("; ")})` : "execution failed before completing (details withheld)");
|
|
118
|
+
}
|
|
119
|
+
else if (issue.code === "TARGET_MUTATED") {
|
|
120
|
+
const kinds = [...new Set((error.immutability?.mutations ?? []).map((mutation) => mutation.kind))].sort();
|
|
121
|
+
parts.push(`the target changed during execution${kinds.length > 0 ? ` (${kinds.join(", ")})` : ""}; it was not restored`);
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
parts.push(GENERIC_ISSUE_TEXT[issue.code] ?? `issue ${issue.code}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return `Local subject execution failed (${error.code}): ${parts.join("; ")}.`;
|
|
128
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { realpath as nativeRealpath } from "node:fs";
|
|
2
|
+
import { lstat, mkdir, mkdtemp, rm } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { relativeWithinRoot } from "../../../core/pathSafety.js";
|
|
6
|
+
import { LocalSubjectExecutionError } from "./localSubjectErrors.js";
|
|
7
|
+
const realpath = promisify(nativeRealpath.native);
|
|
8
|
+
export const defaultPrivateScratchIo = {
|
|
9
|
+
removeDirectory: (directory) => rm(directory, { recursive: true, force: true }),
|
|
10
|
+
};
|
|
11
|
+
/** Physical path of a path that may not exist yet: realpath of the deepest existing ancestor plus the remainder. */
|
|
12
|
+
async function resolvePhysicalPath(candidate) {
|
|
13
|
+
const resolved = path.resolve(candidate);
|
|
14
|
+
const remainder = [];
|
|
15
|
+
let current = resolved;
|
|
16
|
+
for (;;) {
|
|
17
|
+
try {
|
|
18
|
+
const physical = await realpath(current);
|
|
19
|
+
return remainder.length === 0 ? physical : path.join(physical, ...remainder.reverse());
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
const code = error.code;
|
|
23
|
+
const parent = path.dirname(current);
|
|
24
|
+
if ((code !== "ENOENT" && code !== "ENOTDIR") || parent === current)
|
|
25
|
+
return resolved;
|
|
26
|
+
remainder.push(path.basename(current));
|
|
27
|
+
current = parent;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function isSameOrInside(root, candidate) {
|
|
32
|
+
try {
|
|
33
|
+
relativeWithinRoot(root, candidate);
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/** True when both paths resolve to the same physical directory (follows links; case-insensitive on Windows). */
|
|
41
|
+
export async function isSamePhysicalDirectory(left, right) {
|
|
42
|
+
const [physicalLeft, physicalRight] = await Promise.all([resolvePhysicalPath(left), resolvePhysicalPath(right)]);
|
|
43
|
+
return path.relative(physicalLeft, physicalRight) === "";
|
|
44
|
+
}
|
|
45
|
+
/** Rejects a Lab-owned work root that equals or lies inside the target. Performs no writes. */
|
|
46
|
+
export async function assertWorkRootOutsideTarget(workRoot, targetRoot) {
|
|
47
|
+
const physicalTarget = await resolvePhysicalPath(targetRoot);
|
|
48
|
+
const physicalWorkRoot = await resolvePhysicalPath(workRoot);
|
|
49
|
+
if (isSameOrInside(physicalTarget, physicalWorkRoot)) {
|
|
50
|
+
throw new LocalSubjectExecutionError([
|
|
51
|
+
{ code: "WORK_ROOT_INSIDE_TARGET", message: "the Lab work root must be outside the local subject repository." },
|
|
52
|
+
]);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Creates a short run-owned private scratch directory under the work root and re-verifies it is outside the target. */
|
|
56
|
+
export async function createPrivateScratch(workRoot, targetRoot) {
|
|
57
|
+
await mkdir(path.resolve(workRoot), { recursive: true });
|
|
58
|
+
const scratchPath = await mkdtemp(path.join(await realpath(path.resolve(workRoot)), "s-"));
|
|
59
|
+
if (isSameOrInside(await resolvePhysicalPath(targetRoot), await resolvePhysicalPath(scratchPath))) {
|
|
60
|
+
await rm(scratchPath, { recursive: true, force: true });
|
|
61
|
+
throw new LocalSubjectExecutionError([
|
|
62
|
+
{ code: "WORK_ROOT_INSIDE_TARGET", message: "the private scratch directory resolved inside the local subject repository." },
|
|
63
|
+
]);
|
|
64
|
+
}
|
|
65
|
+
return { path: scratchPath };
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Removes the scratch and verifies it is gone. Returns a failure description instead of throwing so the caller can
|
|
69
|
+
* keep the primary error; never retries and never sleeps.
|
|
70
|
+
*/
|
|
71
|
+
export async function removePrivateScratch(scratch, io = defaultPrivateScratchIo) {
|
|
72
|
+
try {
|
|
73
|
+
await io.removeDirectory(scratch.path);
|
|
74
|
+
}
|
|
75
|
+
catch (error) {
|
|
76
|
+
return `private scratch could not be removed: ${error instanceof Error ? error.message : String(error)}`;
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
await lstat(scratch.path);
|
|
80
|
+
return "private scratch still exists after removal.";
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
return error.code === "ENOENT" ? null : "private scratch state could not be verified after removal.";
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -4,9 +4,9 @@ export const CONTEXT_WINDOW_SCALING_TREATMENT_IDS = ["raw-full-file", "my-dev-ki
|
|
|
4
4
|
export const contextWindowScalingMetadata = {
|
|
5
5
|
id: CONTEXT_WINDOW_SCALING_PLUGIN_ID,
|
|
6
6
|
name: "Context Window Scaling",
|
|
7
|
-
description: "Measure raw-full-file and my-dev-kit-guided strategies under increasing deterministic estimated-context-token budgets, over the bundled fixed scaling corpus
|
|
7
|
+
description: "Measure raw-full-file and my-dev-kit-guided strategies under increasing deterministic estimated-context-token budgets, over the bundled fixed scaling corpus, caller-supplied deterministic synthetic repositories (--synthetic-config), or an explicitly selected local Git repository (--target with --local-subject-config). Operational context-fit measurement only; it does not measure retrieval precision, recall, or ranking quality.",
|
|
8
8
|
schemaVersion: "1.0.0",
|
|
9
9
|
status: "experimental",
|
|
10
|
-
supportedTargets: ["self"],
|
|
10
|
+
supportedTargets: ["self", "external-local"],
|
|
11
11
|
supportedOutputs: ["json", "text", "html", "plot"],
|
|
12
12
|
};
|
|
@@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises";
|
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { tokenCountMethod } from "../../../core/countTokens.js";
|
|
4
4
|
import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
5
|
+
import { serializeLocalRepositorySubjectManifest, } from "../../../evaluation/localRepositorySubject/index.js";
|
|
5
6
|
import { summarizeExperimentRun } from "../../results.js";
|
|
6
7
|
import { aggregateStatus } from "../warmIndexReuse/execution.js";
|
|
7
8
|
import { contextWindowScalingConfigDefinition, defaultContextWindowScalingConfig, validateContextWindowScalingConfig, } from "./config.js";
|
|
@@ -9,7 +10,13 @@ import { executeContextWindowScalingCases, } from "./execution.js";
|
|
|
9
10
|
import { buildContextWindowScalingExecutionArtifact, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
|
|
10
11
|
import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS, contextWindowScalingMetadata } from "./metadata.js";
|
|
11
12
|
import { aggregateContextWindowScaling, toAggregateExperimentMetrics, toOutcomeBudgetMetrics, } from "./metrics.js";
|
|
13
|
+
import { LocalSubjectExecutionError } from "./localSubjectErrors.js";
|
|
14
|
+
import { executeLocalRepositorySubjectContextWindowScaling } from "./localSubjectExecution.js";
|
|
15
|
+
import { describeLocalSubjectFailureForPersistence, projectEvidenceForExternalLocalPersistence, projectExternalLocalTarget, redactKnownPaths, } from "./localSubjectPrivacy.js";
|
|
16
|
+
import { isSamePhysicalDirectory } from "./localSubjectScratch.js";
|
|
12
17
|
import { resolveScalingProjectProfiles } from "./projectProfile.js";
|
|
18
|
+
/** Privacy-safe Batch 1 manifest persisted beside the execution artifact for external-local runs. */
|
|
19
|
+
export const LOCAL_REPOSITORY_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
|
|
13
20
|
const VARIANTS = [
|
|
14
21
|
{
|
|
15
22
|
id: "raw-full-file",
|
|
@@ -31,6 +38,14 @@ export const contextWindowScalingPlugin = {
|
|
|
31
38
|
validateConfig: validateContextWindowScalingConfig,
|
|
32
39
|
async run(context) {
|
|
33
40
|
const startedAt = context.startedAt.toISOString();
|
|
41
|
+
const localSubject = readLocalSubjectInput(context.inputs);
|
|
42
|
+
if (localSubject)
|
|
43
|
+
return runLocalSubject(context, localSubject, startedAt);
|
|
44
|
+
if (context.target.kind === "external-local") {
|
|
45
|
+
// Never fall back to the legacy raw/guided path against an external repository: it is neither
|
|
46
|
+
// ignore-aware nor symlink-aware and writes artifacts it must not.
|
|
47
|
+
throw new Error("External-local context-window-scaling targets require a loaded local repository subject (--local-subject-config).");
|
|
48
|
+
}
|
|
34
49
|
const cases = readCasesInput(context.inputs);
|
|
35
50
|
const projectProfiles = resolveScalingProjectProfiles(cases, readProjectProfilesInput(context.inputs));
|
|
36
51
|
const outDir = context.outputRoot ?? path.resolve(context.toolRoot, "lab-output", "context-window-scaling");
|
|
@@ -72,6 +87,84 @@ export const contextWindowScalingPlugin = {
|
|
|
72
87
|
return result.summary ?? summarizeExperimentRun(result);
|
|
73
88
|
},
|
|
74
89
|
};
|
|
90
|
+
function readLocalSubjectInput(inputs) {
|
|
91
|
+
const value = inputs?.localSubject;
|
|
92
|
+
return value && typeof value === "object" ? value : undefined;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* External-local path: the Batch 2 safe seam owns execution, immutability and scratch cleanup; this function only
|
|
96
|
+
* projects the privacy-lean result for persistence and feeds the same artifact and run builders as the legacy path.
|
|
97
|
+
*/
|
|
98
|
+
async function runLocalSubject(context, subject, startedAt) {
|
|
99
|
+
if (context.target.isSelf || context.target.kind !== "external-local") {
|
|
100
|
+
throw new Error("Local-repository subject mode requires an external-local target.");
|
|
101
|
+
}
|
|
102
|
+
if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
|
|
103
|
+
throw new Error("The selected --target is not the repository the local subject was loaded from.");
|
|
104
|
+
}
|
|
105
|
+
const outDir = context.outputRoot;
|
|
106
|
+
if (!outDir)
|
|
107
|
+
throw new Error("Local-repository subject mode requires an experiment output directory.");
|
|
108
|
+
const knownFiles = [
|
|
109
|
+
...subject.eligibleFiles,
|
|
110
|
+
...subject.runtimeSafetyExclusions.gitIgnoredFiles,
|
|
111
|
+
...subject.runtimeSafetyExclusions.oversizedFiles,
|
|
112
|
+
];
|
|
113
|
+
const privateRoots = [subject.repositoryRoot, context.target.targetRoot, outDir, context.toolRoot];
|
|
114
|
+
let localResult;
|
|
115
|
+
try {
|
|
116
|
+
localResult = await executeLocalRepositorySubjectContextWindowScaling({
|
|
117
|
+
subject,
|
|
118
|
+
contextBudgets: context.config.contextBudgets,
|
|
119
|
+
kitCommand: context.config.kitCommand,
|
|
120
|
+
workRoot: outDir,
|
|
121
|
+
env: readEnvInput(context.inputs),
|
|
122
|
+
dependencies: readDependenciesInput(context.inputs),
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
if (error instanceof LocalSubjectExecutionError)
|
|
127
|
+
throw new Error(describeLocalSubjectFailureForPersistence(error));
|
|
128
|
+
throw new Error(redactKnownPaths(error instanceof Error ? error.message : String(error), knownFiles, privateRoots));
|
|
129
|
+
}
|
|
130
|
+
const startedAtIso = startedAt;
|
|
131
|
+
const evidence = projectEvidenceForExternalLocalPersistence(localResult.caseEvidence, { knownFiles, privateRoots });
|
|
132
|
+
await mkdir(outDir, { recursive: true });
|
|
133
|
+
const completedAt = new Date().toISOString();
|
|
134
|
+
const artifact = buildContextWindowScalingExecutionArtifact({
|
|
135
|
+
runId: context.runId,
|
|
136
|
+
pluginId: contextWindowScalingMetadata.id,
|
|
137
|
+
pluginSchemaVersion: contextWindowScalingMetadata.schemaVersion,
|
|
138
|
+
startedAt: startedAtIso,
|
|
139
|
+
completedAt,
|
|
140
|
+
tokenCountMethod,
|
|
141
|
+
contextBudgets: context.config.contextBudgets,
|
|
142
|
+
cases: evidence,
|
|
143
|
+
});
|
|
144
|
+
const artifactPath = resolveWithinRoot(outDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
|
|
145
|
+
await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}
|
|
146
|
+
`, "utf8");
|
|
147
|
+
const manifestPath = resolveWithinRoot(outDir, LOCAL_REPOSITORY_SUBJECT_MANIFEST_FILE);
|
|
148
|
+
await writeFile(manifestPath, serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
|
|
149
|
+
const run = mapExecutionToRun({
|
|
150
|
+
runId: context.runId,
|
|
151
|
+
startedAt: startedAtIso,
|
|
152
|
+
completedAt,
|
|
153
|
+
target: projectExternalLocalTarget(subject.manifest),
|
|
154
|
+
contextBudgets: context.config.contextBudgets,
|
|
155
|
+
executionEvidence: evidence,
|
|
156
|
+
artifactPath,
|
|
157
|
+
});
|
|
158
|
+
run.artifacts.push({
|
|
159
|
+
id: "local-repository-subject-manifest",
|
|
160
|
+
label: "Privacy-safe local repository subject manifest",
|
|
161
|
+
path: manifestPath,
|
|
162
|
+
kind: "artifact",
|
|
163
|
+
mimeType: "application/json",
|
|
164
|
+
description: "Subject identity, Git commit, safety policy and aggregate inventory counts; no paths, source text or file lists.",
|
|
165
|
+
});
|
|
166
|
+
return run;
|
|
167
|
+
}
|
|
75
168
|
export function mapExecutionToRun(args) {
|
|
76
169
|
const cases = args.executionEvidence.map((caseEvidence) => ({
|
|
77
170
|
id: caseEvidence.caseId,
|
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { calculateProjectComplexityScore, computeProjectComplexityMetrics, PROJECT_COMPLEXITY_FORMULA, } from "../../../evaluation/projectComplexity.js";
|
|
3
3
|
import { buildProjectFileTree } from "../../../evaluation/projectFileTree.js";
|
|
4
|
-
|
|
5
|
-
* The fixed scaling project has no entry in the shared benchmark-project-profiles contract. The
|
|
6
|
-
* existing prompt owner requires a profile per case, so a missing one is derived in memory from
|
|
7
|
-
* the existing file-tree and complexity owners. Supplied profiles always win; nothing is written.
|
|
8
|
-
*/
|
|
9
|
-
export function resolveScalingProjectProfiles(cases, supplied) {
|
|
4
|
+
export function resolveScalingProjectProfiles(cases, supplied, options = {}) {
|
|
10
5
|
const profiles = [...supplied];
|
|
11
6
|
const known = new Set(profiles.map((profile) => profile.projectId));
|
|
12
7
|
for (const evaluationCase of cases) {
|
|
@@ -14,13 +9,13 @@ export function resolveScalingProjectProfiles(cases, supplied) {
|
|
|
14
9
|
if (known.has(profileId))
|
|
15
10
|
continue;
|
|
16
11
|
known.add(profileId);
|
|
17
|
-
profiles.push(deriveProfile(profileId, evaluationCase, cases));
|
|
12
|
+
profiles.push(deriveProfile(profileId, evaluationCase, cases, options));
|
|
18
13
|
}
|
|
19
14
|
return profiles;
|
|
20
15
|
}
|
|
21
|
-
function deriveProfile(profileId, evaluationCase, cases) {
|
|
16
|
+
function deriveProfile(profileId, evaluationCase, cases, options) {
|
|
22
17
|
const siblings = cases.filter((candidate) => (candidate.projectProfileRef ?? candidate.benchmarkProject) === profileId);
|
|
23
|
-
const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot);
|
|
18
|
+
const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot, options.eligibleFiles === undefined ? {} : { allowedRelativeFiles: options.eligibleFiles });
|
|
24
19
|
const average = (values) => (values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length);
|
|
25
20
|
const complexityMetrics = computeProjectComplexityMetrics(evaluationCase.absoluteTargetRoot, fileTree, {
|
|
26
21
|
taskCount: siblings.length,
|
|
@@ -3,6 +3,7 @@ import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
|
|
|
3
3
|
import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
|
|
4
4
|
import { renderContextWindowScalingHtml } from "./renderContextWindowScalingHtml.js";
|
|
5
5
|
export function renderPluginExperimentReportHtml(report) {
|
|
6
|
+
const redactedTarget = report.target.privacyProjection === "external-local-redacted";
|
|
6
7
|
return `<!DOCTYPE html>
|
|
7
8
|
<html lang="en">
|
|
8
9
|
<head>
|
|
@@ -49,10 +50,10 @@ export function renderPluginExperimentReportHtml(report) {
|
|
|
49
50
|
["Mode", report.target.mode],
|
|
50
51
|
["Tool root", report.target.toolRoot],
|
|
51
52
|
["Target root", report.target.targetRoot],
|
|
52
|
-
["Target package", report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
|
|
53
|
+
["Target package", redactedTarget ? "redacted" : report.target.packageName ? `${report.target.packageName}@${report.target.packageVersion ?? "unknown"}` : "unavailable"],
|
|
53
54
|
["Target branch/commit", report.target.hasGit ? `${report.target.branch ?? "unknown"} / ${report.target.commit ?? "unknown"}` : "unavailable"],
|
|
54
|
-
["Has package.json", String(report.target.hasPackageJson)],
|
|
55
|
-
["Has lockfile", String(report.target.hasLockfile)],
|
|
55
|
+
["Has package.json", redactedTarget ? "redacted" : String(report.target.hasPackageJson)],
|
|
56
|
+
["Has lockfile", redactedTarget ? "redacted" : String(report.target.hasLockfile)],
|
|
56
57
|
["Output root", report.metadata.outputRoot ?? "unavailable"],
|
|
57
58
|
])}
|
|
58
59
|
</section>
|
|
@@ -17,13 +17,19 @@ export async function writePluginExperimentReports(args) {
|
|
|
17
17
|
textPath: resolveWithinRoot(outDir, "report.txt"),
|
|
18
18
|
};
|
|
19
19
|
await mkdir(outputPaths.outDir, { recursive: true });
|
|
20
|
-
const
|
|
20
|
+
const builtReport = buildPluginExperimentReport({
|
|
21
21
|
run: args.run,
|
|
22
22
|
plugin: args.plugin,
|
|
23
23
|
outputRoot: outputPaths.outDir,
|
|
24
24
|
generatedAt: args.generatedAt,
|
|
25
25
|
});
|
|
26
|
-
|
|
26
|
+
const report = args.redactOutputRoot
|
|
27
|
+
? { ...builtReport, metadata: { ...builtReport.metadata, outputRoot: "[redacted]" } }
|
|
28
|
+
: builtReport;
|
|
29
|
+
const serializedPaths = args.redactOutputRoot
|
|
30
|
+
? { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" }
|
|
31
|
+
: outputPaths;
|
|
32
|
+
await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, "utf8");
|
|
27
33
|
await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), "utf8");
|
|
28
34
|
await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), "utf8");
|
|
29
35
|
return { report, outputPaths };
|
package/docs/METRICS.md
CHANGED
|
@@ -622,6 +622,8 @@ The report artifact schema is `my-dev-kit-lab-context-window-scaling-report-v1`.
|
|
|
622
622
|
|
|
623
623
|
**Synthetic-repository runs (v0.7.1).** The synthetic input introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact schema, the report schema, and the plot set are identical to the bundled-corpus run. Each generated repository's `synthetic-repository-manifest.json` records generation evidence (per-file byte length, character count, SHA-256, generated-content estimated tokens, aggregate totals, and `repositoryContentIdentity`); these are reproducibility measurements, not context-window outcome metrics. The raw-full-file treatment context is measured by the existing baseline, which adds file headers and separators, so its estimated tokens differ from the manifest's generated-content estimate. Retrieval precision/recall remains v0.8.0 scope.
|
|
624
624
|
|
|
625
|
+
**Local-repository runs (v0.7.2).** Running over an explicitly selected local Git repository introduces no new scientific metric: fit, utilization, correctness, success, and omitted-relevant-file semantics, the execution artifact and report schemas, and the plot set are the same. In this mode file identity lists are replaced by numbered placeholders (`<redacted file N>`) with the same length as the real list, and each treatment carries `fileIdentityRedaction: "redacted"`; a placeholder is a withheld identity, not an empty result, and an empty list or `null` keeps its usual meaning. Operational compatibility with a repository, the 1 MiB file-size limit, and the Git-ignore exclusions are safety policy, not quality measures: they do not imply retrieval precision, recall, ranking quality, or a retrieval winner.
|
|
626
|
+
|
|
625
627
|
There is no composite score, winner, or ranking. Budgets are experiment thresholds, not assertions about actual provider context-window capacities. See [COMMANDS.md](COMMANDS.md), [WORKFLOWS.md](WORKFLOWS.md), and [ARCHITECTURE.md](ARCHITECTURE.md).
|
|
626
628
|
|
|
627
629
|
## Incremental-change and staleness evidence (v0.6.2)
|