@dailephd/my-dev-kit-lab 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/README.md +6 -5
  3. package/dist/src/cli/help.js +15 -4
  4. package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
  5. package/dist/src/commands/runExperimentRunCommand.js +117 -18
  6. package/dist/src/core/fileGlobs.js +44 -0
  7. package/dist/src/evaluation/localRepositorySubject/config.js +296 -0
  8. package/dist/src/evaluation/localRepositorySubject/evaluationCase.js +36 -0
  9. package/dist/src/evaluation/localRepositorySubject/expectedFiles.js +51 -0
  10. package/dist/src/evaluation/localRepositorySubject/gitRepository.js +128 -0
  11. package/dist/src/evaluation/localRepositorySubject/index.js +8 -0
  12. package/dist/src/evaluation/localRepositorySubject/inventory.js +127 -0
  13. package/dist/src/evaluation/localRepositorySubject/loadLocalRepositorySubject.js +48 -0
  14. package/dist/src/evaluation/localRepositorySubject/manifest.js +52 -0
  15. package/dist/src/evaluation/localRepositorySubject/types.js +55 -0
  16. package/dist/src/evaluation/projectFileTree.js +45 -1
  17. package/dist/src/evaluation/runMyDevKitRetrieval.js +3 -1
  18. package/dist/src/evaluation/runRawFullFileBaseline.js +18 -4
  19. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +29 -7
  20. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +4 -0
  21. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectErrors.js +35 -0
  22. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExclusions.js +75 -0
  23. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js +136 -0
  24. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +128 -0
  25. package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js +85 -0
  26. package/dist/src/experiments/plugins/contextWindowScaling/metadata.js +2 -2
  27. package/dist/src/experiments/plugins/contextWindowScaling/plugin.js +93 -0
  28. package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js +4 -9
  29. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +4 -3
  30. package/dist/src/report/experiments/writePluginExperimentReports.js +8 -2
  31. package/docs/METRICS.md +2 -0
  32. package/package.json +1 -1
@@ -0,0 +1,48 @@
1
+ import { parseLocalRepositorySubjectConfig } from "./config.js";
2
+ import { buildLocalRepositorySubjectEvaluationCases } from "./evaluationCase.js";
3
+ import { validateExpectedFiles } from "./expectedFiles.js";
4
+ import { readRepositoryIdentity } from "./gitRepository.js";
5
+ import { buildLocalRepositorySubjectInventory, defaultLocalRepositorySubjectFsIo, validateSourceRootOnDisk, } from "./inventory.js";
6
+ import { buildLocalRepositorySubjectManifest } from "./manifest.js";
7
+ import { DEFAULT_LOCAL_REPOSITORY_SUBJECT_MAX_FILE_BYTES, LocalRepositorySubjectConfigError, compareCodeUnits, logicalTargetRootForSubject, } from "./types.js";
8
+ /**
9
+ * Loads a local repository subject in the frozen safety order: validate config, identify the Git repository,
10
+ * validate source roots on disk, inventory (without reading contents), validate required files, then build the
11
+ * privacy-safe manifest and in-memory EvaluationCase objects. Never mutates the subject repository.
12
+ */
13
+ export async function loadLocalRepositorySubject(options) {
14
+ const maxFileBytes = options.maxFileBytes ?? DEFAULT_LOCAL_REPOSITORY_SUBJECT_MAX_FILE_BYTES;
15
+ if (!Number.isSafeInteger(maxFileBytes) || maxFileBytes < 1) {
16
+ throw new LocalRepositorySubjectConfigError([`policy.maxFileBytes: must be a positive safe integer (received ${String(maxFileBytes)}).`]);
17
+ }
18
+ const io = options.io ?? defaultLocalRepositorySubjectFsIo;
19
+ const config = parseLocalRepositorySubjectConfig(options.config);
20
+ const { repositoryRoot, identity } = await readRepositoryIdentity(options.repositoryPath);
21
+ const rootErrors = [];
22
+ for (let caseIndex = 0; caseIndex < config.cases.length; caseIndex += 1) {
23
+ const roots = config.cases[caseIndex].sourceRoots;
24
+ for (let rootIndex = 0; rootIndex < roots.length; rootIndex += 1) {
25
+ rootErrors.push(...(await validateSourceRootOnDisk(repositoryRoot, roots[rootIndex], `config.cases[${caseIndex}].sourceRoots[${rootIndex}]`, io)));
26
+ }
27
+ }
28
+ if (rootErrors.length > 0)
29
+ throw new LocalRepositorySubjectConfigError(rootErrors);
30
+ const unionRoots = [...new Set(config.cases.flatMap((subjectCase) => subjectCase.sourceRoots))];
31
+ const inventory = await buildLocalRepositorySubjectInventory(repositoryRoot, unionRoots, maxFileBytes, io);
32
+ const expectedFileErrors = await validateExpectedFiles(repositoryRoot, config.cases, inventory, io);
33
+ if (expectedFileErrors.length > 0)
34
+ throw new LocalRepositorySubjectConfigError(expectedFileErrors);
35
+ const manifest = buildLocalRepositorySubjectManifest(config, identity, maxFileBytes, inventory);
36
+ return {
37
+ subjectId: config.subjectId,
38
+ logicalTargetRoot: logicalTargetRootForSubject(config.subjectId),
39
+ repositoryRoot,
40
+ manifest,
41
+ evaluationCases: buildLocalRepositorySubjectEvaluationCases(config, repositoryRoot),
42
+ eligibleFiles: inventory.eligibleFiles.map((file) => file.path),
43
+ runtimeSafetyExclusions: {
44
+ gitIgnoredFiles: [...inventory.ignoredFiles].sort(compareCodeUnits),
45
+ oversizedFiles: [...inventory.oversizedFiles].sort(compareCodeUnits),
46
+ },
47
+ };
48
+ }
@@ -0,0 +1,52 @@
1
+ import { LOCAL_REPOSITORY_SUBJECT_MANIFEST_SCHEMA_ID, LOCAL_REPOSITORY_SUBJECT_MANIFEST_SCHEMA_VERSION, LOCAL_REPOSITORY_SUBJECT_SAFETY_POLICY_VERSION, compareCodeUnits, logicalTargetRootForSubject, } from "./types.js";
2
+ /** Lowercase text after the last "." of the basename; "(none)" when absent or only a leading dot. */
3
+ export function extensionOf(filePath) {
4
+ const base = filePath.slice(filePath.lastIndexOf("/") + 1);
5
+ const dot = base.lastIndexOf(".");
6
+ if (dot <= 0 || dot === base.length - 1)
7
+ return "(none)";
8
+ return base.slice(dot + 1).toLowerCase();
9
+ }
10
+ function summarizeExtensions(filePaths) {
11
+ const counts = new Map();
12
+ for (const filePath of filePaths) {
13
+ const extension = extensionOf(filePath);
14
+ counts.set(extension, (counts.get(extension) ?? 0) + 1);
15
+ }
16
+ return [...counts.entries()]
17
+ .sort(([left], [right]) => compareCodeUnits(left, right))
18
+ .map(([extension, fileCount]) => ({ extension, fileCount }));
19
+ }
20
+ /** Builds the privacy-safe manifest model. Inputs that are runtime-only (paths, file lists) are reduced to aggregates. */
21
+ export function buildLocalRepositorySubjectManifest(config, identity, maxFileBytes, inventory) {
22
+ const sourceRoots = [...new Set(config.cases.flatMap((subjectCase) => subjectCase.sourceRoots))].sort(compareCodeUnits);
23
+ return {
24
+ schemaId: LOCAL_REPOSITORY_SUBJECT_MANIFEST_SCHEMA_ID,
25
+ schemaVersion: LOCAL_REPOSITORY_SUBJECT_MANIFEST_SCHEMA_VERSION,
26
+ subjectId: config.subjectId,
27
+ logicalTargetRoot: logicalTargetRootForSubject(config.subjectId),
28
+ repository: { commit: identity.commit, branch: identity.branch, workingTreeDirty: identity.workingTreeDirty },
29
+ safetyPolicy: {
30
+ version: LOCAL_REPOSITORY_SUBJECT_SAFETY_POLICY_VERSION,
31
+ maxFileBytes,
32
+ gitIgnoreAuthority: "git",
33
+ symlinkPolicy: "excluded-not-followed",
34
+ },
35
+ sourceRoots,
36
+ caseCount: config.cases.length,
37
+ caseIds: config.cases.map((subjectCase) => subjectCase.id),
38
+ inventory: {
39
+ eligibleFileCount: inventory.counts.eligibleFileCount,
40
+ eligibleByteCount: inventory.counts.eligibleByteCount,
41
+ gitIgnoredCount: inventory.counts.gitIgnoredCount,
42
+ oversizedCount: inventory.counts.oversizedCount,
43
+ symlinkCount: inventory.counts.symlinkCount,
44
+ otherExcludedCount: inventory.counts.otherExcludedCount,
45
+ extensionSummary: summarizeExtensions(inventory.eligibleFiles.map((file) => file.path)),
46
+ },
47
+ };
48
+ }
49
+ /** Deterministic serialization (fixed key order from the builder, LF terminated). */
50
+ export function serializeLocalRepositorySubjectManifest(manifest) {
51
+ return `${JSON.stringify(manifest, null, 2)}\n`;
52
+ }
@@ -0,0 +1,55 @@
1
+ export const LOCAL_REPOSITORY_SUBJECT_CONFIG_SCHEMA_ID = "my-dev-kit-lab-local-repository-subject-config-v1";
2
+ export const LOCAL_REPOSITORY_SUBJECT_CONFIG_SCHEMA_VERSION = "1.0.0";
3
+ export const LOCAL_REPOSITORY_SUBJECT_MANIFEST_SCHEMA_ID = "my-dev-kit-lab-local-repository-subject-manifest-v1";
4
+ export const LOCAL_REPOSITORY_SUBJECT_MANIFEST_SCHEMA_VERSION = "1.0.0";
5
+ /** Operational safety policy (not a quality metric): files larger than this are never eligible subject material. */
6
+ export const DEFAULT_LOCAL_REPOSITORY_SUBJECT_MAX_FILE_BYTES = 1_048_576;
7
+ export const LOCAL_REPOSITORY_SUBJECT_SAFETY_POLICY_VERSION = "1.0.0";
8
+ export const LOCAL_REPOSITORY_SUBJECT_LIMITS = {
9
+ subjectIdMaxLength: 128,
10
+ };
11
+ export const LOCAL_REPOSITORY_SUBJECT_CONFIG_FIELDS = ["schemaVersion", "subjectId", "cases"];
12
+ export const LOCAL_REPOSITORY_SUBJECT_CASE_FIELDS = [
13
+ "id",
14
+ "title",
15
+ "sourceRoots",
16
+ "query",
17
+ "expectedFiles",
18
+ "expectedSymbols",
19
+ "rawIncludeGlobs",
20
+ "answerKey",
21
+ "expectedFacts",
22
+ "taskLocality",
23
+ "promptComplexityHint",
24
+ "projectComplexityRelevance",
25
+ "notes",
26
+ ];
27
+ /** Thrown when the config, or required paths it names, fail validation. Messages carry logical labels only. */
28
+ export class LocalRepositorySubjectConfigError extends Error {
29
+ errors;
30
+ constructor(errors) {
31
+ super(`Invalid local repository subject configuration: ${errors.join("; ")}`);
32
+ this.name = "LocalRepositorySubjectConfigError";
33
+ this.errors = [...errors];
34
+ }
35
+ }
36
+ /** Thrown when the selected repository cannot be identified or classified safely. Never carries raw Git output. */
37
+ export class LocalRepositorySubjectRepositoryError extends Error {
38
+ code;
39
+ constructor(code, message) {
40
+ super(`Local repository subject repository error (${code}): ${message}`);
41
+ this.name = "LocalRepositorySubjectRepositoryError";
42
+ this.code = code;
43
+ }
44
+ }
45
+ export function logicalTargetRootForSubject(subjectId) {
46
+ return `local-repository:${subjectId}`;
47
+ }
48
+ /** Locale-independent ordering by code unit. */
49
+ export function compareCodeUnits(left, right) {
50
+ if (left < right)
51
+ return -1;
52
+ if (left > right)
53
+ return 1;
54
+ return 0;
55
+ }
@@ -45,8 +45,52 @@ export function countFileLines(filePath) {
45
45
  }
46
46
  return content.split(/\r?\n/).length;
47
47
  }
48
- export function buildProjectFileTree(projectRoot) {
48
+ function buildAllowedFileTree(root, allowedRelativeFiles) {
49
+ const top = { children: new Map(), isFile: false };
50
+ for (const allowed of allowedRelativeFiles) {
51
+ const relativePath = allowed.replace(/\\/g, "/");
52
+ if (isExcludedProjectPath(relativePath))
53
+ continue;
54
+ let node = top;
55
+ const segments = relativePath.split("/");
56
+ segments.forEach((segment, index) => {
57
+ let child = node.children.get(segment);
58
+ if (!child) {
59
+ child = { children: new Map(), isFile: index === segments.length - 1 };
60
+ node.children.set(segment, child);
61
+ }
62
+ node = child;
63
+ });
64
+ }
65
+ const entries = [];
66
+ const walk = (node, prefix) => {
67
+ const names = [...node.children.keys()].sort((a, b) => a.localeCompare(b));
68
+ for (const name of names) {
69
+ const child = node.children.get(name);
70
+ const relativePath = prefix === "" ? name : `${prefix}/${name}`;
71
+ if (child.isFile) {
72
+ entries.push({
73
+ path: relativePath,
74
+ kind: "file",
75
+ role: inferFileRole(relativePath, "file"),
76
+ language: inferFileLanguage(relativePath),
77
+ lines: countFileLines(path.join(root, ...relativePath.split("/")))
78
+ });
79
+ }
80
+ else {
81
+ entries.push({ path: relativePath, kind: "directory", role: inferFileRole(relativePath, "directory") });
82
+ walk(child, relativePath);
83
+ }
84
+ }
85
+ };
86
+ walk(top, "");
87
+ return { entries };
88
+ }
89
+ export function buildProjectFileTree(projectRoot, options = {}) {
49
90
  const root = path.resolve(projectRoot);
91
+ if (options.allowedRelativeFiles !== undefined) {
92
+ return buildAllowedFileTree(root, options.allowedRelativeFiles);
93
+ }
50
94
  const entries = [];
51
95
  function walk(currentDir) {
52
96
  const dirEntries = readdirSync(currentDir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
@@ -177,6 +177,7 @@ export async function buildMyDevKitIndex(options) {
177
177
  "--root",
178
178
  options.target.absoluteTargetRoot,
179
179
  ...options.target.sourceRoots.flatMap((sourceRoot) => ["--src", sourceRoot]),
180
+ ...(options.excludePaths ?? []).flatMap((excludePath) => ["--exclude", excludePath]),
180
181
  "--out",
181
182
  options.indexDir,
182
183
  ...(mode.kind === "incremental" ? ["--incremental", "--refresh-scope", mode.refreshScope] : []),
@@ -352,7 +353,8 @@ export async function runMyDevKitRetrieval(options) {
352
353
  kitCommand: options.kitCommand,
353
354
  indexDir,
354
355
  commandsDir,
355
- requireKit: options.requireKit
356
+ requireKit: options.requireKit,
357
+ excludePaths: options.excludePaths
356
358
  });
357
359
  if (!index.ok) {
358
360
  return skippedRetrieval(options.evaluationCase.id, [...index.warnings], [index.command], Date.now() - started);
@@ -1,7 +1,8 @@
1
- import { readFileSync, statSync } from "node:fs";
2
- import { collectFilesForGlobs } from "../core/fileGlobs.js";
1
+ import { lstatSync, readFileSync, statSync } from "node:fs";
2
+ import { collectFilesForGlobs, selectRelativePathsForGlobs } from "../core/fileGlobs.js";
3
+ import { resolveWithinRoot } from "../core/pathSafety.js";
3
4
  import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
4
- export async function runRawFullFileBaseline(evaluationCase) {
5
+ export async function runRawFullFileBaseline(evaluationCase, options = {}) {
5
6
  const started = Date.now();
6
7
  let stats;
7
8
  try {
@@ -13,7 +14,9 @@ export async function runRawFullFileBaseline(evaluationCase) {
13
14
  if (!stats.isDirectory()) {
14
15
  throw new Error(`Target root is not a directory: ${evaluationCase.targetRoot}`);
15
16
  }
16
- const files = collectFilesForGlobs(evaluationCase.absoluteTargetRoot, evaluationCase.rawIncludeGlobs);
17
+ const files = options.eligibleFiles === undefined
18
+ ? collectFilesForGlobs(evaluationCase.absoluteTargetRoot, evaluationCase.rawIncludeGlobs)
19
+ : selectEligibleFiles(evaluationCase, options.eligibleFiles);
17
20
  const contextText = files
18
21
  .map(({ absolutePath, relativePath }) => `=== FILE: ${relativePath} ===\n${readFileSync(absolutePath, "utf8")}\n`)
19
22
  .join("\n");
@@ -29,3 +32,14 @@ export async function runRawFullFileBaseline(evaluationCase) {
29
32
  durationMs: Date.now() - started
30
33
  };
31
34
  }
35
+ function selectEligibleFiles(evaluationCase, eligibleFiles) {
36
+ return selectRelativePathsForGlobs(eligibleFiles, evaluationCase.rawIncludeGlobs).map((relativePath) => {
37
+ const absolutePath = resolveWithinRoot(evaluationCase.absoluteTargetRoot, relativePath);
38
+ // The eligible list was built earlier; refuse a file that has since been swapped for a link or removed.
39
+ const stats = lstatSync(absolutePath, { throwIfNoEntry: false });
40
+ if (!stats || stats.isSymbolicLink() || !stats.isFile()) {
41
+ throw new Error(`Eligible file is no longer a regular file: ${relativePath}`);
42
+ }
43
+ return { absolutePath, relativePath };
44
+ });
45
+ }
@@ -18,7 +18,7 @@ function isInsideRoot(resolvedRoot, candidate) {
18
18
  const relative = path.relative(resolvedRoot, candidate);
19
19
  return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
20
20
  }
21
- async function snapshotPathEntry(resolvedPath) {
21
+ async function snapshotPathEntry(resolvedPath, maxHashedFileBytes) {
22
22
  let stats;
23
23
  try {
24
24
  stats = await lstat(resolvedPath);
@@ -38,6 +38,15 @@ async function snapshotPathEntry(resolvedPath) {
38
38
  return { state: "directory", sha256: null, symbolicLinkTarget: null };
39
39
  }
40
40
  if (stats.isFile()) {
41
+ if (maxHashedFileBytes !== undefined && stats.size > maxHashedFileBytes) {
42
+ // Safe mode: a file above the bound is fingerprinted by metadata and its contents are never read.
43
+ return {
44
+ state: "file",
45
+ sha256: null,
46
+ symbolicLinkTarget: null,
47
+ contentFingerprint: { sizeBytes: stats.size, mtimeMs: stats.mtimeMs }
48
+ };
49
+ }
41
50
  const contents = await readFile(resolvedPath);
42
51
  return { state: "file", sha256: sha256Hex(contents), symbolicLinkTarget: null };
43
52
  }
@@ -69,8 +78,9 @@ async function runRequiredGit(resolvedRoot, args, targetRootPath) {
69
78
  : `git ${args.join(" ")} failed: ${result.message}`;
70
79
  return { ok: false, code, targetRootPath, message };
71
80
  }
72
- export async function captureTargetSnapshot(config) {
81
+ export async function captureTargetSnapshot(config, options = {}) {
73
82
  const resolvedTargetRootPath = path.resolve(config.targetRootPath);
83
+ const maxHashedFileBytes = options.externalLocalSafe?.maxHashedFileBytes;
74
84
  let rootStats;
75
85
  try {
76
86
  rootStats = await lstat(resolvedTargetRootPath);
@@ -113,13 +123,14 @@ export async function captureTargetSnapshot(config) {
113
123
  };
114
124
  }
115
125
  try {
116
- const entry = await snapshotPathEntry(resolvedPath);
126
+ const entry = await snapshotPathEntry(resolvedPath, maxHashedFileBytes);
117
127
  configuredFiles.push({
118
128
  relativePath,
119
129
  resolvedPath,
120
130
  state: entry.state,
121
131
  sha256: entry.sha256,
122
- symbolicLinkTarget: entry.symbolicLinkTarget
132
+ symbolicLinkTarget: entry.symbolicLinkTarget,
133
+ ...(entry.contentFingerprint ? { contentFingerprint: entry.contentFingerprint } : {})
123
134
  });
124
135
  }
125
136
  catch (error) {
@@ -183,15 +194,25 @@ export async function captureTargetSnapshot(config) {
183
194
  const untrackedFiles = [];
184
195
  for (const untrackedRelativePath of untrackedPaths) {
185
196
  const resolvedPath = path.resolve(resolvedTargetRootPath, untrackedRelativePath);
186
- const entry = await snapshotPathEntry(resolvedPath);
197
+ const entry = await snapshotPathEntry(resolvedPath, maxHashedFileBytes);
187
198
  const state = entry.state === "directory" ? "other" : entry.state;
188
199
  untrackedFiles.push({
189
200
  path: untrackedRelativePath,
190
201
  state,
191
202
  sha256: entry.sha256,
192
- symbolicLinkTarget: entry.symbolicLinkTarget
203
+ symbolicLinkTarget: entry.symbolicLinkTarget,
204
+ ...(entry.contentFingerprint ? { contentFingerprint: entry.contentFingerprint } : {})
193
205
  });
194
206
  }
207
+ let ignoredPaths;
208
+ if (maxHashedFileBytes !== undefined) {
209
+ // Safe mode: Lab-created files in ignored locations (for example a stray index directory) are invisible to
210
+ // `git status`, so record the ignored path set by name. Contents are never read.
211
+ const ignoredOutcome = await runRequiredGit(resolvedTargetRootPath, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], config.targetRootPath);
212
+ if (!ignoredOutcome.ok)
213
+ return ignoredOutcome;
214
+ ignoredPaths = ignoredOutcome.stdout.split("\0").filter((entry) => entry.length > 0).sort();
215
+ }
195
216
  const git = {
196
217
  availability: "available",
197
218
  branch,
@@ -199,7 +220,8 @@ export async function captureTargetSnapshot(config) {
199
220
  statusEntries,
200
221
  worktreeDiffSha256,
201
222
  stagedDiffSha256,
202
- untrackedFiles
223
+ untrackedFiles,
224
+ ...(ignoredPaths !== undefined ? { ignoredPaths } : {})
203
225
  };
204
226
  return {
205
227
  ok: true,
@@ -38,6 +38,10 @@ export function compareTargetSnapshots(before, after) {
38
38
  mutations.push(mutation(`git.untracked:${afterEntry.path}`, "git-untracked-file", `git.untrackedFiles.${afterEntry.path}`, null, afterEntry));
39
39
  }
40
40
  }
41
+ if ((before.git.ignoredPaths !== undefined || after.git.ignoredPaths !== undefined) &&
42
+ !isDeepStrictEqual(before.git.ignoredPaths ?? null, after.git.ignoredPaths ?? null)) {
43
+ mutations.push(mutation("git.ignored-paths", "git-ignored-paths", "git.ignoredPaths", (before.git.ignoredPaths ?? null), (after.git.ignoredPaths ?? null)));
44
+ }
41
45
  const afterConfiguredByPath = new Map(after.configuredFiles.map((entry) => [entry.relativePath, entry]));
42
46
  for (const beforeEntry of before.configuredFiles) {
43
47
  const afterEntry = afterConfiguredByPath.get(beforeEntry.relativePath);
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Thrown when a local-subject execution did not complete cleanly. The primary code is the first issue in the
3
+ * ordered list (execution, mutation, snapshot, cleanup); every issue is retained so a secondary failure never
4
+ * masks the primary one. `caseEvidence` and `immutability` are runtime-only partial evidence for forensics and
5
+ * must not be persisted without the later privacy projection.
6
+ */
7
+ export class LocalSubjectExecutionError extends Error {
8
+ code;
9
+ issues;
10
+ caseEvidence;
11
+ immutability;
12
+ constructor(issues, partial = {}) {
13
+ if (issues.length === 0)
14
+ throw new Error("LocalSubjectExecutionError requires at least one issue.");
15
+ super(`Local subject execution failed (${issues[0].code}): ${issues.map((issue) => issue.message).join("; ")}`);
16
+ this.name = "LocalSubjectExecutionError";
17
+ this.code = issues[0].code;
18
+ this.issues = [...issues];
19
+ this.caseEvidence = partial.caseEvidence ?? null;
20
+ this.immutability = partial.immutability ?? null;
21
+ }
22
+ }
23
+ /** Replaces absolute private roots in a message so errors never carry the target or scratch location. */
24
+ export function redactPrivatePaths(message, privatePaths) {
25
+ let redacted = message;
26
+ for (const privatePath of privatePaths) {
27
+ if (!privatePath)
28
+ continue;
29
+ const variants = new Set([privatePath, privatePath.replace(/\\/g, "/"), privatePath.replace(/\//g, "\\")]);
30
+ for (const variant of variants) {
31
+ redacted = redacted.split(variant).join("<private-path>");
32
+ }
33
+ }
34
+ return redacted;
35
+ }
@@ -0,0 +1,75 @@
1
+ import { compareCodeUnits } from "../../../evaluation/localRepositorySubject/index.js";
2
+ import { LocalSubjectExecutionError } from "./localSubjectErrors.js";
3
+ /** Fail-closed ceilings so the exclusion arguments stay well inside Windows command-line limits. */
4
+ export const GUIDED_EXCLUSION_MAX_ENTRIES = 500;
5
+ export const GUIDED_EXCLUSION_MAX_CHARACTERS = 24_000;
6
+ /**
7
+ * Pure policy: turns Batch 1 runtime exclusion evidence into the exact `--exclude` paths for the guided index.
8
+ *
9
+ * Each excluded file is covered by the shallowest ancestor directory that contains no eligible file (so a fully
10
+ * ignored directory becomes one entry), otherwise by the file path itself. Bare names are never produced because
11
+ * my-dev-kit treats a bare name as "every directory with this name", which would drop eligible material. If the
12
+ * result is too large the call throws instead of truncating: an incomplete exclusion list would be unsafe.
13
+ */
14
+ export function deriveGuidedIndexExclusions(input) {
15
+ const eligibleDirectories = new Set();
16
+ const eligibleSegments = new Set();
17
+ for (const eligible of input.eligibleFiles) {
18
+ const segments = eligible.split("/");
19
+ segments.forEach((segment) => eligibleSegments.add(segment));
20
+ for (let depth = 1; depth < segments.length; depth += 1) {
21
+ eligibleDirectories.add(segments.slice(0, depth).join("/"));
22
+ }
23
+ }
24
+ const candidates = new Set();
25
+ for (const excluded of input.excludedFiles) {
26
+ const segments = excluded.split("/");
27
+ let covering = excluded;
28
+ for (let depth = 1; depth < segments.length; depth += 1) {
29
+ const directory = segments.slice(0, depth).join("/");
30
+ if (!eligibleDirectories.has(directory)) {
31
+ covering = directory;
32
+ break;
33
+ }
34
+ }
35
+ // A single-segment entry is a name match in my-dev-kit (any depth), and "./name" is silently ignored, so it is
36
+ // only safe when no eligible path uses that name. Otherwise descend to a multi-segment path, which is exact.
37
+ if (!covering.includes("/") && eligibleSegments.has(covering)) {
38
+ if (covering !== excluded && segments.length > 2) {
39
+ covering = segments.slice(0, 2).join("/");
40
+ }
41
+ else if (covering !== excluded) {
42
+ covering = excluded;
43
+ }
44
+ else {
45
+ throw new LocalSubjectExecutionError([
46
+ {
47
+ code: "GUIDED_EXCLUSION_UNREPRESENTABLE",
48
+ message: `an excluded top-level entry shares its name with eligible files and cannot be excluded exactly (${excluded}).`,
49
+ },
50
+ ]);
51
+ }
52
+ }
53
+ candidates.add(covering);
54
+ }
55
+ const exclusions = [...candidates]
56
+ .filter((candidate) => {
57
+ const segments = candidate.split("/");
58
+ for (let depth = 1; depth < segments.length; depth += 1) {
59
+ if (candidates.has(segments.slice(0, depth).join("/")))
60
+ return false;
61
+ }
62
+ return true;
63
+ })
64
+ .sort(compareCodeUnits);
65
+ const characters = exclusions.reduce((total, entry) => total + entry.length + 1, 0);
66
+ if (exclusions.length > GUIDED_EXCLUSION_MAX_ENTRIES || characters > GUIDED_EXCLUSION_MAX_CHARACTERS) {
67
+ throw new LocalSubjectExecutionError([
68
+ {
69
+ code: "GUIDED_EXCLUSION_LIMIT",
70
+ message: `guided index exclusions exceed the safe limit (${exclusions.length} entries, ${characters} characters).`,
71
+ },
72
+ ]);
73
+ }
74
+ return exclusions;
75
+ }
@@ -0,0 +1,136 @@
1
+ import path from "node:path";
2
+ import { relativeWithinRoot } from "../../../core/pathSafety.js";
3
+ import { runMyDevKitRetrieval } from "../../../evaluation/runMyDevKitRetrieval.js";
4
+ import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
5
+ import { captureTargetSnapshot, compareTargetSnapshots } from "../../../evaluation/targetImmutability/index.js";
6
+ import { executeContextWindowScalingCases, } from "./execution.js";
7
+ import { LocalSubjectExecutionError, redactPrivatePaths, } from "./localSubjectErrors.js";
8
+ import { deriveGuidedIndexExclusions } from "./localSubjectExclusions.js";
9
+ import { assertWorkRootOutsideTarget, createPrivateScratch, defaultPrivateScratchIo, removePrivateScratch, } from "./localSubjectScratch.js";
10
+ import { resolveScalingProjectProfiles } from "./projectProfile.js";
11
+ function toRepositoryRelative(repositoryRoot, file) {
12
+ const slashed = file.replace(/\\/g, "/");
13
+ if (path.isAbsolute(file)) {
14
+ try {
15
+ return relativeWithinRoot(repositoryRoot, file);
16
+ }
17
+ catch {
18
+ return slashed;
19
+ }
20
+ }
21
+ return slashed.replace(/^\.\//, "");
22
+ }
23
+ /** Safe local constructors: raw context from eligible files only; guided retrieval over the original root. */
24
+ function createSafeLocalDependencies(subject, guidedExclusions) {
25
+ const eligible = new Set(subject.eligibleFiles);
26
+ return {
27
+ constructRawContext: (evaluationCase) => runRawFullFileBaseline(evaluationCase, { eligibleFiles: subject.eligibleFiles }),
28
+ constructGuidedContext: async ({ evaluationCase, kitCommand, outputDir }) => {
29
+ const result = await runMyDevKitRetrieval({
30
+ evaluationCase,
31
+ kitCommand,
32
+ outputDir,
33
+ requireKit: false,
34
+ excludePaths: guidedExclusions,
35
+ });
36
+ const returned = [...result.filesRead, ...(result.selectedFile ? [result.selectedFile] : [])];
37
+ const outside = returned.filter((file) => !eligible.has(toRepositoryRelative(subject.repositoryRoot, file)));
38
+ if (outside.length > 0) {
39
+ throw new Error(`Guided retrieval returned ${outside.length} file(s) outside the eligible subject universe: ${outside
40
+ .map((file) => toRepositoryRelative(subject.repositoryRoot, file))
41
+ .join(", ")}`);
42
+ }
43
+ return result;
44
+ },
45
+ };
46
+ }
47
+ /**
48
+ * Internal programmatic seam: runs a loaded Batch 1 local subject through the existing context-window core with
49
+ * safe raw and guided constructors, a private scratch outside the target, and pre/post target-immutability
50
+ * enforcement. Order: validate work root, derive exclusions, capture before, create scratch, execute, capture after,
51
+ * compare, remove scratch, then decide. Success is returned only when execution, immutability and cleanup all pass.
52
+ * A mutated target is never reverted.
53
+ */
54
+ export async function executeLocalRepositorySubjectContextWindowScaling(args) {
55
+ const { subject } = args;
56
+ await assertWorkRootOutsideTarget(args.workRoot, subject.repositoryRoot);
57
+ const guidedExclusions = deriveGuidedIndexExclusions({
58
+ eligibleFiles: subject.eligibleFiles,
59
+ excludedFiles: [...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
60
+ });
61
+ const snapshotConfig = { targetRootPath: subject.repositoryRoot, relativeFilePaths: subject.eligibleFiles };
62
+ const snapshotOptions = { externalLocalSafe: { maxHashedFileBytes: subject.manifest.safetyPolicy.maxFileBytes } };
63
+ const before = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
64
+ if (!before.ok) {
65
+ throw new LocalSubjectExecutionError([
66
+ { code: "BEFORE_SNAPSHOT_FAILED", message: `target snapshot before execution failed (${before.code}).` },
67
+ ]);
68
+ }
69
+ const scratch = await createPrivateScratch(args.workRoot, subject.repositoryRoot);
70
+ const privatePaths = [subject.repositoryRoot, scratch.path, path.resolve(args.workRoot)];
71
+ const redact = (message) => redactPrivatePaths(message, privatePaths);
72
+ const issues = [];
73
+ let caseEvidence = null;
74
+ try {
75
+ const projectProfiles = resolveScalingProjectProfiles(subject.evaluationCases, [], { eligibleFiles: subject.eligibleFiles });
76
+ caseEvidence = await executeContextWindowScalingCases({
77
+ cases: subject.evaluationCases,
78
+ contextBudgets: args.contextBudgets,
79
+ kitCommand: args.kitCommand,
80
+ outputRoot: scratch.path,
81
+ projectProfiles,
82
+ cwd: scratch.path,
83
+ env: args.env,
84
+ dependencies: { ...createSafeLocalDependencies(subject, guidedExclusions), ...args.dependencies },
85
+ });
86
+ }
87
+ catch (error) {
88
+ issues.push({
89
+ code: "EXECUTION_FAILED",
90
+ message: redact(`execution threw: ${error instanceof Error ? error.message : String(error)}`),
91
+ });
92
+ }
93
+ if (caseEvidence) {
94
+ for (const caseResult of caseEvidence) {
95
+ for (const treatment of caseResult.treatments) {
96
+ if (treatment.status === "failed" || treatment.errors.length > 0) {
97
+ issues.push({
98
+ code: "EXECUTION_FAILED",
99
+ message: redact(`case ${caseResult.caseId} treatment ${treatment.variantId} failed: ${treatment.errors
100
+ .map((entry) => `${entry.code}: ${entry.message}`)
101
+ .join("; ")}`),
102
+ });
103
+ }
104
+ }
105
+ }
106
+ }
107
+ let comparison = null;
108
+ const after = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
109
+ if (!after.ok) {
110
+ issues.push({ code: "AFTER_SNAPSHOT_FAILED", message: `target snapshot after execution failed (${after.code}).` });
111
+ }
112
+ else {
113
+ comparison = compareTargetSnapshots(before.snapshot, after.snapshot);
114
+ if (comparison.status === "mutated") {
115
+ issues.push({
116
+ code: "TARGET_MUTATED",
117
+ message: `the target changed during execution (${comparison.mutations.map((mutation) => mutation.id).join(", ")}); it was not restored.`,
118
+ });
119
+ }
120
+ }
121
+ const cleanupFailure = await removePrivateScratch(scratch, args.scratchIo ?? defaultPrivateScratchIo);
122
+ if (cleanupFailure) {
123
+ issues.push({ code: "SCRATCH_CLEANUP_FAILED", message: redact(cleanupFailure) });
124
+ }
125
+ if (issues.length > 0 || !caseEvidence || !comparison) {
126
+ throw new LocalSubjectExecutionError(issues, { caseEvidence, immutability: comparison });
127
+ }
128
+ return {
129
+ caseEvidence,
130
+ immutability: {
131
+ status: "unchanged",
132
+ preExistingGitStatusEntryCount: comparison.preExistingGitStatusEntryCount,
133
+ newMutationCount: 0,
134
+ },
135
+ };
136
+ }