@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/dist/src/evaluation/agentSuccess/validatePatchPolicy.js

@@ -0,0 +1,66 @@
+import { normalizeProjectRelativePath } from "./taskPaths.js";
+import { DEFAULT_PATCH_BOUNDS } from "./patchTypes.js";
+const MAX_REPORTED_REJECTIONS = 20;
+const SYMLINK_MODE = "120000";
+const GITLINK_MODE = "160000";
+function describeStatus(file) {
+    if (file.isNew || (file.oldPath === null && file.newPath !== null))
+        return "added";
+    if (file.isDelete || (file.newPath === null && file.oldPath !== null))
+        return "deleted";
+    return "modified";
+}
+/**
+ * Applies the Lab's patch policy to a parsed diff. Everything a patch could use to escape the benchmark
+ * copy or to alter protected files is rejected here, before Git sees the patch. A file outside the task's
+ * allowed edit scope is NOT rejected: such edits are legitimate evidence for later measurement.
+ */
+export function validatePatchPolicy(files, options) {
+    const bounds = options.bounds ?? DEFAULT_PATCH_BOUNDS;
+    const rejections = [];
+    const reject = (rejection) => {
+        if (rejections.length < MAX_REPORTED_REJECTIONS)
+            rejections.push(rejection);
+    };
+    const protectedSet = new Set();
+    for (const entry of options.protectedFiles) {
+        const normalized = normalizeProjectRelativePath(entry);
+        if (normalized.ok)
+            protectedSet.add(normalized.path);
+    }
+    const accepted = new Map();
+    for (const file of files) {
+        const label = file.newPath ?? file.oldPath ?? file.gitHeaderPath ?? undefined;
+        if (file.binary)
+            reject({ code: "BINARY_PATCH", path: label, message: "binary patches are not accepted." });
+        if (file.rename || file.copy)
+            reject({ code: "RENAME_OR_COPY", path: label, message: "rename and copy patches are not accepted." });
+        if (file.oldPath !== null && file.newPath !== null && file.oldPath !== file.newPath) {
+            reject({ code: "PATH_MISMATCH", path: label, message: "the old and new paths differ." });
+        }
+        if (file.oldMode === SYMLINK_MODE || file.newMode === SYMLINK_MODE) {
+            reject({ code: "SYMLINK_MODE", path: label, message: "symbolic-link entries are not accepted." });
+        }
+        if (file.oldMode === GITLINK_MODE || file.newMode === GITLINK_MODE || file.subproject) {
+            reject({ code: "SUBMODULE", path: label, message: "submodule/gitlink changes are not accepted." });
+        }
+        const candidates = new Set([file.oldPath, file.newPath, file.oldPath === null && file.newPath === null ? file.gitHeaderPath : null].filter((value) => value !== null));
+        for (const candidate of candidates) {
+            const normalized = normalizeProjectRelativePath(candidate);
+            if (!normalized.ok) {
+                reject({ code: "UNSAFE_PATH", message: `unsafe patch path (${normalized.code}).` });
+                continue;
+            }
+            if (protectedSet.has(normalized.path)) {
+                reject({ code: "PROTECTED_PATH", path: normalized.path, message: "the patch touches a protected file." });
+            }
+            accepted.set(normalized.path, describeStatus(file));
+        }
+    }
+    if (accepted.size > bounds.maxChangedPaths) {
+        rejections.unshift({ code: "TOO_MANY_PATHS", message: `the patch changes more than ${bounds.maxChangedPaths} paths.` });
+    }
+    if (rejections.length > 0)
+        return { ok: false, rejections: rejections.slice(0, MAX_REPORTED_REJECTIONS) };
+    return { ok: true, files: [...accepted].map(([path, status]) => ({ path, status })).sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)) };
+}

package/dist/src/evaluation/agentSuccess/verificationTypes.js

@@ -0,0 +1 @@
+export const VERIFICATION_OUTPUT_LIMIT_BYTES = 1_048_576;

package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js

@@ -0,0 +1,127 @@
+import { constants as fsConstants } from "node:fs";
+import { copyFile, lstat, mkdir, realpath, rm, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../core/pathSafety.js";
+import { BenchmarkSandboxError } from "./errors.js";
+import { excerptGitFailure, runSandboxGit } from "./gitExecutor.js";
+import { BENCHMARK_SANDBOX_EXCLUDED_NAMES, isSameOrInside, resolvePhysicalPath, SANDBOX_ID_PATTERN } from "./pathPolicy.js";
+import { digestTreeSnapshot, scanProjectTree, snapshotProjectTree } from "./treeSnapshot.js";
+import { BENCHMARK_SANDBOX_BASELINE_BRANCH, BENCHMARK_SANDBOX_OWNER_MARKER_FILE, BENCHMARK_SANDBOX_SCHEMA_VERSION } from "./types.js";
+const BASELINE_COMMIT_MESSAGE = "benchmark sandbox baseline";
+/** Fixed dates make the baseline commit id a pure function of the copied content, so repeated runs agree. */
+const BASELINE_COMMIT_DATE_ENV = Object.freeze({ GIT_AUTHOR_DATE: "2000-01-01T00:00:00Z", GIT_COMMITTER_DATE: "2000-01-01T00:00:00Z" });
+async function pathExists(target) {
+    try {
+        await lstat(target);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function resolveCanonicalProjectRoot(canonicalProjectRoot) {
+    let stats;
+    try {
+        stats = await lstat(path.resolve(canonicalProjectRoot));
+    }
+    catch {
+        throw new BenchmarkSandboxError("CANONICAL_ROOT_INVALID", "the canonical project root does not exist.");
+    }
+    if (stats.isSymbolicLink() || !(await stat(path.resolve(canonicalProjectRoot))).isDirectory()) {
+        throw new BenchmarkSandboxError("CANONICAL_ROOT_INVALID", "the canonical project root must be a real directory.");
+    }
+    return realpath(path.resolve(canonicalProjectRoot));
+}
+async function mustGit(target, args, label, env) {
+    const result = await runSandboxGit(target, args, { label, env });
+    if (result.unavailable) {
+        throw new BenchmarkSandboxError("GIT_UNAVAILABLE", "the git executable is not available.");
+    }
+    if (!result.ok) {
+        throw new BenchmarkSandboxError("GIT_FAILED", `git ${args[0] ?? ""} failed: ${excerptGitFailure(result)}`);
+    }
+    return result;
+}
+/**
+ * Creates one independent, Git-backed disposable copy of a controlled benchmark project inside the
+ * caller-supplied runtime root. Nothing is ever written to the canonical project, and the runtime root is
+ * never defaulted. Order: validate -> scan source (reject links) -> copy -> manifest -> fresh Git baseline.
+ * Any failure after the sandbox directory exists removes that (owned) directory before rethrowing.
+ */
+export async function createBenchmarkSandbox(options) {
+    if (!SANDBOX_ID_PATTERN.test(options.sandboxId)) {
+        throw new BenchmarkSandboxError("INVALID_SANDBOX_ID", `sandbox id must match ${SANDBOX_ID_PATTERN.source}.`);
+    }
+    const canonicalProjectRoot = await resolveCanonicalProjectRoot(options.canonicalProjectRoot);
+    const runtimeRoot = path.resolve(options.runtimeRoot);
+    const physicalRuntimeRoot = await resolvePhysicalPath(runtimeRoot);
+    if (isSameOrInside(canonicalProjectRoot, physicalRuntimeRoot) || isSameOrInside(physicalRuntimeRoot, canonicalProjectRoot)) {
+        throw new BenchmarkSandboxError("RUNTIME_OVERLAP", "the runtime root and the canonical project must not overlap.");
+    }
+    const sandboxRoot = resolveWithinRoot(runtimeRoot, options.sandboxId);
+    if (await pathExists(sandboxRoot)) {
+        throw new BenchmarkSandboxError("SANDBOX_EXISTS", "refusing to overwrite an existing sandbox directory.");
+    }
+    // Reject links and unsupported entries before anything is created under the runtime root.
+    const sourceFiles = await scanProjectTree(canonicalProjectRoot, BENCHMARK_SANDBOX_EXCLUDED_NAMES);
+    if (sourceFiles.length === 0) {
+        throw new BenchmarkSandboxError("EMPTY_PROJECT", "the canonical project has no copyable files.");
+    }
+    await mkdir(runtimeRoot, { recursive: true });
+    await mkdir(sandboxRoot);
+    try {
+        const marker = {
+            schemaVersion: BENCHMARK_SANDBOX_SCHEMA_VERSION,
+            sandboxId: options.sandboxId,
+            runtimeRoot,
+            canonicalProjectRoot
+        };
+        await writeFile(path.join(sandboxRoot, BENCHMARK_SANDBOX_OWNER_MARKER_FILE), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
+        const projectRoot = path.join(sandboxRoot, "project");
+        const evidenceRoot = path.join(sandboxRoot, "evidence");
+        await mkdir(projectRoot);
+        await mkdir(evidenceRoot);
+        for (const relativePath of sourceFiles) {
+            const segments = relativePath.split("/");
+            const destination = path.join(projectRoot, ...segments);
+            await mkdir(path.dirname(destination), { recursive: true });
+            await copyFile(path.join(canonicalProjectRoot, ...segments), destination, fsConstants.COPYFILE_EXCL);
+        }
+        const manifest = await snapshotProjectTree(projectRoot, { excludedNames: [".git"] });
+        if (manifest.length !== sourceFiles.length || manifest.some((entry, index) => entry.path !== sourceFiles[index])) {
+            throw new BenchmarkSandboxError("COPY_MISMATCH", "the copied tree does not match the scanned source file list.");
+        }
+        const git = { projectRoot, evidenceRoot };
+        await mustGit(git, ["init", "-q", `--initial-branch=${BENCHMARK_SANDBOX_BASELINE_BRANCH}`], "init");
+        await mustGit(git, ["config", "user.name", "my-dev-kit-lab sandbox"], "config-name");
+        await mustGit(git, ["config", "user.email", "sandbox@my-dev-kit-lab.invalid"], "config-email");
+        await mustGit(git, ["config", "commit.gpgsign", "false"], "config-gpg");
+        await mustGit(git, ["config", "core.autocrlf", "false"], "config-autocrlf");
+        await mustGit(git, ["config", "core.longpaths", "true"], "config-longpaths");
+        await mustGit(git, ["config", "core.symlinks", "false"], "config-symlinks");
+        await mustGit(git, ["add", "-A", "--force"], "add");
+        await mustGit(git, ["commit", "-q", "--no-verify", "-m", BASELINE_COMMIT_MESSAGE], "commit", BASELINE_COMMIT_DATE_ENV);
+        const commit = (await mustGit(git, ["rev-parse", "HEAD"], "rev-parse")).stdout.trim();
+        const status = await mustGit(git, ["status", "--porcelain"], "status");
+        if (status.stdout.trim() !== "") {
+            throw new BenchmarkSandboxError("BASELINE_NOT_CLEAN", "git status is not clean after the baseline commit.");
+        }
+        return {
+            schemaVersion: BENCHMARK_SANDBOX_SCHEMA_VERSION,
+            sandboxId: options.sandboxId,
+            sandboxRoot,
+            projectRoot,
+            evidenceRoot,
+            runtimeRoot,
+            canonicalProjectRoot,
+            baseline: { commit, fileCount: manifest.length, digest: digestTreeSnapshot(manifest), manifest }
+        };
+    }
+    catch (error) {
+        // Only this call's own directory exists at this point; removal cannot touch anything else.
+        await rm(sandboxRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }).catch(() => undefined);
+        if (error instanceof BenchmarkSandboxError)
+            throw error;
+        throw new BenchmarkSandboxError("COPY_FAILED", error instanceof Error ? error.message : String(error));
+    }
+}

package/dist/src/evaluation/benchmarkSandbox/errors.js

@@ -0,0 +1,11 @@
+const MAX_MESSAGE_LENGTH = 500;
+/** Bounded, typed failure of sandbox creation, baseline, or tree inspection. */
+export class BenchmarkSandboxError extends Error {
+    code;
+    constructor(code, message) {
+        const bounded = message.length > MAX_MESSAGE_LENGTH ? `${message.slice(0, MAX_MESSAGE_LENGTH)}...` : message;
+        super(`Benchmark sandbox error (${code}): ${bounded}`);
+        this.name = "BenchmarkSandboxError";
+        this.code = code;
+    }
+}

package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js

@@ -0,0 +1,60 @@
+import path from "node:path";
+import { runMeasuredCommand } from "../../core/runMeasuredCommand.js";
+import { buildMinimalHostEnv } from "./minimalHostEnv.js";
+// Git (including Git for Windows) opens this as an empty config file; os.devNull (\\.\nul) is rejected by Git on Windows.
+const NULL_CONFIG_PATH = process.platform === "win32" ? "NUL" : "/dev/null";
+const DEFAULT_GIT_TIMEOUT_MS = 60_000;
+const GIT_OUTPUT_LIMIT_BYTES = 8 * 1024 * 1024;
+const sequenceByEvidenceRoot = new Map();
+function nextSequence(evidenceRoot) {
+    const next = (sequenceByEvidenceRoot.get(evidenceRoot) ?? 0) + 1;
+    sequenceByEvidenceRoot.set(evidenceRoot, next);
+    return next;
+}
+/**
+ * Runs Git for a sandbox through the shared measured-command owner (structured arguments, shell:false,
+ * bounded output, awaited tree termination, evidence files). The environment is isolated from the user's
+ * and the system's Git configuration so hashes and diffs do not depend on the developer machine.
+ */
+export async function runSandboxGit(target, args, options = {}) {
+    const label = (options.label ?? args[0] ?? "git").replace(/[^A-Za-z0-9_-]/g, "-").slice(0, 32);
+    const commandId = `git-${String(nextSequence(target.evidenceRoot)).padStart(4, "0")}-${label}`;
+    const env = buildMinimalHostEnv(process.env, {
+        GIT_CONFIG_NOSYSTEM: "1",
+        GIT_CONFIG_GLOBAL: NULL_CONFIG_PATH,
+        GIT_TERMINAL_PROMPT: "0",
+        GIT_OPTIONAL_LOCKS: "0",
+        HOME: target.evidenceRoot,
+        USERPROFILE: target.evidenceRoot,
+        ...options.env
+    });
+    const result = await runMeasuredCommand({
+        commandId,
+        executable: "git",
+        args: ["-c", "core.quotepath=false", "-c", "core.autocrlf=false", "-c", "core.fsmonitor=false", ...args],
+        cwd: target.projectRoot,
+        outDir: path.join(target.evidenceRoot, "git"),
+        env,
+        inheritParentEnv: false,
+        stdinText: options.stdin,
+        timeoutMs: options.timeoutMs ?? DEFAULT_GIT_TIMEOUT_MS,
+        stdoutMaxBytes: GIT_OUTPUT_LIMIT_BYTES,
+        stderrMaxBytes: GIT_OUTPUT_LIMIT_BYTES,
+        terminateProcessTree: true
+    });
+    const unavailable = result.resolvedCommand?.resolutionKind === "unavailable";
+    return {
+        ok: result.ok,
+        exitCode: result.exitCode,
+        stdout: result.stdout,
+        stderr: result.stderr,
+        timedOut: result.timedOut === true,
+        unavailable,
+        error: result.error
+    };
+}
+/** Bounded single-line excerpt of Git diagnostics for error messages. */
+export function excerptGitFailure(result) {
+    const text = (result.stderr || result.stdout || result.error || "no output").trim().replace(/\s+/g, " ");
+    return text.length > 300 ? `${text.slice(0, 300)}...` : text;
+}

package/dist/src/evaluation/benchmarkSandbox/index.js

@@ -0,0 +1,8 @@
+export * from "./errors.js";
+export * from "./types.js";
+export { createBenchmarkSandbox } from "./createBenchmarkSandbox.js";
+export { removeBenchmarkSandbox } from "./removeBenchmarkSandbox.js";
+export { digestTreeSnapshot, scanProjectTree, sha256OfBuffer, snapshotProjectTree } from "./treeSnapshot.js";
+export { buildMinimalHostEnv } from "./minimalHostEnv.js";
+export { excerptGitFailure, runSandboxGit } from "./gitExecutor.js";
+export { BENCHMARK_SANDBOX_EXCLUDED_NAMES, SANDBOX_ID_PATTERN } from "./pathPolicy.js";

package/dist/src/evaluation/benchmarkSandbox/minimalHostEnv.js

@@ -0,0 +1,37 @@
+/**
+ * Variables a child process needs to locate and start `node` or `git` on the supported platforms. Nothing
+ * else is forwarded from the parent: provider keys, tokens, cookies and project secrets must never reach
+ * Lab-run verification or Git processes. Names are matched case-insensitively (Windows environments are).
+ */
+const FORWARDED_NAMES = [
+    "path",
+    "systemroot",
+    "systemdrive",
+    "windir",
+    "comspec",
+    "pathext",
+    "temp",
+    "tmp",
+    "tmpdir",
+    "lang",
+    "lc_all"
+];
+/** Defense in depth: even an allowed name is dropped if it looks credential-like. */
+const CREDENTIAL_LIKE = /(key|token|secret|passw|credential|auth|cookie|session)/i;
+export function buildMinimalHostEnv(source = process.env, extras = {}) {
+    const env = {};
+    const actualNames = Object.keys(source);
+    for (const wanted of FORWARDED_NAMES) {
+        const actual = actualNames.find((name) => name.toLowerCase() === wanted);
+        if (actual === undefined)
+            continue;
+        const value = source[actual];
+        if (value === undefined || CREDENTIAL_LIKE.test(actual))
+            continue;
+        env[actual] = value;
+    }
+    for (const [name, value] of Object.entries(extras)) {
+        env[name] = value;
+    }
+    return env;
+}

package/dist/src/evaluation/benchmarkSandbox/pathPolicy.js

@@ -0,0 +1,49 @@
+import { realpath } from "node:fs/promises";
+import path from "node:path";
+/** Sandbox identities become one directory name beneath the caller's runtime root. */
+export const SANDBOX_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+/**
+ * Directory/file names never copied into a sandbox and never part of its baseline. `.git` would import the
+ * canonical history; the rest are generated or Lab-internal outputs that are not benchmark source.
+ */
+export const BENCHMARK_SANDBOX_EXCLUDED_NAMES = Object.freeze([
+    ".git",
+    "node_modules",
+    "dist",
+    "build",
+    "coverage",
+    "lab-output",
+    ".my-dev-kit-context",
+    ".my-dev-kit-orchestrator"
+]);
+/**
+ * Physical path of a path that may not exist yet: the realpath of the deepest existing ancestor plus the
+ * not-yet-created remainder. Resolving physically (not lexically) is what makes overlap checks hold across
+ * links, junctions, and Windows drive aliases.
+ */
+export async function resolvePhysicalPath(candidate) {
+    const resolved = path.resolve(candidate);
+    const remainder = [];
+    let current = resolved;
+    for (;;) {
+        try {
+            const physical = await realpath(current);
+            return remainder.length === 0 ? physical : path.join(physical, ...remainder.reverse());
+        }
+        catch (error) {
+            const code = error.code;
+            const parent = path.dirname(current);
+            if ((code !== "ENOENT" && code !== "ENOTDIR") || parent === current)
+                return resolved;
+            remainder.push(path.basename(current));
+            current = parent;
+        }
+    }
+}
+/** True when `candidate` equals `root` or lies beneath it. Both arguments must already be physical paths. */
+export function isSameOrInside(root, candidate) {
+    const relative = path.relative(root, candidate);
+    if (relative === "")
+        return true;
+    return !(relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative));
+}

package/dist/src/evaluation/benchmarkSandbox/removeBenchmarkSandbox.js

@@ -0,0 +1,74 @@
+import { lstat, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../core/pathSafety.js";
+import { isSameOrInside, resolvePhysicalPath, SANDBOX_ID_PATTERN } from "./pathPolicy.js";
+import { BENCHMARK_SANDBOX_OWNER_MARKER_FILE, BENCHMARK_SANDBOX_SCHEMA_VERSION } from "./types.js";
+/**
+ * Removes one sandbox, and only a sandbox this subsystem created: the directory must be a real directory
+ * directly inside the runtime root and carry an ownership marker whose schema, sandbox id and runtime root
+ * match. Any mismatch is a refusal that leaves everything on disk untouched. The runtime root itself is
+ * never removed.
+ */
+export async function removeBenchmarkSandbox(options) {
+    if (!SANDBOX_ID_PATTERN.test(options.sandboxId)) {
+        return { removed: false, reason: "invalid sandbox id." };
+    }
+    const runtimeRoot = path.resolve(options.runtimeRoot);
+    let sandboxRoot;
+    try {
+        sandboxRoot = resolveWithinRoot(runtimeRoot, options.sandboxId);
+    }
+    catch {
+        return { removed: false, reason: "sandbox path escapes the runtime root." };
+    }
+    let stats;
+    try {
+        stats = await lstat(sandboxRoot);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            try {
+                await lstat(runtimeRoot);
+                return { removed: true, alreadyAbsent: true };
+            }
+            catch {
+                return { removed: false, reason: "the runtime root does not exist." };
+            }
+        }
+        return { removed: false, reason: "the sandbox path could not be inspected." };
+    }
+    if (stats.isSymbolicLink() || !stats.isDirectory()) {
+        return { removed: false, reason: "the sandbox path is not a plain directory." };
+    }
+    if (!isSameOrInside(await resolvePhysicalPath(runtimeRoot), await resolvePhysicalPath(sandboxRoot))) {
+        return { removed: false, reason: "the sandbox resolves outside the runtime root." };
+    }
+    let marker;
+    try {
+        marker = JSON.parse(await readFile(path.join(sandboxRoot, BENCHMARK_SANDBOX_OWNER_MARKER_FILE), "utf8"));
+    }
+    catch {
+        return { removed: false, reason: "no readable ownership marker; the directory is not a recognized sandbox." };
+    }
+    if (marker.schemaVersion !== BENCHMARK_SANDBOX_SCHEMA_VERSION ||
+        marker.sandboxId !== options.sandboxId ||
+        typeof marker.runtimeRoot !== "string" ||
+        path.resolve(marker.runtimeRoot) !== runtimeRoot) {
+        return { removed: false, reason: "the ownership marker does not match this sandbox." };
+    }
+    try {
+        await rm(sandboxRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+    }
+    catch (error) {
+        return { removed: false, reason: `removal failed: ${error instanceof Error ? error.message : String(error)}` };
+    }
+    try {
+        await lstat(sandboxRoot);
+        return { removed: false, reason: "the sandbox directory still exists after removal." };
+    }
+    catch (error) {
+        return error.code === "ENOENT"
+            ? { removed: true, alreadyAbsent: false }
+            : { removed: false, reason: "the sandbox state could not be verified after removal." };
+    }
+}

package/dist/src/evaluation/benchmarkSandbox/treeSnapshot.js

@@ -0,0 +1,57 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { BenchmarkSandboxError } from "./errors.js";
+export function sha256OfBuffer(data) {
+    return createHash("sha256").update(data).digest("hex");
+}
+function compareCodeUnits(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+/**
+ * Lists every regular file below `root` as sorted project-relative POSIX paths. Entries whose name is in
+ * `excludedNames` are skipped at any depth. A symbolic link or junction anywhere else is rejected rather
+ * than followed (a link could point outside the project), as is any entry that is neither a file nor a
+ * directory.
+ */
+export async function scanProjectTree(root, excludedNames) {
+    const excluded = new Set(excludedNames);
+    const files = [];
+    async function walk(absoluteDir, relativeDir) {
+        const entries = await readdir(absoluteDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (excluded.has(entry.name))
+                continue;
+            const relativePath = relativeDir === "" ? entry.name : `${relativeDir}/${entry.name}`;
+            if (entry.isSymbolicLink()) {
+                throw new BenchmarkSandboxError("SYMLINK_REJECTED", `symbolic link or junction at ${relativePath}`);
+            }
+            if (entry.isDirectory()) {
+                await walk(path.join(absoluteDir, entry.name), relativePath);
+            }
+            else if (entry.isFile()) {
+                files.push(relativePath);
+            }
+            else {
+                throw new BenchmarkSandboxError("UNSUPPORTED_ENTRY", `unsupported entry type at ${relativePath}`);
+            }
+        }
+    }
+    await walk(root, "");
+    return files.sort(compareCodeUnits);
+}
+/** Deterministic content identity of a project tree: sorted paths with sha256 and size of every file. */
+export async function snapshotProjectTree(root, options) {
+    const files = await scanProjectTree(root, options.excludedNames);
+    const entries = [];
+    for (const relativePath of files) {
+        const data = await readFile(path.join(root, ...relativePath.split("/")));
+        entries.push({ path: relativePath, sha256: sha256OfBuffer(data), size: data.length });
+    }
+    return entries;
+}
+/** Aggregate digest over sorted "path<TAB>sha256<LF>" lines (same convention as the staleness source state). */
+export function digestTreeSnapshot(entries) {
+    const serialized = entries.map((entry) => `${entry.path}\t${entry.sha256}\n`).join("");
+    return createHash("sha256").update(Buffer.from(serialized, "utf8")).digest("hex");
+}

package/dist/src/evaluation/benchmarkSandbox/types.js

@@ -0,0 +1,3 @@
+export const BENCHMARK_SANDBOX_SCHEMA_VERSION = "my-dev-kit-lab-benchmark-sandbox-v1";
+export const BENCHMARK_SANDBOX_OWNER_MARKER_FILE = "sandbox-owner.json";
+export const BENCHMARK_SANDBOX_BASELINE_BRANCH = "sandbox-baseline";

package/dist/src/evaluation/changeSet/captureChangeSet.js

@@ -0,0 +1,98 @@
+import { BenchmarkSandboxError, excerptGitFailure, runSandboxGit, snapshotProjectTree } from "../benchmarkSandbox/index.js";
+import { CHANGE_SET_SCHEMA_VERSION, ChangeSetError } from "./types.js";
+function compareCodeUnits(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+async function git(sandbox, args, label) {
+    const result = await runSandboxGit(sandbox, args, { label });
+    if (!result.ok) {
+        throw new ChangeSetError("GIT_FAILED", `git ${args[0] ?? ""} failed: ${excerptGitFailure(result)}`);
+    }
+    return result;
+}
+/** Parses `git diff --numstat -z --no-renames`: records of "added<TAB>deleted<TAB>path" separated by NUL. */
+function parseNumstat(output) {
+    const stats = new Map();
+    for (const record of output.split("\0")) {
+        if (record === "")
+            continue;
+        const firstTab = record.indexOf("\t");
+        const secondTab = firstTab < 0 ? -1 : record.indexOf("\t", firstTab + 1);
+        if (firstTab < 0 || secondTab < 0) {
+            throw new ChangeSetError("INCONSISTENT_EVIDENCE", "unrecognized numstat record.");
+        }
+        const added = record.slice(0, firstTab);
+        const deleted = record.slice(firstTab + 1, secondTab);
+        stats.set(record.slice(secondTab + 1), {
+            additions: added === "-" ? null : Number(added),
+            deletions: deleted === "-" ? null : Number(deleted)
+        });
+    }
+    return stats;
+}
+/**
+ * Captures what changed in a sandbox relative to its own baseline commit. File status and hashes come from
+ * comparing the baseline manifest with the working tree (independent of Git ignore rules and of anything the
+ * agent claimed); line counts and the cumulative diff come from Git against the baseline commit with rename
+ * detection disabled. The two views must agree or capture fails rather than reporting doubtful evidence.
+ *
+ * Call it right after patch application: it stages the working tree (`git add -A --force`) and reflects
+ * whatever exists on disk at that moment, including files produced by later verification runs.
+ */
+export async function captureChangeSet(options) {
+    const { sandbox } = options;
+    let working;
+    try {
+        working = await snapshotProjectTree(sandbox.projectRoot, { excludedNames: [".git"] });
+    }
+    catch (error) {
+        if (error instanceof BenchmarkSandboxError && (error.code === "SYMLINK_REJECTED" || error.code === "UNSUPPORTED_ENTRY")) {
+            throw new ChangeSetError("SYMLINK_IN_SANDBOX", error.message);
+        }
+        throw error;
+    }
+    const baselineByPath = new Map(sandbox.baseline.manifest.map((entry) => [entry.path, entry.sha256]));
+    const workingByPath = new Map(working.map((entry) => [entry.path, entry.sha256]));
+    const records = [];
+    for (const [relativePath, before] of baselineByPath) {
+        const after = workingByPath.get(relativePath);
+        if (after === undefined) {
+            records.push({ relativePath, status: "deleted", beforeSha256: before, afterSha256: null });
+        }
+        else if (after !== before) {
+            records.push({ relativePath, status: "modified", beforeSha256: before, afterSha256: after });
+        }
+    }
+    for (const [relativePath, after] of workingByPath) {
+        if (!baselineByPath.has(relativePath)) {
+            records.push({ relativePath, status: "added", beforeSha256: null, afterSha256: after });
+        }
+    }
+    records.sort((left, right) => compareCodeUnits(left.relativePath, right.relativePath));
+    await git(sandbox, ["add", "-A", "--force"], "stage");
+    const numstat = parseNumstat((await git(sandbox, ["diff", "--cached", "--numstat", "--no-renames", "-z", sandbox.baseline.commit], "numstat")).stdout);
+    const diffResult = await git(sandbox, ["diff", "--cached", "--no-renames", "--no-ext-diff", "--no-color", "--src-prefix=a/", "--dst-prefix=b/", sandbox.baseline.commit], "diff");
+    const changedFiles = records.map((record) => {
+        const stats = numstat.get(record.relativePath);
+        if (stats === undefined) {
+            throw new ChangeSetError("INCONSISTENT_EVIDENCE", `Git reports no change for ${record.relativePath}, which differs from the baseline manifest.`);
+        }
+        return { ...record, additions: stats.additions, deletions: stats.deletions };
+    });
+    const count = (status) => changedFiles.filter((file) => file.status === status).length;
+    const sum = (pick) => changedFiles.reduce((total, file) => total + (pick(file) ?? 0), 0);
+    const diff = diffResult.stdout === "" || diffResult.stdout.endsWith("\n") ? diffResult.stdout : `${diffResult.stdout}\n`;
+    return {
+        schemaVersion: CHANGE_SET_SCHEMA_VERSION,
+        sandboxId: sandbox.sandboxId,
+        baselineCommit: sandbox.baseline.commit,
+        changedFiles,
+        addedCount: count("added"),
+        modifiedCount: count("modified"),
+        deletedCount: count("deleted"),
+        changedCount: changedFiles.length,
+        totalAdditions: sum((file) => file.additions),
+        totalDeletions: sum((file) => file.deletions),
+        diff
+    };
+}

package/dist/src/evaluation/changeSet/index.js

@@ -0,0 +1,2 @@
+export * from "./types.js";
+export { captureChangeSet } from "./captureChangeSet.js";

package/dist/src/evaluation/changeSet/types.js

@@ -0,0 +1,9 @@
+export const CHANGE_SET_SCHEMA_VERSION = "my-dev-kit-lab-change-set-v1";
+export class ChangeSetError extends Error {
+    code;
+    constructor(code, message) {
+        super(`Change set error (${code}): ${message.length > 500 ? `${message.slice(0, 500)}...` : message}`);
+        this.name = "ChangeSetError";
+        this.code = code;
+    }
+}

package/dist/src/evaluation/runMyDevKitRetrieval.js

@@ -183,6 +183,7 @@ export async function buildMyDevKitIndex(options) {
             ...(options.excludePaths ?? []).flatMap((excludePath) => ["--exclude", excludePath]),
             "--out",
             options.indexDir,
+            ...(options.callGraph === true ? ["--call-graph"] : []),
             ...(mode.kind === "incremental" ? ["--incremental", "--refresh-scope", mode.refreshScope] : []),
             "--json"
         ]

package/dist/src/experiments/defaultRegistry.js

@@ -1,5 +1,7 @@
 import { ExperimentPluginRegistry } from "./registry.js";
+import { agentSuccessRatePlugin } from "./plugins/agentSuccessRate/index.js";
 import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
+import { contextPackGenerationPlugin } from "./plugins/contextPackGeneration/index.js";
 import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
 import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
 import { retrievalPrecisionRecallPlugin } from "./plugins/retrievalPrecisionRecall/index.js";
@@ -13,5 +15,7 @@ export function createDefaultExperimentPluginRegistry() {
     registry.register(contextWindowScalingPlugin);
     registry.register(retrievalPrecisionRecallPlugin);
     registry.register(retrievalQueryStrategyComparisonPlugin);
+    registry.register(contextPackGenerationPlugin);
+    registry.register(agentSuccessRatePlugin);
     return registry;
 }

package/dist/src/experiments/plugins/agentSuccessRate/agentTaskProjection.js

@@ -0,0 +1,12 @@
+export const AGENT_FACING_TASK_FIELDS = ["caseId", "benchmarkProject", "title", "instruction", "query", "taskLocality"];
+/** Builds a fresh object from named fields. It never spreads, serializes or iterates the source task. */
+export function projectAgentFacingTask(task) {
+    return Object.freeze({
+        caseId: task.id,
+        benchmarkProject: task.benchmarkProject,
+        title: task.title,
+        instruction: task.instruction,
+        query: task.query,
+        taskLocality: task.taskLocality
+    });
+}