@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/dist/src/evaluation/benchmarkSandbox/removeBenchmarkSandbox.js

@@ -0,0 +1,74 @@
+import { lstat, readFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../core/pathSafety.js";
+import { isSameOrInside, resolvePhysicalPath, SANDBOX_ID_PATTERN } from "./pathPolicy.js";
+import { BENCHMARK_SANDBOX_OWNER_MARKER_FILE, BENCHMARK_SANDBOX_SCHEMA_VERSION } from "./types.js";
+/**
+ * Removes one sandbox, and only a sandbox this subsystem created: the directory must be a real directory
+ * directly inside the runtime root and carry an ownership marker whose schema, sandbox id and runtime root
+ * match. Any mismatch is a refusal that leaves everything on disk untouched. The runtime root itself is
+ * never removed.
+ */
+export async function removeBenchmarkSandbox(options) {
+    if (!SANDBOX_ID_PATTERN.test(options.sandboxId)) {
+        return { removed: false, reason: "invalid sandbox id." };
+    }
+    const runtimeRoot = path.resolve(options.runtimeRoot);
+    let sandboxRoot;
+    try {
+        sandboxRoot = resolveWithinRoot(runtimeRoot, options.sandboxId);
+    }
+    catch {
+        return { removed: false, reason: "sandbox path escapes the runtime root." };
+    }
+    let stats;
+    try {
+        stats = await lstat(sandboxRoot);
+    }
+    catch (error) {
+        if (error.code === "ENOENT") {
+            try {
+                await lstat(runtimeRoot);
+                return { removed: true, alreadyAbsent: true };
+            }
+            catch {
+                return { removed: false, reason: "the runtime root does not exist." };
+            }
+        }
+        return { removed: false, reason: "the sandbox path could not be inspected." };
+    }
+    if (stats.isSymbolicLink() || !stats.isDirectory()) {
+        return { removed: false, reason: "the sandbox path is not a plain directory." };
+    }
+    if (!isSameOrInside(await resolvePhysicalPath(runtimeRoot), await resolvePhysicalPath(sandboxRoot))) {
+        return { removed: false, reason: "the sandbox resolves outside the runtime root." };
+    }
+    let marker;
+    try {
+        marker = JSON.parse(await readFile(path.join(sandboxRoot, BENCHMARK_SANDBOX_OWNER_MARKER_FILE), "utf8"));
+    }
+    catch {
+        return { removed: false, reason: "no readable ownership marker; the directory is not a recognized sandbox." };
+    }
+    if (marker.schemaVersion !== BENCHMARK_SANDBOX_SCHEMA_VERSION ||
+        marker.sandboxId !== options.sandboxId ||
+        typeof marker.runtimeRoot !== "string" ||
+        path.resolve(marker.runtimeRoot) !== runtimeRoot) {
+        return { removed: false, reason: "the ownership marker does not match this sandbox." };
+    }
+    try {
+        await rm(sandboxRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+    }
+    catch (error) {
+        return { removed: false, reason: `removal failed: ${error instanceof Error ? error.message : String(error)}` };
+    }
+    try {
+        await lstat(sandboxRoot);
+        return { removed: false, reason: "the sandbox directory still exists after removal." };
+    }
+    catch (error) {
+        return error.code === "ENOENT"
+            ? { removed: true, alreadyAbsent: false }
+            : { removed: false, reason: "the sandbox state could not be verified after removal." };
+    }
+}

package/dist/src/evaluation/benchmarkSandbox/treeSnapshot.js

@@ -0,0 +1,57 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { BenchmarkSandboxError } from "./errors.js";
+export function sha256OfBuffer(data) {
+    return createHash("sha256").update(data).digest("hex");
+}
+function compareCodeUnits(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+/**
+ * Lists every regular file below `root` as sorted project-relative POSIX paths. Entries whose name is in
+ * `excludedNames` are skipped at any depth. A symbolic link or junction anywhere else is rejected rather
+ * than followed (a link could point outside the project), as is any entry that is neither a file nor a
+ * directory.
+ */
+export async function scanProjectTree(root, excludedNames) {
+    const excluded = new Set(excludedNames);
+    const files = [];
+    async function walk(absoluteDir, relativeDir) {
+        const entries = await readdir(absoluteDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (excluded.has(entry.name))
+                continue;
+            const relativePath = relativeDir === "" ? entry.name : `${relativeDir}/${entry.name}`;
+            if (entry.isSymbolicLink()) {
+                throw new BenchmarkSandboxError("SYMLINK_REJECTED", `symbolic link or junction at ${relativePath}`);
+            }
+            if (entry.isDirectory()) {
+                await walk(path.join(absoluteDir, entry.name), relativePath);
+            }
+            else if (entry.isFile()) {
+                files.push(relativePath);
+            }
+            else {
+                throw new BenchmarkSandboxError("UNSUPPORTED_ENTRY", `unsupported entry type at ${relativePath}`);
+            }
+        }
+    }
+    await walk(root, "");
+    return files.sort(compareCodeUnits);
+}
+/** Deterministic content identity of a project tree: sorted paths with sha256 and size of every file. */
+export async function snapshotProjectTree(root, options) {
+    const files = await scanProjectTree(root, options.excludedNames);
+    const entries = [];
+    for (const relativePath of files) {
+        const data = await readFile(path.join(root, ...relativePath.split("/")));
+        entries.push({ path: relativePath, sha256: sha256OfBuffer(data), size: data.length });
+    }
+    return entries;
+}
+/** Aggregate digest over sorted "path<TAB>sha256<LF>" lines (same convention as the staleness source state). */
+export function digestTreeSnapshot(entries) {
+    const serialized = entries.map((entry) => `${entry.path}\t${entry.sha256}\n`).join("");
+    return createHash("sha256").update(Buffer.from(serialized, "utf8")).digest("hex");
+}

package/dist/src/evaluation/benchmarkSandbox/types.js

@@ -0,0 +1,3 @@
+export const BENCHMARK_SANDBOX_SCHEMA_VERSION = "my-dev-kit-lab-benchmark-sandbox-v1";
+export const BENCHMARK_SANDBOX_OWNER_MARKER_FILE = "sandbox-owner.json";
+export const BENCHMARK_SANDBOX_BASELINE_BRANCH = "sandbox-baseline";

package/dist/src/evaluation/changeSet/captureChangeSet.js

@@ -0,0 +1,98 @@
+import { BenchmarkSandboxError, excerptGitFailure, runSandboxGit, snapshotProjectTree } from "../benchmarkSandbox/index.js";
+import { CHANGE_SET_SCHEMA_VERSION, ChangeSetError } from "./types.js";
+function compareCodeUnits(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+async function git(sandbox, args, label) {
+    const result = await runSandboxGit(sandbox, args, { label });
+    if (!result.ok) {
+        throw new ChangeSetError("GIT_FAILED", `git ${args[0] ?? ""} failed: ${excerptGitFailure(result)}`);
+    }
+    return result;
+}
+/** Parses `git diff --numstat -z --no-renames`: records of "added<TAB>deleted<TAB>path" separated by NUL. */
+function parseNumstat(output) {
+    const stats = new Map();
+    for (const record of output.split("\0")) {
+        if (record === "")
+            continue;
+        const firstTab = record.indexOf("\t");
+        const secondTab = firstTab < 0 ? -1 : record.indexOf("\t", firstTab + 1);
+        if (firstTab < 0 || secondTab < 0) {
+            throw new ChangeSetError("INCONSISTENT_EVIDENCE", "unrecognized numstat record.");
+        }
+        const added = record.slice(0, firstTab);
+        const deleted = record.slice(firstTab + 1, secondTab);
+        stats.set(record.slice(secondTab + 1), {
+            additions: added === "-" ? null : Number(added),
+            deletions: deleted === "-" ? null : Number(deleted)
+        });
+    }
+    return stats;
+}
+/**
+ * Captures what changed in a sandbox relative to its own baseline commit. File status and hashes come from
+ * comparing the baseline manifest with the working tree (independent of Git ignore rules and of anything the
+ * agent claimed); line counts and the cumulative diff come from Git against the baseline commit with rename
+ * detection disabled. The two views must agree or capture fails rather than reporting doubtful evidence.
+ *
+ * Call it right after patch application: it stages the working tree (`git add -A --force`) and reflects
+ * whatever exists on disk at that moment, including files produced by later verification runs.
+ */
+export async function captureChangeSet(options) {
+    const { sandbox } = options;
+    let working;
+    try {
+        working = await snapshotProjectTree(sandbox.projectRoot, { excludedNames: [".git"] });
+    }
+    catch (error) {
+        if (error instanceof BenchmarkSandboxError && (error.code === "SYMLINK_REJECTED" || error.code === "UNSUPPORTED_ENTRY")) {
+            throw new ChangeSetError("SYMLINK_IN_SANDBOX", error.message);
+        }
+        throw error;
+    }
+    const baselineByPath = new Map(sandbox.baseline.manifest.map((entry) => [entry.path, entry.sha256]));
+    const workingByPath = new Map(working.map((entry) => [entry.path, entry.sha256]));
+    const records = [];
+    for (const [relativePath, before] of baselineByPath) {
+        const after = workingByPath.get(relativePath);
+        if (after === undefined) {
+            records.push({ relativePath, status: "deleted", beforeSha256: before, afterSha256: null });
+        }
+        else if (after !== before) {
+            records.push({ relativePath, status: "modified", beforeSha256: before, afterSha256: after });
+        }
+    }
+    for (const [relativePath, after] of workingByPath) {
+        if (!baselineByPath.has(relativePath)) {
+            records.push({ relativePath, status: "added", beforeSha256: null, afterSha256: after });
+        }
+    }
+    records.sort((left, right) => compareCodeUnits(left.relativePath, right.relativePath));
+    await git(sandbox, ["add", "-A", "--force"], "stage");
+    const numstat = parseNumstat((await git(sandbox, ["diff", "--cached", "--numstat", "--no-renames", "-z", sandbox.baseline.commit], "numstat")).stdout);
+    const diffResult = await git(sandbox, ["diff", "--cached", "--no-renames", "--no-ext-diff", "--no-color", "--src-prefix=a/", "--dst-prefix=b/", sandbox.baseline.commit], "diff");
+    const changedFiles = records.map((record) => {
+        const stats = numstat.get(record.relativePath);
+        if (stats === undefined) {
+            throw new ChangeSetError("INCONSISTENT_EVIDENCE", `Git reports no change for ${record.relativePath}, which differs from the baseline manifest.`);
+        }
+        return { ...record, additions: stats.additions, deletions: stats.deletions };
+    });
+    const count = (status) => changedFiles.filter((file) => file.status === status).length;
+    const sum = (pick) => changedFiles.reduce((total, file) => total + (pick(file) ?? 0), 0);
+    const diff = diffResult.stdout === "" || diffResult.stdout.endsWith("\n") ? diffResult.stdout : `${diffResult.stdout}\n`;
+    return {
+        schemaVersion: CHANGE_SET_SCHEMA_VERSION,
+        sandboxId: sandbox.sandboxId,
+        baselineCommit: sandbox.baseline.commit,
+        changedFiles,
+        addedCount: count("added"),
+        modifiedCount: count("modified"),
+        deletedCount: count("deleted"),
+        changedCount: changedFiles.length,
+        totalAdditions: sum((file) => file.additions),
+        totalDeletions: sum((file) => file.deletions),
+        diff
+    };
+}

package/dist/src/evaluation/changeSet/index.js

@@ -0,0 +1,2 @@
+export * from "./types.js";
+export { captureChangeSet } from "./captureChangeSet.js";

package/dist/src/evaluation/changeSet/types.js

@@ -0,0 +1,9 @@
+export const CHANGE_SET_SCHEMA_VERSION = "my-dev-kit-lab-change-set-v1";
+export class ChangeSetError extends Error {
+    code;
+    constructor(code, message) {
+        super(`Change set error (${code}): ${message.length > 500 ? `${message.slice(0, 500)}...` : message}`);
+        this.name = "ChangeSetError";
+        this.code = code;
+    }
+}

package/dist/src/experiments/defaultRegistry.js

@@ -1,4 +1,5 @@
 import { ExperimentPluginRegistry } from "./registry.js";
+import { agentSuccessRatePlugin } from "./plugins/agentSuccessRate/index.js";
 import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
 import { contextPackGenerationPlugin } from "./plugins/contextPackGeneration/index.js";
 import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
@@ -15,5 +16,6 @@ export function createDefaultExperimentPluginRegistry() {
     registry.register(retrievalPrecisionRecallPlugin);
     registry.register(retrievalQueryStrategyComparisonPlugin);
     registry.register(contextPackGenerationPlugin);
+    registry.register(agentSuccessRatePlugin);
     return registry;
 }

package/dist/src/experiments/plugins/agentSuccessRate/agentTaskProjection.js

@@ -0,0 +1,12 @@
+export const AGENT_FACING_TASK_FIELDS = ["caseId", "benchmarkProject", "title", "instruction", "query", "taskLocality"];
+/** Builds a fresh object from named fields. It never spreads, serializes or iterates the source task. */
+export function projectAgentFacingTask(task) {
+    return Object.freeze({
+        caseId: task.id,
+        benchmarkProject: task.benchmarkProject,
+        title: task.title,
+        instruction: task.instruction,
+        query: task.query,
+        taskLocality: task.taskLocality
+    });
+}