@tangle-network/agent-bench 0.11.1 → 0.11.3

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.11.3
+
+Uses Runtime 0.209.0, Eval 0.180.0, and Knowledge 15.0.3 together, including named resource accounting and source-attributed resource reports.
+SWE-bench setup, prompts, and patch extraction use the writable session workspace instead of a root-level directory.
+Official grading behavior is unchanged.
+
+## 0.11.2
+
+Supports SWE-bench 5.x by detecting removed cache and namespace evaluator flags while retaining the 4.x path.
+
 ## 0.11.1
 
 The published benchmark package now admits sandbox SDK 0.38.x, including consumer-ready runtime-edge readiness.

package/dist/benchmarks/swe-bench.js

@@ -14,13 +14,8 @@ import { join } from "node:path";
 * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the
 * predictions-file → run_evaluation argv → report-shape mapping.
 */
-/**
-* Fixed in-box path the agent clones the instance repo into. It is the SINGLE
-* source of truth shared by the prompt template (which tells the agent to clone
-* here) and `boxExtract` (which runs `git diff` here after the shot) — so the
-* harness always knows exactly where the agent's edits live, for any instance.
-*/
-const SWE_REPO_DIR = "/work";
+/** Root-level directories are not writable in every sandbox; use the session workspace. */
+const SWE_REPO_DIR = "./swe-bench-repo";
 /**
 * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
 *
@@ -97,7 +92,10 @@ function scoreSweReport(taskId, value) {
 	};
 }
 function sweEvaluationArgv(args) {
-	return [
+	return sweEvaluationArgvForHarness(args, true);
+}
+function sweEvaluationArgvForHarness(args, legacyFlags) {
+	const common = [
 		"-m",
 		"swebench.harness.run_evaluation",
 		"--dataset_name",
@@ -109,7 +107,11 @@ function sweEvaluationArgv(args) {
 		"--instance_ids",
 		args.instanceId,
 		"--max_workers",
-		"1",
+		"1"
+	];
+	if (!legacyFlags) return common;
+	return [
+		...common,
 		"--namespace",
 		args.namespace ?? scorerNamespace(),
 		"--cache_level",
@@ -135,6 +137,7 @@ function createSweBenchAdapter(options = {}) {
 	if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error("swe-bench: invalid cacheLevel");
 	if (options.captureEvaluatorArtifacts !== void 0 && typeof options.captureEvaluatorArtifacts !== "function") throw new Error("swe-bench: captureEvaluatorArtifacts must be a function");
 	let attemptSequence = 0;
+	let legacyFlags;
 	return {
 		name: "swe-bench-verified",
 		output: swePatchOutput,
@@ -149,8 +152,9 @@ function createSweBenchAdapter(options = {}) {
 			await preflightVenvImports({
 				modules: ["swebench"],
 				requireDocker: true,
-				fix: "Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; (2) ensure the Docker daemon is running (the judge builds per-instance images)."
+				fix: "Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install 'swebench>=4,<6' ; (2) ensure the Docker daemon is running (the judge builds per-instance images)."
 			});
+			legacyFlags = await detectLegacyFlags();
 		},
 		async loadTasks(opts = {}) {
 			const limit = opts.limit ?? 10;
@@ -180,10 +184,10 @@ print(json.dumps(out))
 				prompt: [
 					`Repository: ${r.repo} @ ${r.base_commit}`,
 					"",
-					`The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
+					`The repository is ALREADY cloned at ${SWE_REPO_DIR} relative to your initial session workspace, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
 					"",
 					"Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.",
-					"Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.",
+					"Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your source edits directly from the repo, including uncommitted changes.",
 					"",
 					"--- Issue ---",
 					String(r.problem_statement ?? "")
@@ -196,6 +200,7 @@ print(json.dumps(out))
 			return typeof gold === "string" ? gold : void 0;
 		},
 		async judge(task, artifact) {
+			const useLegacyFlags = await ensureLegacyFlags();
 			const runId = safeRunId("bench", task.id);
 			const capture = options.captureEvaluatorArtifacts?.({
 				taskId: task.id,
@@ -214,13 +219,13 @@ print(json.dumps(out))
 						model_patch: artifact
 					}]));
 				},
-				argv: (dir) => sweEvaluationArgv({
+				argv: (dir) => sweEvaluationArgvForHarness({
 					predictionsPath: join(dir, "preds.json"),
 					runId,
 					instanceId: task.id,
 					cacheLevel,
 					namespace: scorerNamespace()
-				}),
+				}, useLegacyFlags),
 				async parseReport(dir) {
 					const report = await readJsonReport(join(dir, `agent-runtime-bench.${runId}.json`));
 					return scoreSweReport(task.id, report);
@@ -228,6 +233,19 @@ print(json.dumps(out))
 			});
 		}
 	};
+	async function detectLegacyFlags() {
+		return (await runVenvPython(`
+import subprocess, sys
+result = subprocess.run([sys.executable, '-m', 'swebench.harness.run_evaluation', '--help'], capture_output=True, text=True)
+if result.returncode != 0:
+    raise SystemExit(result.stderr or result.stdout or str(result.returncode))
+print('legacy' if '--namespace' in result.stdout and '--cache_level' in result.stdout else 'modern')
+`)).trim() === "legacy";
+	}
+	async function ensureLegacyFlags() {
+		if (legacyFlags === void 0) legacyFlags = await detectLegacyFlags();
+		return legacyFlags;
+	}
 }
 //#endregion
 export { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv, swePatchOutput };

package/dist/benchmarks/swe-bench.js.map

@@ -1 +1 @@
-{"version":3,"file":"swe-bench.js","names":[],"sources":["../../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/**\n * Fixed in-box path the agent clones the instance repo into. It is the SINGLE\n * source of truth shared by the prompt template (which tells the agent to clone\n * here) and `boxExtract` (which runs `git diff` here after the shot) — so the\n * harness always knows exactly where the agent's edits live, for any instance.\n */\nconst SWE_REPO_DIR = '/work'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n    '--namespace', args.namespace ?? scorerNamespace(),\n    '--cache_level', args.cacheLevel,\n  ]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgv({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAiCA,MAAM,eAAe;;;;;;;;;;AAWrB,MAAa,iBAAwC,EACnD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAKA,QADa,CADG,GAAG,KAAK,SAAS,uCAAuC,CACtD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MACb,KAAA,CAAM,KAAK;AAC7B,EACF;AAEA,MAAM,UAAU;AAsBhB,MAAM,mCAAmB,IAAI,IAAwB;CAAC;CAAQ;CAAQ;CAAO;AAAU,CAAC;AAExF,SAAS,kBAAuC;CAC9C,MAAM,YAAY,QAAQ,IAAI,sBAAsB;CACpD,IAAI,cAAc,cAAc,cAAc,QAC5C,MAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;CAEhF,OAAO;AACT;AACA,MAAM,qBAAqB;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;CAClF,MAAM,QAAQ,OAAO;CACrB,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;CACjC,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,UAAU,OAAO,UAAU,QAAQ,GAC1E,MAAM,IAAI,MAAM,wBAAwB,KAAK;CAE/C,OAAO;AACT;;AAGA,SAAgB,eAAe,QAAgB,OAA4B;CACzE,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAC5D,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,SAAS;CACf,MAAM,YAAY;EAChB,UAAU,UAAU,QAAQ,cAAc;EAC1C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,YAAY,UAAU,QAAQ,iBAAiB;EAC/C,WAAW,UAAU,QAAQ,eAAe;EAC5C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,OAAO,UAAU,QAAQ,WAAW;CACtC;CACA,MAAM,YAAY,UAAU,QAAQ,eAAe;CAEnD,IADkB,OAAO,OAAO,SAAS,CAAC,CAAC,KAEjC,CAAC,CAAC,MAAM,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,OAAO,SAExE,MAAM,IAAI,MAAM,2CAA2C,QAAQ;CAErE,IAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAC1E,MAAM,IAAI,MAAM,mCAAmC,QAAQ;CAE7D,MAAM,WAAW;EACf,UAAU,SAAS,SAAS,MAAM;EAClC,UAAU,WAAW,SAAS,MAAM;EACpC,UAAU,WAAW,SAAS,MAAM;CACtC;CACA,IAAI,SAAS,OAAO,OAAO,CAAC,CAAC,WAAW,GACtC,MAAM,IAAI,MAAM,+CAA+C,QAAQ;CAEzE,KAAK,SAAS,MAAM,SAAS,OAAO,CAAC,UAAU,UAAU,SAAS,MAAM,GACtE,MAAM,IAAI,MAAM,sDAAsD,QAAQ;CAEhF,MAAM,WAAW,SAAS;CAC1B,OAAO;EAAE;EAAU,OAAO,WAAW,IAAI;EAAG,QAAQ,KAAK,UAAU,MAAM;CAAE;AAC7E;AAEA,SAAgB,kBAAkB,MAMrB;CACX,OAAO;EACL;EAAM;EACN;EAAkB;EAClB;EAAsB,KAAK;EAC3B;EAAY,KAAK;EACjB;EAAkB,KAAK;EACvB;EAAiB;EACjB;EAAe,KAAK,aAAa,gBAAgB;EACjD;EAAiB,KAAK;CACxB;AACF;AAEA,SAAS,WAAW,OAAuB;CACzC,OAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,EAAE;AAC1C;AAEA,SAAS,YAAY,MAAiD;CACpE,MAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;CAC7C,MAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;CACpD,IAAI,CAAC,qCAAqC,KAAK,IAAI,GACjD,MAAM,IAAI,MAAM,wCAAwC,KAAK,GAAG,IAAI,MAAM;CAE5E,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAChC,MAAM,IAAI,MAAM,+CAA+C,KAAK,GAAG,IAAI,MAAM;CAEnF,OAAO;EAAE;EAAM;CAAK;AACtB;AAEA,SAAgB,sBAAsB,UAAkC,CAAC,GAAqB;CAC5F,IACE,QAAQ,cAAc,KAAA,MAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,IACrE,MAAM,IAAI,MAAM,iDAAiD;CACnE,MAAM,aAAa,QAAQ,cAAc;CACzC,IAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,+BAA+B;CACtF,IACE,QAAQ,8BAA8B,KAAA,KACnC,OAAO,QAAQ,8BAA8B,YAChD,MAAM,IAAI,MAAM,yDAAyD;CAC3E,IAAI,kBAAkB;CACtB,OAAO;EACL,MAAM;EACN,QAAQ;EAWR,SAAS,MAAM;GACb,MAAM,EAAE,MAAM,SAAS,YAAY,IAAI;GACvC,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,wBAAwB,WAAW,sBAAsB,MAAM,EAAE,GAAG,WAAW,YAAY,EAAE,aAAa,WAAW,YAAY,EAAE,oBAAoB,WAAW,IAAI,IACpN;EACF;EACA,aAAa;GACX,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,oBAAoB,WAAW,YAAY,EAAE,sBAAsB,qBACjH;EACF;EAEA,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAEJ,CAAC;EACH;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,MAAM,QAAQ,KAAK,SAAS;GAqB5B,MAAM,SAAS,MAAM,cAAc;;;oBAhBrB,KAAK,UAAU,OAAO,EAAE,UAAU,KAAK,UAAU,KAAK,EAAE;;;;;;;;;;;;qCAYvC,MAAM;;;GAIM,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAErF,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KACT,OAAkB;IACjB,IAAI,OAAO,EAAE,WAAW;IACxB;IACA,QAAQ;KACN,eAAe,EAAE,KAAK,KAAK,EAAE;KAC7B;KACA,uCAAuC,aAAa,0BAA0B,EAAE,YAAY,8BAA8B,aAAa;KACvI;KACA;KACA;KACA;KACA;KACA,OAAO,EAAE,qBAAqB,EAAE;IAClC,CAAC,CAAC,KAAK,IAAI;IACX,UAAU;GACZ,EACF;EACF;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,KAAK,UAAU;GAC5B,OAAO,OAAO,SAAS,WAAW,OAAO,KAAA;EAC3C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;GACxC,MAAM,UAAU,QAAQ,4BAA4B;IAClD,QAAQ,KAAK;IACb;IACA,iBAAiB,EAAE;GACrB,CAAC;GACD,OAAO,eAAe;IACpB,WAAW;IACX,GAAI,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;IAC1E,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;IAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;IACzD,MAAM,MAAM,KAAK;KACf,MAAM,UACJ,KAAK,KAAK,YAAY,GACtB,KAAK,UAAU,CACb;MAAE,aAAa,KAAK;MAAI,oBAAoB;MAAuB,aAAa;KAAS,CAC3F,CAAC,CACH;IACF;IAGA,OAAO,QAAQ,kBAAkB;KAC/B,iBAAiB,KAAK,KAAK,YAAY;KACvC;KACA,YAAY,KAAK;KACjB;KACA,WAAW,gBAAgB;IAC7B,CAAC;IACD,MAAM,YAAY,KAAK;KAErB,MAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,MAAM,MAAM,CAAC;KAC7F,OAAO,eAAe,KAAK,IAAI,MAAM;IACvC;GACF,CAAC;EACH;CACF;AACF"}
+{"version":3,"file":"swe-bench.js","names":[],"sources":["../../src/benchmarks/swe-bench.ts"],"sourcesContent":["/**\n * SWE-bench Verified adapter. Worker artifact = a unified-diff patch. Judge =\n * the official `swebench` harness: apply the patch in the instance's Docker\n * image, run FAIL_TO_PASS + PASS_TO_PASS, report `resolved`. Fully deterministic\n * — no LLM judge.\n *\n * Requires: the bench `.venv` with `swebench` installed + a running Docker\n * daemon (per-instance images are pulled/built on first run).\n *\n * Process/Docker/report plumbing is shared via ./_harness; this file owns the\n * SWE-specific pieces: the patch OutputAdapter, the dataset dump, and the\n * predictions-file → run_evaluation argv → report-shape mapping.\n */\n\nimport { join } from 'node:path'\nimport type { OutputAdapter } from '@tangle-network/agent-runtime/kernel'\nimport {\n  preflightVenvImports,\n  readJsonReport,\n  runStagedJudge,\n  runVenvPython,\n  safeRunId,\n  stageFile,\n  type StagedRunCaptureSpec,\n} from './_harness'\nimport type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'\n\n/** Root-level directories are not writable in every sandbox; use the session workspace. */\nconst SWE_REPO_DIR = './swe-bench-repo'\n\n/**\n * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.\n *\n * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's\n * actual edits, read from the cloned repo's STATE inside the box (standard\n * SWE-bench practice). This event-stream parse only runs when that diff is empty\n * — a model that edited the source correctly but never printed a fenced diff (the\n * exact failure this replaces) still scores off its real changes, not its prose.\n */\nexport const swePatchOutput: OutputAdapter<string> = {\n  parse(events) {\n    let text = ''\n    for (const ev of events) {\n      const d = (ev as { data?: Record<string, unknown> })?.data\n      const t = d?.finalText ?? d?.text ?? d?.result\n      if (typeof t === 'string' && t.length > 0) text = t\n    }\n    // Last ```diff/```patch fenced block (the contract the prompt asks for);\n    // fall back to the raw text so a fence-less but valid diff still reaches the judge.\n    const fences = [...text.matchAll(/```(?:diff|patch)?\\s*\\n([\\s\\S]*?)```/g)]\n    const last = fences.at(-1)?.[1]\n    return (last ?? text).trim()\n  },\n}\n\nconst DATASET = 'princeton-nlp/SWE-bench_Verified'\nexport type SweBenchCacheLevel = 'none' | 'base' | 'env' | 'instance'\n\nexport interface SweBenchArtifactCaptureContext {\n  readonly taskId: string\n  readonly runId: string\n  /** One-based sequence unique within this adapter instance. */\n  readonly attemptSequence: number\n}\n\nexport interface SweBenchAdapterOptions {\n  readonly timeoutMs?: number\n  readonly cacheLevel?: SweBenchCacheLevel\n  /**\n   * Return a unique destination for any attempt whose complete official\n   * evaluator directory and process logs should be retained.\n   */\n  readonly captureEvaluatorArtifacts?: (\n    context: SweBenchArtifactCaptureContext,\n  ) => StagedRunCaptureSpec | undefined\n}\n\nconst SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])\n\nfunction scorerNamespace(): 'swebench' | 'none' {\n  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'\n  if (namespace !== 'swebench' && namespace !== 'none') {\n    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got \"${namespace}\"`)\n  }\n  return namespace\n}\nconst TEST_FILE_EXCLUDES = [\n  \"':(exclude,glob)**/tests/**'\",\n  \"':(exclude,glob)**/test/**'\",\n  \"':(exclude,glob)test_*.py'\",\n  \"':(exclude,glob)**/test_*.py'\",\n  \"':(exclude,glob)*_test.py'\",\n  \"':(exclude,glob)**/*_test.py'\",\n  \"':(exclude,glob)conftest.py'\",\n  \"':(exclude,glob)**/conftest.py'\",\n].join(' ')\n\ninterface SweReport {\n  resolved_instances?: number\n  resolved_ids?: string[]\n  unresolved_ids?: string[]\n  empty_patch_ids?: string[]\n  completed_ids?: string[]\n  incomplete_ids?: string[]\n  error_ids?: string[]\n  submitted_ids?: string[]\n}\n\nfunction stringIds(report: Record<string, unknown>, key: keyof SweReport): string[] {\n  const value = report[key]\n  if (value === undefined) return []\n  if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) {\n    throw new Error(`swe-bench: malformed ${key}`)\n  }\n  return value\n}\n\n/** Convert one official report into a score without turning evaluator failures into agent failures. */\nexport function scoreSweReport(taskId: string, value: unknown): BenchScore {\n  if (!value || typeof value !== 'object' || Array.isArray(value)) {\n    throw new Error('swe-bench: report must be an object')\n  }\n  const report = value as Record<string, unknown>\n  const statusIds = {\n    resolved: stringIds(report, 'resolved_ids'),\n    unresolved: stringIds(report, 'unresolved_ids'),\n    emptyPatch: stringIds(report, 'empty_patch_ids'),\n    completed: stringIds(report, 'completed_ids'),\n    incomplete: stringIds(report, 'incomplete_ids'),\n    error: stringIds(report, 'error_ids'),\n  }\n  const submitted = stringIds(report, 'submitted_ids')\n  const mentioned = Object.values(statusIds).flat()\n  if (\n    mentioned.some((id) => id !== taskId)\n    || (submitted.length > 0 && (submitted.length !== 1 || submitted[0] !== taskId))\n  ) {\n    throw new Error(`swe-bench: report identity mismatch for ${taskId}`)\n  }\n  if (statusIds.error.includes(taskId) || statusIds.incomplete.includes(taskId)) {\n    throw new Error(`swe-bench: evaluator failed for ${taskId}`)\n  }\n  const outcomes = [\n    statusIds.resolved.includes(taskId),\n    statusIds.unresolved.includes(taskId),\n    statusIds.emptyPatch.includes(taskId),\n  ]\n  if (outcomes.filter(Boolean).length !== 1) {\n    throw new Error(`swe-bench: report has no unique outcome for ${taskId}`)\n  }\n  if ((outcomes[0] || outcomes[1]) && !statusIds.completed.includes(taskId)) {\n    throw new Error(`swe-bench: report lacks a completed evaluation for ${taskId}`)\n  }\n  const resolved = outcomes[0]\n  return { resolved, score: resolved ? 1 : 0, detail: JSON.stringify(report) }\n}\n\nexport function sweEvaluationArgv(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}): string[] {\n  return sweEvaluationArgvForHarness(args, true)\n}\n\nfunction sweEvaluationArgvForHarness(args: {\n  readonly predictionsPath: string\n  readonly runId: string\n  readonly instanceId: string\n  readonly cacheLevel: SweBenchCacheLevel\n  readonly namespace?: 'swebench' | 'none'\n}, legacyFlags: boolean): string[] {\n  const common = [\n    '-m', 'swebench.harness.run_evaluation',\n    '--dataset_name', DATASET,\n    '--predictions_path', args.predictionsPath,\n    '--run_id', args.runId,\n    '--instance_ids', args.instanceId,\n    '--max_workers', '1',\n  ]\n  if (!legacyFlags) return common\n  return [...common, '--namespace', args.namespace ?? scorerNamespace(), '--cache_level', args.cacheLevel]\n}\n\nfunction shellQuote(value: string): string {\n  return `'${value.replace(/'/g, `'\\\\''`)}'`\n}\n\nfunction sweMetadata(task: BenchTask): { repo: string; base: string } {\n  const repo = String(task.metadata?.repo ?? '')\n  const base = String(task.metadata?.base_commit ?? '')\n  if (!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repo)) {\n    throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`)\n  }\n  if (!/^[0-9a-f]{7,40}$/i.test(base)) {\n    throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`)\n  }\n  return { repo, base }\n}\n\nexport function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): BenchmarkAdapter {\n  if (\n    options.timeoutMs !== undefined\n    && (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs <= 0)\n  ) throw new Error('swe-bench: timeoutMs must be a positive integer')\n  const cacheLevel = options.cacheLevel ?? 'env'\n  if (!SWE_CACHE_LEVELS.has(cacheLevel)) throw new Error('swe-bench: invalid cacheLevel')\n  if (\n    options.captureEvaluatorArtifacts !== undefined\n    && typeof options.captureEvaluatorArtifacts !== 'function'\n  ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')\n  let attemptSequence = 0\n  let legacyFlags: boolean | undefined\n  return {\n    name: 'swe-bench-verified',\n    output: swePatchOutput,\n\n    // Extract the patch from repo STATE, not printed text: stage every edit the\n    // agent made in the cloned repo and diff it against the checked-out\n    // base_commit (`HEAD`). Test files are excluded — the judge applies the gold\n    // `test_patch` itself, so an agent edit to a test would collide on apply. The\n    // paths come out `a/<repo-relative>` (cwd = repo root), matching the gold\n    // patch format the swebench judge's `git apply` expects.\n    // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the\n    // agent only edits (the harness owns the checkout — a stochastic model can't be\n    // trusted to clone to an exact path). `--quiet` keeps the exec output small.\n    boxSetup(task) {\n      const { repo, base } = sweMetadata(task)\n      return {\n        command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`,\n      }\n    },\n    boxExtract() {\n      return {\n        command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`,\n      }\n    },\n\n    async preflight() {\n      await preflightVenvImports({\n        modules: ['swebench'],\n        requireDocker: true,\n        fix:\n          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install 'swebench>=4,<6' ; ` +\n          `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,\n      })\n      legacyFlags = await detectLegacyFlags()\n    },\n\n    async loadTasks(opts: LoadOptions = {}) {\n      const limit = opts.limit ?? 10\n      const split = opts.split ?? 'test'\n      // Dump instances as JSON via the datasets loader (HF download on first run).\n      const script = `\nimport json, sys\nfrom datasets import load_dataset\nds = load_dataset(${JSON.stringify(DATASET)}, split=${JSON.stringify(split)})\nids = set(json.loads(sys.argv[1])) if len(sys.argv) > 1 and sys.argv[1] else None\nout = []\nfor r in ds:\n    if ids is not None and r[\"instance_id\"] not in ids:\n        continue\n    out.append({\n        \"instance_id\": r[\"instance_id\"], \"repo\": r[\"repo\"], \"base_commit\": r[\"base_commit\"],\n        \"problem_statement\": r[\"problem_statement\"], \"patch\": r[\"patch\"], \"test_patch\": r[\"test_patch\"],\n        \"FAIL_TO_PASS\": r[\"FAIL_TO_PASS\"], \"PASS_TO_PASS\": r[\"PASS_TO_PASS\"],\n        \"version\": r.get(\"version\"), \"environment_setup_commit\": r.get(\"environment_setup_commit\"),\n    })\n    if ids is None and len(out) >= ${limit}:\n        break\nprint(json.dumps(out))\n`\n      const stdout = await runVenvPython(script, [opts.ids ? JSON.stringify(opts.ids) : ''])\n      const rows = JSON.parse(stdout) as Array<Record<string, unknown>>\n      return rows.map(\n        (r): BenchTask => ({\n          id: String(r.instance_id),\n          split,\n          prompt: [\n            `Repository: ${r.repo} @ ${r.base_commit}`,\n            '',\n            `The repository is ALREADY cloned at ${SWE_REPO_DIR} relative to your initial session workspace, checked out at commit ${r.base_commit}. Work there directly (\\`cd ${SWE_REPO_DIR}\\`); do not re-clone.`,\n            '',\n            'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',\n            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your source edits directly from the repo, including uncommitted changes.',\n            '',\n            '--- Issue ---',\n            String(r.problem_statement ?? ''),\n          ].join('\\n'),\n          metadata: r,\n        }),\n      )\n    },\n\n    async goldArtifact(task: BenchTask) {\n      const gold = task.metadata?.patch\n      return typeof gold === 'string' ? gold : undefined\n    },\n\n    async judge(task: BenchTask, artifact: string): Promise<BenchScore> {\n      const useLegacyFlags = await ensureLegacyFlags()\n      const runId = safeRunId('bench', task.id)\n      const capture = options.captureEvaluatorArtifacts?.({\n        taskId: task.id,\n        runId,\n        attemptSequence: ++attemptSequence,\n      })\n      return runStagedJudge({\n        tmpPrefix: 'swebench-',\n        ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),\n        ...(capture === undefined ? {} : { capture }),\n        // Debug: retain the staged dir (holds swebench's per-instance apply/run\n        // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default.\n        ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}),\n        async stage(dir) {\n          await stageFile(\n            join(dir, 'preds.json'),\n            JSON.stringify([\n              { instance_id: task.id, model_name_or_path: 'agent-runtime-bench', model_patch: artifact },\n            ]),\n          )\n        },\n        // The official evaluation harness. Pulls/builds the instance image, applies\n        // the patch, runs the test spec, writes a per-run report JSON in cwd.\n        argv: (dir) => sweEvaluationArgvForHarness({\n          predictionsPath: join(dir, 'preds.json'),\n          runId,\n          instanceId: task.id,\n          cacheLevel,\n          namespace: scorerNamespace(),\n        }, useLegacyFlags),\n        async parseReport(dir) {\n          // Report file: agent-runtime-bench.<run_id>.json\n          const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))\n          return scoreSweReport(task.id, report)\n        },\n      })\n    },\n  }\n\n  async function detectLegacyFlags(): Promise<boolean> {\n    const script = `\nimport subprocess, sys\nresult = subprocess.run([sys.executable, '-m', 'swebench.harness.run_evaluation', '--help'], capture_output=True, text=True)\nif result.returncode != 0:\n    raise SystemExit(result.stderr or result.stdout or str(result.returncode))\nprint('legacy' if '--namespace' in result.stdout and '--cache_level' in result.stdout else 'modern')\n`\n    return (await runVenvPython(script)).trim() === 'legacy'\n  }\n\n  async function ensureLegacyFlags(): Promise<boolean> {\n    if (legacyFlags === undefined) legacyFlags = await detectLegacyFlags()\n    return legacyFlags\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AA4BA,MAAM,eAAe;;;;;;;;;;AAWrB,MAAa,iBAAwC,EACnD,MAAM,QAAQ;CACZ,IAAI,OAAO;CACX,KAAK,MAAM,MAAM,QAAQ;EACvB,MAAM,IAAK,IAA2C;EACtD,MAAM,IAAI,GAAG,aAAa,GAAG,QAAQ,GAAG;EACxC,IAAI,OAAO,MAAM,YAAY,EAAE,SAAS,GAAG,OAAO;CACpD;CAKA,QADa,CADG,GAAG,KAAK,SAAS,uCAAuC,CACtD,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MACb,KAAA,CAAM,KAAK;AAC7B,EACF;AAEA,MAAM,UAAU;AAsBhB,MAAM,mCAAmB,IAAI,IAAwB;CAAC;CAAQ;CAAQ;CAAO;AAAU,CAAC;AAExF,SAAS,kBAAuC;CAC9C,MAAM,YAAY,QAAQ,IAAI,sBAAsB;CACpD,IAAI,cAAc,cAAc,cAAc,QAC5C,MAAM,IAAI,MAAM,kDAAkD,UAAU,EAAE;CAEhF,OAAO;AACT;AACA,MAAM,qBAAqB;CACzB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF,CAAC,CAAC,KAAK,GAAG;AAaV,SAAS,UAAU,QAAiC,KAAgC;CAClF,MAAM,QAAQ,OAAO;CACrB,IAAI,UAAU,KAAA,GAAW,OAAO,CAAC;CACjC,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,MAAM,MAAM,UAAU,OAAO,UAAU,QAAQ,GAC1E,MAAM,IAAI,MAAM,wBAAwB,KAAK;CAE/C,OAAO;AACT;;AAGA,SAAgB,eAAe,QAAgB,OAA4B;CACzE,IAAI,CAAC,SAAS,OAAO,UAAU,YAAY,MAAM,QAAQ,KAAK,GAC5D,MAAM,IAAI,MAAM,qCAAqC;CAEvD,MAAM,SAAS;CACf,MAAM,YAAY;EAChB,UAAU,UAAU,QAAQ,cAAc;EAC1C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,YAAY,UAAU,QAAQ,iBAAiB;EAC/C,WAAW,UAAU,QAAQ,eAAe;EAC5C,YAAY,UAAU,QAAQ,gBAAgB;EAC9C,OAAO,UAAU,QAAQ,WAAW;CACtC;CACA,MAAM,YAAY,UAAU,QAAQ,eAAe;CAEnD,IADkB,OAAO,OAAO,SAAS,CAAC,CAAC,KAEjC,CAAC,CAAC,MAAM,OAAO,OAAO,MAAM,KAChC,UAAU,SAAS,MAAM,UAAU,WAAW,KAAK,UAAU,OAAO,SAExE,MAAM,IAAI,MAAM,2CAA2C,QAAQ;CAErE,IAAI,UAAU,MAAM,SAAS,MAAM,KAAK,UAAU,WAAW,SAAS,MAAM,GAC1E,MAAM,IAAI,MAAM,mCAAmC,QAAQ;CAE7D,MAAM,WAAW;EACf,UAAU,SAAS,SAAS,MAAM;EAClC,UAAU,WAAW,SAAS,MAAM;EACpC,UAAU,WAAW,SAAS,MAAM;CACtC;CACA,IAAI,SAAS,OAAO,OAAO,CAAC,CAAC,WAAW,GACtC,MAAM,IAAI,MAAM,+CAA+C,QAAQ;CAEzE,KAAK,SAAS,MAAM,SAAS,OAAO,CAAC,UAAU,UAAU,SAAS,MAAM,GACtE,MAAM,IAAI,MAAM,sDAAsD,QAAQ;CAEhF,MAAM,WAAW,SAAS;CAC1B,OAAO;EAAE;EAAU,OAAO,WAAW,IAAI;EAAG,QAAQ,KAAK,UAAU,MAAM;CAAE;AAC7E;AAEA,SAAgB,kBAAkB,MAMrB;CACX,OAAO,4BAA4B,MAAM,IAAI;AAC/C;AAEA,SAAS,4BAA4B,MAMlC,aAAgC;CACjC,MAAM,SAAS;EACb;EAAM;EACN;EAAkB;EAClB;EAAsB,KAAK;EAC3B;EAAY,KAAK;EACjB;EAAkB,KAAK;EACvB;EAAiB;CACnB;CACA,IAAI,CAAC,aAAa,OAAO;CACzB,OAAO;EAAC,GAAG;EAAQ;EAAe,KAAK,aAAa,gBAAgB;EAAG;EAAiB,KAAK;CAAU;AACzG;AAEA,SAAS,WAAW,OAAuB;CACzC,OAAO,IAAI,MAAM,QAAQ,MAAM,OAAO,EAAE;AAC1C;AAEA,SAAS,YAAY,MAAiD;CACpE,MAAM,OAAO,OAAO,KAAK,UAAU,QAAQ,EAAE;CAC7C,MAAM,OAAO,OAAO,KAAK,UAAU,eAAe,EAAE;CACpD,IAAI,CAAC,qCAAqC,KAAK,IAAI,GACjD,MAAM,IAAI,MAAM,wCAAwC,KAAK,GAAG,IAAI,MAAM;CAE5E,IAAI,CAAC,oBAAoB,KAAK,IAAI,GAChC,MAAM,IAAI,MAAM,+CAA+C,KAAK,GAAG,IAAI,MAAM;CAEnF,OAAO;EAAE;EAAM;CAAK;AACtB;AAEA,SAAgB,sBAAsB,UAAkC,CAAC,GAAqB;CAC5F,IACE,QAAQ,cAAc,KAAA,MAClB,CAAC,OAAO,cAAc,QAAQ,SAAS,KAAK,QAAQ,aAAa,IACrE,MAAM,IAAI,MAAM,iDAAiD;CACnE,MAAM,aAAa,QAAQ,cAAc;CACzC,IAAI,CAAC,iBAAiB,IAAI,UAAU,GAAG,MAAM,IAAI,MAAM,+BAA+B;CACtF,IACE,QAAQ,8BAA8B,KAAA,KACnC,OAAO,QAAQ,8BAA8B,YAChD,MAAM,IAAI,MAAM,yDAAyD;CAC3E,IAAI,kBAAkB;CACtB,IAAI;CACJ,OAAO;EACL,MAAM;EACN,QAAQ;EAWR,SAAS,MAAM;GACb,MAAM,EAAE,MAAM,SAAS,YAAY,IAAI;GACvC,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,wBAAwB,WAAW,sBAAsB,MAAM,EAAE,GAAG,WAAW,YAAY,EAAE,aAAa,WAAW,YAAY,EAAE,oBAAoB,WAAW,IAAI,IACpN;EACF;EACA,aAAa;GACX,OAAO,EACL,SAAS,UAAU,WAAW,YAAY,EAAE,oBAAoB,WAAW,YAAY,EAAE,sBAAsB,qBACjH;EACF;EAEA,MAAM,YAAY;GAChB,MAAM,qBAAqB;IACzB,SAAS,CAAC,UAAU;IACpB,eAAe;IACf,KACE;GAEJ,CAAC;GACD,cAAc,MAAM,kBAAkB;EACxC;EAEA,MAAM,UAAU,OAAoB,CAAC,GAAG;GACtC,MAAM,QAAQ,KAAK,SAAS;GAC5B,MAAM,QAAQ,KAAK,SAAS;GAqB5B,MAAM,SAAS,MAAM,cAAc;;;oBAhBrB,KAAK,UAAU,OAAO,EAAE,UAAU,KAAK,UAAU,KAAK,EAAE;;;;;;;;;;;;qCAYvC,MAAM;;;GAIM,CAAC,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,IAAI,EAAE,CAAC;GAErF,OADa,KAAK,MAAM,MACd,CAAC,CAAC,KACT,OAAkB;IACjB,IAAI,OAAO,EAAE,WAAW;IACxB;IACA,QAAQ;KACN,eAAe,EAAE,KAAK,KAAK,EAAE;KAC7B;KACA,uCAAuC,aAAa,qEAAqE,EAAE,YAAY,8BAA8B,aAAa;KAClL;KACA;KACA;KACA;KACA;KACA,OAAO,EAAE,qBAAqB,EAAE;IAClC,CAAC,CAAC,KAAK,IAAI;IACX,UAAU;GACZ,EACF;EACF;EAEA,MAAM,aAAa,MAAiB;GAClC,MAAM,OAAO,KAAK,UAAU;GAC5B,OAAO,OAAO,SAAS,WAAW,OAAO,KAAA;EAC3C;EAEA,MAAM,MAAM,MAAiB,UAAuC;GAClE,MAAM,iBAAiB,MAAM,kBAAkB;GAC/C,MAAM,QAAQ,UAAU,SAAS,KAAK,EAAE;GACxC,MAAM,UAAU,QAAQ,4BAA4B;IAClD,QAAQ,KAAK;IACb;IACA,iBAAiB,EAAE;GACrB,CAAC;GACD,OAAO,eAAe;IACpB,WAAW;IACX,GAAI,QAAQ,cAAc,KAAA,IAAY,CAAC,IAAI,EAAE,WAAW,QAAQ,UAAU;IAC1E,GAAI,YAAY,KAAA,IAAY,CAAC,IAAI,EAAE,QAAQ;IAG3C,GAAI,QAAQ,IAAI,oBAAoB,EAAE,SAAS,KAAK,IAAI,CAAC;IACzD,MAAM,MAAM,KAAK;KACf,MAAM,UACJ,KAAK,KAAK,YAAY,GACtB,KAAK,UAAU,CACb;MAAE,aAAa,KAAK;MAAI,oBAAoB;MAAuB,aAAa;KAAS,CAC3F,CAAC,CACH;IACF;IAGA,OAAO,QAAQ,4BAA4B;KACzC,iBAAiB,KAAK,KAAK,YAAY;KACvC;KACA,YAAY,KAAK;KACjB;KACA,WAAW,gBAAgB;IAC7B,GAAG,cAAc;IACjB,MAAM,YAAY,KAAK;KAErB,MAAM,SAAS,MAAM,eAA0B,KAAK,KAAK,uBAAuB,MAAM,MAAM,CAAC;KAC7F,OAAO,eAAe,KAAK,IAAI,MAAM;IACvC;GACF,CAAC;EACH;CACF;CAEA,eAAe,oBAAsC;EAQnD,QAAQ,MAAM,cAAc;;;;;;CAAM,EAAA,CAAG,KAAK,MAAM;CAClD;CAEA,eAAe,oBAAsC;EACnD,IAAI,gBAAgB,KAAA,GAAW,cAAc,MAAM,kBAAkB;EACrE,OAAO;CACT;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.11.1",
+  "version": "0.11.3",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {
@@ -25,11 +25,11 @@
     }
   },
   "dependencies": {
-    "@tangle-network/agent-eval": ">=0.179.0 <0.180.0",
+    "@tangle-network/agent-eval": ">=0.180.0 <0.181.0",
     "@tangle-network/agent-interface": "^2.6.0",
-    "@tangle-network/agent-knowledge": "^15.0.2",
+    "@tangle-network/agent-knowledge": "^15.0.3",
     "@tangle-network/sandbox": ">=0.36.4 <0.39.0",
-    "@tangle-network/agent-runtime": "^0.204.1"
+    "@tangle-network/agent-runtime": "^0.209.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "0.18.5",

package/src/benchmarks/swe-bench.test.mts

@@ -1,5 +1,9 @@
 import assert from 'node:assert/strict'
 import test from 'node:test'
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
 import { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv } from './swe-bench'
 
 const taskId = 'django__django-12345'
@@ -59,3 +63,48 @@ test('SWE evaluation command preserves the requested instance image', () => {
     /invalid cacheLevel/,
   )
 })
+
+test('SWE setup and extraction stay in the session workspace and exclude test edits', () => {
+  const root = mkdtempSync(join(tmpdir(), 'swe-workspace-'))
+  try {
+    const origin = join(root, 'origin')
+    const workspace = join(root, 'session')
+    mkdirSync(origin)
+    mkdirSync(workspace)
+    const git = (args: string[], input?: string) => execFileSync('git', args, { cwd: origin, input, encoding: 'utf8' }).trim()
+    git(['init', '--quiet'])
+    const blob = git(['hash-object', '-w', '--stdin'], 'before\n')
+    const tree = git(['mktree'], `100644 blob ${blob}\tsource.py\n`)
+    // Construct fixture history without changing the developer's Git identity or configuration.
+    const base = git(['hash-object', '-t', 'commit', '-w', '--stdin'],
+      `tree ${tree}\nauthor Fixture <fixture@example.invalid> 1 +0000\ncommitter Fixture <fixture@example.invalid> 1 +0000\n\nfixture\n`)
+    git(['update-ref', 'HEAD', base])
+    const task = { id: taskId, prompt: 'fix', metadata: { repo: 'fixture/repo', base_commit: base } }
+    const adapter = createSweBenchAdapter()
+    const setup = adapter.boxSetup!(task)
+    const extract = adapter.boxExtract!(task)
+    assert.match(setup.command, /^rm -rf '\.\//)
+    assert.equal(setup.cwd, undefined)
+    assert.equal(extract.cwd, undefined)
+    const env = {
+      ...process.env,
+      GIT_CONFIG_COUNT: '2',
+      GIT_CONFIG_KEY_0: `url.file://${origin}.insteadOf`,
+      GIT_CONFIG_VALUE_0: 'https://github.com/fixture/repo',
+      GIT_CONFIG_KEY_1: 'protocol.file.allow',
+      GIT_CONFIG_VALUE_1: 'always',
+    }
+    execFileSync('sh', ['-c', setup.command], { cwd: workspace, env })
+    const repo = join(workspace, 'swe-bench-repo')
+    assert.ok(existsSync(join(repo, '.git')))
+    writeFileSync(join(repo, 'source.py'), 'after\n')
+    mkdirSync(join(repo, 'tests'))
+    writeFileSync(join(repo, 'tests', 'test_fix.py'), 'hidden-test-edit\n')
+    const patch = execFileSync('sh', ['-c', extract.command], { cwd: workspace, env, encoding: 'utf8' })
+    assert.match(patch, /diff --git a\/source.py b\/source.py/)
+    assert.match(patch, /\+after/)
+    assert.doesNotMatch(patch, /hidden-test-edit|test_fix/)
+  } finally {
+    rmSync(root, { recursive: true, force: true })
+  }
+})

package/src/benchmarks/swe-bench.ts

@@ -25,13 +25,8 @@ import {
 } from './_harness'
 import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types'
 
-/**
- * Fixed in-box path the agent clones the instance repo into. It is the SINGLE
- * source of truth shared by the prompt template (which tells the agent to clone
- * here) and `boxExtract` (which runs `git diff` here after the shot) — so the
- * harness always knows exactly where the agent's edits live, for any instance.
- */
-const SWE_REPO_DIR = '/work'
+/** Root-level directories are not writable in every sandbox; use the session workspace. */
+const SWE_REPO_DIR = './swe-bench-repo'
 
 /**
  * The SWE deliverable's FALLBACK parser, from the agent's event STREAM.
@@ -167,16 +162,26 @@ export function sweEvaluationArgv(args: {
   readonly cacheLevel: SweBenchCacheLevel
   readonly namespace?: 'swebench' | 'none'
 }): string[] {
-  return [
+  return sweEvaluationArgvForHarness(args, true)
+}
+
+function sweEvaluationArgvForHarness(args: {
+  readonly predictionsPath: string
+  readonly runId: string
+  readonly instanceId: string
+  readonly cacheLevel: SweBenchCacheLevel
+  readonly namespace?: 'swebench' | 'none'
+}, legacyFlags: boolean): string[] {
+  const common = [
     '-m', 'swebench.harness.run_evaluation',
     '--dataset_name', DATASET,
     '--predictions_path', args.predictionsPath,
     '--run_id', args.runId,
     '--instance_ids', args.instanceId,
     '--max_workers', '1',
-    '--namespace', args.namespace ?? scorerNamespace(),
-    '--cache_level', args.cacheLevel,
   ]
+  if (!legacyFlags) return common
+  return [...common, '--namespace', args.namespace ?? scorerNamespace(), '--cache_level', args.cacheLevel]
 }
 
 function shellQuote(value: string): string {
@@ -207,6 +212,7 @@ export function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): Ben
     && typeof options.captureEvaluatorArtifacts !== 'function'
   ) throw new Error('swe-bench: captureEvaluatorArtifacts must be a function')
   let attemptSequence = 0
+  let legacyFlags: boolean | undefined
   return {
     name: 'swe-bench-verified',
     output: swePatchOutput,
@@ -237,9 +243,10 @@ export function createSweBenchAdapter(options: SweBenchAdapterOptions = {}): Ben
         modules: ['swebench'],
         requireDocker: true,
         fix:
-          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install swebench ; ` +
+          `Fix: (1) python3 -m venv bench/.venv && bench/.venv/bin/pip install 'swebench>=4,<6' ; ` +
           `(2) ensure the Docker daemon is running (the judge builds per-instance images).`,
       })
+      legacyFlags = await detectLegacyFlags()
     },
 
     async loadTasks(opts: LoadOptions = {}) {
@@ -274,10 +281,10 @@ print(json.dumps(out))
           prompt: [
             `Repository: ${r.repo} @ ${r.base_commit}`,
             '',
-            `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
+            `The repository is ALREADY cloned at ${SWE_REPO_DIR} relative to your initial session workspace, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`,
             '',
             'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.',
-            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.',
+            'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your source edits directly from the repo, including uncommitted changes.',
             '',
             '--- Issue ---',
             String(r.problem_statement ?? ''),
@@ -293,6 +300,7 @@ print(json.dumps(out))
     },
 
     async judge(task: BenchTask, artifact: string): Promise<BenchScore> {
+      const useLegacyFlags = await ensureLegacyFlags()
       const runId = safeRunId('bench', task.id)
       const capture = options.captureEvaluatorArtifacts?.({
         taskId: task.id,
@@ -316,13 +324,13 @@ print(json.dumps(out))
         },
         // The official evaluation harness. Pulls/builds the instance image, applies
         // the patch, runs the test spec, writes a per-run report JSON in cwd.
-        argv: (dir) => sweEvaluationArgv({
+        argv: (dir) => sweEvaluationArgvForHarness({
           predictionsPath: join(dir, 'preds.json'),
           runId,
           instanceId: task.id,
           cacheLevel,
           namespace: scorerNamespace(),
-        }),
+        }, useLegacyFlags),
         async parseReport(dir) {
           // Report file: agent-runtime-bench.<run_id>.json
           const report = await readJsonReport<SweReport>(join(dir, `agent-runtime-bench.${runId}.json`))
@@ -331,4 +339,20 @@ print(json.dumps(out))
       })
     },
   }
+
+  async function detectLegacyFlags(): Promise<boolean> {
+    const script = `
+import subprocess, sys
+result = subprocess.run([sys.executable, '-m', 'swebench.harness.run_evaluation', '--help'], capture_output=True, text=True)
+if result.returncode != 0:
+    raise SystemExit(result.stderr or result.stdout or str(result.returncode))
+print('legacy' if '--namespace' in result.stdout and '--cache_level' in result.stdout else 'modern')
+`
+    return (await runVenvPython(script)).trim() === 'legacy'
+  }
+
+  async function ensureLegacyFlags(): Promise<boolean> {
+    if (legacyFlags === undefined) legacyFlags = await detectLegacyFlags()
+    return legacyFlags
+  }
 }