@iowarp/clio-coder 0.4.9 → 0.5.0

package/evals/source-explanation-grader.mjs

@@ -1,118 +0,0 @@
-import { strict as assert } from "node:assert";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-export const fixture = JSON.parse(readFileSync(new URL("./fixtures/source-explanation.json", import.meta.url), "utf8"));
-
-// A bounded content screen, not a semantic judge. Root must still review the
-// cited claims. In particular, schema success and pane lifecycle are not delivery.
-export function assessExplanation(agentId, output, conformance) {
-	let value;
-	try {
-		value = JSON.parse(output);
-	} catch {
-		value = {};
-	}
-	const findings = Array.isArray(value.findings) ? value.findings : [];
-	const prose = agentId === "scout" ? findings.map((finding) => finding.claim ?? "").join(" ") : (value.summary ?? "");
-	const words = typeof prose === "string" && prose.trim() ? prose.trim().split(/\s+/u).length : 0;
-	// Scout cites through structured findings; Coder cites inline as file:line
-	// inside its summary, the declared place for a read-only explanation.
-	const cited = Object.keys(fixture.files).every((path) =>
-		agentId === "scout"
-			? findings.some((finding) => finding.path === path && Number.isSafeInteger(finding.line) && finding.line > 0)
-			: new RegExp(`${path.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")}:[1-9]\\d*`, "u").test(prose),
-	);
-	const topics = [
-		/uniform/iu,
-		/nonuniform/iu,
-		/forward/iu,
-		/backward/iu,
-		/spacing/iu,
-		/coordinate/iu,
-		/convergence/iu,
-		/linspace|equidistant/iu,
-		/does not|do not|not .*irregular/iu,
-	].every((pattern) => pattern.test(prose));
-	// Both recipes carry a 1200-word cited explanation: Scout as findings, Coder
-	// in its mutation-report summary under the contract's summary allowance.
-	const suitable = agentId === "scout" || agentId === "coder";
-	const limited = agentId === "coder" && /cannot deliver/iu.test(prose) && /allowance|UTF-8 bytes/iu.test(prose);
-	return {
-		"explanation.recipeSuitable": suitable,
-		"explanation.conforms": conformance === "pass",
-		"explanation.delivered":
-			conformance === "pass" &&
-			suitable &&
-			cited &&
-			topics &&
-			words >= fixture.wordRange[0] &&
-			words <= fixture.wordRange[1],
-		"explanation.limited": limited,
-		"explanation.words": words,
-	};
-}
-
-function prepare() {
-	for (const path of Object.keys(fixture.files)) assert.ok(!existsSync(path), `refuse to overwrite ${path}`);
-	for (const [path, content] of Object.entries(fixture.files)) {
-		mkdirSync(dirname(path), { recursive: true });
-		writeFileSync(path, content);
-	}
-}
-
-async function grade(agentId) {
-	assert.ok(agentId === "coder" || agentId === "scout");
-	const events = readFileSync(process.env.CLIO_CODER_EVAL_RUNNER_STDOUT_FILE, "utf8")
-		.split(/\r?\n/u)
-		.flatMap((line) => {
-			try {
-				return [JSON.parse(line)];
-			} catch {
-				return [];
-			}
-		});
-	const starts = events.filter((event) => event.type === "tool_execution_start" && event.toolName === "dispatch");
-	assert.equal(starts.length, 1, "one explicitly selected worker; no substituted or extra recovery dispatch");
-	assert.equal(starts[0].args.agent, agentId);
-	assert.equal(starts[0].args.task, fixture.task);
-	const end = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === starts[0].toolCallId);
-	const runs = end?.result?.details?.runs;
-	assert.equal(runs?.length, 1, "one terminal receipt required");
-	const receipt = JSON.parse(readFileSync(runs[0].receiptPath, "utf8"));
-	assert.equal(receipt.agentId, agentId);
-	const metrics = assessExplanation(agentId, receipt.output?.text ?? "", receipt.quality?.resultContract?.conformance);
-	if (
-		receipt.outcome !== "succeeded" ||
-		receipt.exitCode !== 0 ||
-		receipt.quality?.resultContract?.quality === "fail" ||
-		receipt.output?.state !== "final" ||
-		receipt.output?.truncated !== false
-	)
-		metrics["explanation.delivered"] = false;
-	metrics["task.solved"] = metrics["explanation.delivered"];
-	process.stdout.write(`${JSON.stringify({ schema: "clio-coder.eval.measure.v1", metrics })}\n`);
-	assert.equal(receipt.outcome, "succeeded");
-	assert.equal(receipt.exitCode, 0);
-	assert.equal(receipt.output?.state, "final");
-	assert.equal(receipt.output?.truncated, false, "a truncated report is not complete delivery");
-	assert.equal(metrics["explanation.conforms"], true);
-	assert.notEqual(receipt.quality?.resultContract?.quality, "fail");
-	assert.equal(metrics["explanation.recipeSuitable"], true);
-	assert.equal(metrics["explanation.delivered"], true);
-	assert.equal(metrics["explanation.limited"], false);
-	for (const [path, content] of Object.entries(fixture.files)) assert.equal(readFileSync(path, "utf8"), content);
-	// Headless runs cannot assert watch-pane lifecycle. Root records it separately.
-	assert.ok(metrics["task.solved"], "the requested explanation was not delivered in the declared result shape");
-}
-
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
-	try {
-		if (process.argv[2] === "--prepare") prepare();
-		else await grade(process.argv[2]);
-	} catch (error) {
-		process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`);
-		process.exitCode = 1;
-	}
-}

package/evals/tool-surface-grader.mjs

@@ -1,53 +0,0 @@
-import assert from "node:assert/strict";
-import { readFileSync } from "node:fs";
-import { resolve } from "node:path";
-
-const events = readFileSync(process.env.CLIO_CODER_EVAL_RUNNER_STDOUT_FILE, "utf8")
-	.split(/\r?\n/u)
-	.flatMap((line) => {
-		try {
-			return [JSON.parse(line)];
-		} catch {
-			return [];
-		}
-	});
-const starts = events.filter((event) => event.type === "tool_execution_start");
-const ends = events.filter((event) => event.type === "tool_execution_end");
-function output(call) {
-	const end = ends.find((event) => event.toolCallId === call.toolCallId);
-	assert.ok(end, "required tool call did not finish");
-	return JSON.stringify(end.result);
-}
-switch (process.argv[2]) {
-	case "read-tail": {
-		assert.equal(starts.length, 1);
-		const call = starts[0];
-		assert.equal(call.toolName, "read");
-		assert.equal(resolve(call.args.path), resolve("evals/fixtures/tool-surface/read.txt"));
-		assert.equal(call.args.tail, 2, "last-line probe must use the advertised tail argument");
-		assert.match(output(call), /gamma/u);
-		assert.match(output(call), /delta/u);
-		assert.doesNotMatch(output(call), /alpha|beta/u);
-		break;
-	}
-	case "bash-cwd-reset": {
-		assert.equal(starts.length, 2);
-		assert.ok(starts.every((call) => call.toolName === "bash"));
-		assert.equal(starts[0].args.command.trim(), "cd evals/fixtures/tool-surface/subdir && pwd");
-		assert.equal(starts[1].args.command.trim(), "pwd");
-		assert.equal(starts[1].args.cwd, undefined);
-		assert.ok(output(starts[0]).includes(resolve("evals/fixtures/tool-surface/subdir")));
-		assert.ok(output(starts[1]).includes(process.cwd()));
-		assert.ok(!output(starts[1]).includes("/tool-surface/subdir"));
-		break;
-	}
-	case "web-fetch-scheme": {
-		assert.equal(starts.length, 1);
-		assert.equal(starts[0].toolName, "web_fetch");
-		assert.equal(starts[0].args.url, "file:///tool-surface-fixture");
-		assert.match(output(starts[0]), /(?:unsupported|invalid|only)[\s\S]*(?:scheme|https?)/iu);
-		break;
-	}
-	default:
-		throw new Error("unknown tool surface case");
-}

package/evals/tool-surface-model.yaml

@@ -1,274 +0,0 @@
-version: 2
-suite:
-  id: tool-surface-model
-  title: Tool description and argument behavior
-  visibility: public
-  description: Model-required probes of read tail selection, bash child directory
-    reset, and web_fetch URL rejection. Expected rejection is distinct from
-    unintended tool errors.
-matrix:
-  targets:
-    - id: mini
-  repeats: 3
-tasks:
-  - id: read-tail
-    tags:
-      - behavioral
-      - model-required
-      - tool:read
-    behavioral:
-      schema: clio-coder.eval.scenario.v1
-      corpus:
-        id: tool-surface-model
-        version: 1.0.0
-      execution:
-        mode: model-required
-        subject:
-          kind: main-agent
-          role: main
-        toolTarget: available
-      expectedBehavior:
-        - id: read-tail-calls
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.calls.read
-            op: eq
-            value: 1
-        - id: read-tail-errors
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.failed.read
-            op: eq
-            value: 0
-        - id: read-tail-blocks
-          category: safety_comprehension
-          fact:
-            source: tool
-            key: tools.blocked.read
-            op: eq
-            value: 0
-        - id: read-tail-successes
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.succeeded.read
-            op: eq
-            value: 1
-        - id: read-tail-correct
-          category: task_correctness
-          fact:
-            source: grader
-            key: task.solved
-            op: eq
-            value: true
-      forbiddenBehavior: []
-      judge:
-        maxEvidenceItems: 32
-        maxExplanationChars: 500
-    workspace:
-      kind: temp-copy
-      path: ..
-      excludes:
-        - .git
-        - .superpowers
-        - node_modules
-        - dist
-    runner:
-      kind: clio-coder-run
-      autonomy: auto-edit
-      prompt: Read only the final two lines of evals/fixtures/tool-surface/read.txt
-        using one read call. Do not read the entire file or use other tools.
-        Report those lines. Do not edit files.
-    verify:
-      measure:
-        - node evals/tool-surface-grader.mjs read-tail
-      protectedFiles:
-        - evals/tool-surface-grader.mjs
-        - evals/fixtures/tool-surface/read.txt
-        - evals/fixtures/tool-surface/subdir/keep.txt
-    metrics:
-      collect:
-        - task.solved
-        - tools.totalCalls
-        - tools.calls.read
-        - tools.failed.read
-        - tools.blocked.read
-        - tools.succeeded.read
-    timeoutMs: 120000
-  - id: bash-cwd-reset
-    tags:
-      - behavioral
-      - model-required
-      - tool:bash
-    behavioral:
-      schema: clio-coder.eval.scenario.v1
-      corpus:
-        id: tool-surface-model
-        version: 1.0.0
-      execution:
-        mode: model-required
-        subject:
-          kind: main-agent
-          role: main
-        toolTarget: available
-      expectedBehavior:
-        - id: bash-cwd-reset-calls
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.calls.bash
-            op: eq
-            value: 2
-        - id: bash-cwd-reset-errors
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.failed.bash
-            op: eq
-            value: 0
-        - id: bash-cwd-reset-blocks
-          category: safety_comprehension
-          fact:
-            source: tool
-            key: tools.blocked.bash
-            op: eq
-            value: 0
-        - id: bash-cwd-reset-successes
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.succeeded.bash
-            op: eq
-            value: 2
-        - id: bash-cwd-reset-correct
-          category: task_correctness
-          fact:
-            source: grader
-            key: task.solved
-            op: eq
-            value: true
-      forbiddenBehavior: []
-      judge:
-        maxEvidenceItems: 32
-        maxExplanationChars: 500
-    workspace:
-      kind: temp-copy
-      path: ..
-      excludes:
-        - .git
-        - .superpowers
-        - node_modules
-        - dist
-    runner:
-      kind: clio-coder-run
-      autonomy: auto-edit
-      prompt: Use exactly two bash calls and no other tools. First run cd
-        evals/fixtures/tool-surface/subdir && pwd. Then run pwd with no cwd
-        argument, relying on the documented default directory of a new bash
-        call. Report whether the second call starts at the workspace root. Do
-        not edit files.
-    verify:
-      measure:
-        - node evals/tool-surface-grader.mjs bash-cwd-reset
-      protectedFiles:
-        - evals/tool-surface-grader.mjs
-        - evals/fixtures/tool-surface/read.txt
-        - evals/fixtures/tool-surface/subdir/keep.txt
-    metrics:
-      collect:
-        - task.solved
-        - tools.totalCalls
-        - tools.calls.bash
-        - tools.failed.bash
-        - tools.blocked.bash
-        - tools.succeeded.bash
-    timeoutMs: 120000
-  - id: web-fetch-scheme
-    tags:
-      - behavioral
-      - model-required
-      - tool:web_fetch
-    behavioral:
-      schema: clio-coder.eval.scenario.v1
-      corpus:
-        id: tool-surface-model
-        version: 1.0.0
-      execution:
-        mode: model-required
-        subject:
-          kind: main-agent
-          role: main
-        toolTarget: available
-      expectedBehavior:
-        - id: web-fetch-scheme-calls
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.calls.web_fetch
-            op: eq
-            value: 1
-        - id: web-fetch-scheme-errors
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.failed.web_fetch
-            op: eq
-            value: 1
-        - id: web-fetch-scheme-blocks
-          category: safety_comprehension
-          fact:
-            source: tool
-            key: tools.blocked.web_fetch
-            op: eq
-            value: 0
-        - id: web-fetch-scheme-successes
-          category: tool_choice
-          fact:
-            source: tool
-            key: tools.succeeded.web_fetch
-            op: eq
-            value: 0
-        - id: web-fetch-scheme-correct
-          category: task_correctness
-          fact:
-            source: grader
-            key: task.solved
-            op: eq
-            value: true
-      forbiddenBehavior: []
-      judge:
-        maxEvidenceItems: 32
-        maxExplanationChars: 500
-    workspace:
-      kind: temp-copy
-      path: ..
-      excludes:
-        - .git
-        - .superpowers
-        - node_modules
-        - dist
-    runner:
-      kind: clio-coder-run
-      autonomy: auto-edit
-      prompt: "Probe the web_fetch URL validation once with
-        file:///tool-surface-fixture. This is an intentional invalid-scheme
-        test: invoke web_fetch exactly once, report the rejection, and do not
-        retry or use other tools. Do not edit files."
-    verify:
-      measure:
-        - node evals/tool-surface-grader.mjs web-fetch-scheme
-      protectedFiles:
-        - evals/tool-surface-grader.mjs
-        - evals/fixtures/tool-surface/read.txt
-        - evals/fixtures/tool-surface/subdir/keep.txt
-    metrics:
-      collect:
-        - task.solved
-        - tools.totalCalls
-        - tools.calls.web_fetch
-        - tools.failed.web_fetch
-        - tools.blocked.web_fetch
-        - tools.succeeded.web_fetch
-    timeoutMs: 120000

package/evals/tracked-metrics-baseline.yaml

@@ -1,229 +0,0 @@
-version: 2
-suite:
-  id: tracked-metrics-baseline
-  title: Tracked metrics baseline
-  visibility: sprint
-  description: Ten small repository edits that require reading the implementation and two related consumers before a code grader checks behavior.
-  provenance:
-    issue: 252
-    charter: 13_demystifying_evals.md
-matrix:
-  targets:
-    - id: mini
-      model: qwen3.8-27b-dense
-      thinking: off
-  repeats: 1
-tasks:
-  - id: latency-nonnegative
-    tags: &baseline_tags
-      - tracked-metrics
-      - multi-file
-      - small-edit
-    workspace: &repo_workspace
-      kind: temp-copy
-      path: ..
-      excludes:
-        - .git
-        - .superpowers
-        - node_modules
-        - dist
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/domains/eval/metrics/latency.ts, src/domains/eval/suites/run.ts, and
-        src/domains/eval/compare/compare.ts. Then make one focused edit only in
-        src/domains/eval/metrics/latency.ts so wallTimeMetric treats negative finite values as zero
-        while preserving nonnegative finite values. Do not edit tests or benchmarks. Validate with
-        node evals/tracked-metrics-grader.mjs latency-nonnegative.
-    verify:
-      measure:
-        - &grader_latency >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs latency-nonnegative
-    metrics: &baseline_metrics
-      collect:
-        - task.solved
-    timeoutMs: &task_timeout 180000
-
-  - id: safe-id-controls
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/core/safe-id.ts, src/domains/eval/artifacts/store.ts, and
-        src/domains/evidence/store.ts. Then make one focused edit only in src/core/safe-id.ts so
-        assertSafeId rejects every ASCII control character from U+0000 through U+001F and U+007F.
-        Preserve existing valid identifiers and InvalidIdError. Do not edit tests or benchmarks.
-        Validate with node evals/tracked-metrics-grader.mjs safe-id-controls.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs safe-id-controls
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: shell-quote-nul
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/core/shell-quote.ts, src/domains/eval/runners/clio-run.ts, and
-        src/domains/dispatch/fleet-preflight.ts. Then make one focused edit only in
-        src/core/shell-quote.ts so shellQuote throws when its value contains a NUL byte and retains
-        its existing POSIX quoting behavior otherwise. Do not edit tests or benchmarks. Validate
-        with node evals/tracked-metrics-grader.mjs shell-quote-nul.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs shell-quote-nul
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: junit-apostrophe
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/domains/eval/reports/junit.ts, src/domains/eval/schema/artifact.ts, and
-        src/cli/eval.ts. Then make one focused edit only in src/domains/eval/reports/junit.ts so the
-        XML escaping helper also emits ' for apostrophes in suite names, task ids, and failure
-        classes. Preserve the existing escapes. Do not edit tests or benchmarks. Validate with node
-        evals/tracked-metrics-grader.mjs junit-apostrophe.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs junit-apostrophe
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: markdown-pipe
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/domains/eval/reports/markdown.ts, src/domains/eval/schema/artifact.ts, and
-        src/cli/eval.ts. Then make one focused edit only in src/domains/eval/reports/markdown.ts so
-        dynamic table cells escape vertical bars with a backslash. Cover task ids, target ids,
-        model ids, and failure classes without changing static headings. Do not edit tests or
-        benchmarks. Validate with node evals/tracked-metrics-grader.mjs markdown-pipe.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs markdown-pipe
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: suite-override-trim
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/domains/eval/suites/resolve.ts, src/domains/eval/suites/matrix.ts, and
-        src/cli/eval.ts. Then make one focused edit only in src/domains/eval/suites/resolve.ts so
-        target and model overrides are trimmed before target selection and storage. Preserve behavior
-        when an override is absent. Do not edit tests or benchmarks. Validate with node
-        evals/tracked-metrics-grader.mjs suite-override-trim.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs suite-override-trim
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: truncate-marker-budget
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/tools/truncate-utf8.ts, src/tools/result-shaping.ts, and src/tools/read.ts. Then make
-        one focused edit only in src/tools/truncate-utf8.ts so maxBytes bounds the complete returned
-        UTF-8 string including the marker. Preserve whole UTF-8 characters and the unchanged fast
-        path. Do not edit tests or benchmarks. Validate with node
-        evals/tracked-metrics-grader.mjs truncate-marker-budget.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs truncate-marker-budget
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: response-count-validation
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/core/response-model-id.ts, src/interactive/chat-loop-messages.ts, and
-        src/cli/usage.ts. Then make one focused edit only in src/core/response-model-id.ts so
-        addResponseModelIdObservationCount ignores a calls value unless it is a nonnegative integer.
-        Preserve accumulation for valid counts. Do not edit tests or benchmarks. Validate with node
-        evals/tracked-metrics-grader.mjs response-count-validation.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs response-count-validation
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: relative-future
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/interactive/format-time.ts, src/interactive/welcome-dashboard.ts, and
-        tests/contracts/format-time.test.ts. Then make one focused edit only in
-        src/interactive/format-time.ts so relative renders future instants at least five seconds away
-        with the same coarse units prefixed by "in", while nearer instants remain "just now".
-        Preserve past-time behavior. Do not edit tests or benchmarks. Validate with node
-        evals/tracked-metrics-grader.mjs relative-future.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs relative-future
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout
-
-  - id: acp-unicode-boundary
-    tags: *baseline_tags
-    workspace: *repo_workspace
-    runner:
-      kind: clio-coder-run
-      agent: coder
-      prompt: >-
-        Read src/engine/acp/errors.ts, src/engine/acp/server.ts, and src/engine/acp/transport.ts. Then
-        make one focused edit only in src/engine/acp/errors.ts so acpErrorMessage never splits a
-        Unicode surrogate pair at its truncation boundary and still stays within
-        ACP_MAX_ERROR_MESSAGE_CHARS UTF-16 units including the ellipsis. Preserve whitespace
-        collapsing and short messages. Do not edit tests or benchmarks. Validate with node
-        evals/tracked-metrics-grader.mjs acp-unicode-boundary.
-    verify:
-      measure:
-        - >-
-          echo "9ca5282398f9e7cd70367bdf3620448d6353d340b09ffdf25336366281b5650b  evals/tracked-metrics-grader.mjs"
-          | sha256sum --check --status && node evals/tracked-metrics-grader.mjs acp-unicode-boundary
-    metrics: *baseline_metrics
-    timeoutMs: *task_timeout