@maxiaochao/pi-toolkit 0.9.3 → 0.9.6

package/packages/codex-edit/tests/benchmark.test.mjs

@@ -0,0 +1,81 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import test from "node:test";
+import { incidentCases } from "../benchmark/incident-cases.mjs";
+import { interleaveNeutralAndChallenge } from "../benchmark/schedule.mjs";
+import { applyUpdate, parsePatch } from "../src/parser.ts";
+import { comparisonCases } from "./comparison-cases.mjs";
+import { runPatchCase } from "./comparison-harness.mjs";
+
+const manifest = JSON.parse(readFileSync(new URL("../benchmark/explicit-edit-100.json", import.meta.url), "utf8"));
+const evidence = JSON.parse(readFileSync(new URL("../benchmark/results/explicit-edit-100-sol-low-v3.json", import.meta.url), "utf8"));
+
+test("frozen 100-task selection keeps the same task IDs and fixture hashes", () => {
+  assert.equal(manifest.included.length, 100);
+  assert.equal(new Set(manifest.included.map(({ id }) => id)).size, 100);
+  const digest = createHash("sha256")
+    .update(manifest.included.map(({ id, fixtureSha256 }) => `${id}\0${fixtureSha256}`).sort().join("\n"))
+    .digest("hex");
+  assert.equal(digest, "3a5da55315806cfd7984264e6afb5f5b69739d2101b7128cb25ff35f834afc3b");
+  assert.equal(evidence.completed, 300);
+  assert.equal(evidence.results.length, 300);
+  assert.equal(evidence.identities.focusedSelectionSha256, evidence.identities.taskManifestSha256);
+
+  const outputDirectory = mkdtempSync(join(tmpdir(), "codex-edit-report-"));
+  try {
+    execFileSync(process.execPath, [new URL("./generate-test-report.mjs", import.meta.url).pathname], {
+      env: { ...process.env, CODEX_EDIT_REPORT_DIR: outputDirectory, CODEX_EDIT_REPORT_SKIP_CHECKS: "1" },
+      stdio: "pipe",
+    });
+    for (const filename of ["test-summary.md", "test-report.md", "test-report.html"]) {
+      assert.equal(
+        readFileSync(join(outputDirectory, filename), "utf8"),
+        readFileSync(new URL(`../docs/${filename}`, import.meta.url), "utf8"),
+        `${filename} must regenerate byte-for-byte from committed evidence`,
+      );
+    }
+  } finally {
+    rmSync(outputDirectory, { recursive: true, force: true });
+  }
+});
+
+test("every five scheduled tasks contain four neutral and one challenge task", () => {
+  assert.equal(manifest.selection.scheduling, "four-neutral-then-one-challenge-v1");
+  for (let index = 0; index < manifest.included.length; index += 5) {
+    const cohorts = manifest.included.slice(index, index + 5).map(({ cohort }) => cohort);
+    assert.deepEqual(cohorts, ["neutral", "neutral", "neutral", "neutral", "challenge"]);
+  }
+  const budgetPrefix = manifest.included.slice(0, 47);
+  assert.equal(budgetPrefix.filter(({ cohort }) => cohort === "neutral").length, 38);
+  assert.equal(budgetPrefix.filter(({ cohort }) => cohort === "challenge").length, 9);
+});
+
+test("scheduler rejects malformed ratios and unknown cohorts", () => {
+  assert.throws(() => interleaveNeutralAndChallenge([{ cohort: "neutral" }]), /Expected a 4:1/);
+  assert.throws(() => interleaveNeutralAndChallenge([{ cohort: "incident" }]), /Unknown cohort/);
+  assert.throws(() => interleaveNeutralAndChallenge([], 0), /positive integer/);
+});
+
+test("reported production incidents stay linked to executable protocol regressions", () => {
+  assert.equal(incidentCases.length, 5);
+  assert.equal(new Set(incidentCases.map(({ id }) => id)).size, incidentCases.length);
+  assert.equal(incidentCases.filter(({ tier }) => tier === "agent-and-protocol").length, 4);
+  assert.ok(incidentCases.some(({ tier }) => tier === "protocol-only"));
+
+  const comparisonById = new Map(comparisonCases.map((testCase) => [testCase.id, testCase]));
+  for (const incident of incidentCases) {
+    const protocolCase = comparisonById.get(incident.protocolCaseId);
+    assert.ok(protocolCase, `${incident.id} has no protocol regression`);
+    assert.equal(protocolCase.kind, "valid-edit");
+    assert.equal(runPatchCase(protocolCase, parsePatch, applyUpdate, "pass"), "pass");
+    if (incident.tier === "agent-and-protocol") {
+      assert.doesNotMatch(incident.prompt, /apply_patch|Update File|@@/i);
+    }
+  }
+  assert.ok(incidentCases.some(({ protocolCaseId }) => comparisonById.get(protocolCaseId).outcomes.released === "fail"));
+  assert.ok(incidentCases.some(({ protocolCaseId }) => comparisonById.get(protocolCaseId).outcomes.native !== "pass"));
+});

package/packages/codex-edit/tests/comparison-cases.mjs

@@ -0,0 +1,396 @@
+export const comparisonCases = [
+  {
+    id: "unique-replacement",
+    kind: "valid-edit",
+    input: { "app.ts": "const enabled = false;\n" },
+    expected: { "app.ts": "const enabled = true;\n" },
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+-const enabled = false;
++const enabled = true;
+*** End Patch`,
+    nativeCalls: [{ path: "app.ts", edits: [{ oldText: "const enabled = false;", newText: "const enabled = true;" }] }],
+    outcomes: { native: "pass", released: "pass", local: "pass" },
+  },
+  {
+    id: "disjoint-edits-one-file",
+    kind: "valid-edit",
+    input: { "app.ts": "const first = 1;\nconst middle = 2;\nconst last = 3;\n" },
+    expected: { "app.ts": "const first = 10;\nconst middle = 2;\nconst last = 30;\n" },
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+-const first = 1;
++const first = 10;
+@@
+-const last = 3;
++const last = 30;
+*** End Patch`,
+    nativeCalls: [{
+      path: "app.ts",
+      edits: [
+        { oldText: "const first = 1;", newText: "const first = 10;" },
+        { oldText: "const last = 3;", newText: "const last = 30;" },
+      ],
+    }],
+    outcomes: { native: "pass", released: "pass", local: "pass" },
+  },
+  {
+    id: "repeated-markdown-separator",
+    kind: "valid-edit",
+    input: { "SKILL.md": "---\nname: demo\ndescription: old\n---\n\n# Demo\n\n---\nFooter\n" },
+    expected: { "SKILL.md": "---\nname: demo\ndescription: new\n---\nInserted after frontmatter.\n\n# Demo\n\n---\nFooter\n" },
+    patch: `*** Begin Patch
+*** Update File: SKILL.md
+@@
+-description: old
++description: new
+@@
+ ---
++Inserted after frontmatter.
+*** End Patch`,
+    nativeCalls: [{ path: "SKILL.md", edits: [{ oldText: "---", newText: "---\nInserted after frontmatter." }] }],
+    outcomes: { native: "fail", released: "pass", local: "pass" },
+  },
+  {
+    id: "repeated-update-same-path",
+    kind: "valid-edit",
+    input: { "tool.ts": "const state = \"old\";\n" },
+    expected: { "tool.ts": "const state = \"new\";\n" },
+    patch: `*** Begin Patch
+*** Update File: tool.ts
+@@
+-const state = "old";
++const state = "middle";
+*** Update File: tool.ts
+@@
+-const state = "middle";
++const state = "new";
+*** End Patch`,
+    nativeCalls: [
+      { path: "tool.ts", edits: [{ oldText: "const state = \"old\";", newText: "const state = \"middle\";" }] },
+      { path: "tool.ts", edits: [{ oldText: "const state = \"middle\";", newText: "const state = \"new\";" }] },
+    ],
+    outcomes: { native: "pass", released: "fail", local: "pass" },
+  },
+  {
+    id: "ordered-runner-type-renames",
+    kind: "valid-edit",
+    input: {
+      "packages/herdr-subagents/src/runner.ts": `export interface TinySubagentDocument {
+\tpath: string;
+}
+
+export interface TinySubagentReport {
+\tdocuments: TinySubagentDocument[];
+}
+
+export interface TinySubagentOptions {
+\tonUpdate?: (result: TinySubagentResult) => void;
+}
+
+export interface TinySubagentResult {
+\tdocuments: TinySubagentDocument[];
+\tstatus: "running" | "failed";
+}
+
+function compactText(report: TinySubagentReport): string {
+\treturn report.documents.map((document) => document.path).join("\\n");
+}
+
+function failure(cwd: string, startedAt: number, errorMessage: string, fields: Partial<TinySubagentResult> = {}): TinySubagentResult {
+\treturn { documents: [], status: "failed", ...fields };
+}
+
+function validReport(value: unknown): value is TinySubagentReport {
+\treturn Boolean(value);
+}
+
+async function readReport(path: string): Promise<TinySubagentReport | undefined> {
+\treturn undefined;
+}
+
+export async function runTinySubagent(prompt: string, options: TinySubagentOptions = {}): Promise<TinySubagentResult> {
+\tlet result: TinySubagentResult | undefined;
+\tconst common = (): Partial<TinySubagentResult> => ({});
+\tconst update = (status: TinySubagentResult["status"], summary: string) => {};
+\treturn result ?? failure(".", 0, prompt, common());
+}
+`,
+    },
+    expected: {
+      "packages/herdr-subagents/src/runner.ts": `export interface HerdrSubagentsDocument {
+\tpath: string;
+}
+
+export interface HerdrSubagentsReport {
+\tdocuments: HerdrSubagentsDocument[];
+}
+
+export interface HerdrSubagentsOptions {
+\tonUpdate?: (result: HerdrSubagentsResult) => void;
+}
+
+export interface HerdrSubagentsResult {
+\tdocuments: HerdrSubagentsDocument[];
+\tstatus: "running" | "failed";
+}
+
+function compactText(report: HerdrSubagentsReport): string {
+\treturn report.documents.map((document) => document.path).join("\\n");
+}
+
+function failure(cwd: string, startedAt: number, errorMessage: string, fields: Partial<HerdrSubagentsResult> = {}): HerdrSubagentsResult {
+\treturn { documents: [], status: "failed", ...fields };
+}
+
+function validReport(value: unknown): value is HerdrSubagentsReport {
+\treturn Boolean(value);
+}
+
+async function readReport(path: string): Promise<HerdrSubagentsReport | undefined> {
+\treturn undefined;
+}
+
+export async function runHerdrSubagents(prompt: string, options: HerdrSubagentsOptions = {}): Promise<HerdrSubagentsResult> {
+\tlet result: HerdrSubagentsResult | undefined;
+\tconst common = (): Partial<HerdrSubagentsResult> => ({});
+\tconst update = (status: HerdrSubagentsResult["status"], summary: string) => {};
+\treturn result ?? failure(".", 0, prompt, common());
+}
+`,
+    },
+    patch: `*** Begin Patch
+*** Update File: packages/herdr-subagents/src/runner.ts
+@@
+-export interface TinySubagentDocument {
++export interface HerdrSubagentsDocument {
+@@
+-export interface TinySubagentReport {
++export interface HerdrSubagentsReport {
+@@
+-\tdocuments: TinySubagentDocument[];
++\tdocuments: HerdrSubagentsDocument[];
+@@
+-export interface TinySubagentOptions {
++export interface HerdrSubagentsOptions {
+@@
+-\tonUpdate?: (result: TinySubagentResult) => void;
++\tonUpdate?: (result: HerdrSubagentsResult) => void;
+@@
+-export interface TinySubagentResult {
++export interface HerdrSubagentsResult {
+@@
+-\tdocuments: TinySubagentDocument[];
++\tdocuments: HerdrSubagentsDocument[];
+@@
+-function compactText(report: TinySubagentReport): string {
++function compactText(report: HerdrSubagentsReport): string {
+@@
+-function failure(cwd: string, startedAt: number, errorMessage: string, fields: Partial<TinySubagentResult> = {}): TinySubagentResult {
++function failure(cwd: string, startedAt: number, errorMessage: string, fields: Partial<HerdrSubagentsResult> = {}): HerdrSubagentsResult {
+@@
+-function validReport(value: unknown): value is TinySubagentReport {
++function validReport(value: unknown): value is HerdrSubagentsReport {
+@@
+-async function readReport(path: string): Promise<TinySubagentReport | undefined> {
++async function readReport(path: string): Promise<HerdrSubagentsReport | undefined> {
+@@
+-export async function runTinySubagent(prompt: string, options: TinySubagentOptions = {}): Promise<TinySubagentResult> {
++export async function runHerdrSubagents(prompt: string, options: HerdrSubagentsOptions = {}): Promise<HerdrSubagentsResult> {
+@@
+-\tlet result: TinySubagentResult | undefined;
+-\tconst common = (): Partial<TinySubagentResult> => ({});
++\tlet result: HerdrSubagentsResult | undefined;
++\tconst common = (): Partial<HerdrSubagentsResult> => ({});
+@@
+-\tconst update = (status: TinySubagentResult["status"], summary: string) => {};
++\tconst update = (status: HerdrSubagentsResult["status"], summary: string) => {};
+*** End Patch`,
+    nativeCalls: [{
+      path: "packages/herdr-subagents/src/runner.ts",
+      edits: [{
+        oldText: `export interface TinySubagentDocument {
+\tpath: string;
+}
+
+export interface TinySubagentReport {
+\tdocuments: TinySubagentDocument[];
+}
+
+export interface TinySubagentOptions {
+\tonUpdate?: (result: TinySubagentResult) => void;
+}
+
+export interface TinySubagentResult {
+\tdocuments: TinySubagentDocument[];
+\tstatus: "running" | "failed";
+}`,
+        newText: `export interface HerdrSubagentsDocument {
+\tpath: string;
+}
+
+export interface HerdrSubagentsReport {
+\tdocuments: HerdrSubagentsDocument[];
+}
+
+export interface HerdrSubagentsOptions {
+\tonUpdate?: (result: HerdrSubagentsResult) => void;
+}
+
+export interface HerdrSubagentsResult {
+\tdocuments: HerdrSubagentsDocument[];
+\tstatus: "running" | "failed";
+}`,
+      }, {
+        oldText: `function compactText(report: TinySubagentReport): string {
+\treturn report.documents.map((document) => document.path).join("\\n");
+}
+
+function failure(cwd: string, startedAt: number, errorMessage: string, fields: Partial<TinySubagentResult> = {}): TinySubagentResult {
+\treturn { documents: [], status: "failed", ...fields };
+}
+
+function validReport(value: unknown): value is TinySubagentReport {
+\treturn Boolean(value);
+}
+
+async function readReport(path: string): Promise<TinySubagentReport | undefined> {
+\treturn undefined;
+}
+
+export async function runTinySubagent(prompt: string, options: TinySubagentOptions = {}): Promise<TinySubagentResult> {
+\tlet result: TinySubagentResult | undefined;
+\tconst common = (): Partial<TinySubagentResult> => ({});
+\tconst update = (status: TinySubagentResult["status"], summary: string) => {};`,
+        newText: `function compactText(report: HerdrSubagentsReport): string {
+\treturn report.documents.map((document) => document.path).join("\\n");
+}
+
+function failure(cwd: string, startedAt: number, errorMessage: string, fields: Partial<HerdrSubagentsResult> = {}): HerdrSubagentsResult {
+\treturn { documents: [], status: "failed", ...fields };
+}
+
+function validReport(value: unknown): value is HerdrSubagentsReport {
+\treturn Boolean(value);
+}
+
+async function readReport(path: string): Promise<HerdrSubagentsReport | undefined> {
+\treturn undefined;
+}
+
+export async function runHerdrSubagents(prompt: string, options: HerdrSubagentsOptions = {}): Promise<HerdrSubagentsResult> {
+\tlet result: HerdrSubagentsResult | undefined;
+\tconst common = (): Partial<HerdrSubagentsResult> => ({});
+\tconst update = (status: HerdrSubagentsResult["status"], summary: string) => {};`,
+      }],
+    }],
+    outcomes: { native: "pass", released: "pass", local: "pass" },
+  },
+  {
+    id: "multi-file-add-delete-move",
+    kind: "valid-edit",
+    input: { "old.ts": "export const value = 1;\n", "remove.txt": "obsolete\n" },
+    expected: { "new.ts": "export const value = 2;\n", "added.txt": "created\n" },
+    patch: `*** Begin Patch
+*** Update File: old.ts
+*** Move to: new.ts
+@@
+-export const value = 1;
++export const value = 2;
+*** Delete File: remove.txt
+*** Add File: added.txt
++created
+*** End Patch`,
+    nativeCalls: null,
+    outcomes: { native: "unsupported", released: "pass", local: "pass" },
+  },
+  {
+    id: "stale-expected-lines",
+    kind: "safety",
+    input: { "app.ts": "const actual = true;\n" },
+    expected: null,
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+-const stale = false;
++const stale = true;
+*** End Patch`,
+    nativeCalls: [{ path: "app.ts", edits: [{ oldText: "const stale = false;", newText: "const stale = true;" }] }],
+    outcomes: { native: "fail", released: "fail", local: "fail" },
+  },
+  {
+    id: "conflicting-targets",
+    kind: "safety",
+    input: { "app.ts": "old\n" },
+    expected: null,
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+-old
++new
+*** Delete File: app.ts
+*** End Patch`,
+    nativeCalls: null,
+    outcomes: { native: "unsupported", released: "fail", local: "fail" },
+  },
+  {
+    id: "empty-update-chunk",
+    kind: "safety",
+    input: { "app.ts": "unchanged\n" },
+    expected: null,
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+*** End Patch`,
+    nativeCalls: null,
+    outcomes: { native: "unsupported", released: "fail", local: "fail" },
+  },
+  {
+    id: "resolved-path-alias-conflict",
+    kind: "safety",
+    input: { "app.ts": "const value = 1;\n" },
+    expected: null,
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+-const value = 1;
++const value = 2;
+*** Update File: ./app.ts
+@@
+-const value = 2;
++const value = 3;
+*** End Patch`,
+    nativeCalls: null,
+    outcomes: { native: "unsupported", released: "fail", local: "fail" },
+  },
+  {
+    id: "outside-workspace-absolute-path",
+    kind: "safety",
+    input: { "app.ts": "const value = 1;\n" },
+    expected: null,
+    patch: `*** Begin Patch
+*** Update File: /tmp/outside-workspace.ts
+@@
+-const value = 1;
++const value = 2;
+*** End Patch`,
+    nativeCalls: null,
+    outcomes: { native: "unsupported", released: "fail", local: "fail" },
+  },
+  {
+    id: "bom-and-crlf",
+    kind: "valid-edit",
+    input: { "app.ts": "\ufeffconst old = 1;\r\nconst keep = 2;\r\n" },
+    expected: { "app.ts": "\ufeffconst next = 1;\r\nconst keep = 2;\r\n" },
+    patch: `*** Begin Patch
+*** Update File: app.ts
+@@
+-const old = 1;
++const next = 1;
+*** End Patch`,
+    nativeCalls: [{ path: "app.ts", edits: [{ oldText: "const old = 1;", newText: "const next = 1;" }] }],
+    outcomes: { native: "pass", released: "pass", local: "pass" },
+  },
+];

package/packages/codex-edit/tests/comparison-harness.mjs

@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+
+export function applyOperations(input, operations, applyUpdate) {
+  const files = new Map(Object.entries(input));
+  for (const operation of operations) {
+    if (operation.action === "add") {
+      if (files.has(operation.path)) throw new Error(`Cannot add existing file ${operation.path}`);
+      files.set(operation.path, operation.content);
+      continue;
+    }
+    if (!files.has(operation.path)) throw new Error(`Missing source file ${operation.path}`);
+    if (operation.action === "delete") {
+      files.delete(operation.path);
+      continue;
+    }
+    const groups = operation.chunkGroups ?? [operation.chunks];
+    const after = groups.reduce(
+      (content, chunks) => applyUpdate(content, chunks, operation.path),
+      files.get(operation.path),
+    );
+    if (operation.moveTo) {
+      if (files.has(operation.moveTo)) throw new Error(`Cannot move over existing file ${operation.moveTo}`);
+      files.delete(operation.path);
+      files.set(operation.moveTo, after);
+    } else {
+      files.set(operation.path, after);
+    }
+  }
+  return Object.fromEntries([...files].sort(([left], [right]) => left.localeCompare(right)));
+}
+
+export function runPatchCase(testCase, parse, applyUpdate, expectedOutcome) {
+  try {
+    const actual = applyOperations(testCase.input, parse(testCase.patch), applyUpdate);
+    if (expectedOutcome !== "pass") {
+      assert.fail(`${testCase.id}: expected ${expectedOutcome}, but patch succeeded`);
+    }
+    assert.deepEqual(actual, testCase.expected, `${testCase.id}: final filesystem differs`);
+    return "pass";
+  } catch (error) {
+    if (expectedOutcome === "pass") throw error;
+    return "fail";
+  }
+}