@maxiaochao/pi-toolkit 0.9.4 → 0.10.0

package/packages/codex-edit/benchmark/export-evidence.mjs

@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { basename, resolve } from "node:path";
+
+const [runDirectoryArg, outputArg] = process.argv.slice(2);
+if (!runDirectoryArg || !outputArg) {
+  console.error("Usage: export-evidence.mjs <benchmark-run-directory> <output.json>");
+  process.exit(2);
+}
+
+const runDirectory = resolve(runDirectoryArg);
+const output = resolve(outputArg);
+const packageDirectory = resolve(new URL("..", import.meta.url).pathname);
+const [progress, runManifest, taskManifestText] = await Promise.all([
+  readFile(resolve(runDirectory, "progress.json"), "utf8").then(JSON.parse),
+  readFile(resolve(runDirectory, "manifest.json"), "utf8").then(JSON.parse),
+  readFile(resolve(packageDirectory, "benchmark/explicit-edit-100.json"), "utf8"),
+]);
+const taskManifest = JSON.parse(taskManifestText);
+const taskById = new Map(taskManifest.included.map((task) => [task.id, task]));
+const sumAttempts = (result, field) => (result.recovery?.attempts ?? []).reduce(
+  (sum, attempt) => sum + Number(attempt.execution?.[field] ?? 0), 0,
+);
+
+const evidence = {
+  schemaVersion: 1,
+  runId: basename(runDirectory),
+  completed: progress.completed,
+  total: progress.total,
+  identities: {
+    taskManifestSha256: createHash("sha256").update(taskManifestText).digest("hex"),
+    focusedSelectionSha256: runManifest.focusedSelectionSha256,
+    configSha256: runManifest.configSha256,
+    verifierSha256: runManifest.verifierSha256,
+  },
+  source: taskManifest.source,
+  selection: {
+    seed: taskManifest.selection.seed,
+    scheduling: taskManifest.selection.scheduling,
+    neutral: taskManifest.included.filter((task) => task.cohort === "neutral").length,
+    challenge: taskManifest.included.filter((task) => task.cohort === "challenge").length,
+  },
+  run: {
+    contract: runManifest.contract,
+    attempts: runManifest.attempts,
+    oracleRecoveries: runManifest.oracleRecoveries,
+    concurrency: runManifest.concurrency,
+    timeoutMs: runManifest.timeoutMs,
+  },
+  harnesses: Object.fromEntries(Object.entries(runManifest.harnesses).map(([profile, harness]) => [profile, {
+    agentVersion: harness.agentVersion,
+    model: harness.model,
+    thinking: harness.thinking,
+    harnessVersion: harness.harnessVersion,
+    configurationId: harness.configurationId,
+    extensions: harness.configuration?.extensions ?? [],
+  }])),
+  results: progress.results.map((result) => ({
+    taskId: result.taskId,
+    profile: result.profile,
+    category: result.category,
+    cohort: taskById.get(result.taskId)?.cohort,
+    firstAttemptPassed: Boolean(result.recovery?.firstAttemptPassed),
+    eventuallyPassed: Boolean(result.recovery?.eventuallyPassed),
+    recoveriesUsed: Number(result.recovery?.recoveriesUsed ?? 0),
+    costUsd: sumAttempts(result, "costUsd"),
+    totalTokens: sumAttempts(result, "totalTokens"),
+    seconds: Number(result.seconds ?? 0),
+    toolCalls: sumAttempts(result, "toolCalls"),
+    failedToolCalls: sumAttempts(result, "failedToolCalls"),
+    providerFailures: (result.recovery?.attempts ?? []).filter((attempt) => attempt.execution?.providerFailure).length,
+  })),
+};
+
+await writeFile(output, `${JSON.stringify(evidence, null, 2)}\n`, "utf8");
+console.log(`Exported ${evidence.results.length} sanitized results to ${output}`);

package/packages/codex-edit/benchmark/incident-cases.mjs

@@ -0,0 +1,37 @@
+export const incidentCases = [
+  {
+    id: "incident-repeated-markdown-context",
+    protocolCaseId: "repeated-markdown-separator",
+    tier: "agent-and-protocol",
+    issue: "重复 Markdown 分隔符曾触发 ambiguous match",
+    prompt: "更新 SKILL.md 的 frontmatter 描述，并在 frontmatter 结束分隔符后插入指定说明；不要改动正文和页脚。",
+  },
+  {
+    id: "incident-repeated-update-same-path",
+    protocolCaseId: "repeated-update-same-path",
+    tier: "protocol-only",
+    issue: "同一个 raw path 的多个 Update File 块在发布版中冲突",
+    prompt: "验证同一次 apply_patch 调用中，同一路径的多个普通 Update File 块可以按顺序以上一块结果为输入。",
+  },
+  {
+    id: "incident-runner-type-rename",
+    protocolCaseId: "ordered-runner-type-renames",
+    tier: "agent-and-protocol",
+    issue: "runner.ts 中重复类型名的大范围重命名曾触发 ambiguous match",
+    prompt: "将 packages/herdr-subagents/src/runner.ts 中 TinySubagent 系列公开类型和引用完整重命名为 HerdrSubagents，保持其余代码逐字不变。",
+  },
+  {
+    id: "incident-multi-file-structural-edit",
+    protocolCaseId: "multi-file-add-delete-move",
+    tier: "agent-and-protocol",
+    issue: "一次 patch 中的 add/delete/move 需要原子预校验和执行",
+    prompt: "按要求移动并更新 old.ts，删除 remove.txt，再创建 added.txt；不要留下额外文件。",
+  },
+  {
+    id: "incident-bom-crlf",
+    protocolCaseId: "bom-and-crlf",
+    tier: "agent-and-protocol",
+    issue: "UTF-8 BOM 与 CRLF 曾导致原生精确文本匹配失败",
+    prompt: "只修改 app.ts 中的目标标识符，同时逐字保留 UTF-8 BOM、CRLF 和未指定行。",
+  },
+];

package/packages/codex-edit/benchmark/prepare.mjs

@@ -0,0 +1,181 @@
+#!/usr/bin/env node
+
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+
+const PI_VERSION = "0.85.1";
+const MODEL = "openai-codex/gpt-5.6-sol";
+const THINKING = "low";
+const PACKAGE = "@maxiaochao/pi-codex-edit";
+const nodeBinary = realpathSync(execFileSync("which", ["node"], { encoding: "utf8" }).trim());
+const nodeRuntime = dirname(dirname(nodeBinary));
+const root = resolve(".cache/codex-edit-benchmark");
+const authFile = join(homedir(), ".pi/agent/auth.json");
+const modelFile = join(homedir(), ".pi/agent/models.json");
+for (const file of [authFile, modelFile]) {
+  if (!existsSync(file)) throw new Error(`Required Pi state is missing: ${file}`);
+}
+
+const npmJson = (...args) =>
+  JSON.parse(execFileSync("npm", args, { encoding: "utf8", maxBuffer: 10 * 1024 * 1024 }));
+const latest = npmJson("view", `${PACKAGE}@latest`, "version", "dist.integrity", "--json");
+latest.integrity ??= latest["dist.integrity"];
+if (typeof latest.version !== "string" || typeof latest.integrity !== "string") {
+  throw new Error("npm latest did not resolve to an exact version and integrity");
+}
+
+rmSync(root, { recursive: true, force: true });
+mkdirSync(root, { recursive: true });
+const tarballDirectory = join(root, "tarballs");
+mkdirSync(tarballDirectory);
+const [packed] = npmJson(
+  "pack",
+  resolve("packages/codex-edit"),
+  "--pack-destination",
+  tarballDirectory,
+  "--json",
+);
+if (!packed?.filename || !packed?.integrity) throw new Error("Local npm pack returned no integrity");
+const localTarball = join(tarballDirectory, packed.filename);
+
+function installRuntime(name, extensionSpec) {
+  const directory = join(root, "runtimes", name);
+  mkdirSync(directory, { recursive: true });
+  writeFileSync(join(directory, "package.json"), `${JSON.stringify({ private: true })}\n`);
+  const dependencies = [`@earendil-works/pi-coding-agent@${PI_VERSION}`];
+  if (extensionSpec) dependencies.push(extensionSpec);
+  execFileSync(
+    "npm",
+    ["install", "--no-audit", "--no-fund", "--save-exact", ...dependencies],
+    { cwd: directory, stdio: "inherit" },
+  );
+  const piManifest = JSON.parse(
+    readFileSync(join(directory, "node_modules/@earendil-works/pi-coding-agent/package.json")),
+  );
+  if (piManifest.version !== PI_VERSION) throw new Error(`${name}: Pi version drifted`);
+  if (extensionSpec) {
+    const extensionManifest = JSON.parse(
+      readFileSync(join(directory, "node_modules/@maxiaochao/pi-codex-edit/package.json")),
+    );
+    if (!extensionManifest.pi?.extensions?.includes("./extensions/codex-edit.ts")) {
+      throw new Error(`${name}: installed package does not declare Codex Edit`);
+    }
+  }
+  return directory;
+}
+
+const nativeRuntime = installRuntime("native-edit");
+const releasedRuntime = installRuntime("released-codex-edit", `${PACKAGE}@${latest.version}`);
+const localRuntime = installRuntime("local-codex-edit", localTarball);
+const lock = JSON.parse(readFileSync(join(releasedRuntime, "package-lock.json")));
+const releasedLock = lock.packages?.["node_modules/@maxiaochao/pi-codex-edit"];
+if (releasedLock?.version !== latest.version || releasedLock?.integrity !== latest.integrity) {
+  throw new Error("Released Codex Edit lockfile does not match the npm version and integrity pin");
+}
+
+const sourceDigest = createHash("sha256")
+  .update(readFileSync(resolve("packages/codex-edit/src/parser.ts")))
+  .update(readFileSync(resolve("packages/codex-edit/extensions/codex-edit.ts")))
+  .digest("hex");
+const commonArgs = [
+  "--model",
+  MODEL,
+  "--thinking",
+  THINKING,
+  "--no-extensions",
+  "--no-skills",
+  "--no-prompt-templates",
+  "--no-themes",
+  "--no-context-files",
+  "--tools",
+  "read,bash,edit,write",
+  "--session-dir",
+  "/state/pi/sessions",
+  "--mode",
+  "json",
+  "-p",
+];
+function adapter(name, runtime, extension) {
+  const extensionEntry = extension
+    ? join(runtime, "node_modules/@maxiaochao/pi-codex-edit/extensions/codex-edit.ts")
+    : null;
+  const extensionIdentity =
+    name === "released-codex-edit"
+      ? `${PACKAGE}@${latest.version} (${latest.integrity})`
+      : name === "local-codex-edit"
+        ? `${PACKAGE}@${packed.version}+local (${packed.integrity})`
+        : null;
+  return {
+    kind: "pi-default",
+    command: join(runtime, "node_modules/.bin/pi"),
+    args: [
+      ...commonArgs,
+      ...(extensionEntry ? ["--extension", extensionEntry] : []),
+      "--",
+      "{prompt}",
+    ],
+    version: PI_VERSION,
+    model: MODEL,
+    thinking: THINKING,
+    transport: "harness-native",
+    ready: false,
+    readOnly: [nodeRuntime, join(runtime, "node_modules")],
+    seedFiles: { "pi/auth.json": authFile, "pi/models.json": modelFile },
+    env: {
+      PATH: `${dirname(nodeBinary)}:/usr/local/bin:/usr/bin:/bin`,
+      PI_CODING_AGENT_DIR: "/state/pi",
+      PI_TELEMETRY: "0",
+    },
+    agentFamily: "pi",
+    agentVersion: PI_VERSION,
+    modelFamily: "gpt-5.6-sol",
+    modelVersion: "gpt-5.6-sol",
+    provider: "openai-codex",
+    harnessFamily: name,
+    harnessVersion:
+      name === "native-edit"
+        ? PI_VERSION
+        : name === "released-codex-edit"
+          ? latest.version
+          : `${packed.version}+local.${sourceDigest.slice(0, 12)}`,
+    adapterVersion: "1",
+    configurationLabels: [`harness/${name}`, extension ? "editor/apply-patch" : "editor/edit"],
+    configurationId: `${name}/explicit-edit-v1`,
+    configuration: {
+      tools: extension ? ["read", "bash", "write", "apply_patch"] : ["read", "bash", "write", "edit"],
+      extensions: extensionIdentity ? [extensionIdentity] : [],
+      rules: extension ? ["Codex patch grammar", "native write retained"] : [],
+      runtimeFlags: [`thinking=${THINKING}`, "clean-pi-resources", "tool-parity-v1"],
+      environment: [],
+    },
+  };
+}
+
+const config = {
+  harnesses: {
+    "native-edit": adapter("native-edit", nativeRuntime, false),
+    "released-codex-edit": adapter("released-codex-edit", releasedRuntime, true),
+    "local-codex-edit": adapter("local-codex-edit", localRuntime, true),
+  },
+};
+const configPath = join(root, "config.json");
+writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 });
+const metadata = {
+  preparedAt: new Date().toISOString(),
+  piVersion: PI_VERSION,
+  model: MODEL,
+  thinking: THINKING,
+  released: { package: PACKAGE, version: latest.version, integrity: latest.integrity },
+  local: {
+    version: packed.version,
+    integrity: packed.integrity,
+    shasum: packed.shasum,
+    sourceDigest,
+  },
+  config: configPath,
+};
+writeFileSync(join(root, "metadata.json"), `${JSON.stringify(metadata, null, 2)}\n`);
+console.log(JSON.stringify(metadata, null, 2));