@maxiaochao/pi-toolkit 0.10.24 → 0.11.1

package/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 2026-10-12 - v0.11.1
+
+- bundle herdr-subagents v0.4.11: add architecture documentation and clear-output guidance evaluation
+
+## 2026-10-12 - v0.11.0
+
+- bundle worktree v0.4.0: Replace CLI start and prepare with the create-only command
+- Replace `pi-worktree start` and `pi-worktree prepare` with the create-only `pi-worktree create` CLI; `/worktree start` and `enter_worktree` continue to switch the current Pi session.
+
 ## 2026-10-12 - v0.10.24
 
 - bundle herdr-subagents v0.4.10: Stabilize concurrency checks and cancellation synchronization on slower CI runners.
@@ -5,6 +14,7 @@
 ## 2026-10-12 - v0.10.23
 
 - bundle herdr-subagents v0.4.9: Stabilize concurrency checks on slower CI runners.
+
 ## 2026-10-12 - v0.10.22
 
 - bundle herdr-subagents v0.4.8: make blocking behave identically in serial and parallel runs and let respond continue a named failed task from its saved session

package/evaluations/agents-md/README.md

@@ -0,0 +1,19 @@
+# AGENTS.md Clear Output Evaluation
+
+This A/B evaluation measures the effect of the `Clear Output` guidance. The `before` variant uses a minimal coding-assistant prompt. The `after` variant adds only the two `Clear Output` rules.
+
+Run the deterministic scorer tests:
+
+```bash
+npm run eval:agents:test
+```
+
+Run the model evaluation:
+
+```bash
+npm run eval:agents -- --runs 2
+```
+
+Set `PI_EVAL_MODEL` or pass `--model <provider/model>` to pin a model. Results are written to `results/latest.json` and `results/latest.md`.
+
+The scorer measures sentence length, conjunction density, response language, terminology changes, and use of a visual for the branching-flow case. These are proxies. Review the saved outputs for correctness and completeness before accepting a prompt change.

package/evaluations/agents-md/run.mjs

@@ -0,0 +1,217 @@
+import { execFileSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+const CLEAR_OUTPUT = `## Clear Output
+
+- Make outputs easy to understand and verify. Use ASD-STE100-inspired principles: short sentences, one idea per sentence, and consistent terms. Apply its formal language rules only to English.
+- When it improves understanding, use a diagram, HTML artifact, or video instead of long prose.`;
+
+const BASE_SYSTEM = "You are an expert coding assistant. Give a direct and accurate answer. Reply in the same language as the user.";
+
+export const variants = {
+  before: BASE_SYSTEM,
+  after: `${BASE_SYSTEM}\n\n${CLEAR_OUTPUT}`,
+};
+
+export const cases = [
+  {
+    id: "english-explanation",
+    language: "en",
+    prompt: "Explain how optimistic concurrency control prevents lost updates. Include the read, write, conflict, and retry stages. Write for an application developer.",
+  },
+  {
+    id: "chinese-explanation",
+    language: "zh",
+    prompt: "向应用开发者解释乐观并发控制如何防止更新丢失。需要涵盖读取、写入、冲突和重试阶段。",
+  },
+  {
+    id: "term-consistency",
+    language: "en",
+    aliases: ["work item", "job", "task"],
+    prompt: "A queue API uses job, task, and work item for the same object. Explain its lifecycle from creation through processing to completion. Resolve the naming ambiguity in your answer.",
+  },
+  {
+    id: "visual-flow",
+    language: "en",
+    expectVisual: true,
+    prompt: "Explain this request flow: gateway to authentication; failed authentication to rejection; successful authentication to cache; cache miss to database; transient database failure retries twice, then returns an error. Make the branching and retry path easy to verify.",
+  },
+];
+
+function splitSentences(text) {
+  return [...new Intl.Segmenter(undefined, { granularity: "sentence" }).segment(text)]
+    .map(({ segment }) => segment.trim())
+    .filter(Boolean);
+}
+
+function tokenCount(sentence, language) {
+  if (language === "zh") {
+    return (sentence.match(/[\p{Script=Han}]|[A-Za-z0-9]+/gu) ?? []).length;
+  }
+  return (sentence.match(/[\p{L}\p{N}]+(?:[-'][\p{L}\p{N}]+)*/gu) ?? []).length;
+}
+
+export function scoreOutput(text, testCase) {
+  const sentences = splitSentences(text);
+  const lengths = sentences.map((sentence) => tokenCount(sentence, testCase.language));
+  const limit = testCase.language === "zh" ? 35 : 25;
+  const conjunctionPattern = testCase.language === "zh"
+    ? /(?:而且|但是|同时|以及|并且|然而|其中)/g
+    : /\b(?:and|but|while|whereas|which|although|however)\b/gi;
+  const conjunctions = sentences.reduce(
+    (sum, sentence) => sum + (sentence.match(conjunctionPattern) ?? []).length,
+    0,
+  );
+  const firstParagraphEnd = text.indexOf("\n\n");
+  const bodyAfterNaming = firstParagraphEnd >= 0
+    ? text.slice(firstParagraphEnd + 2)
+    : sentences.slice(1).join(" ");
+  const usedAliasesAfterNaming = (testCase.aliases ?? []).filter((alias) =>
+    new RegExp(`\\b${alias.replaceAll(" ", "\\s+")}s?\\b`, "i").test(bodyAfterNaming),
+  );
+  const hasVisual = /```(?:mermaid)?|(?:-->|->|→|↓|├|└)|<svg\b|<html\b|\|[^\n]+\|[^\n]+\|/i.test(text);
+  const hanCharacters = (text.match(/[\p{Script=Han}]/gu) ?? []).length;
+  const latinWords = (text.match(/[A-Za-z]+/g) ?? []).length;
+  const languageMatches = testCase.language === "zh"
+    ? hanCharacters >= 20
+    : hanCharacters <= Math.max(5, latinWords * 0.05);
+
+  return {
+    characters: text.length,
+    sentences: sentences.length,
+    averageSentenceTokens: lengths.length
+      ? Number((lengths.reduce((sum, value) => sum + value, 0) / lengths.length).toFixed(1))
+      : 0,
+    longSentenceRate: lengths.length
+      ? Number((lengths.filter((value) => value > limit).length / lengths.length).toFixed(2))
+      : 0,
+    conjunctionsPerSentence: lengths.length
+      ? Number((conjunctions / lengths.length).toFixed(2))
+      : 0,
+    extraAliasesAfterNaming: Math.max(0, usedAliasesAfterNaming.length - 1),
+    languageMatches,
+    hasVisual,
+  };
+}
+
+function parseArguments(argv) {
+  const options = { runs: 1, model: process.env.PI_EVAL_MODEL };
+  for (let index = 0; index < argv.length; index += 1) {
+    if (argv[index] === "--runs") options.runs = Number(argv[++index]);
+    else if (argv[index] === "--model") options.model = argv[++index];
+    else throw new Error(`Unknown argument: ${argv[index]}`);
+  }
+  if (!Number.isInteger(options.runs) || options.runs < 1) {
+    throw new Error("--runs must be a positive integer");
+  }
+  return options;
+}
+
+function runPi(systemPrompt, prompt, model) {
+  const args = [
+    "--print",
+    "--no-session",
+    "--no-tools",
+    "--no-extensions",
+    "--no-skills",
+    "--no-prompt-templates",
+    "--no-themes",
+    "--no-context-files",
+    "--system-prompt",
+    systemPrompt,
+  ];
+  if (model) args.push("--model", model);
+  args.push("--", prompt);
+  return execFileSync("pi", args, { encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] }).trim();
+}
+
+function average(rows, key) {
+  if (rows.length === 0) return 0;
+  return Number((rows.reduce((sum, row) => sum + Number(row.score[key]), 0) / rows.length).toFixed(2));
+}
+
+export function summarize(results) {
+  return Object.fromEntries(Object.keys(variants).map((variant) => {
+    const rows = results.filter((result) => result.variant === variant);
+    const visualRows = rows.filter((result) => cases.find(({ id }) => id === result.caseId).expectVisual);
+    const terminologyRows = rows.filter((result) => cases.find(({ id }) => id === result.caseId).aliases);
+    return [variant, {
+      averageCharacters: average(rows, "characters"),
+      averageSentenceTokens: average(rows, "averageSentenceTokens"),
+      longSentenceRate: average(rows, "longSentenceRate"),
+      conjunctionsPerSentence: average(rows, "conjunctionsPerSentence"),
+      languageMatchRate: average(rows, "languageMatches"),
+      visualFlowRate: average(visualRows, "hasVisual"),
+      extraAliasesAfterNaming: average(terminologyRows, "extraAliasesAfterNaming"),
+    }];
+  }));
+}
+
+export function markdownReport(payload) {
+  const { summary } = payload;
+  const lines = [
+    "# AGENTS.md Clear Output A/B Evaluation",
+    "",
+    `Runs per case: ${payload.options.runs}`,
+    `Model override: ${payload.options.model ?? "Pi default"}`,
+    "",
+    "| Variant | Mean characters | Mean sentence tokens | Long sentence rate | Conjunctions / sentence | Language match | Visual-flow rate | Extra aliases |",
+    "|---|---:|---:|---:|---:|---:|---:|---:|",
+    ...Object.entries(summary).map(([name, score]) =>
+      `| ${name} | ${score.averageCharacters} | ${score.averageSentenceTokens} | ${score.longSentenceRate} | ${score.conjunctionsPerSentence} | ${score.languageMatchRate} | ${score.visualFlowRate} | ${score.extraAliasesAfterNaming} |`,
+    ),
+    "",
+    "Lower is better for long sentence rate, conjunctions, and extra aliases after naming. Higher is better for language match and visual-flow rate. Character count is descriptive, not a quality score.",
+    "",
+    "## Per-case output",
+    "",
+    ...payload.results.flatMap((result) => [
+      `### ${result.caseId} · ${result.variant} · run ${result.run}`,
+      "",
+      `Score: \`${JSON.stringify(result.score)}\``,
+      "",
+      result.output,
+      "",
+    ]),
+  ];
+  return `${lines.join("\n")}\n`;
+}
+
+async function main() {
+  const options = parseArguments(process.argv.slice(2));
+  const results = [];
+  for (let run = 1; run <= options.runs; run += 1) {
+    for (const testCase of cases) {
+      const order = run % 2 === 1 ? ["before", "after"] : ["after", "before"];
+      for (const variant of order) {
+        process.stderr.write(`Running ${testCase.id} / ${variant} / ${run}\n`);
+        const output = runPi(variants[variant], testCase.prompt, options.model);
+        results.push({
+          caseId: testCase.id,
+          variant,
+          run,
+          output,
+          score: scoreOutput(output, testCase),
+        });
+      }
+    }
+  }
+
+  const payload = {
+    generatedAt: new Date().toISOString(),
+    options,
+    summary: summarize(results),
+    results,
+  };
+  const resultDirectory = fileURLToPath(new URL("./results", import.meta.url));
+  mkdirSync(resultDirectory, { recursive: true });
+  writeFileSync(`${resultDirectory}/latest.json`, `${JSON.stringify(payload, null, 2)}\n`);
+  writeFileSync(`${resultDirectory}/latest.md`, markdownReport(payload));
+  console.log(JSON.stringify(payload.summary, null, 2));
+  console.log(`\nFull report: ${resultDirectory}/latest.md`);
+}
+
+if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) {
+  await main();
+}

package/evaluations/agents-md/tests/run-tests.mjs

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import { scoreOutput } from "../run.mjs";
+
+const concise = scoreOutput(
+  "Read the record. Compare its version. Reject a conflict. Retry the operation.",
+  { language: "en" },
+);
+assert.equal(concise.sentences, 4);
+assert.equal(concise.longSentenceRate, 0);
+assert.equal(concise.conjunctionsPerSentence, 0);
+assert.equal(concise.languageMatches, true);
+
+const dense = scoreOutput(
+  "Read the record and compare its version, but reject the write when another client changed it, while preserving the new value and returning a conflict which the caller can retry after loading the latest record.",
+  { language: "en" },
+);
+assert.equal(dense.sentences, 1);
+assert.equal(dense.longSentenceRate, 1);
+assert.ok(dense.conjunctionsPerSentence >= 4);
+
+const chinese = scoreOutput(
+  "客户端读取记录。客户端比较版本。版本冲突时，服务拒绝写入。客户端重新读取后重试。",
+  { language: "zh" },
+);
+assert.equal(chinese.sentences, 4);
+assert.equal(chinese.longSentenceRate, 0);
+assert.equal(chinese.languageMatches, true);
+
+const terms = scoreOutput(
+  "Create one work item. The worker processes the job and completes the task.",
+  { language: "en", aliases: ["work item", "job", "task"] },
+);
+assert.equal(terms.extraAliasesAfterNaming, 1);
+
+const canonicalTerm = scoreOutput(
+  "Use work item as the canonical term for job, task, and work item. Create the work item. Process the work item. Complete the work item.",
+  { language: "en", aliases: ["work item", "job", "task"] },
+);
+assert.equal(canonicalTerm.extraAliasesAfterNaming, 0);
+
+const wrongLanguage = scoreOutput("这是一个中文回答，包含足够多的中文字符，因此不应匹配英文问题。", {
+  language: "en",
+});
+assert.equal(wrongLanguage.languageMatches, false);
+
+const visual = scoreOutput("gateway --> auth --> cache\n                  ↓\n                reject", {
+  language: "en",
+  expectVisual: true,
+});
+assert.equal(visual.hasVisual, true);
+
+console.log("AGENTS.md evaluation scorer tests passed");

package/global/AGENTS.md

@@ -2,13 +2,18 @@
 
 Behavioral guidelines to reduce common LLM coding mistakes. Project-specific instructions override this file when they conflict.
 
-## 1. Think Before Coding
+## 1. Clear Output
+
+- Make outputs easy to understand and verify. Use ASD-STE100-inspired principles: short sentences, one idea per sentence, and consistent terms. Apply its formal language rules only to English.
+- When it improves understanding, use a diagram, HTML artifact, or video instead of long prose.
+
+## 2. Think Before Coding
 
 - Surface material assumptions, interpretations, and tradeoffs.
 - If unresolved ambiguity would materially change the result, ask. When no human is reachable, choose a defensible interpretation, state it, and proceed with verifiable criteria.
 - Prefer the simpler approach. Name conflicting requirements rather than silently compromising between them.
 
-## 2. Minimal Solution
+## 3. Minimal Solution
 
 Understand the problem and relevant code flow before choosing the first rung that works:
 
@@ -24,11 +29,11 @@ Understand the problem and relevant code flow before choosing the first rung tha
 - For states excluded by enforced invariants, fail visibly instead of inventing fallback behavior.
 - State the ceiling and upgrade trigger of any deliberate simplification that cuts a real corner.
 
-## 3. Skill Boundaries
+## 4. Skill Boundaries
 
 After loading a skill, treat its documented interface as sufficient for normal use. Do not inspect the skill's implementation, installed executable, or package source unless the user asks to review, debug, or modify it; its documented behavior conflicts with observed behavior; or required usage details are missing from the skill.
 
-## 4. Read, Then Change Surgically
+## 5. Read, Then Change Surgically
 
 Before editing, read the affected public API and its immediate callers. For a bug, find every statically discoverable caller and relevant implementation.
 
@@ -38,23 +43,23 @@ Every changed line traces to the user's request or to a defect that request reve
 - Remove imports, variables, and functions that your changes make unused. Leave pre-existing dead code in place and mention it.
 - Fix a faulty shared function once and name affected sibling callers; if only one caller is wrong, fix that caller instead of guarding a correct shared function.
 
-## 5. Goal-Driven Execution
+## 6. Goal-Driven Execution
 
 - Turn work into checkable outcomes: reproduce bugs before fixing them, preserve behavior during refactors, and optimize against a tested reference implementation.
 - For multi-step or materially ambiguous work, state a brief plan and give every step its own verification.
 - Leave the smallest relevant runnable check for new or changed non-trivial logic. Do not introduce a test framework solely for that check.
 
-## 6. Fail Visibly
+## 7. Fail Visibly
 
 - "Completed" is wrong if anything required was silently skipped.
 - "Tests pass" is wrong if any required test was skipped or not run.
 - If a required input, file, or tool is missing, or a step is impossible under the stated constraints, report what is missing and what you need. Never fabricate it to make a check pass.
 
-## 7. Frontend Verification
+## 8. Frontend Verification
 
 For frontend feature development, load the `web-browser` skill and verify the result in a real Chrome browser through CDP. Exercise the relevant user flow with real clicks, capture screenshots of the resulting UI, and test the actual behavior before reporting completion. Static inspection or non-browser tests do not replace this verification.
 
-## 8. Worktree Isolation
+## 9. Worktree Isolation
 
 For a Git-repository task that coordinates substantial changes across multiple files or modules, or implements a feature intended for release, ask before editing whether to isolate it in a worktree. Skip the question when already in a linked worktree.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@maxiaochao/pi-toolkit",
-  "version": "0.10.24",
+  "version": "0.11.1",
   "description": "Personal pi coding agent toolkit: scheduler, /btw, /cache_export, apply_patch, browser automation, and bundled theme/skills.",
   "type": "module",
   "workspaces": [
@@ -12,6 +12,7 @@
   ],
   "files": [
     "extensions",
+    "evaluations/agents-md",
     "packages/cache-export",
     "packages/codex-edit",
     "packages/herdr-subagents",