@event4u/agent-config 9.35.0 → 10.0.0

package/src/scripts/probe_promissory_closing.ts

@@ -0,0 +1,180 @@
+#!/usr/bin/env tsx
+/**
+ * Round 7 § 6.3 — the promissory-closing rate, as a script instead of an ad-hoc probe.
+ *
+ * `verify-before-complete` § Turn-completion forbids ending a turn on a promise of
+ * unexecuted work ("I'll…", "als nächstes mache ich…"). Round 7 measured the rate
+ * to decide whether to build a mechanism, and the answer was: don't — a blocking
+ * `detectPromissory` ALREADY ships in `turn-end-gate`.
+ *
+ * That inverts the reading of a low number, and the inversion is the reason this
+ * script exists. Round 7's first draft declined the class as "below any threshold
+ * worth a gate"; with a gate already in place, 0.6 % is plausibly the mechanism
+ * WORKING, not the class being rare. A rate over a gated class measures residual
+ * leakage. Read it that way or not at all.
+ *
+ * WHAT IS COUNTED: only a HAND-BACK turn — an assistant turn immediately followed
+ * by a genuine user turn, i.e. where the agent actually stopped. A promise in a
+ * mid-turn message is not a violation; the agent kept going. A turn that ASKS is
+ * also excluded: a question is a legitimate stop condition.
+ *
+ * HONEST BOUND: the predicate is deliberately narrow (first-person + future,
+ * anchored in the closing lines), so the figure is a FLOOR. A looser predicate
+ * over the same corpus read 1.2 % against this one's 0.6 %; both are printed so a
+ * reader can see the width of the bracket rather than trust one edge of it.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/probe_promissory_closing [--limit N] [--store PATH]
+ *
+ * Exit codes: 0 always — a measurement, not a gate.
+ */
+import fs from "node:fs";
+import path from "node:path";
+
+import { defaultStore, HARNESS_TEXT, isInjectedBody } from "./conformance_scan.js";
+import { isSyntheticPrompt } from "./_lib/prompt_shape.js";
+
+/** First person + intent, at the start of a closing sentence. Narrow on purpose. */
+export const PROMISE_NARROW: readonly RegExp[] = [
+  /\bich (mache|baue|schreibe|starte|prüfe|räume|setze|lege|fixe|committe|pushe|nehme|ziehe|hole|führe|erstelle|implementiere)\b[^.?!]{0,80}\b(jetzt|als nächstes|gleich|danach|anschließend|im nächsten schritt)\b/i,
+  /\b(als nächstes|im nächsten schritt|danach) (mache|baue|schreibe|starte|prüfe|nehme|committe|pushe|erstelle|implementiere) ich\b/i,
+  /\bi'?(ll| will) (now |next |then )?(start|build|write|check|run|fix|commit|push|create|implement|add|wire|measure)\b/i,
+  /\bnext,? i'?(ll| will)\b/i,
+  /\bi'?m going to (start|build|write|check|run|fix|commit|push|create|implement)\b/i,
+];
+
+/** The bracket's other edge — any future-intent marker at all. */
+export const PROMISE_LOOSE =
+  /\b(ich werde|ich mache|ich baue|ich starte|ich prüfe|als nächstes|next up|i'll |i will |i'm going to|nun mache ich|jetzt mache ich)/i;
+
+/** A reply that asks is a legitimate stop condition, never a promissory closing. */
+const ASKS = /(\?\s*$)|(^\s*\d\.\s)/m;
+
+interface Turn {
+  role: "user" | "assistant";
+  text: string;
+  at: string;
+}
+
+function turns(lines: string[]): Turn[] {
+  const out: Turn[] = [];
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    let e: Record<string, unknown>;
+    try {
+      e = JSON.parse(line) as Record<string, unknown>;
+    } catch {
+      continue;
+    }
+    if (e["isSidechain"] === true) continue;
+    if (e["type"] === "user" && e["toolUseResult"] !== undefined) continue;
+    if (e["isCompactSummary"] === true) continue;
+    const msg = e["message"] as { role?: string; content?: unknown } | undefined;
+    const role = msg?.role;
+    if (role !== "user" && role !== "assistant") continue;
+    const c = msg?.content;
+    const text =
+      typeof c === "string"
+        ? c
+        : Array.isArray(c)
+          ? c
+              .filter((b: { type?: string }) => b?.type === "text")
+              .map((b: { text?: string }) => b.text ?? "")
+              .join("\n")
+          : "";
+    if (!text.trim()) continue;
+    if (role === "assistant" && HARNESS_TEXT.test(text.trim())) continue;
+    // A skill / slash-command body arrives in the user role and is not a chat
+    // message, so it must not close a hand-back.
+    if (role === "user" && (isSyntheticPrompt(text) || isInjectedBody(text))) continue;
+    out.push({ role, text, at: String(e["timestamp"] ?? "") });
+  }
+  return out;
+}
+
+export interface PromissoryResult {
+  /** Hand-back turns that could carry a promissory closing — asking ones excluded. */
+  handbacks: number;
+  /** Asking hand-backs, excluded from the population. R2 finding 6. */
+  asked: number;
+  narrow: { session: string; at: string; span: string }[];
+  loose: number;
+}
+
+export function measure(store: string, limit: number): PromissoryResult {
+  const result: PromissoryResult = { handbacks: 0, asked: 0, narrow: [], loose: 0 };
+  const files = fs
+    .readdirSync(store)
+    .filter((f) => f.endsWith(".jsonl"))
+    .map((f) => ({ f, m: fs.statSync(path.join(store, f)).mtimeMs }))
+    .sort((a, b) => b.m - a.m)
+    .slice(0, limit);
+
+  for (const { f } of files) {
+    const session = f.replace(/\.jsonl$/, "").slice(0, 8);
+    const ts = turns(fs.readFileSync(path.join(store, f), "utf8").split("\n"));
+    for (let i = 0; i < ts.length; i += 1) {
+      const t = ts[i] as Turn;
+      if (t.role !== "assistant") continue;
+      const next = ts[i + 1];
+      // A hand-back: the next turn is the user's, or the session ends here.
+      if (next !== undefined && next.role !== "user") continue;
+      const tail = t.text.trimEnd().split("\n").slice(-4).join("\n");
+      // R2 finding 6: `handbacks` was incremented BEFORE this guard, so every
+      // asking hand-back sat in the denominator while being excluded from both
+      // numerators — understating both rates by the ask share. An asking closing
+      // is a legitimate stop condition and cannot be a promissory closing, so it
+      // is not in the population at all. Counted separately, because a reader who
+      // cannot see the ask share cannot judge the denominator.
+      if (ASKS.test(tail)) {
+        result.asked += 1;
+        continue;
+      }
+      result.handbacks += 1;
+      if (PROMISE_LOOSE.test(tail)) result.loose += 1;
+      const hit = PROMISE_NARROW.find((re) => re.test(tail));
+      if (hit) {
+        result.narrow.push({ session, at: t.at, span: (tail.match(hit)?.[0] ?? "").slice(0, 110) });
+      }
+    }
+  }
+  return result;
+}
+
+export function main(argv?: string[]): number {
+  const args = argv ?? process.argv.slice(2);
+  const arg = (flag: string, dflt: string): string => {
+    const i = args.indexOf(flag);
+    return i >= 0 && args[i + 1] !== undefined ? (args[i + 1] as string) : dflt;
+  };
+  const store = arg("--store", defaultStore(process.cwd()));
+  const limit = Number(arg("--limit", "30"));
+
+  if (!fs.existsSync(store)) {
+    process.stdout.write(`probe:promissory-closing · store not found: ${store}\n`);
+    return 0;
+  }
+  const r = measure(store, limit);
+  process.stdout.write(
+    `probe:promissory-closing · ${r.handbacks} eligible hand-back turn(s) ` +
+      `(+${r.asked} that ASK, excluded — a question is a legitimate stop condition) in ${store}\n`,
+  );
+  for (const h of r.narrow) {
+    process.stdout.write(`  ${h.session} ${h.at.slice(0, 16)}  ${JSON.stringify(h.span)}\n`);
+  }
+  const pct = (n: number): string => (r.handbacks === 0 ? "—" : `${((100 * n) / r.handbacks).toFixed(1)}%`);
+  process.stdout.write(
+    `  narrow predicate: ${r.narrow.length} (${pct(r.narrow.length)}) · ` +
+      `loose predicate: ${r.loose} (${pct(r.loose)})\n`,
+  );
+  process.stdout.write(
+    "  The class is ALREADY GATED — `turn-end-gate` ships a blocking `detectPromissory`.\n" +
+      "  So a low figure here is residual leakage past a working mechanism, not evidence\n" +
+      "  the class is rare. The narrow figure is a floor; the loose one is the other edge.\n",
+  );
+  return 0;
+}
+
+if (process.argv[1] !== undefined && import.meta.url.endsWith(path.basename(process.argv[1]))) {
+  process.exit(main());
+}

package/src/scripts/probe_session_canary.ts

@@ -0,0 +1,180 @@
+#!/usr/bin/env tsx
+/**
+ * Round 7 § 6.3 — the opening-canary rate, as a script instead of an ad-hoc probe.
+ *
+ * `session-canary` obliges the agent to address the user by name in the first
+ * reply of a session AND of each new task within it. The rule's own text records
+ * two audits reading it as broadly missed (~13/15, then 24/29 task starts) and
+ * concludes the reminder carrier does not move it.
+ *
+ * This measures the half that is DECIDABLE from a transcript: the per-SESSION
+ * instance. The per-TASK instance is not measurable at all — no task boundary is
+ * recorded anywhere in a transcript — which is why the mechanism the rule
+ * pre-registers ("a check at delivery that rejects a task-start reply carrying no
+ * greeting") cannot be built as specified. Round 7 states that as an honest
+ * downgrade rather than an unbuilt promise.
+ *
+ * Two instrument defects this probe was born with, fixed and recorded:
+ *   1. Its first version omitted `You've hit your` from the harness predicate, so
+ *      a spend-limit banner counted as a missed greeting: 24/28 instead of 25/28.
+ *      The predicate is now the same one `conformance_scan` uses.
+ *   2. It reads the first assistant turn carrying PROSE, not the first assistant
+ *      entry: a session opening with 30 tool calls has no prose to greet in.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/probe_session_canary [--limit N] [--store PATH] [--name NAME]
+ *
+ * Exit codes: 0 always — this is a measurement, not a gate. A rate is not a
+ * verdict, and the round-7 reading (96 % post-carrier) is not a number anyone
+ * should gate on before it has a second window to compare against.
+ */
+import fs from "node:fs";
+import path from "node:path";
+
+import { load_agent_settings } from "./_lib/agent_settings.js";
+import { defaultStore, HARNESS_TEXT } from "./conformance_scan.js";
+
+/** The first assistant turn that carries prose, or null. */
+function firstProse(lines: string[]): { text: string; at: string } | null {
+  for (const line of lines) {
+    if (!line.trim()) continue;
+    let e: Record<string, unknown>;
+    try {
+      e = JSON.parse(line) as Record<string, unknown>;
+    } catch {
+      continue;
+    }
+    if (e["isSidechain"] === true) continue;
+    const msg = e["message"] as { role?: string; content?: unknown } | undefined;
+    if (msg?.role !== "assistant") continue;
+    const c = msg.content;
+    if (!Array.isArray(c)) continue;
+    const text = c
+      .filter((b: { type?: string }) => b?.type === "text")
+      .map((b: { text?: string }) => b.text ?? "")
+      .join("\n")
+      .trim();
+    if (!text) continue;
+    // A harness banner occupies an assistant turn without being the assistant
+    // writing. Same predicate as the scan's, imported rather than re-typed.
+    if (HARNESS_TEXT.test(text)) continue;
+    return { text, at: String(e["timestamp"] ?? "") };
+  }
+  return null;
+}
+
+/**
+ * The name the obligation actually uses, read through the same three layers the
+ * rule names: project `personal.canary_name` → user-global `personal.canary_name`
+ * → user-global `identity.name`. Empty string when no layer carries one, which is
+ * exactly when the rule is inert.
+ *
+ * Deliberately tolerant of every read failure: a probe that throws because a
+ * settings file is malformed measures the file, not the behaviour.
+ */
+export function resolveCanaryName(): string {
+  try {
+    const s = load_agent_settings({ cwd: process.cwd() }) as Record<string, unknown>;
+    const personal = s["personal"];
+    if (typeof personal === "object" && personal !== null && !Array.isArray(personal)) {
+      const n = (personal as Record<string, unknown>)["canary_name"];
+      if (typeof n === "string" && n.trim() !== "") return n.trim();
+    }
+    const identity = s["identity"];
+    if (typeof identity === "object" && identity !== null && !Array.isArray(identity)) {
+      const n = (identity as Record<string, unknown>)["name"];
+      if (typeof n === "string" && n.trim() !== "") return n.trim();
+    }
+  } catch {
+    // no settings, unreadable settings, malformed settings — all mean "no name".
+  }
+  return "";
+}
+
+export interface CanaryRow {
+  session: string;
+  at: string;
+  greeted: boolean;
+  opener: string;
+}
+
+export function measure(store: string, limit: number, name: string): CanaryRow[] {
+  const re = new RegExp(`\\b${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`, "i");
+  return fs
+    .readdirSync(store)
+    .filter((f) => f.endsWith(".jsonl"))
+    .map((f) => ({ f, m: fs.statSync(path.join(store, f)).mtimeMs }))
+    .sort((a, b) => b.m - a.m)
+    .slice(0, limit)
+    .flatMap(({ f }) => {
+      const lines = fs.readFileSync(path.join(store, f), "utf8").split("\n");
+      const first = firstProse(lines);
+      if (first === null) return [];
+      // The name has to be in the OPENING, not anywhere in a long first reply —
+      // a greeting is a greeting only where the reader looks for it.
+      const head = first.text.split("\n").slice(0, 3).join(" ");
+      return [
+        {
+          session: f.replace(/\.jsonl$/, "").slice(0, 8),
+          at: first.at,
+          greeted: re.test(head),
+          opener: first.text.split("\n")[0]?.slice(0, 70).replace(/\s+/g, " ") ?? "",
+        },
+      ];
+    });
+}
+
+export function main(argv?: string[]): number {
+  const args = argv ?? process.argv.slice(2);
+  const arg = (flag: string, dflt: string): string => {
+    const i = args.indexOf(flag);
+    return i >= 0 && args[i + 1] !== undefined ? (args[i + 1] as string) : dflt;
+  };
+  const store = arg("--store", defaultStore(process.cwd()));
+  const limit = Number(arg("--limit", "30"));
+  // R2 finding 7: the default used to be a maintainer's own nickname, hardcoded
+  // in `src/` — which breaks the portability Iron Law and made the figure
+  // irreproducible for anyone else (a name that never appears greps to a silent
+  // 0 %). The rule this probe measures resolves the name through the settings
+  // cascade, so the probe resolves it the same way and REFUSES rather than
+  // guessing when no layer carries one.
+  const name = arg("--name", resolveCanaryName());
+
+  if (name === "") {
+    process.stdout.write(
+      "probe:session-canary · no canary name configured — nothing to measure.\n" +
+        "  The obligation itself is inert without one (`session-canary`: no name on any\n" +
+        "  layer ⇒ the rule does not fire), so a rate here would be a rate over an\n" +
+        "  obligation that does not exist. Pass --name <name> to measure anyway.\n",
+    );
+    return 0;
+  }
+  if (!fs.existsSync(store)) {
+    process.stdout.write(`probe:session-canary · store not found: ${store}\n`);
+    return 0;
+  }
+  const rows = measure(store, limit, name);
+  if (rows.length === 0) {
+    process.stdout.write(`probe:session-canary · 0 session(s) with assistant prose in ${store}\n`);
+    return 0;
+  }
+  const hit = rows.filter((r) => r.greeted).length;
+  process.stdout.write(`probe:session-canary · ${rows.length} session(s) · name=${name}\n`);
+  for (const r of rows.filter((r) => !r.greeted).sort((a, b) => a.at.localeCompare(b.at))) {
+    process.stdout.write(`  MISS ${r.session} ${r.at.slice(0, 16)}  "${r.opener}"\n`);
+  }
+  process.stdout.write(
+    `  opening canary present in ${hit}/${rows.length} sessions ` +
+      `(${((100 * hit) / rows.length).toFixed(1)}%)\n`,
+  );
+  process.stdout.write(
+    "  Per-SESSION only. The per-TASK instance the rule also obliges is NOT decidable\n" +
+      "  from a transcript — no task boundary is recorded — so this figure is a floor on\n" +
+      "  compliance, never a verdict on the obligation as a whole.\n",
+  );
+  return 0;
+}
+
+if (process.argv[1] !== undefined && import.meta.url.endsWith(path.basename(process.argv[1]))) {
+  process.exit(main());
+}