@event4u/agent-config 8.8.0 → 8.10.0

package/src/scripts/_lib/orchestration_savings.ts

@@ -5,16 +5,28 @@
  * Consumes the `orchestration` sub-object of audit-log-v1 JSONL lines
  * (contract: `src/agent-src/contexts/execution/orchestration-telemetry.md`).
  *
- * HONEST LIMIT — the telemetry records `token_delta` (net dispatch cost vs the
- * in-session baseline; negative = saved) but NOT the absolute baseline. A clean
- * "% of session saved" is therefore NOT derivable from this data alone — only
- * ABSOLUTE net tokens saved, plus the measured-vs-estimated provenance split.
- * A percentage would need an absolute-baseline field on the telemetry (see the
- * report `notes` and the PR's follow-up section).
+ * TWO metrics, kept distinct on purpose:
+ * 1. MEASURED token counts — `token_delta` (net token-COUNT delta vs the
+ *    in-session baseline; negative = fewer tokens). Real, but blind to the
+ *    downshift RATE win: the same work on a cheaper tier uses ~the same token
+ *    COUNT, so token_delta ≈ 0 for a pure downshift even though cost dropped.
+ * 2. MODELED cost-% (`modeled_cost`) — captures exactly that downshift win via
+ *    provider-neutral tier weights + absolute `dispatch_tokens`. It is a MODEL
+ *    (weights + a same-token-count assumption), NOT a measured $ figure, and is
+ *    clearly labelled as such in the report notes.
  */
 
 export type TokenDeltaProvenance = 'measured' | 'estimated';
 
+/**
+ * Relative, provider-NEUTRAL cost weights per model tier — a tunable ratio, not
+ * $ prices. Default reflects the ~5×/~15× cheap-vs-frontier ratios from the
+ * cost-downshift council; override per project via the report's `--weights`.
+ * Used ONLY for the modeled cost-%, never for the measured token counts.
+ */
+export type TierWeights = Record<string, number>;
+export const DEFAULT_TIER_WEIGHTS: TierWeights = { lite: 1, medium: 5, high: 15 };
+
 /** The `orchestration` sub-object shape we read (subset — additive fields only). */
 export interface OrchestrationRecord {
     spawn_count?: number;
@@ -23,6 +35,14 @@ export interface OrchestrationRecord {
     tier_chosen?: string | null;
     task_class?: string | null;
     outcome?: string;
+    /** Absolute measured tokens the dispatched slice consumed (for the modeled cost-%). */
+    dispatch_tokens?: number | null;
+    /** The orchestrator's own tier — the baseline the downshift is measured against. */
+    session_tier?: string | null;
+    /** QUALITY: subagent return adopted without parent rework. null/absent = pre-extension line / not measured. */
+    first_pass_success?: boolean | null;
+    /** QUALITY: slice retried on a higher tier after a verification failure. null/absent = pre-extension line / not measured. */
+    escalated?: boolean | null;
 }
 
 /** A parsed audit-log-v1 line (only the fields this aggregator touches). */
@@ -37,6 +57,33 @@ export interface ProvenanceStat {
     net_token_delta: number;
 }
 
+export interface ModeledCost {
+    /** Dispatches with dispatch_tokens + session_tier + tier_chosen all weight-resolvable. */
+    covered_dispatches: number;
+    /** Σ dispatch_tokens × weight(session_tier) — same tokens costed on the baseline tier. */
+    baseline_cost_units: number;
+    /** Σ dispatch_tokens × weight(tier_chosen) — modeled cost of the downshifted run. */
+    delegated_cost_units: number;
+    /** (baseline − delegated) / baseline, 0..1; null when no covered dispatches. MODELED, not measured $. */
+    cost_reduction_pct: number | null;
+    weights: TierWeights;
+}
+
+export interface QualityStats {
+    /** Dispatches carrying at least one of the two quality booleans (the n behind the ≥20 pairing gate). */
+    quality_lines: number;
+    /** Dispatches carrying a boolean first_pass_success. */
+    first_pass_lines: number;
+    first_pass_success_count: number;
+    /** first_pass_success_count / first_pass_lines, 0..1; null when no line carries the field. */
+    first_pass_success_rate: number | null;
+    /** Dispatches carrying a boolean escalated. */
+    escalated_lines: number;
+    escalated_count: number;
+    /** escalated_count / escalated_lines, 0..1; null when no line carries the field. */
+    escalation_rate: number | null;
+}
+
 export interface SavingsReport {
     /** Lines carrying an orchestration object with spawn_count > 0. */
     dispatches: number;
@@ -54,6 +101,10 @@ export interface SavingsReport {
     by_task_class: Record<string, number>;
     /** measured dispatches / dispatches, 0..1 (0 when no dispatches). */
     measured_share: number;
+    /** Modeled downshift cost-% (tier-weight based). Distinct from the measured token counts. */
+    modeled_cost: ModeledCost;
+    /** QUALITY dimension — paired with cost per council verdict; never report savings alone. */
+    quality: QualityStats;
     /** Honest caveats about what the number does and does not mean. */
     notes: string[];
 }
@@ -70,7 +121,10 @@ function emptyProvenance(): ProvenanceStat {
  * dispatch). Unknown/absent fields default safely (token_delta → 0, provenance
  * → estimated) so a pre-extension line never throws.
  */
-export function aggregateOrchestrationSavings(lines: AuditLine[]): SavingsReport {
+export function aggregateOrchestrationSavings(
+    lines: AuditLine[],
+    weights: TierWeights = DEFAULT_TIER_WEIGHTS,
+): SavingsReport {
     const report: SavingsReport = {
         dispatches: 0,
         total_spawns: 0,
@@ -81,6 +135,16 @@ export function aggregateOrchestrationSavings(lines: AuditLine[]): SavingsReport
         by_tier: {},
         by_task_class: {},
         measured_share: 0,
+        modeled_cost: { covered_dispatches: 0, baseline_cost_units: 0, delegated_cost_units: 0, cost_reduction_pct: null, weights },
+        quality: {
+            quality_lines: 0,
+            first_pass_lines: 0,
+            first_pass_success_count: 0,
+            first_pass_success_rate: null,
+            escalated_lines: 0,
+            escalated_count: 0,
+            escalation_rate: null,
+        },
         notes: [],
     };
 
@@ -109,18 +173,59 @@ export function aggregateOrchestrationSavings(lines: AuditLine[]): SavingsReport
 
         const cls = o.task_class ?? 'unclassified';
         report.by_task_class[cls] = (report.by_task_class[cls] ?? 0) + delta;
+
+        // Modeled cost — the downshift RATE win token_delta (counts) cannot see:
+        // the same dispatch_tokens on a cheaper tier cost less by the weight ratio.
+        const dt = o.dispatch_tokens ?? 0;
+        const wChosen = o.tier_chosen != null ? weights[o.tier_chosen] : undefined;
+        const wSession = o.session_tier != null ? weights[o.session_tier] : undefined;
+        if (dt > 0 && wChosen !== undefined && wSession !== undefined) {
+            report.modeled_cost.covered_dispatches += 1;
+            report.modeled_cost.baseline_cost_units += dt * wSession;
+            report.modeled_cost.delegated_cost_units += dt * wChosen;
+        }
+
+        // Quality dimension — only booleans count; null/absent = pre-extension
+        // line / not measured (never a silent false).
+        const q = report.quality;
+        const fps = typeof o.first_pass_success === 'boolean';
+        const esc = typeof o.escalated === 'boolean';
+        if (fps || esc) q.quality_lines += 1;
+        if (fps) {
+            q.first_pass_lines += 1;
+            if (o.first_pass_success === true) q.first_pass_success_count += 1;
+        }
+        if (esc) {
+            q.escalated_lines += 1;
+            if (o.escalated === true) q.escalated_count += 1;
+        }
     }
 
     report.measured_share = report.dispatches === 0 ? 0 : report.by_provenance.measured.dispatches / report.dispatches;
 
+    const mc = report.modeled_cost;
+    mc.cost_reduction_pct = mc.baseline_cost_units > 0 ? (mc.baseline_cost_units - mc.delegated_cost_units) / mc.baseline_cost_units : null;
+
+    const q = report.quality;
+    q.first_pass_success_rate = q.first_pass_lines > 0 ? q.first_pass_success_count / q.first_pass_lines : null;
+    q.escalation_rate = q.escalated_lines > 0 ? q.escalated_count / q.escalated_lines : null;
+
     // Honest caveats — always attached so a consumer never over-reads the number.
     if (report.dispatches === 0) {
-        report.notes.push('No orchestration telemetry yet (0 dispatches). The number accrues as `subagents.auto: on` runs delegate real slices.');
+        report.notes.push('No orchestration telemetry yet (0 dispatches). Data accrues as `subagents.auto: on` delegates real slices (recorded via orchestration_record).');
     } else {
-        report.notes.push('Reports ABSOLUTE net token_delta (negative = saved). A "% of session saved" is NOT derivable — the telemetry records net delta, not the absolute baseline. A percentage needs an absolute-baseline field on the telemetry.');
+        report.notes.push('Token counts are MEASURED but blind to the downshift rate win (same tokens on a cheaper tier ≈ same count). The MODELED cost-% below captures that axis.');
         if (report.by_provenance.estimated.dispatches > 0) {
             report.notes.push(`${report.by_provenance.estimated.dispatches}/${report.dispatches} dispatch(es) use ESTIMATED token_delta (chars/4, lossy). Prefer measured (host usage metadata).`);
         }
+        if (mc.covered_dispatches > 0 && mc.cost_reduction_pct !== null) {
+            report.notes.push(`MODELED cost reduction ${(mc.cost_reduction_pct * 100).toFixed(0)}% over ${mc.covered_dispatches}/${report.dispatches} dispatch(es) that carry tier data — from provider-neutral tier weights ${JSON.stringify(mc.weights)} + a same-token-count assumption. A MODEL, NOT a measured $ figure; tune with --weights.`);
+        } else {
+            report.notes.push('MODELED cost-% unavailable: no dispatch yet carries dispatch_tokens + session_tier + tier_chosen. Record those (orchestration_record --dispatch-tokens/--session-tier/--tier-chosen) to populate it.');
+        }
+        if (q.quality_lines === 0) {
+            report.notes.push('QUALITY columns empty: no dispatch yet carries first_pass_success / escalated. Record them (orchestration_record --first-pass-success/--escalated) — cost and quality are reported as a PAIR, never savings alone.');
+        }
     }
 
     return report;

package/src/scripts/_lib/retrieval_sanitize.ts

@@ -0,0 +1,71 @@
+/**
+ * Sanitize floor for retrieval read-surfaces
+ * (road-to-retrieval-substrate-hardening B6).
+ *
+ * Corpus content — especially user-ingested knowledge chunks under
+ * `agents/memory/knowledge/` — flows into the agent context via `retrieve_v1`
+ * / `memory_get_v1`. An attacker (or a compromised ingested source) can embed
+ * hidden-instruction vectors (bidi controls, zero-width chars, Unicode Tag
+ * block) or control-char noise that the model may act on — the untrusted-input
+ * / lethal-trifecta injection surface. This floor strips those vectors from
+ * every string field before it is emitted, treating retrieved content as DATA.
+ *
+ * It deliberately does NOT rewrite visible text (that would corrupt legitimate
+ * rule/chunk bodies and code snippets) — the injection risk is the *invisible*
+ * layer + unbounded length; visible content is the agent's to read as data.
+ * Codepoint classes are shared with `lint_hidden_unicode` (one source of truth
+ * for what counts as a hidden-instruction vector).
+ */
+import { _classify } from '../lint_hidden_unicode.js';
+
+/** Hard per-field length cap — bounds a runaway/adversarial body. */
+export const MAX_FIELD_CHARS = 8192;
+
+/** True for a C0/C1 control char that is NOT a benign whitespace (\t, \n). */
+function _isStrippableControl(cp: number): boolean {
+    if (cp === 0x09 || cp === 0x0a) return false; // keep tab + newline
+    if (cp <= 0x1f) return true; // C0 controls (incl. CR 0x0d — normalised out)
+    if (cp === 0x7f) return true; // DEL
+    if (cp >= 0x80 && cp <= 0x9f) return true; // C1 controls
+    return false;
+}
+
+/**
+ * Strip hidden-instruction vectors (bidi / zero-width / Unicode-tag /
+ * deprecated-format, per `_classify`) and stray control chars from a string,
+ * then cap its length. Preserves all visible content + tabs/newlines.
+ */
+export function sanitize_text(s: string): string {
+    let out = '';
+    for (const ch of s) {
+        const cp = ch.codePointAt(0);
+        if (cp === undefined) continue;
+        if (_classify(cp) !== null) continue; // hidden-instruction vector → drop
+        if (_isStrippableControl(cp)) continue; // control-char noise → drop
+        out += ch;
+    }
+    return out.length > MAX_FIELD_CHARS ? out.slice(0, MAX_FIELD_CHARS) : out;
+}
+
+/**
+ * Return a shallow copy of a retrieval entry with every string value (and
+ * one level of string-array values) sanitized. Non-string values pass through
+ * unchanged (numbers, booleans, the PyFloat marker, nested plain objects are
+ * sanitized recursively).
+ */
+export function sanitize_entry(entry: Record<string, unknown>): Record<string, unknown> {
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(entry)) {
+        out[k] = _sanitize_value(v);
+    }
+    return out;
+}
+
+function _sanitize_value(v: unknown): unknown {
+    if (typeof v === 'string') return sanitize_text(v);
+    if (Array.isArray(v)) return v.map((x) => (typeof x === 'string' ? sanitize_text(x) : _sanitize_value(x)));
+    if (v && typeof v === 'object' && v.constructor === Object) {
+        return sanitize_entry(v as Record<string, unknown>);
+    }
+    return v; // numbers, booleans, null, marker classes (PyFloat/PyTimestamp) untouched
+}

package/src/scripts/_lib/security_lint.ts

@@ -52,6 +52,7 @@ export const DEFAULT_SCAN_ROOTS: readonly string[] = [
     'src/rules',
     'src/agent-src',
     'src/domains',
+    'dist/agent-src',
 ];
 
 // A path is "example/teaching" (0.25x weight) when it lives under docs/ or

package/src/scripts/_lib/stat_index.ts

@@ -0,0 +1,98 @@
+/**
+ * Stat-index — skip an expensive scan when the input tree is unchanged
+ * (road-to-retrieval-substrate-hardening B5a).
+ *
+ * The memory / knowledge / discovery scanners re-read their whole file set on
+ * every `stop` / `session_start` hook, even when nothing changed. A stat-index
+ * captures a cheap signature of the input files (size + mtime_ns) and caches
+ * the scan's result beside it; a later call with the same signature returns the
+ * cached payload without re-scanning. The cache is version-namespaced (so a
+ * tool bump invalidates cleanly, versioned-cache lint B5b) and written
+ * atomically (pid-temp + rename) so concurrent hook invocations stay
+ * idempotent (council Q3).
+ *
+ * `size + mtime_ns` is sufficient for gating rebuild decisions (the council
+ * withdrew the crypto-binding demand as a non-problem); a `--force` bypass and
+ * an explicit `clear()` cover the rare stale-mtime case (e.g. `cp -p`).
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+
+/** A deterministic signature over a file set: `<count>:<Σsize>:<maxMtimeNs>:<hashOfPaths>`. */
+export function statSignature(files: readonly string[]): string {
+    let count = 0;
+    let totalSize = 0;
+    let maxMtimeNs = 0n;
+    const sortedPaths = [...files].sort();
+    for (const f of sortedPaths) {
+        let st: fs.Stats;
+        try {
+            st = fs.statSync(f, { bigint: true }) as unknown as fs.Stats;
+        } catch {
+            continue; // missing file → excluded from the signature
+        }
+        count += 1;
+        totalSize += Number((st as unknown as { size: bigint }).size);
+        const m = (st as unknown as { mtimeNs: bigint }).mtimeNs;
+        if (m > maxMtimeNs) maxMtimeNs = m;
+    }
+    // A stable hash of the path list so add/remove/rename shifts the signature
+    // even when count + size + mtime coincide.
+    let h = 5381;
+    for (const p of sortedPaths) {
+        for (let i = 0; i < p.length; i++) h = ((h * 33) ^ p.charCodeAt(i)) >>> 0;
+    }
+    return `${count}:${totalSize}:${maxMtimeNs.toString()}:${h.toString(16)}`;
+}
+
+interface CacheFile<T> {
+    schema_version: number;
+    signature: string;
+    payload: T;
+}
+
+/**
+ * Return the cached payload when the file-set signature is unchanged; otherwise
+ * run `compute`, persist the result under `signature`, and return it.
+ *
+ * @param cachePath  version-namespaced cache file (e.g. `…-index-v1.json`)
+ * @param files      the input file set whose stat signature gates the cache
+ * @param compute    the expensive scan, run only on a miss
+ * @param force      bypass the cache and always recompute (then persist)
+ */
+export function scanCached<T>(
+    cachePath: string,
+    files: readonly string[],
+    compute: () => T,
+    force = false,
+): T {
+    const signature = statSignature(files);
+    if (!force) {
+        try {
+            const cached = JSON.parse(fs.readFileSync(cachePath, 'utf-8')) as CacheFile<T>;
+            if (cached.signature === signature) return cached.payload;
+        } catch {
+            /* miss → recompute */
+        }
+    }
+    const payload = compute();
+    writeCacheAtomic<T>(cachePath, { schema_version: 1, signature, payload });
+    return payload;
+}
+
+function writeCacheAtomic<T>(cachePath: string, data: CacheFile<T>): void {
+    fs.mkdirSync(path.dirname(cachePath), { recursive: true });
+    const tmp = `${cachePath}.${process.pid}.tmp`;
+    fs.writeFileSync(tmp, JSON.stringify(data));
+    fs.renameSync(tmp, cachePath);
+}
+
+/** Drop a stat-index cache file (for `--force`-style resets / tests). */
+export function clear(cachePath: string): void {
+    try {
+        fs.rmSync(cachePath);
+    } catch {
+        /* already absent */
+    }
+}

package/src/scripts/benchmark.ts

@@ -0,0 +1,122 @@
+#!/usr/bin/env tsx
+/**
+ * `agent-config benchmark` — self-measuring token-reduction report
+ * (road-to-retrieval-substrate-hardening B7a).
+ *
+ * Reports the context-token reduction of the retrieval economy against the
+ * REAL alternative: the FULL always-loaded projection (council Q4) — the
+ * honest "what the user pays if everything loads eagerly + full descriptions".
+ * NOT a synthetic full-corpus strawman (explicitly rejected).
+ *
+ * Baseline components (from the pinned internal/bench/reports/token-baseline.json):
+ *   eager_rule_load + skill_descriptions + command_descriptions + mcp_schemas
+ * Reduced: eager→thin rule projection, descriptions/schemas unchanged.
+ *
+ * Deterministic (reads the pinned baseline). Every emitted number binds to
+ * docs/CLAIMS.md with a method line (council Q4 hard condition).
+ *
+ * Usage: benchmark.ts [--baseline PATH] [--format text|json]
+ * Exit codes: 0 ok, 1 baseline missing/malformed, 2 usage error.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+const PROG = 'benchmark.ts';
+const _HERE = fileURLToPath(import.meta.url);
+export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+export const BASELINE = path.join(REPO_ROOT, 'internal', 'bench', 'reports', 'token-baseline.json');
+
+interface Metrics {
+    eager_rule_load: number;
+    thin_rule_load: number;
+    skill_descriptions: number;
+    command_descriptions: number;
+    mcp_schemas: number;
+}
+export interface BenchmarkReport {
+    baseline_full_projection: number;
+    reduced_projection: number;
+    saved: number;
+    reduction_ratio: number; // 0..1
+    method: string;
+    components: Metrics;
+}
+
+function _num(v: unknown): number {
+    return typeof v === 'number' && Number.isFinite(v) ? v : 0;
+}
+
+/** Compute the reduction report from a token-baseline metrics object. */
+export function computeReport(metrics: Metrics): BenchmarkReport {
+    const shared = metrics.skill_descriptions + metrics.command_descriptions + metrics.mcp_schemas;
+    const full = metrics.eager_rule_load + shared;
+    const reduced = metrics.thin_rule_load + shared;
+    const saved = full - reduced;
+    const ratio = full > 0 ? Math.round((saved / full) * 10000) / 10000 : 0;
+    return {
+        baseline_full_projection: full,
+        reduced_projection: reduced,
+        saved,
+        reduction_ratio: ratio,
+        method:
+            'Baseline = FULL always-loaded projection (eager rule load + skill/command ' +
+            'descriptions + MCP schemas) from token-baseline.json; reduced = thin rule ' +
+            'projection, shared surfaces unchanged. Not a synthetic strawman (council Q4).',
+        components: metrics,
+    };
+}
+
+export function loadMetrics(baselinePath: string): Metrics {
+    const parsed = JSON.parse(fs.readFileSync(baselinePath, 'utf-8')) as { metrics?: Record<string, unknown> };
+    const m = parsed.metrics ?? {};
+    return {
+        eager_rule_load: _num(m['eager_rule_load']),
+        thin_rule_load: _num(m['thin_rule_load']),
+        skill_descriptions: _num(m['skill_descriptions']),
+        command_descriptions: _num(m['command_descriptions']),
+        mcp_schemas: _num(m['mcp_schemas']),
+    };
+}
+
+export function main(argv: string[]): number {
+    let baselinePath = BASELINE;
+    let format: 'text' | 'json' = 'text';
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i] as string;
+        if (a === '--baseline') baselinePath = argv[++i] ?? baselinePath;
+        else if (a === '--format') format = (argv[++i] as 'text' | 'json') ?? 'text';
+        else if (a === '-h' || a === '--help') {
+            process.stdout.write(`usage: ${PROG} [--baseline PATH] [--format text|json]\n`);
+            return 0;
+        } else {
+            process.stderr.write(`${PROG}: error: unknown argument ${a}\n`);
+            return 2;
+        }
+    }
+    let report: BenchmarkReport;
+    try {
+        report = computeReport(loadMetrics(baselinePath));
+    } catch (exc) {
+        process.stderr.write(`${PROG}: cannot read baseline ${baselinePath}: ${String(exc)}\n`);
+        return 1;
+    }
+    if (format === 'json') {
+        process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+        return 0;
+    }
+    const pct = (report.reduction_ratio * 100).toFixed(1);
+    process.stdout.write(
+        `context token reduction (vs full always-loaded projection):\n` +
+            `  full projection   ${report.baseline_full_projection} tokens\n` +
+            `  reduced (thin)    ${report.reduced_projection} tokens\n` +
+            `  saved             ${report.saved} tokens (${pct}%)\n` +
+            `  method            ${report.method}\n`,
+    );
+    return 0;
+}
+
+if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? '')) {
+    process.exit(main(process.argv.slice(2)));
+}

package/src/scripts/build_proof.ts

@@ -26,6 +26,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
 import { REPO, LEDGER_REL, load_ledger, pointer_unresolved } from './check_claims.js';
 import { collectSkillGaps } from './check_skill_gaps.js';
 import { loadRows as loadComparisonRows } from './check_comparison.js';
+import { computeStatus as domainSoundnessStatus } from './domain_soundness_status.js';
 
 const _FILE = fileURLToPath(import.meta.url);
 const OUT_REL = 'docs/proof.md';
@@ -118,30 +119,42 @@ function render(): string {
     L.push('checks the wrong property is worse than none), so the number grows');
     L.push('deliberately, not overnight.');
     L.push('');
+    const ds = domainSoundnessStatus();
     L.push('**Non-coding domain soundness — scoped, not proven.** The `finance` /');
     L.push('`founder` / `ops` / `content` profiles sell concrete domain value (DCF,');
     L.push('runway, RICE, incident command, messaging), but the skills are forged on');
     L.push('TS/PHP — "promising, not proven" off those stacks. A disclaimer floor');
     L.push('bounds *liability*, not *correctness*: a skill can be format-correct,');
-    L.push('disclaimered, and still embed a wrong domain assumption. Today **0 of 20**');
+    L.push(`disclaimered, and still embed a wrong domain assumption. Today **${ds.validated} of ${ds.total}**`);
     L.push('default-surface domain skills carry a sourced `domain-truth` fixture');
-    L.push('(`./scripts-run src/scripts/domain_soundness_status`); all are labeled');
-    L.push('`unvalidated` and the validated count is CI-ratcheted. Authoring the');
-    L.push('answer keys needs domain competence (a cited method, not the skill\'s own');
-    L.push('output), so validation lands deliberately — the gap is published, never');
-    L.push('implied away.');
+    L.push('(`./scripts-run src/scripts/domain_soundness_status`); the rest are labeled');
+    L.push('`unvalidated` and the validated count is CI-ratcheted. The fixtures landed');
+    L.push('so far are the **deterministic** targets (runway, unit-economics, DCF,');
+    L.push('forecasting, scenario band/sensitivity), whose answer keys are computed');
+    L.push('from cited standard formulas — never the skill\'s own output; the');
+    L.push('**rubric** targets (incident command, messaging, fundraising, editorial)');
+    L.push('need domain-competent grounding and remain unvalidated, so validation');
+    L.push('lands deliberately — the gap is published, never implied away.');
     L.push('');
-    L.push('**Second-brain substrate — continuity convenience, task-lift unmeasured.**');
-    L.push('The memory substrate is built (working-memory continuity across');
-    L.push('compaction, promotable knowledge cards, contradiction surfacing), but');
-    L.push('whether it beats a no-memory baseline on a reproducible task is **not yet');
-    L.push('measured**. The measurement rig exists — a deterministic multi-session');
-    L.push('recall corpus + scorer (`./scripts-run src/scripts/second_brain_score');
-    L.push('--dry-run`, no model-in-the-loop grading) — but the paired');
-    L.push('`memory-on`/`off`/`placebo` run is spend-bearing and pending. So no');
-    L.push('"second brain" capability claim is made anywhere; the substrate ships as');
-    L.push('continuity convenience until a paired run backs a lift. Boundary vs a');
-    L.push('human PKM is stated in [`docs/second-brain-scope.md`](second-brain-scope.md).');
+    L.push('**Second-brain substrate — measured recall lift, honestly bounded.** On a');
+    L.push('deterministic multi-session recall corpus, the memory substrate beats a');
+    L.push('no-memory baseline AND an equal-byte placebo: memory-on **27/27** vs');
+    L.push('no-memory **10/27** vs placebo **9/27** (claude-haiku-4-5, 9 tasks × 3');
+    L.push('seeds, sign test **p = 0.031** for both pairings) — a real,');
+    L.push('placebo-controlled lift (`internal/bench/reports/second-brain-delta.json`).');
+    L.push('Scoped, not oversold: this is the context-value upper bound (perfect');
+    L.push('retrieval on a one-fact-per-task corpus), NOT retrieval precision under a');
+    L.push('large store; the lift concentrates exactly where memory is the only source');
+    L.push('and ties where the prompt self-contains the fact. Boundary vs a human PKM,');
+    L.push('and why the Obsidian export stays declined, are in');
+    L.push('[`docs/second-brain-scope.md`](second-brain-scope.md).');
+    L.push('');
+    L.push('A follow-up removed the perfect-retrieval assumption: against a store of');
+    L.push('keyword-overlapping confusers the REAL retrieval recalls the needed');
+    L.push('decision into the top-5 (9/9) and the model disambiguates it — retrieval-on');
+    L.push('**27/27** vs no-memory **5/27** and vs placebo **5/27** (p=0.008 both). The');
+    L.push('honest limit: the keyword scorer recalls but does not rank (mean tie-set');
+    L.push('3.3), which is the SQLite-FTS5 activation signal (ADR-116) at scale.');
     L.push('');
     L.push('## 3. Known limits (published, witness-tested)');
     L.push('');

package/src/scripts/check_memory.ts

@@ -81,7 +81,9 @@ const CRITICAL_WARN_THRESHOLD = 10;
 const CRITICAL_STALE_DAYS = 90;
 const KNOWN_TYPES: ReadonlySet<string> = new Set([
     'domain-invariants',
+    'historical-patterns',
     'incident-learnings',
+    'ownership',
     'product-rules',
 ]);
 

package/src/scripts/check_quality_regression.ts

@@ -108,6 +108,57 @@ export function evaluatePair(
   return { id: task.id, winner, length_delta, winner_is_longer };
 }
 
+/**
+ * Cohen's Kappa — chance-corrected agreement between two raters over the same
+ * categorical labels (road-to-retrieval-substrate-hardening B7b). 1 = perfect,
+ * 0 = chance-level, <0 = worse than chance. `NaN` when there is no data.
+ */
+export function cohensKappa(labelsA: readonly string[], labelsB: readonly string[]): number {
+  const n = Math.min(labelsA.length, labelsB.length);
+  if (n === 0) return Number.NaN;
+  const cats = new Set<string>();
+  for (let i = 0; i < n; i++) {
+    cats.add(labelsA[i] as string);
+    cats.add(labelsB[i] as string);
+  }
+  let observedAgree = 0;
+  const countA = new Map<string, number>();
+  const countB = new Map<string, number>();
+  for (let i = 0; i < n; i++) {
+    const a = labelsA[i] as string;
+    const b = labelsB[i] as string;
+    if (a === b) observedAgree += 1;
+    countA.set(a, (countA.get(a) ?? 0) + 1);
+    countB.set(b, (countB.get(b) ?? 0) + 1);
+  }
+  const po = observedAgree / n;
+  let pe = 0;
+  for (const c of cats) pe += ((countA.get(c) ?? 0) / n) * ((countB.get(c) ?? 0) / n);
+  if (pe === 1) return 1; // both raters unanimous on one label → perfect by convention
+  return Math.round(((po - pe) / (1 - pe)) * 10000) / 10000;
+}
+
+/**
+ * Second-independent-judge validation: Cohen's Kappa over the two judges'
+ * per-pair winner labels (aligned by task id). A LOW kappa means the win-rate
+ * verdict rests on an unreliable grader — the trustworthiness gap the
+ * token-saving live run surfaced (33% judge-inconsistency), distinct from the
+ * effect-significance the Wilcoxon/McNemar gates already test.
+ */
+export function judgeKappa(resultsA: readonly PairResult[], resultsB: readonly PairResult[]): number {
+  const byIdB = new Map(resultsB.map((r) => [r.id, r.winner]));
+  const a: string[] = [];
+  const b: string[] = [];
+  for (const r of resultsA) {
+    const bw = byIdB.get(r.id);
+    if (bw !== undefined) {
+      a.push(r.winner);
+      b.push(bw);
+    }
+  }
+  return cohensKappa(a, b);
+}
+
 export interface QualityAggregate {
   total: number;
   thin_wins: number;