@event4u/agent-config 9.9.0 → 9.11.0

package/src/scripts/measure_scope_dedup.ts

@@ -0,0 +1,224 @@
+#!/usr/bin/env tsx
+/**
+ * C-3 measurement harness — scope de-duplication of the rule projection.
+ *
+ * Pre-registered in `road-to-cache-economy.md` § "C-3
+ * pre-registration" BEFORE this script produced a number. Metric, threshold and
+ * honest-null consequence are fixed there; this file only measures.
+ *
+ * What it does, deterministically and without touching the real HOME:
+ *
+ *   1. Builds a two-scope FIXTURE: a temp home whose `.claude/rules/` is a
+ *      byte-identical copy of the projected rule source. That is the condition
+ *      the reduction targets — a consumer carrying the same package version at
+ *      user scope and project scope. (On a maintainer machine the two scopes
+ *      hold different releases, so the dedup is correctly inert; that case is
+ *      measured too, as the control.)
+ *   2. Censuses the payload with the dedup OFF, then ON.
+ *   3. Reports the delta against BOTH denominators: the measured median
+ *      cold-start (what a spawn actually pays) and the file-measurable subtotal.
+ *
+ * It never writes into `~`, never regenerates the repo's own projection, and
+ * makes no network call.
+ */
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import { censusRuleDir } from './preamble_byte_census.js';
+
+const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..', '..');
+const RULES_SOURCE = path.join(REPO_ROOT, 'dist', 'agent-src', 'rules');
+
+/** Estimated tokens, the same chars/4 basis the census uses. */
+function tokens(chars: number): number {
+    return Math.round(chars / 4);
+}
+
+interface ScopeSnapshot {
+    files: number;
+    chars: number;
+    tokens: number;
+}
+
+function snapshot(dir: string): ScopeSnapshot {
+    const c = censusRuleDir(dir);
+    return { files: c.files, chars: c.chars, tokens: tokens(c.chars) };
+}
+
+/** Copy every `*.md` byte-for-byte — the fixture must be byte-identical to matter. */
+function copyRules(from: string, to: string): number {
+    fs.mkdirSync(to, { recursive: true });
+    let n = 0;
+    for (const entry of fs.readdirSync(from)) {
+        if (!entry.endsWith('.md')) continue;
+        fs.copyFileSync(path.join(from, entry), path.join(to, entry));
+        n += 1;
+    }
+    return n;
+}
+
+/**
+ * Which rules a byte-identity-gated dedup would skip at project scope. Mirrors
+ * `condense._dedupable_rules`'s predicate rather than importing it, so the
+ * measurement fails loudly if the two ever disagree instead of moving together.
+ */
+function dedupableCount(userDir: string, sourceDir: string): { skipped: number; chars: number } {
+    let skipped = 0;
+    let chars = 0;
+    if (!fs.existsSync(userDir)) return { skipped, chars };
+    for (const entry of fs.readdirSync(sourceDir)) {
+        if (!entry.endsWith('.md')) continue;
+        const twin = path.join(userDir, entry);
+        const source = path.join(sourceDir, entry);
+        if (!fs.existsSync(twin)) continue;
+        const a = fs.readFileSync(twin);
+        const b = fs.readFileSync(source);
+        if (a.equals(b)) {
+            skipped += 1;
+            chars += b.byteLength;
+        }
+    }
+    return { skipped, chars };
+}
+
+/**
+ * Why the real user scope misses the fixture condition. Version drift is the
+ * benign half; the structural half is that `install.ts` stamps ownership
+ * frontmatter into every installed rule (`_set_key(fm_lines, 'package', …)` at
+ * install.ts:2723, `source_path` at install.ts:2725) while the in-repo
+ * projection stamps nothing — so a provenance-only difference survives even a
+ * perfect version alignment. Classifying the two apart is what turns "0/110"
+ * from a puzzle into an answer: `provenanceOnly == total` means aligning
+ * versions would buy exactly nothing.
+ */
+const OWNERSHIP_KEYS = ['package', 'source_path'] as const;
+
+function stripOwnershipKeys(buf: Buffer): string {
+    return buf
+        .toString('utf-8')
+        .split('\n')
+        .filter((line) => !OWNERSHIP_KEYS.some((k) => line.startsWith(`${k}:`)))
+        .join('\n');
+}
+
+interface ReachabilitySplit {
+    total: number;
+    identical: number;
+    provenanceOnly: number;
+    bodyDiff: number;
+    missing: number;
+}
+
+function classifyReachability(userDir: string, sourceDir: string): ReachabilitySplit {
+    const split: ReachabilitySplit = {
+        total: 0,
+        identical: 0,
+        provenanceOnly: 0,
+        bodyDiff: 0,
+        missing: 0,
+    };
+    for (const entry of fs.readdirSync(sourceDir)) {
+        if (!entry.endsWith('.md')) continue;
+        split.total += 1;
+        const twin = path.join(userDir, entry);
+        let a: Buffer;
+        try {
+            a = fs.readFileSync(twin);
+        } catch {
+            split.missing += 1;
+            continue;
+        }
+        const b = fs.readFileSync(path.join(sourceDir, entry));
+        if (a.equals(b)) split.identical += 1;
+        else if (stripOwnershipKeys(a) === stripOwnershipKeys(b)) split.provenanceOnly += 1;
+        else split.bodyDiff += 1;
+    }
+    return split;
+}
+
+function main(argv: readonly string[]): number {
+    const medianArg = argv.find((a) => a.startsWith('--measured-median='));
+    // The measured median cold-start payload, from the transcript census. Passed
+    // in rather than re-derived so this harness stays byte-only and offline.
+    const measuredMedian = medianArg ? Number(medianArg.split('=')[1]) : 230556;
+
+    const fixtureHome = fs.mkdtempSync(path.join(os.tmpdir(), 'scope-dedup-'));
+    const fixtureUserRules = path.join(fixtureHome, '.claude', 'rules');
+    const copied = copyRules(RULES_SOURCE, fixtureUserRules);
+
+    const source = snapshot(RULES_SOURCE);
+    const fixtureUser = snapshot(fixtureUserRules);
+
+    // --- the two conditions ---------------------------------------------------
+    const fixture = dedupableCount(fixtureUserRules, RULES_SOURCE);
+    const realUserRules = path.join(process.env['HOME'] ?? os.homedir(), '.claude', 'rules');
+    const control = dedupableCount(realUserRules, RULES_SOURCE);
+    const reach = fs.existsSync(realUserRules)
+        ? classifyReachability(realUserRules, RULES_SOURCE)
+        : { total: 0, identical: 0, provenanceOnly: 0, bodyDiff: 0, missing: 0 };
+
+    const before = fixtureUser.tokens + source.tokens; // both scopes load
+    const after = fixtureUser.tokens + (source.tokens - tokens(fixture.chars));
+    const removed = before - after;
+
+    const pct = (part: number, whole: number): string => `${((part / whole) * 100).toFixed(1)}%`;
+
+    const lines: string[] = [];
+    lines.push('C-3 measurement — scope de-duplication of the rule projection');
+    lines.push(`  metric basis: chars/4 token estimate; rule source ${RULES_SOURCE}`);
+    lines.push('');
+    lines.push(`FIXTURE (consumer condition — same version at both scopes, ${copied} files copied)`);
+    lines.push(`  user scope:            ${fixtureUser.files} files, ${fixtureUser.tokens} tok`);
+    lines.push(`  project scope:         ${source.files} files, ${source.tokens} tok`);
+    lines.push(`  byte-identical twins:  ${fixture.skipped}/${source.files}`);
+    lines.push(`  rules payload before:  ${before} tok`);
+    lines.push(`  rules payload after:   ${after} tok`);
+    lines.push(`  removed:               ${removed} tok`);
+    lines.push('');
+    lines.push('REDUCTION vs the pre-registered denominators');
+    lines.push(`  of measured median cold-start (${measuredMedian} tok): ${pct(removed, measuredMedian)}`);
+    lines.push(`  of the two-scope rules payload (${before} tok):       ${pct(removed, before)}`);
+    lines.push('');
+    lines.push('CONTROL (this machine — the dedup must be inert unless both scopes match)');
+    lines.push(`  byte-identical twins:  ${control.skipped}/${source.files}`);
+    lines.push(`  removed:               ${tokens(control.chars)} tok`);
+    lines.push('');
+    lines.push(`threshold: >= 15% of the measured median  ->  ${
+        removed / measuredMedian >= 0.15 ? 'MET' : 'NOT MET'
+    }`);
+    lines.push('');
+    lines.push('REACHABILITY of the fixture condition (why the control is not the fixture)');
+    lines.push(`  identical:             ${reach.identical}/${reach.total}`);
+    lines.push(`  differ in body:        ${reach.bodyDiff}/${reach.total}  (closed by aligning versions)`);
+    lines.push(`  differ ONLY in the ownership stamp: ${reach.provenanceOnly}/${reach.total}`);
+    lines.push(`  absent at user scope:  ${reach.missing}/${reach.total}`);
+    if (reach.identical === 0 && reach.bodyDiff + reach.provenanceOnly === reach.total) {
+        lines.push('');
+        lines.push('  -> Aligning versions would close the body diffs and leave the ownership');
+        lines.push('     stamp, so the reachable twin count is 0, NOT the fixture\'s figure.');
+        lines.push('     install.ts:2723/2725 stamp package:/source_path: into every installed');
+        lines.push('     rule unconditionally; the in-repo projection stamps nothing. Two');
+        lines.push('     writers, deliberately different output. Making this reachable was');
+        lines.push('     DECIDED AGAINST on 2026-07-31 (real install-path risk, empty recipient');
+        lines.push('     set) — see contexts/dedup-reachability-refusal.md for the analysis and');
+        lines.push('     the five reopen conditions. This 0 is expected, not a defect.');
+        lines.push('     Do NOT relax the byte predicate to make this number move: that');
+        lines.push('     predicate IS the content-neutrality argument.');
+    }
+    lines.push('');
+    lines.push('Content safety: only byte-identical twins are removed, so the host still');
+    lines.push('loads the same rule text in full — once instead of twice. No rule becomes');
+    lines.push('trigger-gated, which is what separates this from the disabled thin projection.');
+
+    process.stdout.write(lines.join('\n') + '\n');
+    fs.rmSync(fixtureHome, { recursive: true, force: true });
+    return 0;
+}
+
+if (process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`) {
+    process.exit(main(process.argv.slice(2)));
+}
+
+export { classifyReachability, dedupableCount, main };
+export type { ReachabilitySplit };

package/src/scripts/mine_session.ts

@@ -34,6 +34,8 @@ import * as os from 'node:os';
 import * as path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
+import { appendGlobalObservation, applySharedFactCap } from './_lib/user_global_observations.js';
+
 const _HERE = fileURLToPath(import.meta.url);
 // src/scripts/mine_session.py → parent.parent.parent == repo root.
 export const ROOT = path.resolve(path.dirname(_HERE), '..', '..');
@@ -296,6 +298,64 @@ function _nowIsoSeconds(): string {
     return new Date().toISOString().replace(/\.\d{3}Z$/, '+00:00');
 }
 
+/** One candidate the user-scoped channel below may propose to the global observation buffer. */
+export interface UserObservationSignal {
+    readonly ts: string;
+    readonly suggest: string;
+    readonly evidence: string;
+}
+
+/**
+ * The second, user-scoped channel (road-to-global-user-memory Phase 2).
+ * Scans the SAME entries `mine()` scans for the Preference family
+ * (`SIGNAL_FAMILIES`'s `'convention'` entry) and, for every match `mine()`
+ * would have silently discarded — because `_normalise` found no
+ * project-scoped path/symbol token, i.e. the match is about the USER, not
+ * the project — shapes a candidate observation instead of dropping it.
+ *
+ * Deliberately independent of `mine()`: it does not read or mutate `mine()`'s
+ * `facts` array, does not change `mine()`'s signature, return shape, or
+ * output, and does not touch disk. `mine()`'s byte-parity contract with the
+ * retired Python implementation is therefore untouched by this addition —
+ * this is a wholly new, additive export.
+ *
+ * The caller is responsible for the shared ≤5-per-cycle cap
+ * (`user_global_observations.applySharedFactCap`, counting THIS output
+ * together with `mine()`'s `facts.length`) and for guard evaluation +
+ * writing (`user_global_observations.appendGlobalObservation`) — this
+ * function only detects and shapes candidates.
+ */
+export function mineUserObservationCandidates(
+    entries: Iterable<Record<string, Json>>,
+    since: Date,
+): UserObservationSignal[] {
+    const preferenceFamily = SIGNAL_FAMILIES.find(([tag]) => tag === 'convention')?.[1];
+    if (!preferenceFamily) {
+        return [];
+    }
+    const out: UserObservationSignal[] = [];
+    for (const turn of entries) {
+        const tsStr = _turnTs(turn);
+        if (!_withinWindow(tsStr, since)) {
+            continue;
+        }
+        const text = _turnText(turn);
+        if (!text || !_search(preferenceFamily, text)) {
+            continue;
+        }
+        // Project-scoped → mine()'s job, not this channel.
+        if (_search(PATH_TOKEN, text) || _search(SYMBOL_TOKEN, text)) {
+            continue;
+        }
+        const cleaned = _redact(text, []).replace(/\s+/g, ' ').trim().slice(0, 240);
+        if (!cleaned) {
+            continue;
+        }
+        out.push({ ts: tsStr || _nowIsoSeconds(), suggest: cleaned, evidence: cleaned });
+    }
+    return out;
+}
+
 export function renderPreview(
     facts: Array<Record<string, Json>>,
     project: string,
@@ -619,6 +679,44 @@ export function main(argv: string[] | null = null): number {
                 'files.\n   Next: /memory promote to lift validated lines ' +
                 'into curated YAML.\n',
         );
+        // Second channel (road-to-global-user-memory Phase 2): the preference
+        // signals `mine()` discards because they are about the USER, not the
+        // project, reach the global buffer instead of /dev/null. The ≤5 cap is
+        // SHARED — the project channel's facts are counted first, so this channel
+        // only ever uses the leftover headroom.
+        //
+        // `entries` is a consumed generator by now, so the source is re-read.
+        // Appending still goes through every capture guard and the redaction gate;
+        // nothing here can write `profile.md` — only a human `accept` does that.
+        const observationEntries =
+            kind === 'chat-history' ? _iterChatHistory(srcPath) : _iterClaudeCodeJsonl(srcPath);
+        const candidates = applySharedFactCap(
+            facts.length,
+            mineUserObservationCandidates(observationEntries, since),
+        );
+        let appended = 0;
+        const refused: string[] = [];
+        for (const candidate of candidates) {
+            const result = appendGlobalObservation({
+                ts: candidate.ts,
+                field: 'notes',
+                suggest: candidate.suggest,
+                source: 'mine_session',
+                evidence: candidate.evidence,
+            });
+            if (result.written) {
+                appended += 1;
+            } else if (result.category !== undefined) {
+                refused.push(result.category);
+            }
+        }
+        if (candidates.length > 0) {
+            const detail = refused.length > 0 ? ` (${refused.length} refused: ${refused.join(', ')})` : '';
+            process.stdout.write(
+                `✅ Appended ${appended} user observation(s) to the global buffer${detail}.\n` +
+                    '   Next: /agents user review to accept or drop them.\n',
+            );
+        }
         return 0;
     }
 

package/src/scripts/orchestration_payload_hash_drift.ts

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+/**
+ * Payload-hash × cache-hit drift report — CLI.
+ *
+ * `road-to-cache-economy.md` Phase 3, steps 6-7. Reads the
+ * accumulated audit-log-v1 telemetry (reuses `readAuditLines` from
+ * `orchestration_savings_report.ts` — the same reader, not a reimplementation)
+ * and joins `orchestration.payload_hash` against `orchestration.cache_hit`
+ * per {@link aggregatePayloadHashDrift}'s cohort split, so prefix-stability
+ * drift becomes visible.
+ *
+ * REFUSAL (step 7 — report it, never act on it): `cache_hit` is a
+ * provider-reported proxy for HOST-controlled cache behaviour, not a metric
+ * this package causes or controls. Per ADR-118's automation threshold (§1:
+ * a direct measure, a low false-positive rate, and no unique human
+ * judgement required — all three, or it stays manual), a host-controlled
+ * proxy fails condition 1 outright. This report NEVER wires a measure→adjust
+ * step off these two fields — no default flip, no tier change, no dispatch
+ * throttle. It is read-only, always.
+ *
+ * Zero recorded lines with BOTH fields is the CURRENT expected state — both
+ * fields are lean-init extensions with no caller wiring a real value into
+ * them yet (per the roadmap's own defect list). This report says so
+ * explicitly rather than presenting empty input as a green result.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/orchestration_payload_hash_drift [--dir <path>] [--format text|json]
+ *
+ * Default dir: agents/runtime/state/audit (the audit-log-v1 path, matching
+ * `orchestration_savings_report.ts`'s default).
+ */
+import * as path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import { aggregatePayloadHashDrift, type PayloadHashDriftReport, type PayloadHashLine } from './_lib/payload_hash_drift.js';
+import { readAuditLines } from './orchestration_savings_report.js';
+
+const DEFAULT_DIR = 'agents/runtime/state/audit';
+
+export interface Options {
+    dir: string;
+    format: 'text' | 'json';
+}
+
+export function parseArgs(argv: string[]): Options {
+    const opts: Options = { dir: DEFAULT_DIR, format: 'text' };
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === undefined) continue;
+        if (a === '--dir') opts.dir = argv[++i] ?? opts.dir;
+        else if (a.startsWith('--dir=')) opts.dir = a.slice('--dir='.length);
+        else if (a === '--format') opts.format = (argv[++i] as Options['format']) ?? opts.format;
+        else if (a.startsWith('--format=')) opts.format = a.slice('--format='.length) as Options['format'];
+    }
+    if (opts.format !== 'text' && opts.format !== 'json') opts.format = 'text';
+    return opts;
+}
+
+interface RawOrchestrationHashFields {
+    payload_hash?: unknown;
+    cache_hit?: unknown;
+}
+interface RawHashLine {
+    input_kind?: unknown;
+    orchestration?: RawOrchestrationHashFields;
+}
+
+/** Reads only the two fields this report needs, tolerant of any pre-extension line missing them. */
+function toPayloadHashLine(raw: unknown): PayloadHashLine {
+    const line = (raw ?? {}) as RawHashLine;
+    if (line.input_kind !== 'orchestration' || !line.orchestration) {
+        return { payload_hash: null, cache_hit: null };
+    }
+    const hash = line.orchestration.payload_hash;
+    const hit = line.orchestration.cache_hit;
+    return {
+        payload_hash: typeof hash === 'string' && hash.length > 0 ? hash : null,
+        cache_hit: typeof hit === 'boolean' ? hit : null,
+    };
+}
+
+export function buildReport(dir: string): PayloadHashDriftReport {
+    const rawLines: unknown[] = readAuditLines(dir);
+    const lines = rawLines.map(toPayloadHashLine);
+    return aggregatePayloadHashDrift(lines);
+}
+
+function pct(v: number): string {
+    return `${(v * 100).toFixed(1)}%`;
+}
+
+export function renderText(r: PayloadHashDriftReport): string {
+    const out: string[] = [];
+    out.push('Payload-hash × cache-hit drift report');
+    out.push(`  total audit-log-v1 lines seen: ${r.total_lines}`);
+    out.push(`  lines carrying BOTH payload_hash and cache_hit: ${r.lines_with_data}`);
+    out.push('');
+
+    if (r.lines_with_data === 0) {
+        out.push(
+            'No recorded lines carry both fields yet — this is the CURRENT expected state (both are lean-init',
+        );
+        out.push(
+            'extensions; no caller wires a real value into them today). The reader works; there is nothing to',
+        );
+        out.push('report on. This is NOT presented as a green pass — it is an honest empty state.');
+    } else {
+        out.push(
+            `stable cohort (repeat occurrences of a hash — the prior write should already be cached): n=${r.stable_cohort.n} ` +
+                `hit_rate=${r.stable_cohort.hit_rate !== null ? pct(r.stable_cohort.hit_rate) : 'n/a'}`,
+        );
+        out.push(
+            `unstable cohort (hashes seen exactly once — a payload that never repeats): n=${r.unstable_cohort.n} ` +
+                `hit_rate=${r.unstable_cohort.hit_rate !== null ? pct(r.unstable_cohort.hit_rate) : 'n/a'}`,
+        );
+        out.push(`drift_visible (stable hit_rate > unstable hit_rate): ${r.drift_visible ? 'YES' : 'no'}`);
+        out.push('');
+        out.push('Per-hash breakdown (most-occurrences first):');
+        for (const g of r.groups) {
+            out.push(`  ${g.payload_hash}: occurrences=${g.occurrences} hit_rate=${pct(g.hit_rate)}`);
+        }
+    }
+
+    out.push('');
+    out.push('REFUSAL (report, never act): cache_hit is a proxy for host-controlled cache behaviour, not a');
+    out.push('metric this package controls. Per ADR-118 §1, a host-controlled proxy fails the direct-measure');
+    out.push('condition outright — this report never wires a measure→adjust step off these two fields.');
+    return out.join('\n');
+}
+
+export function main(argv: string[] = process.argv.slice(2)): number {
+    const opts = parseArgs(argv);
+    const report = buildReport(opts.dir);
+    if (opts.format === 'json') {
+        process.stdout.write(JSON.stringify(report, null, 2) + '\n');
+    } else {
+        process.stdout.write(renderText(report) + '\n');
+    }
+    return 0;
+}
+
+if (process.argv[1] !== undefined) {
+    const invokedUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+    if (import.meta.url === invokedUrl) process.exit(main());
+}