@event4u/agent-config 9.8.0 → 9.10.0

package/src/scripts/mcp_telemetry_store.ts

@@ -22,10 +22,14 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import type { DatabaseSync } from 'node:sqlite';
-import type * as NodeSqlite from 'node:sqlite';
 
 import { TELEMETRY_FILENAME, TELEMETRY_REL_DIR } from './mcp_server/telemetry.js';
+import { loadSqlite, readUserVersion, stampUserVersion } from './_lib/sqlite_guard.js';
 
+// cache-invalidation: the path carries no version segment (existing CLI
+// intent tests pin this exact string in stdout); invalidation instead runs
+// through `PRAGMA user_version = SCHEMA_VERSION`, stamped on every connect
+// by `_connect()` below (road-to-reachable-code-memory Phase 7).
 export const DEFAULT_DB_REL = 'agents/runtime/mcp-telemetry/calls.sqlite3';
 export const SCHEMA_VERSION = 1;
 
@@ -173,42 +177,18 @@ export function resolve_db(consumer_root?: string | null): string {
     return path.join(root, DEFAULT_DB_REL);
 }
 
-// Python's stdlib sqlite3 emits nothing on stderr; node:sqlite is flagged
-// experimental on node 22 and prints an `ExperimentalWarning: SQLite …` line
-// to stderr on first import (stable / silent on node >= 23). Drop only that
-// specific warning so stderr stays byte-stable across node versions (tests
-// pin the exact bytes). Installed once, narrowly matched.
-let _sqliteWarningSilenced = false;
-function _silenceSqliteExperimentalWarning(): void {
-    if (_sqliteWarningSilenced) return;
-    _sqliteWarningSilenced = true;
-    const orig = process.emitWarning.bind(process);
-    process.emitWarning = ((warning: string | Error, ...rest: unknown[]): void => {
-        const text = typeof warning === 'string' ? warning : (warning?.message ?? '');
-        if (/SQLite is an experimental/i.test(text)) return;
-        (orig as (w: string | Error, ...a: unknown[]) => void)(warning, ...rest);
-    }) as typeof process.emitWarning;
-}
-
-/** Lazy `node:sqlite` import — keeps the module loadable on a runtime without it. */
-async function _loadSqlite(): Promise<typeof NodeSqlite> {
-    _silenceSqliteExperimentalWarning();
-    try {
-        return await import('node:sqlite');
-    } catch (exc) {
-        const message = exc instanceof Error ? exc.message : String(exc);
-        throw new Error(
-            `node:sqlite is unavailable in this runtime (${message}). ` +
-                'mcp_telemetry_store requires Node with the built-in SQLite module.',
-        );
+/** Remove a possibly-corrupt DB file + its WAL/SHM sidecars, best-effort. */
+function _discardDb(db_path: string): void {
+    for (const p of [db_path, `${db_path}-wal`, `${db_path}-shm`]) {
+        try {
+            fs.rmSync(p, { force: true });
+        } catch {
+            /* best-effort */
+        }
     }
 }
 
-/** Open + schema-init the DB. Mirrors the Python `_connect` helper. */
-async function _connect(db_path: string): Promise<DatabaseSync> {
-    fs.mkdirSync(path.dirname(db_path), { recursive: true });
-    const { DatabaseSync } = await _loadSqlite();
-    const conn = new DatabaseSync(db_path);
+function _createSchema(conn: DatabaseSync): void {
     conn.exec('PRAGMA journal_mode=WAL');
     conn.exec(
         `
@@ -226,6 +206,56 @@ async function _connect(db_path: string): Promise<DatabaseSync> {
     conn.exec('CREATE INDEX IF NOT EXISTS idx_calls_tool ON calls(tool_name)');
     conn.exec('CREATE INDEX IF NOT EXISTS idx_calls_ts ON calls(ts)');
     conn.exec('CREATE INDEX IF NOT EXISTS idx_calls_outcome ON calls(outcome)');
+}
+
+/**
+ * Open + schema-init the DB. Mirrors the Python `_connect` helper.
+ *
+ * road-to-reachable-code-memory Phase 7 — fixes the previously never-read
+ * `SCHEMA_VERSION` constant: `PRAGMA user_version` is stamped on create AND
+ * checked on open.
+ *
+ * - Absent file, or `user_version == 0` (a pre-Phase-7 DB written before this
+ *   fix existed — same table shape, just never versioned): create/open
+ *   normally, then stamp the current version. This is the "legacy-path
+ *   sweep on first run".
+ * - `user_version` present but NOT `SCHEMA_VERSION`: a real future schema
+ *   change landed. The DB is a derived, idempotently-rebuildable view of the
+ *   JSONL sink (source of truth) — silently discard it and rebuild fresh
+ *   rather than attempt an in-place migration for a schema this code
+ *   doesn't know.
+ * - File exists but fails to even OPEN (truncated / corrupt): same
+ *   discard-and-rebuild path. Corruption is a non-event, not an error the
+ *   caller has to handle.
+ */
+async function _connect(db_path: string): Promise<DatabaseSync> {
+    fs.mkdirSync(path.dirname(db_path), { recursive: true });
+    const { DatabaseSync } = await loadSqlite('mcp_telemetry_store');
+
+    if (fs.existsSync(db_path)) {
+        let probe: DatabaseSync | null = null;
+        let drift = false;
+        try {
+            probe = new DatabaseSync(db_path);
+            const existing = readUserVersion(probe);
+            drift = existing !== 0 && existing !== SCHEMA_VERSION;
+        } catch {
+            drift = true; // corrupt / unreadable → rebuild
+        } finally {
+            try {
+                probe?.close();
+            } catch {
+                /* best-effort */
+            }
+        }
+        if (drift) {
+            _discardDb(db_path);
+        }
+    }
+
+    const conn = new DatabaseSync(db_path);
+    _createSchema(conn);
+    stampUserVersion(conn, SCHEMA_VERSION);
     return conn;
 }
 

package/src/scripts/measure_scope_dedup.ts

@@ -0,0 +1,224 @@
+#!/usr/bin/env tsx
+/**
+ * C-3 measurement harness — scope de-duplication of the rule projection.
+ *
+ * Pre-registered in `road-to-cache-economy.md` § "C-3
+ * pre-registration" BEFORE this script produced a number. Metric, threshold and
+ * honest-null consequence are fixed there; this file only measures.
+ *
+ * What it does, deterministically and without touching the real HOME:
+ *
+ *   1. Builds a two-scope FIXTURE: a temp home whose `.claude/rules/` is a
+ *      byte-identical copy of the projected rule source. That is the condition
+ *      the reduction targets — a consumer carrying the same package version at
+ *      user scope and project scope. (On a maintainer machine the two scopes
+ *      hold different releases, so the dedup is correctly inert; that case is
+ *      measured too, as the control.)
+ *   2. Censuses the payload with the dedup OFF, then ON.
+ *   3. Reports the delta against BOTH denominators: the measured median
+ *      cold-start (what a spawn actually pays) and the file-measurable subtotal.
+ *
+ * It never writes into `~`, never regenerates the repo's own projection, and
+ * makes no network call.
+ */
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+
+import { censusRuleDir } from './preamble_byte_census.js';
+
+const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..', '..');
+const RULES_SOURCE = path.join(REPO_ROOT, 'dist', 'agent-src', 'rules');
+
+/** Estimated tokens, the same chars/4 basis the census uses. */
+function tokens(chars: number): number {
+    return Math.round(chars / 4);
+}
+
+interface ScopeSnapshot {
+    files: number;
+    chars: number;
+    tokens: number;
+}
+
+function snapshot(dir: string): ScopeSnapshot {
+    const c = censusRuleDir(dir);
+    return { files: c.files, chars: c.chars, tokens: tokens(c.chars) };
+}
+
+/** Copy every `*.md` byte-for-byte — the fixture must be byte-identical to matter. */
+function copyRules(from: string, to: string): number {
+    fs.mkdirSync(to, { recursive: true });
+    let n = 0;
+    for (const entry of fs.readdirSync(from)) {
+        if (!entry.endsWith('.md')) continue;
+        fs.copyFileSync(path.join(from, entry), path.join(to, entry));
+        n += 1;
+    }
+    return n;
+}
+
+/**
+ * Which rules a byte-identity-gated dedup would skip at project scope. Mirrors
+ * `condense._dedupable_rules`'s predicate rather than importing it, so the
+ * measurement fails loudly if the two ever disagree instead of moving together.
+ */
+function dedupableCount(userDir: string, sourceDir: string): { skipped: number; chars: number } {
+    let skipped = 0;
+    let chars = 0;
+    if (!fs.existsSync(userDir)) return { skipped, chars };
+    for (const entry of fs.readdirSync(sourceDir)) {
+        if (!entry.endsWith('.md')) continue;
+        const twin = path.join(userDir, entry);
+        const source = path.join(sourceDir, entry);
+        if (!fs.existsSync(twin)) continue;
+        const a = fs.readFileSync(twin);
+        const b = fs.readFileSync(source);
+        if (a.equals(b)) {
+            skipped += 1;
+            chars += b.byteLength;
+        }
+    }
+    return { skipped, chars };
+}
+
+/**
+ * Why the real user scope misses the fixture condition. Version drift is the
+ * benign half; the structural half is that `install.ts` stamps ownership
+ * frontmatter into every installed rule (`_set_key(fm_lines, 'package', …)` at
+ * install.ts:2723, `source_path` at install.ts:2725) while the in-repo
+ * projection stamps nothing — so a provenance-only difference survives even a
+ * perfect version alignment. Classifying the two apart is what turns "0/110"
+ * from a puzzle into an answer: `provenanceOnly == total` means aligning
+ * versions would buy exactly nothing.
+ */
+const OWNERSHIP_KEYS = ['package', 'source_path'] as const;
+
+function stripOwnershipKeys(buf: Buffer): string {
+    return buf
+        .toString('utf-8')
+        .split('\n')
+        .filter((line) => !OWNERSHIP_KEYS.some((k) => line.startsWith(`${k}:`)))
+        .join('\n');
+}
+
+interface ReachabilitySplit {
+    total: number;
+    identical: number;
+    provenanceOnly: number;
+    bodyDiff: number;
+    missing: number;
+}
+
+function classifyReachability(userDir: string, sourceDir: string): ReachabilitySplit {
+    const split: ReachabilitySplit = {
+        total: 0,
+        identical: 0,
+        provenanceOnly: 0,
+        bodyDiff: 0,
+        missing: 0,
+    };
+    for (const entry of fs.readdirSync(sourceDir)) {
+        if (!entry.endsWith('.md')) continue;
+        split.total += 1;
+        const twin = path.join(userDir, entry);
+        let a: Buffer;
+        try {
+            a = fs.readFileSync(twin);
+        } catch {
+            split.missing += 1;
+            continue;
+        }
+        const b = fs.readFileSync(path.join(sourceDir, entry));
+        if (a.equals(b)) split.identical += 1;
+        else if (stripOwnershipKeys(a) === stripOwnershipKeys(b)) split.provenanceOnly += 1;
+        else split.bodyDiff += 1;
+    }
+    return split;
+}
+
+function main(argv: readonly string[]): number {
+    const medianArg = argv.find((a) => a.startsWith('--measured-median='));
+    // The measured median cold-start payload, from the transcript census. Passed
+    // in rather than re-derived so this harness stays byte-only and offline.
+    const measuredMedian = medianArg ? Number(medianArg.split('=')[1]) : 230556;
+
+    const fixtureHome = fs.mkdtempSync(path.join(os.tmpdir(), 'scope-dedup-'));
+    const fixtureUserRules = path.join(fixtureHome, '.claude', 'rules');
+    const copied = copyRules(RULES_SOURCE, fixtureUserRules);
+
+    const source = snapshot(RULES_SOURCE);
+    const fixtureUser = snapshot(fixtureUserRules);
+
+    // --- the two conditions ---------------------------------------------------
+    const fixture = dedupableCount(fixtureUserRules, RULES_SOURCE);
+    const realUserRules = path.join(process.env['HOME'] ?? os.homedir(), '.claude', 'rules');
+    const control = dedupableCount(realUserRules, RULES_SOURCE);
+    const reach = fs.existsSync(realUserRules)
+        ? classifyReachability(realUserRules, RULES_SOURCE)
+        : { total: 0, identical: 0, provenanceOnly: 0, bodyDiff: 0, missing: 0 };
+
+    const before = fixtureUser.tokens + source.tokens; // both scopes load
+    const after = fixtureUser.tokens + (source.tokens - tokens(fixture.chars));
+    const removed = before - after;
+
+    const pct = (part: number, whole: number): string => `${((part / whole) * 100).toFixed(1)}%`;
+
+    const lines: string[] = [];
+    lines.push('C-3 measurement — scope de-duplication of the rule projection');
+    lines.push(`  metric basis: chars/4 token estimate; rule source ${RULES_SOURCE}`);
+    lines.push('');
+    lines.push(`FIXTURE (consumer condition — same version at both scopes, ${copied} files copied)`);
+    lines.push(`  user scope:            ${fixtureUser.files} files, ${fixtureUser.tokens} tok`);
+    lines.push(`  project scope:         ${source.files} files, ${source.tokens} tok`);
+    lines.push(`  byte-identical twins:  ${fixture.skipped}/${source.files}`);
+    lines.push(`  rules payload before:  ${before} tok`);
+    lines.push(`  rules payload after:   ${after} tok`);
+    lines.push(`  removed:               ${removed} tok`);
+    lines.push('');
+    lines.push('REDUCTION vs the pre-registered denominators');
+    lines.push(`  of measured median cold-start (${measuredMedian} tok): ${pct(removed, measuredMedian)}`);
+    lines.push(`  of the two-scope rules payload (${before} tok):       ${pct(removed, before)}`);
+    lines.push('');
+    lines.push('CONTROL (this machine — the dedup must be inert unless both scopes match)');
+    lines.push(`  byte-identical twins:  ${control.skipped}/${source.files}`);
+    lines.push(`  removed:               ${tokens(control.chars)} tok`);
+    lines.push('');
+    lines.push(`threshold: >= 15% of the measured median  ->  ${
+        removed / measuredMedian >= 0.15 ? 'MET' : 'NOT MET'
+    }`);
+    lines.push('');
+    lines.push('REACHABILITY of the fixture condition (why the control is not the fixture)');
+    lines.push(`  identical:             ${reach.identical}/${reach.total}`);
+    lines.push(`  differ in body:        ${reach.bodyDiff}/${reach.total}  (closed by aligning versions)`);
+    lines.push(`  differ ONLY in the ownership stamp: ${reach.provenanceOnly}/${reach.total}`);
+    lines.push(`  absent at user scope:  ${reach.missing}/${reach.total}`);
+    if (reach.identical === 0 && reach.bodyDiff + reach.provenanceOnly === reach.total) {
+        lines.push('');
+        lines.push('  -> Aligning versions would close the body diffs and leave the ownership');
+        lines.push('     stamp, so the reachable twin count is 0, NOT the fixture\'s figure.');
+        lines.push('     install.ts:2723/2725 stamp package:/source_path: into every installed');
+        lines.push('     rule unconditionally; the in-repo projection stamps nothing. Two');
+        lines.push('     writers, deliberately different output. Making this reachable was');
+        lines.push('     DECIDED AGAINST on 2026-07-31 (real install-path risk, empty recipient');
+        lines.push('     set) — see contexts/dedup-reachability-refusal.md for the analysis and');
+        lines.push('     the five reopen conditions. This 0 is expected, not a defect.');
+        lines.push('     Do NOT relax the byte predicate to make this number move: that');
+        lines.push('     predicate IS the content-neutrality argument.');
+    }
+    lines.push('');
+    lines.push('Content safety: only byte-identical twins are removed, so the host still');
+    lines.push('loads the same rule text in full — once instead of twice. No rule becomes');
+    lines.push('trigger-gated, which is what separates this from the disabled thin projection.');
+
+    process.stdout.write(lines.join('\n') + '\n');
+    fs.rmSync(fixtureHome, { recursive: true, force: true });
+    return 0;
+}
+
+if (process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`) {
+    process.exit(main(process.argv.slice(2)));
+}
+
+export { classifyReachability, dedupableCount, main };
+export type { ReachabilitySplit };

package/src/scripts/memory_learn_hook.ts

@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+/**
+ * session_end concern — memory learning sidecar aggregation
+ * (road-to-reachable-code-memory Phase 4).
+ *
+ * Joins the EXISTING `session_end` concern set in hook_manifest.yaml — no
+ * new platform hook wiring. On session end it aggregates
+ * `agents/memory/intake/signals-*.jsonl` through the learning sidecar
+ * (`learning_sidecar.ts`: 30-day decay, ≥2-origin corroboration, dead-end
+ * ledger) into the gitignored `agents/memory/.agent-learning.json` +
+ * // cache-invalidation: path owned by learning_sidecar.ts (schema_version field inside the JSON; name pinned there)
+ * `LESSONS.md`. Local-only, budget-capped, fail-open:
+ *
+ * - Default-OFF. Fires only when `memory.learn_on_session_end: true` in
+ *   `.agent-settings.yml` (council decision 2026-07-27: ships off; the
+ *   default flip is proposed only after the 30-day dogfood shows
+ *   non-trivial signal AND session-end p95 < 2 s).
+ * - Budget-capped: a wall-clock budget (default 2000 ms) aborts the write
+ *   and exits 0 — a slow aggregation must never tax session teardown.
+ * - Fail-open: any error → exit 0, no output. `fail_closed: false`.
+ * - Promotion stays human: this hook only refreshes the sidecar; a
+ *   PROMOTED lesson is surfaced as a one-line visibility marker pointing
+ *   at the existing /memory:propose flow. Nothing auto-writes curated
+ *   `agents/memory/*.yml`.
+ *
+ * Exit codes (dispatcher contract): 0 always (allow).
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+import {
+    buildSidecar,
+    LESSONS_NAME,
+    renderLessonsMd,
+    SIDECAR_NAME,
+} from './learning_sidecar.js';
+
+const SETTINGS_FILE = '.agent-settings.yml';
+/** Wall-clock budget for the whole aggregation (ms). */
+export const BUDGET_MS = 2000;
+
+/**
+ * Minimal `.agent-settings.yml` reader for `memory.learn_on_session_end`
+ * (same hand-rolled mini-parser pattern as the sibling hooks — no YAML
+ * dependency in the hook hot path).
+ */
+export function enabled(root: string): boolean {
+    const p = path.join(root, SETTINGS_FILE);
+    let text: string;
+    try {
+        text = fs.readFileSync(p, 'utf8');
+    } catch {
+        return false;
+    }
+    let inMemory = false;
+    for (const rawLine of text.split('\n')) {
+        const line = rawLine.replace(/\s+$/u, '');
+        if (/^memory:\s*(#.*)?$/u.test(line)) {
+            inMemory = true;
+            continue;
+        }
+        if (inMemory) {
+            if (/^\S/u.test(line) && line !== '') {
+                inMemory = false; // left the memory: block
+                continue;
+            }
+            const m = /^\s{2,}learn_on_session_end:\s*(\S+)/u.exec(line);
+            if (m) return m[1] === 'true';
+        }
+    }
+    return false;
+}
+
+/** Run the aggregation. Returns the one-line marker to emit, or null. */
+export function runLearn(root: string, nowIso: string, budgetMs: number = BUDGET_MS): string | null {
+    const started = Date.now();
+    const intakeDir = path.join(root, 'agents', 'memory', 'intake');
+    const outDir = path.join(root, 'agents', 'memory');
+    if (!fs.existsSync(intakeDir)) return null;
+    const sidecar = buildSidecar(intakeDir, nowIso);
+    if (Date.now() - started > budgetMs) return null; // over budget — skip the write
+    if (sidecar.lessons.length === 0) return null;
+    fs.writeFileSync(path.join(outDir, SIDECAR_NAME), `${JSON.stringify(sidecar, null, 2)}\n`, 'utf8');
+    fs.writeFileSync(path.join(outDir, LESSONS_NAME), renderLessonsMd(sidecar), 'utf8');
+    const promoted = sidecar.lessons.filter((l) => l.verdict === 'preferred').length;
+    const deadEnds = sidecar.lessons.filter((l) => l.verdict === 'dead_end').length;
+    // Memory-visibility contract shape: one line, ids/counters only, no bodies.
+    return (
+        `🧠 Memory: sidecar refreshed — ${sidecar.lessons.length} lesson(s) ` +
+        `(${promoted} preferred · ${deadEnds} dead-end); promote via /memory:propose`
+    );
+}
+
+export function main(): number {
+    try {
+        const root = process.cwd();
+        if (!enabled(root)) return 0;
+        const marker = runLearn(root, new Date().toISOString());
+        if (marker) process.stdout.write(`${marker}\n`);
+    } catch {
+        // fail-open: session teardown must never break on a learning hook.
+    }
+    return 0;
+}
+
+// Bundle-safety: never auto-run when inlined into an esbuild bundle, where
+// every module shares the bundle's `import.meta.url` (see cmd_migrate.ts).
+declare const __AGENT_CONFIG_BUNDLE__: boolean | undefined;
+const _isMain = (() => {
+    if (typeof __AGENT_CONFIG_BUNDLE__ !== 'undefined' && __AGENT_CONFIG_BUNDLE__) return false;
+    if (process.argv[1] === undefined) return false;
+    try {
+        return (
+            pathToFileURL(fs.realpathSync(path.resolve(process.argv[1]))).href ===
+            import.meta.url
+        );
+    } catch {
+        return pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url;
+    }
+})();
+if (_isMain) {
+    process.exit(main());
+}
+
+export const _HERE = fileURLToPath(import.meta.url);