@event4u/agent-config 7.2.0 → 7.4.0

package/src/scripts/check_token_regression.ts

@@ -0,0 +1,266 @@
+#!/usr/bin/env tsx
+/**
+ * Token-regression gate (token-saving Phase 0).
+ *
+ * Fails CI when a tracked always-loaded / projected token surface grows more
+ * than the tolerance (default 5%) over a committed baseline. This is the
+ * RELATIVE guard — distinct from `audit-tokens-budget` (absolute budgets) and
+ * from the cache-prefix byte-stability guard (Phase 5).
+ *
+ * Source of truth is `internal/bench/reports/projection-cost.json`, which is
+ * now produced with EXACT counts via the real tokenizer
+ * (`src/scripts/_lib/token_count.ts` → js-tiktoken cl100k_base). Run
+ * `task audit-tokens` first so the report reflects the working tree.
+ *
+ * Gated metrics (all `tokens_gpt`):
+ *   - eager_rule_load    — thin_projection.eager_gpt (the eager always-on load)
+ *   - thin_rule_load     — thin_projection.thin_gpt
+ *   - skill_descriptions — description_catalog.skills_core_source.tokens_gpt
+ *   - command_descriptions — description_catalog.commands_core_source.tokens_gpt
+ *   - mcp_schemas        — Σ mcp_tool_schemas[*].tokens_gpt
+ *
+ * A baseline captured under a different `token_method` (e.g. proxy → exact)
+ * is a legitimate reset, not a regression — the gate WARNS and asks for a
+ * `--update-baseline` rather than failing on the method switch alone.
+ *
+ * CLI:
+ *   ./scripts-run src/scripts/check_token_regression            # gate (read-only)
+ *   ./scripts-run src/scripts/check_token_regression --json     # machine output
+ *   ./scripts-run src/scripts/check_token_regression --tolerance 0.05
+ *   ./scripts-run src/scripts/check_token_regression --update-baseline  # record current as baseline
+ *
+ * Exit codes:
+ *   0 — within tolerance (or baseline written / warmup: no baseline yet)
+ *   1 — argument / file error
+ *   2 — regression: a metric exceeded baseline by more than the tolerance
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const _HERE = fileURLToPath(import.meta.url);
+// REPO_ROOT = <repo>/src/scripts/<file> → up two.
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+
+const PROJECTION_REPORT = path.join(
+  REPO_ROOT,
+  'internal/bench/reports/projection-cost.json',
+);
+const BASELINE_FILE = path.join(
+  REPO_ROOT,
+  'internal/bench/reports/token-baseline.json',
+);
+const DEFAULT_TOLERANCE = 0.05;
+
+type Json = Record<string, unknown>;
+
+interface Metrics {
+  token_method: string;
+  metrics: Record<string, number>;
+}
+
+function _num(obj: unknown, ...keys: string[]): number {
+  let cur: unknown = obj;
+  for (const k of keys) {
+    if (cur && typeof cur === 'object' && k in (cur as Json)) {
+      cur = (cur as Json)[k];
+    } else {
+      return 0;
+    }
+  }
+  return typeof cur === 'number' ? cur : 0;
+}
+
+/** Extract the flat gated-metric map from a projection-cost report. */
+function extract_metrics(report: Json): Metrics {
+  const mcp = (report['mcp_tool_schemas'] as Json) ?? {};
+  let mcp_total = 0;
+  for (const server of Object.values(mcp)) {
+    mcp_total += _num(server, 'tokens_gpt');
+  }
+  return {
+    token_method: String(report['token_method'] ?? 'unknown'),
+    metrics: {
+      eager_rule_load: _num(report, 'thin_projection', 'eager_gpt'),
+      thin_rule_load: _num(report, 'thin_projection', 'thin_gpt'),
+      skill_descriptions: _num(
+        report,
+        'description_catalog',
+        'skills_core_source',
+        'tokens_gpt',
+      ),
+      command_descriptions: _num(
+        report,
+        'description_catalog',
+        'commands_core_source',
+        'tokens_gpt',
+      ),
+      mcp_schemas: mcp_total,
+    },
+  };
+}
+
+function _readJson(file: string): Json | null {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf-8')) as Json;
+  } catch {
+    return null;
+  }
+}
+
+interface Row {
+  id: string;
+  current: number;
+  baseline: number;
+  pct: number; // signed fraction, e.g. 0.07 = +7%
+  regressed: boolean;
+}
+
+export interface GateResult {
+  status: 'ok' | 'regression' | 'warmup' | 'method-changed';
+  tolerance: number;
+  current_method: string;
+  baseline_method: string | null;
+  rows: Row[];
+}
+
+export function evaluate(
+  current: Metrics,
+  baseline: Metrics | null,
+  tolerance: number,
+): GateResult {
+  if (baseline === null) {
+    return {
+      status: 'warmup',
+      tolerance,
+      current_method: current.token_method,
+      baseline_method: null,
+      rows: [],
+    };
+  }
+  const method_changed = baseline.token_method !== current.token_method;
+  const rows: Row[] = [];
+  for (const [id, cur] of Object.entries(current.metrics)) {
+    const base = baseline.metrics[id] ?? 0;
+    const pct = base > 0 ? (cur - base) / base : 0;
+    rows.push({
+      id,
+      current: cur,
+      baseline: base,
+      pct,
+      // A method switch is a legitimate baseline reset, not a regression.
+      regressed: !method_changed && base > 0 && pct > tolerance,
+    });
+  }
+  rows.sort((a, b) => b.pct - a.pct);
+  const anyRegressed = rows.some((r) => r.regressed);
+  const status: GateResult['status'] = anyRegressed
+    ? 'regression'
+    : method_changed
+      ? 'method-changed'
+      : 'ok';
+  return {
+    status,
+    tolerance,
+    current_method: current.token_method,
+    baseline_method: baseline.token_method,
+    rows,
+  };
+}
+
+function _fmtPct(p: number): string {
+  const sign = p >= 0 ? '+' : '';
+  return `${sign}${(p * 100).toFixed(1)}%`;
+}
+
+function main(argv: string[]): number {
+  let tolerance = DEFAULT_TOLERANCE;
+  let asJson = false;
+  let updateBaseline = false;
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a === '--json') asJson = true;
+    else if (a === '--update-baseline') updateBaseline = true;
+    else if (a === '--tolerance') {
+      const v = Number(argv[(i += 1)]);
+      if (!Number.isFinite(v) || v < 0) {
+        process.stderr.write(`error: invalid --tolerance\n`);
+        return 1;
+      }
+      tolerance = v;
+    } else if (a === '-h' || a === '--help') {
+      process.stdout.write(
+        'usage: check_token_regression [--json] [--tolerance F] [--update-baseline]\n',
+      );
+      return 0;
+    } else {
+      process.stderr.write(`error: unknown argument: ${a}\n`);
+      return 1;
+    }
+  }
+
+  const report = _readJson(PROJECTION_REPORT);
+  if (report === null) {
+    process.stderr.write(
+      `error: cannot read ${path.relative(REPO_ROOT, PROJECTION_REPORT)} — run \`task audit-tokens\` first\n`,
+    );
+    return 1;
+  }
+  const current = extract_metrics(report);
+
+  if (updateBaseline) {
+    const payload = JSON.stringify(current, null, 2) + '\n';
+    fs.writeFileSync(BASELINE_FILE, payload);
+    process.stdout.write(
+      `✅  token baseline written: ${path.relative(REPO_ROOT, BASELINE_FILE)} (method: ${current.token_method})\n`,
+    );
+    return 0;
+  }
+
+  const baselineRaw = _readJson(BASELINE_FILE);
+  const baseline: Metrics | null =
+    baselineRaw === null ? null : (baselineRaw as unknown as Metrics);
+  const result = evaluate(current, baseline, tolerance);
+
+  if (asJson) {
+    process.stdout.write(JSON.stringify(result, null, 2) + '\n');
+  } else if (result.status === 'warmup') {
+    process.stdout.write(
+      `⚠️  no token baseline yet — run \`./scripts-run src/scripts/check_token_regression --update-baseline\` to record one.\n`,
+    );
+  } else {
+    for (const r of result.rows) {
+      const mark = r.regressed ? '❌' : '✅';
+      process.stdout.write(
+        `${mark}  ${r.id}: ${r.current} vs baseline ${r.baseline} (${_fmtPct(r.pct)})\n`,
+      );
+    }
+    if (result.status === 'method-changed') {
+      process.stdout.write(
+        `⚠️  token_method changed (${result.baseline_method} → ${result.current_method}); ` +
+          `baseline reset is legitimate — run --update-baseline to re-anchor.\n`,
+      );
+    } else if (result.status === 'regression') {
+      process.stdout.write(
+        `❌  token regression: a projection grew >${(tolerance * 100).toFixed(0)}% over baseline.\n`,
+      );
+    } else {
+      process.stdout.write(
+        `✅  token surfaces within +${(tolerance * 100).toFixed(0)}% of baseline.\n`,
+      );
+    }
+  }
+
+  return result.status === 'regression' ? 2 : 0;
+}
+
+const _IS_MAIN =
+  process.argv[1] !== undefined &&
+  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
+
+if (_IS_MAIN) {
+  process.exit(main(process.argv.slice(2)));
+}
+
+export { extract_metrics, main };

package/src/scripts/cross_model_smoke.ts

@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+/**
+ * cross_model_smoke — the Phase-0 keystone (roadmap `road-to-operator-runtime-harvest`,
+ * T-004/T-006). Runs the existing trigger-routing eval across multiple host
+ * vendors (Anthropic / OpenAI / Gemini) over plain HTTP and emits, PER HOST:
+ *
+ *   - routing pass-rate + precision/recall (the parity signal),
+ *   - input/output tokens + cost estimate,
+ *   - output-shape parse-rate (did the host return parseable `would_load` JSON?
+ *     — the format-governance signal, T-004 hardening #2),
+ *   - negative-control catch-rate (did the eval correctly reject the
+ *     should-NOT-trigger queries? — the discrimination signal, T-003).
+ *
+ * Honesty couplings (3rd-pass council): the result names the coverage
+ * denominator (N skills / M fixtures) and the read of outcome (a)/(b)/(c) is
+ * left to a human in T-006 — this script reports, it does not declare parity.
+ *
+ * Modes:
+ *   --dry-run            use MockRouter for every host (no keys, no spend) — CI path.
+ *   --skills a,b,c       fixture skills to run (must have evals/triggers.json).
+ *   --hosts anthropic,openai,gemini   subset of hosts (default: all three).
+ *
+ * Live mode reads keys from ~/.event4u/agent-config/{anthropic,openai,gemini}.key
+ * and makes real, paid API calls — keep --skills small.
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+import {
+    type EvalResult,
+    type Query,
+    type SkillMeta,
+    type TriggerRouter,
+    MockRouter,
+    compute_metrics,
+    estimate_cost,
+    load_skill_metas,
+    load_triggers,
+} from './skill_trigger_eval.js';
+import {
+    AnthropicFetchRouter,
+    GeminiRouter,
+    OpenAiRouter,
+    loadKeyFromFile,
+} from './_lib/trigger_routers.js';
+
+type AsyncRouter = TriggerRouter & {
+    routeAsync?: (q: string, skills: SkillMeta[]) => Promise<[string[], number, number]>;
+};
+
+interface HostSummary {
+    host: string;
+    model: string;
+    queries: number;
+    passed: number;
+    pass_rate: number;
+    input_tokens: number;
+    output_tokens: number;
+    cost_usd_estimate: number;
+    // output-shape signal: fraction of calls that produced a parseable would_load
+    // (a host that returned 0 loaded AND 0 tokens is treated as an unparseable / failed call)
+    parse_ok: number;
+    parse_rate: number;
+    // discrimination signal: should-NOT-trigger queries correctly rejected
+    neg_controls: number;
+    neg_caught: number;
+}
+
+/** Mirror run_eval but await routeAsync when present (fetch routers); fall back to sync route (MockRouter). */
+async function runEvalAsync(
+    skill: string,
+    queries: Query[],
+    router: AsyncRouter,
+    skills: SkillMeta[],
+    model: string,
+): Promise<{ result: EvalResult; parse_ok: number; neg_controls: number; neg_caught: number }> {
+    const result: EvalResult = {
+        skill,
+        model,
+        timestamp: '',
+        router: router.name,
+        queries: [],
+        metrics: { true_positive: 0, false_positive: 0, true_negative: 0, false_negative: 0, precision: 0, recall: 0 },
+        input_tokens: 0,
+        output_tokens: 0,
+        cost_usd_estimate: 0.0,
+    };
+    let parse_ok = 0;
+    let neg_controls = 0;
+    let neg_caught = 0;
+    for (const q of queries) {
+        const [loaded, inTok, outTok] =
+            typeof router.routeAsync === 'function'
+                ? await router.routeAsync(q.q, skills)
+                : router.route(q.q, skills);
+        const observed = loaded.includes(skill);
+        const passed = observed === q.trigger;
+        // a call that returned at least one token OR a non-empty load is treated as parseable
+        if (inTok > 0 || outTok > 0 || loaded.length > 0) parse_ok += 1;
+        if (!q.trigger) {
+            neg_controls += 1;
+            if (passed) neg_caught += 1; // correctly rejected a should-not query
+        }
+        result.queries.push({ q: q.q, expected: q.trigger, observed, loaded_skills: [...loaded], passed });
+        result.input_tokens += inTok;
+        result.output_tokens += outTok;
+    }
+    result.metrics = compute_metrics(result.queries);
+    result.cost_usd_estimate = estimate_cost(model, result.input_tokens, result.output_tokens);
+    return { result, parse_ok, neg_controls, neg_caught };
+}
+
+function buildRouter(host: string, dryRun: boolean): AsyncRouter {
+    if (dryRun) {
+        // Deterministic stub: loads a skill iff the query mentions its name.
+        return new MockRouter((q, skills) => skills.filter((s) => q.toLowerCase().includes(s.name)).map((s) => s.name)) as AsyncRouter;
+    }
+    if (host === 'openai') return new OpenAiRouter({ apiKey: loadKeyFromFile('openai.key') }) as AsyncRouter;
+    if (host === 'gemini') return new GeminiRouter({ apiKey: loadKeyFromFile('gemini.key') }) as AsyncRouter;
+    if (host === 'anthropic') return new AnthropicFetchRouter({ apiKey: loadKeyFromFile('anthropic.key') }) as AsyncRouter;
+    throw new Error(`unknown host: ${host}`);
+}
+
+function modelFor(host: string): string {
+    if (host === 'openai') return 'gpt-4o-mini';
+    if (host === 'gemini') return 'gemini-2.5-flash';
+    if (host === 'anthropic') return 'claude-haiku-4-5-20251001';
+    return 'mock';
+}
+
+export async function runSmoke(opts: {
+    skills: string[];
+    hosts: string[];
+    dryRun: boolean;
+}): Promise<{ catalogue_size: number; fixtures: number; hosts: HostSummary[] }> {
+    // Read the catalogue from the source-of-truth `src/skills` tree explicitly.
+    // load_skill_metas' built-in default points at a legacy tree that is absent
+    // in fresh clones / CI, so we pass the path rather than rely on the default.
+    // Keeps the catalogue and the fixture paths on the same tree.
+    const catalogue = load_skill_metas(path.join('src', 'skills'));
+    const summaries: HostSummary[] = [];
+    let totalFixtures = 0;
+
+    for (const host of opts.hosts) {
+        const router = buildRouter(host, opts.dryRun);
+        const model = opts.dryRun ? 'mock' : modelFor(host);
+        let q = 0,
+            passed = 0,
+            inTok = 0,
+            outTok = 0,
+            cost = 0,
+            parseOk = 0,
+            negC = 0,
+            negCaught = 0,
+            fixtures = 0;
+        for (const skill of opts.skills) {
+            const trigPath = path.join('src', 'skills', skill, 'evals', 'triggers.json');
+            if (!fs.existsSync(trigPath)) continue;
+            // Some fixtures use a legacy shape (`should_trigger`/`should_not_trigger`)
+            // instead of the unified `queries[]`. Skip-and-warn rather than crash the
+            // whole run — and never silently: the skip is surfaced on stderr.
+            let skillName: string;
+            let queries: Query[];
+            try {
+                [skillName, queries] = load_triggers(trigPath);
+            } catch (e) {
+                process.stderr.write(`skip ${skill}: unparseable triggers.json — ${(e as Error).message}\n`);
+                continue;
+            }
+            fixtures += 1;
+            const r = await runEvalAsync(skillName, queries, router, catalogue, model);
+            q += r.result.queries.length;
+            passed += r.result.queries.filter((x) => x.passed).length;
+            inTok += r.result.input_tokens;
+            outTok += r.result.output_tokens;
+            cost += r.result.cost_usd_estimate;
+            parseOk += r.parse_ok;
+            negC += r.neg_controls;
+            negCaught += r.neg_caught;
+        }
+        totalFixtures = fixtures;
+        summaries.push({
+            host,
+            model,
+            queries: q,
+            passed,
+            pass_rate: q ? passed / q : 0,
+            input_tokens: inTok,
+            output_tokens: outTok,
+            cost_usd_estimate: cost,
+            parse_ok: parseOk,
+            parse_rate: q ? parseOk / q : 0,
+            neg_controls: negC,
+            neg_caught: negCaught,
+        });
+    }
+    return { catalogue_size: catalogue.length, fixtures: totalFixtures, hosts: summaries };
+}
+
+function parseArgs(argv: string[]): { skills: string[]; hosts: string[]; dryRun: boolean } {
+    let skills = ['image-analyser'];
+    let hosts = ['anthropic', 'openai', 'gemini'];
+    let dryRun = false;
+    for (let i = 0; i < argv.length; i++) {
+        if (argv[i] === '--dry-run') dryRun = true;
+        else if (argv[i] === '--skills') skills = (argv[++i] ?? '').split(',').filter(Boolean);
+        else if (argv[i] === '--hosts') hosts = (argv[++i] ?? '').split(',').filter(Boolean);
+    }
+    return { skills, hosts, dryRun };
+}
+
+async function main(): Promise<number> {
+    const opts = parseArgs(process.argv.slice(2));
+    const out = await runSmoke(opts);
+    const lines: string[] = [];
+    lines.push(`cross_model_smoke · catalogue=${out.catalogue_size} skills · fixtures=${out.fixtures} · ${opts.dryRun ? 'DRY-RUN (mock)' : 'LIVE'}`);
+    lines.push('host        model                       queries  pass%   parse%  neg-ctrl  in_tok   out_tok  ~$');
+    for (const h of out.hosts) {
+        lines.push(
+            `${h.host.padEnd(11)} ${h.model.padEnd(27)} ${String(h.queries).padStart(7)}  ${(h.pass_rate * 100).toFixed(0).padStart(5)}  ${(h.parse_rate * 100).toFixed(0).padStart(6)}  ${String(h.neg_caught + '/' + h.neg_controls).padStart(8)}  ${String(h.input_tokens).padStart(7)}  ${String(h.output_tokens).padStart(7)}  ${h.cost_usd_estimate.toFixed(4)}`,
+        );
+    }
+    process.stdout.write(lines.join('\n') + '\n');
+    return 0;
+}
+
+// Entry guard: only run main when invoked directly (not when imported by tests).
+const _invokedDirectly = (() => {
+    try {
+        const argv1 = process.argv[1];
+        if (!argv1) return false;
+        return fs.realpathSync(argv1) === fs.realpathSync(new URL(import.meta.url).pathname);
+    } catch {
+        return false;
+    }
+})();
+if (_invokedDirectly) {
+    main().then(
+        (code) => process.exit(code),
+        (err) => {
+            process.stderr.write(String(err?.stack ?? err) + '\n');
+            process.exit(1);
+        },
+    );
+}

package/src/scripts/hook_manifest.yaml

@@ -64,6 +64,18 @@ concerns:
     script: src/scripts/hooks/block_no_verify.ts
     args: []
     fail_closed: true
+  # token-saving Phase 3 — PreToolUse RTK-wrap nudge. DETERMINISTIC: keys off a
+  # live `which rtk` probe (not the self-reported flag). When rtk is installed
+  # and the agent is about to run a single verbose CLI command that is NOT
+  # completeness-critical, warns "re-run wrapped with rtk" (60–90% fewer output
+  # tokens). Warn only — never blocks (the v1 contract has no transparent
+  # rewrite); never fires when rtk is absent (silent plain-command fallback).
+  # Default-OFF (hooks.rtk_wrap.enabled). fail_closed: false — never break a
+  # command for a token optimization.
+  rtk-wrap:
+    script: src/scripts/hooks/rtk_wrap_hook.ts
+    args: []
+    fail_closed: false
   # Phase 2 of road-to-hooks-actually-fire-in-consumers — session_start
   # gate that surfaces the marketplace-install-but-unscaffolded shape.
   # Writes .augment/.first-run-action-needed.md + one stderr line so
@@ -96,7 +108,7 @@ platforms:
     session_start:    [chat-history, first-run-gate, onboarding-gate, verify-before-complete, minimal-safe-diff, profile-staleness, wrapper-freshness]
     session_end:      [chat-history]
     stop:             [chat-history, verify-before-complete]
-    pre_tool_use:     [block-no-verify]
+    pre_tool_use:     [block-no-verify, rtk-wrap]
     post_tool_use:    [chat-history, roadmap-progress, context-hygiene, verify-before-complete, minimal-safe-diff, injection-scan]
 
   claude:
@@ -104,7 +116,7 @@ platforms:
     session_end:      [chat-history]
     stop:             [chat-history, verify-before-complete]
     user_prompt_submit: [chat-history, verify-before-complete, minimal-safe-diff]
-    pre_tool_use:     [block-no-verify]
+    pre_tool_use:     [block-no-verify, rtk-wrap]
     post_tool_use:    [chat-history, roadmap-progress, context-hygiene, verify-before-complete, minimal-safe-diff, injection-scan]
 
   # Cowork — the Claude desktop app's local-agent-mode runtime, built
@@ -126,7 +138,7 @@ platforms:
     session_end:      [chat-history]
     stop:             [chat-history, verify-before-complete]
     user_prompt_submit: [chat-history, verify-before-complete, minimal-safe-diff]
-    pre_tool_use:     [block-no-verify]
+    pre_tool_use:     [block-no-verify, rtk-wrap]
     post_tool_use:    [chat-history, roadmap-progress, context-hygiene, verify-before-complete, minimal-safe-diff, injection-scan]
 
   # Phase 7.5 — Cursor. `.cursor/hooks.json` (project) is read by the