@event4u/agent-config 8.8.0 → 8.10.0

package/src/scripts/mcp_server/tools.ts

@@ -477,6 +477,18 @@ async function _memoryLookupHandler(
         throw new Error("'detail' must be 'index' or 'full'");
     }
 
+    // B1 (road-to-retrieval-substrate-hardening): optional token budget →
+    // compact one-line rows, hard-cut with a truncation hint. Absent → the
+    // envelope is unchanged (v1 default-full contract holds).
+    const opts: { detail: 'full' | 'index'; token_budget?: number } = { detail: detailRaw };
+    if (args.token_budget !== undefined) {
+        const tb = args.token_budget;
+        if (typeof tb !== 'number' || !Number.isInteger(tb) || typeof tb === 'boolean' || tb < 1) {
+            throw new Error("'token_budget' must be a positive integer");
+        }
+        opts.token_budget = tb;
+    }
+
     const prevCwd = process.cwd();
     let envelope: Record<string, unknown>;
     try {
@@ -485,7 +497,7 @@ async function _memoryLookupHandler(
             [...(types as string[])],
             [...(keys as string[])],
             limitRaw,
-            { detail: detailRaw },
+            opts,
         );
     } finally {
         process.chdir(prevCwd);
@@ -1400,6 +1412,17 @@ export const ALLOWLIST: Record<string, BuiltinTool> = {
                         "this first, then memory_get the ids you need. " +
                         "'full' (default) returns complete entries.",
                 },
+                token_budget: {
+                    type: 'integer',
+                    minimum: 1,
+                    description:
+                        'Optional token budget. When set, entries are ' +
+                        'rendered as one-line compact rows (id, type, ' +
+                        'confidence, `line`) and the row set is hard-cut at ' +
+                        'token_budget × 4 chars; omitted hits appear as a ' +
+                        'top-level `truncation` hint naming a concrete next ' +
+                        'step. Absent → the envelope is unchanged.',
+                },
             },
             required: ['types'],
             additionalProperties: false,

package/src/scripts/measure_lexical_ranking.ts

@@ -0,0 +1,173 @@
+#!/usr/bin/env tsx
+/**
+ * Ranking-lift measurement for the hand-rolled lexical index
+ * (road-to-retrieval-substrate-hardening B2, deterministic + free).
+ *
+ * The gate on shipping the index (roadmap Phase 2): it must measurably rank
+ * BETTER than the current `_score` bucket scorer, else honest-null. This
+ * script runs both scorers over the SAME retrieval-precision corpus + store
+ * (internal/bench/second-brain/retrieval-store/) and reports, per task and in
+ * aggregate:
+ *
+ *   - mean top tie-set size — how many entries share the TOP score. `_score`
+ *     gives 0.8 to any keyword match, so the needed entry ties with its
+ *     confusers (the "recalls but does not rank" finding, mean 3.3). A good
+ *     ranker drives this toward 1.
+ *   - precision@1 / @k — was the NEEDED entry actually first / in the top-k.
+ *
+ * No model, no spend: pure retrieval scoring. Exit 0 ok / 1 error / 2 usage.
+ *
+ * Usage: measure_lexical_ranking.ts [--write] [--format text|json]
+ *   --write  persist the report to internal/bench/reports/lexical-ranking.json
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import * as YAML from 'yaml';
+
+import { LexicalIndex } from './_lib/lexical_index.js';
+import { _score, _setIntakeRoot, _setKnowledgeRoot, _setMemoryRoot, retrieve } from './memory_lookup.js';
+
+const _HERE = fileURLToPath(import.meta.url);
+export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+export const STORE_DIR = path.join(REPO_ROOT, 'internal', 'bench', 'second-brain', 'retrieval-store');
+export const CORPUS = path.join(STORE_DIR, 'retrieval-corpus.yml');
+export const REPORT = path.join(REPO_ROOT, 'internal', 'bench', 'reports', 'lexical-ranking.json');
+
+interface Task {
+    id: string;
+    query: string[];
+    needed: string;
+}
+
+interface TaskResult {
+    id: string;
+    needed: string;
+    baseline: { rank: number; tieSet: number };
+    index: { rank: number; tieSet: number };
+}
+
+interface RankingSummary {
+    tasks: number;
+    k: number;
+    baseline_mean_tie_set: number;
+    index_mean_tie_set: number;
+    baseline_precision_at_1: number;
+    index_precision_at_1: number;
+    baseline_precision_at_k: number;
+    index_precision_at_k: number;
+}
+
+/** Text an entry contributes to the index — the fields `_score` also reads. */
+function entryText(entry: Record<string, unknown>): string {
+    const parts: string[] = [];
+    for (const f of ['path', 'key', 'symptom', 'feature', 'rule', 'body']) {
+        const v = entry[f];
+        if (typeof v === 'string') parts.push(v);
+        else if (Array.isArray(v)) for (const x of v) parts.push(String(x));
+    }
+    return parts.join(' | ');
+}
+
+/** Top tie-set size + rank of the needed id in a sorted (id, score) list. */
+function diagnose(ranked: Array<{ id: string; score: number }>, needed: string): { rank: number; tieSet: number } {
+    if (ranked.length === 0) return { rank: -1, tieSet: 0 };
+    const top = ranked[0]?.score ?? 0;
+    const tieSet = ranked.filter((r) => r.score === top).length;
+    const rank = ranked.findIndex((r) => r.id === needed);
+    return { rank, tieSet };
+}
+
+export function measure(): { tasks: TaskResult[]; summary: RankingSummary } {
+    _setMemoryRoot(STORE_DIR);
+    _setKnowledgeRoot(path.join(STORE_DIR, 'knowledge-none'));
+    _setIntakeRoot(path.join(STORE_DIR, 'intake-none'));
+
+    const doc = YAML.parse(fs.readFileSync(CORPUS, 'utf-8')) as { type: string; k: number; tasks: Task[] };
+    const type = doc.type;
+    const k = doc.k;
+    const results: TaskResult[] = [];
+
+    for (const task of doc.tasks) {
+        // Shared universe: every store entry `_score` considers relevant.
+        const hits = retrieve([type], task.query, 100);
+
+        // Baseline — rank by the current bucket scorer, ties broken by the
+        // order `retrieve` already returns (store order), exactly as today.
+        const baselineRanked = hits.map((h) => ({ id: h.id, score: h.score }));
+        const baseline = diagnose(baselineRanked, task.needed);
+
+        // Index — BM25 over the same entries' text.
+        const index = new LexicalIndex(hits.map((h) => ({ id: h.id, text: entryText(h.entry) })));
+        const indexRanked = index.rank(task.query);
+        const idx = diagnose(indexRanked, task.needed);
+
+        results.push({ id: task.id, needed: task.needed, baseline, index: idx });
+        void _score; // referenced for provenance — retrieve() applies it internally
+    }
+
+    const n = results.length;
+    const mean = (sel: (r: TaskResult) => number): number =>
+        n === 0 ? 0 : Math.round((results.reduce((a, r) => a + sel(r), 0) / n) * 1000) / 1000;
+    const precAt = (sel: (r: TaskResult) => number, kk: number): number =>
+        n === 0 ? 0 : Math.round((results.filter((r) => sel(r) >= 0 && sel(r) < kk).length / n) * 1000) / 1000;
+
+    const summary = {
+        tasks: n,
+        k,
+        baseline_mean_tie_set: mean((r) => r.baseline.tieSet),
+        index_mean_tie_set: mean((r) => r.index.tieSet),
+        baseline_precision_at_1: precAt((r) => r.baseline.rank, 1),
+        index_precision_at_1: precAt((r) => r.index.rank, 1),
+        baseline_precision_at_k: precAt((r) => r.baseline.rank, k),
+        index_precision_at_k: precAt((r) => r.index.rank, k),
+    };
+    return { tasks: results, summary };
+}
+
+export function main(argv: string[]): number {
+    let write = false;
+    let format: 'text' | 'json' = 'text';
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i] as string;
+        if (a === '--write') write = true;
+        else if (a === '--format') format = (argv[++i] as 'text' | 'json') ?? 'text';
+        else if (a === '-h' || a === '--help') {
+            process.stdout.write('usage: measure_lexical_ranking.ts [--write] [--format text|json]\n');
+            return 0;
+        } else {
+            process.stderr.write(`measure_lexical_ranking: unknown argument ${a}\n`);
+            return 2;
+        }
+    }
+
+    let out: { tasks: TaskResult[]; summary: RankingSummary };
+    try {
+        out = measure();
+    } catch (exc) {
+        process.stderr.write(`measure_lexical_ranking: error: ${String(exc)}\n`);
+        return 1;
+    }
+
+    if (write) {
+        fs.mkdirSync(path.dirname(REPORT), { recursive: true });
+        fs.writeFileSync(REPORT, JSON.stringify({ schema_version: 1, ...out }, null, 2) + '\n');
+    }
+    if (format === 'json') {
+        process.stdout.write(JSON.stringify(out, null, 2) + '\n');
+        return 0;
+    }
+    const s = out.summary;
+    process.stdout.write(
+        `lexical-ranking (${s.tasks} tasks, k=${s.k}):\n` +
+            `  mean top tie-set   baseline=${s.baseline_mean_tie_set}  index=${s.index_mean_tie_set}\n` +
+            `  precision@1        baseline=${s.baseline_precision_at_1}  index=${s.index_precision_at_1}\n` +
+            `  precision@${s.k}        baseline=${s.baseline_precision_at_k}  index=${s.index_precision_at_k}\n`,
+    );
+    return 0;
+}
+
+if (fileURLToPath(import.meta.url) === path.resolve(process.argv[1] ?? '')) {
+    process.exit(main(process.argv.slice(2)));
+}

package/src/scripts/orchestration_record.ts

@@ -0,0 +1,156 @@
+/**
+ * Orchestration telemetry recorder — CLI.
+ *
+ * Appends ONE validated audit-log-v1 line (with an `orchestration` object) to
+ * the monthly audit log, from the counts the orchestrating agent has after an
+ * auto-dispatch. One command instead of hand-authored JSON — the reliable,
+ * lock-respecting (2026-06-30 no-hook decision) capture path.
+ *
+ * Usage:
+ *   ./scripts-run src/scripts/orchestration_record \
+ *     --spawn-count 1 --token-delta -72000 --provenance measured \
+ *     --tier-chosen lite --tier-source inferred --task-class read-only-fanout \
+ *     [--tiers sonnet,opus] [--wall-clock-ms 18500] [--dispatch-outcome DONE] \
+ *     [--first-pass-success true|false] [--escalated true|false] \
+ *     [--dir <audit-dir>] [--dry-run]
+ *
+ * Read by `src/scripts/orchestration_savings_report.ts`.
+ */
+
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import {
+    buildOrchestrationLine,
+    type Provenance,
+    type TierChosen,
+    type TierSource,
+    type DispatchOutcome,
+    type VerifyMode,
+    type Band,
+    type LinePhase,
+    type LineOutcome,
+    type RecordInput,
+} from './_lib/orchestration_record.js';
+
+const DEFAULT_DIR = 'agents/runtime/state/audit';
+
+type Flags = Record<string, string | boolean>;
+
+function parseFlags(argv: string[]): Flags {
+    const flags: Flags = {};
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === undefined || !a.startsWith('--')) continue;
+        const eq = a.indexOf('=');
+        if (eq !== -1) {
+            flags[a.slice(2, eq)] = a.slice(eq + 1);
+            continue;
+        }
+        const key = a.slice(2);
+        const next = argv[i + 1];
+        if (next !== undefined && !next.startsWith('--')) {
+            flags[key] = next;
+            i++;
+        } else {
+            flags[key] = true; // boolean flag (e.g. --dry-run)
+        }
+    }
+    return flags;
+}
+
+function str(flags: Flags, key: string): string | undefined {
+    const v = flags[key];
+    return typeof v === 'string' ? v : undefined;
+}
+
+/**
+ * Parse an explicit boolean flag value. `--key true|false` → boolean; a bare
+ * `--key` counts as `true`; any other value is passed through so the lib's
+ * non-boolean validation rejects it with a clear error.
+ */
+function bool(flags: Flags, key: string): boolean | undefined {
+    const v = flags[key];
+    if (v === undefined) return undefined;
+    if (v === true || v === 'true') return true;
+    if (v === 'false') return false;
+    return v as unknown as boolean; // invalid string → caught by the lib's typeof validation
+}
+
+function int(flags: Flags, key: string): number | undefined {
+    const v = str(flags, key);
+    if (v === undefined) return undefined;
+    const n = Number(v);
+    return Number.isFinite(n) ? n : NaN; // NaN → caught by the lib's isInt validation
+}
+
+/** Deterministic content-hash id (contract allows "ULID or content hash"). */
+function contentHashId(input: RecordInput): string {
+    const canonical = JSON.stringify({
+        ts: input.ts,
+        spawn_count: input.spawn_count,
+        token_delta: input.token_delta,
+        tier_chosen: input.tier_chosen ?? null,
+        task_class: input.task_class ?? null,
+    });
+    return crypto.createHash('sha256').update(canonical).digest('hex').slice(0, 26).toUpperCase();
+}
+
+export function main(argv: string[] = process.argv.slice(2)): number {
+    const flags = parseFlags(argv);
+    const dir = str(flags, 'dir') ?? DEFAULT_DIR;
+    const dryRun = flags['dry-run'] === true;
+    const ts = new Date().toISOString();
+
+    const partial: Omit<RecordInput, 'id'> = {
+        spawn_count: int(flags, 'spawn-count') ?? NaN,
+        token_delta: int(flags, 'token-delta') ?? NaN,
+        token_delta_provenance: str(flags, 'provenance') as Provenance | undefined,
+        tiers: str(flags, 'tiers')?.split(',').map((s) => s.trim()).filter(Boolean),
+        tier_chosen: str(flags, 'tier-chosen') as TierChosen | undefined,
+        tier_source: str(flags, 'tier-source') as TierSource | undefined,
+        task_class: str(flags, 'task-class'),
+        task_size_estimate: int(flags, 'task-size-estimate'),
+        wall_clock_ms: int(flags, 'wall-clock-ms'),
+        dispatch_tokens: int(flags, 'dispatch-tokens'),
+        session_tier: str(flags, 'session-tier'),
+        dispatch_outcome: str(flags, 'dispatch-outcome') as DispatchOutcome | undefined,
+        verify_mode: str(flags, 'verify-mode') as VerifyMode | undefined,
+        first_pass_success: bool(flags, 'first-pass-success'),
+        escalated: bool(flags, 'escalated'),
+        phase: str(flags, 'phase') as LinePhase | undefined,
+        outcome: str(flags, 'outcome') as LineOutcome | undefined,
+        confidence_band: str(flags, 'confidence-band') as Band | undefined,
+        risk_class: str(flags, 'risk-class') as Band | undefined,
+        persona: str(flags, 'persona'),
+        work_id: str(flags, 'work-id'),
+        ts,
+    };
+    const input: RecordInput = { ...partial, id: contentHashId({ ...partial, id: '' }) };
+
+    const { line, errors } = buildOrchestrationLine(input);
+    if (errors.length || line === null) {
+        process.stderr.write('orchestration_record: refused — invalid input:\n');
+        for (const e of errors) process.stderr.write(`  - ${e}\n`);
+        return 1;
+    }
+
+    const serialized = JSON.stringify(line);
+    if (dryRun) {
+        process.stdout.write(serialized + '\n');
+        return 0;
+    }
+
+    fs.mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, `${ts.slice(0, 7)}.jsonl`);
+    fs.appendFileSync(file, serialized + '\n', 'utf8');
+    process.stdout.write(`orchestration telemetry recorded → ${file} (id ${line.id})\n`);
+    return 0;
+}
+
+if (process.argv[1] !== undefined) {
+    const invokedUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+    if (import.meta.url === invokedUrl) process.exit(main());
+}

package/src/scripts/orchestration_savings_report.ts

@@ -5,11 +5,18 @@
  * and prints an aggregate token-savings report. Read-only; no network, no
  * mutation. Handles a missing/empty audit dir gracefully.
  *
+ * QUALITY × COST PAIRING (council verdict): the savings figures never render
+ * without the quality columns (first_pass_success_rate / escalation_rate).
+ * When ≥ 20 lines in the window carry the quality fields the real rates
+ * render; below 20 the columns render as "n/a (n=<count>)" beside savings.
+ *
  * Usage:
- *   ./scripts-run src/scripts/orchestration_savings_report [--dir <path>] [--format text|json]
+ *   ./scripts-run src/scripts/orchestration_savings_report \
+ *     [--dir <path>] [--format text|json] [--weights lite=1,medium=5,high=15]
  *
  * Default dir: agents/runtime/state/audit (the audit-log-v1 path where the
- * orchestration layer writes telemetry lines). See
+ * orchestration layer writes telemetry lines). `--weights` tunes the
+ * provider-neutral tier weights behind the MODELED cost-%. See
  * `src/agent-src/contexts/execution/orchestration-telemetry.md`.
  */
 
@@ -17,17 +24,30 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { pathToFileURL } from 'node:url';
 
-import { aggregateOrchestrationSavings, type AuditLine, type SavingsReport } from './_lib/orchestration_savings.js';
+import { aggregateOrchestrationSavings, DEFAULT_TIER_WEIGHTS, type AuditLine, type SavingsReport, type TierWeights } from './_lib/orchestration_savings.js';
 
 const DEFAULT_DIR = 'agents/runtime/state/audit';
 
 interface Options {
     dir: string;
     format: 'text' | 'json';
+    weights: TierWeights;
+}
+
+/** Parse `lite=1,medium=5,high=15` → weight map. Returns undefined if nothing valid parsed. */
+function parseWeights(spec: string | undefined): TierWeights | undefined {
+    if (!spec) return undefined;
+    const w: TierWeights = {};
+    for (const pair of spec.split(',')) {
+        const [k, v] = pair.split('=');
+        const n = Number(v);
+        if (k && Number.isFinite(n)) w[k.trim()] = n;
+    }
+    return Object.keys(w).length ? w : undefined;
 }
 
 function parseArgs(argv: string[]): Options {
-    const opts: Options = { dir: DEFAULT_DIR, format: 'text' };
+    const opts: Options = { dir: DEFAULT_DIR, format: 'text', weights: DEFAULT_TIER_WEIGHTS };
     for (let i = 0; i < argv.length; i++) {
         const a = argv[i];
         if (a === undefined) continue;
@@ -35,6 +55,8 @@ function parseArgs(argv: string[]): Options {
         else if (a.startsWith('--dir=')) opts.dir = a.slice('--dir='.length);
         else if (a === '--format') opts.format = (argv[++i] as Options['format']) ?? opts.format;
         else if (a.startsWith('--format=')) opts.format = a.slice('--format='.length) as Options['format'];
+        else if (a === '--weights') opts.weights = parseWeights(argv[++i]) ?? opts.weights;
+        else if (a.startsWith('--weights=')) opts.weights = parseWeights(a.slice('--weights='.length)) ?? opts.weights;
     }
     if (opts.format !== 'text' && opts.format !== 'json') opts.format = 'text';
     return opts;
@@ -60,13 +82,41 @@ export function readAuditLines(dir: string): AuditLine[] {
     return lines;
 }
 
-function renderText(r: SavingsReport, dir: string): string {
+/** Dispatches carrying the quality fields at/above this render the real rates; below, "n/a (n=<count>)". */
+export const QUALITY_GATE_MIN_LINES = 20;
+
+function pct(v: number): string {
+    return `${(v * 100).toFixed(0)}%`;
+}
+
+/**
+ * Render the paired cost × quality block. Council verdict: savings NEVER render
+ * without the quality columns — at n ≥ QUALITY_GATE_MIN_LINES the real rates
+ * render; below, the quality columns render as "n/a (n=<count>)" alongside the
+ * savings figures. There is no savings-only render path.
+ */
+function renderPairedCostQuality(r: SavingsReport): string[] {
+    const q = r.quality;
+    const gated = q.quality_lines >= QUALITY_GATE_MIN_LINES;
+    const fps = gated && q.first_pass_success_rate !== null
+        ? `${pct(q.first_pass_success_rate)} (n=${q.first_pass_lines})`
+        : `n/a (n=${q.first_pass_lines})`;
+    const esc = gated && q.escalation_rate !== null
+        ? `${pct(q.escalation_rate)} (n=${q.escalated_lines})`
+        : `n/a (n=${q.escalated_lines})`;
+    return [
+        '  cost × quality (paired — never savings alone):',
+        `    net token_delta: ${r.net_token_delta} (negative = net saved)   | first_pass_success_rate: ${fps}`,
+        `    tokens saved: ${r.tokens_saved} · tokens added: ${r.tokens_added}   | escalation_rate: ${esc}`,
+    ];
+}
+
+export function renderText(r: SavingsReport, dir: string): string {
     const out: string[] = [];
     out.push('Orchestration savings report');
     out.push(`  source: ${dir}`);
     out.push(`  dispatches: ${r.dispatches}  (total spawns: ${r.total_spawns})`);
-    out.push(`  net token_delta: ${r.net_token_delta}  (negative = net saved)`);
-    out.push(`  tokens saved: ${r.tokens_saved}   tokens added: ${r.tokens_added}`);
+    out.push(...renderPairedCostQuality(r));
     out.push(`  provenance: measured ${r.by_provenance.measured.dispatches} (Δ ${r.by_provenance.measured.net_token_delta}) · estimated ${r.by_provenance.estimated.dispatches} (Δ ${r.by_provenance.estimated.net_token_delta})`);
     out.push(`  measured share: ${(r.measured_share * 100).toFixed(0)}%`);
     if (Object.keys(r.by_tier).length) {
@@ -77,6 +127,13 @@ function renderText(r: SavingsReport, dir: string): string {
         out.push('  by task_class (Δ tokens):');
         for (const [cls, d] of Object.entries(r.by_task_class).sort()) out.push(`    ${cls}: ${d}`);
     }
+    const mc = r.modeled_cost;
+    if (mc.cost_reduction_pct !== null) {
+        out.push(`  MODELED cost reduction: ${(mc.cost_reduction_pct * 100).toFixed(0)}%  (over ${mc.covered_dispatches}/${r.dispatches} dispatch(es) with tier data)`);
+        out.push(`    weights ${JSON.stringify(mc.weights)} · baseline ${mc.baseline_cost_units} → delegated ${mc.delegated_cost_units} cost-units · MODEL, not measured $`);
+    } else {
+        out.push('  MODELED cost reduction: n/a (needs dispatch_tokens + session_tier + tier_chosen on a dispatch)');
+    }
     out.push('  notes:');
     for (const n of r.notes) out.push(`    - ${n}`);
     return out.join('\n');
@@ -85,7 +142,7 @@ function renderText(r: SavingsReport, dir: string): string {
 export function main(argv: string[] = process.argv.slice(2)): number {
     const opts = parseArgs(argv);
     const lines = readAuditLines(opts.dir);
-    const report = aggregateOrchestrationSavings(lines);
+    const report = aggregateOrchestrationSavings(lines, opts.weights);
     if (opts.format === 'json') {
         process.stdout.write(JSON.stringify({ ...report, source: opts.dir }, null, 2) + '\n');
     } else {

package/src/scripts/schemas/skill.schema.json

@@ -13,6 +13,10 @@
       "pattern": "^[a-z][a-z0-9-]*$",
       "description": "Lowercase, hyphenated slug; must match the parent directory name."
     },
+    "user-invocable": {
+      "type": "boolean",
+      "description": "Claude Code Agent-Skills extension: false = the skill claims no /name slash command (only the model can load it). REQUIRED (false) when name equals a Claude Code built-in command or bundled skill (see scripts/_lib/claude_builtin_names.ts) so the suite never shadows the host's own /name. Other tools ignore the key."
+    },
     "description": {
       "type": "string",
       "minLength": 1,