@event4u/agent-config 8.6.0 → 8.8.0

package/src/scripts/_lib/orchestration_savings.ts

@@ -0,0 +1,127 @@
+/**
+ * Orchestration savings aggregator (pure, no-I/O).
+ *
+ * Aggregates realized orchestration telemetry into a token-savings report.
+ * Consumes the `orchestration` sub-object of audit-log-v1 JSONL lines
+ * (contract: `src/agent-src/contexts/execution/orchestration-telemetry.md`).
+ *
+ * HONEST LIMIT — the telemetry records `token_delta` (net dispatch cost vs the
+ * in-session baseline; negative = saved) but NOT the absolute baseline. A clean
+ * "% of session saved" is therefore NOT derivable from this data alone — only
+ * ABSOLUTE net tokens saved, plus the measured-vs-estimated provenance split.
+ * A percentage would need an absolute-baseline field on the telemetry (see the
+ * report `notes` and the PR's follow-up section).
+ */
+
+export type TokenDeltaProvenance = 'measured' | 'estimated';
+
+/** The `orchestration` sub-object shape we read (subset — additive fields only). */
+export interface OrchestrationRecord {
+    spawn_count?: number;
+    token_delta?: number;
+    token_delta_provenance?: TokenDeltaProvenance;
+    tier_chosen?: string | null;
+    task_class?: string | null;
+    outcome?: string;
+}
+
+/** A parsed audit-log-v1 line (only the fields this aggregator touches). */
+export interface AuditLine {
+    input_kind?: string;
+    orchestration?: OrchestrationRecord;
+}
+
+export interface ProvenanceStat {
+    dispatches: number;
+    /** Σ token_delta for this provenance (negative = net saved). */
+    net_token_delta: number;
+}
+
+export interface SavingsReport {
+    /** Lines carrying an orchestration object with spawn_count > 0. */
+    dispatches: number;
+    total_spawns: number;
+    /** Σ token_delta across all dispatches (negative = net saved). */
+    net_token_delta: number;
+    /** Magnitude of tokens saved (sum of negative deltas, as a positive number). */
+    tokens_saved: number;
+    /** Sum of positive deltas (orchestration that cost MORE than baseline). */
+    tokens_added: number;
+    by_provenance: { measured: ProvenanceStat; estimated: ProvenanceStat };
+    /** tier_chosen → Σ token_delta (negative = net saved on that tier). */
+    by_tier: Record<string, number>;
+    /** task_class → Σ token_delta. */
+    by_task_class: Record<string, number>;
+    /** measured dispatches / dispatches, 0..1 (0 when no dispatches). */
+    measured_share: number;
+    /** Honest caveats about what the number does and does not mean. */
+    notes: string[];
+}
+
+function emptyProvenance(): ProvenanceStat {
+    return { dispatches: 0, net_token_delta: 0 };
+}
+
+/**
+ * Aggregate orchestration telemetry lines into a savings report.
+ *
+ * A line counts as a dispatch only when it carries an `orchestration` object
+ * with `spawn_count > 0` (spawn_count 0 / absent = handled in-session, not a
+ * dispatch). Unknown/absent fields default safely (token_delta → 0, provenance
+ * → estimated) so a pre-extension line never throws.
+ */
+export function aggregateOrchestrationSavings(lines: AuditLine[]): SavingsReport {
+    const report: SavingsReport = {
+        dispatches: 0,
+        total_spawns: 0,
+        net_token_delta: 0,
+        tokens_saved: 0,
+        tokens_added: 0,
+        by_provenance: { measured: emptyProvenance(), estimated: emptyProvenance() },
+        by_tier: {},
+        by_task_class: {},
+        measured_share: 0,
+        notes: [],
+    };
+
+    for (const line of lines) {
+        const o = line.orchestration;
+        if (!o) continue;
+        const spawns = o.spawn_count ?? 0;
+        if (!(spawns > 0)) continue; // in-session, not a dispatch
+
+        report.dispatches += 1;
+        report.total_spawns += spawns;
+
+        const delta = o.token_delta ?? 0;
+        report.net_token_delta += delta;
+        if (delta < 0) report.tokens_saved += -delta;
+        else report.tokens_added += delta;
+
+        // Provenance defaults to `estimated` when unstated (matches the
+        // telemetry contract: estimated is the lossy fallback).
+        const prov: TokenDeltaProvenance = o.token_delta_provenance === 'measured' ? 'measured' : 'estimated';
+        report.by_provenance[prov].dispatches += 1;
+        report.by_provenance[prov].net_token_delta += delta;
+
+        const tier = o.tier_chosen ?? 'unknown';
+        report.by_tier[tier] = (report.by_tier[tier] ?? 0) + delta;
+
+        const cls = o.task_class ?? 'unclassified';
+        report.by_task_class[cls] = (report.by_task_class[cls] ?? 0) + delta;
+    }
+
+    report.measured_share = report.dispatches === 0 ? 0 : report.by_provenance.measured.dispatches / report.dispatches;
+
+    // Honest caveats — always attached so a consumer never over-reads the number.
+    if (report.dispatches === 0) {
+        report.notes.push('No orchestration telemetry yet (0 dispatches). The number accrues as `subagents.auto: on` runs delegate real slices.');
+    } else {
+        report.notes.push('Reports ABSOLUTE net token_delta (negative = saved). A "% of session saved" is NOT derivable — the telemetry records net delta, not the absolute baseline. A percentage needs an absolute-baseline field on the telemetry.');
+        if (report.by_provenance.estimated.dispatches > 0) {
+            report.notes.push(`${report.by_provenance.estimated.dispatches}/${report.dispatches} dispatch(es) use ESTIMATED token_delta (chars/4, lossy). Prefer measured (host usage metadata).`);
+        }
+    }
+
+    return report;
+}

package/src/scripts/build_proof.ts

@@ -107,6 +107,42 @@ function render(): string {
         L.push('Benchmark results are published under `docs/` including measured nulls.');
     }
     L.push('');
+    L.push('**Behavioural-eval coverage — the honest baseline.** Skill *quality* is only');
+    L.push('as good as its measurement. Today **2 of 264** skills carry a behavioural');
+    L.push('`evals.json`, and the highest-traffic / highest-cost tiers (default-surface +');
+    L.push('`rich` + routers, **0 of 35**) are not yet covered. We publish that gap rather');
+    L.push('than imply "264 evaluated skills": coverage is measured per tier');
+    L.push('(`./scripts-run src/scripts/skill_eval_coverage`) and **CI-ratcheted so it can');
+    L.push('only rise** — a merged change can never lower it. Authoring the priority-tier');
+    L.push('evals is gated on per-case human ratification (a generated assertion that');
+    L.push('checks the wrong property is worse than none), so the number grows');
+    L.push('deliberately, not overnight.');
+    L.push('');
+    L.push('**Non-coding domain soundness — scoped, not proven.** The `finance` /');
+    L.push('`founder` / `ops` / `content` profiles sell concrete domain value (DCF,');
+    L.push('runway, RICE, incident command, messaging), but the skills are forged on');
+    L.push('TS/PHP — "promising, not proven" off those stacks. A disclaimer floor');
+    L.push('bounds *liability*, not *correctness*: a skill can be format-correct,');
+    L.push('disclaimered, and still embed a wrong domain assumption. Today **0 of 20**');
+    L.push('default-surface domain skills carry a sourced `domain-truth` fixture');
+    L.push('(`./scripts-run src/scripts/domain_soundness_status`); all are labeled');
+    L.push('`unvalidated` and the validated count is CI-ratcheted. Authoring the');
+    L.push('answer keys needs domain competence (a cited method, not the skill\'s own');
+    L.push('output), so validation lands deliberately — the gap is published, never');
+    L.push('implied away.');
+    L.push('');
+    L.push('**Second-brain substrate — continuity convenience, task-lift unmeasured.**');
+    L.push('The memory substrate is built (working-memory continuity across');
+    L.push('compaction, promotable knowledge cards, contradiction surfacing), but');
+    L.push('whether it beats a no-memory baseline on a reproducible task is **not yet');
+    L.push('measured**. The measurement rig exists — a deterministic multi-session');
+    L.push('recall corpus + scorer (`./scripts-run src/scripts/second_brain_score');
+    L.push('--dry-run`, no model-in-the-loop grading) — but the paired');
+    L.push('`memory-on`/`off`/`placebo` run is spend-bearing and pending. So no');
+    L.push('"second brain" capability claim is made anywhere; the substrate ships as');
+    L.push('continuity convenience until a paired run backs a lift. Boundary vs a');
+    L.push('human PKM is stated in [`docs/second-brain-scope.md`](second-brain-scope.md).');
+    L.push('');
     L.push('## 3. Known limits (published, witness-tested)');
     L.push('');
     const skillGaps = collectSkillGaps();
@@ -164,6 +200,8 @@ function render(): string {
     L.push('task check-refs     # no broken internal references');
     L.push('task check-skill-gaps    # every logged known-limit cites a real witness test');
     L.push('task check-comparison    # every comparison-table "our evidence" pointer resolves');
+    L.push('./scripts-run src/scripts/skill_eval_coverage         # behavioural-eval coverage, per tier');
+    L.push('./scripts-run src/scripts/skill_eval_coverage --check # the ratchet: coverage may not drop');
     L.push('task build-proof-check   # this page is in sync with its sources');
     L.push('```');
     L.push('');

package/src/scripts/chat_history.ts

@@ -43,6 +43,8 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
+import { _entry_tokens_estimate } from './memory_lookup.js';
+
 import { load_agent_settings } from './_lib/agent_settings.js';
 
 export const DEFAULT_FILE = 'agents/runtime/.agent-chat-history';
@@ -708,6 +710,26 @@ export function read_entries(
         agent?: string | null | undefined;
     } = {},
 ): Entry[] {
+    return read_entries_with_refs(options).map((p) => p.entry);
+}
+
+/**
+ * Like {@link read_entries} but each entry carries its `ref` — the 0-based
+ * ordinal in the file's FULL chronological entry list (header excluded,
+ * counted BEFORE any session/agent/last filter), so a ref from an index row
+ * stays valid regardless of the filters a later call applies. Refs are
+ * stable for the append-only log; file rotation invalidates them (documented
+ * — a ref is a within-file locator, not a durable id).
+ * (road-to-memory-retrieval-economy Phase 3.)
+ */
+export function read_entries_with_refs(
+    options: {
+        last?: number | null | undefined;
+        path?: string | null | undefined;
+        session?: string | null | undefined;
+        agent?: string | null | undefined;
+    } = {},
+): Array<{ entry: Entry; ref: number }> {
     const target = options.path ?? file_path();
     const last = options.last ?? null;
     const session = options.session ?? null;
@@ -715,8 +737,9 @@ export function read_entries(
     if (!_isFile(target)) {
         return [];
     }
-    let entries: Entry[] = [];
+    let pairs: Array<{ entry: Entry; ref: number }> = [];
     const lines = _readLines(target);
+    let ref = 0;
     for (let i = 0; i < lines.length; i++) {
         const line = _strip(lines[i] ?? '');
         if (!line) {
@@ -732,23 +755,72 @@ export function read_entries(
             continue;
         }
         if (_isPlainObject(obj)) {
-            entries.push(obj);
+            pairs.push({ entry: obj, ref });
+            ref += 1;
         }
     }
     if (session !== null) {
-        entries = entries.filter((e) => e.s === session);
+        pairs = pairs.filter((p) => p.entry.s === session);
     }
     if (agent !== null) {
         if (agent === AGENT_UNKNOWN) {
-            entries = entries.filter((e) => !e.agent);
+            pairs = pairs.filter((p) => !p.entry.agent);
         } else {
-            entries = entries.filter((e) => e.agent === agent);
+            pairs = pairs.filter((p) => p.entry.agent === agent);
         }
     }
     if (last !== null && last >= 0) {
-        entries = entries.slice(entries.length - last);
+        pairs = pairs.slice(pairs.length - last);
+    }
+    return pairs;
+}
+
+/**
+ * Timeline anchor: the entries around `ref` in the FULL chronological list —
+ * `before`/`after` neighbours plus the anchor itself. JSONL is chronological,
+ * so this is slicing, not indexing (road-to-memory-retrieval-economy P3).
+ * An out-of-range ref returns [].
+ */
+export function slice_around(
+    path_: string | null,
+    ref: number,
+    before = 3,
+    after = 3,
+): Array<{ entry: Entry; ref: number }> {
+    const all = read_entries_with_refs({ path: path_ });
+    if (ref < 0 || ref >= all.length) {
+        return [];
+    }
+    const start = Math.max(0, ref - Math.max(0, before));
+    const end = Math.min(all.length, ref + Math.max(0, after) + 1);
+    return all.slice(start, end);
+}
+
+/**
+ * Compact index row for a history entry: ref + type tag + timestamp-ish
+ * fields when present + a ~100-char preview + the real-tokenizer cost of
+ * fetching the full entry (same lazy-tokenizer contract as memory_lookup's
+ * index rows).
+ */
+export function history_index_row(pair: { entry: Entry; ref: number }): Record<string, unknown> {
+    const e = pair.entry;
+    const text = typeof e.text === 'string' ? e.text : '';
+    const row: Record<string, unknown> = {
+        ref: pair.ref,
+        t: e.t ?? '',
+        preview: text.slice(0, 100),
+        tokens_estimate: _entry_tokens_estimate(e as Record<string, unknown>),
+    };
+    if (typeof e.s === 'string') {
+        row.s = e.s;
+    }
+    for (const tsField of ['ts', 'time', 'started']) {
+        if (typeof (e as Record<string, unknown>)[tsField] === 'string') {
+            row[tsField] = (e as Record<string, unknown>)[tsField];
+            break;
+        }
     }
-    return entries;
+    return row;
 }
 
 export function read_entries_for_current(

package/src/scripts/check_trigger_eval_presence.ts

@@ -0,0 +1,171 @@
+#!/usr/bin/env tsx
+/**
+ * Trigger-eval PRESENCE ratchet (road-to-composition-ratchet Phase 1).
+ *
+ * `check_trigger_evals.ts` validates freshness + structure of triggers.json
+ * files that EXIST; it cannot flag a new skill that ships without one, even
+ * though the drafting protocol mandates the stub. This gate closes that gap
+ * with a shrink-only grandfather allowlist
+ * (`src/scripts/trigger_eval_grandfather.json`, frozen 2026-07-08 at 221
+ * entries):
+ *
+ *   1. Every skill under `src/skills/` must either carry
+ *      `evals/triggers.json` or appear in the allowlist.
+ *   2. An allowlist entry whose skill directory no longer exists is stale
+ *      and must be removed.
+ *   3. An allowlist entry whose skill HAS a triggers.json must be removed —
+ *      the list may only shrink (ratchet), never regrow.
+ *
+ * Structure + freshness of existing files stay owned by
+ * `check_trigger_evals.ts`; this gate checks presence only.
+ *
+ * Exit codes: 0 ok · 1 violation(s) · 2 usage / unreadable allowlist.
+ * `--quiet` suppresses the ✅ summary (errors always print to stderr).
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+const SKILLS_DIR = path.join(REPO_ROOT, 'src', 'skills');
+const ALLOWLIST_PATH = path.join(REPO_ROOT, 'src', 'scripts', 'trigger_eval_grandfather.json');
+
+interface Allowlist {
+    skills: string[];
+}
+
+function load_allowlist(p: string): Allowlist | null {
+    try {
+        const raw = JSON.parse(fs.readFileSync(p, 'utf-8')) as unknown;
+        if (
+            typeof raw !== 'object' ||
+            raw === null ||
+            !Array.isArray((raw as { skills?: unknown }).skills) ||
+            !(raw as { skills: unknown[] }).skills.every((s) => typeof s === 'string')
+        ) {
+            return null;
+        }
+        return { skills: (raw as { skills: string[] }).skills };
+    } catch {
+        return null;
+    }
+}
+
+function list_skill_dirs(root: string): string[] {
+    return fs
+        .readdirSync(root, { withFileTypes: true })
+        .filter((e) => e.isDirectory())
+        .map((e) => e.name)
+        .sort();
+}
+
+function has_triggers(skill: string): boolean {
+    return fs.existsSync(path.join(SKILLS_DIR, skill, 'evals', 'triggers.json'));
+}
+
+interface Findings {
+    errors: string[];
+    covered: number;
+    total: number;
+}
+
+function check(skills: string[], allowlisted: string[]): Findings {
+    const allow = new Set(allowlisted);
+    const skillSet = new Set(skills);
+    const errors: string[] = [];
+    let covered = 0;
+
+    for (const skill of skills) {
+        if (has_triggers(skill)) {
+            covered += 1;
+            if (allow.has(skill)) {
+                errors.push(
+                    `${skill}: has evals/triggers.json but is still in the grandfather allowlist — remove the entry (shrink-only ratchet)`,
+                );
+            }
+        } else if (!allow.has(skill)) {
+            errors.push(
+                `${skill}: missing evals/triggers.json and not grandfathered — new/changed skills must ship a trigger-eval set (see artifact-drafting-protocol Phase C)`,
+            );
+        }
+    }
+
+    for (const entry of allowlisted) {
+        if (!skillSet.has(entry)) {
+            errors.push(
+                `allowlist entry "${entry}" has no matching skill directory — remove the stale entry`,
+            );
+        }
+    }
+
+    return { errors, covered, total: skills.length };
+}
+
+function parse_args(argv: string[]): { quiet: boolean } | null {
+    let quiet = false;
+    for (const a of argv) {
+        if (a === '--quiet' || a === '-q') {
+            quiet = true;
+        } else {
+            return null;
+        }
+    }
+    return { quiet };
+}
+
+function main(): number {
+    const args = parse_args(process.argv.slice(2));
+    if (args === null) {
+        process.stderr.write('usage: check_trigger_eval_presence [--quiet]\n');
+        return 2;
+    }
+
+    const allowlist = load_allowlist(ALLOWLIST_PATH);
+    if (allowlist === null) {
+        process.stderr.write(
+            `❌ check-trigger-eval-presence: unreadable or malformed allowlist at ${path.relative(REPO_ROOT, ALLOWLIST_PATH)}\n`,
+        );
+        return 2;
+    }
+
+    const { errors, covered, total } = check(list_skill_dirs(SKILLS_DIR), allowlist.skills);
+
+    if (errors.length) {
+        process.stderr.write('❌ check-trigger-eval-presence: presence-ratchet violation(s):\n');
+        for (const e of errors) {
+            process.stderr.write(`   - ${e}\n`);
+        }
+        return 1;
+    }
+
+    if (!args.quiet) {
+        process.stdout.write(
+            `✅ check-trigger-eval-presence: ${covered}/${total} skills carry evals/triggers.json (${allowlist.skills.length} grandfathered, shrink-only)\n`,
+        );
+    }
+    return 0;
+}
+
+const _HERE = fileURLToPath(import.meta.url);
+function _isCliEntry(): boolean {
+    if (process.argv[1] === undefined) {
+        return false;
+    }
+    const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
+    if (import.meta.url === argvUrl) {
+        return true;
+    }
+    try {
+        const here = fs.realpathSync(fileURLToPath(import.meta.url));
+        const argv = fs.realpathSync(path.resolve(process.argv[1]));
+        return here === argv;
+    } catch {
+        return false;
+    }
+}
+
+if (_isCliEntry() || process.argv[1] === _HERE) {
+    process.exit(main());
+}
+
+export { ALLOWLIST_PATH, check, load_allowlist, parse_args, main };