@event4u/agent-config 16.4.0 → 17.0.0

package/src/scripts/ai_council/council_egress.ts

@@ -0,0 +1,167 @@
+/**
+ * The council channel's egress state (road-to-one-detector-and-the-senders-we-own
+ * 3.3): one token scope per run, the bundler's replacement through it, and the
+ * final check every member call passes immediately before its request is written.
+ *
+ * The values replaced in this run are held in memory so the final check can
+ * look for them; they are never written, logged or returned. The retained
+ * record carries what the members were sent, so it keeps the placeholders; a
+ * restore is for a display in this process only, and resolves identifiers —
+ * a credential token resolves to nothing (D5).
+ *
+ * Credentials are replaced whatever the settings say: the bundle masked every
+ * credential line before this module existed, so replacing the value instead of
+ * the line changes how much is lost, not whether. Identifiers are replaced — and
+ * checked — only when `egress.placeholders.mode` is `protect` or `strict`;
+ * whether any identifier is replaced by default for consumers is the owner's
+ * decision D8, which this module does not take.
+ */
+
+import {
+    detect,
+    EGRESS_RULES,
+    type EgressRule,
+    type EgressSpan,
+    maskingEnd,
+    prefixesOf,
+    rawKeyPrefixPattern,
+    replaceSpans,
+    resolveOverlaps,
+    findSpans,
+    REVIEW_CREDENTIAL_ROWS,
+} from '../_lib/egress_rules.js';
+import { parseTokens, TokenScope } from '../_lib/egress_token.js';
+import { readEgressMode } from '../hooks/egress_stage.js';
+
+/**
+ * The bundler's own looser readings, kept so moving it from whole-line masks to
+ * spans narrows nothing: an Anthropic key prefix with eight key characters, an
+ * OpenAI or project key glued to a preceding word, and the path of this
+ * package's own key files.
+ */
+const BUNDLER_OWN_ROWS: readonly EgressRule[] = [
+    {
+        rule: 'bundler-short-anthropic-key',
+        type: 'APIKEY',
+        class: 'credential',
+        replacement: 'detection',
+        pattern: rawKeyPrefixPattern(8, prefixesOf('anthropic-api-key')),
+    },
+    {
+        rule: 'bundler-glued-openai-key',
+        type: 'APIKEY',
+        class: 'credential',
+        replacement: 'detection',
+        pattern: rawKeyPrefixPattern(20, [...prefixesOf('openai-api-key'), ...prefixesOf('anthropic-project-key')], {
+            boundary: 'never',
+        }),
+    },
+    {
+        rule: 'bundler-key-path',
+        type: 'KEYPATH',
+        class: 'credential',
+        replacement: 'detection',
+        pattern: /~?\/?\.(?:event4u|config)\/agent-config\/[^/\s]+\.key/gu,
+    },
+];
+
+export const BUNDLE_RULES: readonly EgressRule[] = [...EGRESS_RULES, ...REVIEW_CREDENTIAL_ROWS, ...BUNDLER_OWN_ROWS];
+
+const CLASS_RANK = { credential: 1, identifier: 0 } as const;
+
+/** Why a body was refused; never carries a byte of the body. */
+export interface EgressRefusal {
+    readonly reason: 'replaced-value-present' | 'high-confidence-finding';
+    /** The rule that fired, or the type of the replaced value. */
+    readonly what: string;
+}
+
+export class CouncilEgress {
+    readonly scope = new TokenScope();
+    private readonly replaced = new Set<string>();
+
+    /** `identifiers`: replace and check the identifier class too (mode `protect` or `strict`). */
+    constructor(private readonly opts: { readonly identifiers: boolean } = { identifiers: false }) {}
+
+    /**
+     * Every detected value becomes a token; a key block whose end cannot be
+     * proven is replaced through the nearest END marker, as the line masks did.
+     */
+    replace(text: string): string {
+        const spans = resolveOverlaps(findSpans(text, BUNDLE_RULES))
+            .filter((s) => s.class === 'credential' || this.opts.identifiers)
+            .map((s) => ({ ...s, end: maskingEnd(text, s) }));
+        spans.sort((a, b) => a.start - b.start);
+        const merged: EgressSpan[] = [];
+        for (const s of spans) {
+            const last = merged[merged.length - 1];
+            if (last !== undefined && s.start < last.end) {
+                const keep = CLASS_RANK[s.class] > CLASS_RANK[last.class] ? s : last;
+                merged[merged.length - 1] = { ...keep, start: last.start, end: Math.max(last.end, s.end) };
+            } else {
+                merged.push(s);
+            }
+        }
+        const tokens = merged.map((s) => {
+            const value = text.slice(s.start, s.end);
+            this.replaced.add(value);
+            return this.scope.mint(s.type, s.class, value);
+        });
+        let out = replaceSpans(text, merged, (_s, i) => tokens[i] as string);
+        // A value detected once is a secret wherever it stands: a password the
+        // rules saw inside a connection URI is the same password on a line of
+        // its own, where no rule fires. Leaving it there sent it anyway — and
+        // `check` then refused the whole payload for it.
+        merged.forEach((s, i) => {
+            const value = text.slice(s.start, s.end);
+            if (out.includes(value)) out = out.split(value).join(tokens[i] as string);
+        });
+        return out;
+    }
+
+    /** The final check: null when the body may be sent. */
+    check(body: string): EgressRefusal | null {
+        for (const value of this.replaced) {
+            if (body.includes(value)) return { reason: 'replaced-value-present', what: 'value replaced in this run' };
+        }
+        for (const s of detect(body)) {
+            const high = s.class === 'credential' ? s.rule !== 'generic-assignment' : s.validated && this.opts.identifiers;
+            if (high) return { reason: 'high-confidence-finding', what: s.rule };
+        }
+        return null;
+    }
+
+    /** For a display in this process only: identifier tokens back to their spelling. */
+    restoreForDisplay(text: string): string {
+        let out = text;
+        for (const t of parseTokens(text).reverse()) {
+            const value = this.scope.resolve(text.slice(t.start, t.end));
+            if (value !== null) out = out.slice(0, t.start) + value + out.slice(t.end);
+        }
+        return out;
+    }
+}
+
+let current: CouncilEgress | null = null;
+
+function fromSettings(): CouncilEgress {
+    let mode: string;
+    try {
+        mode = readEgressMode(process.cwd());
+    } catch {
+        mode = 'protect';
+    }
+    return new CouncilEgress({ identifiers: mode === 'protect' || mode === 'strict' });
+}
+
+/** The run's egress state; one council run is one process. */
+export function councilEgress(): CouncilEgress {
+    current ??= fromSettings();
+    return current;
+}
+
+/** A fresh scope, for a test or a second run in one process; the settings are read again unless `opts` is given. */
+export function resetCouncilEgress(opts?: { readonly identifiers: boolean }): CouncilEgress {
+    current = opts === undefined ? fromSettings() : new CouncilEgress(opts);
+    return current;
+}

package/src/scripts/ai_council/exposure_ledger.ts

@@ -0,0 +1,117 @@
+/**
+ * How much of the project each route has seen, so no single provider collects
+ * enough fragments to rebuild it.
+ *
+ * Counts distinct project files and symbols per route in gitignored runtime
+ * state. Entries are digests, so the ledger holds no path or name in clear. The
+ * chooser prefers the least-exposed admissible route and refuses one this call
+ * would push past the share cap. This raises reconstruction cost; it never
+ * replaces the content ceiling, which is what keeps project content off a
+ * public-artifact route.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+/** The share of tracked files one route may have seen; a constant, not a setting. */
+export const EXPOSURE_SHARE_CAP = 0.2;
+
+export const EXPOSURE_LEDGER_RELPATH = path.join('agents', 'runtime', 'state', 'hand-exposure.json');
+
+export interface Exposure {
+    readonly files: readonly string[];
+    readonly symbols: readonly string[];
+}
+
+interface RouteSeen {
+    files: string[];
+    symbols: string[];
+}
+
+function digest(kind: string, value: string): string {
+    return createHash('sha256').update(`${kind}\u0000${value}`).digest('hex').slice(0, 16);
+}
+
+export class ExposureLedger {
+    private readonly routes = new Map<string, { files: Set<string>; symbols: Set<string> }>();
+
+    /** `file` null keeps the ledger in memory only. */
+    constructor(private readonly file: string | null) {}
+
+    static load(file: string): ExposureLedger {
+        const ledger = new ExposureLedger(file);
+        try {
+            const raw = JSON.parse(fs.readFileSync(file, 'utf8')) as { routes?: Record<string, RouteSeen> };
+            for (const [name, seen] of Object.entries(raw.routes ?? {})) {
+                ledger.routes.set(name, { files: new Set(seen.files ?? []), symbols: new Set(seen.symbols ?? []) });
+            }
+        } catch {
+            /* absent or unreadable: start empty */
+        }
+        return ledger;
+    }
+
+    private entry(route: string): { files: Set<string>; symbols: Set<string> } {
+        let e = this.routes.get(route);
+        if (e === undefined) {
+            e = { files: new Set(), symbols: new Set() };
+            this.routes.set(route, e);
+        }
+        return e;
+    }
+
+    seen(route: string): { files: number; symbols: number } {
+        const e = this.routes.get(route);
+        return { files: e?.files.size ?? 0, symbols: e?.symbols.size ?? 0 };
+    }
+
+    /** Distinct files the route would have seen after `exposure` is sent to it. */
+    filesAfter(route: string, exposure: Exposure): number {
+        const known = this.routes.get(route)?.files ?? new Set<string>();
+        let n = known.size;
+        for (const f of new Set(exposure.files)) if (!known.has(digest('file', f))) n++;
+        return n;
+    }
+
+    /** A payload that was sent cannot be recalled, so this runs whatever the call returned. */
+    record(route: string, exposure: Exposure): void {
+        if (exposure.files.length === 0 && exposure.symbols.length === 0) return;
+        const e = this.entry(route);
+        for (const f of exposure.files) e.files.add(digest('file', f));
+        for (const s of exposure.symbols) e.symbols.add(digest('symbol', s));
+        if (this.file === null) return;
+        const routes: Record<string, RouteSeen> = {};
+        for (const [name, seen] of this.routes) routes[name] = { files: [...seen.files], symbols: [...seen.symbols] };
+        try {
+            fs.mkdirSync(path.dirname(this.file), { recursive: true });
+            fs.writeFileSync(this.file, `${JSON.stringify({ schema_version: 1, routes })}\n`, 'utf8');
+        } catch {
+            /* accounting never blocks the call; the in-memory ledger still holds */
+        }
+    }
+}
+
+/**
+ * Each route's projected share of tracked files if this exposure is sent to it;
+ * null when the denominator is unknown, which the chooser treats as refusal for
+ * an unpaid route.
+ */
+export function exposureShareFn(
+    ledger: ExposureLedger,
+    exposure: Exposure,
+    trackedFiles: number | null,
+): (route: string) => number | null {
+    return (route) => (trackedFiles === null || trackedFiles <= 0 ? null : ledger.filesAfter(route, exposure) / trackedFiles);
+}
+
+/** Files git tracks under `root`; null outside a repository. */
+export function trackedFileCount(root: string): number | null {
+    try {
+        const out = execFileSync('git', ['-C', root, 'ls-files', '-z'], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+        return out.split('\0').filter(Boolean).length;
+    } catch {
+        return null;
+    }
+}

package/src/scripts/ai_council/free_hand_shadow.ts

@@ -0,0 +1,276 @@
+#!/usr/bin/env node
+/**
+ * Shadow mode for free hands (free-hands 6.2): a free route answers the same
+ * typed slice beside the paid route, the paid answer is the one used, the head
+ * verifies both, and one row per call goes to a gitignored ledger. The ledger
+ * is the data both pre-registered claims are read from (`free-hands-limit-relief`,
+ * `free-hands-quality-held`) and the only input a per-kind flip may act on.
+ *
+ * Bounds, each failing toward not calling the free side:
+ *   - the free side must be unpaid (`free-quota` or `self-hosted`); a metered
+ *     "free" route would spend, so it is never shadowed;
+ *   - the payload must be `public-artifact`, the ceiling every free route sits at;
+ *   - at most `SHADOW_CALL_CAP` rows (decision A5); past it the paid side still
+ *     answers and nothing else is called.
+ *
+ * Nothing here calls anything on its own: a caller hands in both members and
+ * the head's verify. A row holds routes, verdicts and token counts, never the
+ * prompt or either answer.
+ *
+ * Usage: ./scripts-run src/scripts/ai_council/free_hand_shadow --report [--json]
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+import type { ExternalAIClient } from './clients.js';
+import { admitsContent, type ContentClass } from './content_ceiling.js';
+import { handSystemPrompt, parseHandReturn } from './hand_schemas.js';
+import { routeCandidateFromMember } from './route_chooser.js';
+import { classifyRouteOutcome, recordRouteOutcome, type RouteOutcome } from './route_liveness.js';
+import { isUnpaid, type HandKind, type RouteBilling, type VerifyVerdict } from '../_lib/hand_kinds.js';
+
+/** Decision A5: enough for per-kind rates at the 5 pp margin; spend stays USD 0.00. */
+export const SHADOW_CALL_CAP = 2000;
+
+export const SHADOW_LEDGER_RELPATH = path.join('agents', 'runtime', 'state', 'free-hand-shadow.jsonl');
+
+export interface ShadowSide {
+    readonly route: string;
+    readonly billing: RouteBilling;
+    readonly outcome: RouteOutcome;
+    readonly schema_valid: boolean;
+    readonly verified: VerifyVerdict;
+    /** Billable tokens the side's call reported. */
+    readonly tokens: number;
+    /** Tokens the head spent verifying this side's return. */
+    readonly verify_tokens: number;
+}
+
+export interface ShadowRow {
+    readonly ts: string;
+    readonly kind: HandKind;
+    readonly paid: ShadowSide;
+    readonly free: ShadowSide;
+}
+
+/** The head's check of one parsed return. `side` is for accounting only. */
+export type HeadVerify = (
+    kind: HandKind,
+    parsed: unknown,
+    side: 'paid' | 'free',
+) => { verdict: VerifyVerdict; tokens: number } | Promise<{ verdict: VerifyVerdict; tokens: number }>;
+
+export interface ShadowAskOptions {
+    readonly kind: HandKind;
+    readonly contentClass: ContentClass;
+    readonly paid: ExternalAIClient;
+    readonly free: ExternalAIClient;
+    readonly verify: HeadVerify;
+    /** Root of the gitignored state the ledger and route liveness live in. */
+    readonly stateRoot: string;
+    readonly now?: () => Date;
+}
+
+export type ShadowSkip = 'free-side-metered' | 'paid-side-unpaid' | 'content-ceiling' | 'cap-reached';
+
+export interface ShadowAnswer {
+    readonly answer: string;
+    readonly parsed: unknown;
+}
+
+export interface ShadowResult {
+    /** The paid answer — the one the caller uses. Null when the paid side failed. */
+    readonly answer: ShadowAnswer | null;
+    readonly row: ShadowRow | null;
+    readonly skipped?: ShadowSkip;
+}
+
+interface Called {
+    readonly outcome: RouteOutcome;
+    readonly parsed: { ok: true; value: unknown } | { ok: false };
+    readonly text: string;
+    readonly tokens: number;
+}
+
+async function _call(member: ExternalAIClient, kind: HandKind, prompt: string): Promise<Called> {
+    try {
+        const raw = member.ask(handSystemPrompt(kind), prompt);
+        const r = raw instanceof Promise ? await raw : raw;
+        const outcome = classifyRouteOutcome(r?.error ?? null);
+        const text = (r?.text ?? '').trim();
+        const tokens = (r?.input_tokens ?? 0) + (r?.output_tokens ?? 0);
+        if (outcome !== 'ok' || text === '') return { outcome, parsed: { ok: false }, text: '', tokens };
+        const p = parseHandReturn(kind, text);
+        return { outcome, parsed: p.ok ? { ok: true, value: p.value } : { ok: false }, text, tokens };
+    } catch {
+        return { outcome: 'other', parsed: { ok: false }, text: '', tokens: 0 };
+    }
+}
+
+async function _side(member: ExternalAIClient, c: Called, kind: HandKind, which: 'paid' | 'free', verify: HeadVerify): Promise<ShadowSide> {
+    const route = routeCandidateFromMember(member);
+    // An answer that never parsed has already failed; the head is not asked about it.
+    const v = c.parsed.ok ? await verify(kind, c.parsed.value, which) : { verdict: 'fail' as const, tokens: 0 };
+    return {
+        route: route.name,
+        billing: route.billing,
+        outcome: c.outcome,
+        schema_valid: c.parsed.ok,
+        verified: v.verdict,
+        tokens: c.tokens,
+        verify_tokens: v.tokens,
+    };
+}
+
+function _ledgerPath(stateRoot: string): string {
+    return path.join(stateRoot, SHADOW_LEDGER_RELPATH);
+}
+
+function _rowCount(stateRoot: string): number {
+    try {
+        return fs.readFileSync(_ledgerPath(stateRoot), 'utf8').split('\n').filter((l) => l.trim() !== '').length;
+    } catch {
+        return 0;
+    }
+}
+
+function _append(stateRoot: string, row: ShadowRow): void {
+    const file = _ledgerPath(stateRoot);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.appendFileSync(file, `${JSON.stringify(row)}\n`, 'utf8');
+}
+
+function _routeModel(member: ExternalAIClient): string | null {
+    const m = member as unknown as Record<string, unknown>;
+    return typeof m['base_url'] === 'string' && typeof m['model'] === 'string' && m['model'] !== '' ? m['model'] : null;
+}
+
+export async function shadowAsk(prompt: string, opts: ShadowAskOptions): Promise<ShadowResult> {
+    const paidRoute = routeCandidateFromMember(opts.paid);
+    const freeRoute = routeCandidateFromMember(opts.free);
+    const usePaid = async (skipped: ShadowSkip): Promise<ShadowResult> => {
+        const c = await _call(opts.paid, opts.kind, prompt);
+        return { answer: c.parsed.ok ? { answer: c.text, parsed: c.parsed.value } : null, row: null, skipped };
+    };
+    if (isUnpaid(paidRoute.billing)) return { answer: null, row: null, skipped: 'paid-side-unpaid' };
+    if (!isUnpaid(freeRoute.billing)) return usePaid('free-side-metered');
+    if (!admitsContent(freeRoute.ceiling, opts.contentClass).admitted) return usePaid('content-ceiling');
+    if (_rowCount(opts.stateRoot) >= SHADOW_CALL_CAP) return usePaid('cap-reached');
+
+    const [p, f] = await Promise.all([_call(opts.paid, opts.kind, prompt), _call(opts.free, opts.kind, prompt)]);
+    const model = _routeModel(opts.free);
+    if (model !== null) recordRouteOutcome(opts.stateRoot, freeRoute.name, model, f.outcome);
+    const row: ShadowRow = {
+        ts: (opts.now ?? (() => new Date()))().toISOString(),
+        kind: opts.kind,
+        paid: await _side(opts.paid, p, opts.kind, 'paid', opts.verify),
+        free: await _side(opts.free, f, opts.kind, 'free', opts.verify),
+    };
+    _append(opts.stateRoot, row);
+    return { answer: p.parsed.ok ? { answer: p.text, parsed: p.parsed.value } : null, row };
+}
+
+export function readShadowLedger(stateRoot: string): ShadowRow[] {
+    let raw: string;
+    try {
+        raw = fs.readFileSync(_ledgerPath(stateRoot), 'utf8');
+    } catch {
+        return [];
+    }
+    const rows: ShadowRow[] = [];
+    for (const line of raw.split('\n')) {
+        if (line.trim() === '') continue;
+        try {
+            const r = JSON.parse(line) as ShadowRow;
+            if (typeof r.kind === 'string' && r.paid !== undefined && r.free !== undefined) rows.push(r);
+        } catch {
+            /* a torn line is skipped, never guessed at */
+        }
+    }
+    return rows;
+}
+
+/** An escalation or a failed verify: a `skipped` verify on a return counts as an escalation. */
+export function sideFailed(s: ShadowSide): boolean {
+    return s.verified !== 'pass';
+}
+
+/** ISO-8601 week, `YYYY-Www`. */
+export function isoWeek(ts: string): string {
+    const d = new Date(ts);
+    const day = (d.getUTCDay() + 6) % 7;
+    const thursday = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate() - day + 3));
+    const yearStart = Date.UTC(thursday.getUTCFullYear(), 0, 1);
+    const week = Math.ceil(((thursday.getTime() - yearStart) / 86_400_000 + 1) / 7);
+    return `${String(thursday.getUTCFullYear())}-W${String(week).padStart(2, '0')}`;
+}
+
+export interface KindShadowStats {
+    readonly kind: HandKind;
+    readonly n: number;
+    readonly free_rate: number;
+    readonly paid_rate: number;
+    /** Paid tokens of the calls whose free return passed the head's verify. */
+    readonly tokens_avoided: number;
+    /** The head's tokens spent verifying free returns. */
+    readonly verify_tokens: number;
+    readonly net: number;
+    readonly weeks: { week: string; net: number }[];
+}
+
+export function shadowReport(rows: readonly ShadowRow[]): KindShadowStats[] {
+    const byKind = new Map<HandKind, ShadowRow[]>();
+    for (const r of rows) byKind.set(r.kind, [...(byKind.get(r.kind) ?? []), r]);
+    const out: KindShadowStats[] = [];
+    for (const [kind, rs] of [...byKind.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        const weeks = new Map<string, number>();
+        let avoided = 0;
+        let verify = 0;
+        for (const r of rs) {
+            const saved = r.free.verified === 'pass' ? r.paid.tokens : 0;
+            avoided += saved;
+            verify += r.free.verify_tokens;
+            const w = isoWeek(r.ts);
+            weeks.set(w, (weeks.get(w) ?? 0) + saved - r.free.verify_tokens);
+        }
+        const rate = (pick: (r: ShadowRow) => ShadowSide): number => rs.filter((r) => sideFailed(pick(r))).length / rs.length;
+        out.push({
+            kind,
+            n: rs.length,
+            free_rate: rate((r) => r.free),
+            paid_rate: rate((r) => r.paid),
+            tokens_avoided: avoided,
+            verify_tokens: verify,
+            net: avoided - verify,
+            weeks: [...weeks.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([week, net]) => ({ week, net })),
+        });
+    }
+    return out;
+}
+
+export function main(argv: readonly string[], cwd: string = process.cwd()): number {
+    if (!argv.includes('--report')) {
+        process.stderr.write('usage: free_hand_shadow --report [--json]\n');
+        return 2;
+    }
+    const rows = readShadowLedger(cwd);
+    const report = shadowReport(rows);
+    if (argv.includes('--json')) {
+        process.stdout.write(`${JSON.stringify({ rows: rows.length, cap: SHADOW_CALL_CAP, kinds: report }, null, 2)}\n`);
+        return 0;
+    }
+    process.stdout.write(`free-hand shadow: ${String(rows.length)} / ${String(SHADOW_CALL_CAP)} calls\n`);
+    for (const k of report) {
+        process.stdout.write(
+            `  ${k.kind}: n=${String(k.n)} free=${k.free_rate.toFixed(3)} paid=${k.paid_rate.toFixed(3)} ` +
+                `avoided=${String(k.tokens_avoided)} verify=${String(k.verify_tokens)} net=${String(k.net)}\n`,
+        );
+    }
+    return 0;
+}
+
+if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+    process.exit(main(process.argv.slice(2)));
+}