@event4u/agent-config 14.12.0 → 14.14.0

package/src/scripts/_lib/empty_cycles.ts

@@ -0,0 +1,88 @@
+/**
+ * A double trigger is one event, not two outcomes.
+ *
+ * Hosts fire the same slot more than once for a single logical event —
+ * `run_continuation_hook.ts` documents the same quirk on `stop` and dedupes it
+ * with an ordinal + open-count + short-window key. The consequence here is
+ * arithmetic rather than cosmetic: if a duplicate fire writes a second outcome
+ * record, every per-asset rate computed over the stream is inflated by exactly
+ * the duplication rate, and nothing downstream can separate the two.
+ *
+ * But a duplicate is not nothing, and discarding it silently is the other
+ * failure. The number of times a trigger fired without producing new work is
+ * itself a signal — it is how an idle loop looks from the outside. So a
+ * duplicate produces NO second outcome and DOES increment an empty-cycle
+ * counter, which is reported as its own quantity rather than folded into
+ * either the numerator or the denominator of a success rate.
+ *
+ * Pure and host-free on purpose: this is the seam the hook consumes, so the
+ * arithmetic can be tested without constructing a host payload.
+ */
+
+/** Default coincidence window. Same value the stop-slot dedupe uses. */
+export const DUPLICATE_WINDOW_MS = 60 * 1000;
+
+export interface TriggerFire {
+    /**
+     * Identity of the logical event. Two fires sharing a key inside the window
+     * are the same event seen twice. Id-shaped — never free-form text.
+     */
+    key: string;
+    /** Epoch milliseconds. */
+    at: number;
+}
+
+export interface CycleState {
+    /** The most recent fire that produced an outcome, or `null` before any. */
+    last: TriggerFire | null;
+    /** Fires suppressed as duplicates so far. */
+    empty_cycles: number;
+}
+
+export function initialCycleState(): CycleState {
+    return { last: null, empty_cycles: 0 };
+}
+
+export interface CycleDecision {
+    /** Whether this fire should produce an outcome record. */
+    emit_outcome: boolean;
+    /** The state to carry forward. */
+    state: CycleState;
+}
+
+/**
+ * Decide what one fire means.
+ *
+ * A fire is a duplicate when it carries the SAME key as the last
+ * outcome-producing fire AND lands inside the window. Both conjuncts are
+ * required and neither is sufficient:
+ *
+ * - Key alone would collapse a genuine repeat of the same work an hour later
+ *   into an empty cycle, erasing a real outcome.
+ * - Window alone would collapse two DIFFERENT events that happen to be close
+ *   together, which is the common case under any kind of fan-out.
+ *
+ * A duplicate does not advance `last`. That is deliberate: three fires inside
+ * one window are one outcome and two empty cycles, not one outcome, one empty
+ * cycle, and a third fire that has drifted out of the window and looks new.
+ */
+export function observeFire(
+    state: CycleState,
+    fire: TriggerFire,
+    windowMs: number = DUPLICATE_WINDOW_MS,
+): CycleDecision {
+    const prev = state.last;
+    const isDuplicate =
+        prev !== null && prev.key === fire.key && fire.at - prev.at < windowMs;
+
+    if (isDuplicate) {
+        return {
+            emit_outcome: false,
+            state: { last: prev, empty_cycles: state.empty_cycles + 1 },
+        };
+    }
+    return {
+        emit_outcome: true,
+        state: { last: fire, empty_cycles: state.empty_cycles },
+    };
+}

package/src/scripts/_lib/estate_carrier_cases.ts

@@ -0,0 +1,72 @@
+/**
+ * The three `check_estate_count` self-test cases that exercise `status: carrier`.
+ *
+ * They live outside that gate because it sits at the 1,500-line source ceiling
+ * `check_source_size_budget` enforces, and because the doctrine that gate states
+ * for an oversized file is a split rather than a raised baseline. The reasoning
+ * behind the fixture shapes is in `carrier_status.ts` § Fixture design.
+ */
+
+import type { SelfTestCase } from './gate_self_test.js';
+
+/** What the caller must supply: its own fixture harness and roadmap bodies. */
+export interface EstateCaseDeps {
+    fixture: (opts: {
+        roadmaps: number;
+        base?: string;
+        before?: (dir: string) => void;
+        after: (dir: string) => void;
+    }) => number;
+    write: (dir: string, rel: string, body: string) => void;
+    remove: (dir: string, rel: string) => void;
+    roadmap: (name: string, extra?: string) => string;
+    CARRIER: string;
+    DRAFT: string;
+}
+
+export function carrierEstateCases(d: EstateCaseDeps): SelfTestCase[] {
+    const { fixture, write, remove, roadmap, CARRIER, DRAFT } = d;
+    return [
+        {
+            name: 'deleting a carrier does not offset a new draft → reject',
+            expect: 'reject',
+            run: () =>
+                fixture({
+                    roadmaps: 3,
+                    base: 'main',
+                    before: (dir) => write(dir, 'agents/roadmaps/road-to-carrier.md', CARRIER),
+                    after: (dir) => {
+                        remove(dir, 'agents/roadmaps/road-to-carrier.md');
+                        write(dir, 'agents/roadmaps/road-to-new.md', DRAFT);
+                    },
+                }),
+        },
+        {
+            // The control that proves the case above tests the STATUS rather
+            // than the deletion: only the removed file's status differs.
+            name: 'deleting an ordinary roadmap still offsets a new draft → accept',
+            expect: 'accept',
+            run: () =>
+                fixture({
+                    roadmaps: 3,
+                    base: 'main',
+                    before: (dir) => write(dir, 'agents/roadmaps/road-to-spare.md', roadmap('S')),
+                    after: (dir) => {
+                        remove(dir, 'agents/roadmaps/road-to-spare.md');
+                        write(dir, 'agents/roadmaps/road-to-new.md', DRAFT);
+                    },
+                }),
+        },
+        {
+            name: 'flipping a carrier back to an ordinary roadmap costs nothing → accept',
+            expect: 'accept',
+            run: () =>
+                fixture({
+                    roadmaps: 3,
+                    base: 'main',
+                    before: (dir) => write(dir, 'agents/roadmaps/road-to-2.md', CARRIER),
+                    after: (dir) => write(dir, 'agents/roadmaps/road-to-2.md', roadmap('R2')),
+                }),
+        },
+    ];
+}

package/src/scripts/_lib/eval_publication.ts

@@ -0,0 +1,354 @@
+/**
+ * Comparison arms that can discriminate, and results honest enough to publish.
+ *
+ * `road-to-skill-ecosystem-eval-integrity` Phases 4 and 5. Every function here
+ * exists because a benchmark can be run correctly and still produce a number
+ * nobody should act on — and each of those routes is a separate mechanism, not
+ * a matter of care.
+ */
+import { createHash } from 'node:crypto';
+
+// ── Phase 4.2 — a fixture the control arm cannot also satisfy ───────────────
+
+export interface PlantedItem {
+    id: string;
+    /**
+     * True when this item is recoverable ONLY via the behaviour the artifact
+     * under test prescribes. At least one such item must exist, or the control
+     * arm scores the same and the eval adjudicates nothing.
+     */
+    requires_artifact_behaviour: boolean;
+}
+
+/**
+ * Can this fixture set discriminate at all?
+ *
+ * The check runs BEFORE a run rather than after, because the answer does not
+ * depend on the outcome: a fixture set where every item is recoverable by
+ * general competence measures general competence, whatever the arms score. A
+ * run over such a set is not a weak result — it is not a result.
+ */
+export function discriminationDeficit(items: readonly PlantedItem[]): string | null {
+    if (items.length === 0) return 'no fixture items — nothing to recover';
+    if (!items.some((i) => i.requires_artifact_behaviour)) {
+        return (
+            `none of ${String(items.length)} fixture item(s) is recoverable only via the behaviour ` +
+            'under test, so the control arm can score identically and the eval cannot adjudicate'
+        );
+    }
+    return null;
+}
+
+// ── Phase 4.3 — a missed plant is a zero, never an absence ──────────────────
+
+export interface RubricScoreInput {
+    /** Items the run actually reported on. */
+    reported: readonly { id: string; score: number }[];
+    /** Every item that was planted, whether reported or not. */
+    planted: readonly string[];
+}
+
+export interface RubricScore {
+    /** Sum over PLANTED items, missing ones counted as zero. */
+    total: number;
+    /** Denominator is the planted count, never the reported count. */
+    denominator: number;
+    ratio: number;
+    missed: readonly string[];
+}
+
+/**
+ * Score a rubric with missed plants counted as implicit zeros.
+ *
+ * Dividing by what was REPORTED is the defect: a run that finds one item and
+ * says nothing about the other nine scores 100 %, so under-reporting buys a
+ * higher ratio. The denominator has to be the plant.
+ */
+export function scoreWithImplicitZeros(input: RubricScoreInput): RubricScore {
+    const byId = new Map(input.reported.map((r) => [r.id, r.score]));
+    const missed = input.planted.filter((p) => !byId.has(p));
+    const total = input.planted.reduce((a, id) => a + (byId.get(id) ?? 0), 0);
+    const denominator = input.planted.length;
+    return {
+        total,
+        denominator,
+        ratio: denominator === 0 ? 0 : total / denominator,
+        missed,
+    };
+}
+
+// ── Phase 4.4 — did the run read what it was denied? ────────────────────────
+
+export interface LeakScanInput {
+    /** Paths the run must not have read: ground truth, generator, scorers, prior trials. */
+    denied: readonly string[];
+    /** Every tool input the run recorded, in order. */
+    toolInputs: readonly string[];
+}
+
+export interface LeakFinding {
+    denied: string;
+    /** The tool input that matched. */
+    evidence: string;
+}
+
+/**
+ * Walk a run transcript for reads of denied paths.
+ *
+ * Runs AFTER the trial, on the recorded transcript, because a denial that is
+ * only configured is a denial nobody verified. Matching is substring on a
+ * path with separators normalised — a leak arriving through a symlink or a
+ * parent-directory hop is not something this catches, and saying so is cheaper
+ * than implying it does.
+ */
+export function scanLeaks(input: LeakScanInput): LeakFinding[] {
+    const out: LeakFinding[] = [];
+    const norm = (s: string): string => s.split('\\').join('/');
+    for (const d of input.denied) {
+        const needle = norm(d);
+        if (needle === '') continue;
+        for (const t of input.toolInputs) {
+            if (norm(t).includes(needle)) {
+                out.push({ denied: d, evidence: t.slice(0, 200) });
+                break;
+            }
+        }
+    }
+    return out;
+}
+
+// ── Phase 4.5 — a cache key that includes the CRITERIA ──────────────────────
+
+/** Field delimiter for the identity hash. An escape, never a raw byte. */
+const FIELD_SEP = '\u001e';
+/** Element delimiter inside one array field. Different from FIELD_SEP on purpose. */
+const ELEMENT_SEP = '\u001f';
+
+export interface BaselineIdentity {
+    prompt: string;
+    fixtures: readonly string[];
+    /** The half most likely to be omitted, which is why it is required here. */
+    rubric: readonly string[];
+    assertions: readonly string[];
+    toolExpectations: readonly string[];
+    turnLimit: number;
+    tokenLimit: number;
+}
+
+/**
+ * Composite identity hash for a cached baseline.
+ *
+ * Keying on prompt and fixtures alone is the common shape and the wrong one: an
+ * edited rubric changes what a pass MEANS while the key stays identical, so the
+ * cache serves a baseline scored under criteria that no longer exist. The
+ * criteria half is required by the type rather than optional, because an
+ * optional field that is easy to omit is a field that gets omitted.
+ */
+export function baselineIdentity(id: BaselineIdentity): string {
+    const h = createHash('sha256');
+    const feed = (label: string, v: string | readonly string[] | number): void => {
+        // The separators are ESCAPES, never raw control bytes: a raw U+1F in a
+        // text source makes grep, file(1) and git treat the file as binary and
+        // skip it silently, which `lint_hidden_unicode` refuses. A field
+        // delimiter and an element delimiter that differ are what keep two
+        // fields concatenating to the same text from producing one hash.
+        h.update(FIELD_SEP + label + FIELD_SEP);
+        h.update(Array.isArray(v) ? [...v].join(ELEMENT_SEP) : String(v));
+    };
+    feed('prompt', id.prompt);
+    feed('fixtures', id.fixtures);
+    feed('rubric', id.rubric);
+    feed('assertions', id.assertions);
+    feed('tools', id.toolExpectations);
+    feed('turns', id.turnLimit);
+    feed('tokens', id.tokenLimit);
+    return h.digest('hex');
+}
+
+// ── Phase 5.1 / 5.2 — first attempts, and the completeness precondition ─────
+
+export interface Receipt {
+    caseId: string;
+    configuration: string;
+    /** 1 for the first attempt. A later correction carries 2, 3, … */
+    attempt: number;
+    outcome: 'pass' | 'fail' | 'aborted';
+    /** Required when `outcome` is `aborted`, and must be an allow-listed reason. */
+    failureReason?: string | undefined;
+}
+
+/**
+ * Reasons an aborted run may record. A closed set on purpose.
+ *
+ * The alternative to an allow-list is free text, and free text lets an abort be
+ * recorded as anything — including something that reads like a result. A run
+ * that stopped is a truthful receipt with its gates marked not-run, never a
+ * dropped run and never a fabricated artifact.
+ */
+export const ABORT_REASONS = [
+    'quota-exhausted',
+    'credential-missing',
+    'dependency-unavailable',
+    'timeout',
+    'harness-error',
+    'operator-cancelled',
+] as const;
+export type AbortReason = (typeof ABORT_REASONS)[number];
+
+/** Attempt-one-only accounting: a later correction is diagnosis, never a result. */
+export function firstAttempts(receipts: readonly Receipt[]): Receipt[] {
+    const best = new Map<string, Receipt>();
+    for (const r of receipts) {
+        // The separator is load-bearing, not cosmetic: without one, a
+        // configuration "ab" with case "c" collides with "a" and "bc", which
+        // silently drops a receipt — and the completeness precondition below
+        // would then report a missing first attempt for a case that has one.
+        const key = r.configuration + FIELD_SEP + r.caseId;
+        const seen = best.get(key);
+        if (seen === undefined || r.attempt < seen.attempt) best.set(key, r);
+    }
+    return [...best.values()];
+}
+
+export interface CompletenessVerdict {
+    publishable: boolean;
+    /** Every reason it is not, so one fix does not reveal the next one a day later. */
+    reasons: string[];
+}
+
+/**
+ * May this result be published?
+ *
+ * The precondition is exact and deliberately unforgiving: every configuration
+ * has exactly ONE first-attempt receipt for every case. A missing receipt is the
+ * shape that produces a published number over a silently smaller denominator,
+ * and a duplicate first attempt means two runs disagree about which one counted.
+ */
+export function completenessVerdict(
+    receipts: readonly Receipt[],
+    configurations: readonly string[],
+    cases: readonly string[],
+): CompletenessVerdict {
+    const reasons: string[] = [];
+    const firsts = receipts.filter((r) => r.attempt === 1);
+    for (const cfg of configurations) {
+        for (const c of cases) {
+            const n = firsts.filter((r) => r.configuration === cfg && r.caseId === c).length;
+            if (n === 0) reasons.push(`${cfg} / ${c}: no first-attempt receipt`);
+            else if (n > 1) reasons.push(`${cfg} / ${c}: ${String(n)} first-attempt receipts`);
+        }
+    }
+    for (const r of receipts) {
+        if (r.outcome !== 'aborted') continue;
+        const reason = r.failureReason;
+        if (reason === undefined || !(ABORT_REASONS as readonly string[]).includes(reason)) {
+            reasons.push(
+                `${r.configuration} / ${r.caseId}: aborted with ` +
+                    `${reason === undefined ? 'no reason' : `an unlisted reason "${reason}"`} — ` +
+                    `allowed: ${ABORT_REASONS.join(', ')}`,
+            );
+        }
+    }
+    return { publishable: reasons.length === 0, reasons };
+}
+
+// ── Phase 5.7 — a denominator that excludes what cannot self-activate ───────
+
+export interface CoverageInput {
+    /** Every artifact in the population. */
+    artifacts: readonly { id: string; can_self_activate: boolean; covered: boolean }[];
+}
+
+export interface CoverageReport {
+    /** Coverage over artifacts that CAN self-activate. */
+    rate: number | null;
+    denominator: number;
+    /** Reported separately, never folded into the rate. */
+    dependency_level: readonly string[];
+}
+
+/**
+ * Coverage, with non-self-activating artifacts excluded from the denominator.
+ *
+ * An arm that loads only an artifact which cannot activate itself runs
+ * treatment-equals-control by construction, so its score is judge noise. Folding
+ * such artifacts into a coverage rate publishes that noise as coverage. They are
+ * still reported — as dependency-level coverage, which is a real and different
+ * claim.
+ */
+export function coverageExcludingNonActivating(input: CoverageInput): CoverageReport {
+    const activating = input.artifacts.filter((a) => a.can_self_activate);
+    const dependency = input.artifacts.filter((a) => !a.can_self_activate).map((a) => a.id);
+    return {
+        rate:
+            activating.length === 0
+                ? null
+                : activating.filter((a) => a.covered).length / activating.length,
+        denominator: activating.length,
+        dependency_level: dependency,
+    };
+}
+
+// ── Phase 5.8 — the indeterminate branch, declared before the run ───────────
+
+export interface ThresholdDeclaration {
+    metric: string;
+    /** At or above this, the pre-registered claim is supported. */
+    pass: number;
+    /** At or below this, it is refuted. */
+    refute: number;
+}
+
+export type ThresholdOutcome = 'supported' | 'refuted' | 'indeterminate';
+
+export interface ThresholdVerdict {
+    outcome: ThresholdOutcome;
+    /** What a null here does and does not license, stated at declaration time. */
+    interpretation_limit: string;
+}
+
+/**
+ * Evaluate a measurement against a pre-registered threshold with a DECLARED
+ * indeterminate band.
+ *
+ * A single cut point forces every measurement into support or refutation, so a
+ * value that lands between them is read as whichever the reader hoped for. This
+ * package's own honest-null history is the argument: naming the indeterminate
+ * outcome BEFORE the run is what makes a null reportable rather than
+ * embarrassing.
+ *
+ * Throws when the band is inverted, because a declaration that cannot produce an
+ * indeterminate outcome is the single cut point wearing two numbers.
+ */
+export function evaluateThreshold(d: ThresholdDeclaration, measured: number): ThresholdVerdict {
+    if (d.refute >= d.pass) {
+        throw new Error(
+            `threshold declaration for "${d.metric}" has refute (${String(d.refute)}) >= pass ` +
+                `(${String(d.pass)}) — that leaves no indeterminate band, which is a single cut ` +
+                'point with two numbers written on it',
+        );
+    }
+    if (measured >= d.pass) {
+        return {
+            outcome: 'supported',
+            interpretation_limit:
+                'supports the pre-registered claim at the declared bar; it does not license a claim ' +
+                'about any metric that was not measured',
+        };
+    }
+    if (measured <= d.refute) {
+        return {
+            outcome: 'refuted',
+            interpretation_limit:
+                'refutes the pre-registered claim at the declared bar; it does not establish that ' +
+                'the opposite claim holds',
+        };
+    }
+    return {
+        outcome: 'indeterminate',
+        interpretation_limit:
+            `landed between the declared bars (${String(d.refute)} .. ${String(d.pass)}): the run ` +
+            'neither supports nor refutes. A null here licenses NO directional reading — it says ' +
+            'the instrument could not separate the arms at this sample size.',
+    };
+}