@event4u/agent-config 14.12.0 → 14.13.0

package/src/scripts/_lib/delivery_criticality.ts

@@ -0,0 +1,126 @@
+/**
+ * The fail-closed delivery ladder (`road-to-runtime-context-floors` step 2.3).
+ *
+ * An obligation moved out of the always-loaded prefix onto a runtime carrier is
+ * cheaper and can be MISSING. This module is the single encoding of what happens
+ * then; the prose lives in `docs/contracts/resident-process-floors.md` § 3 and
+ * the wire shape in `src/scripts/schemas/delivery-manifest.schema.json`.
+ *
+ * Three values, not a boolean, because "may this migrate" and "what if the
+ * carrier is down" are different questions and collapsing them loses the middle
+ * case that matters:
+ *
+ *   critical-A  never migrates; stays in standing context
+ *   critical-B  may migrate; carrier down ⇒ delivered EAGERLY, never dropped
+ *   standard    may migrate; carrier down ⇒ may fail open
+ *
+ * THE DEFAULT IS `critical-B`. An obligation whose class nobody declared is one
+ * nobody classified, and the cheap reading of an unclassified obligation must not
+ * be "droppable" — that is the same fail-closed choice
+ * `check_prefix_stable_mutation` makes for an undecidable write target.
+ */
+
+export const DELIVERY_CRITICALITY = ['critical-A', 'critical-B', 'standard'] as const;
+export type DeliveryCriticality = (typeof DELIVERY_CRITICALITY)[number];
+
+/** What nobody declared. Deliberately the middle rung, never the cheap one. */
+export const DEFAULT_CRITICALITY: DeliveryCriticality = 'critical-B';
+
+export interface DeliveryEntry {
+    /** Logical obligation id — the rule, gate or floor being delivered. */
+    id: string;
+    /** Omitted ⇒ {@link DEFAULT_CRITICALITY}. */
+    criticality?: DeliveryCriticality;
+    /** The runtime carrier that would deliver it. */
+    carrier?: string;
+}
+
+export type DeliveryOutcome =
+    /** Never left standing context; the carrier is irrelevant to it. */
+    | 'standing'
+    /** The carrier delivered it. */
+    | 'carrier'
+    /** The carrier is unavailable and the obligation was delivered the expensive way. */
+    | 'eager-fallback'
+    /** The carrier is unavailable and the obligation may be omitted. */
+    | 'fail-open';
+
+export interface DeliveryDecision {
+    id: string;
+    criticality: DeliveryCriticality;
+    /** True when the value was defaulted rather than declared. */
+    defaulted: boolean;
+    outcome: DeliveryOutcome;
+    reason: string;
+}
+
+export function isDeliveryCriticality(v: unknown): v is DeliveryCriticality {
+    return typeof v === 'string' && (DELIVERY_CRITICALITY as readonly string[]).includes(v);
+}
+
+/**
+ * Resolve one obligation against carrier availability.
+ *
+ * An UNRECOGNISED declared value is not honoured and does not throw: it
+ * defaults, exactly as an omitted one does. A typo must not be able to buy a
+ * weaker rung than the default — which is the direction a throw would also
+ * prevent, but a throw would take the whole delivery down with it, and losing
+ * every obligation to protect one is the wrong trade.
+ */
+export function resolveDelivery(entry: DeliveryEntry, carrierAvailable: boolean): DeliveryDecision {
+    const declared = entry.criticality;
+    const known = isDeliveryCriticality(declared);
+    const criticality: DeliveryCriticality = known ? declared : DEFAULT_CRITICALITY;
+    const defaulted = !known;
+
+    if (criticality === 'critical-A') {
+        return {
+            id: entry.id,
+            criticality,
+            defaulted,
+            outcome: 'standing',
+            reason: 'critical-A never migrates off standing context, so carrier availability cannot reach it',
+        };
+    }
+    if (carrierAvailable) {
+        return {
+            id: entry.id,
+            criticality,
+            defaulted,
+            outcome: 'carrier',
+            reason: `delivered by carrier ${entry.carrier ?? '(unnamed)'}`,
+        };
+    }
+    if (criticality === 'critical-B') {
+        return {
+            id: entry.id,
+            criticality,
+            defaulted,
+            outcome: 'eager-fallback',
+            reason:
+                `carrier ${entry.carrier ?? '(unnamed)'} is unavailable and this obligation is critical-B` +
+                (defaulted ? ' by default (its class was not declared)' : '') +
+                ' — delivered eagerly rather than dropped',
+        };
+    }
+    return {
+        id: entry.id,
+        criticality,
+        defaulted,
+        outcome: 'fail-open',
+        reason: `carrier ${entry.carrier ?? '(unnamed)'} is unavailable and this obligation is standard — may fail open`,
+    };
+}
+
+/** Resolve a whole manifest. Order is preserved so a report reads deterministically. */
+export function resolveDeliveryManifest(
+    entries: readonly DeliveryEntry[],
+    carrierAvailable: boolean,
+): DeliveryDecision[] {
+    return entries.map((e) => resolveDelivery(e, carrierAvailable));
+}
+
+/** The obligations a carrier failure must NOT silently drop. */
+export function droppedObligations(decisions: readonly DeliveryDecision[]): DeliveryDecision[] {
+    return decisions.filter((d) => d.outcome === 'fail-open');
+}

package/src/scripts/_lib/doctor_runtime_checks.ts

@@ -0,0 +1,70 @@
+/**
+ * The two runtime-presence checks of `agent-config doctor`.
+ *
+ * PURE MOVE out of `cmd_doctor.ts`, made under the extraction discipline that
+ * file's own `_lib` siblings follow: it is ~3,700 lines and
+ * `check_source_size_budget` counts every line above 1,500, so a change that
+ * adds lines there must pay for them by taking lines away. Wiring the Phase-1
+ * install-reach checks in (`road-to-consumer-repo-reality` 1.1) charged five
+ * lines; this move returns more than that.
+ *
+ * Both checks are cohesive with each other and with nothing else in that file:
+ * each answers "is a runtime present, and what does the package do when it is
+ * not", and neither reads the manifest, the drift sets, or the project root.
+ * Their one dependency is a `which` lookup, and it is passed IN rather than
+ * imported: `shutilWhich` lives in `cmd_doctor.ts` and importing it back would
+ * make the two modules circular. A parameter is cheaper than moving a shared
+ * utility that four other call sites already resolve from there.
+ *
+ * Behaviour is unchanged — same ids, same statuses, same message strings.
+ */
+/** The doctor's structured check row. Mirrors `cmd_doctor`'s internal `Dict`. */
+export interface RuntimeCheck {
+    id: string;
+    status: 'ok' | 'warn' | 'fail' | 'skipped';
+    message: string;
+    remedy: string;
+    /** Structural compatibility with `cmd_doctor`'s `Dict` row type. */
+    [k: string]: string;
+}
+
+export function checkPythonRuntime(): RuntimeCheck {
+    // Post-teardown: the package runtime is TypeScript-on-`tsx`. python3 is no
+    // longer a runtime dependency, so this check no longer probes for an
+    // interpreter (spawning python3 here would be misleading in a python-free
+    // package). The check id is retained for the doctor's stable report shape;
+    // it always reports `ok`.
+    return {
+        id: 'python-runtime',
+        status: 'ok',
+        message: 'python3 is not a runtime dependency (TS runtime via tsx)',
+        remedy: '',
+    };
+}
+
+export function checkHumanizerRuntime(which: (name: string) => string | null): RuntimeCheck {
+    // Write-engine step 4b (humanize audit) runs `detect_ai_tells.ts` via a
+    // Node/tsx runtime "when available", degrading to a prose-only audit
+    // otherwise. This check surfaces which path a consumer install gets so the
+    // default-on behavior is not a silent surprise. Either state is healthy —
+    // the fallback is graceful by design — so it never fails the run.
+    const node = which('node');
+    if (node === null) {
+        return {
+            id: 'humanizer-runtime',
+            status: 'ok',
+            message:
+                'no Node runtime on PATH — write-engine step 4b uses the ' +
+                'prose-only humanize audit (graceful fallback)',
+            remedy:
+                'install Node to enable the mechanical detect_ai_tells.ts pass; ' +
+                'the prose audit runs regardless',
+        };
+    }
+    return {
+        id: 'humanizer-runtime',
+        status: 'ok',
+        message: `Node runtime present (${node}) — step 4b runs the mechanical detect_ai_tells.ts pass`,
+        remedy: '',
+    };
+}

package/src/scripts/_lib/empty_cycles.ts

@@ -0,0 +1,88 @@
+/**
+ * A double trigger is one event, not two outcomes.
+ *
+ * Hosts fire the same slot more than once for a single logical event —
+ * `run_continuation_hook.ts` documents the same quirk on `stop` and dedupes it
+ * with an ordinal + open-count + short-window key. The consequence here is
+ * arithmetic rather than cosmetic: if a duplicate fire writes a second outcome
+ * record, every per-asset rate computed over the stream is inflated by exactly
+ * the duplication rate, and nothing downstream can separate the two.
+ *
+ * But a duplicate is not nothing, and discarding it silently is the other
+ * failure. The number of times a trigger fired without producing new work is
+ * itself a signal — it is how an idle loop looks from the outside. So a
+ * duplicate produces NO second outcome and DOES increment an empty-cycle
+ * counter, which is reported as its own quantity rather than folded into
+ * either the numerator or the denominator of a success rate.
+ *
+ * Pure and host-free on purpose: this is the seam the hook consumes, so the
+ * arithmetic can be tested without constructing a host payload.
+ */
+
+/** Default coincidence window. Same value the stop-slot dedupe uses. */
+export const DUPLICATE_WINDOW_MS = 60 * 1000;
+
+export interface TriggerFire {
+    /**
+     * Identity of the logical event. Two fires sharing a key inside the window
+     * are the same event seen twice. Id-shaped — never free-form text.
+     */
+    key: string;
+    /** Epoch milliseconds. */
+    at: number;
+}
+
+export interface CycleState {
+    /** The most recent fire that produced an outcome, or `null` before any. */
+    last: TriggerFire | null;
+    /** Fires suppressed as duplicates so far. */
+    empty_cycles: number;
+}
+
+export function initialCycleState(): CycleState {
+    return { last: null, empty_cycles: 0 };
+}
+
+export interface CycleDecision {
+    /** Whether this fire should produce an outcome record. */
+    emit_outcome: boolean;
+    /** The state to carry forward. */
+    state: CycleState;
+}
+
+/**
+ * Decide what one fire means.
+ *
+ * A fire is a duplicate when it carries the SAME key as the last
+ * outcome-producing fire AND lands inside the window. Both conjuncts are
+ * required and neither is sufficient:
+ *
+ * - Key alone would collapse a genuine repeat of the same work an hour later
+ *   into an empty cycle, erasing a real outcome.
+ * - Window alone would collapse two DIFFERENT events that happen to be close
+ *   together, which is the common case under any kind of fan-out.
+ *
+ * A duplicate does not advance `last`. That is deliberate: three fires inside
+ * one window are one outcome and two empty cycles, not one outcome, one empty
+ * cycle, and a third fire that has drifted out of the window and looks new.
+ */
+export function observeFire(
+    state: CycleState,
+    fire: TriggerFire,
+    windowMs: number = DUPLICATE_WINDOW_MS,
+): CycleDecision {
+    const prev = state.last;
+    const isDuplicate =
+        prev !== null && prev.key === fire.key && fire.at - prev.at < windowMs;
+
+    if (isDuplicate) {
+        return {
+            emit_outcome: false,
+            state: { last: prev, empty_cycles: state.empty_cycles + 1 },
+        };
+    }
+    return {
+        emit_outcome: true,
+        state: { last: fire, empty_cycles: state.empty_cycles },
+    };
+}

package/src/scripts/_lib/eval_publication.ts

@@ -0,0 +1,354 @@
+/**
+ * Comparison arms that can discriminate, and results honest enough to publish.
+ *
+ * `road-to-skill-ecosystem-eval-integrity` Phases 4 and 5. Every function here
+ * exists because a benchmark can be run correctly and still produce a number
+ * nobody should act on — and each of those routes is a separate mechanism, not
+ * a matter of care.
+ */
+import { createHash } from 'node:crypto';
+
+// ── Phase 4.2 — a fixture the control arm cannot also satisfy ───────────────
+
+export interface PlantedItem {
+    id: string;
+    /**
+     * True when this item is recoverable ONLY via the behaviour the artifact
+     * under test prescribes. At least one such item must exist, or the control
+     * arm scores the same and the eval adjudicates nothing.
+     */
+    requires_artifact_behaviour: boolean;
+}
+
+/**
+ * Can this fixture set discriminate at all?
+ *
+ * The check runs BEFORE a run rather than after, because the answer does not
+ * depend on the outcome: a fixture set where every item is recoverable by
+ * general competence measures general competence, whatever the arms score. A
+ * run over such a set is not a weak result — it is not a result.
+ */
+export function discriminationDeficit(items: readonly PlantedItem[]): string | null {
+    if (items.length === 0) return 'no fixture items — nothing to recover';
+    if (!items.some((i) => i.requires_artifact_behaviour)) {
+        return (
+            `none of ${String(items.length)} fixture item(s) is recoverable only via the behaviour ` +
+            'under test, so the control arm can score identically and the eval cannot adjudicate'
+        );
+    }
+    return null;
+}
+
+// ── Phase 4.3 — a missed plant is a zero, never an absence ──────────────────
+
+export interface RubricScoreInput {
+    /** Items the run actually reported on. */
+    reported: readonly { id: string; score: number }[];
+    /** Every item that was planted, whether reported or not. */
+    planted: readonly string[];
+}
+
+export interface RubricScore {
+    /** Sum over PLANTED items, missing ones counted as zero. */
+    total: number;
+    /** Denominator is the planted count, never the reported count. */
+    denominator: number;
+    ratio: number;
+    missed: readonly string[];
+}
+
+/**
+ * Score a rubric with missed plants counted as implicit zeros.
+ *
+ * Dividing by what was REPORTED is the defect: a run that finds one item and
+ * says nothing about the other nine scores 100 %, so under-reporting buys a
+ * higher ratio. The denominator has to be the plant.
+ */
+export function scoreWithImplicitZeros(input: RubricScoreInput): RubricScore {
+    const byId = new Map(input.reported.map((r) => [r.id, r.score]));
+    const missed = input.planted.filter((p) => !byId.has(p));
+    const total = input.planted.reduce((a, id) => a + (byId.get(id) ?? 0), 0);
+    const denominator = input.planted.length;
+    return {
+        total,
+        denominator,
+        ratio: denominator === 0 ? 0 : total / denominator,
+        missed,
+    };
+}
+
+// ── Phase 4.4 — did the run read what it was denied? ────────────────────────
+
+export interface LeakScanInput {
+    /** Paths the run must not have read: ground truth, generator, scorers, prior trials. */
+    denied: readonly string[];
+    /** Every tool input the run recorded, in order. */
+    toolInputs: readonly string[];
+}
+
+export interface LeakFinding {
+    denied: string;
+    /** The tool input that matched. */
+    evidence: string;
+}
+
+/**
+ * Walk a run transcript for reads of denied paths.
+ *
+ * Runs AFTER the trial, on the recorded transcript, because a denial that is
+ * only configured is a denial nobody verified. Matching is substring on a
+ * path with separators normalised — a leak arriving through a symlink or a
+ * parent-directory hop is not something this catches, and saying so is cheaper
+ * than implying it does.
+ */
+export function scanLeaks(input: LeakScanInput): LeakFinding[] {
+    const out: LeakFinding[] = [];
+    const norm = (s: string): string => s.split('\\').join('/');
+    for (const d of input.denied) {
+        const needle = norm(d);
+        if (needle === '') continue;
+        for (const t of input.toolInputs) {
+            if (norm(t).includes(needle)) {
+                out.push({ denied: d, evidence: t.slice(0, 200) });
+                break;
+            }
+        }
+    }
+    return out;
+}
+
+// ── Phase 4.5 — a cache key that includes the CRITERIA ──────────────────────
+
+/** Field delimiter for the identity hash. An escape, never a raw byte. */
+const FIELD_SEP = '\u001e';
+/** Element delimiter inside one array field. Different from FIELD_SEP on purpose. */
+const ELEMENT_SEP = '\u001f';
+
+export interface BaselineIdentity {
+    prompt: string;
+    fixtures: readonly string[];
+    /** The half most likely to be omitted, which is why it is required here. */
+    rubric: readonly string[];
+    assertions: readonly string[];
+    toolExpectations: readonly string[];
+    turnLimit: number;
+    tokenLimit: number;
+}
+
+/**
+ * Composite identity hash for a cached baseline.
+ *
+ * Keying on prompt and fixtures alone is the common shape and the wrong one: an
+ * edited rubric changes what a pass MEANS while the key stays identical, so the
+ * cache serves a baseline scored under criteria that no longer exist. The
+ * criteria half is required by the type rather than optional, because an
+ * optional field that is easy to omit is a field that gets omitted.
+ */
+export function baselineIdentity(id: BaselineIdentity): string {
+    const h = createHash('sha256');
+    const feed = (label: string, v: string | readonly string[] | number): void => {
+        // The separators are ESCAPES, never raw control bytes: a raw U+1F in a
+        // text source makes grep, file(1) and git treat the file as binary and
+        // skip it silently, which `lint_hidden_unicode` refuses. A field
+        // delimiter and an element delimiter that differ are what keep two
+        // fields concatenating to the same text from producing one hash.
+        h.update(FIELD_SEP + label + FIELD_SEP);
+        h.update(Array.isArray(v) ? [...v].join(ELEMENT_SEP) : String(v));
+    };
+    feed('prompt', id.prompt);
+    feed('fixtures', id.fixtures);
+    feed('rubric', id.rubric);
+    feed('assertions', id.assertions);
+    feed('tools', id.toolExpectations);
+    feed('turns', id.turnLimit);
+    feed('tokens', id.tokenLimit);
+    return h.digest('hex');
+}
+
+// ── Phase 5.1 / 5.2 — first attempts, and the completeness precondition ─────
+
+export interface Receipt {
+    caseId: string;
+    configuration: string;
+    /** 1 for the first attempt. A later correction carries 2, 3, … */
+    attempt: number;
+    outcome: 'pass' | 'fail' | 'aborted';
+    /** Required when `outcome` is `aborted`, and must be an allow-listed reason. */
+    failureReason?: string | undefined;
+}
+
+/**
+ * Reasons an aborted run may record. A closed set on purpose.
+ *
+ * The alternative to an allow-list is free text, and free text lets an abort be
+ * recorded as anything — including something that reads like a result. A run
+ * that stopped is a truthful receipt with its gates marked not-run, never a
+ * dropped run and never a fabricated artifact.
+ */
+export const ABORT_REASONS = [
+    'quota-exhausted',
+    'credential-missing',
+    'dependency-unavailable',
+    'timeout',
+    'harness-error',
+    'operator-cancelled',
+] as const;
+export type AbortReason = (typeof ABORT_REASONS)[number];
+
+/** Attempt-one-only accounting: a later correction is diagnosis, never a result. */
+export function firstAttempts(receipts: readonly Receipt[]): Receipt[] {
+    const best = new Map<string, Receipt>();
+    for (const r of receipts) {
+        // The separator is load-bearing, not cosmetic: without one, a
+        // configuration "ab" with case "c" collides with "a" and "bc", which
+        // silently drops a receipt — and the completeness precondition below
+        // would then report a missing first attempt for a case that has one.
+        const key = r.configuration + FIELD_SEP + r.caseId;
+        const seen = best.get(key);
+        if (seen === undefined || r.attempt < seen.attempt) best.set(key, r);
+    }
+    return [...best.values()];
+}
+
+export interface CompletenessVerdict {
+    publishable: boolean;
+    /** Every reason it is not, so one fix does not reveal the next one a day later. */
+    reasons: string[];
+}
+
+/**
+ * May this result be published?
+ *
+ * The precondition is exact and deliberately unforgiving: every configuration
+ * has exactly ONE first-attempt receipt for every case. A missing receipt is the
+ * shape that produces a published number over a silently smaller denominator,
+ * and a duplicate first attempt means two runs disagree about which one counted.
+ */
+export function completenessVerdict(
+    receipts: readonly Receipt[],
+    configurations: readonly string[],
+    cases: readonly string[],
+): CompletenessVerdict {
+    const reasons: string[] = [];
+    const firsts = receipts.filter((r) => r.attempt === 1);
+    for (const cfg of configurations) {
+        for (const c of cases) {
+            const n = firsts.filter((r) => r.configuration === cfg && r.caseId === c).length;
+            if (n === 0) reasons.push(`${cfg} / ${c}: no first-attempt receipt`);
+            else if (n > 1) reasons.push(`${cfg} / ${c}: ${String(n)} first-attempt receipts`);
+        }
+    }
+    for (const r of receipts) {
+        if (r.outcome !== 'aborted') continue;
+        const reason = r.failureReason;
+        if (reason === undefined || !(ABORT_REASONS as readonly string[]).includes(reason)) {
+            reasons.push(
+                `${r.configuration} / ${r.caseId}: aborted with ` +
+                    `${reason === undefined ? 'no reason' : `an unlisted reason "${reason}"`} — ` +
+                    `allowed: ${ABORT_REASONS.join(', ')}`,
+            );
+        }
+    }
+    return { publishable: reasons.length === 0, reasons };
+}
+
+// ── Phase 5.7 — a denominator that excludes what cannot self-activate ───────
+
+export interface CoverageInput {
+    /** Every artifact in the population. */
+    artifacts: readonly { id: string; can_self_activate: boolean; covered: boolean }[];
+}
+
+export interface CoverageReport {
+    /** Coverage over artifacts that CAN self-activate. */
+    rate: number | null;
+    denominator: number;
+    /** Reported separately, never folded into the rate. */
+    dependency_level: readonly string[];
+}
+
+/**
+ * Coverage, with non-self-activating artifacts excluded from the denominator.
+ *
+ * An arm that loads only an artifact which cannot activate itself runs
+ * treatment-equals-control by construction, so its score is judge noise. Folding
+ * such artifacts into a coverage rate publishes that noise as coverage. They are
+ * still reported — as dependency-level coverage, which is a real and different
+ * claim.
+ */
+export function coverageExcludingNonActivating(input: CoverageInput): CoverageReport {
+    const activating = input.artifacts.filter((a) => a.can_self_activate);
+    const dependency = input.artifacts.filter((a) => !a.can_self_activate).map((a) => a.id);
+    return {
+        rate:
+            activating.length === 0
+                ? null
+                : activating.filter((a) => a.covered).length / activating.length,
+        denominator: activating.length,
+        dependency_level: dependency,
+    };
+}
+
+// ── Phase 5.8 — the indeterminate branch, declared before the run ───────────
+
+export interface ThresholdDeclaration {
+    metric: string;
+    /** At or above this, the pre-registered claim is supported. */
+    pass: number;
+    /** At or below this, it is refuted. */
+    refute: number;
+}
+
+export type ThresholdOutcome = 'supported' | 'refuted' | 'indeterminate';
+
+export interface ThresholdVerdict {
+    outcome: ThresholdOutcome;
+    /** What a null here does and does not license, stated at declaration time. */
+    interpretation_limit: string;
+}
+
+/**
+ * Evaluate a measurement against a pre-registered threshold with a DECLARED
+ * indeterminate band.
+ *
+ * A single cut point forces every measurement into support or refutation, so a
+ * value that lands between them is read as whichever the reader hoped for. This
+ * package's own honest-null history is the argument: naming the indeterminate
+ * outcome BEFORE the run is what makes a null reportable rather than
+ * embarrassing.
+ *
+ * Throws when the band is inverted, because a declaration that cannot produce an
+ * indeterminate outcome is the single cut point wearing two numbers.
+ */
+export function evaluateThreshold(d: ThresholdDeclaration, measured: number): ThresholdVerdict {
+    if (d.refute >= d.pass) {
+        throw new Error(
+            `threshold declaration for "${d.metric}" has refute (${String(d.refute)}) >= pass ` +
+                `(${String(d.pass)}) — that leaves no indeterminate band, which is a single cut ` +
+                'point with two numbers written on it',
+        );
+    }
+    if (measured >= d.pass) {
+        return {
+            outcome: 'supported',
+            interpretation_limit:
+                'supports the pre-registered claim at the declared bar; it does not license a claim ' +
+                'about any metric that was not measured',
+        };
+    }
+    if (measured <= d.refute) {
+        return {
+            outcome: 'refuted',
+            interpretation_limit:
+                'refutes the pre-registered claim at the declared bar; it does not establish that ' +
+                'the opposite claim holds',
+        };
+    }
+    return {
+        outcome: 'indeterminate',
+        interpretation_limit:
+            `landed between the declared bars (${String(d.refute)} .. ${String(d.pass)}): the run ` +
+            'neither supports nor refutes. A null here licenses NO directional reading — it says ' +
+            'the instrument could not separate the arms at this sample size.',
+    };
+}