@event4u/agent-config 14.11.0 → 14.13.0

package/src/scripts/_lib/activation_states.ts

@@ -0,0 +1,88 @@
+/**
+ * The five states a per-asset outcome can occupy, and why three would not do.
+ *
+ * A per-asset "win rate" computed over two states — worked / did not work —
+ * is uninterpretable, and the failure is not cosmetic. Take a skill whose
+ * content is excellent and which the router never activates. Its success rate
+ * is low. Every fix that number suggests is a change to the skill, and every
+ * one of them would improve nothing, because the skill was never reached. The
+ * split below is what makes the number point at the right layer.
+ *
+ * Three rungs are observed independently:
+ *
+ *   available -> was the asset present in this install at all?
+ *   activated -> did the router/agent actually load it for this task?
+ *   followed  -> did the work that followed conform to what it said?
+ *
+ * Each rung is `true`, `false`, or `null` for "not observed". `null` is not a
+ * pessimistic `false`: an unobserved rung means the instrument did not see,
+ * which is a statement about the instrument. Collapsing it into either
+ * direction manufactures a signal, so `classify` returns `'unknown'` the moment
+ * any rung it would need is unobserved.
+ */
+
+/** Ordered from least to most engaged. `unknown` is not on that ladder. */
+export const ACTIVATION_STATES = [
+    /** The asset is not present in this install. Nothing about its quality follows. */
+    'not-available',
+    /** Present, and the router never loaded it. A content fix cannot help this case. */
+    'available-not-activated',
+    /** Loaded, and the work did not conform to it. This is the case a content or clarity fix addresses. */
+    'activated-not-followed',
+    /** Loaded and conformed to. The only state that may count toward a win rate. */
+    'activated-followed',
+    /**
+     * At least one rung was not observed. Reported as its own share, never
+     * folded into a success or a failure — a report that hides its own blind
+     * spot is worse than one that has none, because the reader cannot see the
+     * hole.
+     */
+    'unknown',
+] as const;
+
+export type ActivationState = (typeof ACTIVATION_STATES)[number];
+
+export function isActivationState(v: unknown): v is ActivationState {
+    return typeof v === 'string' && (ACTIVATION_STATES as readonly string[]).includes(v);
+}
+
+/** One observation per rung. `null` means "not observed", never "no". */
+export interface ActivationObservation {
+    available: boolean | null;
+    activated: boolean | null;
+    followed: boolean | null;
+}
+
+/**
+ * Classify one observation.
+ *
+ * The short-circuits are deliberate and are the reason this is a function
+ * rather than a lookup table: once `available` is observed `false`, the other
+ * two rungs are IRRELEVANT rather than unknown — an absent asset cannot have
+ * been activated — so an unobserved `followed` must not drag the answer to
+ * `'unknown'`. The same holds one rung down for `activated: false`.
+ *
+ * Everywhere else an unobserved rung wins, because the alternative is to invent
+ * the observation.
+ */
+export function classify(obs: ActivationObservation): ActivationState {
+    if (obs.available === null) return 'unknown';
+    if (obs.available === false) return 'not-available';
+
+    if (obs.activated === null) return 'unknown';
+    if (obs.activated === false) return 'available-not-activated';
+
+    if (obs.followed === null) return 'unknown';
+    return obs.followed ? 'activated-followed' : 'activated-not-followed';
+}
+
+/**
+ * True for the states that may enter a win-rate denominator.
+ *
+ * `unknown` is excluded because it is not an outcome, and `not-available` is
+ * excluded because it is not a fact about the asset's quality. Exported so a
+ * report cannot quietly choose a different denominator per column.
+ */
+export function countsTowardWinRate(state: ActivationState): boolean {
+    return state === 'activated-followed' || state === 'activated-not-followed';
+}

package/src/scripts/_lib/adherence_detectors.ts

@@ -0,0 +1,143 @@
+/**
+ * Deterministic adherence detection, and a hard refusal to guess.
+ *
+ * Step 5.1 splits a per-asset outcome into five states. The hardest of its
+ * rungs to observe is `followed`: whether the work actually conformed to the
+ * rule that was loaded. It is tempting to have a model read the transcript and
+ * say. That is precisely what this module refuses to do.
+ *
+ * The rule, from step 5.2: where a rule has an OBSERVABLE FOOTPRINT, adherence
+ * is computed from that footprint. Where it has none, adherence is `unknown` —
+ * never inferred, never a model's guess. An inferred adherence figure is worse
+ * than a missing one, because a missing one is visible in the `unknown` share
+ * and an inferred one is not.
+ *
+ * A DETECTOR ANSWERS FOR ONE CLAUSE, NOT FOR A RULE. Rules in this tree carry
+ * many clauses; `think-before-action` alone spans a read set, a verification
+ * matrix, and a test-first preference. A detector that observed one clause and
+ * reported "rule followed" would be overclaiming by construction, so every
+ * registration below names the clause it decides and the registry key is the
+ * pair, not the rule id.
+ */
+
+import type { ActivationState } from './activation_states.js';
+
+/** The subset of {@link ActivationState} a detector may return. */
+export type Adherence = Extract<
+    ActivationState,
+    'activated-followed' | 'activated-not-followed' | 'unknown'
+>;
+
+/**
+ * One observed write, reduced to the two things a detector may see: where it
+ * went, and in what order relative to the others.
+ *
+ * Deliberately carries NO content field — a detector that could read a diff
+ * body would be a second place for file contents to travel, and the audit
+ * privacy floor exists to stop exactly that.
+ */
+export interface WriteObservation {
+    /** Repository-relative path. */
+    path: string;
+    /** Monotonic ordering key within one task. Absolute time is not required. */
+    seq: number;
+}
+
+export interface DetectorInput {
+    writes: readonly WriteObservation[];
+}
+
+export interface ClauseDetector {
+    /** The rule whose clause this decides. */
+    rule_id: string;
+    /** Which clause — a rule has many, and a detector answers for one. */
+    clause: string;
+    /** Where that clause is written, so a reader can check the detector against it. */
+    source: string;
+    detect(input: DetectorInput): Adherence;
+}
+
+/** A path is a test path when it sits under a `tests/` segment or is named `*.test.*` / `*.spec.*`. */
+export function isTestPath(p: string): boolean {
+    const normalized = p.replace(/\\/g, '/');
+    if (/(^|\/)tests?(\/|$)/.test(normalized)) return true;
+    return /\.(test|spec)\.[cm]?[jt]sx?$/.test(normalized);
+}
+
+/**
+ * `think-before-action` § "When behavior can be defined → prefer test-first /
+ * TDD" (`dist/agent-src/rules/think-before-action.md:43`).
+ *
+ * The footprint is write ORDER: a test file written before the production file
+ * it covers is test-first; the reverse is not.
+ *
+ * The `unknown` branch is the load-bearing one and is wider than it first
+ * looks. If only test writes are observed, no production change happened and
+ * the clause did not apply. If only production writes are observed, it is
+ * tempting to answer `activated-not-followed` — and that would be a GUESS: the
+ * test may have been written in an earlier task, or the change may be one the
+ * clause's own "when behavior can be defined" qualifier excludes. Both
+ * one-sided cases are therefore `unknown`, which is the honest answer and the
+ * conservative one in the direction that matters — it never manufactures a
+ * violation.
+ */
+export const testFirstDetector: ClauseDetector = {
+    rule_id: 'think-before-action',
+    clause: 'prefer test-first / TDD when behavior can be defined',
+    source: 'dist/agent-src/rules/think-before-action.md:43',
+    detect({ writes }) {
+        let firstTest: number | null = null;
+        let firstProd: number | null = null;
+        for (const w of writes) {
+            if (isTestPath(w.path)) {
+                if (firstTest === null || w.seq < firstTest) firstTest = w.seq;
+            } else if (firstProd === null || w.seq < firstProd) {
+                firstProd = w.seq;
+            }
+        }
+        // Either side missing means the order was never observed, not that it
+        // was violated. See the note above on why the production-only case is
+        // deliberately NOT a violation.
+        if (firstTest === null || firstProd === null) return 'unknown';
+        return firstTest < firstProd ? 'activated-followed' : 'activated-not-followed';
+    },
+};
+
+/**
+ * The registry. Keyed `${rule_id}#${clause}` so one rule may gain a second
+ * detector later without either silently replacing the other.
+ */
+const DETECTORS: readonly ClauseDetector[] = [testFirstDetector];
+
+export function detectorsFor(ruleId: string): readonly ClauseDetector[] {
+    return DETECTORS.filter((d) => d.rule_id === ruleId);
+}
+
+/**
+ * Adherence for one rule.
+ *
+ * A rule with NO registered detector returns `unknown`. That is the whole
+ * point: 118 of this tree's 119 rules have no observable footprint today, and
+ * the honest report says so rather than filling the column with inference.
+ *
+ * With several detectors registered for one rule, a single observed violation
+ * wins over a `followed` — a rule is not adhered to if any decided clause was
+ * broken — and `unknown` only survives when nothing was decided at all.
+ */
+export function detectAdherence(ruleId: string, input: DetectorInput): Adherence {
+    const detectors = detectorsFor(ruleId);
+    if (detectors.length === 0) return 'unknown';
+
+    let sawFollowed = false;
+    for (const d of detectors) {
+        const verdict = d.detect(input);
+        if (verdict === 'activated-not-followed') return 'activated-not-followed';
+        if (verdict === 'activated-followed') sawFollowed = true;
+    }
+    return sawFollowed ? 'activated-followed' : 'unknown';
+}
+
+/** Rule ids that have at least one detector. Everything else reports `unknown`. */
+export function rulesWithDetectors(): readonly string[] {
+    return [...new Set(DETECTORS.map((d) => d.rule_id))];
+}

package/src/scripts/_lib/arith_claims.ts

@@ -0,0 +1,181 @@
+/**
+ * Check an arithmetic claim written in prose against itself.
+ *
+ * The defect this exists for, observed rather than imagined: a fixture states
+ * the same number twice — once as a declared `expected` and once inside the
+ * rationale that derives it — and the two disagree. A judge then prefers the
+ * declared value over the recomputed one and scores a correct response as a
+ * loss. Both numbers are in the file, so nothing has to be inferred; they
+ * simply were never compared.
+ *
+ * The scanner reads CHAINS (`a x b / c = d / c = e`), because that is how the
+ * derivations in this tree are actually written. Comparing only the outermost
+ * pair would mis-read every intermediate step as a claim of its own — the
+ * first version of this check did exactly that and produced three false
+ * findings on a corpus that was correct.
+ *
+ * Prose rounding is legitimate and is tolerated to the PRECISION THE PROSE
+ * ITSELF USES: `1.333` agrees with 4/3, `1.33` does not have to. A fixed
+ * epsilon would either reject honest rounding or accept a real disagreement,
+ * depending on the magnitude of the numbers, which is the wrong axis.
+ */
+
+/** One prose arithmetic chain and what it evaluates to. */
+export interface ArithChain {
+    /** The chain as written. */
+    text: string;
+    /** Every `=`-separated segment that evaluated to a number. */
+    values: readonly { segment: string; value: number }[];
+}
+
+/** A chain whose segments disagree beyond the precision the prose claims. */
+export interface ArithDisagreement {
+    text: string;
+    leftSegment: string;
+    left: number;
+    rightSegment: string;
+    right: number;
+    /** The tolerance the right-hand segment's own precision earns it. */
+    tolerance: number;
+}
+
+const CHAIN = /(?:[-+]?[\d,]*\.?\d+[\s\d,.+\-*/x×^()%]*=\s*)+[-+]?[\d,]*\.?\d+/g;
+
+/**
+ * Evaluate one prose segment, or `null` when it is not pure arithmetic.
+ *
+ * Deliberately narrow: digits, the four operators, `^`, parentheses and the
+ * separators prose uses. Anything else returns `null` rather than being
+ * coerced — a segment carrying a word is a sentence, not a claim, and guessing
+ * at it is how a checker starts inventing findings.
+ */
+export function evaluateSegment(segment: string): number | null {
+    const t = segment
+        .trim()
+        .replace(/,/g, '')
+        .replace(/[×x]/g, '*')
+        .replace(/%/g, '')
+        .replace(/\$/g, '');
+    if (!/\d/.test(t)) return null;
+    if (!/^[\d\s.+\-*/^()]+$/.test(t)) return null;
+    // `^` is exponentiation in prose and XOR in JS — translate before eval.
+    const js = t.replace(/\^/g, '**');
+    try {
+        // Input is whitelisted above to digits, operators and parentheses — a
+        // segment carrying anything else returned null before reaching here.
+        const fn = new Function(`"use strict"; return (${js});`) as () => unknown;
+        const v = fn();
+        return typeof v === 'number' && Number.isFinite(v) ? v : null;
+    } catch {
+        return null;
+    }
+}
+
+/** Decimal places written in a literal, used as the rounding tolerance. */
+function precisionTolerance(segment: string, magnitude: number): number {
+    const m = /\.(\d+)\s*$/.exec(segment.trim());
+    const decimals = m?.[1];
+    if (decimals !== undefined) return 0.5 * 10 ** -decimals.length;
+    // An integer-looking claim is tolerated to half a unit, plus float noise
+    // proportional to the magnitude — `20,000` may legitimately stand for
+    // 20000.000000001 produced by a chain of divisions.
+    return Math.max(0.5, Math.abs(magnitude) * 1e-9);
+}
+
+/** Every arithmetic chain in a piece of prose. */
+export function findArithChains(prose: string): ArithChain[] {
+    const out: ArithChain[] = [];
+    for (const m of prose.matchAll(CHAIN)) {
+        const values: { segment: string; value: number }[] = [];
+        for (const segment of m[0].split('=')) {
+            const value = evaluateSegment(segment);
+            if (value !== null) values.push({ segment, value });
+        }
+        if (values.length >= 2) out.push({ text: m[0], values });
+    }
+    return out;
+}
+
+/**
+ * Every chain whose own segments disagree.
+ *
+ * Each segment is compared against the FIRST evaluable one rather than against
+ * its neighbour: a chain is a claim that all of its forms are the same number,
+ * and comparing neighbours lets an error accumulate quietly across three steps.
+ */
+export function findArithDisagreements(prose: string): ArithDisagreement[] {
+    const out: ArithDisagreement[] = [];
+    for (const chain of findArithChains(prose)) {
+        const first = chain.values[0];
+        if (first === undefined) continue;
+        for (const later of chain.values.slice(1)) {
+            const tolerance = precisionTolerance(later.segment, first.value);
+            if (Math.abs(later.value - first.value) > tolerance) {
+                out.push({
+                    text: chain.text,
+                    leftSegment: first.segment.trim(),
+                    left: first.value,
+                    rightSegment: later.segment.trim(),
+                    right: later.value,
+                    tolerance,
+                });
+                break;
+            }
+        }
+    }
+    return out;
+}
+
+/**
+ * Does a prose derivation ever state the declared value?
+ *
+ * Returns `null` when the prose carries no evaluable chain at all — an absent
+ * derivation is not a disagreement, and treating it as one would make the
+ * check fire on every fixture that simply cites a source instead.
+ */
+export function chainSupportsExpected(
+    prose: string,
+    expected: number,
+    tolerance: number,
+): boolean | null {
+    const chains = findArithChains(prose);
+    if (chains.length === 0) return null;
+    for (const chain of chains) {
+        for (const v of chain.values) {
+            const tol = Math.max(tolerance, precisionTolerance(v.segment, expected));
+            if (Math.abs(v.value - expected) <= tol) return true;
+            if (sameMantissa(v.value, expected, tol)) return true;
+        }
+    }
+    return false;
+}
+
+/**
+ * Do two numbers agree once a UNIT SCALE is allowed?
+ *
+ * A measured false positive is what this exists for, not a hypothetical: a
+ * fixture derives `4,200,000,000 x 0.006 = 25,200,000 = $25.2M` and declares
+ * `expected: 25.2`, because the scenario asks for the figure "in $M". The
+ * derivation works in dollars and the expectation in millions. Nothing in the
+ * file states the unit in a machine-readable way, so demanding exact agreement
+ * reports a correct fixture as broken.
+ *
+ * **The weakening this accepts, stated rather than buried.** A real defect that
+ * happens to be a pure power of ten away — `0.85` written where `8.5` is meant —
+ * is no longer caught by THIS check. It is a narrow hole: a transcription or
+ * arithmetic error almost always changes the digit sequence, and the
+ * self-consistency check above still reads the derivation on its own terms,
+ * where no scale tolerance applies at all. The alternative was worse: a check
+ * that fires on every unit-scaled fixture is a check that gets suppressed.
+ *
+ * Bounded to 10^±12 so an unrelated number cannot be dragged into agreement by
+ * an arbitrarily large exponent.
+ */
+function sameMantissa(a: number, b: number, tolerance: number): boolean {
+    if (a === 0 || b === 0) return false;
+    const ratio = Math.abs(a / b);
+    const exp = Math.round(Math.log10(ratio));
+    if (!Number.isFinite(exp) || Math.abs(exp) > 12 || exp === 0) return false;
+    const scaled = a / 10 ** exp;
+    return Math.abs(scaled - b) <= Math.max(tolerance, Math.abs(b) * 1e-9);
+}

package/src/scripts/_lib/arm_ranking.ts

@@ -0,0 +1,62 @@
+/**
+ * Arm ranking metrics (`road-to-runtime-context-floors` step 4.2).
+ *
+ * `tokens` stays the default and `cost-per-solved` becomes AVAILABLE — not a
+ * replacement. The reason it has to exist at all is the same one behind the
+ * mandatory cache fields: an arm can reduce token volume and cost more, because
+ * rewriting a prefix pays the cache-write rate rather than the cache-read rate.
+ * A comparison that can only rank on tokens cannot see that.
+ *
+ * The reason it must NOT become the default is the sibling risk: a single
+ * mandatory score invites tuning the score. Keeping both available, and keeping
+ * the cache ratio diagnostic rather than scored, is what stops this from being
+ * the Goodhart shape that `road-to-cache-economy`'s falsified C-5 already
+ * demonstrated for token count.
+ */
+
+export const RANKING_METRICS = ['tokens', 'cost-per-solved'] as const;
+export type RankingMetric = (typeof RANKING_METRICS)[number];
+export const DEFAULT_RANKING_METRIC: RankingMetric = 'tokens';
+
+export interface ArmResult {
+    arm: string;
+    /** Total tokens consumed by the arm across the run. */
+    tokens: number;
+    /** Total cost in USD. */
+    cost_usd: number;
+    /** Tasks the arm actually solved — the denominator of cost-per-solved. */
+    solved: number;
+}
+
+export interface RankedArm extends ArmResult {
+    /** The value ranked on. `null` when the metric is undefined for this arm. */
+    score: number | null;
+    /** Present only when `score` is null. */
+    reason?: string;
+}
+
+export function isRankingMetric(v: unknown): v is RankingMetric {
+    return typeof v === 'string' && (RANKING_METRICS as readonly string[]).includes(v);
+}
+
+function score(arm: ArmResult, metric: RankingMetric): { score: number | null; reason?: string } {
+    if (metric === 'tokens') return { score: arm.tokens };
+    if (arm.solved === 0) {
+        // Not Infinity and not 0: an arm that solved nothing has no
+        // cost-per-solved, and ranking it as "infinitely expensive" would be a
+        // number nobody measured.
+        return { score: null, reason: 'the arm solved nothing, so cost-per-solved is undefined' };
+    }
+    return { score: arm.cost_usd / arm.solved };
+}
+
+/**
+ * Rank ascending — lower is better under both metrics. Arms with an undefined
+ * score sort last, in input order, and carry their reason.
+ */
+export function rankArms(arms: readonly ArmResult[], metric: RankingMetric = DEFAULT_RANKING_METRIC): RankedArm[] {
+    const scored: RankedArm[] = arms.map((a) => ({ ...a, ...score(a, metric) }));
+    const defined = scored.filter((a) => a.score !== null).sort((a, b) => (a.score as number) - (b.score as number) || a.arm.localeCompare(b.arm));
+    const undef = scored.filter((a) => a.score === null);
+    return [...defined, ...undef];
+}