create-agent-rig 0.8.0 → 0.9.1

package/packages/cli/dist/policy/benchmark/corpus.js

@@ -0,0 +1,165 @@
+import { CAPABILITY_STATES } from '../core/vocabulary.js';
+export const BENCHMARK_CLASSIFICATIONS = Object.freeze([
+    'equivalent',
+    'intentional-degradation',
+    'unsupported',
+]);
+const MEASUREMENT_KINDS = Object.freeze([
+    'capability-probe',
+    'guard-enforcement',
+    'compatibility-rejection',
+]);
+const REQUIRED_SCENARIO_IDS = [
+    'real-wiring',
+    'unwired-enforcement',
+    'disabled-enforcement',
+    'narrowed-enforcement',
+    'bypassed-enforcement',
+    'unreadable-input',
+    'hook-input',
+    'protected-rulebook',
+    'widening',
+    'foreign-major',
+];
+const requiredScenarioIds = new Set(REQUIRED_SCENARIO_IDS);
+const scenario = (id, fixture, mutation, action, measurementKind, expectedStates) => Object.freeze({
+    id,
+    fixture,
+    mutation,
+    action,
+    measurementKind,
+    expectedStates: Object.freeze([...expectedStates]),
+});
+export const BENCHMARK_CORPUS = Object.freeze({
+    contractMajor: 1,
+    scenarios: Object.freeze([
+        scenario('real-wiring', 'universal-rig', 'none', 'exercise protected write', 'capability-probe', ['SUPPORTED']),
+        scenario('unwired-enforcement', 'universal-rig', 'remove hook registration', 'exercise protected write', 'capability-probe', ['UNSUPPORTED']),
+        scenario('disabled-enforcement', 'universal-rig', 'disable all hooks', 'exercise protected write', 'capability-probe', ['UNSUPPORTED']),
+        scenario('narrowed-enforcement', 'universal-rig', 'narrow matcher', 'exercise omitted protected write', 'capability-probe', ['DEGRADED']),
+        scenario('bypassed-enforcement', 'universal-rig', 'bypass guard command', 'exercise protected write', 'capability-probe', ['INTEGRATION-FAILED']),
+        scenario('unreadable-input', 'universal-rig', 'make snapshot unreadable', 'exercise protected write', 'capability-probe', ['INTEGRATION-FAILED']),
+        scenario('hook-input', 'universal-rig', 'make tool_input unreadable', 'exercise protected write', 'guard-enforcement', ['SUPPORTED']),
+        scenario('protected-rulebook', 'universal-rig', 'none', 'edit protected rulebook', 'guard-enforcement', ['SUPPORTED']),
+        scenario('widening', 'universal-rig', 'widen authorization allow-list', 'attempt unauthorized widening', 'guard-enforcement', ['SUPPORTED']),
+        scenario('foreign-major', 'contracts/session-messaging/v1/fixtures/negative/envelope-foreign-major.json', 'foreign schema major', 'check policy compatibility', 'compatibility-rejection', ['UNSUPPORTED']),
+    ]),
+});
+const canonicalScenarios = new Map(BENCHMARK_CORPUS.scenarios.map((entry) => [entry.id, entry]));
+export const classifyCapability = (state) => {
+    switch (state) {
+        case 'SUPPORTED':
+            return { classification: 'equivalent', integrationFailed: false };
+        case 'DEGRADED':
+            return { classification: 'intentional-degradation', integrationFailed: false };
+        case 'UNSUPPORTED':
+            return { classification: 'unsupported', integrationFailed: false };
+        case 'INTEGRATION-FAILED':
+            return { classification: 'unsupported', integrationFailed: true };
+        default:
+            throw new Error(`Unknown capability state: ${String(state)}`);
+    }
+};
+const isOwnDataRecord = (value) => {
+    if (typeof value !== 'object' || value === null || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    if (prototype !== Object.prototype && prototype !== null)
+        return false;
+    return Object.values(Object.getOwnPropertyDescriptors(value)).every((descriptor) => descriptor.enumerable === true && 'value' in descriptor);
+};
+const isDataArray = (value) => {
+    if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype)
+        return false;
+    for (let index = 0; index < value.length; index += 1) {
+        const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
+        if (descriptor === undefined || descriptor.enumerable !== true || !('value' in descriptor))
+            return false;
+    }
+    return true;
+};
+const hasOnlyKeys = (value, keys) => {
+    const actual = Object.keys(value);
+    return actual.length === keys.length && keys.every((key) => Object.hasOwn(value, key));
+};
+const nonBlank = (value) => typeof value === 'string' && value.trim() !== '';
+const invalidCorpus = (reason) => {
+    throw new Error(`Invalid benchmark corpus: ${reason}`);
+};
+export const parseBenchmarkCorpus = (input) => {
+    if (!isOwnDataRecord(input) || !hasOnlyKeys(input, ['contractMajor', 'scenarios'])) {
+        return invalidCorpus('must be a plain corpus with contractMajor and scenarios');
+    }
+    if (input.contractMajor !== 1)
+        return invalidCorpus('unsupported contract major; expected 1');
+    if (!isDataArray(input.scenarios))
+        return invalidCorpus('scenarios must be a data array');
+    if (input.scenarios.length !== REQUIRED_SCENARIO_IDS.length) {
+        return invalidCorpus('must include every required scenario exactly once');
+    }
+    const ids = new Set();
+    const scenarios = [];
+    for (const value of input.scenarios) {
+        if (!isOwnDataRecord(value) ||
+            !hasOnlyKeys(value, [
+                'id',
+                'fixture',
+                'mutation',
+                'action',
+                'measurementKind',
+                'expectedStates',
+            ])) {
+            return invalidCorpus('each scenario must have only id, fixture, mutation, action, measurementKind, and expectedStates');
+        }
+        const { id, fixture, mutation, action, measurementKind, expectedStates } = value;
+        if (!nonBlank(id) || !requiredScenarioIds.has(id))
+            return invalidCorpus(`unknown scenario ${String(id)}`);
+        if (ids.has(id))
+            return invalidCorpus(`scenario ${id} is duplicated`);
+        if (!nonBlank(fixture) || !nonBlank(mutation) || !nonBlank(action)) {
+            return invalidCorpus(`scenario ${id} has empty metadata`);
+        }
+        if (typeof measurementKind !== 'string' ||
+            !MEASUREMENT_KINDS.includes(measurementKind)) {
+            return invalidCorpus(`scenario ${id} has an unknown measurement kind`);
+        }
+        if (!isDataArray(expectedStates) || expectedStates.length === 0) {
+            return invalidCorpus(`scenario ${id} must name expected capability states`);
+        }
+        const states = new Set();
+        for (const expectedState of expectedStates) {
+            if (typeof expectedState !== 'string' ||
+                !CAPABILITY_STATES.includes(expectedState)) {
+                return invalidCorpus(`scenario ${id} has an unknown capability state`);
+            }
+            if (states.has(expectedState))
+                return invalidCorpus(`scenario ${id} repeats a capability state`);
+            states.add(expectedState);
+        }
+        const canonical = canonicalScenarios.get(id);
+        if (fixture !== canonical.fixture ||
+            mutation !== canonical.mutation ||
+            action !== canonical.action ||
+            measurementKind !== canonical.measurementKind) {
+            return invalidCorpus(`scenario ${id} changes its canonical metadata`);
+        }
+        if (expectedStates.length !== canonical.expectedStates.length ||
+            expectedStates.some((state, index) => state !== canonical.expectedStates[index])) {
+            return invalidCorpus(`scenario ${id} changes its expected capability states`);
+        }
+        ids.add(id);
+        scenarios.push(Object.freeze({
+            id,
+            fixture,
+            mutation,
+            action,
+            measurementKind: measurementKind,
+            expectedStates: Object.freeze([...states]),
+        }));
+    }
+    for (const id of REQUIRED_SCENARIO_IDS) {
+        if (!ids.has(id))
+            return invalidCorpus(`required scenario ${id} is missing`);
+    }
+    return Object.freeze({ contractMajor: 1, scenarios: Object.freeze(scenarios) });
+};

package/packages/cli/dist/policy/core/coverage.js

@@ -0,0 +1,253 @@
+/**
+ * The capability & degradation contract (RP-36): what each declared policy is
+ * worth on one harness surface, how that answer is established, and how it is
+ * maintained afterwards.
+ *
+ * ⚠ A LIBRARY, not a running contract. Nothing in this rig calls any of it
+ * yet: `doctor` rendering the coverage report is its own task, and so is the
+ * caller that would report observed traffic. The present tense below describes
+ * what the functions do when called, not something happening today. The
+ * rationale and the full list of what this deliberately does not do are in
+ * `docs/decisions/capability-coverage.md`.
+ *
+ * A status is established by `./probe.ts` on an occasion named from
+ * `./vocabulary.ts` (`PROBE_TRIGGERS`), and maintained from what real traffic
+ * was expected to show and did not. Three rules follow, and they are the whole
+ * design.
+ *
+ * 🔴 **A timer alone never degrades anything.** `observeExpectedSignal` takes
+ * an OBSERVATION, never an elapsed time, and this module exports no function
+ * that ages, expires or sweeps an entry — pinned by
+ * `packages/cli/test/policy-coverage.test.ts` › "exports nothing that ages,
+ * expires or sweeps an entry, so only an observation can move it", and
+ * measured end-to-end by › "reports no downgrade when a year passes and the
+ * same wiring is probed again, because elapsed time is not evidence" against
+ * its positive control. No traffic is not a failure: it is no evidence.
+ *
+ * 🔴 **Traffic lowers a status; it never raises one.** A missing signal is
+ * evidence the mechanism did not act. A present signal is evidence it acted
+ * once — not that a wiring defect the probe found has been repaired. So the
+ * only route back up is another probe: › "never raises a status on traffic: a
+ * degraded surface stays degraded until a probe says otherwise".
+ *
+ * 🔴 **Traffic that CONTRADICTS the record is its own answer.** A signal
+ * observed on a surface the map records as `UNSUPPORTED` is not a promotion
+ * and not a miss — the wiring and the traffic disagree, which is what
+ * `INTEGRATION-FAILED` names: › "a signal observed where the map says nothing
+ * is wired is a contradiction, not a pass". It is still not a pass;
+ * `qualifierFor` maps it to `UNVERIFIABLE`.
+ *
+ * **Why this module throws where its neighbours return a `Validation`.** The
+ * line is what the value IS, not how bad it is: outside data read off a disk
+ * becomes a RESULT — `snapshot: unknown` is never trusted and never throws, it
+ * becomes `INTEGRATION-FAILED` — while a value the CALLER chose (a word from a
+ * closed vocabulary, its own clock, its own claim about which build it saw) is
+ * a programming error and throws. `./evidence-matrix.ts` sits on the first
+ * side, this module's two entry points on the second.
+ *
+ * Two limits worth stating rather than leaving to be discovered. `DEGRADED`
+ * carries no verdict qualifier, so an operation on precisely the tool a
+ * degraded matcher lost does not require a qualifier from this library — the
+ * item scopes the `UNVERIFIABLE` requirement to the unenforceable states; the
+ * caller owns the operation verdict. And a `reason`
+ * supplied with a miss that does not cross the threshold is not retained;
+ * only the miss that degrades records one — › "discards the reason given with
+ * a miss that does not degrade, and records the one given with the miss that
+ * does".
+ */
+import { probePolicy } from './probe.js';
+import { EXACT_VERSION_EXPECTED, isExactVersion, ISO_8601 } from './validation.js';
+import { PROBE_TRIGGERS, UNENFORCEABLE_STATES } from './vocabulary.js';
+/**
+ * How many consecutive expected-but-absent signals degrade a supported policy.
+ *
+ * Three, not one: a single miss is as easily an operation that never reached
+ * the mechanism as a mechanism that failed to act, and a contract that
+ * degrades on it reports noise. It is a DEFAULT and not a constant — every
+ * caller may set its own, because what counts as enough evidence depends on
+ * how much traffic the surface sees.
+ */
+export const DEFAULT_DEGRADATION_THRESHOLD = 3;
+/**
+ * How much a state can enforce, as an order, for deciding what counts as a
+ * fall. `UNSUPPORTED` and `INTEGRATION-FAILED` rank together on purpose: both
+ * mean the policy cannot be relied on here, they differ in whether anything is
+ * wired at all, and calling a move between them a downgrade would report a
+ * change of diagnosis as a loss of capability.
+ *
+ * This is a DISPLAY ordering and nothing else reads it. What qualifies a
+ * verdict is `UNENFORCEABLE_STATES` in `./vocabulary.ts`, which
+ * `./decision-record.ts` reads too — keying qualification off a rank would
+ * mean a future re-rank silently changed which verdicts are UNVERIFIABLE.
+ */
+const ENFORCEMENT_RANK = {
+    SUPPORTED: 0,
+    DEGRADED: 1,
+    UNSUPPORTED: 2,
+    'INTEGRATION-FAILED': 2,
+};
+const requireTimestamp = (field, value) => {
+    if (!ISO_8601.test(value)) {
+        throw new Error(`${field} must be an ISO-8601 date-time with an explicit zone; got ${JSON.stringify(value)}`);
+    }
+};
+/**
+ * The probe's answer for every policy, as one map.
+ *
+ * `trigger` is required, and that is the point: `PROBE_TRIGGERS` names only
+ * changes to the surface, so a caller must say which change occasioned this
+ * read and cannot name an interval. `at` is the caller's clock — nothing here
+ * reads one — and it is validated, because a status whose time cannot be
+ * ordered against a change is not auditable. The surface harness and path must
+ * also name the adapter doing the read; otherwise valid wiring from one
+ * harness could be labeled as support on another.
+ */
+export function coverageFromProbe(args) {
+    const { surface, policies, adapter, snapshot, at, trigger, evidencePointer } = args;
+    if (!PROBE_TRIGGERS.includes(trigger)) {
+        throw new Error(`a probe is occasioned by one of ${PROBE_TRIGGERS.join(', ')}; got ${JSON.stringify(trigger)}`);
+    }
+    requireTimestamp('at', at);
+    if (surface.harness !== adapter.harness || surface.surface !== adapter.surfaceFile) {
+        throw new Error(`surface identity must match the adapter being probed; expected ${JSON.stringify(adapter.harness)} at ${JSON.stringify(adapter.surfaceFile)}, got ${JSON.stringify(surface.harness)} at ${JSON.stringify(surface.surface)}`);
+    }
+    // The identity is the caller's claim about WHICH build was observed, and
+    // three documents called it exact while nothing read it. A map is the shape
+    // that carries it, so this is where it is refused.
+    if (!isExactVersion(surface.harnessVersion)) {
+        throw new Error(`surface.harnessVersion ${EXACT_VERSION_EXPECTED}; got ${JSON.stringify(surface.harnessVersion)}`);
+    }
+    const entries = policies.map((policy) => {
+        const result = probePolicy(policy, adapter, snapshot);
+        const entry = {
+            policyId: policy.policyId,
+            policyVersion: policy.policyVersion,
+            mechanism: policy.mechanism,
+            status: result.state,
+            verifiedAt: at,
+            verifiedBy: 'probe',
+            triggeredBy: trigger,
+            consecutiveMisses: 0,
+        };
+        if (result.reason !== undefined)
+            entry.degradationReason = result.reason;
+        if (evidencePointer !== undefined)
+            entry.evidencePointer = evidencePointer;
+        return entry;
+    });
+    return { surface, entries };
+}
+/**
+ * Fold one observation of real traffic into an entry.
+ *
+ * `seen` is the whole input: the operation the policy judges happened, and the
+ * observable signal the mechanism should have produced either appeared or did
+ * not. There is no parameter for "and this much time has passed", because time
+ * is not evidence about a mechanism.
+ */
+export function observeExpectedSignal(entry, observation) {
+    const { seen, at, threshold = DEFAULT_DEGRADATION_THRESHOLD, reason } = observation;
+    if (!Number.isInteger(threshold) || threshold < 1) {
+        throw new Error(`the degradation threshold must be a whole number of observations, at least 1; got ${String(threshold)}`);
+    }
+    requireTimestamp('at', at);
+    if (seen) {
+        const next = {
+            ...entry,
+            consecutiveMisses: 0,
+            verifiedAt: at,
+            verifiedBy: 'traffic',
+        };
+        // A signal seen where the record says nothing is wired is not a promotion
+        // and not a miss: the wiring and the traffic disagree, and that is the
+        // state the item reserves for it. The status is not raised — see the
+        // second rule in this file's header — so the only status this branch
+        // produces is the one naming the contradiction.
+        if (entry.status === 'UNSUPPORTED') {
+            next.status = 'INTEGRATION-FAILED';
+            next.degradationReason = `the expected signal was observed although the recorded status was UNSUPPORTED, so the wiring and the traffic disagree`;
+        }
+        return next;
+    }
+    const consecutiveMisses = entry.consecutiveMisses + 1;
+    const degrades = entry.status === 'SUPPORTED' && consecutiveMisses >= threshold;
+    const next = {
+        ...entry,
+        consecutiveMisses,
+        verifiedAt: at,
+        verifiedBy: 'traffic',
+    };
+    if (!degrades)
+        return next;
+    next.status = 'DEGRADED';
+    next.degradationReason =
+        reason ??
+            `the expected observable signal was absent on ${String(consecutiveMisses)} consecutive operations`;
+    return next;
+}
+/** The entry for one policy, or `null` — never an invented status. */
+export function statusOf(map, policyId) {
+    return map.entries.find((entry) => entry.policyId === policyId) ?? null;
+}
+/**
+ * The qualifier a verdict must carry when it was produced under this state.
+ *
+ * The unenforceable states mean the question could not be put to a working
+ * mechanism, so an `allow` under either is UNVERIFIABLE rather than a pass —
+ * `./decision-record.ts` refuses the unqualified record, and
+ * `packages/cli/test/policy-coverage.test.ts` › "refuses the silent pass an
+ * unwired surface would otherwise produce, and accepts it once qualifierFor
+ * speaks" holds the two modules to it together. How far that refusal reaches is
+ * stated once, at `UNENFORCEABLE_STATES` in `./vocabulary.ts` — read it there.
+ * This sentence used to paraphrase it as well, and the paraphrase is what went
+ * stale: it still said the refusal was "a property of records built the
+ * ordinary way" after RP-153 had made it hold against a hand-built prototype
+ * too. A second copy of a fact the same sentence claims is spelled once
+ * (`rules/invariants.md`, "One mechanism, one implementation").
+ */
+export function qualifierFor(state) {
+    return UNENFORCEABLE_STATES.includes(state)
+        ? 'UNVERIFIABLE'
+        : undefined;
+}
+/**
+ * Every policy whose status fell between two maps of the same surface.
+ *
+ * Only the same policy id and declaration version present in both are compared:
+ * an entry that appeared, vanished or changed semantics is a change of what is
+ * being measured, not a capability that fell.
+ * Two maps of DIFFERENT surfaces are refused rather than diffed — the result
+ * is attributed to one surface identity, and attributing one surface's fall to
+ * another is a report an operator would act on in the wrong place.
+ */
+export function downgradesBetween(before, after) {
+    // Deliberately NOT harnessVersion. A probe before and after an upgrade
+    // carries two versions of one surface, and that pair is exactly what the
+    // `upgrade` trigger exists to compare — refusing it disabled the comparison
+    // for the case it was added for. Harness, surface and OS are what make it
+    // the same place.
+    const sameSurface = before.surface.harness === after.surface.harness &&
+        before.surface.surface === after.surface.surface &&
+        before.surface.os === after.surface.os;
+    if (!sameSurface) {
+        throw new Error('two coverage maps of different surfaces cannot be compared: a downgrade is attributed to one surface, and these name two');
+    }
+    const downgrades = [];
+    for (const next of after.entries) {
+        const previous = before.entries.find((entry) => entry.policyId === next.policyId && entry.policyVersion === next.policyVersion) ?? null;
+        if (previous === null)
+            continue;
+        if (ENFORCEMENT_RANK[next.status] <= ENFORCEMENT_RANK[previous.status])
+            continue;
+        downgrades.push({
+            surface: after.surface,
+            policyId: next.policyId,
+            from: previous.status,
+            to: next.status,
+            mechanism: next.mechanism,
+            reason: next.degradationReason ?? 'the entry recorded no reason',
+            at: next.verifiedAt,
+        });
+    }
+    return downgrades;
+}