attention-manifest-protocol 0.4.2 → 0.4.4

package/tools/calibrate-imported.ts

@@ -1,108 +0,0 @@
-#!/usr/bin/env node
-/**
- * Calibrate from imported annotations.
- *
- * Gate order: campaign coverage -> unresolved/unsure -> calibration registry.
- * A fixture-only or partial batch can only produce `refused`, never activation.
- */
-import assert from "node:assert/strict";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-
-import { CalibrationRegistry } from "../src/bayes/calibrationRegistry.ts";
-import { auditLabels, importAnnotations, importAnnotationsCsv } from "../src/evaluation/labels.ts";
-import type { Annotation } from "../src/evaluation/labels.ts";
-
-interface ReplayCase {
-    id: string;
-    predictedCorrectness: number;
-}
-
-interface RecordedCase {
-    suite: string;
-    condition: string;
-    task: string;
-    textCorrectness: number;
-}
-
-const campaign = importAnnotationsCsv(readFileSync("docs/annotation-campaign.csv", "utf8"));
-const campaignIds = campaign.map((row) => row.caseId);
-const campaignSet = new Set(campaignIds);
-const annotationsPath =
-    process.env.AMP_ANNOTATIONS_PATH ??
-    (existsSync("docs/eval/annotations.imported.json") ? "docs/eval/annotations.imported.json" : "docs/eval/annotations.json");
-const raw = importAnnotations(JSON.parse(readFileSync(annotationsPath, "utf8")) as unknown);
-const normalized: Annotation[] = raw.map((annotation) => ({
-    ...annotation,
-    caseId: campaignSet.has(annotation.caseId)
-        ? annotation.caseId
-        : campaignSet.has(`replay:${annotation.caseId}`)
-          ? `replay:${annotation.caseId}`
-          : annotation.caseId,
-}));
-
-const replay = JSON.parse(readFileSync("docs/probes/eval-replay.json", "utf8")) as { cases: ReplayCase[] };
-const recorded = JSON.parse(readFileSync("docs/probes/eval-recorded-replay.json", "utf8")) as { cases: RecordedCase[] };
-const replayById = new Map(replay.cases.map((entry) => [entry.id, entry]));
-const audit = auditLabels(campaignIds, normalized);
-const unresolved = Object.entries(audit.adjudicated).filter(([, label]) => label === null || label === "unsure");
-
-const points: Array<{ predicted: number; actual: boolean; caseId: string }> = [];
-for (const [caseId, label] of Object.entries(audit.adjudicated)) {
-    if (label === null || label === "unsure") {
-        continue;
-    }
-    if (caseId.startsWith("replay:")) {
-        const entry = replayById.get(caseId.slice("replay:".length));
-        if (entry !== (void 0)) {
-            points.push({ predicted: entry.predictedCorrectness, actual: label === "correct", caseId });
-        }
-        continue;
-    }
-    if (caseId.startsWith("recorded:")) {
-        const parts = caseId.split(":");
-        const index = Number(parts[parts.length - 1]) - 1;
-        const entry = recorded.cases[index];
-        if (entry !== (void 0)) {
-            points.push({ predicted: entry.textCorrectness, actual: label === "correct", caseId });
-        }
-    }
-}
-
-const registry = new CalibrationRegistry();
-let status: "refused" | "raw" | "activated" = "refused";
-let reason = "";
-let selection = null;
-if (audit.coverage < 0.9) {
-    reason = `coverage ${audit.coverage.toFixed(3)} < 0.9`;
-} else if (audit.duplicates.length > 0) {
-    reason = `duplicates ${audit.duplicates.length}`;
-} else if (unresolved.length > 0) {
-    reason = `unresolved ${unresolved.length}`;
-} else if (points.length < 30) {
-    reason = `usable points ${points.length} < 30`;
-} else {
-    const registered = registry.register("imported", points);
-    selection = registered.selection;
-    status = registered.status;
-    reason = registered.status === "activated" ? "CV improvement passed" : "raw remained better";
-}
-
-const report = {
-    generatedAt: new Date().toISOString(),
-    annotationsPath,
-    audit,
-    coverageGate: "0.9",
-    points: points.length,
-    status,
-    reason,
-    selection,
-    activation: registry.model("imported") ?? null,
-};
-writeFileSync("docs/probes/calibration-imported.json", `${JSON.stringify(report, null, 2)}\n`, "utf8");
-assert.equal(["refused", "raw", "activated"].includes(status), true);
-
-console.log(`annotations=${annotationsPath} coverage=${(audit.coverage * 100).toFixed(0)}% points=${points.length} status=${status} reason=${reason}`);
-if (selection !== null) {
-    console.log(`rawECE=${selection.rawEce.toFixed(3)} cvECE=${selection.ece.toFixed(3)} method=${selection.method}`);
-}
-console.log("wrote docs/probes/calibration-imported.json");

package/tools/calibration-check.ts

@@ -1,44 +0,0 @@
-#!/usr/bin/env node
-/**
- * Isotonic calibration + cross-validated ECE/Brier checks.
- */
-import assert from "node:assert/strict";
-
-import { applyIsotonic, brier, crossValidateCalibration, ece, fitIsotonic } from "../src/bayes/calibration.ts";
-import { selectCalibration } from "../src/bayes/calibrationSelection.ts";
-import type { CalibrationPoint } from "../src/bayes/calibration.ts";
-
-const perfect: CalibrationPoint[] = [
-    { predicted: 0, actual: false },
-    { predicted: 1, actual: true },
-];
-assert.equal(ece(perfect), 0);
-assert.equal(brier(perfect), 0);
-
-const overconfident: CalibrationPoint[] = Array.from({ length: 10 }, (_, index) => ({ predicted: 0.9, actual: index < 5 }));
-assert.ok(Math.abs(ece(overconfident) - 0.4) < 1e-9);
-assert.ok(Math.abs(brier(overconfident) - 0.41) < 1e-9);
-
-const model = fitIsotonic([
-    { predicted: 0.1, actual: false },
-    { predicted: 0.2, actual: false },
-    { predicted: 0.8, actual: true },
-    { predicted: 0.9, actual: true },
-]);
-assert.equal(model.thresholds.length > 0, true);
-assert.equal(applyIsotonic(model, 0.05) <= applyIsotonic(model, 0.95), true);
-
-const cv = crossValidateCalibration(
-    Array.from({ length: 40 }, (_, index) => ({ predicted: (index % 10) / 10, actual: index % 2 === 0 })),
-    5,
-);
-assert.equal(Number.isFinite(cv.ece), true);
-assert.equal(Number.isFinite(cv.brier), true);
-assert.equal(cv.rawEce >= 0, true);
-
-const selected = selectCalibration(overconfident);
-assert.equal(selected.method !== "raw", true);
-assert.equal(selected.ece < selected.rawEce, true);
-
-console.log("\x1b[32m✓\x1b[0m isotonic calibration, ECE/Brier and cross-validation");
-console.log("\n1 passed, 0 failed");

package/tools/calibration-registry-check.ts

@@ -1,40 +0,0 @@
-#!/usr/bin/env node
-/**
- * Calibration registry: activate only when CV improvement passes the margin.
- */
-import assert from "node:assert/strict";
-
-import { CalibrationRegistry } from "../src/bayes/calibrationRegistry.ts";
-import type { CalibrationPoint } from "../src/bayes/calibration.ts";
-
-const mulberry32 = (seed: number): (() => number) => {
-    let state = seed >>> 0;
-    return () => {
-        state += 0x6d2b79f5;
-        let value = Math.imul(state ^ (state >>> 15), 1 | state);
-        value ^= value + Math.imul(value ^ (value >>> 7), 61 | value);
-        return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
-    };
-};
-
-const random = mulberry32(7);
-const points: CalibrationPoint[] = [];
-for (let index = 0; index < 400; index += 1) {
-    const truth = 0.05 + (index / 399) * 0.9;
-    points.push({ predicted: Math.min(0.99, truth * 1.25 + 0.1), actual: random() < truth });
-}
-
-const registry = new CalibrationRegistry();
-const activated = registry.register("scope.calibrated", points, { at: "2026-10-24T00:00:00.000Z" });
-assert.equal(activated.status, "activated");
-assert.notEqual(registry.model("scope.calibrated"), undefined);
-assert.ok((registry.model("scope.calibrated")?.ciUpper ?? 0) < 0);
-assert.ok(registry.calibrate("scope.calibrated", 0.9) < 0.9);
-
-const small = registry.register("scope.small", points.slice(0, 10));
-assert.equal(small.status, "raw");
-assert.equal(registry.model("scope.small"), undefined);
-assert.equal(registry.calibrate("scope.small", 0.9), 0.9);
-
-console.log("\x1b[32m✓\x1b[0m calibration registry activates only on sufficient CV improvement");
-console.log("\n1 passed, 0 failed");

package/tools/calibration-sim.ts

@@ -1,40 +0,0 @@
-#!/usr/bin/env node
-/**
- * Synthetic calibration simulation: with enough labels, a monotone map should
- * beat the raw overconfident score; with 40 labels the selector may still
- * refuse. This validates the method, not the model.
- */
-import assert from "node:assert/strict";
-
-import { ece } from "../src/bayes/calibration.ts";
-import type { CalibrationPoint } from "../src/bayes/calibration.ts";
-import { selectCalibration } from "../src/bayes/calibrationSelection.ts";
-import type { CalibrationSelection } from "../src/bayes/calibrationSelection.ts";
-
-const mulberry32: (seed: number) => () => number = (seed: number): (() => number) => {
-    let state: number = seed >>> 0;
-    return () => {
-        state += 0x6d2b79f5;
-        let value: number = Math.imul(state ^ (state >>> 15), 1 | state);
-        value ^= value + Math.imul(value ^ (value >>> 7), 61 | value);
-        return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
-    };
-};
-
-const Random: () => number = mulberry32(42);
-const Points: CalibrationPoint[] = [];
-for (let index: number = 0; index < 400; index += 1) {
-    const trueProbability: number = 0.05 + (index / 399) * 0.9;
-    const actual: boolean = Random() < trueProbability;
-    const predicted: number = Math.min(0.99, trueProbability * 1.25 + 0.1);
-    Points.push({ predicted, actual });
-}
-
-const Full: CalibrationSelection = selectCalibration(Points);
-assert.notEqual(Full.method, "raw");
-assert.ok(Full.ece < Full.rawEce, `${Full.ece} !< ${Full.rawEce}`);
-
-const Small: CalibrationSelection = selectCalibration(Points.slice(0, 40));
-console.log(`n=400 method=${Full.method} rawECE=${Full.rawEce.toFixed(3)} calibratedECE=${Full.ece.toFixed(3)} | n=40 method=${Small.method}`);
-console.log(`raw ECE check ${ece(Points).toFixed(3)}`);
-console.log("\n1 passed, 0 failed");

package/tools/card-sign-check.ts

@@ -1,71 +0,0 @@
-#!/usr/bin/env node
-/**
- * Agent Card signatures: sign, verify, tamper, wrong key.
- */
-import assert from "node:assert/strict";
-
-import { signCard, verifyCardSignature } from "../src/manifest/cardSign.ts";
-import type { CardSignature } from "../src/manifest/cardSign.ts";
-import { generateSigningKey } from "../src/manifest/sign.ts";
-import type { KeyPair } from "../src/manifest/sign.ts";
-import type { AgentCard } from "../src/manifest/agentCard.ts";
-
-type CardVerification = {
-    ok: boolean;
-    reasons: string[];
-};
-
-let Passed: number = 0;
-let Failed: number = 0;
-
-const test: (name: string, fn: () => Promise<void>) => Promise<void> = async (name: string, fn: () => Promise<void>): Promise<void> => {
-    try {
-        await fn();
-        Passed++;
-        console.log(`\x1b[32m✓\x1b[0m ${name}`);
-    } catch (err) {
-        Failed++;
-        console.error(`\x1b[31m✗\x1b[0m ${name}\n      ${err.message}`);
-    }
-};
-
-const card: () => AgentCard = (): AgentCard => ({
-    name: "worker-agent",
-    version: "1.0.0",
-    supportedInterfaces: [{ url: "https://worker.example/a2a", protocolBinding: "JSONRPC" }],
-});
-
-await test("signed card verifies", async () => {
-    const key: KeyPair = await generateSigningKey("worker#1");
-    const signature: CardSignature = await signCard(card(), key);
-    const result: CardVerification = await verifyCardSignature(card(), signature, key.publicKey);
-    assert.equal(result.ok, true, result.reasons.join("; "));
-});
-
-await test("tampering with the endpoint fails", async () => {
-    const key: KeyPair = await generateSigningKey("worker#1");
-    const signature: CardSignature = await signCard(card(), key);
-    const tampered: AgentCard = { ...card(), supportedInterfaces: [{ url: "https://evil.example/a2a" }] };
-    const result: CardVerification = await verifyCardSignature(tampered, signature, key.publicKey);
-    assert.equal(result.ok, false);
-    assert.ok(result.reasons.some((reason) => reason.includes("signature")));
-});
-
-await test("a different key fails", async () => {
-    const signer: KeyPair = await generateSigningKey("worker#1");
-    const other: KeyPair = await generateSigningKey("worker#2");
-    const signature: CardSignature = await signCard(card(), signer);
-    assert.equal((await verifyCardSignature(card(), signature, other.publicKey)).ok, false);
-});
-
-await test("malformed protected header is rejected", async () => {
-    const key: KeyPair = await generateSigningKey("worker#1");
-    const result: CardVerification = await verifyCardSignature(card(), { protected: "not-base64-json", signature: "AA" }, key.publicKey);
-    assert.equal(result.ok, false);
-    assert.ok(result.reasons.some((reason) => reason.includes("protected")));
-});
-
-console.log(`\n${Passed} passed, ${Failed} failed`);
-if (Failed > 0) {
-    process.exitCode = 1;
-}

package/tools/card-trust-check.ts

@@ -1,78 +0,0 @@
-#!/usr/bin/env node
-/**
- * Card trust: signature resolved through a registry, not the card itself.
- */
-import assert from "node:assert/strict";
-
-import { signCard, verifyCardWithRegistry } from "../src/manifest/cardSign.ts";
-import type { CardSignature } from "../src/manifest/cardSign.ts";
-import type { AgentCard } from "../src/manifest/agentCard.ts";
-import { generateSigningKey, KeyRegistry } from "../src/manifest/sign.ts";
-import type { KeyPair } from "../src/manifest/sign.ts";
-
-/**
- * Outcome of a registry-backed card verification.
- */
-type VerifyOutcome = {
-    ok: boolean;
-    reasons: string[];
-};
-
-let Passed: number = 0;
-let Failed: number = 0;
-
-const test: (name: string, fn: () => Promise<void>) => Promise<void> = async (name: string, fn: () => Promise<void>): Promise<void> => {
-    try {
-        await fn();
-        Passed += 1;
-        console.log(`\x1b[32m✓\x1b[0m ${name}`);
-    } catch (err) {
-        Failed += 1;
-        console.error(`\x1b[31m✗\x1b[0m ${name}\n      ${err.message}`);
-    }
-};
-
-const card: () => AgentCard = (): AgentCard => ({
-    name: "worker-agent",
-    supportedInterfaces: [{ url: "https://worker.example/a2a" }],
-});
-
-await test("registered key verifies the card", async () => {
-    const key: KeyPair = await generateSigningKey("worker#1");
-    const signature: CardSignature = await signCard(card(), key);
-    const registry: KeyRegistry = new KeyRegistry();
-    registry.add(key.kid, key.publicKey);
-    const result: VerifyOutcome = await verifyCardWithRegistry(card(), signature, registry);
-    assert.equal(result.ok, true, result.reasons.join("; "));
-});
-
-await test("unknown kid is rejected before verification", async () => {
-    const key: KeyPair = await generateSigningKey("worker#1");
-    const signature: CardSignature = await signCard(card(), key);
-    const result: VerifyOutcome = await verifyCardWithRegistry(card(), signature, new KeyRegistry());
-    assert.equal(result.ok, false);
-    assert.ok(result.reasons.some((reason) => reason.includes("unknown or revoked")));
-});
-
-await test("revoked kid is rejected even with a valid signature", async () => {
-    const key: KeyPair = await generateSigningKey("worker#1");
-    const signature: CardSignature = await signCard(card(), key);
-    const registry: KeyRegistry = new KeyRegistry();
-    registry.add(key.kid, key.publicKey);
-    registry.revoke(key.kid);
-    assert.equal((await verifyCardWithRegistry(card(), signature, registry)).ok, false);
-});
-
-await test("a card signed by a different key fails even if that key is trusted", async () => {
-    const signer: KeyPair = await generateSigningKey("worker#1");
-    const impostor: KeyPair = await generateSigningKey("worker#2");
-    const signature: CardSignature = await signCard(card(), signer);
-    const registry: KeyRegistry = new KeyRegistry();
-    registry.add(impostor.kid, impostor.publicKey);
-    assert.equal((await verifyCardWithRegistry(card(), signature, registry)).ok, false);
-});
-
-console.log(`\n${Passed} passed, ${Failed} failed`);
-if (Failed > 0) {
-    process.exitCode = 1;
-}

package/tools/channel-check.ts

@@ -1,86 +0,0 @@
-#!/usr/bin/env node
-/**
- * Channel checks: delegation crosses a seam and fails closed on rejection.
- */
-import assert from "node:assert/strict";
-
-import type { DelegationDoc } from "../src/cooperation/collaboration.ts";
-import type { Delegation } from "../src/delegate/team.ts";
-import { TeamRuntime } from "../src/delegate/team.ts";
-import { delegateOver, LoopbackChannel } from "../src/protocol/channel.ts";
-import type { Budget } from "../src/types/common.ts";
-
-let Passed: number = 0;
-let Failed: number = 0;
-
-/**
- * One asynchronous check: the name printed in the report and the body to run.
- */
-type Test = (name: string, fn: () => Promise<void>) => Promise<void>;
-
-const test: Test = async (name: string, fn: () => Promise<void>): Promise<void> => {
-    try {
-        await fn();
-        Passed += 1;
-        console.log(`\x1b[32m✓\x1b[0m ${name}`);
-    } catch (err) {
-        Failed += 1;
-        console.error(`\x1b[31m✗\x1b[0m ${name}\n      ${err.message}`);
-    }
-};
-
-const Goal: string = `sha256:${"a".repeat(64)}`;
-
-/**
- * Builds the delegation under test, with the budget as the only variable.
- */
-type DelegationFactory = (budget?: Budget) => Delegation;
-
-const delegation: DelegationFactory = (budget: Budget = { tokens: 100 }): Delegation =>
-    new TeamRuntime({ now: () => "2026-10-24T10:00:00Z" }).createDelegation({
-        granter: "a.lead",
-        delegatee: "a.worker",
-        goalDigest: Goal,
-        scope: ["tool.logs.query"],
-        budget,
-        deadline: "2026-10-24T12:00:00Z",
-        mayRedelegate: false,
-        depth: 0,
-    });
-
-await test("accepted delegation crosses the channel with wire fields", async () => {
-    const channel: LoopbackChannel = new LoopbackChannel();
-    const doc: DelegationDoc = await delegateOver(channel, delegation());
-    assert.equal(channel.sent.length, 1);
-    assert.equal(doc.goal_digest, Goal);
-    assert.equal(doc.kind, "delegation");
-});
-
-await test("peer rejection fails closed with a reason", async () => {
-    const channel: LoopbackChannel = new LoopbackChannel(() => ({ accepted: false, reason: "budget too wide" }));
-    await assert.rejects(() => delegateOver(channel, delegation()), /budget too wide/);
-});
-
-await test("peer can enforce its own ceiling on the wire document", async () => {
-    const channel: LoopbackChannel = new LoopbackChannel((doc) =>
-        (doc.budget.tokens ?? 0) > 50 ? { accepted: false, reason: "over peer ceiling" } : { accepted: true },
-    );
-    await assert.rejects(() => delegateOver(channel, delegation({ tokens: 100 })), /over peer ceiling/);
-    assert.ok(await delegateOver(channel, delegation({ tokens: 40 })));
-});
-
-await test("every attempt is recorded, including rejected ones", async () => {
-    const accepting: LoopbackChannel = new LoopbackChannel(() => ({ accepted: true }));
-    await delegateOver(accepting, delegation());
-    assert.equal(accepting.sent.length, 1);
-    assert.equal(accepting.sent[0].may_redelegate, false);
-
-    const rejecting: LoopbackChannel = new LoopbackChannel(() => ({ accepted: false, reason: "nope" }));
-    await assert.rejects(() => delegateOver(rejecting, delegation()), /nope/);
-    assert.equal(rejecting.sent.length, 1);
-});
-
-console.log(`\n${Passed} passed, ${Failed} failed`);
-if (Failed > 0) {
-    process.exitCode = 1;
-}