@byollm/conformance 0.1.0-alpha.1 → 0.1.0-alpha.100

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { JobKind, JobPayload, Audience, JobState, MustId } from '@byollm/protocol';
-import { Backend, BackendRequest, BackendResult, Runner, Allowlist, IngressLog } from 'byollm';
+import { JobKind, JobPayload, Audience, JobState, MustId, StoredKeys, PublicIdentity } from '@byollm/protocol';
+import { Backend, BackendRequest, BackendResult, Runner, IngressLog, SpendLedger, LoadedConfig } from 'byollm';
 
 /**
  * What a server implementation must expose to be certified.
@@ -145,8 +145,26 @@ declare function certify(target: ConformanceTarget, options?: {
     only?: readonly string[];
     onProgress?: (result: CheckResult) => void;
 }): Promise<CertificationReport>;
-/** MUSTs with no check asserting them. */
+/**
+ * `conformance`-kind MUSTs with no check asserting them.
+ *
+ * This counts only the MUSTs the kit is *able* to assert. It used to count
+ * all of them, which made a permanent structural fact — the kit certifies a
+ * server, and a third of the MUSTs are properties of a daemon — look like a
+ * backlog of ten missing tests. A number that can never reach zero gets
+ * ignored, and a number that is ignored is not a check.
+ *
+ * This one should be zero, and CI keeps it there.
+ */
 declare function uncoveredMusts(checks?: readonly Check[]): MustId[];
+/**
+ * MUSTs a check claims but which are not verifiable by conformance.
+ *
+ * The opposite error, and the one that would quietly overstate what a
+ * certification means: a check asserting an `operator`-kind MUST would put
+ * "verified" next to something no third party can check from outside.
+ */
+declare function miscoveredMusts(checks?: readonly Check[]): MustId[];
 /** A human-readable report. */
 declare function formatReport(report: CertificationReport): string;
 
@@ -159,12 +177,20 @@ declare function formatReport(report: CertificationReport): string;
  * every respect except the thing at the very end of the call.
  */
 declare class EchoBackend implements Backend {
+    readonly stopReasons: {
+        kind: "unavailable";
+        why: string;
+    };
     readonly id: "openai-http";
     readonly class: "http";
     /** Prompts this backend was asked to run, in order. */
     readonly seen: string[];
     /** Set to make the next call hang, for lease and cancel checks. */
     hangMs: number;
+    /** Set false to simulate the model not being installed or not running. */
+    healthy: boolean;
+    /** What the backend reports it can serve. Empty means "does not enumerate". */
+    models: string[];
     health(): Promise<{
         healthy: boolean;
         models: string[];
@@ -174,11 +200,27 @@ declare class EchoBackend implements Backend {
 interface HarnessDaemon {
     readonly runner: Runner;
     readonly backend: EchoBackend;
-    readonly allowlist: Allowlist;
     readonly runnerId: string;
     readonly owner: string;
+    /** This daemon's keys, so a check can sign as it — or deliberately not. */
+    readonly keys: StoredKeys;
+    /** This daemon's keys, and the site identities it pinned at pairing. */
+    identityKeys(): Promise<StoredKeys>;
+    /**
+     * The site a single-site check seals to and verifies against.
+     *
+     * A pairing covers a set now, and every check in this kit pairs with one
+     * upstream serving one site — so this is that entry, read from the set
+     * rather than kept beside it. Two copies of "which key opens this" is the
+     * bug the set exists to remove.
+     */
+    readonly sitePinned: PublicIdentity;
     readonly home: string;
     readonly ingress: IngressLog;
+    /** The owner's spend ledger, so a check can drive it past its ceiling. */
+    readonly spend: SpendLedger;
+    /** The resolved config — the effective offer scope lives here. */
+    readonly loaded: LoadedConfig;
     /** Stop cleanly: cancel in-flight work and clean up. */
     dispose(): Promise<void>;
     /**
@@ -198,12 +240,40 @@ interface HarnessDaemon {
  * pairing exchange, with the shipped allowlist and budget checks. Only the
  * model at the far end is substituted.
  */
+/**
+ * A paid backend, and what the owner said about spending on it — byollm_007.
+ *
+ * The kit needs this because "who pays" is visible on the wire: a daemon
+ * advertises the *effective* offer scope, so a metered backend nobody
+ * consented to share shows up to the server as `self` and the server is
+ * obliged to act on that.
+ */
+interface MeteredOptions {
+    /**
+     * `openai` takes its cost from the registry; `openai-http` has it inferred
+     * from {@link MeteredOptions.baseUrl}.
+     */
+    readonly provider?: "openai" | "openai-http";
+    readonly baseUrl?: string;
+    readonly acknowledged?: boolean;
+    readonly dailyCapCents?: number;
+}
 declare function pairDaemon(target: ConformanceTarget, options: {
     owner: string;
     label?: string;
-    offer?: "self" | "named" | "public";
+    /**
+     * **Required — no default.** A harness default is part of every test's
+     * claim (ruled 2026-08-26), and this one decides whether the device's
+     * admission check runs at all. The relay suite's equivalent defaulted to
+     * `public` and silently disabled admission in every cross-user check it
+     * had; this one defaulted to the safe direction and was still a value no
+     * reader of a call site could see.
+     */
+    offer: "private" | "team";
     /** Use the subscription-class backend, to exercise the self-lock. */
     subscription?: boolean;
+    /** Use a paid backend, to exercise the cost rules. */
+    metered?: MeteredOptions;
 }): Promise<HarnessDaemon>;
 /**
  * The id this target uses for a person, given the friendly name the checks
@@ -217,10 +287,110 @@ declare function waitFor(predicate: () => boolean | Promise<boolean>, options?:
     what?: string;
 }): Promise<void>;
 declare function sleep(ms: number): Promise<void>;
+declare function advance(target: ConformanceTarget, ms: number): Promise<void>;
+
 /**
- * Move the target's clock forward, faking it if the target can and genuinely
- * waiting if it cannot.
+ * The deployment posture audit — what an outsider can do to a running relay.
+ *
+ * ## Why this exists as a separate surface
+ *
+ * `certify` drives a real daemon against a {@link ConformanceTarget}, and a
+ * target may be an in-process handler or an HTTP server: "deliberately
+ * transport-agnostic", which is the right call for certifying a *protocol*.
+ *
+ * It is also a blind spot, and byollm_009's ninth finding lived in it. Eight
+ * freeze-gate findings came from tests where the site reached the relay by
+ * calling `handle()` on an object it held a reference to — and a harness that
+ * invokes the system under test directly cannot see anything about how the
+ * system is *reached*. The site plane had no authentication at all. Nothing
+ * noticed, because nothing in the suite was ever a stranger.
+ *
+ * So this suite is a stranger. It holds no credential, no key the deployment
+ * knows, and no reference to any object inside it. It has a URL, which is
+ * exactly what an attacker has. Every check asks the question that form of
+ * access makes available:
+ *
+ * - can I enqueue work into someone's machines?
+ * - can I read who is online and what they are holding?
+ * - can I make a signature that is *well-formed* and be believed?
+ * - is anything served that should not be on the internet?
+ * - can I reach a handler by dressing a path up to look like one?
+ *
+ * ## What this is not
+ *
+ * Not a penetration test, and not exhaustive — it cannot be, because the next
+ * hole will be in whatever gets added next. It is the specific class that has
+ * already bitten, turned into something that runs. That is the same move as
+ * every other check in this kit: a finding becomes a check so its *shape*
+ * cannot recur silently.
+ *
+ * It is also deliberately **safe to run against production**: nothing here
+ * writes, nothing floods, and every request is one an ordinary scanner would
+ * make. A posture audit you are nervous about running is one nobody runs.
  */
-declare function advance(target: ConformanceTarget, ms: number): Promise<void>;
+/** One thing a stranger tried. */
+interface PostureCheck {
+    /** Stable id, cited in output. */
+    readonly id: string;
+    /** What a person should understand from a failure. */
+    readonly title: string;
+    /** MUSTs this exercises, where one applies. Empty is honest, not a gap. */
+    readonly cites: readonly string[];
+    run(context: PostureContext): Promise<PostureOutcome>;
+}
+interface PostureContext {
+    /** The origin, as an outsider would type it. */
+    readonly origin: string;
+    /** Where the daemon plane is mounted. */
+    readonly basePath: string;
+    /** Injectable, so a test can drive this without a network. */
+    readonly fetch: typeof fetch;
+    /**
+     * The origin's own address, behind whatever edge fronts it — `D008`.
+     *
+     * Optional because most deployments have no separate origin, and a check
+     * that guessed one would report a posture it never tested.
+     */
+    readonly originAddress?: string;
+}
+interface PostureOutcome {
+    readonly passed: boolean;
+    /**
+     * Did this check actually put its question? Absent means yes.
+     *
+     * A posture that could not be measured does not pass — a gate that waves
+     * through what it could not look at passes hardest when it is blindest —
+     * and it is not a breach either. Unproven is a third state.
+     */
+    readonly measured?: boolean;
+    /** What actually happened, in a sentence someone can act on. */
+    readonly detail: string;
+}
+interface PostureResult extends PostureOutcome {
+    readonly id: string;
+    readonly title: string;
+    readonly cites: readonly string[];
+}
+interface PostureReport {
+    readonly origin: string;
+    readonly passed: boolean;
+    readonly results: readonly PostureResult[];
+}
+declare const POSTURE_CHECKS: readonly PostureCheck[];
+/**
+ * Audit a running deployment. Holds nothing it was not given a URL for.
+ *
+ * Every check runs even after one fails, because a posture report's job is to
+ * be a complete picture rather than the first thing that went wrong.
+ */
+declare function auditDeployment(options: {
+    url: string;
+    basePath?: string;
+    /** The origin behind the edge, for `D008`. */
+    originAddress?: string;
+    fetch?: typeof fetch;
+    onProgress?: (result: PostureResult) => void;
+}): Promise<PostureReport>;
+declare function formatPostureReport(report: PostureReport): string;
 
-export { CHECKS, type CertificationReport, type Check, type CheckResult, type ConformanceTarget, EchoBackend, type HarnessDaemon, advance, certify, formatReport, ownerIdFor, pairDaemon, sleep, uncoveredMusts, waitFor };
+export { CHECKS, type CertificationReport, type Check, type CheckResult, type ConformanceTarget, EchoBackend, type HarnessDaemon, POSTURE_CHECKS, type PostureCheck, type PostureContext, type PostureOutcome, type PostureReport, type PostureResult, advance, auditDeployment, certify, formatPostureReport, formatReport, miscoveredMusts, ownerIdFor, pairDaemon, sleep, uncoveredMusts, waitFor };

package/dist/index.js

@@ -4,18 +4,28 @@ import {
   advance,
   certify,
   formatReport,
+  miscoveredMusts,
   ownerIdFor,
   pairDaemon,
   sleep,
   uncoveredMusts,
   waitFor
-} from "./chunk-PDQJJ3Q2.js";
+} from "./chunk-OPJBI7WK.js";
+import {
+  POSTURE_CHECKS,
+  auditDeployment,
+  formatPostureReport
+} from "./chunk-2JXKI5TH.js";
 export {
   CHECKS,
   EchoBackend,
+  POSTURE_CHECKS,
   advance,
+  auditDeployment,
   certify,
+  formatPostureReport,
   formatReport,
+  miscoveredMusts,
   ownerIdFor,
   pairDaemon,
   sleep,

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@byollm/conformance",
-  "version": "0.1.0-alpha.1",
+  "version": "0.1.0-alpha.100",
   "description": "The BYOLLM compatibility contract — drive a real daemon against any server and assert every protocol MUST.",
   "license": "MIT",
   "type": "module",
   "bin": {
-    "byollm-certify": "./dist/cli.js"
+    "byollm-certify": "./dist/cli.js",
+    "byollm-audit-deployment": "./dist/audit-cli.js"
   },
   "exports": {
     ".": {
@@ -21,23 +22,32 @@
     "README.md"
   ],
   "engines": {
-    "node": ">=22.12"
+    "node": ">=22.14"
   },
   "dependencies": {
-    "byollm": "0.1.0-alpha.1",
-    "@byollm/protocol": "0.1.0-alpha.1"
+    "@byollm/protocol": "0.1.0-alpha.100",
+    "byollm": "0.1.0-alpha.100"
   },
   "devDependencies": {
     "@supabase/supabase-js": "^2.112.2",
-    "@byollm/server": "0.1.0-alpha.1"
+    "@byollm/server": "0.1.0-alpha.100"
   },
   "publishConfig": {
     "access": "public"
   },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/oftomorrowinc/byollm.git",
+    "directory": "packages/conformance"
+  },
+  "bugs": {
+    "url": "https://github.com/oftomorrowinc/byollm/issues"
+  },
   "scripts": {
     "build": "tsup",
     "clean": "rm -rf dist .tsbuild",
     "certify:reference": "vitest run --project conformance --root ../..",
-    "certify:supabase": "node --experimental-strip-types targets/supabase.ts"
+    "certify:supabase": "node --experimental-strip-types targets/supabase.ts",
+    "audit:deployed": "node dist/audit-cli.js"
   }
 }

package/targets/supabase.ts

@@ -24,7 +24,11 @@ import {
 // The built package, not `../src`: Node's type stripping does not rewrite the
 // `.js` specifiers the source uses, and certifying the published entry points
 // is closer to what a consumer actually gets.
-import { ByollmApp, createFetchHandler } from "@byollm/server";
+import {
+  ByollmApp,
+  createFetchHandler,
+  generateSiteKeys,
+} from "@byollm/server";
 import { supabaseStore } from "@byollm/server/supabase";
 
 const SUPABASE_URL = process.env["SUPABASE_URL"] ?? "http://127.0.0.1:54421";
@@ -37,6 +41,12 @@ const ORIGIN = "https://supabase.byollm.test";
 /** Short, because a real Postgres clock cannot be faked forward. */
 const LEASE_MS = 2_000;
 const TTL_MS = 1_500;
+/**
+ * Short for the same reason as the two above: with no fakeable clock, an
+ * expiry check waits for real. The product default is ten minutes, which is
+ * right for a human reading a code off a screen and wrong for a suite.
+ */
+const PAIRING_TTL_MS = 2_000;
 
 if (SERVICE_KEY === "") {
   process.stderr.write(
@@ -89,12 +99,18 @@ async function ensureUser(name: string): Promise<string> {
 
 const toName = (id: string): string => ownerNames.get(id) ?? id;
 
+// Generated per run: this target is one process, so there is no scale-out
+// identity problem to model here. A real deployment supplies these.
+const SITE_KEYS = generateSiteKeys();
+
 const store = supabaseStore({ client, defaultTtlMs: TTL_MS });
-const app = new ByollmApp({ store });
+const app = new ByollmApp({ store, siteKeys: SITE_KEYS });
 const handler = createFetchHandler({
   store,
   verificationUrl: `${ORIGIN}/settings/runners`,
+  siteKeys: SITE_KEYS,
   leaseMs: LEASE_MS,
+  pairingTtlMs: PAIRING_TTL_MS,
 });
 
 const target: ConformanceTarget = {