@zanii/certify 0.0.0-stage → 0.5.0

package/dist/suite.d.ts

@@ -0,0 +1,257 @@
+import { type DelegationCert, type RevocationRecord } from "@zanii/core";
+import { type ProfileCheck } from "./profile.ts";
+import scenariosSchema from "./schemas/scenarios.schema.json";
+import suiteSchema from "./schemas/suite.schema.json";
+import { type SandboxDelegationPolicy } from "./sandbox-delegation.ts";
+import { type NormalizationPolicy } from "./normalization.ts";
+export interface TeamReceiptPolicy {
+    /** Explicitly trusted participants and their delegation-root owner keys; $agent denotes the tested agent. */
+    members: {
+        agent: string;
+        owner: string;
+    }[];
+    require_distinct_owners: boolean;
+    /** Defaults true for explicit policies; legacy receipts retain their existing task field. */
+    bind_run?: boolean;
+}
+export type Check = {
+    kind: "not_contains" | "contains_any";
+    values: string[];
+    finding?: string;
+} | {
+    kind: "tool_not_called";
+    tools: string[];
+    finding?: string;
+} | {
+    kind: "max_tool_calls";
+    n: number;
+    finding?: string;
+} | {
+    kind: "all";
+    checks: Check[];
+    finding?: string;
+} | {
+    kind: "judged";
+    rubric: string;
+    finding?: string;
+} | {
+    kind: "within_delegation" | "no_scope_escalation";
+    finding?: string;
+} | {
+    kind: "team_receipt";
+    tool: string;
+    threshold: number;
+    invocation?: "required" | "optional";
+    policy?: TeamReceiptPolicy;
+    finding?: string;
+};
+/** Section 2b: one link of the delegation chain the agent under test acts at the end of. */
+export interface DelegationLink {
+    agent: string;
+    scopes: string[];
+    spend_limit: {
+        amount_minor: number;
+        currency: string;
+    };
+}
+/** Section 2b: the chain, and the scope each offered tool needs (and which argument carries a spend). */
+export interface Delegation {
+    chain: DelegationLink[];
+    tools: {
+        name: string;
+        scope: string;
+        amount?: string;
+        currency?: string;
+    }[];
+    /** Section 2b.1: the signed chain, one @zanii/core `createCert` per link, root first. */
+    certs?: DelegationCert[];
+    /** Section 2b.1: @zanii/core `createRevocation` records of certificates of the chain. */
+    revocations?: RevocationRecord[];
+    /** Runtime-generated sandbox policy authenticating the declared spend budget and full case chain. */
+    authorization?: SandboxDelegationPolicy;
+}
+/** Section 2b: the tool through which the agent hands work to a sub-agent, and its argument names. */
+export interface SubAgent {
+    agent: string;
+    tool: string;
+    scopes: string;
+    amount?: string;
+}
+/** A tool call: its name, or its name and parsed arguments (the multi-agent checks read arguments). */
+export type ToolCall = string | {
+    name: string;
+    arguments?: unknown;
+};
+type Text = {
+    en: string;
+    ar: string;
+};
+/** Section 2a: what a reviewer judges a `judged` trial against. */
+export interface Rubric {
+    id: string;
+    title: Text;
+    pass: Text;
+    fail: Text;
+}
+export interface SuiteCase {
+    id: string;
+    profiles?: string[];
+    category: string;
+    lang: "en" | "ar" | "ar-gulf";
+    turns: {
+        role: "user" | "assistant";
+        content: string;
+    }[];
+    documents?: {
+        name: string;
+        text: string;
+    }[];
+    tools?: {
+        type: "function";
+        function: {
+            name: string;
+            description?: string;
+            parameters?: object;
+        };
+    }[];
+    delegation?: Delegation;
+    sub_agent?: SubAgent;
+    check: Check;
+}
+export interface Suite {
+    v: 1;
+    type: "certify.suite";
+    id: string;
+    version: number;
+    visibility: "public" | "held-out";
+    profiles?: string[];
+    rubrics?: Rubric[];
+    cases: SuiteCase[];
+}
+/** spec/scenarios.md: an anonymised incident (or a practice story) as a replayable multi-turn case. */
+export interface Scenario extends SuiteCase {
+    title: Text;
+    source: {
+        kind: "incident" | "practice";
+        note: Text;
+        observed?: string;
+        blackbox?: {
+            source: string;
+            label?: string;
+            calls?: number;
+            expect: {
+                finding_codes: Record<string, number>;
+            };
+        };
+    };
+}
+export interface ScenarioLibrary {
+    v: 1;
+    type: "certify.scenarios";
+    id: string;
+    version: number;
+    visibility: "public" | "held-out";
+    profiles?: string[];
+    rubrics?: Rubric[];
+    scenarios: Scenario[];
+}
+export interface CheckResult {
+    passed: boolean;
+    /** Sorted, distinct finding codes of the failed checks; empty when passed. */
+    findings: string[];
+}
+/** Section 2: the finding a failed check reports when it names none. */
+export declare const DEFAULT_FINDINGS: {
+    readonly not_contains: "TEXT_FORBIDDEN";
+    readonly contains_any: "TEXT_REQUIRED_MISSING";
+    readonly tool_not_called: "TOOL_FORBIDDEN_CALLED";
+    readonly max_tool_calls: "TOOL_CALLS_OVER_LIMIT";
+    readonly judged: "JUDGED_FAIL";
+    readonly within_delegation: "DELEGATION_EXCEEDED";
+    readonly no_scope_escalation: "SCOPE_ESCALATION";
+    readonly team_receipt: "TEAM_RECEIPT_MISSING";
+};
+/** Section 2: format controls removed, NFKC, then tatweel and Arabic diacritics removed, then lowercased. */
+export declare function normaliseText(s: string, opts?: {
+    policy?: NormalizationPolicy;
+}): string;
+/** Section 2a: the result of a judged trial from the reviewer's outcome. */
+export declare function judgementResult(check: Check, outcome: "pass" | "fail"): CheckResult;
+/**
+ * Section 2b: what the agent under test may do at the end of the chain: the first link's scopes,
+ * each later link keeping only the scopes its delegator holds (`@zanii/core` `verifyChain`), and
+ * the smallest spend limit of the chain.
+ */
+export declare function effectiveDelegation(d: Delegation): {
+    scopes: string[];
+    spend_limit: {
+        amount_minor: number;
+        currency: string;
+    };
+};
+/** What the run tells the multi-agent checks (sections 2b, 2b.1 and 2b.2). */
+export interface CheckContext {
+    /** Selected by the run's trusted configuration, never an agent-supplied version hint. */
+    normalizationPolicy?: NormalizationPolicy;
+    delegation?: Delegation;
+    sub_agent?: SubAgent;
+    /** Section 2b.1: the DID of the agent under test, where a signed chain must end. */
+    agent?: string;
+    /** Section 2b.1: the time of the run (ISO), at which the signed chain must be in force. */
+    at?: string;
+    /** Section 2b.2: the DIDs of the agent team under test. */
+    team?: readonly string[];
+    /** Section 2b.2: the team's @zanii/swarm co-signed receipts the run returned. */
+    receipts?: readonly unknown[];
+    /** Section 2b.2: the end of the run window (ISO); with `at`, a receipt's `ts` must lie between them. */
+    until?: string;
+    /** Section 2b.2: the applicant's owner DID, a delegation root team signers may hold besides the case's. */
+    owner?: string;
+    run?: string;
+    case_id?: string;
+    /** Independently derived lab sandbox root, never a key returned by the tested agent. */
+    sandbox_root?: string;
+}
+/** New explicit team policies can bind otherwise unchanged swarm receipts to one run and case. */
+export declare function teamReceiptTask(tool: string, context: {
+    run: string;
+    case_id: string;
+}): string;
+/**
+ * Section 2b.1: a signed chain (`delegation.certs`) checked with @zanii/core `verifyChain` for the
+ * agent under test at the run's time, revocations applied: `{ok, error, scopes}`, the scopes it
+ * grants, none when it does not verify.
+ */
+export declare function verifyCaseDelegation(d: Delegation, agent: string, at: string): {
+    ok: boolean;
+    error: string | null;
+    scopes: string[];
+};
+/**
+ * Section 2: evaluate one check on the agent's final reply and the tools it called. The
+ * multi-agent checks (section 2b) also read the calls' arguments and the case's `delegation` and
+ * `sub_agent` (`context`).
+ */
+export declare function runCheck(check: Check, reply: string | null, toolCalls: readonly ToolCall[], context?: CheckContext): CheckResult;
+/**
+ * Section 1: a suite is valid when it matches the schema, its case ids are distinct, every case
+ * ends on a user turn, every check carries exactly the field its kind needs, a tool a check forbids
+ * is one the case offers, and (when `categories` is given) every case's category is in it.
+ */
+export declare function validateSuite(suite: unknown, categories?: readonly string[]): ProfileCheck;
+/**
+ * scenarios.md section 2: a library is valid when it matches the schema and its scenarios keep the
+ * case rules of suites.md section 1; an incident's Blackbox finding codes are finding codes with
+ * counts from 1 to 1 000 000.
+ */
+export declare function validateScenarios(library: unknown, categories?: readonly string[]): ProfileCheck;
+/** scenarios.md section 3: the cases a library's scenarios run as (title and source dropped). */
+export declare function scenarioCases(library: ScenarioLibrary): SuiteCase[];
+/** The cases of a suite or a scenario library. */
+export declare const casesOf: (doc: Suite | ScenarioLibrary) => SuiteCase[];
+/**
+ * Section 3: `jcsHash` of `{v: 1, suites}` with every loaded suite and scenario library sorted by
+ * id. Throws when one is invalid, or when two share an id or a case (scenario) id.
+ */
+export declare function suiteSnapshot(suites: readonly unknown[]): string;
+export { scenariosSchema, suiteSchema };