@alisio/sdk 0.6.0 → 0.8.0

package/dist/index.d.ts

@@ -792,6 +792,19 @@ export interface RunEventDataMap {
         latencyMs: number;
         reason: DecisionFallbackReason;
     };
+    /** A Proof criterion changed derived status or gained evidence. */
+    proof_updated: {
+        criterionId: string;
+        status: ProofStatus;
+        stale: boolean;
+        evidenceId?: string;
+    };
+    /** The final text claims something the Proof ledger does not back (display only). */
+    proof_divergence: {
+        claim: string;
+        criterionId?: string;
+        actual: ProofStatus;
+    };
 }
 /** Every `RunEvent.type` the core emits today. `RunEvent.type` itself stays `string`. */
 export type RunEventType = keyof RunEventDataMap;
@@ -1155,6 +1168,11 @@ export interface PlanReview {
     artifact?: ArtifactRef;
     /** The diagrams that were accepted for this revision (absent without diagrams). */
     diagrams?: PlanDiagramInfo[];
+    /**
+     * Acceptance criteria the model proposed (drafts). Approving the plan activates exactly these;
+     * any other decision discards them (absent when none were proposed).
+     */
+    acceptanceCriteria?: PlanCriterionDraft[];
 }
 /** What a diagram illustrates: a purpose, not a Mermaid syntax. */
 export type PlanDiagramType = "overview" | "flow" | "components" | "architecture" | "sequence" | "data" | "state" | "other";
@@ -1218,6 +1236,8 @@ export interface PlanManifest {
         id: string;
         title: string;
     }>;
+    /** The acceptance-criteria drafts of this revision (absent when none were proposed). */
+    acceptanceCriteria?: PlanCriterionDraft[];
 }
 /** Lifecycle of a background task (`bg_run`); a terminal state is never left. */
 export type BackgroundTaskStatus = "queued" | "running" | "stopping" | "succeeded" | "failed" | "cancelled" | "lost";
@@ -1270,7 +1290,9 @@ export interface BackgroundTaskOutput {
 /** The state of a session goal (`/goal`); `complete` and `budget_limited` are final unless re-armed. */
 export type GoalStatus = "active" | "paused" | "blocked" | "budget_limited" | "complete";
 /** Why a goal is in its state: a closed set of codes (the UIs translate them). */
-export type GoalReason = "created" | "resumed" | "edited" | "user_paused" | "user_interrupt" | "model_complete" | "model_blocked" | "policy_denied" | "run_error" | "token_budget" | "max_turns" | "max_wall" | "no_progress" | "restart";
+export type GoalReason = "created" | "resumed" | "edited" | "user_paused" | "user_interrupt" | "model_complete" | "model_blocked" | "policy_denied" | "run_error" | "token_budget" | "max_turns" | "max_wall" | "no_progress" | "restart"
+/** The user completed the goal over unmet required criteria (Proof override). */
+ | "user_override";
 /** What an active goal is waiting for before its next continuation (never stored: computed). */
 export type GoalWaiting = "approval" | "question" | "user_input" | "plan_mode" | "background_tasks";
 /** What the user can do to a goal (the actions the UIs offer per state). */
@@ -1279,6 +1301,151 @@ export type GoalAction = "pause" | "resume" | "edit" | "clear" | "budget";
 export interface GoalEvidence {
     kind: "file" | "test" | "log" | "command" | "denied" | "other";
     detail: string;
+    /** Links this declaration to captured Proof evidence; without it the text is a free claim. */
+    evidenceId?: string;
+}
+/** Derived state of an acceptance criterion (never stored as an independent truth). */
+export type ProofStatus = "verified" | "failed" | "not_checked" | "manual_review" | "not_applicable";
+/** How a criterion is checked. Consumers must ignore verifier kinds they do not know. */
+export type ProofVerifierSpec = {
+    kind: "command";
+    command: string;
+    cwd?: string;
+    successExitCodes?: number[];
+    outputPattern?: string;
+} | {
+    kind: "file";
+    path: string;
+    sha256?: string;
+    pattern?: string;
+} | {
+    kind: "data-check";
+    datasetId: string;
+    check: "row_count" | "sum" | "equals";
+    column?: string;
+    expected: number | string;
+    /** Table to query; `data` (CSV/TSV/JSON/JSONL) when absent, `s_<sheet>` for XLSX. */
+    table?: string;
+} | {
+    kind: "coverage";
+    command: string;
+    minPercent: number;
+} | {
+    kind: "manual";
+} | {
+    kind: "plugin";
+    verifierId: string;
+    input?: unknown;
+};
+/**
+ * What a plugin verifier returns. The evidence is the `outputHash` (sha256 hex of the output the
+ * verdict rests on) and, when the verifier made core tool calls, the `callId` of the one that
+ * backs the verdict. A result without a valid `outputHash`, or naming a call the session never
+ * made, is degraded to `manual_review`: a verdict nobody can trace is not a verification.
+ */
+export interface ProofResult {
+    ok: boolean;
+    outputHash: string;
+    callId?: string;
+    /** Short human-readable summary; redacted and bounded by the host. */
+    excerpt?: string;
+}
+/** What the host hands a plugin verifier. It carries no permission of its own. */
+export interface ProofVerifierContext {
+    sessionId: string;
+    criterionId: string;
+    workspace: string;
+    signal: AbortSignal;
+    /**
+     * Runs a core tool through the same gate as a model call (permission mode, `--read-only`,
+     * path policy). A refused or failed call resolves with `result.isError` set and nothing ran.
+     * `callId` is what a `ProofResult` may reference.
+     */
+    callTool(name: string, input: Record<string, unknown>): Promise<{
+        callId: string;
+        result: ToolResult;
+    }>;
+}
+/** A verifier a plugin contributes to Proof; criteria bind to it as `<pluginId>:<id>`. */
+export interface ProofVerifierRegistration {
+    /** Lowercase letters, digits, `.`, `_` and `-` (max 64). */
+    id: string;
+    describe: string;
+    run(input: unknown, ctx: ProofVerifierContext): Promise<ProofResult>;
+}
+/** One acceptance criterion with its derived status. */
+export interface ProofCriterion {
+    criterionId: string;
+    sessionId: string;
+    goalId?: string;
+    text: string;
+    required: boolean;
+    origin: "plan" | "goal" | "user" | "inferred" | "cli";
+    verifier?: ProofVerifierSpec;
+    /** Globs of the files the criterion covers; absent means the whole workspace. */
+    scope?: string[];
+    status: ProofStatus;
+    /** True when the criterion had successful evidence that a later edit in its scope invalidated. */
+    stale?: boolean;
+    evidenceIds: string[];
+    notApplicableReason?: string;
+}
+/** Immutable record that a verifier ran. A new run creates a new evidence, never an edit. */
+export interface ProofEvidence {
+    evidenceId: string;
+    criterionId: string;
+    verifierKind: ProofVerifierSpec["kind"];
+    runId: string;
+    callId: string;
+    ok: boolean;
+    commandFingerprint?: string;
+    exitCode?: number;
+    durationMs: number;
+    outputHash: string;
+    excerpt?: string;
+    treeFingerprint: string;
+    filesTouched: string[];
+    capturedAt: string;
+}
+/** The criteria and evidence of one session, as shown by every surface. */
+export interface ProofLedger {
+    sessionId: string;
+    criteria: ProofCriterion[];
+    evidence: ProofEvidence[];
+    divergences: {
+        claim: string;
+        criterionId?: string;
+        actual: ProofStatus;
+    }[];
+    generatedAt: string;
+}
+/**
+ * What a completed goal records about its Proof criteria: a compact snapshot taken inside the
+ * completing transaction (never a second source of truth: the ledger derives statuses on read).
+ */
+export interface GoalProofInfo {
+    criteria: {
+        criterionId: string;
+        required: boolean;
+        status: ProofStatus;
+    }[];
+    /** Required criteria that were `failed` or `not_checked` at completion. */
+    unmet: string[];
+    /** Evidence ids the model declared that resolved to captured evidence of the session. */
+    linkedEvidence: string[];
+    /** Present only when the USER completed the goal over unmet criteria. */
+    override?: {
+        reason: string;
+        at: number;
+    };
+}
+/** A criterion the model proposed in `exit_plan`: a draft until the user approves the plan. */
+export interface PlanCriterionDraft {
+    text: string;
+    required: boolean;
+    verifier?: ProofVerifierSpec;
+    scope?: string[];
+    origin: "inferred";
 }
 /** A session goal as shown by routes and UIs. */
 export interface GoalInfo {
@@ -1303,6 +1470,8 @@ export interface GoalInfo {
     /** The model's last `update_goal` report. */
     summary?: string;
     evidence?: GoalEvidence[];
+    /** The Proof snapshot of a completed goal that had criteria (absent without criteria). */
+    proof?: GoalProofInfo;
     /** What the next continuation waits for (set by the host: it knows the live state). */
     waiting?: GoalWaiting;
     /** The user actions allowed in this state. */
@@ -1689,6 +1858,13 @@ export interface PluginAPI {
     views?: {
         register(view: ViewDefinition): () => void;
     };
+    /**
+     * Proof verifiers (specs/011). Absent on a core that predates Proof: feature-detect with
+     * `api.proof?.registerVerifier(...)` so the plugin keeps working there.
+     */
+    proof?: {
+        registerVerifier(verifier: ProofVerifierRegistration): () => void;
+    };
     /**
      * Decision Intelligence: register a `DecisionProvider` or consume decisions. Absent on a core
      * that predates it: feature-detect with `api.decisions?.registerProvider(...)`.
@@ -1937,6 +2113,10 @@ export type ServerFrame = {
     };
     /** The session's goal, when it has one (so a reload shows the goal bar at once). */
     goal?: GoalInfo;
+    /** How many Proof criteria the session has; absent without any (the card loads on demand). */
+    proof?: {
+        criteria: number;
+    };
 }
 /** Durable event; its SSE `id` is `event.eventId`. */
  | {
@@ -2775,4 +2955,265 @@ export interface ForgeProgress {
         durationMs?: number;
     };
 }
+/** Hard limits of one reliability-lab task (`specs/010`). A run that exceeds one ends `budget_exceeded`. */
+export interface EvalBudget {
+    maxTurns: number;
+    maxTokens?: number;
+    maxWallMs: number;
+    maxToolCalls?: number;
+}
+/** Assertion over the sequence of tool events of one run. */
+export type ToolSequenceRule = 
+/** `first` must occur before `then`. */
+{
+    type: "before";
+    first: string;
+    then: string;
+} | {
+    type: "never";
+    tool: string;
+} | {
+    type: "max-calls";
+    tool: string;
+    max: number;
+} | {
+    type: "no-repeat-failure";
+    tool: string;
+};
+/** Deterministic, local check of a task result. `kind` stays open for future adapters. */
+export type EvalVerifier = {
+    kind: "command";
+    id: string;
+    command: string[];
+    timeoutMs: number;
+    expectExit?: number;
+    expectStdoutIncludes?: string;
+} | {
+    kind: "file-equals";
+    id: string;
+    path: string;
+    sha256: string;
+} | {
+    kind: "file-contains";
+    id: string;
+    path: string;
+    text: string;
+} | {
+    kind: "file-absent";
+    id: string;
+    path: string;
+} | {
+    kind: "tree-diff-within";
+    id: string;
+    allowed: string[];
+} | {
+    kind: "tool-sequence";
+    id: string;
+    rule: ToolSequenceRule;
+};
+export interface EvalTask {
+    id: string;
+    prompt: string;
+    /** Fixture directory, relative to the suite directory. */
+    fixture: string;
+    agent?: string;
+    budget: EvalBudget;
+    /**
+     * `allow` lists extra effects on top of the mode: `write`, `process`, `external` or `analysis`
+     * (never MCP or agents). A task can only equal or restrict what the invoker already holds.
+     */
+    policy: {
+        mode: "ask" | "auto" | "full";
+        readOnly?: boolean;
+        allow?: string[];
+    };
+    verifiers: EvalVerifier[];
+    tags?: string[];
+    /** Initialize a git repository (identity set, no global config) in the run copy of the fixture. */
+    fixtureGit?: boolean;
+    /**
+     * Replay-only noise filter, versioned with the suite: each `pattern` (a JavaScript regular
+     * expression, applied globally) found in a string of a request is replaced by `replace` BEFORE
+     * the request is fingerprinted, after the lab's own path and id normalization. Use it for
+     * non-deterministic fixture output (a timestamp a command prints). Over-broad rules hide real
+     * regressions; keep them narrow.
+     */
+    replayNormalize?: Array<{
+        pattern: string;
+        replace: string;
+    }>;
+}
+/** Content of a versioned `suite.json`. */
+export interface EvalSuite {
+    schemaVersion: 1;
+    suiteId: string;
+    suiteVersion: string;
+    description: string;
+    tasks: EvalTask[];
+}
+/** One repetition of one task, with the conditions it was measured under. */
+export interface EvalRunRecord {
+    schemaVersion: 1;
+    runId: string;
+    suiteId: string;
+    suiteVersion: string;
+    taskId: string;
+    repetition: number;
+    seed: number;
+    harnessVersion: string;
+    configHash: string;
+    provider: string;
+    model: string;
+    mode: "live" | "replay";
+    startedAt: number;
+    endedAt: number;
+    status: "pass" | "fail" | "budget_exceeded" | "error";
+    /** Which budget ended the run; present only when `status` is `budget_exceeded`. */
+    budgetExceeded?: "maxTurns" | "maxTokens" | "maxWallMs" | "maxToolCalls";
+    /**
+     * Why the run ended `error` (infrastructure or refusal), or the detail of a failure the model
+     * caused (`fail` with a `run_aborted` incident). Never contains file contents.
+     */
+    error?: string;
+    /**
+     * `sha256:` of the compaction-related configuration the run was measured under. Compaction
+     * calls are not in `measured` tokens, so two runs with different compaction settings are not
+     * comparable on tokens or cost.
+     */
+    compactionHash?: string;
+    verifiers: Array<{
+        id: string;
+        status: "pass" | "fail" | "error";
+        detail?: string;
+    }>;
+    /** Provider-reported usage, wall time and event counts. Never mixed with `estimated`. */
+    measured: {
+        inputTokens?: number;
+        outputTokens?: number;
+        /**
+         * True when only SOME turns reported usage: the token totals above understate the run and are
+         * treated as not reported by every aggregation.
+         */
+        usagePartial?: true;
+        wallMs: number;
+        turns: number;
+        toolCalls: number;
+    };
+    /** Derived values (cost from configured rates, approximate tokens). */
+    estimated?: {
+        costUsd?: number;
+        approxTokens?: number;
+    };
+    incidents: Array<{
+        kind: string;
+        count: number;
+    }>;
+    /**
+     * Present on a replay run that ended `error` with `error: "replay_divergence"`: the harness asked
+     * the provider for something the cassette did not record.
+     */
+    divergence?: EvalReplayDivergence;
+}
+/** One field of a request that differs from the recorded one (excerpts are redacted, at most ~240 characters). */
+export interface EvalDivergenceEntry {
+    call: number;
+    /** For example `instructions`, `messages[2].tool.result`, `tools[read_file].description`, `model`. */
+    field: string;
+    excerpt: {
+        before: string;
+        after: string;
+    };
+}
+/** Why a replay stopped: a changed request, a call past the recording, or recorded calls never asked for. */
+export interface EvalReplayDivergence {
+    kind: "fingerprint" | "exhausted" | "unconsumed";
+    message: string;
+    /** Empty unless `kind` is `fingerprint`; at most 10 entries. */
+    diff: EvalDivergenceEntry[];
+}
+/**
+ * Rates the caller supplies to turn measured tokens into an estimated cost (USD per million
+ * tokens). Alisio never ships or guesses rates: without them no cost is computed.
+ */
+export interface EvalRates {
+    inputPerMTokUsd: number;
+    outputPerMTokUsd: number;
+}
+/**
+ * Reliability metrics of one task over its repetitions. Runs that ended `error` (infrastructure)
+ * are not repetitions of the task: they are excluded from every number here and counted in
+ * `errored`. `k` is the number of judged runs (pass, fail or budget_exceeded).
+ */
+export interface EvalTaskMetrics {
+    taskId: string;
+    k: number;
+    /** Judged runs only. With `k: 0` the value is 0 and the interval [0, 1] carries no information. */
+    successRate: {
+        value: number;
+        wilsonLow: number;
+        wilsonHigh: number;
+    };
+    passAtK: boolean;
+    passPowK: boolean;
+    /** Runs of this task that ended `error` and were excluded from the numbers (omitted when 0). */
+    errored?: number;
+    /** `configHash` the task was measured under (a hash covers the task's own policy and budget). */
+    configHash?: string;
+    /**
+     * Provider-reported usage and wall time only. A value that cannot be computed honestly is
+     * omitted, never invented: the median needs one judged run, a variance two samples, the
+     * per-success figures at least one success (and usage on every judged run for tokens).
+     */
+    measured: {
+        medianWallMs?: number;
+        varianceWallMs?: number;
+        varianceTokens?: number;
+        tokensPerSuccess?: number;
+        wallMsPerSuccess?: number;
+    };
+    /** Present only when the caller supplied rates and every judged run reported usage. */
+    estimated?: {
+        costUsdPerSuccess?: number;
+    };
+}
+export interface EvalReport {
+    schemaVersion: 1;
+    suiteId: string;
+    suiteVersion: string;
+    conditions: {
+        harnessVersion: string;
+        configHash: string;
+        provider: string;
+        model: string;
+        mode: "live" | "replay";
+        k: number;
+        seedBase: number;
+        /** Compaction identity shared by every run (omitted for reports of older runs). */
+        compactionHash?: string;
+    };
+    /** The variant this report was measured as (`eval variants`): its id and config overrides. */
+    variant?: {
+        id: string;
+        overrides: Record<string, unknown>;
+    };
+    /** Caveats about what the numbers mean (for example which tokens are counted). */
+    notes?: string[];
+    tasks: EvalTaskMetrics[];
+    /** IDs of the `EvalRunRecord`s, errored ones included (the detail lives in `runs.jsonl`). */
+    runs: string[];
+    /**
+     * Counts of RUNS, except `tasks`: `passed`, `failed`, `errored` and `budgetExceeded` add up to
+     * `runs`. `errored` runs are infrastructure and are excluded from every rate and spread.
+     */
+    summary: {
+        tasks: number;
+        passed: number;
+        errored: number;
+        budgetExceeded: number;
+        runs?: number;
+        failed?: number;
+    };
+    /** The rates behind every `estimated` figure; absent when none was supplied. */
+    rates?: EvalRates;
+}
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alisio/sdk",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "description": "Typed plugin SDK for Alisio: the stable contract for tools, commands, context, compaction and session hooks, model completions and storage. Types only plus tiny helpers; zero runtime dependencies.",
   "author": "Gustavo Gutiérrez",
   "license": "MIT",