@alisio/sdk 0.5.2 → 0.7.0

package/dist/index.d.ts

@@ -1848,6 +1848,8 @@ export interface InflightState {
         startedAt: number;
         /** Latest progress output, bounded. */
         tail: string;
+        /** Latest `ForgeProgress` of a `js_run`/`python_run` call; never part of `tail`. */
+        forge?: ForgeProgress;
     }>;
 }
 /** An approval waiting for a decision from a web client. */
@@ -1871,8 +1873,11 @@ export interface PendingApproval {
     capability?: AnalysisCapability;
     /** A short preview for capability approvals (the first 40 lines of the script). */
     preview?: string;
-    /** Capability approvals: `managed` (not sandboxed) or `oci` (container). */
-    runtime?: "managed" | "oci";
+    /**
+     * Capability approvals: `managed` (not sandboxed), `oci` (container), `wasm` (Pyodide in a
+     * restricted process) or `auto` (the most isolated backend that is ready, decided at run time).
+     */
+    runtime?: "auto" | "managed" | "oci" | "wasm";
     /** `analysis.install` approvals: only `once` and `deny` are offered. */
     install?: InstallPreview;
 }
@@ -2359,7 +2364,12 @@ export interface AnalysisStatus {
     /** `analysis.enabled` and not `--read-only`: python_run is registered. */
     enabled: boolean;
     readOnly: boolean;
-    mode: "managed" | "oci";
+    /** `analysis.runtime`: `auto` picks the most isolated backend that is ready. */
+    mode: "auto" | "managed" | "oci" | "wasm";
+    /** `analysis.isolation`: what `auto` may fall back to. */
+    isolation?: "preferred" | "required" | "strict";
+    /** The Python backends with their live state (isolation, ready, profiles, reason). */
+    backends?: EngineDescriptor[];
     /** The discovered (or `--python`) interpreter and the extras environment. */
     python: {
         found: true;
@@ -2649,4 +2659,381 @@ export declare const textResult: (text: string, isError?: boolean) => ToolResult
  * raw bytes or structured blocks never reach the model prompt.
  */
 export declare function textProjection(result: ToolResult): ToolResult;
+/** The language a Forja engine runs. */
+export type EngineLanguage = "javascript" | "python";
+/** Stable identifier of a Forja execution engine. */
+export type EngineId = "javascript-quickjs" | "python-managed" | "python-oci" | "python-wasm";
+/** Isolation an engine declares and has verified, ordered from the strongest guarantee to none. */
+export type IsolationLevel = "os-container" | "wasm-isolate" | "wasm-process" | "none";
+/** What an engine reports about itself. An engine that is not `ready` never runs code. */
+export interface EngineDescriptor {
+    id: EngineId;
+    language: EngineLanguage;
+    /** Engine build, e.g. `quickjs-wasi@3.6.2` or `cpython-3.12.4`. */
+    version: string;
+    isolation: IsolationLevel;
+    ready: boolean;
+    /** Actionable reason the engine is not ready. */
+    reason?: string;
+    capabilities: {
+        cooperativeInterrupt: boolean;
+        /** The host can terminate the guest without its cooperation. */
+        hardTermination: boolean;
+        hardMemoryLimit: boolean;
+        networkDeniedByOS: boolean;
+        /** The guest can call core tools. */
+        toolCalls: boolean;
+    };
+    /** Package profiles available (Python). */
+    profiles?: string[];
+}
+/**
+ * `GET /api/forge`: the Forja engines with their live state. `engines` lists the JavaScript engine
+ * first and then the Python backends (`python-oci`, `python-wasm`, `python-managed`), each with its
+ * isolation, readiness, reason and capabilities (`networkDeniedByOS` comes from a self-probe).
+ */
+export interface ForgeOverview {
+    engines: EngineDescriptor[];
+    javascript: {
+        verified: boolean;
+        verification?: string;
+        enabled: boolean;
+        host: "worker" | "process";
+        hostSetting: "auto" | "worker" | "process";
+        wasmSha256: string;
+        version: string;
+        limits: {
+            maxSourceBytes: number;
+            timeoutMs: number;
+            memoryBytes: number;
+            stackBytes: number;
+            maxToolCalls: number;
+            maxConcurrency: number;
+            maxCallInputBytes: number;
+            maxCallResponseBytes: number;
+            maxRetainedOutputBytes: number;
+            maxOutputChars: number;
+        };
+    };
+    /** The scratchpad settings (`forge.scratch.*`); the folder is a host path and never listed. */
+    scratch: {
+        enabled: boolean;
+        retentionDays: number;
+        maxBytes: number;
+    };
+    state: {
+        enabled: boolean;
+    };
+    guides: {
+        enabled: boolean;
+    };
+    /** Python settings only the user decides, read only: `analysis.runtime` and `analysis.isolation`. */
+    python?: {
+        runtime: "auto" | "managed" | "oci" | "wasm";
+        isolation: "preferred" | "required" | "strict";
+    };
+}
+/** One file of a session's scratchpad (`GET /api/sessions/:sid/scratch`); never an absolute path. */
+export interface ScratchFile {
+    /** Path relative to the scratchpad, `/`-separated; also the tail of the download URL. */
+    name: string;
+    /** Bytes. */
+    size: number;
+    /** Last modification, ms epoch. */
+    mtime: number;
+}
+/** Answer of `GET /api/sessions/:sid/scratch`. */
+export interface ScratchListing {
+    /** False while Forja's JavaScript engine or its scratchpad is off: there is nothing to list. */
+    available: boolean;
+    files: ScratchFile[];
+    /** Bytes the quota counts (`forge.scratch.maxBytes`), symlinks as links. */
+    usedBytes: number;
+    quotaBytes: number;
+    /** The list was cut at 2 000 files; `usedBytes` still counts all of them. */
+    truncated?: boolean;
+}
+/** Answer of `DELETE /api/sessions/:sid/scratch`. */
+export interface ScratchClearResult {
+    /** Files (and links) removed. */
+    removed: number;
+    freedBytes: number;
+}
+/** Typed payload of a `tool_progress` event emitted by a Forja execution. */
+export interface ForgeProgress {
+    kind: "forge";
+    executionId: string;
+    engine: EngineId;
+    phase: "preparing" | "running" | "call" | "collecting";
+    message?: string;
+    completed?: number;
+    total?: number;
+    call?: {
+        callId: number;
+        tool: string;
+        status: "started" | "succeeded" | "failed" | "denied";
+        durationMs?: number;
+    };
+}
+/** Hard limits of one reliability-lab task (`specs/010`). A run that exceeds one ends `budget_exceeded`. */
+export interface EvalBudget {
+    maxTurns: number;
+    maxTokens?: number;
+    maxWallMs: number;
+    maxToolCalls?: number;
+}
+/** Assertion over the sequence of tool events of one run. */
+export type ToolSequenceRule = 
+/** `first` must occur before `then`. */
+{
+    type: "before";
+    first: string;
+    then: string;
+} | {
+    type: "never";
+    tool: string;
+} | {
+    type: "max-calls";
+    tool: string;
+    max: number;
+} | {
+    type: "no-repeat-failure";
+    tool: string;
+};
+/** Deterministic, local check of a task result. `kind` stays open for future adapters. */
+export type EvalVerifier = {
+    kind: "command";
+    id: string;
+    command: string[];
+    timeoutMs: number;
+    expectExit?: number;
+    expectStdoutIncludes?: string;
+} | {
+    kind: "file-equals";
+    id: string;
+    path: string;
+    sha256: string;
+} | {
+    kind: "file-contains";
+    id: string;
+    path: string;
+    text: string;
+} | {
+    kind: "file-absent";
+    id: string;
+    path: string;
+} | {
+    kind: "tree-diff-within";
+    id: string;
+    allowed: string[];
+} | {
+    kind: "tool-sequence";
+    id: string;
+    rule: ToolSequenceRule;
+};
+export interface EvalTask {
+    id: string;
+    prompt: string;
+    /** Fixture directory, relative to the suite directory. */
+    fixture: string;
+    agent?: string;
+    budget: EvalBudget;
+    /**
+     * `allow` lists extra effects on top of the mode: `write`, `process`, `external` or `analysis`
+     * (never MCP or agents). A task can only equal or restrict what the invoker already holds.
+     */
+    policy: {
+        mode: "ask" | "auto" | "full";
+        readOnly?: boolean;
+        allow?: string[];
+    };
+    verifiers: EvalVerifier[];
+    tags?: string[];
+    /** Initialize a git repository (identity set, no global config) in the run copy of the fixture. */
+    fixtureGit?: boolean;
+    /**
+     * Replay-only noise filter, versioned with the suite: each `pattern` (a JavaScript regular
+     * expression, applied globally) found in a string of a request is replaced by `replace` BEFORE
+     * the request is fingerprinted, after the lab's own path and id normalization. Use it for
+     * non-deterministic fixture output (a timestamp a command prints). Over-broad rules hide real
+     * regressions; keep them narrow.
+     */
+    replayNormalize?: Array<{
+        pattern: string;
+        replace: string;
+    }>;
+}
+/** Content of a versioned `suite.json`. */
+export interface EvalSuite {
+    schemaVersion: 1;
+    suiteId: string;
+    suiteVersion: string;
+    description: string;
+    tasks: EvalTask[];
+}
+/** One repetition of one task, with the conditions it was measured under. */
+export interface EvalRunRecord {
+    schemaVersion: 1;
+    runId: string;
+    suiteId: string;
+    suiteVersion: string;
+    taskId: string;
+    repetition: number;
+    seed: number;
+    harnessVersion: string;
+    configHash: string;
+    provider: string;
+    model: string;
+    mode: "live" | "replay";
+    startedAt: number;
+    endedAt: number;
+    status: "pass" | "fail" | "budget_exceeded" | "error";
+    /** Which budget ended the run; present only when `status` is `budget_exceeded`. */
+    budgetExceeded?: "maxTurns" | "maxTokens" | "maxWallMs" | "maxToolCalls";
+    /**
+     * Why the run ended `error` (infrastructure or refusal), or the detail of a failure the model
+     * caused (`fail` with a `run_aborted` incident). Never contains file contents.
+     */
+    error?: string;
+    /**
+     * `sha256:` of the compaction-related configuration the run was measured under. Compaction
+     * calls are not in `measured` tokens, so two runs with different compaction settings are not
+     * comparable on tokens or cost.
+     */
+    compactionHash?: string;
+    verifiers: Array<{
+        id: string;
+        status: "pass" | "fail" | "error";
+        detail?: string;
+    }>;
+    /** Provider-reported usage, wall time and event counts. Never mixed with `estimated`. */
+    measured: {
+        inputTokens?: number;
+        outputTokens?: number;
+        /**
+         * True when only SOME turns reported usage: the token totals above understate the run and are
+         * treated as not reported by every aggregation.
+         */
+        usagePartial?: true;
+        wallMs: number;
+        turns: number;
+        toolCalls: number;
+    };
+    /** Derived values (cost from configured rates, approximate tokens). */
+    estimated?: {
+        costUsd?: number;
+        approxTokens?: number;
+    };
+    incidents: Array<{
+        kind: string;
+        count: number;
+    }>;
+    /**
+     * Present on a replay run that ended `error` with `error: "replay_divergence"`: the harness asked
+     * the provider for something the cassette did not record.
+     */
+    divergence?: EvalReplayDivergence;
+}
+/** One field of a request that differs from the recorded one (excerpts are redacted, at most ~240 characters). */
+export interface EvalDivergenceEntry {
+    call: number;
+    /** For example `instructions`, `messages[2].tool.result`, `tools[read_file].description`, `model`. */
+    field: string;
+    excerpt: {
+        before: string;
+        after: string;
+    };
+}
+/** Why a replay stopped: a changed request, a call past the recording, or recorded calls never asked for. */
+export interface EvalReplayDivergence {
+    kind: "fingerprint" | "exhausted" | "unconsumed";
+    message: string;
+    /** Empty unless `kind` is `fingerprint`; at most 10 entries. */
+    diff: EvalDivergenceEntry[];
+}
+/**
+ * Rates the caller supplies to turn measured tokens into an estimated cost (USD per million
+ * tokens). Alisio never ships or guesses rates: without them no cost is computed.
+ */
+export interface EvalRates {
+    inputPerMTokUsd: number;
+    outputPerMTokUsd: number;
+}
+/**
+ * Reliability metrics of one task over its repetitions. Runs that ended `error` (infrastructure)
+ * are not repetitions of the task: they are excluded from every number here and counted in
+ * `errored`. `k` is the number of judged runs (pass, fail or budget_exceeded).
+ */
+export interface EvalTaskMetrics {
+    taskId: string;
+    k: number;
+    /** Judged runs only. With `k: 0` the value is 0 and the interval [0, 1] carries no information. */
+    successRate: {
+        value: number;
+        wilsonLow: number;
+        wilsonHigh: number;
+    };
+    passAtK: boolean;
+    passPowK: boolean;
+    /** Runs of this task that ended `error` and were excluded from the numbers (omitted when 0). */
+    errored?: number;
+    /** `configHash` the task was measured under (a hash covers the task's own policy and budget). */
+    configHash?: string;
+    /**
+     * Provider-reported usage and wall time only. A value that cannot be computed honestly is
+     * omitted, never invented: the median needs one judged run, a variance two samples, the
+     * per-success figures at least one success (and usage on every judged run for tokens).
+     */
+    measured: {
+        medianWallMs?: number;
+        varianceWallMs?: number;
+        varianceTokens?: number;
+        tokensPerSuccess?: number;
+        wallMsPerSuccess?: number;
+    };
+    /** Present only when the caller supplied rates and every judged run reported usage. */
+    estimated?: {
+        costUsdPerSuccess?: number;
+    };
+}
+export interface EvalReport {
+    schemaVersion: 1;
+    suiteId: string;
+    suiteVersion: string;
+    conditions: {
+        harnessVersion: string;
+        configHash: string;
+        provider: string;
+        model: string;
+        mode: "live" | "replay";
+        k: number;
+        seedBase: number;
+        /** Compaction identity shared by every run (omitted for reports of older runs). */
+        compactionHash?: string;
+    };
+    /** The variant this report was measured as (`eval variants`): its id and config overrides. */
+    variant?: {
+        id: string;
+        overrides: Record<string, unknown>;
+    };
+    /** Caveats about what the numbers mean (for example which tokens are counted). */
+    notes?: string[];
+    tasks: EvalTaskMetrics[];
+    /** IDs of the `EvalRunRecord`s, errored ones included (the detail lives in `runs.jsonl`). */
+    runs: string[];
+    /**
+     * Counts of RUNS, except `tasks`: `passed`, `failed`, `errored` and `budgetExceeded` add up to
+     * `runs`. `errored` runs are infrastructure and are excluded from every rate and spread.
+     */
+    summary: {
+        tasks: number;
+        passed: number;
+        errored: number;
+        budgetExceeded: number;
+        runs?: number;
+        failed?: number;
+    };
+    /** The rates behind every `estimated` figure; absent when none was supplied. */
+    rates?: EvalRates;
+}
 export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alisio/sdk",
-  "version": "0.5.2",
+  "version": "0.7.0",
   "description": "Typed plugin SDK for Alisio: the stable contract for tools, commands, context, compaction and session hooks, model completions and storage. Types only plus tiny helpers; zero runtime dependencies.",
   "author": "Gustavo Gutiérrez",
   "license": "MIT",