@theokit/sdk 4.4.2 → 4.5.1

package/dist/interactive/index.d.cts

@@ -0,0 +1,12 @@
+/**
+ * `@theokit/sdk/interactive` — pluggable interactive-session backend seam.
+ *
+ * The streaming twin of `@theokit/sdk/sandbox` (one-shot `execute`) and sibling
+ * of `@theokit/sdk/filesystem`. Ship an `InteractiveBackend` (e.g. the local
+ * `@theokit/sdk-pty`, a container/E2B backend for the cluster, or a Tauri
+ * backend) to give agent shell tools a surface-agnostic REPL/stdin capability —
+ * with NO native dependency in core.
+ *
+ * @public
+ */
+export { InteractiveBackend, type InteractiveProvider, InteractiveUnavailableError, NoSuchSessionError, resolveInteractive, type StartInteractiveOptions, type StartInteractiveResult, type WriteStdinOptions, type WriteStdinResult, } from "./types.js";

package/dist/interactive/types.d.cts

@@ -0,0 +1,85 @@
+/**
+ * Interactive-session backend protocol — the streaming twin of `SandboxBackend`
+ * (which is one-shot `execute`). A surface-agnostic contract for driving a
+ * long-lived interactive process (a REPL, `git rebase -i`, any command that
+ * PROMPTS for stdin): start → `session_id`, write to stdin, read incremental
+ * output, kill.
+ *
+ * Injected exactly like {@link FilesystemProvider} (SE31) — the tool depends on
+ * the interface, the HOST supplies the implementation, so the SAME tool runs on
+ * a local PTY (`@theokit/sdk-pty`), a container/E2B backend (cluster/web), or a
+ * desktop backend (Tauri) with NO tool change and NO native dependency in core.
+ *
+ * @public
+ */
+/** Thrown when the interactive path is requested but no backend can provide it
+ *  (no provider injected, or a local backend whose native module / spawn failed).
+ *  The caller falls back to non-interactive exec. */
+export declare class InteractiveUnavailableError extends Error {
+    readonly code: "interactive_unavailable";
+    constructor(message: string);
+}
+/** Thrown (typed) when a write/kill targets an unknown or already-exited session,
+ *  so callers branch on the type instead of string-matching a message. */
+export declare class NoSuchSessionError extends Error {
+    readonly code: "no_such_session";
+    constructor(sessionId: string);
+}
+/** Result of starting a session: its id + whatever the program printed on startup. */
+export interface StartInteractiveResult {
+    sessionId: string;
+    output: string;
+}
+/** Result of writing to a session: the output produced during the yield window + liveness. */
+export interface WriteStdinResult {
+    output: string;
+    alive: boolean;
+}
+/** Bounds a start call. All optional; a backend clamps/defaults each. */
+export interface StartInteractiveOptions {
+    /** Working directory for the session. Defaults to the backend's root. */
+    cwd?: string;
+    /** How long to wait, in ms, before returning the startup output (clamped by the backend). */
+    yieldMs?: number;
+    /** Idle time, in ms, after which the backend reaps a forgotten session. */
+    ttlMs?: number;
+    /** Cap on the returned output bytes (tail kept). */
+    maxBytes?: number;
+    /** Terminal geometry, when the backend allocates a real TTY. */
+    cols?: number;
+    rows?: number;
+}
+/** Bounds a write call. */
+export interface WriteStdinOptions {
+    yieldMs?: number;
+    ttlMs?: number;
+    maxBytes?: number;
+}
+/**
+ * Pluggable interactive-session backend. Implement the three abstract methods.
+ * A backend that cannot provide interactive sessions should not be constructed —
+ * callers detect absence by catching {@link InteractiveUnavailableError}.
+ *
+ * @public
+ */
+export declare abstract class InteractiveBackend {
+    /** Spawn `command` as an interactive session; resolve after the yield window with the
+     *  `session_id` + startup output. Throws {@link InteractiveUnavailableError} when the
+     *  session cannot be allocated. */
+    abstract startInteractive(command: string, opts?: StartInteractiveOptions): Promise<StartInteractiveResult>;
+    /** Write `chars` to a live session's stdin; resolve after the yield window with the output it
+     *  produced + whether it is still alive. Throws {@link NoSuchSessionError} on an unknown session. */
+    abstract writeStdin(sessionId: string, chars: string, opts?: WriteStdinOptions): Promise<WriteStdinResult>;
+    /** Kill a session (idempotent) and free its slot. */
+    abstract kill(sessionId: string): void;
+}
+/**
+ * A backend OR a per-request resolver of one — mirrors {@link FilesystemProvider}. A resolver runs
+ * at tool-execution time (request scope), so multi-tenant / multi-role agents get a distinct backend
+ * per request without a shared mutable one.
+ *
+ * @public
+ */
+export type InteractiveProvider<Ctx = unknown> = InteractiveBackend | ((ctx: Ctx) => InteractiveBackend | Promise<InteractiveBackend>);
+/** Resolve an {@link InteractiveProvider} to a concrete backend for `ctx`. */
+export declare function resolveInteractive<Ctx>(provider: InteractiveProvider<Ctx>, ctx: Ctx): Promise<InteractiveBackend>;

package/dist/internal/eval/assert.d.ts

@@ -0,0 +1,26 @@
+/**
+ * SE41 — `assertEval(run, thresholds)`: the CI gate for evals.
+ *
+ * A pure function over a completed {@link EvalRun}. It reads only
+ * `run.aggregate`, collects EVERY unmet threshold (not just the first), and
+ * throws {@link EvalThresholdError} carrying the full failure list. Passing
+ * silently returns `void` — drop it straight into a Vitest `it(...)` or a
+ * standalone eval script whose non-zero exit fails the CI job.
+ *
+ * @public
+ */
+import type { EvalRun, EvalThresholdFailure, EvalThresholds } from "../../types/eval.js";
+/** Thrown by {@link assertEval} when a run misses one or more thresholds. */
+export declare class EvalThresholdError extends Error {
+    readonly name = "EvalThresholdError";
+    /** The eval's name (`EvalRun.name`). */
+    readonly evalName: string;
+    /** Every unmet threshold, in check order. */
+    readonly failures: readonly EvalThresholdFailure[];
+    constructor(evalName: string, failures: readonly EvalThresholdFailure[]);
+}
+/**
+ * Assert a run meets every set threshold. Throws {@link EvalThresholdError}
+ * with the complete list of failures when it does not; returns `void` on pass.
+ */
+export declare function assertEval(run: EvalRun, thresholds: EvalThresholds): void;

package/dist/internal/eval/trials.d.ts

@@ -0,0 +1,19 @@
+/**
+ * SE41 — trial expansion + collapse for `EvalOptions.trials`.
+ *
+ * Strategy: EXPAND each dataset entry into `trials` tagged copies, run them
+ * through the existing execution paths untouched, then COLLAPSE the resulting
+ * per-trial rows back into one row per original entry. Per-scorer score is the
+ * mean over the trials — an errored trial contributes 0 (a reliability signal),
+ * so the denominator is always `trials`, not the count of successful trials.
+ *
+ * @internal
+ */
+/** Reserved metadata key: the original dataset-row index a trial belongs to. */
+export declare const TRIAL_INDEX_KEY = "__evalRowIndex";
+/** Reserved metadata key: the 0-based trial number within a row. */
+export declare const TRIAL_NUM_KEY = "__evalTrial";
+/** Repeat each entry `trials` times, tagging each copy with its origin + trial number. */
+export declare function expandForTrials(entries: ReadonlyArray<DatasetEntry>, trials: number): DatasetEntry[];
+/** Group per-trial rows by their original entry index and collapse each group. */
+export declare function collapseTrials(rows: ReadonlyArray<EvalRowResult>, trials: number): EvalRowResult[];

package/dist/internal/scorers/levenshtein.d.ts

@@ -0,0 +1,11 @@
+/**
+ * SE41 — Levenshtein edit distance for `Scorers.levenshtein`.
+ *
+ * Classic two-row dynamic-programming distance (O(n) memory). Callers MUST
+ * bound input length via {@link LEVENSHTEIN_MAX_LEN} before calling — the
+ * algorithm is O(n*m) time, so unbounded LLM output would be a DoS vector.
+ *
+ * @internal
+ */
+/** Minimum single-edit distance between `a` and `b` (two-row DP, O(n) memory). */
+export declare function levenshteinDistance(a: string, b: string): number;

package/dist/scorers.d.ts

@@ -15,6 +15,40 @@
 import type { ZodType } from "zod";
 import { type LlmJudgeOptions } from "./internal/scorers/llm-judge.js";
 import type { NamedScorer, VerifyGateOptions } from "./types/eval.js";
+/** SE41 — options for the deterministic fuzzy `Scorers.levenshtein`. */
+interface LevenshteinOptions {
+    /** Case-sensitive compare. Default: false (fuzzy matching is usually case-insensitive). */
+    caseSensitive?: boolean;
+    /**
+     * When set, binarize: `score = normalizedSimilarity >= threshold ? 1 : 0`.
+     * When omitted, return the continuous similarity in `[0, 1]`.
+     */
+    threshold?: number;
+}
+/** SE41 — options for the deterministic `Scorers.numericDiff`. */
+interface NumericDiffOptions {
+    /**
+     * When set, binarize: `score = |output - expected| <= tolerance ? 1 : 0`.
+     * When omitted, return the continuous relative closeness in `[0, 1]`.
+     */
+    tolerance?: number;
+}
+/** SE41 — options for `Scorers.embeddingSimilarity`. */
+interface EmbeddingSimilarityOptions {
+    /** Embedding API key. Defaults to `OPENROUTER_API_KEY` from the environment. */
+    apiKey?: string;
+    /** Embedding model id (OpenRouter catalog). Default: `openai/text-embedding-3-small`. */
+    model?: string;
+    /** Override the embeddings HTTP base URL. */
+    baseUrl?: string;
+    /** When set, binarize: `score = cosine >= threshold ? 1 : 0`. Else continuous. */
+    threshold?: number;
+    /**
+     * Inject an embedding function (DIP): `(texts) => vectors`. When provided, the
+     * OpenRouter runtime is NOT constructed — used by tests and custom providers.
+     */
+    embed?: (texts: ReadonlyArray<string>) => Promise<number[][]>;
+}
 interface ExactMatchOptions {
     /** Case-sensitive compare. Default: true. */
     caseSensitive?: boolean;
@@ -47,6 +81,29 @@ export declare const Scorers: {
      * inputs before using in production.
      */
     regex(pattern: RegExp): NamedScorer;
+    /**
+     * SE41 — normalized Levenshtein similarity: `1 - editDistance / max(len)`.
+     * Deterministic (no LLM), so it always runs in CI. `threshold` binarizes.
+     *
+     * Refuses empty/non-string `expected` (EC-1 parity) and caps input at
+     * {@link LEVENSHTEIN_MAX_LEN} chars to bound the O(n*m) cost on adversarial output.
+     */
+    levenshtein(opts?: LevenshteinOptions): NamedScorer;
+    /**
+     * SE41 — numeric closeness. Parses `output` and `expected` as numbers and
+     * scores continuous relative closeness `1 - |o-e| / max(|o|,|e|)` (both 0 ⇒ 1),
+     * or a binary pass when `tolerance` is set. Deterministic (no LLM).
+     */
+    numericDiff(opts?: NumericDiffOptions): NamedScorer;
+    /**
+     * SE41 — semantic similarity via embeddings: cosine of `embed(output)` vs
+     * `embed(expected)`, clamped to `[0, 1]` (negatives → 0). `threshold` binarizes.
+     *
+     * By default routes through OpenRouter's embeddings endpoint
+     * (`OPENROUTER_API_KEY`); inject `embed` to use another provider or to test
+     * deterministically. Each scored row costs one embeddings call.
+     */
+    embeddingSimilarity(opts: EmbeddingSimilarityOptions): NamedScorer;
     /**
      * Parse `output` as JSON and validate against a Zod schema.
      *

package/dist/types/eval.d.ts

@@ -71,6 +71,15 @@ export interface EvalOptions {
      * `[1, 64]` (EC-3 — 0 deadlocks the semaphore, Infinity DoSs the provider).
      */
     readonly concurrency?: number;
+    /**
+     * Repeat each dataset row `trials` times to smooth non-determinism (SE41).
+     * Default 1 (no repeat; behavior byte-identical to a non-trialed run). MUST
+     * be an integer in `[1, 100]`. With `trials > 1` the returned `EvalRun.rows`
+     * still has ONE row per dataset entry — see {@link EvalRowResult.trialCount}
+     * for the collapse semantics. Reserved metadata keys `__evalTrial` /
+     * `__evalRowIndex` are attached to each per-trial persisted row.
+     */
+    readonly trials?: number;
     /** Optional metadata persisted to `EvalRun.metadata` (tags, env, version). */
     readonly metadata?: Record<string, unknown>;
     /** Optional progress / lifecycle hooks. */
@@ -98,6 +107,14 @@ export interface EvalRowResult {
      * Persisted alongside the row when `persist` is set.
      */
     readonly outcome?: string;
+    /**
+     * How many trials produced this row (SE41). Present only when
+     * `EvalOptions.trials > 1`: the row is a COLLAPSE of `trialCount` executions
+     * of the same dataset entry — each scorer's `score` is the mean over the
+     * trials (an errored trial contributes 0, a reliability signal), and
+     * `durationMs` / `tokensIn` / `tokensOut` are summed. Absent ⇒ single run.
+     */
+    readonly trialCount?: number;
     /**
      * Captured code change (M6-4): the working-tree `git diff` an agent produced
      * and whether it reverse-applies cleanly. Produced by `captureArtifact`; the
@@ -193,6 +210,34 @@ export interface EvalPersistOptions {
      */
     readonly resume?: boolean;
 }
+/**
+ * Threshold contract for `assertEval(run, thresholds)` (SE41) — the CI gate.
+ * Every field is optional; only the ones you set are checked. A run passes
+ * iff EVERY set threshold is satisfied; otherwise `assertEval` throws
+ * `EvalThresholdError` carrying the full list of failures.
+ */
+export interface EvalThresholds {
+    /** `aggregate.meanScore` MUST be `>=` this. */
+    readonly minMeanScore?: number;
+    /** `aggregate.passRatio` (rows with meanScore >= 0.5) MUST be `>=` this. */
+    readonly minPassRatio?: number;
+    /** `errorRows / totalRows` MUST be `<=` this (0 rows ⇒ ratio 0). */
+    readonly maxErrorRatio?: number;
+    /**
+     * Per-scorer floor on `aggregate.perScorer[name].mean`. A named scorer that
+     * never ran (absent from `perScorer`) is itself a failure ("scorer not found").
+     */
+    readonly perScorer?: Readonly<Record<string, number>>;
+}
+/** One unmet threshold, surfaced on `EvalThresholdError.failures`. */
+export interface EvalThresholdFailure {
+    /** Stable metric id, e.g. `"meanScore"`, `"passRatio"`, `"perScorer.exact-match"`. */
+    readonly metric: string;
+    /** The floor (or ceiling, for `maxErrorRatio`) that was required. */
+    readonly threshold: number;
+    /** The observed value that violated the threshold (`NaN` when the scorer was absent). */
+    readonly actual: number;
+}
 /** Per-call options for `eval.run(...)`. */
 export interface EvalRunOptions {
     /** Cancels pending rows; in-flight rows complete (D140 pattern). */

package/docs/error-codes.md

@@ -0,0 +1,157 @@
+# `@theokit/sdk` Error Codes Reference
+
+Canonical reference for `AgentRunError.code` values + provider-to-code mapping (Production-Readiness #3, ADRs D311-D314).
+
+## `AgentRunErrorCode` union (16 codes)
+
+| Code | Origin | Retriable | When |
+|---|---|:---:|---|
+| `auth_failed` | provider HTTP 401/403 | no | bad API key, revoked token |
+| `rate_limit` | provider HTTP 429 | **yes** | back off using `retryAfterMs` |
+| `quota_exceeded` | provider HTTP 402 / billing body code | no | billing limit hit |
+| `invalid_request` | provider HTTP 400 (generic) | no | malformed payload |
+| `invalid_model` | provider HTTP 400 + "model not found" | no | model id wrong/unavailable |
+| `context_too_long` | provider 400 + context_length code | no | input exceeds model window |
+| `content_filtered` | provider safety filter | no | safety filter blocked |
+| `safety_blocked` | provider safety filter (alias) | no | reserved for stricter mapping |
+| `model_unavailable` | provider 400 + model_unavailable code | no | model temporarily unavailable |
+| `timeout` | HTTP 408 | **yes** | retry with backoff |
+| `network` | DNS/TCP/transport | **yes** | retry with backoff |
+| `server_error` | HTTP 5xx | **yes** | retry with backoff |
+| `provider_unreachable` | DNS/TCP/timeout/5xx (alias) | **yes** | reserved for stricter mapping |
+| `tool_runtime_error` | tool handler throw inside dispatch | no | bug in handler |
+| `aborted` | `AbortSignal` fired (Phase 4) | no | user/lifecycle cancel |
+| `unknown` | unmapped | no | provider returned shape we don't recognize |
+
+## Exhaustive `switch` pattern
+
+```ts
+import { AgentRunError } from "@theokit/sdk";
+
+try {
+  await agent.send(message);
+} catch (err) {
+  if (!(err instanceof AgentRunError)) throw err;
+  switch (err.code) {
+    case "auth_failed":
+      // bad key — show login UI
+      break;
+    case "rate_limit":
+      if (err.retryAfterMs !== undefined) {
+        setTimeout(retry, err.retryAfterMs);
+      }
+      break;
+    case "quota_exceeded":
+      // billing — upsell page
+      break;
+    case "tool_runtime_error":
+      // handler bug — log + tell user
+      break;
+    case "aborted":
+      // user cancelled — no UI noise
+      break;
+    default:
+      // unknown / new code — generic fallback
+      break;
+  }
+}
+```
+
+## Provider mapping table
+
+### OpenAI (and OpenAI-compat: OpenRouter, DeepSeek, Together, Mistral, Voyage, DeepInfra)
+
+| Status | Body hint | → ErrorCode | → AgentRunErrorCode |
+|---|---|---|---|
+| 401 / 403 | any | `auth_failed` | `auth_failed` |
+| 429 | any | `rate_limit` | `rate_limit` |
+| 402 | any | `invalid_request` | `quota_exceeded` (at AgentRunError layer) |
+| 400 | `code: "context_length_exceeded"` | `context_too_long` | `context_too_long` |
+| 400 | `code: "content_policy_violation"` | `content_filtered` | `content_filtered` |
+| 400 | `code: "model_not_found"` | `model_unavailable` | `model_unavailable` |
+| 400 | `code: "insufficient_quota"` | `invalid_request` | `quota_exceeded` |
+| 400 | other | `invalid_request` | `invalid_request` |
+| 408 | any | `timeout` | `timeout` |
+| 5xx | any | `server_error` | `server_error` |
+| other | any | `unknown` | `unknown` |
+
+### Anthropic
+
+Same status-based map as OpenAI. Body code hints are dialect-specific (`overloaded_error` → `server_error`, etc.) — see `internal/error-mappers/anthropic.ts` for the authoritative list.
+
+### Vertex AI
+
+| GCP status | Canonical | Code |
+|---|---|---|
+| `429` / `RESOURCE_EXHAUSTED` | `RateLimitError` | `rate_limit` |
+| `401` / `UNAUTHENTICATED` | `AuthenticationError` | `auth_failed` |
+| `403` / `PERMISSION_DENIED` | `AuthenticationError` | `auth_failed` |
+| `400` / `INVALID_ARGUMENT` | `ConfigurationError` | `invalid_request` |
+| `408` / `DEADLINE_EXCEEDED` | `NetworkError` | `timeout` |
+| `5xx` | `NetworkError` | `server_error` |
+| other | `UnknownAgentError` | `unknown` |
+
+### Bedrock
+
+| AWS status / type | Canonical | Code |
+|---|---|---|
+| 429 / `ThrottlingException` | `RateLimitError` | `rate_limit` |
+| 401/403 / `AccessDeniedException` | `AuthenticationError` | `auth_failed` |
+| 400 / `ValidationException` | `ConfigurationError` | `invalid_request` |
+| 5xx | `NetworkError` | `server_error` |
+
+### Ollama (local)
+
+| Failure mode | Code |
+|---|---|
+| Connection refused | `network` |
+| Timeout | `timeout` |
+| Unknown response | `unknown` |
+
+Ollama has no billing, no rate limit, no auth → `quota_exceeded`/`rate_limit`/`auth_failed` never fire.
+
+## Fields
+
+```ts
+class AgentRunError extends TheokitAgentError {
+  readonly code: AgentRunErrorCode;
+  readonly provider?: string;
+  readonly raw?: string;
+  readonly requestId?: string;      // x-request-id / request-id header
+  readonly conversationId?: string; // SDK agentId where error fired
+
+  get retriable(): boolean;          // alias for isRetryable
+  get retryAfterMs(): number | undefined;  // metadata.retryAfter * 1000
+  get providerError(): unknown;       // metadata.raw alias
+
+  readonly metadata?: ErrorMetadata;  // full structured context
+}
+```
+
+## `retryAfterMs` semantics
+
+Returns milliseconds derived from `metadata.retryAfter` (seconds). Use with `setTimeout`:
+
+```ts
+if (err.retryAfterMs !== undefined) {
+  setTimeout(retry, err.retryAfterMs);
+}
+```
+
+**EC-11: Use `=== undefined` check, NOT truthy check.** `retryAfterMs === 0` is a legitimate value (provider asked for immediate retry — `setTimeout(0)` is valid).
+
+## Anti-leak invariant
+
+`AgentRunError.message` NEVER contains `providerError` content (raw response body may carry sensitive data — internal field names, log fragments, etc). To inspect the raw body:
+
+```ts
+console.log(err.providerError);    // alias for err.metadata?.raw
+console.log(err.metadata?.raw);    // same value (already redacted via D68)
+```
+
+The redacted body is safe to log — `redactSecrets` strips known secret patterns (API keys, JWT, Authorization headers).
+
+## See also
+
+- `internal/error-mappers/` — per-provider mapping implementations
+- ADRs D311-D314, D65-D68 (the broader error system)