archal 0.9.19 → 0.10.0

package/dist/sdk/index.d.cts

@@ -0,0 +1,626 @@
+interface FieldChange {
+    field: string;
+    before: unknown;
+    after: unknown;
+}
+interface EntityChange {
+    entityId: number;
+    entityType: string;
+    fields: FieldChange[];
+}
+interface StateDiff {
+    added: Record<string, unknown[]>;
+    modified: Record<string, unknown[]>;
+    removed: Record<string, (string | number)[]>;
+    fieldChanges?: EntityChange[];
+}
+interface CloneEvent {
+    type: string;
+    payload: unknown;
+    timestamp: string;
+    sourceRule?: string;
+}
+
+interface TraceLink {
+    traceId: string;
+    spanId: string;
+    type: 'retry' | 'read_after_write' | 'write_after_write' | 'fan_out' | 'async' | 'manual';
+}
+interface RetryPolicy {
+    maxAttempts?: number;
+    backoffMs?: number;
+    strategy?: 'fixed' | 'linear' | 'exponential' | 'unknown';
+}
+interface OutboundHttpTrace {
+    method?: string;
+    scheme?: 'http' | 'https';
+    host?: string;
+    route?: string;
+    path?: string;
+    statusCode?: number;
+    latencyMs?: number;
+    requestBytes?: number;
+    responseBytes?: number;
+}
+interface CausalLink {
+    traceId: string;
+    spanId: string;
+    type: 'retry' | 'read_after_write' | 'write_after_write';
+}
+type TraceErrorKind = 'validation_error' | 'not_found' | 'authentication_error' | 'permission_denied' | 'rate_limited' | 'timeout' | 'network_error' | 'server_error' | 'agent_error' | 'unknown';
+interface TraceError {
+    code: string;
+    message: string;
+    kind?: TraceErrorKind;
+    class?: string;
+    normalizedCode?: string;
+    name?: string;
+    statusCode?: number;
+    retryable?: boolean;
+    retryAfterSeconds?: number;
+    stack?: string;
+    stackHash?: string;
+    causeChain?: Array<{
+        class?: string;
+        code?: string;
+        message?: string;
+    }>;
+    retryPolicy?: RetryPolicy;
+    details?: unknown;
+}
+interface TraceEntry {
+    id: string;
+    toolName: string;
+    traceId?: string;
+    spanId?: string;
+    parentSpanId?: string | null;
+    links?: TraceLink[];
+    runIndex?: number;
+    twinName?: string;
+    sessionId?: string;
+    input: Record<string, unknown>;
+    output: unknown;
+    startTimestamp?: string;
+    endTimestamp?: string;
+    queuedAt?: string;
+    dequeuedAt?: string;
+    startedAt?: string;
+    endedAt?: string;
+    uploadedAt?: string;
+    timestamp: string;
+    durationMs: number;
+    stateMutations: StateDiff | null;
+    error: TraceError | null;
+    sequenceIndex?: number;
+    requestBytes?: number;
+    responseBytes?: number;
+    entitiesRead?: Array<{
+        collection: string;
+        id: string | number;
+    }>;
+    entitiesWritten?: Array<{
+        collection: string;
+        id: string | number;
+        action: 'create' | 'update' | 'delete';
+    }>;
+    isRetry?: boolean;
+    retriedFromId?: string;
+    retryCount?: number;
+    rateLimitRemaining?: number;
+    triggeredRateLimit?: boolean;
+    errorCategory?: 'agent_error' | 'validation_error' | 'not_found' | 'permission_denied' | 'rate_limited' | 'server_error' | 'none';
+    causalLinks?: string[];
+    causalLinkDetails?: CausalLink[];
+    outboundHttp?: OutboundHttpTrace;
+    agentInternals?: {
+        providerRequestId?: string;
+        toolChoiceRationaleSummary?: string;
+        tokenUsage?: {
+            input?: number;
+            output?: number;
+            total?: number;
+        };
+        costUsd?: number;
+        latencyMs?: number;
+    };
+}
+
+interface SuccessCriterion {
+    id: string;
+    description: string;
+    type: 'deterministic' | 'probabilistic';
+    critical?: boolean;
+    structuredAssertion?: Record<string, unknown>;
+    judge?: JudgeCriterionConfig;
+}
+interface JudgeCriterionRubricExample {
+    evidence: string;
+    verdict: 'pass' | 'fail' | 'partial' | 'inconclusive';
+    rationale: string;
+}
+interface JudgeCriterionRubric {
+    instructions?: string;
+    pass?: string;
+    partial?: string;
+    fail?: string;
+    inconclusive?: string;
+    examples?: JudgeCriterionRubricExample[];
+}
+interface JudgeCriterionConfig {
+    rubric?: JudgeCriterionRubric;
+    promptVersion?: string;
+}
+/**
+ * Reasons why an LLM-judged criterion could not be evaluated and must be
+ * surfaced as `status: 'error'` (i.e. excluded from the weighted satisfaction
+ * score and called out in the report). See #2507.
+ *
+ * `rate_limited`      — Upstream 429 (Gemini RPM/TPM blown, or direct provider
+ *                       rate limit). Keep retrying a later run.
+ * `upstream_5xx`      — 5xx from upstream including proxy 502 that wraps
+ *                       "Gemini API error: 429 Resource exhausted" (our proxy
+ *                       translates 429 to 502 on the wire).
+ * `auth_error`        — 401/403 from upstream (misconfigured or revoked key).
+ * `provider_error`    — Generic/unknown provider failure (kept for back-compat,
+ *                       no longer silently degrades to 0% score).
+ */
+type LlmEvalErrorReason = 'rate_limited' | 'upstream_5xx' | 'auth_error' | 'provider_error';
+type JudgeFailureClass = 'none' | 'agent_model_behavior' | 'clone_fidelity_gap' | 'routing_sandbox_gap' | 'evaluator_scenario_gap' | 'infrastructure_gap' | 'missing_evidence' | 'unknown';
+type JudgeEvidenceKind = 'span' | 'state_before' | 'state_after' | 'state_diff' | 'agent_response' | 'agent_trace' | 'clone_event';
+interface JudgeEvidenceRef {
+    kind: JudgeEvidenceKind;
+    id: string;
+    note?: string;
+}
+interface JudgeBenchmarkRecommendation {
+    shouldPromote: boolean;
+    benchmarkName: string;
+    scenarioFamily: string;
+    tags: string[];
+    rationale: string;
+    similarScenarioQuery: string[];
+}
+interface JudgeRegressionRisk {
+    status: 'not_assessed' | 'no_regression_evidence' | 'regression_detected' | 'unknown';
+    explanation: string;
+    evidenceRefs: JudgeEvidenceRef[];
+}
+interface JudgeDiagnosis {
+    /**
+     * Where this diagnosis came from. `model` means the LLM supplied the structured
+     * diagnosis. `fallback` means Archal filled defaults for a legacy/small-shape
+     * response. `infrastructure` means Archal generated metadata for judge
+     * unavailability or an evaluator-side skip.
+     */
+    source?: 'model' | 'fallback' | 'infrastructure';
+    whatBroke: string;
+    howOften: string;
+    affectedUsers: string[];
+    why: string;
+    failureClass: JudgeFailureClass;
+    evidenceRefs: JudgeEvidenceRef[];
+    missingEvidence: string[];
+    benchmarkRecommendation: JudgeBenchmarkRecommendation;
+    suggestedFix: string;
+    regressionRisk: JudgeRegressionRisk;
+}
+interface EvaluationResult {
+    criterionId: string;
+    /**
+     * `error` means the criterion could NOT be evaluated due to upstream LLM
+     * failure (429/5xx/auth). It must not be treated as a pass or a fail for the
+     * satisfaction calculation — the final report should surface "N criteria
+     * unevaluated, results incomplete" rather than silently under-reporting.
+     *
+     * `skipped` means evaluation was deliberately short-circuited as a cost
+     * optimization — currently only when every deterministic criterion already
+     * failed, so spending judge calls on the probabilistic criteria is wasteful.
+     * Like `error`, it is EXCLUDED from the weighted satisfaction average (it is
+     * not evidence of failure). Unlike `error`, it does NOT mark the run
+     * incomplete/untrustworthy: the deterministic verdict is conclusive, so the
+     * score stands. See `calculateOverallScore` and `score-trust.ts`.
+     *
+     * `inconclusive` means the judge ran but evidence could not prove a verdict.
+     */
+    status: 'pass' | 'fail' | 'partial' | 'inconclusive' | 'error' | 'skipped';
+    confidence: number;
+    explanation: string;
+    fallbackRecommended?: boolean;
+    fallbackFailureReason?: 'missing_credentials' | 'rate_limited' | 'context_too_large' | 'provider_error';
+    /** Structured reason when `status === 'error'`. Surfaced in the report summary. */
+    errorReason?: LlmEvalErrorReason;
+    /** Continuous [0,1] grade for partial credit or shaped deterministic rewards. */
+    partialScore?: number;
+    /**
+     * Structured LLM-judge diagnosis. This is explanatory metadata only:
+     * satisfaction scoring still depends solely on `status` and `partialScore`.
+     */
+    diagnosis?: JudgeDiagnosis;
+}
+
+/**
+ * Inline seed object: entity collections keyed by table, the exact JSON shape
+ * the clone `/state` PUT accepts (the file-seed loader's shape without the
+ * file).
+ */
+type InlineSeed = Record<string, unknown[]>;
+/**
+ * A named catalog seed (resolved server-side at provision time) or an inline
+ * state object (pushed client-side after provision). File refs arrive with
+ * eval-file support.
+ */
+type SeedRef = string | InlineSeed;
+interface SdkAuth {
+    /** Bearer token. Defaults to the ARCHAL_TOKEN environment variable. */
+    token?: string;
+}
+/** Command line for a spawned agent process. */
+interface AgentCommand {
+    command: string;
+    args?: string[];
+    cwd?: string;
+    env?: Record<string, string>;
+}
+interface ExecOptions {
+    /** The task handed to the agent as AGENT_TASK. */
+    task: string;
+    /** Agent timeout in seconds. Defaults to 300. */
+    timeout?: number;
+    /** Advertised to the agent as AGENT_MODEL. */
+    model?: string;
+}
+/**
+ * How a spawned agent process ended. `exec()` reports agent failures here
+ * instead of throwing so the caller can still collect and judge the
+ * `outcome()`; it throws only when the process cannot be started at all.
+ */
+interface ExecResult {
+    exitCode: number | null;
+    timedOut: boolean;
+    durationMs: number;
+    /** The agent's final text, when extractable from its output. */
+    response?: string;
+}
+interface ClonesOptions {
+    /** Per-service seed, keyed by service name. */
+    seed?: Record<string, SeedRef>;
+    auth?: SdkAuth;
+    /** Session time-to-live in seconds. Defaults to the hosted-session default. */
+    ttl?: number;
+}
+/**
+ * Everything observable about the session since the last `reset()` (or since
+ * `clones()` when never reset). `trace`, `stateAfter`, and `events` are read
+ * back from the clones; `stateBefore` and `stateDiff` come from the baseline
+ * the Session snapshotted client-side at `clones()`/`reset()` time.
+ */
+interface Outcome {
+    /** Every tool call, recorded by the clones (not agent self-report). */
+    trace: TraceEntry[];
+    /** Combined clone state keyed by service name. */
+    stateBefore: Record<string, unknown>;
+    stateAfter: Record<string, unknown>;
+    stateDiff: StateDiff;
+    /** The agent's final text. Absent unless the caller supplies it. */
+    response?: string;
+    /** Clone event logs keyed by service name. Empty for clones without an events endpoint. */
+    events: Record<string, CloneEvent[]>;
+}
+/**
+ * A provisioned set of clones behind one hosted session (spec 12).
+ *
+ * Semantics:
+ * - `reset()` is the only run boundary. Clone trace buffers are cumulative
+ *   and clear only on a state load, so two agent executions with no
+ *   `reset()` between them produce one combined `outcome()`.
+ * - The session is billed wall-clock until `stop()`. Use
+ *   `await using session = await clones(...)` so it never leaks; the library
+ *   installs no process-global handlers.
+ * - The first `stop()` call fences every other operation permanently, even
+ *   if the hosted delete fails (retry `stop()` in that case). An operation
+ *   already in flight when `stop()` is called is not cancelled; it fails
+ *   with clone HTTP errors once the session is gone.
+ * - `exec()` is sequential per session: one session means one shared trace
+ *   and state buffer per clone, so a concurrent `exec()` throws. Parallel
+ *   runs are parallel sessions. In-process callers skip `exec()` entirely
+ *   and drive their own agent against `url()`/`env()`.
+ */
+interface Session {
+    id: string;
+    /** Clone REST base URL for SDK/HTTP clients (Octokit baseUrl, etc.). */
+    url(service: string): string;
+    /** Clone routing env for spawning an agent process: AGENT_CLONE_URLS, ARCHAL_CLONE_NAMES, per-service URL vars, route auth. */
+    env(): Record<string, string>;
+    /** Spawn an agent with the scrubbed env contract and wait for it to finish. */
+    exec(agent: AgentCommand, opts: ExecOptions): Promise<ExecResult>;
+    outcome(extras?: {
+        response?: string;
+    }): Promise<Outcome>;
+    /** Re-seed every clone and clear traces, starting the next run. */
+    reset(): Promise<void>;
+    /** End the session (idempotent). Billing stops here. */
+    stop(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+}
+
+declare function clones(services: string[], options?: ClonesOptions): Promise<Session>;
+
+declare const ASSERTION_TYPES: readonly ["exact_count", "min_count", "max_count", "exists", "not_exists", "state_check", "no_errors", "no_canary_leaks", "trace_count", "trace_contains", "output_contains", "content_check"];
+interface ParsedAssertion {
+    type: (typeof ASSERTION_TYPES)[number];
+    subject: string;
+    value?: number;
+    predicate?: string;
+    labelFilter?: string;
+    contentPatterns?: string[];
+    /** For count assertions: case-insensitive title/summary substring filter. */
+    titleIncludes?: string;
+    /** For trace_contains: exact tool names accepted for structured service actions. */
+    toolNames?: string[];
+    requireAllContentPatterns?: boolean;
+    /** For content_check: restrict matching to these text-bearing fields. */
+    contentFields?: string[];
+    negated?: boolean;
+    /** For count assertions: restrict matching entities to a channel. */
+    channelFilter?: string;
+    /** For count assertions: the target clone/service scope */
+    targetService?: string;
+    /** For state_check: whether ALL items must match (true for "remain" assertions) */
+    allMustMatch?: boolean;
+    /** For entity-specific assertions: numeric ID extracted from "Issue #42", "PR #7", etc. */
+    entityId?: number;
+    /** For entity-specific assertions with natural keys, e.g. Jira "CHG-5". */
+    entityKey?: string;
+    /**
+     * For `exists` assertions: when true, the criterion is only satisfied if the
+     * subject was newly created during the run (i.e., present in `after` but not
+     * in `before`). Set by the parser when the natural-language phrasing implies
+     * agent action (e.g., "was created", "was posted", "was sent", "was added",
+     * "was opened", "has been created"). When false/undefined, evaluator performs
+     * a pure presence check against `after` only -- appropriate for criteria like
+     * "X exists" where the agent's job is to verify existing state.
+     */
+    requireDiff?: boolean;
+}
+
+declare const CRITERION: unique symbol;
+interface CriterionOptions {
+    critical?: boolean;
+    rubric?: JudgeCriterionConfig['rubric'];
+    promptVersion?: string;
+}
+/**
+ * An SDK success criterion. Opaque — construct with {@link criterion} (an
+ * LLM-judged natural-language statement), {@link check} (structured
+ * deterministic assertion), or service/entity builders.
+ */
+interface Criterion {
+    readonly [CRITERION]: true;
+    /** @internal consumed by judge() when building SuccessCriterion entries. */
+    readonly description: string;
+    /** @internal consumed by judge() when building SuccessCriterion entries. */
+    readonly type: SuccessCriterion['type'];
+    /** @internal consumed by judge() when building SuccessCriterion entries. */
+    readonly critical?: boolean;
+    /** @internal consumed by judge() when building SuccessCriterion entries. */
+    readonly structuredAssertion?: ParsedAssertion;
+    /** @internal consumed by judge() when building SuccessCriterion entries. */
+    readonly judge?: JudgeCriterionConfig;
+}
+/**
+ * An LLM-judged natural-language statement about the outcome.
+ */
+declare function criterion(text: string, options?: CriterionOptions): Criterion;
+interface CountOptions {
+    count?: number;
+    atLeast?: number;
+    atMost?: number;
+    titleIncludes?: string;
+    label?: string;
+    channel?: string;
+}
+type CheckDefinition = ParsedAssertion & {
+    critical?: boolean;
+    description?: string;
+    judge?: JudgeCriterionConfig;
+};
+interface EntityCriteriaBuilder {
+    exists(options?: CriterionOptions): Criterion;
+    notExists(options?: CriterionOptions): Criterion;
+    created(count?: CountOptions, options?: CriterionOptions): Criterion;
+    updated(count?: CountOptions, options?: CriterionOptions): Criterion;
+    deleted(count?: CountOptions, options?: CriterionOptions): Criterion;
+}
+declare function check(definition: CheckDefinition, options?: CriterionOptions): Criterion;
+declare function entity(service: string, subject: string): EntityCriteriaBuilder;
+
+interface JudgeOptions {
+    /** Expected-behavior context handed to the LLM judge. */
+    expected?: string;
+    /** Judge model. Defaults to the shared canonical judge model. */
+    model?: string;
+    /**
+     * Bearer token for managed judge routing. Defaults to the ARCHAL_TOKEN
+     * environment variable. Without a token the LLM judge cannot run and every
+     * criterion reports status 'error' with a suggested fix.
+     */
+    auth?: {
+        token?: string;
+    };
+}
+/** Result of judging one {@link Outcome} against a set of criteria. */
+interface Judgment {
+    /** Weighted 0-100 score for this single outcome. */
+    score: number;
+    /**
+     * True when `score` >= 80 and every criterion evaluated conclusively
+     * (none reported 'error', 'inconclusive', or 'skipped').
+     */
+    passed: boolean;
+    /** Per-criterion status, evidence, and confidence. */
+    criteria: EvaluationResult[];
+}
+/**
+ * Grade an outcome: `evaluateRun` from `@archal/evaluator`, nothing more.
+ * No execution, no upload, no markdown. Safe to call concurrently — the
+ * resolved token is scoped to the call, not stored globally.
+ */
+declare function judge(outcome: Outcome, criteria: Criterion[], options?: JudgeOptions): Promise<Judgment>;
+
+interface AgentMetrics {
+    inputTokens: number;
+    outputTokens: number;
+    llmCallCount: number;
+    provider?: string;
+    model?: string;
+}
+interface ReportAgentMetricsOptions {
+    metricsFile?: string;
+}
+/**
+ * Write agent-side token usage for `archal run`.
+ *
+ * Archal sets AGENT_METRICS_FILE when it launches a harness. Calling this
+ * helper before the harness exits lets the dashboard show agent input/output
+ * token counts without exposing the artifact wire format to user code.
+ */
+declare function reportAgentMetrics(metrics: AgentMetrics, options?: ReportAgentMetricsOptions): Promise<boolean>;
+
+interface ScenarioDefinition {
+    name: string;
+    clones: string[];
+    task: string;
+    expected?: string;
+    agent: AgentCommand;
+    criteria?: Criterion[];
+    /**
+     * Agent execution timeout in seconds. Run options take precedence; when
+     * both are absent the SDK session default applies. Workspace-derived
+     * scenarios carry the runner's timeout here so both paths score alike.
+     */
+    timeout?: number;
+}
+interface ScenarioRunOptions extends Pick<ExecOptions, 'model' | 'timeout'> {
+    runs?: number;
+    auth?: SdkAuth;
+}
+interface ScenarioAttemptResult {
+    index: number;
+    exec: ExecResult;
+    outcome: Outcome;
+    judgment?: Judgment;
+    /**
+     * False when the judgment (or exec fallback) failed, and always false when
+     * `toolCallCount` is 0: an agent that never touched a clone did not
+     * exercise the scenario, whatever the judge scored — the seeded state can
+     * satisfy criteria on its own. `satisfaction` stays report-only.
+     */
+    passed: boolean;
+    satisfaction: number;
+    /**
+     * Agent tool calls recorded by the clones for this run, with synthetic
+     * proxy/canary trace entries excluded.
+     */
+    toolCallCount: number;
+}
+interface ScenarioResult {
+    name: string;
+    passed: boolean;
+    satisfaction: number;
+    runs: ScenarioAttemptResult[];
+}
+interface ScenarioHandle {
+    definition: ScenarioDefinition;
+    run(options?: ScenarioRunOptions): Promise<ScenarioResult>;
+}
+declare function scenario(definition: ScenarioDefinition): ScenarioHandle;
+
+interface BenchmarkDefinition {
+    name: string;
+    description?: string;
+    scenarios: Array<ScenarioDefinition | ScenarioHandle>;
+}
+interface BenchmarkRunOptions extends ScenarioRunOptions {
+    /**
+     * Correlates every scenario submission of one run() call into a single
+     * `benchmark_runs` row. Defaults to a fresh UUID per run() call.
+     */
+    invocationId?: string;
+    /** Workspace to submit results to. Defaults to ARCHAL_WORKSPACE_ID. */
+    workspaceId?: string;
+}
+interface BenchmarkScenarioResult {
+    definition: ScenarioDefinition;
+    result: ScenarioResult;
+}
+interface BenchmarkResult {
+    name: string;
+    slug: string;
+    invocationId: string;
+    passed: boolean;
+    /** Mean of scenario satisfactions, 0-100. */
+    satisfaction: number;
+    scenarios: BenchmarkScenarioResult[];
+    /** Cloud run id, present when results were submitted. */
+    runId?: string;
+    /** Absolute dashboard URL for the cloud run, present when submitted. */
+    dashboardUrl?: string;
+}
+interface BenchmarkHandle {
+    definition: BenchmarkDefinition;
+    run(options?: BenchmarkRunOptions): Promise<BenchmarkResult>;
+}
+interface WorkspaceBenchmarkOptions {
+    /** Agent command the caller supplies for every member scenario. */
+    agent: AgentCommand;
+    auth?: SdkAuth;
+    /** Workspace to load the benchmark from. Defaults to ARCHAL_WORKSPACE_ID. */
+    workspaceId?: string;
+}
+interface PublishOptions {
+    auth?: SdkAuth;
+    /** Workspace to submit results to. Defaults to ARCHAL_WORKSPACE_ID. */
+    workspaceId?: string;
+    /** See {@link BenchmarkRunOptions.invocationId}. */
+    invocationId?: string;
+}
+interface PublishResult {
+    runId: string;
+    url: string;
+}
+/**
+ * A benchmark is a named group of scenarios. Two forms, one noun:
+ *
+ * - `benchmark({ name, scenarios })` groups code-first scenarios and returns
+ *   a handle synchronously.
+ * - `benchmark('payment-flows', { agent })` loads a dashboard-authored
+ *   benchmark by slug, converts each member scenario's markdown into a
+ *   ScenarioDefinition, and resolves to the same kind of handle (async — it
+ *   fetches from the workspace). Requires a token (auth.token or
+ *   ARCHAL_TOKEN) and a workspace id (option or ARCHAL_WORKSPACE_ID).
+ *
+ * Either way, `handle.run()` runs each scenario (a scenario run is one agent
+ * execution plus a judge pass) and, when credentials are present, streams
+ * results to the cloud under a single benchmark run.
+ */
+declare function benchmark(definition: BenchmarkDefinition): BenchmarkHandle;
+declare function benchmark(workspaceSlug: string, options: WorkspaceBenchmarkOptions): Promise<BenchmarkHandle>;
+/**
+ * Submit one already-executed scenario result to the cloud as a
+ * single-scenario benchmark run. Unlike benchmark().run(), missing
+ * credentials are an error here — publishing IS the upload.
+ */
+declare function publish(result: ScenarioResult, definition: ScenarioDefinition, options?: PublishOptions): Promise<PublishResult>;
+
+/**
+ * Code-first SDK primitives (spec 12: provision / execute / collect / judge).
+ *
+ * This entry is the extraction target for the SDK-first surface: it must stay
+ * light (no pg, no sandbox-runtime, no fidelity in the static graph) and
+ * must not touch the ambient credentials store — callers pass tokens
+ * explicitly or via ARCHAL_TOKEN.
+ */
+
+declare const version: string;
+
+export { type AgentCommand, type AgentMetrics, type BenchmarkDefinition, type BenchmarkHandle, type BenchmarkResult, type BenchmarkRunOptions, type BenchmarkScenarioResult, type CheckDefinition, type ClonesOptions, type CountOptions, type Criterion, type CriterionOptions, type EntityCriteriaBuilder, type ExecOptions, type ExecResult, type InlineSeed, type JudgeOptions, type Judgment, type Outcome, type PublishOptions, type PublishResult, type ReportAgentMetricsOptions, type ScenarioAttemptResult, type ScenarioDefinition, type ScenarioHandle, type ScenarioResult, type ScenarioRunOptions, type SdkAuth, type SeedRef, type Session, type WorkspaceBenchmarkOptions, benchmark, check, clones, criterion, entity, judge, publish, reportAgentMetrics, scenario, version };