archal 0.9.18 → 0.9.20

package/dist/scenarios.d.cts

@@ -0,0 +1,562 @@
+interface FieldChange {
+    field: string;
+    before: unknown;
+    after: unknown;
+}
+interface EntityChange {
+    entityId: number;
+    entityType: string;
+    fields: FieldChange[];
+}
+interface StateDiff {
+    added: Record<string, unknown[]>;
+    modified: Record<string, unknown[]>;
+    removed: Record<string, (string | number)[]>;
+    fieldChanges?: EntityChange[];
+}
+interface CloneEvent {
+    type: string;
+    payload: unknown;
+    timestamp: string;
+    sourceRule?: string;
+}
+
+interface TraceLink {
+    traceId: string;
+    spanId: string;
+    type: 'retry' | 'read_after_write' | 'write_after_write' | 'fan_out' | 'async' | 'manual';
+}
+interface RetryPolicy {
+    maxAttempts?: number;
+    backoffMs?: number;
+    strategy?: 'fixed' | 'linear' | 'exponential' | 'unknown';
+}
+interface OutboundHttpTrace {
+    method?: string;
+    scheme?: 'http' | 'https';
+    host?: string;
+    route?: string;
+    path?: string;
+    statusCode?: number;
+    latencyMs?: number;
+    requestBytes?: number;
+    responseBytes?: number;
+}
+interface CausalLink {
+    traceId: string;
+    spanId: string;
+    type: 'retry' | 'read_after_write' | 'write_after_write';
+}
+type TraceErrorKind = 'validation_error' | 'not_found' | 'authentication_error' | 'permission_denied' | 'rate_limited' | 'timeout' | 'network_error' | 'server_error' | 'agent_error' | 'unknown';
+interface TraceError {
+    code: string;
+    message: string;
+    kind?: TraceErrorKind;
+    class?: string;
+    normalizedCode?: string;
+    name?: string;
+    statusCode?: number;
+    retryable?: boolean;
+    retryAfterSeconds?: number;
+    stack?: string;
+    stackHash?: string;
+    causeChain?: Array<{
+        class?: string;
+        code?: string;
+        message?: string;
+    }>;
+    retryPolicy?: RetryPolicy;
+    details?: unknown;
+}
+interface TraceEntry {
+    id: string;
+    toolName: string;
+    traceId?: string;
+    spanId?: string;
+    parentSpanId?: string | null;
+    links?: TraceLink[];
+    runIndex?: number;
+    twinName?: string;
+    sessionId?: string;
+    input: Record<string, unknown>;
+    output: unknown;
+    startTimestamp?: string;
+    endTimestamp?: string;
+    queuedAt?: string;
+    dequeuedAt?: string;
+    startedAt?: string;
+    endedAt?: string;
+    uploadedAt?: string;
+    timestamp: string;
+    durationMs: number;
+    stateMutations: StateDiff | null;
+    error: TraceError | null;
+    sequenceIndex?: number;
+    requestBytes?: number;
+    responseBytes?: number;
+    entitiesRead?: Array<{
+        collection: string;
+        id: string | number;
+    }>;
+    entitiesWritten?: Array<{
+        collection: string;
+        id: string | number;
+        action: 'create' | 'update' | 'delete';
+    }>;
+    isRetry?: boolean;
+    retriedFromId?: string;
+    retryCount?: number;
+    rateLimitRemaining?: number;
+    triggeredRateLimit?: boolean;
+    errorCategory?: 'agent_error' | 'validation_error' | 'not_found' | 'permission_denied' | 'rate_limited' | 'server_error' | 'none';
+    causalLinks?: string[];
+    causalLinkDetails?: CausalLink[];
+    outboundHttp?: OutboundHttpTrace;
+    agentInternals?: {
+        providerRequestId?: string;
+        toolChoiceRationaleSummary?: string;
+        tokenUsage?: {
+            input?: number;
+            output?: number;
+            total?: number;
+        };
+        costUsd?: number;
+        latencyMs?: number;
+    };
+}
+
+/**
+ * Reasons why an LLM-judged criterion could not be evaluated and must be
+ * surfaced as `status: 'error'` (i.e. excluded from the weighted satisfaction
+ * score and called out in the report). See #2507.
+ *
+ * `rate_limited`      — Upstream 429 (Gemini RPM/TPM blown, or direct provider
+ *                       rate limit). Keep retrying a later run.
+ * `upstream_5xx`      — 5xx from upstream including proxy 502 that wraps
+ *                       "Gemini API error: 429 Resource exhausted" (our proxy
+ *                       translates 429 to 502 on the wire).
+ * `auth_error`        — 401/403 from upstream (misconfigured or revoked key).
+ * `provider_error`    — Generic/unknown provider failure (kept for back-compat,
+ *                       no longer silently degrades to 0% score).
+ */
+type LlmEvalErrorReason = 'rate_limited' | 'upstream_5xx' | 'auth_error' | 'provider_error';
+interface EvaluationResult {
+    criterionId: string;
+    /**
+     * `error` means the criterion could NOT be evaluated due to upstream LLM
+     * failure (429/5xx/auth). It must not be treated as a pass or a fail for the
+     * satisfaction calculation — the final report should surface "N criteria
+     * unevaluated, results incomplete" rather than silently under-reporting.
+     *
+     * `skipped` means evaluation was deliberately short-circuited as a cost
+     * optimization — currently only when every deterministic criterion already
+     * failed, so spending judge calls on the probabilistic criteria is wasteful.
+     * Like `error`, it is EXCLUDED from the weighted satisfaction average (it is
+     * not evidence of failure). Unlike `error`, it does NOT mark the run
+     * incomplete/untrustworthy: the deterministic verdict is conclusive, so the
+     * score stands. See `calculateOverallScore` and `score-trust.ts`.
+     */
+    status: 'pass' | 'fail' | 'partial' | 'error' | 'skipped';
+    confidence: number;
+    explanation: string;
+    fallbackRecommended?: boolean;
+    fallbackFailureReason?: 'missing_credentials' | 'rate_limited' | 'context_too_large' | 'provider_error';
+    /** Structured reason when `status === 'error'`. Surfaced in the report summary. */
+    errorReason?: LlmEvalErrorReason;
+    /**
+     * Continuous [0,1] grade for criteria that admit partial credit (currently
+     * the count assertions: exact/min/max_count). 1 means fully satisfied; a
+     * fraction expresses how close the run came (e.g. 3 of 5 expected = 0.6).
+     *
+     * This is INERT metadata for the satisfaction calculation: `getCriterionScore`
+     * and `calculateOverallScore` switch on `status` only and never read this
+     * field, so `overallScore` is byte-identical whether or not it is present.
+     * Only the RL reward path (`shapedScore` / `denseReward`, gated behind the
+     * `--rl-reward` flag) consumes it, to give an RL agent a gradient instead of
+     * a binary pass/fail. Always computed on count results because it costs
+     * nothing and gating it would couple the pure assertion evaluators to run
+     * config.
+     */
+    partialScore?: number;
+}
+interface TokenUsageSummary {
+    inputTokens: number;
+    outputTokens: number;
+    llmCallCount?: number;
+    provider?: string;
+    model?: string;
+}
+type AgentTokenUsageStatus = 'reported' | 'not_reported_by_harness';
+interface ScoreTrustSummary {
+    status: 'complete' | 'incomplete';
+    evaluatorErrorCount: number;
+    unscoredRunFailureCount: number;
+    toolErrorCount: number;
+    scoredRunCount: number;
+    totalRunCount: number;
+}
+type CapabilityMissSubkind = 'missing_handler' | 'unknown_tool' | 'unimplemented_route' | 'unsupported_method' | 'shim_target_missing';
+type CapabilityMissSurface = 'mcp' | 'tools_call' | 'rest' | 'graphql_shim' | 'alias_shim';
+type CapabilityMissSeverity = 'high' | 'low';
+type CapabilityMissAlertChannel = 'none' | 'derived';
+type CapabilityMissDetails = {
+    subkind: 'unknown_tool';
+    requestedTool: string;
+} | {
+    subkind: 'missing_handler';
+    resolvedTool: string;
+} | {
+    subkind: 'unsupported_method';
+    method: string;
+    route: string;
+} | {
+    subkind: 'unimplemented_route';
+    method: string;
+    route: string;
+} | {
+    subkind: 'shim_target_missing';
+    shim: string;
+    resolvedTool: string;
+};
+interface CapabilityMissEvent {
+    version: 1;
+    event: 'capability.miss';
+    occurredAt: string;
+    run: {
+        sessionId?: string;
+        runId?: string;
+        scenarioId?: string;
+    };
+    miss: {
+        subkind: CapabilityMissSubkind;
+        surface: CapabilityMissSurface;
+        severity: CapabilityMissSeverity;
+        fingerprint: string;
+    };
+    target: {
+        twin: string;
+        toolName?: string;
+        method?: string;
+        route?: string;
+    };
+    details: CapabilityMissDetails;
+    handling: {
+        abortedRun: boolean;
+        alerted: boolean;
+        alertChannel: CapabilityMissAlertChannel;
+    };
+}
+interface CapabilityMissShadowReportRow {
+    fingerprint: string;
+    twin: string;
+    surface: CapabilityMissSurface;
+    subkind: CapabilityMissSubkind;
+    severity: CapabilityMissSeverity;
+    runsAffected: number;
+    occurrenceCount: number;
+    firstSeenAt: string;
+    lastSeenAt: string;
+    preventableClass: 'tools_surface_parity' | 'rest_surface_parity' | 'shim_wiring' | 'runtime_only_unknown_tool' | 'not_preventable_yet';
+}
+type RunOutcome = 'completed' | 'degraded' | 'failed_agent' | 'no_tool_calls' | 'insufficient_action' | 'inconclusive_infrastructure' | 'inconclusive_seed' | 'twin_auth_failed';
+interface AgentTraceStep {
+    step: number;
+    thinking: string | null;
+    text: string | null;
+    toolCalls: Array<{
+        name: string;
+        arguments: unknown;
+    }>;
+    durationMs: number;
+}
+type FailureReasonCode = 'capability_miss' | 'agent_no_tool_calls' | 'managed_llm_unavailable' | 'twin_auth_failed' | 'evaluator_unavailable' | 'agent_process_failed' | 'unknown';
+interface RunResult {
+    runIndex: number;
+    evaluations: EvaluationResult[];
+    overallScore: number;
+    trace: TraceEntry[];
+    durationMs: number;
+    error?: string;
+    outcome?: RunOutcome;
+    failureReasonCode?: FailureReasonCode;
+    stateBefore?: Record<string, unknown>;
+    stateAfter?: Record<string, unknown>;
+    stateDiff?: StateDiff;
+    agentLog?: string;
+    agentTrace?: AgentTraceStep[];
+    tokenUsage?: TokenUsageSummary;
+    judgeTokenUsage?: TokenUsageSummary;
+    events?: Record<string, CloneEvent[]>;
+    capabilityMisses?: CapabilityMissEvent[];
+    agentResponseText?: string;
+    /**
+     * Terminal shaped reward in [0,100]: the same weighted criterion average as
+     * `overallScore` but WITHOUT the critical-failure short-circuit (and with
+     * partial credit from `EvaluationResult.partialScore`). It gives an RL agent
+     * a gradient even when a critical gate trips `overallScore` to 0.
+     * `shapedScore >= overallScore` always. Only populated when the run was
+     * evaluated with the `--rl-reward` flag; `overallScore` is unaffected.
+     */
+    shapedScore?: number;
+    /**
+     * Step-indexed reward in [0,1], one scalar per trace entry, produced by
+     * re-running the DETERMINISTIC criteria against successive trace prefixes
+     * (trace[0..1], trace[0..2], ...). `denseReward[i]` is the graded weighted
+     * average after step `i+1`. Deterministic-only (never invokes the LLM judge).
+     * Only populated under the `--rl-reward` flag. NOTE: the state snapshot is
+     * terminal (held constant across prefixes), so the gradient comes from
+     * trace-sensitive criteria (no_errors, trace_contains, trace counts);
+     * pure state-assertion criteria contribute their terminal value at every
+     * step. Per-step state reconstruction is a documented follow-up.
+     */
+    denseReward?: number[];
+}
+interface ScenarioReport {
+    scenarioTitle: string;
+    satisfactionScore: number;
+    scoreTrust?: ScoreTrustSummary;
+    criterionDescriptions?: Record<string, string>;
+    criterionTypes?: Record<string, 'deterministic' | 'probabilistic'>;
+    /**
+     * Canonical list of clones exercised by this scenario. Prefer this field
+     * over `twinNames` for new code; both fields are populated during the
+     * twin→clone rename deprecation window.
+     */
+    cloneNames?: string[];
+    /** Legacy alias for `cloneNames`. Will be removed once external consumers migrate. */
+    twinNames?: string[];
+    runs: RunResult[];
+    summary: string;
+    failureAnalysis?: string;
+    agentTokenUsageStatus?: AgentTokenUsageStatus;
+    agentTokenUsage?: TokenUsageSummary;
+    judgeTokenUsage?: TokenUsageSummary;
+    /**
+     * User-supplied evaluator model name. Managed evaluator model names are
+     * intentionally omitted from public reports.
+     */
+    evaluatorModel?: string;
+    judgeMode?: 'managed' | 'user_supplied';
+    capabilityMissShadowReport?: CapabilityMissShadowReportRow[];
+    rootTraceId?: string;
+    timestamp: string;
+}
+
+/**
+ * Follow-up to #2520. Failure analysis is the most operator-valuable
+ * output when a run under-performs — the reviewer on #2507 called it
+ * out explicitly. The pre-fix catch block here silently dropped on
+ * upstream LLM errors (429/5xx/auth), so an operator would see a run
+ * with unexplained `status: 'error'` criteria AND no analysis block,
+ * and have no way to distinguish "infra outage" from "analysis
+ * suppressed because all criteria passed". Surface the same error
+ * taxonomy the primary evaluator uses (#2520's classifyLlmError).
+ */
+interface FailureAnalysisError {
+    reason: LlmEvalErrorReason;
+    message: string;
+}
+
+type ManagedScenarioReport = ScenarioReport & {
+    scenarioSlug?: string;
+    traceId?: string;
+    rootTraceId?: string;
+    /** Managed run id (`run-<uuid>`) — feed to `archal export run` / `inspect run`. */
+    runId?: string;
+    /**
+     * #2520 follow-up — set when the failure-analysis LLM call was
+     * suppressed by an upstream error (429/5xx/auth). When present,
+     * `failureAnalysis` will be undefined; callers that render the run
+     * summary should surface a "Failure analysis unavailable: <reason>"
+     * line so operators aren't left guessing.
+     */
+    failureAnalysisError?: FailureAnalysisError;
+};
+
+type EnvValue = string | undefined;
+/**
+ * Extra host path mounted read-only (default) into the agent container under the
+ * Docker harness. Structurally identical to the engine's `DockerHarnessMount`;
+ * kept local so the public SDK surface does not depend on engine internals.
+ */
+interface RuntimeMount {
+    source: string;
+    target: string;
+    readonly?: boolean;
+}
+interface RuntimeConfig {
+    command: [string, ...string[]];
+    cwd?: string;
+    env?: Record<string, EnvValue>;
+    dockerfile?: string;
+    mounts?: ReadonlyArray<RuntimeMount>;
+}
+interface RuntimeDefinition {
+    kind: 'runtime';
+    command: string;
+    args: string[];
+    env?: Record<string, string>;
+    cwd: string;
+    dockerfile?: string;
+    mounts?: ReadonlyArray<RuntimeMount>;
+}
+interface FileSeed {
+    kind: 'file';
+    path: string;
+}
+interface CloneDefinition {
+    name: string;
+    seed?: string | FileSeed;
+}
+type CloneSpec = string | CloneDefinition;
+interface FileSeedOptions {
+    relativeTo?: string | URL;
+}
+interface RunDefaults {
+    runs?: number;
+    timeout?: string | number;
+    passThreshold?: number;
+    judge?: {
+        model?: string;
+    };
+}
+interface TraceMatcher {
+    subject: string;
+    toolNames?: string[];
+}
+interface CountScenarioAssertion {
+    type: 'exact_count' | 'min_count' | 'max_count';
+    subject: string;
+    value: number;
+    predicate?: string;
+    targetService?: string;
+    requireDiff?: boolean;
+    titleIncludes?: string;
+    labelFilter?: string;
+    channelFilter?: string;
+}
+type StructuredScenarioAssertion = CountScenarioAssertion | {
+    type: 'exists';
+    subject: string;
+    requireDiff?: boolean;
+} | {
+    type: 'github_issue';
+    subject: 'github issue';
+    titleIncludes?: string;
+    labelFilter?: string;
+    requireDiff?: boolean;
+} | {
+    type: 'output_contains';
+    subject: string;
+    contentPatterns: string[];
+    negated?: boolean;
+} | {
+    type: 'trace_contains';
+    subject: string;
+    toolNames: string[];
+    negated?: boolean;
+} | {
+    type: 'trace_order';
+    subject: string;
+    first: TraceMatcher;
+    second: TraceMatcher;
+};
+interface ScenarioCheck {
+    description: string;
+    type: 'deterministic' | 'probabilistic';
+    structuredAssertion?: StructuredScenarioAssertion;
+    traceMatcher?: TraceMatcher;
+}
+interface ScenarioDefinition {
+    name: string;
+    task: string;
+    checks: ScenarioCheck[];
+    clones?: CloneDefinition[];
+    run?: RunDefaults;
+}
+interface SuiteDefinition {
+    name: string;
+    agent: RuntimeDefinition;
+    clones?: CloneDefinition[];
+    run?: RunDefaults;
+    tags?: string[];
+    scenarios: ScenarioDefinition[];
+}
+interface ScenarioRunResult {
+    name: string;
+    passed: boolean;
+    satisfaction: number;
+    report: ManagedScenarioReport;
+    runs: RunResult[];
+    traceId?: string;
+    rootTraceId?: string;
+}
+interface SuiteRunResult {
+    name: string;
+    passed: boolean;
+    satisfaction: number;
+    scenarios: ScenarioRunResult[];
+    reports: ManagedScenarioReport[];
+}
+interface ServiceAction {
+    toolName: string;
+    aliases?: string[];
+}
+interface CountOptions {
+    count?: number;
+    atLeast?: number;
+    atMost?: number;
+}
+
+declare function before(first: ScenarioCheck, second: ScenarioCheck): ScenarioCheck;
+declare const response: {
+    includes(text: string): ScenarioCheck;
+    doesNotInclude(text: string): ScenarioCheck;
+};
+declare const activity: {
+    didCall(action: ServiceAction): ScenarioCheck;
+    didNotCall(action: ServiceAction): ScenarioCheck;
+};
+declare function judge(description: string): ScenarioCheck;
+type GithubFactory = ((options?: {
+    seed?: string | FileSeed;
+}) => CloneDefinition) & {
+    issue: {
+        created(options?: CountOptions & {
+            titleIncludes?: string;
+        }): ScenarioCheck;
+        closed(options?: CountOptions): ScenarioCheck;
+        notClosed(): ScenarioCheck;
+        exists(options?: {
+            titleIncludes?: string;
+        }): ScenarioCheck;
+        hasLabel(options: {
+            titleIncludes?: string;
+            label: string;
+        }): ScenarioCheck;
+    };
+    issues: Record<'create' | 'delete' | 'close' | 'list', ServiceAction>;
+};
+declare const github: GithubFactory;
+type SlackFactory = ((options?: {
+    seed?: string | FileSeed;
+}) => CloneDefinition) & {
+    message: {
+        posted(options?: CountOptions & {
+            channel?: string;
+        }): ScenarioCheck;
+        notPosted(options?: {
+            channel?: string;
+        }): ScenarioCheck;
+    };
+    messages: Record<'post' | 'history', ServiceAction>;
+};
+declare const slack: SlackFactory;
+
+declare function runtime(config: RuntimeConfig): RuntimeDefinition;
+declare function suite(name: string, definition: Omit<SuiteDefinition, 'name'>): SuiteDefinition;
+declare function run(definition: SuiteDefinition): Promise<SuiteRunResult>;
+declare function file(path: string | URL, options?: FileSeedOptions): FileSeed;
+declare function clones(...specs: CloneSpec[]): CloneDefinition[];
+
+export { type CloneDefinition, type CloneSpec, type CountOptions, type CountScenarioAssertion, type EnvValue, type FileSeed, type FileSeedOptions, type RunDefaults, type RuntimeConfig, type RuntimeDefinition, type RuntimeMount, type ScenarioCheck, type ScenarioDefinition, type ScenarioRunResult, type ServiceAction, type StructuredScenarioAssertion, type SuiteDefinition, type SuiteRunResult, type TraceMatcher, activity, before, clones, file, github, judge, response, run, runtime, slack, suite };