@langwatch/scenario 0.4.1 → 0.4.3

package/dist/index.d.ts

@@ -3,6 +3,7 @@ import { ModelMessage, UserModelMessage, AssistantModelMessage, ToolModelMessage
 import { z } from 'zod/v4';
 import { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import { RealtimeSession } from '@openai/agents/realtime';
+import { SetupObservabilityOptions } from 'langwatch/observability/node';
 import { Observable } from 'rxjs';
 import { z as z$1 } from 'zod';
 
@@ -20,6 +21,18 @@ declare enum AgentRole {
     JUDGE = "Judge"
 }
 declare const allAgentRoles: readonly [AgentRole.USER, AgentRole.AGENT, AgentRole.JUDGE];
+/**
+ * Encapsulates a request for the judge agent to evaluate the conversation.
+ *
+ * When present on AgentInput, signals the judge to produce a verdict.
+ * Optionally carries inline criteria that override the judge's own criteria.
+ */
+interface JudgmentRequest {
+    /**
+     * Optional criteria to evaluate, overriding the judge agent's configured criteria.
+     */
+    criteria?: string[];
+}
 /**
  * Input provided to an agent's `call` method.
  */
@@ -41,9 +54,9 @@ interface AgentInput {
      */
     requestedRole: AgentRole;
     /**
-     * Whether a judgment is being requested in this turn.
+     * When set, requests the judge to produce a verdict, optionally with inline criteria.
      */
-    judgmentRequest: boolean;
+    judgmentRequest?: JudgmentRequest;
     /**
      * The current state of the scenario execution.
      */
@@ -166,6 +179,12 @@ interface ScenarioConfig {
      * If not provided, the scenario will not be grouped into a set.
      */
     setId?: string;
+    /**
+     * Optional metadata to attach to the scenario run.
+     * Accepts arbitrary key-value pairs (e.g. prompt IDs, environments, versions).
+     * The `langwatch` key is reserved for platform-internal use.
+     */
+    metadata?: Record<string, unknown>;
 }
 /**
  * Final, normalized scenario configuration.
@@ -215,10 +234,12 @@ interface ScenarioExecutionLike {
     agent(content?: string | ModelMessage): Promise<void>;
     /**
      * Invokes the judge agent to evaluate the current state.
-     * @param content Optional message to the judge.
+     * @param options Optional options with inline criteria to evaluate as a checkpoint.
      * @returns The result of the scenario if the judge makes a final decision.
      */
-    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
+    judge(options?: {
+        criteria?: string[];
+    }): Promise<ScenarioResult | null>;
     /**
      * Proceeds with the scenario automatically for a number of turns.
      * @param turns The number of turns to proceed. Defaults to running until the scenario ends.
@@ -251,6 +272,10 @@ type ScriptStep = (state: ScenarioExecutionStateLike, executor: ScenarioExecutio
  *
  */
 interface ScenarioResult {
+    /**
+     * Unique identifier for this scenario run.
+     */
+    runId: string;
     /**
      * Indicates whether the scenario was successful.
      */
@@ -343,6 +368,13 @@ interface ScenarioExecutionStateLike {
     hasToolCall(toolName: string): boolean;
 }
 
+/**
+ * Schema for the scenario project configuration file (scenario.config.js).
+ *
+ * The `observability` field accepts a subset of `SetupObservabilityOptions`
+ * from the langwatch SDK. It uses `z.custom()` to avoid strict validation
+ * on the passthrough object while keeping the outer config strict.
+ */
 declare const scenarioProjectConfigSchema: z.ZodObject<{
     defaultModel: z.ZodOptional<z.ZodObject<{
         model: z.ZodCustom<ai.LanguageModel, ai.LanguageModel>;
@@ -350,6 +382,7 @@ declare const scenarioProjectConfigSchema: z.ZodObject<{
         maxTokens: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>>;
     headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    observability: z.ZodOptional<z.ZodCustom<Partial<SetupObservabilityOptions>, Partial<SetupObservabilityOptions>>>;
 }, z.core.$strict>;
 type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
 declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
@@ -364,6 +397,7 @@ declare const domain_DEFAULT_MAX_TURNS: typeof DEFAULT_MAX_TURNS;
 declare const domain_DEFAULT_VERBOSE: typeof DEFAULT_VERBOSE;
 type domain_JudgeAgentAdapter = JudgeAgentAdapter;
 declare const domain_JudgeAgentAdapter: typeof JudgeAgentAdapter;
+type domain_JudgmentRequest = JudgmentRequest;
 type domain_ScenarioConfig = ScenarioConfig;
 type domain_ScenarioConfigFinal = ScenarioConfigFinal;
 type domain_ScenarioExecutionLike = ScenarioExecutionLike;
@@ -377,7 +411,7 @@ declare const domain_allAgentRoles: typeof allAgentRoles;
 declare const domain_defineConfig: typeof defineConfig;
 declare const domain_scenarioProjectConfigSchema: typeof scenarioProjectConfigSchema;
 declare namespace domain {
-  export { domain_AgentAdapter as AgentAdapter, type domain_AgentInput as AgentInput, type domain_AgentReturnTypes as AgentReturnTypes, domain_AgentRole as AgentRole, domain_DEFAULT_MAX_TURNS as DEFAULT_MAX_TURNS, domain_DEFAULT_VERBOSE as DEFAULT_VERBOSE, domain_JudgeAgentAdapter as JudgeAgentAdapter, type domain_ScenarioConfig as ScenarioConfig, type domain_ScenarioConfigFinal as ScenarioConfigFinal, type domain_ScenarioExecutionLike as ScenarioExecutionLike, type domain_ScenarioExecutionStateLike as ScenarioExecutionStateLike, type domain_ScenarioProjectConfig as ScenarioProjectConfig, type domain_ScenarioResult as ScenarioResult, type domain_ScriptStep as ScriptStep, domain_UserSimulatorAgentAdapter as UserSimulatorAgentAdapter, domain_allAgentRoles as allAgentRoles, domain_defineConfig as defineConfig, domain_scenarioProjectConfigSchema as scenarioProjectConfigSchema };
+  export { domain_AgentAdapter as AgentAdapter, type domain_AgentInput as AgentInput, type domain_AgentReturnTypes as AgentReturnTypes, domain_AgentRole as AgentRole, domain_DEFAULT_MAX_TURNS as DEFAULT_MAX_TURNS, domain_DEFAULT_VERBOSE as DEFAULT_VERBOSE, domain_JudgeAgentAdapter as JudgeAgentAdapter, type domain_JudgmentRequest as JudgmentRequest, type domain_ScenarioConfig as ScenarioConfig, type domain_ScenarioConfigFinal as ScenarioConfigFinal, type domain_ScenarioExecutionLike as ScenarioExecutionLike, type domain_ScenarioExecutionStateLike as ScenarioExecutionStateLike, type domain_ScenarioProjectConfig as ScenarioProjectConfig, type domain_ScenarioResult as ScenarioResult, type domain_ScriptStep as ScriptStep, domain_UserSimulatorAgentAdapter as UserSimulatorAgentAdapter, domain_allAgentRoles as allAgentRoles, domain_defineConfig as defineConfig, domain_scenarioProjectConfigSchema as scenarioProjectConfigSchema };
 }
 
 /**
@@ -452,6 +486,13 @@ declare class JudgeSpanCollector implements SpanProcessor {
     onEnd(span: ReadableSpan): void;
     forceFlush(): Promise<void>;
     shutdown(): Promise<void>;
+    /**
+     * Removes all spans associated with a specific thread.
+     * Call this after a scenario run completes to prevent memory growth
+     * in long-lived processes.
+     * @param threadId - The thread identifier whose spans should be cleared
+     */
+    clearSpansForThread(threadId: string): void;
     /**
      * Retrieves all spans associated with a specific thread.
      * @param threadId - The thread identifier to filter spans by
@@ -475,7 +516,7 @@ interface JudgeAgentConfig extends TestingAgentConfig {
     /**
      * The criteria that the judge will use to evaluate the conversation.
      */
-    criteria: string[];
+    criteria?: string[];
     /**
      * Optional span collector for telemetry. Defaults to global singleton.
      */
@@ -554,7 +595,7 @@ declare class JudgeAgent extends JudgeAgentAdapter {
  * main();
  * ```
  */
-declare const judgeAgent: (cfg: JudgeAgentConfig) => JudgeAgent;
+declare const judgeAgent: (cfg?: JudgeAgentConfig) => JudgeAgent;
 
 /**
  * Transforms OpenTelemetry spans into a complete plain-text digest for judge evaluation.
@@ -899,35 +940,39 @@ declare const scenarioEventSchema: z$1.ZodDiscriminatedUnion<"type", [z$1.ZodObj
     metadata: z$1.ZodObject<{
         name: z$1.ZodOptional<z$1.ZodString>;
         description: z$1.ZodOptional<z$1.ZodString>;
-    }, "strip", z$1.ZodTypeAny, {
-        description?: string | undefined;
-        name?: string | undefined;
-    }, {
-        description?: string | undefined;
-        name?: string | undefined;
-    }>;
+    }, "strip", z$1.ZodUnknown, z$1.objectOutputType<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, z$1.ZodUnknown, "strip">, z$1.objectInputType<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, z$1.ZodUnknown, "strip">>;
 }, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_STARTED;
+    metadata: {
+        name?: string | undefined;
+        description?: string | undefined;
+    } & {
+        [k: string]: unknown;
+    };
     timestamp: number;
     batchRunId: string;
     scenarioId: string;
     scenarioRunId: string;
     scenarioSetId: string;
-    metadata: {
-        description?: string | undefined;
-        name?: string | undefined;
-    };
     rawEvent?: any;
 }, {
     type: ScenarioEventType.RUN_STARTED;
+    metadata: {
+        name?: string | undefined;
+        description?: string | undefined;
+    } & {
+        [k: string]: unknown;
+    };
     timestamp: number;
     batchRunId: string;
     scenarioId: string;
     scenarioRunId: string;
-    metadata: {
-        description?: string | undefined;
-        name?: string | undefined;
-    };
     rawEvent?: any;
     scenarioSetId?: string | undefined;
 }>, z$1.ZodObject<{
@@ -1334,6 +1379,8 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
     private currentTurnSpan?;
     /** Timestamp when execution started (for total time calculation) */
     private totalStartTime;
+    /** Accumulated results from inline judge checkpoints */
+    private checkpointResults;
     /** Event stream for monitoring scenario progress */
     private eventSubject;
     /**
@@ -1346,13 +1393,18 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - RUN_FINISHED: When scenario execution completes (success/failure/error)
      */
     readonly events$: Observable<ScenarioEvent>;
+    /** Batch run ID for grouping scenario runs */
+    private batchRunId;
+    /** The run ID for the current execution */
+    private scenarioRunId?;
     /**
      * Creates a new ScenarioExecution instance.
      *
      * @param config - The scenario configuration containing agents, settings, and metadata
      * @param script - The ordered sequence of script steps that define the test flow
+     * @param batchRunId - Batch run ID for grouping scenario runs
      */
-    constructor(config: ScenarioConfig, script: ScriptStep[]);
+    constructor(config: ScenarioConfig, script: ScriptStep[], batchRunId: string);
     /**
      * Gets the complete conversation history as an array of messages.
      *
@@ -1554,24 +1606,23 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional message to pass to the judge agent for additional context
+     * @param options - Optional options with inline criteria to evaluate as a checkpoint.
      * @returns A promise that resolves with:
      *   - ScenarioResult if the judge makes a final decision, or
      *   - Null if the conversation should continue
      *
      * @example
      * ```typescript
-     * // Let judge evaluate current state
+     * // Let judge evaluate with its configured criteria
      * const result = await execution.judge();
-     * if (result) {
-     *   console.log(`Judge decided: ${result.success ? 'pass' : 'fail'}`);
-     * }
      *
-     * // Provide additional context to judge
-     * const result = await execution.judge("Please consider the user's satisfaction level");
+     * // Evaluate inline criteria as a checkpoint
+     * const result = await execution.judge({ criteria: ["Agent responded helpfully"] });
      * ```
      */
-    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
+    judge(options?: {
+        criteria?: string[];
+    }): Promise<ScenarioResult | null>;
     /**
      * Lets the scenario proceed automatically for a specified number of turns.
      *
@@ -1712,12 +1763,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - Creates a new ScenarioExecutionState with the current config
      * - Sets up the thread ID (generates new one if not provided)
      * - Initializes all agents
-     * - Starts the first turn
+     * - Initializes turn state (pending agents/roles) without creating a trace span
      * - Records the start time for performance tracking
      * - Clears any pending messages
      * - Clears the result from any previous execution
      */
     private reset;
+    /** Compiles all accumulated checkpoint results into aggregated met/unmet criteria. */
+    private get compiledCheckpoints();
     private nextAgentForRole;
     /**
      * Starts a new turn in the scenario execution.
@@ -1889,6 +1942,25 @@ declare namespace execution {
   export { execution_ScenarioExecution as ScenarioExecution, execution_ScenarioExecutionState as ScenarioExecutionState, type execution_StateChangeEvent as StateChangeEvent, execution_StateChangeEventType as StateChangeEventType };
 }
 
+/**
+ * Configuration for LangWatch event reporting.
+ * All fields are optional — any omitted fields fall back to environment variables.
+ */
+interface LangwatchConfig {
+    /** The endpoint URL to send events to. Falls back to LANGWATCH_ENDPOINT env var. */
+    endpoint?: string;
+    /** The API key for authentication. Falls back to LANGWATCH_API_KEY env var. */
+    apiKey?: string;
+}
+/**
+ * Options for running a scenario.
+ */
+interface RunOptions {
+    /** LangWatch configuration for event reporting. Overrides environment variables. */
+    langwatch?: LangwatchConfig;
+    /** Batch run ID for grouping scenario runs. Overrides SCENARIO_BATCH_RUN_ID env var. */
+    batchRunId?: string;
+}
 /**
  * High-level interface for running a scenario test.
  *
@@ -1938,11 +2010,13 @@ declare namespace execution {
  * main();
  * ```
  */
-declare function run(cfg: ScenarioConfig): Promise<ScenarioResult>;
+declare function run(cfg: ScenarioConfig, options?: RunOptions): Promise<ScenarioResult>;
 
+type runner_LangwatchConfig = LangwatchConfig;
+type runner_RunOptions = RunOptions;
 declare const runner_run: typeof run;
 declare namespace runner {
-  export { runner_run as run };
+  export { type runner_LangwatchConfig as LangwatchConfig, type runner_RunOptions as RunOptions, runner_run as run };
 }
 
 /**
@@ -1980,15 +2054,20 @@ declare const agent: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Invoke the judge agent to evaluate the current conversation state.
  *
- * This function forces the judge agent to make a decision about whether
- * the scenario should continue or end with a success/failure verdict.
- * The judge will evaluate based on its configured criteria.
+ * When criteria are provided inline, the judge evaluates only those criteria
+ * as a checkpoint: if all pass, the scenario continues; if any fail, the
+ * scenario fails immediately. This is the preferred way to pass criteria
+ * when using scripts.
  *
- * @param content Optional message content for the judge. Usually undefined to let
- *                the judge evaluate based on its criteria.
+ * When no criteria are provided, the judge uses its own configured criteria
+ * and returns a final verdict (success or failure), ending the scenario.
+ *
+ * @param options Optional options object with inline criteria to evaluate.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const judge: (content?: string | ModelMessage) => ScriptStep;
+declare const judge: (options?: {
+    criteria: string[];
+}) => ScriptStep;
 /**
  * Generate or specify a user message in the conversation.
  *
@@ -2045,7 +2124,107 @@ declare namespace script {
   export { script_agent as agent, script_fail as fail, script_judge as judge, script_message as message, script_proceed as proceed, script_succeed as succeed, script_user as user };
 }
 
+/**
+ * Explicitly set up tracing for @langwatch/scenario.
+ *
+ * Call this before any `run()` invocations when you want full control
+ * over the observability configuration. If called, `run()` will skip
+ * its own lazy initialization.
+ *
+ * The `judgeSpanCollector` is always added as a span processor regardless
+ * of the user-provided options.
+ *
+ * @param options - Optional `SetupObservabilityOptions` forwarded to the
+ *   langwatch SDK `setupObservability()` function.
+ *
+ * @example
+ * ```typescript
+ * import { setupScenarioTracing } from "@langwatch/scenario";
+ *
+ * setupScenarioTracing({
+ *   instrumentations: [],          // disable auto-instrumentation
+ *   spanProcessors: [myProcessor], // add custom processors
+ * });
+ * ```
+ */
+declare function setupScenarioTracing(options?: Partial<SetupObservabilityOptions>): void;
+
+/**
+ * Criteria for matching spans by instrumentation scope name or span name.
+ * Within each field, matchers use OR semantics (any match succeeds).
+ * Across fields, AND semantics apply (all specified fields must match).
+ */
+interface TraceFilterCriteria {
+    instrumentationScopeName?: TraceFilterMatch[];
+    name?: TraceFilterMatch[];
+}
+/**
+ * A single match rule for string comparison.
+ */
+interface TraceFilterMatch {
+    equals?: string;
+    startsWith?: string;
+    matches?: RegExp;
+    ignoreCase?: boolean;
+}
+/**
+ * A filter rule for controlling which spans are exported.
+ *
+ * Compatible with the langwatch SDK's `TraceFilter` type used by
+ * `LangWatchTraceExporter`.
+ */
+type TraceFilter = {
+    preset: "vercelAIOnly" | "excludeHttpRequests";
+} | {
+    include: TraceFilterCriteria;
+} | {
+    exclude: TraceFilterCriteria;
+};
+/**
+ * Preset filter that only keeps spans from the @langwatch/scenario instrumentation scope.
+ * Use this to prevent unrelated server spans (HTTP, middleware, etc.) from being exported.
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig, scenarioOnly } from "@langwatch/scenario";
+ * import { LangWatchTraceExporter } from "langwatch/observability";
+ *
+ * export default defineConfig({
+ *   observability: {
+ *     traceExporter: new LangWatchTraceExporter({
+ *       filters: scenarioOnly,
+ *     }),
+ *     instrumentations: [], // disable auto-instrumentation
+ *   },
+ * });
+ * ```
+ */
+declare const scenarioOnly: TraceFilter[];
+/**
+ * Creates a filter that keeps spans from the @langwatch/scenario scope
+ * plus any additional custom instrumentation scopes.
+ *
+ * @param scopes - Additional instrumentation scope names to include
+ * @returns Array of TraceFilter rules
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig, withCustomScopes } from "@langwatch/scenario";
+ * import { LangWatchTraceExporter } from "langwatch/observability";
+ *
+ * export default defineConfig({
+ *   observability: {
+ *     traceExporter: new LangWatchTraceExporter({
+ *       filters: withCustomScopes("my-app/database", "my-app/agent"),
+ *     }),
+ *     instrumentations: [], // disable auto-instrumentation
+ *   },
+ * });
+ * ```
+ */
+declare function withCustomScopes(...scopes: string[]): TraceFilter[];
+
 type ScenarioApi = typeof agents & typeof domain & typeof execution & typeof runner & typeof script;
 declare const scenario: ScenarioApi;
 
-export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, type AudioResponseEvent, DEFAULT_MAX_TURNS, DEFAULT_VERBOSE, type FinishTestArgs, type InvokeLLMParams, type InvokeLLMResult, JudgeAgentAdapter, type JudgeAgentConfig, type JudgeResult, JudgeSpanCollector, JudgeSpanDigestFormatter, RealtimeAgentAdapter, type RealtimeAgentAdapterConfig, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type StateChangeEvent, StateChangeEventType, type TestingAgentConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, scenario as default, defineConfig, fail, judge, judgeAgent, judgeSpanCollector, judgeSpanDigestFormatter, message, proceed, run, scenario, scenarioProjectConfigSchema, succeed, user, userSimulatorAgent };
+export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, type AudioResponseEvent, DEFAULT_MAX_TURNS, DEFAULT_VERBOSE, type FinishTestArgs, type InvokeLLMParams, type InvokeLLMResult, JudgeAgentAdapter, type JudgeAgentConfig, type JudgeResult, JudgeSpanCollector, JudgeSpanDigestFormatter, type JudgmentRequest, type LangwatchConfig, RealtimeAgentAdapter, type RealtimeAgentAdapterConfig, type RunOptions, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type StateChangeEvent, StateChangeEventType, type TestingAgentConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, scenario as default, defineConfig, fail, judge, judgeAgent, judgeSpanCollector, judgeSpanDigestFormatter, message, proceed, run, scenario, scenarioOnly, scenarioProjectConfigSchema, setupScenarioTracing, succeed, user, userSimulatorAgent, withCustomScopes };