@langwatch/scenario 0.4.2 → 0.4.4

package/README.md

@@ -97,7 +97,7 @@ describe("Weather Agent", () => {
       role: AgentRole.AGENT,
       call: async (input) => {
         const response = await generateText({
-          model: openai("gpt-4.1"),
+          model: openai("gpt-4.1-mini"),
           system: `You are a helpful assistant that may help the user with weather information.`,
           messages: input.messages,
           tools: { get_current_weather: getCurrentWeather },
@@ -150,7 +150,7 @@ describe("Weather Agent", () => {
         "The user asks for the weather in a specific city, and the agent should use the weather tool to find it.",
       agents: [
         weatherAgent,
-        scenario.userSimulatorAgent({ model: openai("gpt-4.1") }),
+        scenario.userSimulatorAgent({ model: openai("gpt-4.1-mini") }),
       ],
       script: [
         scenario.user("What's the weather like in Barcelona?"),

package/dist/index.d.mts

@@ -3,6 +3,7 @@ import { ModelMessage, UserModelMessage, AssistantModelMessage, ToolModelMessage
 import { z } from 'zod/v4';
 import { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import { RealtimeSession } from '@openai/agents/realtime';
+import { SetupObservabilityOptions } from 'langwatch/observability/node';
 import { Observable } from 'rxjs';
 import { z as z$1 } from 'zod';
 
@@ -178,6 +179,12 @@ interface ScenarioConfig {
      * If not provided, the scenario will not be grouped into a set.
      */
     setId?: string;
+    /**
+     * Optional metadata to attach to the scenario run.
+     * Accepts arbitrary key-value pairs (e.g. prompt IDs, environments, versions).
+     * The `langwatch` key is reserved for platform-internal use.
+     */
+    metadata?: Record<string, unknown>;
 }
 /**
  * Final, normalized scenario configuration.
@@ -265,6 +272,10 @@ type ScriptStep = (state: ScenarioExecutionStateLike, executor: ScenarioExecutio
  *
  */
 interface ScenarioResult {
+    /**
+     * Unique identifier for this scenario run.
+     */
+    runId: string;
     /**
      * Indicates whether the scenario was successful.
      */
@@ -357,6 +368,13 @@ interface ScenarioExecutionStateLike {
     hasToolCall(toolName: string): boolean;
 }
 
+/**
+ * Schema for the scenario project configuration file (scenario.config.js).
+ *
+ * The `observability` field accepts a subset of `SetupObservabilityOptions`
+ * from the langwatch SDK. It uses `z.custom()` to avoid strict validation
+ * on the passthrough object while keeping the outer config strict.
+ */
 declare const scenarioProjectConfigSchema: z.ZodObject<{
     defaultModel: z.ZodOptional<z.ZodObject<{
         model: z.ZodCustom<ai.LanguageModel, ai.LanguageModel>;
@@ -364,6 +382,7 @@ declare const scenarioProjectConfigSchema: z.ZodObject<{
         maxTokens: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>>;
     headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+    observability: z.ZodOptional<z.ZodCustom<Partial<SetupObservabilityOptions>, Partial<SetupObservabilityOptions>>>;
 }, z.core.$strict>;
 type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
 declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
@@ -467,6 +486,13 @@ declare class JudgeSpanCollector implements SpanProcessor {
     onEnd(span: ReadableSpan): void;
     forceFlush(): Promise<void>;
     shutdown(): Promise<void>;
+    /**
+     * Removes all spans associated with a specific thread.
+     * Call this after a scenario run completes to prevent memory growth
+     * in long-lived processes.
+     * @param threadId - The thread identifier whose spans should be cleared
+     */
+    clearSpansForThread(threadId: string): void;
     /**
      * Retrieves all spans associated with a specific thread.
      * @param threadId - The thread identifier to filter spans by
@@ -495,6 +521,22 @@ interface JudgeAgentConfig extends TestingAgentConfig {
      * Optional span collector for telemetry. Defaults to global singleton.
      */
     spanCollector?: JudgeSpanCollector;
+    /**
+     * Token threshold for switching to structure-only trace rendering.
+     * When the full trace digest exceeds this estimated token count,
+     * the judge receives a structure-only view with expand_trace and
+     * grep_trace tools for progressive discovery.
+     *
+     * @default 8192
+     */
+    tokenThreshold?: number;
+    /**
+     * Maximum number of tool-calling steps for progressive trace discovery.
+     * Only applies when the trace exceeds the token threshold.
+     *
+     * @default 10
+     */
+    maxDiscoverySteps?: number;
 }
 /**
  * Agent that evaluates conversations against success criteria.
@@ -509,6 +551,8 @@ declare class JudgeAgent extends JudgeAgentAdapter {
     private readonly cfg;
     private logger;
     private readonly spanCollector;
+    private readonly tokenThreshold;
+    private readonly maxDiscoverySteps;
     role: AgentRole;
     criteria: string[];
     /**
@@ -517,7 +561,19 @@ declare class JudgeAgent extends JudgeAgentAdapter {
     invokeLLM: (params: InvokeLLMParams) => Promise<InvokeLLMResult>;
     constructor(cfg: JudgeAgentConfig);
     call(input: AgentInput): Promise<JudgeResult | null>;
-    private getOpenTelemetryTracesDigest;
+    /**
+     * Builds the trace digest, choosing between full inline rendering
+     * and structure-only mode based on estimated token count.
+     */
+    private buildTraceDigest;
+    /**
+     * Invokes the LLM, enabling multi-step tool execution for large traces.
+     * In multi-step mode, the AI SDK loops automatically: the judge can call
+     * expand_trace/grep_trace tools multiple times before reaching a terminal
+     * tool (finish_test/continue_test) or hitting the step limit.
+     */
+    private invokeLLMWithDiscovery;
+    private parseToolCalls;
 }
 /**
  * Factory function for creating JudgeAgent instances.
@@ -578,6 +634,15 @@ declare const judgeAgent: (cfg?: JudgeAgentConfig) => JudgeAgent;
 declare class JudgeSpanDigestFormatter {
     private readonly logger;
     private readonly deduplicator;
+    /**
+     * Formats spans into a structure-only digest showing span tree hierarchy
+     * without attributes, events, or content. Used for large traces that
+     * exceed the token threshold, paired with expand_trace/grep_trace tools.
+     *
+     * @param spans - All spans for a thread
+     * @returns Plain text digest with only structural information
+     */
+    formatStructureOnly(spans: ReadableSpan[]): string;
     /**
      * Formats spans into a complete digest with full content and nesting.
      * @param spans - All spans for a thread
@@ -586,20 +651,18 @@ declare class JudgeSpanDigestFormatter {
     format(spans: ReadableSpan[]): string;
     private sortByStartTime;
     private buildHierarchy;
+    private renderStructureNode;
     private renderNode;
     private getTreePrefix;
     private getAttrIndent;
-    private cleanAttributes;
-    private formatValue;
-    private transformValue;
-    private transformString;
-    private looksLikeJson;
-    private hrTimeToMs;
-    private calculateSpanDuration;
+    /**
+     * Formats a value with deduplication applied. Used by the `format()` method
+     * to reduce token usage by replacing repeated strings with markers.
+     */
+    private formatValueWithDedup;
+    private transformValueWithDedup;
+    private transformStringWithDedup;
     private calculateTotalDuration;
-    private formatDuration;
-    private formatTimestamp;
-    private getStatusIndicator;
     private collectErrors;
 }
 /**
@@ -607,6 +670,45 @@ declare class JudgeSpanDigestFormatter {
  */
 declare const judgeSpanDigestFormatter: JudgeSpanDigestFormatter;
 
+/**
+ * Default token threshold for switching to structure-only trace rendering.
+ * Traces exceeding this estimated token count will be rendered in
+ * structure-only mode with expand/grep tools available to the judge.
+ *
+ */
+declare const DEFAULT_TOKEN_THRESHOLD = 8192;
+/**
+ * Estimates the number of tokens in a text string using a byte-based heuristic.
+ * Uses UTF-8 byte length divided by 4, which accounts for multi-byte characters
+ * (emojis, CJK, etc.) that typically consume more tokens than ASCII text.
+ *
+ * @param text - The text to estimate token count for
+ * @returns Estimated token count
+ */
+declare function estimateTokens(text: string): number;
+
+/**
+ * Expands one or more spans from a trace, returning their full details
+ * (attributes, events, status) with tree position context.
+ *
+ * @param spans - The full array of ReadableSpan objects for the trace
+ * @param options - Either a single `index` or a `range` string like "10-15"
+ * @returns Formatted string with full span details, truncated to ~4000 tokens
+ */
+declare function expandTrace(spans: ReadableSpan[], { index, range }: {
+    index?: number;
+    range?: string;
+}): string;
+/**
+ * Searches across all span attributes, events, and content for a pattern.
+ * Returns matching spans with their tree position and matching content.
+ *
+ * @param spans - The full array of ReadableSpan objects for the trace
+ * @param pattern - Case-insensitive search pattern
+ * @returns Formatted string with matches, limited to 20 results and ~4000 tokens
+ */
+declare function grepTrace(spans: ReadableSpan[], pattern: string): string;
+
 declare class UserSimulatorAgent extends UserSimulatorAgentAdapter {
     private readonly cfg?;
     private logger;
@@ -855,6 +957,7 @@ declare class RealtimeAgentAdapter extends AgentAdapter {
 }
 
 type agents_AudioResponseEvent = AudioResponseEvent;
+declare const agents_DEFAULT_TOKEN_THRESHOLD: typeof DEFAULT_TOKEN_THRESHOLD;
 type agents_FinishTestArgs = FinishTestArgs;
 type agents_InvokeLLMParams = InvokeLLMParams;
 type agents_InvokeLLMResult = InvokeLLMResult;
@@ -868,12 +971,15 @@ type agents_RealtimeAgentAdapter = RealtimeAgentAdapter;
 declare const agents_RealtimeAgentAdapter: typeof RealtimeAgentAdapter;
 type agents_RealtimeAgentAdapterConfig = RealtimeAgentAdapterConfig;
 type agents_TestingAgentConfig = TestingAgentConfig;
+declare const agents_estimateTokens: typeof estimateTokens;
+declare const agents_expandTrace: typeof expandTrace;
+declare const agents_grepTrace: typeof grepTrace;
 declare const agents_judgeAgent: typeof judgeAgent;
 declare const agents_judgeSpanCollector: typeof judgeSpanCollector;
 declare const agents_judgeSpanDigestFormatter: typeof judgeSpanDigestFormatter;
 declare const agents_userSimulatorAgent: typeof userSimulatorAgent;
 declare namespace agents {
-  export { type agents_AudioResponseEvent as AudioResponseEvent, type agents_FinishTestArgs as FinishTestArgs, type agents_InvokeLLMParams as InvokeLLMParams, type agents_InvokeLLMResult as InvokeLLMResult, type agents_JudgeAgentConfig as JudgeAgentConfig, type agents_JudgeResult as JudgeResult, agents_JudgeSpanCollector as JudgeSpanCollector, agents_JudgeSpanDigestFormatter as JudgeSpanDigestFormatter, agents_RealtimeAgentAdapter as RealtimeAgentAdapter, type agents_RealtimeAgentAdapterConfig as RealtimeAgentAdapterConfig, type agents_TestingAgentConfig as TestingAgentConfig, agents_judgeAgent as judgeAgent, agents_judgeSpanCollector as judgeSpanCollector, agents_judgeSpanDigestFormatter as judgeSpanDigestFormatter, agents_userSimulatorAgent as userSimulatorAgent };
+  export { type agents_AudioResponseEvent as AudioResponseEvent, agents_DEFAULT_TOKEN_THRESHOLD as DEFAULT_TOKEN_THRESHOLD, type agents_FinishTestArgs as FinishTestArgs, type agents_InvokeLLMParams as InvokeLLMParams, type agents_InvokeLLMResult as InvokeLLMResult, type agents_JudgeAgentConfig as JudgeAgentConfig, type agents_JudgeResult as JudgeResult, agents_JudgeSpanCollector as JudgeSpanCollector, agents_JudgeSpanDigestFormatter as JudgeSpanDigestFormatter, agents_RealtimeAgentAdapter as RealtimeAgentAdapter, type agents_RealtimeAgentAdapterConfig as RealtimeAgentAdapterConfig, type agents_TestingAgentConfig as TestingAgentConfig, agents_estimateTokens as estimateTokens, agents_expandTrace as expandTrace, agents_grepTrace as grepTrace, agents_judgeAgent as judgeAgent, agents_judgeSpanCollector as judgeSpanCollector, agents_judgeSpanDigestFormatter as judgeSpanDigestFormatter, agents_userSimulatorAgent as userSimulatorAgent };
 }
 
 /**
@@ -914,35 +1020,39 @@ declare const scenarioEventSchema: z$1.ZodDiscriminatedUnion<"type", [z$1.ZodObj
     metadata: z$1.ZodObject<{
         name: z$1.ZodOptional<z$1.ZodString>;
         description: z$1.ZodOptional<z$1.ZodString>;
-    }, "strip", z$1.ZodTypeAny, {
-        description?: string | undefined;
-        name?: string | undefined;
-    }, {
-        description?: string | undefined;
-        name?: string | undefined;
-    }>;
+    }, "strip", z$1.ZodUnknown, z$1.objectOutputType<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, z$1.ZodUnknown, "strip">, z$1.objectInputType<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, z$1.ZodUnknown, "strip">>;
 }, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_STARTED;
+    metadata: {
+        name?: string | undefined;
+        description?: string | undefined;
+    } & {
+        [k: string]: unknown;
+    };
     timestamp: number;
     batchRunId: string;
     scenarioId: string;
     scenarioRunId: string;
     scenarioSetId: string;
-    metadata: {
-        description?: string | undefined;
-        name?: string | undefined;
-    };
     rawEvent?: any;
 }, {
     type: ScenarioEventType.RUN_STARTED;
+    metadata: {
+        name?: string | undefined;
+        description?: string | undefined;
+    } & {
+        [k: string]: unknown;
+    };
     timestamp: number;
     batchRunId: string;
     scenarioId: string;
     scenarioRunId: string;
-    metadata: {
-        description?: string | undefined;
-        name?: string | undefined;
-    };
     rawEvent?: any;
     scenarioSetId?: string | undefined;
 }>, z$1.ZodObject<{
@@ -1363,13 +1473,18 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - RUN_FINISHED: When scenario execution completes (success/failure/error)
      */
     readonly events$: Observable<ScenarioEvent>;
+    /** Batch run ID for grouping scenario runs */
+    private batchRunId;
+    /** The run ID for the current execution */
+    private scenarioRunId?;
     /**
      * Creates a new ScenarioExecution instance.
      *
      * @param config - The scenario configuration containing agents, settings, and metadata
      * @param script - The ordered sequence of script steps that define the test flow
+     * @param batchRunId - Batch run ID for grouping scenario runs
      */
-    constructor(config: ScenarioConfig, script: ScriptStep[]);
+    constructor(config: ScenarioConfig, script: ScriptStep[], batchRunId: string);
     /**
      * Gets the complete conversation history as an array of messages.
      *
@@ -1728,7 +1843,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - Creates a new ScenarioExecutionState with the current config
      * - Sets up the thread ID (generates new one if not provided)
      * - Initializes all agents
-     * - Starts the first turn
+     * - Initializes turn state (pending agents/roles) without creating a trace span
      * - Records the start time for performance tracking
      * - Clears any pending messages
      * - Clears the result from any previous execution
@@ -1907,6 +2022,25 @@ declare namespace execution {
   export { execution_ScenarioExecution as ScenarioExecution, execution_ScenarioExecutionState as ScenarioExecutionState, type execution_StateChangeEvent as StateChangeEvent, execution_StateChangeEventType as StateChangeEventType };
 }
 
+/**
+ * Configuration for LangWatch event reporting.
+ * All fields are optional — any omitted fields fall back to environment variables.
+ */
+interface LangwatchConfig {
+    /** The endpoint URL to send events to. Falls back to LANGWATCH_ENDPOINT env var. */
+    endpoint?: string;
+    /** The API key for authentication. Falls back to LANGWATCH_API_KEY env var. */
+    apiKey?: string;
+}
+/**
+ * Options for running a scenario.
+ */
+interface RunOptions {
+    /** LangWatch configuration for event reporting. Overrides environment variables. */
+    langwatch?: LangwatchConfig;
+    /** Batch run ID for grouping scenario runs. Overrides SCENARIO_BATCH_RUN_ID env var. */
+    batchRunId?: string;
+}
 /**
  * High-level interface for running a scenario test.
  *
@@ -1956,11 +2090,13 @@ declare namespace execution {
  * main();
  * ```
  */
-declare function run(cfg: ScenarioConfig): Promise<ScenarioResult>;
+declare function run(cfg: ScenarioConfig, options?: RunOptions): Promise<ScenarioResult>;
 
+type runner_LangwatchConfig = LangwatchConfig;
+type runner_RunOptions = RunOptions;
 declare const runner_run: typeof run;
 declare namespace runner {
-  export { runner_run as run };
+  export { type runner_LangwatchConfig as LangwatchConfig, type runner_RunOptions as RunOptions, runner_run as run };
 }
 
 /**
@@ -2068,7 +2204,107 @@ declare namespace script {
   export { script_agent as agent, script_fail as fail, script_judge as judge, script_message as message, script_proceed as proceed, script_succeed as succeed, script_user as user };
 }
 
+/**
+ * Explicitly set up tracing for @langwatch/scenario.
+ *
+ * Call this before any `run()` invocations when you want full control
+ * over the observability configuration. If called, `run()` will skip
+ * its own lazy initialization.
+ *
+ * The `judgeSpanCollector` is always added as a span processor regardless
+ * of the user-provided options.
+ *
+ * @param options - Optional `SetupObservabilityOptions` forwarded to the
+ *   langwatch SDK `setupObservability()` function.
+ *
+ * @example
+ * ```typescript
+ * import { setupScenarioTracing } from "@langwatch/scenario";
+ *
+ * setupScenarioTracing({
+ *   instrumentations: [],          // disable auto-instrumentation
+ *   spanProcessors: [myProcessor], // add custom processors
+ * });
+ * ```
+ */
+declare function setupScenarioTracing(options?: Partial<SetupObservabilityOptions>): void;
+
+/**
+ * Criteria for matching spans by instrumentation scope name or span name.
+ * Within each field, matchers use OR semantics (any match succeeds).
+ * Across fields, AND semantics apply (all specified fields must match).
+ */
+interface TraceFilterCriteria {
+    instrumentationScopeName?: TraceFilterMatch[];
+    name?: TraceFilterMatch[];
+}
+/**
+ * A single match rule for string comparison.
+ */
+interface TraceFilterMatch {
+    equals?: string;
+    startsWith?: string;
+    matches?: RegExp;
+    ignoreCase?: boolean;
+}
+/**
+ * A filter rule for controlling which spans are exported.
+ *
+ * Compatible with the langwatch SDK's `TraceFilter` type used by
+ * `LangWatchTraceExporter`.
+ */
+type TraceFilter = {
+    preset: "vercelAIOnly" | "excludeHttpRequests";
+} | {
+    include: TraceFilterCriteria;
+} | {
+    exclude: TraceFilterCriteria;
+};
+/**
+ * Preset filter that only keeps spans from the @langwatch/scenario instrumentation scope.
+ * Use this to prevent unrelated server spans (HTTP, middleware, etc.) from being exported.
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig, scenarioOnly } from "@langwatch/scenario";
+ * import { LangWatchTraceExporter } from "langwatch/observability";
+ *
+ * export default defineConfig({
+ *   observability: {
+ *     traceExporter: new LangWatchTraceExporter({
+ *       filters: scenarioOnly,
+ *     }),
+ *     instrumentations: [], // disable auto-instrumentation
+ *   },
+ * });
+ * ```
+ */
+declare const scenarioOnly: TraceFilter[];
+/**
+ * Creates a filter that keeps spans from the @langwatch/scenario scope
+ * plus any additional custom instrumentation scopes.
+ *
+ * @param scopes - Additional instrumentation scope names to include
+ * @returns Array of TraceFilter rules
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig, withCustomScopes } from "@langwatch/scenario";
+ * import { LangWatchTraceExporter } from "langwatch/observability";
+ *
+ * export default defineConfig({
+ *   observability: {
+ *     traceExporter: new LangWatchTraceExporter({
+ *       filters: withCustomScopes("my-app/database", "my-app/agent"),
+ *     }),
+ *     instrumentations: [], // disable auto-instrumentation
+ *   },
+ * });
+ * ```
+ */
+declare function withCustomScopes(...scopes: string[]): TraceFilter[];
+
 type ScenarioApi = typeof agents & typeof domain & typeof execution & typeof runner & typeof script;
 declare const scenario: ScenarioApi;
 
-export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, type AudioResponseEvent, DEFAULT_MAX_TURNS, DEFAULT_VERBOSE, type FinishTestArgs, type InvokeLLMParams, type InvokeLLMResult, JudgeAgentAdapter, type JudgeAgentConfig, type JudgeResult, JudgeSpanCollector, JudgeSpanDigestFormatter, type JudgmentRequest, RealtimeAgentAdapter, type RealtimeAgentAdapterConfig, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type StateChangeEvent, StateChangeEventType, type TestingAgentConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, scenario as default, defineConfig, fail, judge, judgeAgent, judgeSpanCollector, judgeSpanDigestFormatter, message, proceed, run, scenario, scenarioProjectConfigSchema, succeed, user, userSimulatorAgent };
+export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, type AudioResponseEvent, DEFAULT_MAX_TURNS, DEFAULT_TOKEN_THRESHOLD, DEFAULT_VERBOSE, type FinishTestArgs, type InvokeLLMParams, type InvokeLLMResult, JudgeAgentAdapter, type JudgeAgentConfig, type JudgeResult, JudgeSpanCollector, JudgeSpanDigestFormatter, type JudgmentRequest, type LangwatchConfig, RealtimeAgentAdapter, type RealtimeAgentAdapterConfig, type RunOptions, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type StateChangeEvent, StateChangeEventType, type TestingAgentConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, scenario as default, defineConfig, estimateTokens, expandTrace, fail, grepTrace, judge, judgeAgent, judgeSpanCollector, judgeSpanDigestFormatter, message, proceed, run, scenario, scenarioOnly, scenarioProjectConfigSchema, setupScenarioTracing, succeed, user, userSimulatorAgent, withCustomScopes };