langwatch 1.18.0 → 1.20.0

package/README.md

@@ -1,7 +1,7 @@
 # LangWatch TypeScript SDK
 
 <p align="center">
-  <img src="https://github.com/langwatch/langwatch/blob/main/assets/logo-header.webp?raw=true" alt="LangWatch Logo" />
+  <img src="https://github.com/langwatch/langwatch/blob/main/.github/assets/logo-header.webp?raw=true" alt="LangWatch Logo" />
 </p>
 
 <p align="center">
@@ -40,6 +40,8 @@ LangWatch helps you monitor, debug, and optimize your LLM/GenAI applications. Th
 npm install langwatch
 ```
 
+`@opentelemetry/api` is a peer dependency (npm installs it for you). A custom `metricReader` or `instrumentations` needs `@opentelemetry/sdk-metrics` or `@opentelemetry/instrumentation` installed in your app.
+
 ---
 
 ## 🚀 Getting Started
@@ -74,6 +76,7 @@ span.end();
   const tracer = getLangWatchTracer("my-app");
   ```
 - **Start a span and record input/output:**
+
   ```ts
   const span = tracer.startSpan("call-llm");
   span.setType("llm");
@@ -81,9 +84,11 @@ span.end();
   span.setOutput("Model response"); // Main way to record output
   span.end();
   ```
+
   > **Note:** `setInput` and `setOutput` are the primary methods to record input/output. Use `setInputString`/`setOutputString` for plain text, or pass any serializable value.
 
 - **Use withActiveSpan for automatic error handling:**
+
   ```ts
   await tracer.withActiveSpan("my-operation", async (span) => {
     span.setType("llm");
@@ -94,19 +99,27 @@ span.end();
   ```
 
 - **Record an evaluation directly on a span:**
+
   ```ts
   span.addEvaluation({ name: "My Eval", passed: true, score: 1.0 });
   ```
+
   > **Note:** This associates evaluation results with a specific span (operation or model call).
 
 - **(Optional) Add GenAI message events:**
+
   ```ts
   span.addGenAISystemMessageEvent({ content: "You are a helpful assistant." });
   span.addGenAIUserMessageEvent({ content: "Hello!" });
   span.addGenAIAssistantMessageEvent({ content: "Hi! How can I help you?" });
   span.addGenAIToolMessageEvent({ content: "Tool result", id: "tool-1" });
-  span.addGenAIChoiceEvent({ finish_reason: "stop", index: 0, message: { content: "Response" } });
+  span.addGenAIChoiceEvent({
+    finish_reason: "stop",
+    index: 0,
+    message: { content: "Response" },
+  });
   ```
+
   > **Advanced:** The `addGenAI...` methods are optional and mainly for advanced/manual instrumentation. Most users do not need these unless you want fine-grained message event logs.
 
 - **RAG context, metrics, and model information:**
@@ -126,6 +139,7 @@ span.end();
   const compiledPrompt = await getPrompt("prompt-id", { user: "Alice" });
   ```
 - **Fetch a specific prompt version:**
+
   ```ts
   import { getPromptVersion } from "langwatch/prompt";
   const compiledPrompt = await getPromptVersion("prompt-id", "version-id", {
@@ -142,17 +156,18 @@ span.end();
   import { runEvaluation } from "langwatch/evaluation";
   const result = await runEvaluation({
     slug: "helpfulness",
-    data: { input: "Hi", output: "Hello!" }
+    data: { input: "Hi", output: "Hello!" },
   });
   ```
 - **Record a custom evaluation:**
+
   ```ts
   await tracer.withActiveSpan("my-operation", async (span) => {
     span.addEvaluation({
       name: "Manual Eval",
       passed: true,
       score: 0.9,
-      details: "Looks good!"
+      details: "Looks good!",
     });
   });
   ```
@@ -197,16 +212,12 @@ const exporter = new LangWatchTraceExporter({
 ```ts
 // Include only specific scopes
 const exporter = new LangWatchTraceExporter({
-  filters: [
-    { include: { instrumentationScopeName: [{ equals: "ai" }] } },
-  ],
+  filters: [{ include: { instrumentationScopeName: [{ equals: "ai" }] } }],
 });
 
 // Exclude spans by name pattern
 const exporter = new LangWatchTraceExporter({
-  filters: [
-    { exclude: { name: [{ startsWith: "internal." }] } },
-  ],
+  filters: [{ exclude: { name: [{ startsWith: "internal." }] } }],
 });
 
 // Combine filters (AND pipeline)
@@ -221,6 +232,7 @@ const exporter = new LangWatchTraceExporter({
 **Learn more:** See the [Filtering Spans Tutorial](https://docs.langwatch.ai/integration/typescript/tutorials/filtering-spans) for comprehensive examples and best practices.
 
 ### Custom OpenTelemetry Integration
+
 ```ts
 import { FilterableBatchSpanProcessor, LangWatchExporter } from "langwatch";
 
@@ -229,11 +241,12 @@ const processor = new FilterableBatchSpanProcessor(
     apiKey: "your-api-key",
     endpoint: "https://custom.langwatch.com",
   }),
-  excludeRules
+  excludeRules,
 );
 ```
 
 ### Span Processing Rules
+
 ```ts
 const excludeRules: SpanProcessingExcludeRule[] = [
   { attribute: "http.url", value: "/health" },
@@ -242,6 +255,7 @@ const excludeRules: SpanProcessingExcludeRule[] = [
 ```
 
 ### Manual Instrumentation
+
 ```ts
 import { semconv } from "langwatch/observability";
 

package/dist/agent/index.d.mts

@@ -6,14 +6,7 @@ interface Logger {
 }
 
 /**
- * The frames the SDK and the platform exchange over the agent socket.
- *
- * Every frame is one JSON text message with a `type` and the protocol
- * version. The shapes here match the contract table in ADR-128 and the
- * platform's own frame module; the validators are small and hand-written
- * because this file is part of the public `langwatch/agent` surface, where no
- * schema library may cross as a value.
- *
+ * Frames exchanged over the agent socket; shapes match ADR-128.
  * @see dev/docs/adr/128-connected-agents.md
  */
 declare const PROTOCOL_VERSION = 1;
@@ -27,39 +20,22 @@ interface AgentMessage {
 type AgentParameterValue = string | number | boolean;
 /** A JSON Schema object as the SDK sends it in `register`. */
 type JsonSchemaObject = Record<string, unknown>;
+/** The scenario hooks an agent declares (ADR-180). */
+interface AgentHooks {
+    setup: boolean;
+    teardown: boolean;
+}
+/** The test data a scenario's setup made, as a JSON object. */
+type ScenarioState = Record<string, unknown>;
+interface CallRun {
+    scenarioRunId?: string;
+    scenarioName?: string;
+    batchRunId?: string;
+}
 
 /**
- * The connection the client speaks over, behind one small interface so the
- * client never depends on how the frames travel.
- *
- * Two transports carry the same frames. The WebSocket is the default and it
- * needs the `ws` package: the platform authenticates from the request
- * headers of the upgrade, and no global `WebSocket` constructor can send
- * them. HTTP long polling is for a network that blocks WebSockets: one POST
- * registers, a GET waits for the next frames, a POST carries the answers. It
- * speaks through the global `fetch` (Node 20+).
- */
-declare const AGENT_TRANSPORTS: readonly ["websocket", "http"];
-type AgentTransport = (typeof AGENT_TRANSPORTS)[number];
-/**
- * The transport to start with: the explicit option, then
- * `LANGWATCH_AGENT_TRANSPORT`, else the WebSocket. Anything that is not
- * `http` is the WebSocket, which falls back to HTTP on its own when the
- * upgrade is refused.
- */
-declare function resolveTransport({ explicit, env, }: {
-    explicit?: string;
-    env?: NodeJS.ProcessEnv;
-}): AgentTransport;
-
-/**
- * The run parameters an agent declares, and the values a call supplies.
- *
- * Three forms are accepted: a definition map, any Standard JSON Schema object
- * (read through `"~standard".jsonSchema`, so zod 4, valibot and arktype work
- * without this package importing them), or a plain JSON Schema. A schema
- * library instance that offers no JSON Schema converter is refused with the
- * three forms named, because the SDK never takes a zod instance as a value.
+ * Run parameters: three forms accepted (definition map, Standard JSON Schema,
+ * or plain JSON Schema); no direct library instances.
  */
 
 /** The scalar types a run parameter may hold. */
@@ -78,9 +54,8 @@ interface ParameterDefinition {
 type ParameterDefinitions = Record<string, ParameterDefinition>;
 /**
  * The Standard JSON Schema converter an object exposes under `"~standard"`.
- * Method syntax on purpose: a library narrows `target` to its own union, and
- * a method parameter is checked bivariantly, so zod 4, valibot and arktype
- * all fit without the SDK naming any of them.
+ * Method syntax on purpose: a method parameter is checked bivariantly, so
+ * zod 4, valibot and arktype all fit without the SDK naming any of them.
  */
 interface StandardJsonSchemaConverter {
     input?(options: {
@@ -93,21 +68,20 @@ interface StandardJsonSchemaConverter {
 /** One problem a Standard Schema `validate` reports. */
 interface StandardSchemaIssue {
     readonly message: string;
-    readonly path?: ReadonlyArray<PropertyKey | {
+    readonly path?: readonly (PropertyKey | {
         readonly key: PropertyKey;
-    }> | undefined;
+    })[] | undefined;
 }
 type StandardSchemaResult<O> = {
     readonly value: O;
     readonly issues?: undefined;
 } | {
-    readonly issues: ReadonlyArray<StandardSchemaIssue>;
+    readonly issues: readonly StandardSchemaIssue[];
 };
 /**
- * Any object that implements the Standard JSON Schema interface. When it also
- * implements Standard Schema (`validate`), the values of every call go
- * through it before the handler runs, so a zod 4 schema validates, fills its
- * defaults and types `params` in one place.
+ * Any object implementing the Standard JSON Schema interface. When it also
+ * implements Standard Schema (`validate`), every call's values pass through
+ * it first, so a zod 4 schema validates, fills defaults and types `params`.
  */
 interface StandardJsonSchema<O = unknown> {
     readonly "~standard": {
@@ -120,14 +94,16 @@ interface StandardJsonSchema<O = unknown> {
         } | undefined;
     };
 }
+/** The `output` field of a Standard Schema `types` entry, narrowed to a param record. */
+type StandardOutputOf<T> = NonNullable<T> extends {
+    readonly output: infer O;
+} ? O extends Record<string, unknown> ? O : Record<string, AgentParameterValue> : Record<string, AgentParameterValue>;
 /** The `params` type a Standard Schema object gives the handler: its parsed output. */
 type InferStandardOutput<S> = S extends {
     readonly "~standard": {
         readonly types?: infer T;
     };
-} ? [NonNullable<T>] extends [never] ? Record<string, AgentParameterValue> : NonNullable<T> extends {
-    readonly output: infer O;
-} ? O extends Record<string, unknown> ? O : Record<string, AgentParameterValue> : Record<string, AgentParameterValue> : Record<string, AgentParameterValue>;
+} ? [NonNullable<T>] extends [never] ? Record<string, AgentParameterValue> : StandardOutputOf<T> : Record<string, AgentParameterValue>;
 /** Every form `parameters` accepts. */
 type ParameterInput = ParameterDefinitions | StandardJsonSchema | JsonSchemaObject;
 /** One parameter as the platform lists it, derived from the schema. */
@@ -155,26 +131,42 @@ declare function toParameterSchema(input: ParameterInput | undefined): JsonSchem
  */
 declare function parameterSpecsFromSchema(schema: JsonSchemaObject): ParameterSpec[];
 
+/**
+ * Connection abstraction for frame transport (WebSocket default, HTTP fallback).
+ */
+declare const AGENT_TRANSPORTS: readonly ["websocket", "http"];
+type AgentTransport = (typeof AGENT_TRANSPORTS)[number];
+/**
+ * The transport to start with: the explicit option, then
+ * `LANGWATCH_AGENT_TRANSPORT`, else the WebSocket, which falls back to HTTP
+ * on its own when the upgrade is refused.
+ */
+declare function resolveTransport({ explicit, env, }: {
+    explicit?: string;
+    env?: NodeJS.ProcessEnv;
+}): AgentTransport;
+
 /**
  * `connectAgent`: the function that runs an agent becomes a simulation target.
- *
- * The wrapper resolves the environment and the parameter schema at definition,
- * registers the agent with the process-wide client, and returns a function
- * that is directly callable (for unit tests and local runs) and exposes
- * `disconnect()`.
- *
  * @see specs/typescript-sdk/agent-wrapper.feature
  */
 
 /** The default call timeout, and the cap the platform enforces. */
 declare const DEFAULT_TIMEOUT_MS = 120000;
 declare const MAX_TIMEOUT_MS = 300000;
-/** What a handler may return: a string, one message, a list of messages, or an output with a session. */
+/**
+ * What a handler may return: a string, one message, a list of messages, or an
+ * output with a session.
+ */
 type AgentOutput = string | AgentMessage | AgentMessage[];
-/** The output of one turn plus the session the agent keeps for the next turn of the same thread. */
+/**
+ * The output of one turn, the session the agent keeps for the next turn of the
+ * same thread, and an update to the scenario's state.
+ */
 interface AgentResult {
     output: AgentOutput;
     session?: unknown;
+    state?: ScenarioState;
 }
 type AgentReply = AgentOutput | AgentResult;
 /** The one object a handler receives on every turn. */
@@ -185,36 +177,57 @@ interface AgentCall<P = Record<string, AgentParameterValue>> {
     newMessages: AgentMessage[];
     /** The platform's conversation id. */
     threadId: string;
-    /** The value the handler returned as `session` on the previous turn of this thread, null on the first. */
+    /**
+     * The value the handler returned as `session` on the previous turn of this
+     * thread, null on the first.
+     */
     session: unknown;
+    /** The scenario's state: what setup returned, as the latest turn updated it; null without one. */
+    state: ScenarioState | null;
     /** The run parameters, validated and with defaults filled. */
     params: P;
-    /** The trace id of the turn, so the agent's own spans join it. Empty when the call carries none. */
+    /**
+     * The trace id of the turn, so the agent's own spans join it. Empty when the
+     * call carries none.
+     */
     traceId: string;
 }
 type AgentHandler<P> = (call: AgentCall<P>) => AgentReply | Promise<AgentReply>;
+/** The scenario run a hook is called for. */
+type ScenarioRun = CallRun;
+type HookAnswer = ScenarioState | undefined | void;
+/** Makes the test data before the first turn and returns it as the state. */
+type SetupHook<P> = (run: ScenarioRun, params: P) => HookAnswer | Promise<HookAnswer>;
+/** Removes the test data after the last turn; what it returns is shown on the run. */
+type TeardownHook<P> = (run: ScenarioRun, state: ScenarioState | null, params: P) => HookAnswer | Promise<HookAnswer>;
+/** `agent.scenario`: declares the hooks the platform runs around a scenario's turns (ADR-180). */
+interface ScenarioHooks<P> {
+    setup: (hook: SetupHook<P>) => void;
+    teardown: (hook: TeardownHook<P>) => void;
+}
 /** What a direct call of the wrapped function takes: messages, and anything else is optional. */
 interface DirectAgentCall<P> {
     messages: AgentMessage[];
     newMessages?: AgentMessage[];
     threadId?: string;
     session?: unknown;
+    state?: ScenarioState | null;
     params?: Partial<P>;
     traceId?: string;
 }
 interface ConnectAgentOptions<P extends ParameterInput = ParameterDefinitions> {
     /** The agent name. One row per name and environment on the platform. */
     name: string;
-    /** Resolved from LANGWATCH_AGENT_ENVIRONMENT, APP_ENV, ENVIRONMENT, NODE_ENV, else development. */
+    /**
+     * Resolved from LANGWATCH_AGENT_ENVIRONMENT, APP_ENV, ENVIRONMENT, NODE_ENV,
+     * else development.
+     */
     environment?: string;
     /** A definition map, a Standard JSON Schema object, or a JSON Schema object. */
     parameters?: P;
     /**
-     * Whether this process connects. Takes any boolean, so one expression can
-     * gate the deployments that connect. Without it the default is true, except
-     * when CI is truthy; a value given here replaces that rule rather than adding
-     * to it, so keep the CI half: `process.env.APP_ENV !== "production" && !process.env.CI`.
-     * LANGWATCH_AGENT_CONNECT=0 always disables.
+     * Whether this process connects. Takes any boolean, so one expression can gate the
+     * deployments that connect.
      */
     enabled?: boolean;
     /** Names this instance in the platform. Also LANGWATCH_AGENT_INSTANCE_LABEL. */
@@ -232,37 +245,53 @@ interface ConnectAgentOptions<P extends ParameterInput = ParameterDefinitions> {
     apiKey?: string;
     endpoint?: string;
     projectId?: string;
-    /** `websocket` (default, falls back to HTTP when the upgrade is refused) or `http`. Also LANGWATCH_AGENT_TRANSPORT. */
+    /**
+     * `websocket` (default, falls back to HTTP when the upgrade is refused) or
+     * `http`. Also LANGWATCH_AGENT_TRANSPORT.
+     */
     transport?: AgentTransport;
     logger?: Logger;
 }
-/** The wrapped function: callable, and connected until `disconnect()`. */
-interface ConnectedAgent<P> {
+/**
+ * The wrapped function: callable, and connected until `disconnect()`. `HookP`
+ * is `P` for every caller; only the implementation signature widens it, since
+ * the hook params are covariant and the call params are not.
+ */
+interface ConnectedAgent<P, HookP = P> {
     (call: DirectAgentCall<P>): Promise<AgentResult>;
     readonly name: string;
     readonly environment: string;
     /** The parameter schema as registered. */
     readonly parameters: JsonSchemaObject;
+    /**
+     * Declares setup and teardown. Declared before the process yields, they ride
+     * on the first register; declared later, the agents register again.
+     */
+    readonly scenario: ScenarioHooks<HookP>;
     /** Send deregister and close the socket when this was the last agent of the process. */
     disconnect: () => Promise<void>;
 }
 type Widen<V> = V extends string ? string : V extends number ? number : V extends boolean ? boolean : V;
-type ParameterValueOf<D extends ParameterDefinition> = D extends {
-    options: readonly (infer O extends string)[];
-} ? O : D extends {
+/** The value type when neither `options` nor `type` pin one down: widened from `default`. */
+type ParameterValueFallback<D extends ParameterDefinition> = D extends {
+    default: infer V;
+} ? Widen<V> : string;
+/** The value type driven by the declared `type`, falling back to `default`'s widened type. */
+type ParameterValueByType<D extends ParameterDefinition> = D extends {
     type: "number";
 } ? number : D extends {
     type: "boolean";
 } ? boolean : D extends {
     type: "string";
-} ? string : D extends {
-    default: infer V;
-} ? Widen<V> : string;
+} ? string : ParameterValueFallback<D>;
+type ParameterValueOf<D extends ParameterDefinition> = D extends {
+    options: readonly (infer O extends string)[];
+} ? O : ParameterValueByType<D>;
 /** The `params` type a definition map gives the handler. */
 type InferParameters<P extends ParameterDefinitions> = {
     [K in keyof P]: ParameterValueOf<P[K]>;
 };
-/** One of the four reply shapes as the `{ output, session }` the result frame carries. */
+/** One of the four reply shapes as the `{ output, session, state }` the result frame carries. */
 declare function normalizeReply(reply: unknown): AgentResult;
 /**
  * A schema library object (zod 4, valibot, arktype: anything with Standard
@@ -275,6 +304,20 @@ declare function connectAgent<const S extends StandardJsonSchema>(options: Conne
 declare function connectAgent<const P extends ParameterDefinitions = Record<string, never>>(options: ConnectAgentOptions<P>, handler: AgentHandler<InferParameters<P>>): ConnectedAgent<InferParameters<P>>;
 declare function connectAgent(options: ConnectAgentOptions<JsonSchemaObject>, handler: AgentHandler<Record<string, AgentParameterValue>>): ConnectedAgent<Record<string, AgentParameterValue>>;
 
+/**
+ * The state a scenario hook or a turn answers with: a JSON object under the
+ * platform's cap (ADR-180).
+ * @see dev/docs/adr/180-scenario-run-hooks.md
+ */
+
+/** The cap on a state and on teardown's returned object, the `session` cap. */
+declare const STATE_MAX_BYTES: number;
+/** A state the platform would refuse, answered with its own error code. */
+declare class AgentStateError extends Error {
+    readonly code: "agent_call_failed" | "agent_payload_too_large";
+    constructor(code: "agent_call_failed" | "agent_payload_too_large", message: string);
+}
+
 /**
  * Who and where a connected agent is: its environment, its instance identity
  * and the endpoint it connects to. Every read of the machine is defensive, so
@@ -308,4 +351,4 @@ declare function resolveConnectUrl(endpoint?: string | null): string;
  */
 declare function resolveHttpConnectUrl(endpoint?: string | null): string;
 
-export { AGENT_TRANSPORTS, type AgentCall, type AgentHandler, type AgentMessage, type AgentOutput, AgentParameterError, type AgentParameterValue, type AgentReply, type AgentResult, type AgentTransport, type ConnectAgentOptions, type ConnectedAgent, DEFAULT_TIMEOUT_MS, type DirectAgentCall, type InferParameters, type JsonSchemaObject, MAX_TIMEOUT_MS, PROTOCOL_VERSION, type ParameterDefinition, type ParameterDefinitions, type ParameterInput, type ParameterSpec, type ParameterType, type StandardJsonSchema, connectAgent, normalizeReply, parameterSpecsFromSchema, resolveConnectUrl, resolveEnvironment, resolveHttpConnectUrl, resolveTransport, sanitizeEnvironment, toParameterSchema };
+export { AGENT_TRANSPORTS, type AgentCall, type AgentHandler, type AgentHooks, type AgentMessage, type AgentOutput, AgentParameterError, type AgentParameterValue, type AgentReply, type AgentResult, AgentStateError, type AgentTransport, type ConnectAgentOptions, type ConnectedAgent, DEFAULT_TIMEOUT_MS, type DirectAgentCall, type InferParameters, type JsonSchemaObject, MAX_TIMEOUT_MS, PROTOCOL_VERSION, type ParameterDefinition, type ParameterDefinitions, type ParameterInput, type ParameterSpec, type ParameterType, STATE_MAX_BYTES, type ScenarioHooks, type ScenarioRun, type ScenarioState, type SetupHook, type StandardJsonSchema, type TeardownHook, connectAgent, normalizeReply, parameterSpecsFromSchema, resolveConnectUrl, resolveEnvironment, resolveHttpConnectUrl, resolveTransport, sanitizeEnvironment, toParameterSchema };