@warlock.js/ai 4.4.0 → 4.5.0

package/esm/contracts/team/team-config.type.d.mts

@@ -0,0 +1,127 @@
+import { AgentContract } from "../agent/agent.contract.mjs";
+import { FlowObserveOption } from "../../observe/resolve-observers.mjs";
+import { SnapshotStore } from "../orchestrator/snapshot-store.contract.mjs";
+import { WorkflowInstance } from "../workflow/workflow.contract.mjs";
+import { SystemPromptContract } from "../system-prompt.contract.mjs";
+import { RouterEntry } from "../supervisor/router-entry.type.mjs";
+import { SupervisorConfig, SupervisorEventHandlers } from "../supervisor/supervisor-config.type.mjs";
+import { EvaluateContext, EvaluateResult } from "../supervisor/evaluate-context.type.mjs";
+import { StandardSchemaV1 } from "@standard-schema/spec";
+
+//#region ../@warlock.js/ai/src/contracts/team/team-config.type.d.ts
+/**
+ * A role member — an agent or a workflow. The same shorthand the
+ * supervisor accepts on each `intents` value, narrowed to the two
+ * dispatchable unit shapes `team()` forwards verbatim. (Callback /
+ * full-entry intent shapes still work when forwarded — this alias is
+ * the documented, autocomplete-friendly subset for the common case.)
+ */
+type TeamMemberValue = AgentContract<unknown> | WorkflowInstance<unknown, unknown>;
+/**
+ * The built-in quality-gate strategies. Each desugars to a concrete
+ * {@link SupervisorConfig.evaluate} callback — no new loop or
+ * termination code; both lean entirely on the already-shipped
+ * {@link EvaluateResult} semantics (`satisfied` terminates,
+ * `reassignTo` re-dispatches the fixer, `feedback` threads forward).
+ *
+ * - `"quality"` — review-then-fix loop. After members settle, read the
+ *   reviewer's verdict slice (`state.approved` by default); if not
+ *   approved, `reassignTo` the fixer with the reviewer feedback
+ *   (`state.notes`) threaded forward. Terminates on approval.
+ * - `"verify"` — test-then-fix loop. Same shape but keyed on the
+ *   tester's pass/fail slice (`state.passed` by default) instead of a
+ *   subjective reviewer score.
+ */
+type TeamGate = "quality" | "verify";
+/**
+ * Fully-custom gate — an escape hatch identical to
+ * {@link SupervisorConfig.evaluate}. Supplying a function instead of a
+ * {@link TeamGate} string opts out of the sugar entirely while keeping
+ * the rest of `team()`'s wiring. Forwarded straight through as
+ * `evaluate` with zero wrapping.
+ */
+type TeamGateFn<TState> = (ctx: EvaluateContext<TState>) => EvaluateResult | Promise<EvaluateResult>;
+/**
+ * Config for `ai.team(config)` — thin, transparent sugar over
+ * {@link SupervisorConfig}. `team()` builds a `SupervisorConfig` from
+ * these fields and calls `supervisor(...)`, returning the **unchanged**
+ * `SupervisorContract<TOutput>`. It owns no loop: the manager becomes
+ * `route`/`router`, the members become `intents`, and the gate becomes
+ * `evaluate`. Everything else passes through 1:1.
+ *
+ * @example
+ * const codeTeam = ai.team({
+ *   name: "code-team",
+ *   goal: "Ship a tested module that passes review.",
+ *   manager: techLeadRouter,
+ *   members: { builder, reviewer, fixer },
+ *   gate: "quality",
+ *   output: v.object({ code: v.string() }),
+ *   maxIterations: 6,
+ * });
+ */
+type TeamConfig<TOutput = unknown, TState = TOutput, TMembers extends Record<string, TeamMemberValue> = Record<string, TeamMemberValue>> = {
+  /** Stable identifier — forwarded verbatim to `SupervisorConfig.name`. */name: string; /** Optional dev-curated version string — forwarded to `SupervisorConfig.version`. */
+  version?: string;
+  /**
+   * The manager. Drives dispatch each iteration. Two forms:
+   * - an `AgentContract` / `RouterEntry` → becomes `SupervisorConfig.router`
+   *   (LLM-driven manager).
+   * - `{ route }` → becomes `SupervisorConfig.route` (deterministic manager).
+   * Exactly one is forwarded; mutually exclusive (mirrors the
+   * supervisor's own `router` XOR `route` rule).
+   */
+  manager: AgentContract<unknown> | RouterEntry<TState> | {
+    route: NonNullable<SupervisorConfig<TOutput, TState>["route"]>;
+  };
+  /**
+   * Role members keyed by role name (e.g. `builder`, `reviewer`,
+   * `tester`, `fixer`). Forwarded verbatim as `SupervisorConfig.intents`.
+   * The keys are the role names the manager routes to AND the keys
+   * `ctx.intents.<role>` exposes (escape hatch preserved).
+   */
+  members: TMembers;
+  /**
+   * The quality gate. A {@link TeamGate} string selects a pre-built
+   * `evaluate` strategy; a function is forwarded straight to
+   * `SupervisorConfig.evaluate` (full escape hatch). Required — a team
+   * without a gate is just a supervisor.
+   */
+  gate: TeamGate | TeamGateFn<TState>;
+  /**
+   * Role-name mapping for the built-in string gates. Only consulted
+   * when `gate` is a {@link TeamGate} string. Defaults:
+   * `{ reviewer: "reviewer", fixer: "fixer", tester: "tester" }`.
+   * Override when your `members` keys differ from the canonical role
+   * names.
+   */
+  roles?: {
+    reviewer?: keyof TMembers & string;
+    fixer?: keyof TMembers & string;
+    tester?: keyof TMembers & string;
+  };
+  /**
+   * State key the gate reads to decide satisfaction. Defaults to
+   * `"approved"` for `gate: "quality"` and `"passed"` for
+   * `gate: "verify"`. The named member's `output` schema must write a
+   * boolean into this key.
+   */
+  gateKey?: string; /** Natural-language objective — forwarded verbatim to `SupervisorConfig.goal`. */
+  goal?: string | SystemPromptContract; /** Final-state schema — forwarded verbatim to `SupervisorConfig.output`. */
+  output?: StandardSchemaV1<TOutput>; /** Initial state seed — forwarded verbatim to `SupervisorConfig.state`. */
+  state?: TState; /** Hard iteration cap — forwarded verbatim; defaults to the supervisor's `10`. */
+  maxIterations?: number; /** Durable snapshot store — forwarded verbatim to `SupervisorConfig.snapshotStore`. */
+  snapshotStore?: SnapshotStore; /** Definition-level event handlers — forwarded verbatim to `SupervisorConfig.on`. */
+  on?: SupervisorEventHandlers;
+  /**
+   * Observability for this team — forwarded verbatim to
+   * `SupervisorConfig.observe`, so the underlying supervisor routes its
+   * completed report through the same generic `Observer` seam every
+   * other flow uses. Additive and gated; see `SupervisorConfig.observe`
+   * for the value semantics (`true` / `false` / a flow-local `Observer`).
+   */
+  observe?: FlowObserveOption;
+};
+//#endregion
+export { TeamConfig, TeamGate, TeamGateFn, TeamMemberValue };
+//# sourceMappingURL=team-config.type.d.mts.map

package/esm/contracts/team/team-config.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"team-config.type.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/contracts/team/team-config.type.ts"],"mappings":";;;;;;;;;;;;;AAiBA;;;;AAEoB;KAFR,eAAA,GACR,aAAA,YACA,gBAAgB;;;;AAiBA;AASpB;;;;;;;;;;;KATY,QAAA;;;;;;;AAWgC;KAFhC,UAAA,YACV,GAAA,EAAK,eAAA,CAAgB,MAAA,MAClB,cAAA,GAAiB,OAAA,CAAQ,cAAA;;;;;;;;;;;;;;;;;;;;KAqBlB,UAAA,6BAED,OAAA,mBACQ,MAAA,SAAe,eAAA,IAAmB,MAAA,SAAe,eAAA;EA+CjD,yEA5CjB,IAAA,UA6D0B;EA1D1B,OAAA;EA6DQ;;;;;;;;EAnDR,OAAA,EACI,aAAA,YACA,WAAA,CAAY,MAAA;IACV,KAAA,EAAO,WAAA,CAAY,gBAAA,CAAiB,OAAA,EAAS,MAAA;EAAA;EAnBnB;;;;;;EA2BhC,OAAA,EAAS,QAAA;EATL;;;;;;EAiBJ,IAAA,EAAM,QAAA,GAAW,UAAA,CAAW,MAAA;EAR5B;;;;;;;EAiBA,KAAA;IACE,QAAA,SAAiB,QAAA;IACjB,KAAA,SAAc,QAAA;IACd,MAAA,SAAe,QAAA;EAAA;EAAA;;;;;;EASjB,OAAA,WAWA;EANA,IAAA,YAAgB,oBAAA,EAShB;EANA,MAAA,GAAS,gBAAA,CAAiB,OAAA,GASV;EANhB,KAAA,GAAQ,MAAA,EASH;EANL,aAAA,WAeU;EAZV,aAAA,GAAgB,aAAA,EAYW;EAT3B,EAAA,GAAK,uBAAA;;;;;;;;EASL,OAAA,GAAU,iBAAA;AAAA"}

package/esm/contracts/tool.contract.d.mts

@@ -121,8 +121,10 @@ interface ToolConfig<TInput = unknown, TOutput = unknown> {
    *
    * Two forms:
    * - **string** — static, used as-is.
-   * - **function** — receives the validated input and returns a
-   *   string, so callers can include input data
+   * - **function** — receives the model's **raw, pre-validation**
+   *   input (resolved at the dispatch boundary, before `execute`'s
+   *   schema validation runs) and returns a string, so callers can
+   *   include input data
    *   (`(input) => \`Searching for "${input.query}"\``).
    *
    * Resolved at the framework boundary; consumers receive a

package/esm/contracts/tool.contract.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"tool.contract.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/contracts/tool.contract.ts"],"mappings":";;;;;AAQA;;;;KAAY,QAAA,GAAW,MAAM;AAoC7B;;;;AAAoB;AAmCpB;;;;;;;;;;;;;;;AASsB;AAiBtB;;;;;;;;;;;;;AA7DA,KAAY,QAAA;;;;;;;;;;;;;;;;;;;;;;;;AAmHgD;;;;;;;;;;KAhFhD,WAAA,cAAyB,MAAA;;;;;;;EAOnC,SAAA,EAAW,UAAA;EAEX,MAAA,GAAS,WAAA;AAAA;;;;;;;;;;;;;;;UAiBM,UAAA;;EAEf,IAAA;;EAEA,WAAA;;;;;;;;EAQA,OAAA;;;;;;;;;;;;;;;;EAgBA,MAAA,cAAoB,KAAA,EAAO,MAAA;;EAE3B,IAAA,GAAO,QAAA;;;;;;;EAOP,IAAA,GAAO,QAAA;;;;;;EAMP,KAAA,GAAQ,gBAAA,CAAiB,MAAA;;;;;;;;;;;EAWzB,OAAA,CAAQ,KAAA,EAAO,MAAA,EAAQ,GAAA,GAAM,WAAA,GAAc,OAAA,CAAQ,OAAA;AAAA"}
+{"version":3,"file":"tool.contract.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/contracts/tool.contract.ts"],"mappings":";;;;;AAQA;;;;KAAY,QAAA,GAAW,MAAM;AAoC7B;;;;AAAoB;AAmCpB;;;;;;;;;;;;;;;AASsB;AAiBtB;;;;;;;;;;;;;AA7DA,KAAY,QAAA;;;;;;;;;;;;;;;;;;;;;;;;AAqHgD;;;;;;;;;;KAlFhD,WAAA,cAAyB,MAAA;;;;;;;EAOnC,SAAA,EAAW,UAAA;EAEX,MAAA,GAAS,WAAA;AAAA;;;;;;;;;;;;;;;UAiBM,UAAA;;EAEf,IAAA;;EAEA,WAAA;;;;;;;;EAQA,OAAA;;;;;;;;;;;;;;;;;;EAkBA,MAAA,cAAoB,KAAA,EAAO,MAAA;;EAE3B,IAAA,GAAO,QAAA;;;;;;;EAOP,IAAA,GAAO,QAAA;;;;;;EAMP,KAAA,GAAQ,gBAAA,CAAiB,MAAA;;;;;;;;;;;EAWzB,OAAA,CAAQ,KAAA,EAAO,MAAA,EAAQ,GAAA,GAAM,WAAA,GAAc,OAAA,CAAQ,OAAA;AAAA"}

package/esm/contracts/workflow/step.contract.d.mts

@@ -169,13 +169,38 @@ type StepDefinition<TInput = unknown, TState = Record<string, unknown>, TContext
    */
   retry?: RetryConfig | false;
   /**
-   * Parallel children — run concurrently, share `ctx.state`
-   * (last-write-wins). Mutually exclusive with `run` and `agent`.
-   * The parent's snapshot is atomic: written only after every child
-   * settles. Children are addressable at both `ctx.steps.<child>` and
-   * `ctx.steps.<parent>.steps.<child>`.
+   * Parallel children — run concurrently, each starting from a snapshot
+   * of the parent `ctx.state`. After every child settles, their
+   * resulting states are merged back into the parent in **declaration
+   * order** (the order children appear in this array), so a key written
+   * by more than one child deterministically resolves to the
+   * **last-declared** child's value — regardless of which child finished
+   * first. Supply {@link mergeState} to customize conflict resolution
+   * (e.g. sum counters, concat arrays). Mutually exclusive with `run`
+   * and `agent`. The parent's snapshot is atomic: written only after
+   * every child settles. Children are addressable at both
+   * `ctx.steps.<child>` and `ctx.steps.<parent>.steps.<child>`.
    */
   parallel?: StepDefinition<TInput, TState, TContext>[];
+  /**
+   * Conflict resolver for {@link parallel} children that write the same
+   * `ctx.state` keys. Called once per child, in **declaration order**,
+   * with the accumulator (the parent state being built up), that child's
+   * resolved state, and the child's name. Mutate `acc` in place.
+   *
+   * Omit it to keep the deterministic default (last-declared child wins
+   * per key — a plain `Object.assign` in declaration order). Supply it
+   * for advanced merges:
+   *
+   * ```ts
+   * mergeState: (acc, childState) => {
+   *   acc.total = (acc.total ?? 0) + (childState.total ?? 0); // sum
+   * }
+   * ```
+   *
+   * Only consulted for parallel steps; ignored otherwise.
+   */
+  mergeState?: (acc: TState, childState: TState, childName: string) => void;
   /**
    * Per-step event handlers — fires alongside the workflow-level
    * `workflow.step.*` events. Scope is step-local (no `workflow.`

package/esm/contracts/workflow/step.contract.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"step.contract.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/contracts/workflow/step.contract.ts"],"mappings":";;;;;;;;;;;;AAaA;;KAAY,eAAA;EAAe,yEAEzB,QAAA;IAAY,IAAA;EAAA,GAGV;EADF,QAAA;IACE,IAAA;IACA,OAAA;IACA,aAAA;IACA,SAAA;EAAA,GAGsC;EAAxC,MAAA;IAAU,IAAA;IAAc,KAAA;IAAgB,QAAA;EAAA,GAEY;EAApD,SAAA;IAAa,IAAA;IAAc,MAAA;IAAiB,QAAA;EAAA;AAAA;;;;;;KAQlC,cAAA,GAAiB,mBAAmB;EAqBvC,8DAnBP,MAAM;AAAA;;;;;;;;;;KAYI,cAAA,+CAGD,MAAA;EAGT,OAAA,GACE,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAClC,OAAA,GAAU,OAAA,CAAQ,OAAA;EACvB,MAAA,GAAS,gBAAA,CAAiB,OAAA;AAAA;;;;;;;;;;;AAAO;AAoBnC;;;;;;KAAY,cAAA,4BAED,MAAA;EAkBI;;;;;;EATb,IAAA;EAwB4B;;;;;;;EAf5B,IAAA,IAAQ,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,gBAAwB,OAAA;EAgCvD;;;;;;EAxBd,MAAA,IAAU,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,aAAqB,OAAA;EA+B3D;;;;;EAxBT,GAAA,IAAO,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA;EA4CX;;;;;;;EAnCjC,KAAA,GAAQ,aAAA;EAoDuB;;;;;;EA5C/B,KAAA,IAAS,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAAc,cAAA,GAAiB,OAAA,CAAQ,cAAA;EAsDpD;;;;;EA/CjC,MAAA,GAAS,cAAA,UAAwB,MAAA,EAAQ,MAAA,EAAQ,QAAA;EAgEvB;;;;;;EAxD1B,KAAA,IAAS,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,aAAqB,OAAA;EAgE9D;;;;;;;;;;EApDL,QAAA,IAAY,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAAc,cAAA,GAAiB,OAAA,CAAQ,cAAA;EA3DnD;;;;;;;;;;;;;;EA2ErC,SAAA,IACE,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,GACrC,KAAA,EAAO,OAAA,KACJ,cAAA,GAAiB,OAAA,CAAQ,cAAA;EA/Dc;;;;;;EAuE5C,QAAA,IAAY,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,aAAqB,OAAA;EAtDhC;;;;;;EA8DtC,KAAA,GAAQ,WAAA;EAvDC;;;;;;;EAgET,QAAA,GAAW,cAAA,CAAe,MAAA,EAAQ,MAAA,EAAQ,QAAA;EAxDI;;;;;;EAgE9C,EAAA,GAAK,OAAA,eACS,eAAA,IAAmB,OAAA,EAAS,eAAA,CAAgB,CAAA;AAAA"}
+{"version":3,"file":"step.contract.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/contracts/workflow/step.contract.ts"],"mappings":";;;;;;;;;;;;AAaA;;KAAY,eAAA;EAAe,yEAEzB,QAAA;IAAY,IAAA;EAAA,GAGV;EADF,QAAA;IACE,IAAA;IACA,OAAA;IACA,aAAA;IACA,SAAA;EAAA,GAGsC;EAAxC,MAAA;IAAU,IAAA;IAAc,KAAA;IAAgB,QAAA;EAAA,GAEY;EAApD,SAAA;IAAa,IAAA;IAAc,MAAA;IAAiB,QAAA;EAAA;AAAA;;;;;;KAQlC,cAAA,GAAiB,mBAAmB;EAqBvC,8DAnBP,MAAM;AAAA;;;;;;;;;;KAYI,cAAA,+CAGD,MAAA;EAGT,OAAA,GACE,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAClC,OAAA,GAAU,OAAA,CAAQ,OAAA;EACvB,MAAA,GAAS,gBAAA,CAAiB,OAAA;AAAA;;;;;;;;;;;AAAO;AAoBnC;;;;;;KAAY,cAAA,4BAED,MAAA;EAkBI;;;;;;EATb,IAAA;EAwB4B;;;;;;;EAf5B,IAAA,IAAQ,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,gBAAwB,OAAA;EAgCvD;;;;;;EAxBd,MAAA,IAAU,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,aAAqB,OAAA;EA+B3D;;;;;EAxBT,GAAA,IAAO,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA;EA4CX;;;;;;;EAnCjC,KAAA,GAAQ,aAAA;EAoDuB;;;;;;EA5C/B,KAAA,IAAS,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAAc,cAAA,GAAiB,OAAA,CAAQ,cAAA;EAsDpD;;;;;EA/CjC,MAAA,GAAS,cAAA,UAAwB,MAAA,EAAQ,MAAA,EAAQ,QAAA;EAsEvB;;;;;;EA9D1B,KAAA,IAAS,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,aAAqB,OAAA;EA2FzB;;;;;;;;;;EA/E1C,QAAA,IAAY,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAAc,cAAA,GAAiB,OAAA,CAAQ,cAAA;EA3D3E;;;;;;;;;;;;;;EA2Eb,SAAA,IACE,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,GACrC,KAAA,EAAO,OAAA,KACJ,cAAA,GAAiB,OAAA,CAAQ,cAAA;EA/DF;;;;;;EAuE5B,QAAA,IAAY,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,aAAqB,OAAA;EAtDxD;;;;;;EA8Dd,KAAA,GAAQ,WAAA;EA9D6E;;;;;;;;;;;;;EA6ErF,QAAA,GAAW,cAAA,CAAe,MAAA,EAAQ,MAAA,EAAQ,QAAA;EAlDzB;;;;;;;;;;;;;;;;;;EAsEjB,UAAA,IAAc,GAAA,EAAK,MAAA,EAAQ,UAAA,EAAY,MAAA,EAAQ,SAAA;EA3C/C;;;;;;EAmDA,EAAA,GAAK,OAAA,eACS,eAAA,IAAmB,OAAA,EAAS,eAAA,CAAgB,CAAA;AAAA"}

package/esm/contracts/workflow/workflow.contract.d.mts

@@ -1,6 +1,7 @@
 import { ExecutableContract } from "../executable.contract.mjs";
 import { WorkflowResult } from "../result/workflow-result.type.mjs";
 import { ToolContract } from "../../tool/tool.mjs";
+import { FlowObserveOption } from "../../observe/resolve-observers.mjs";
 import { SnapshotStore } from "../orchestrator/snapshot-store.contract.mjs";
 import { NextStepResult } from "./next-step-result.type.mjs";
 import { RetryConfig } from "./retry-config.type.mjs";
@@ -99,6 +100,21 @@ type WorkflowDefinition<TInput = unknown, TOutput = unknown, TState = Record<str
   nextStep?: (stepName: string, ctx: WorkflowContext<TInput, TState, TContext>) => NextStepResult | Promise<NextStepResult>;
   output?: StepOutputSpec<TOutput, TInput, TState, TContext>;
   on?: WorkflowEventHandlers;
+  /**
+   * Observability for this workflow. Additive and gated — when omitted,
+   * the workflow follows the global observe-all flag (off unless an
+   * observability tool turned it on), so behavior is unchanged by
+   * default.
+   *
+   * - `true` → route this run's completed report to the globally
+   *   registered observers, even when observe-all is off.
+   * - `false` → opt this workflow out entirely, even when observe-all is on.
+   * - an `Observer` object → a flow-local collector receiving only this
+   *   workflow's report. Typed as the structural `Observer` so core stays
+   *   panoptic-agnostic; a panoptic flow-local collector can be passed
+   *   directly. Observer errors are swallowed — they never break the run.
+   */
+  observe?: FlowObserveOption;
 };
 /**
  * Runtime handle returned by `ai.workflow(def)`. Implements the

package/esm/contracts/workflow/workflow.contract.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"workflow.contract.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/contracts/workflow/workflow.contract.ts"],"mappings":";;;;;;;;;;;;;KAYY,oBAAA,iBAAqC,gBAAA,KAC/C,OAAA,EAAS,gBAAA,CAAiB,CAAA;AAAA,KAGhB,qBAAA,GAAwB,OAAA,eACtB,gBAAA,GAAmB,oBAAA,CAAqB,CAAA;;;;;KAO1C,kBAAA;EAXiB,sEAa3B,KAAA,WAd+C;EAgB/C,MAAA,GAAS,WAAA,EAfiB;EAiB1B,EAAA,GAAK,qBAAA;EAjBuB;AAAA;AAG9B;;;;;EAsBE,OAAA,GAAU,QAAA;EAtBwB;;;;;;;EA8BlC,SAAA;AAAA;AA7BqD;AAOvD;;;;AAPuD,KAqC3C,sBAAA,yCAGR,kBAAA,CAAmB,QAAA;EACrB,KAAA,EAAO,MAAA;AAAA;AAAA,KAGG,qBAAA,uBACV,kBAAkB,CAAC,QAAA;EAtCU;;EAyC3B,KAAA;AAAA;AAAA,KAGQ,kBAAA,+CAGD,MAAA;EAGT,IAAA;EApCA;;;;AAQS;EAkCT,WAAA;EA1BgC;;;;;;;;;;EAqChC,WAAA,GAAc,gBAAA,CAAiB,MAAA;EAC/B,OAAA;EACA,KAAA,EAAO,cAAA,CAAe,MAAA,EAAQ,MAAA,EAAQ,QAAA;EAnCzB;AAAA;AAGf;;;;;;;;;EA4CE,aAAA,GAAgB,aAAA,CAAc,gBAAA;EArCpB;;;;;;;;;EA+CV,YAAA,GAAe,WAAA;EACf,QAAA;EACA,aAAA;EACA,QAAA,IACE,QAAA,UACA,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAClC,cAAA,GAAiB,OAAA,CAAQ,cAAA;EAC9B,MAAA,GAAS,cAAA,CAAe,OAAA,EAAS,MAAA,EAAQ,MAAA,EAAQ,QAAA;EACjD,EAAA,GAAK,qBAAA;AAAA;;;;;;;UASU,gBAAA,+CAGN,MAAA,+CAED,kBAAA,CACR,MAAA,EACA,kBAAA,CAAmB,QAAA,GACnB,cAAA,CAAe,OAAA;EAAA,SAEN,IAAA;EAAA,SACA,WAAA;EApBJ;;;;;;EAAA,SA2BI,WAAA,GAAc,gBAAA,CAAiB,MAAA;EAAA,SAC/B,SAAA;EAAA,SACA,OAAA;EAxET;;;;;;;;;;;;;;EAwFA,OAAA,CACE,KAAA,EAAO,MAAA,EACP,OAAA,GAAU,kBAAA,CAAmB,QAAA,IAC5B,OAAA,CAAQ,cAAA,CAAe,OAAA;EAC1B,OAAA,CACE,OAAA,EAAS,sBAAA,CAAuB,MAAA,EAAQ,QAAA,IACvC,OAAA,CAAQ,cAAA,CAAe,OAAA;EAE1B,MAAA,CACE,KAAA,UACA,OAAA,GAAU,qBAAA,CAAsB,QAAA,IAC/B,OAAA,CAAQ,cAAA,CAAe,OAAA;EAE1B,EAAA,iBAAmB,gBAAA,EAAkB,KAAA,EAAO,CAAA,EAAG,OAAA,EAAS,oBAAA,CAAqB,CAAA;EAC7E,GAAA,iBAAoB,gBAAA,EAAkB,KAAA,EAAO,CAAA,EAAG,OAAA,EAAS,oBAAA,CAAqB,CAAA;EA9DvE;;;;;;;;;;;;;;;;EAgFP,MAAA,cAAoB,MAAA,EAAQ,OAAA;IAC1B,WAAA;IACA,WAAA,EAAa,gBAAA,CAAiB,UAAA;EAAA,IAC5B,YAAA,CAAa,UAAA,EAAY,OAAA;AAAA"}
+{"version":3,"file":"workflow.contract.d.mts","names":[],"sources":["../../../../../../../../@warlock.js/ai/src/contracts/workflow/workflow.contract.ts"],"mappings":";;;;;;;;;;;;;;KAaY,oBAAA,iBAAqC,gBAAA,KAC/C,OAAA,EAAS,gBAAA,CAAiB,CAAA;AAAA,KAGhB,qBAAA,GAAwB,OAAA,eACtB,gBAAA,GAAmB,oBAAA,CAAqB,CAAA;;;;;KAO1C,kBAAA;EAXiB,sEAa3B,KAAA,WAd+C;EAgB/C,MAAA,GAAS,WAAA,EAfiB;EAiB1B,EAAA,GAAK,qBAAA;EAjBuB;AAAA;AAG9B;;;;;EAsBE,OAAA,GAAU,QAAA;EAtBwB;;;;;;;EA8BlC,SAAA;AAAA;AA7BqD;AAOvD;;;;AAPuD,KAqC3C,sBAAA,yCAGR,kBAAA,CAAmB,QAAA;EACrB,KAAA,EAAO,MAAA;AAAA;AAAA,KAGG,qBAAA,uBACV,kBAAkB,CAAC,QAAA;EAtCU;;EAyC3B,KAAA;AAAA;AAAA,KAGQ,kBAAA,+CAGD,MAAA;EAGT,IAAA;EApCA;;;;AAQS;EAkCT,WAAA;EA1BgC;;;;;;;;;;EAqChC,WAAA,GAAc,gBAAA,CAAiB,MAAA;EAC/B,OAAA;EACA,KAAA,EAAO,cAAA,CAAe,MAAA,EAAQ,MAAA,EAAQ,QAAA;EAnCzB;AAAA;AAGf;;;;;;;;;EA4CE,aAAA,GAAgB,aAAA,CAAc,gBAAA;EArCpB;;;;;;;;;EA+CV,YAAA,GAAe,WAAA;EACf,QAAA;EACA,aAAA;EACA,QAAA,IACE,QAAA,UACA,GAAA,EAAK,eAAA,CAAgB,MAAA,EAAQ,MAAA,EAAQ,QAAA,MAClC,cAAA,GAAiB,OAAA,CAAQ,cAAA;EAC9B,MAAA,GAAS,cAAA,CAAe,OAAA,EAAS,MAAA,EAAQ,MAAA,EAAQ,QAAA;EACjD,EAAA,GAAK,qBAAA;EAHkC;;;;;;;;;;;;;;EAkBvC,OAAA,GAAU,iBAAA;AAAA;;;;;;;UASK,gBAAA,+CAGN,MAAA,+CAED,kBAAA,CACR,MAAA,EACA,kBAAA,CAAmB,QAAA,GACnB,cAAA,CAAe,OAAA;EAAA,SAEN,IAAA;EAAA,SACA,WAAA;EAjET;;;;;;EAAA,SAwES,WAAA,GAAc,gBAAA,CAAiB,MAAA;EAAA,SAC/B,SAAA;EAAA,SACA,OAAA;EApDM;;;;;;;;;;;;;;EAoEf,OAAA,CACE,KAAA,EAAO,MAAA,EACP,OAAA,GAAU,kBAAA,CAAmB,QAAA,IAC5B,OAAA,CAAQ,cAAA,CAAe,OAAA;EAC1B,OAAA,CACE,OAAA,EAAS,sBAAA,CAAuB,MAAA,EAAQ,QAAA,IACvC,OAAA,CAAQ,cAAA,CAAe,OAAA;EAE1B,MAAA,CACE,KAAA,UACA,OAAA,GAAU,qBAAA,CAAsB,QAAA,IAC/B,OAAA,CAAQ,cAAA,CAAe,OAAA;EAE1B,EAAA,iBAAmB,gBAAA,EAAkB,KAAA,EAAO,CAAA,EAAG,OAAA,EAAS,oBAAA,CAAqB,CAAA;EAC7E,GAAA,iBAAoB,gBAAA,EAAkB,KAAA,EAAO,CAAA,EAAG,OAAA,EAAS,oBAAA,CAAqB,CAAA;EA1E9E;;;;;AAe2B;AAS7B;;;;;;;;;;EAoEE,MAAA,cAAoB,MAAA,EAAQ,OAAA;IAC1B,WAAA;IACA,WAAA,EAAa,gBAAA,CAAiB,UAAA;EAAA,IAC5B,YAAA,CAAa,UAAA,EAAY,OAAA;AAAA"}

package/esm/errors/error-code.type.d.mts

@@ -13,7 +13,7 @@
  *   return agent.execute(input);
  * }
  */
-type AIErrorCode = "AGENT_EXEC_FAILED" | "AGENT_CANCELLED" | "AGENT_MAX_TRIPS" | "SCHEMA_VALIDATION_FAILED" | "TOOL_EXEC_FAILED" | "PROVIDER_ERROR" | "PROVIDER_RATE_LIMIT" | "PROVIDER_QUOTA_EXCEEDED" | "PROVIDER_TIMEOUT" | "CONTEXT_LENGTH_EXCEEDED" | "CONTENT_FILTER" | "PROVIDER_INVALID_REQUEST" | "PROVIDER_AUTH" | "BUDGET_EXCEEDED" | "GUARDRAIL_VIOLATION" | "WORKFLOW_ERROR" | "STEP_FAILED" | "WORKFLOW_DRIFT" | "WORKFLOW_CANCELLED" | "WORKFLOW_MAX_STEPS" | "WORKFLOW_INVALID_GOTO" | "SUPERVISOR_FAILED" | "SUPERVISOR_MAX_ITERATIONS" | "SUPERVISOR_INVALID_ROUTE" | "SUPERVISOR_CANCELLED" | "SUPERVISOR_DRIFT" | "SUPERVISOR_INTENT_DESCRIPTION_REQUIRED" | "SUPERVISOR_INTENT_MIXED_DISPATCH" | "SUPERVISOR_INTENT_STREAM_AND_OUTPUT" | "SUPERVISOR_INTENT_STREAM_TO_REQUIRED" | "SUPERVISOR_INTENT_STREAM_ON_WORKFLOW" | "SUPERVISOR_DISPATCH_CYCLE" | "ORCHESTRATOR_FAILED" | "ORCHESTRATOR_DRIFT" | "ORCHESTRATOR_CONFIG" | "ORCHESTRATOR_CANCELLED" | "PLANNER_FAILED" | "PLANNER_PLAN_INVALID" | "PLANNER_CANCELLED";
+type AIErrorCode = "AGENT_EXEC_FAILED" | "AGENT_CANCELLED" | "AGENT_MAX_TRIPS" | "SCHEMA_VALIDATION_FAILED" | "TOOL_EXEC_FAILED" | "PROVIDER_ERROR" | "PROVIDER_RATE_LIMIT" | "PROVIDER_QUOTA_EXCEEDED" | "PROVIDER_TIMEOUT" | "CONTEXT_LENGTH_EXCEEDED" | "CONTENT_FILTER" | "PROVIDER_INVALID_REQUEST" | "PROVIDER_AUTH" | "BUDGET_EXCEEDED" | "GUARDRAIL_VIOLATION" | "WORKFLOW_ERROR" | "STEP_FAILED" | "WORKFLOW_DRIFT" | "WORKFLOW_CANCELLED" | "WORKFLOW_MAX_STEPS" | "WORKFLOW_INVALID_GOTO" | "SUPERVISOR_FAILED" | "SUPERVISOR_MAX_ITERATIONS" | "SUPERVISOR_INVALID_ROUTE" | "SUPERVISOR_CANCELLED" | "SUPERVISOR_DRIFT" | "SUPERVISOR_INTENT_DESCRIPTION_REQUIRED" | "SUPERVISOR_INTENT_MIXED_DISPATCH" | "SUPERVISOR_INTENT_STREAM_AND_OUTPUT" | "SUPERVISOR_INTENT_STREAM_TO_REQUIRED" | "SUPERVISOR_INTENT_STREAM_ON_WORKFLOW" | "SUPERVISOR_DISPATCH_CYCLE" | "ORCHESTRATOR_FAILED" | "ORCHESTRATOR_DRIFT" | "ORCHESTRATOR_CONFIG" | "ORCHESTRATOR_CANCELLED" | "PLANNER_FAILED" | "PLANNER_PLAN_INVALID" | "PLANNER_CANCELLED" | "VCR_CASSETTE_MISS" | "OUTBOUND_POLICY_BLOCKED";
 //#endregion
 export { AIErrorCode };
 //# sourceMappingURL=error-code.type.d.mts.map

package/esm/errors/index.d.mts

@@ -18,6 +18,7 @@ import { OrchestratorFailedError } from "./orchestrator-failed-error.mjs";
 import { OrchestratorCancelledError, OrchestratorCancelledErrorOptions } from "./orchestrator-cancelled-error.mjs";
 import { OrchestratorConfigError } from "./orchestrator-config-error.mjs";
 import { OrchestratorDriftError, OrchestratorDriftErrorOptions } from "./orchestrator-drift-error.mjs";
+import { OutboundPolicyError } from "./outbound-policy-error.mjs";
 import { PlannerFailedError } from "./planner-failed-error.mjs";
 import { PlannerCancelledError, PlannerCancelledErrorOptions } from "./planner-cancelled-error.mjs";
 import { PlannerPlanInvalidError } from "./planner-plan-invalid-error.mjs";

package/esm/errors/index.mjs

@@ -16,6 +16,7 @@ import { OrchestratorFailedError } from "./orchestrator-failed-error.mjs";
 import { OrchestratorCancelledError } from "./orchestrator-cancelled-error.mjs";
 import { OrchestratorConfigError } from "./orchestrator-config-error.mjs";
 import { OrchestratorDriftError } from "./orchestrator-drift-error.mjs";
+import { OutboundPolicyError } from "./outbound-policy-error.mjs";
 import { PlannerFailedError } from "./planner-failed-error.mjs";
 import { PlannerCancelledError } from "./planner-cancelled-error.mjs";
 import { PlannerPlanInvalidError } from "./planner-plan-invalid-error.mjs";

package/esm/errors/outbound-policy-error.d.mts

@@ -0,0 +1,27 @@
+import { ErrorCategory } from "./error-category.type.mjs";
+import { AIError, AIErrorOptions } from "./ai-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/outbound-policy-error.d.ts
+/**
+ * A server-side outbound request (attachment fetch, URL skill manifest,
+ * RAG loader, …) was blocked by the shared `OutboundPolicy` before it
+ * left the process — a disallowed scheme, a host outside the allowlist, a
+ * private / loopback / link-local / cloud-metadata address resolved after
+ * DNS, an oversized body, or a timeout.
+ *
+ * This is the framework's SSRF / resource-exhaustion guard surfacing: the
+ * request was refused on purpose, not a provider failure. `context`
+ * carries the offending `url` / `host` / `reason` for logs.
+ *
+ * @example
+ * if (error instanceof OutboundPolicyError) {
+ *   logger.warn("blocked outbound fetch", { context: error.context });
+ * }
+ */
+declare class OutboundPolicyError extends AIError {
+  static readonly defaultCategory: ErrorCategory;
+  constructor(message: string, options?: AIErrorOptions);
+}
+//#endregion
+export { OutboundPolicyError };
+//# sourceMappingURL=outbound-policy-error.d.mts.map

package/esm/errors/outbound-policy-error.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbound-policy-error.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/outbound-policy-error.ts"],"mappings":";;;;;;AAmBA;;;;;;;;;;;;;;cAAa,mBAAA,SAA4B,OAAA;EAAA,gBAChB,eAAA,EAAiB,aAAA;cAErB,OAAA,UAAiB,OAAA,GAAU,cAAA;AAAA"}

package/esm/errors/outbound-policy-error.mjs

@@ -0,0 +1,32 @@
+import { AIError } from "./ai-error.mjs";
+
+//#region ../@warlock.js/ai/src/errors/outbound-policy-error.ts
+/**
+* A server-side outbound request (attachment fetch, URL skill manifest,
+* RAG loader, …) was blocked by the shared `OutboundPolicy` before it
+* left the process — a disallowed scheme, a host outside the allowlist, a
+* private / loopback / link-local / cloud-metadata address resolved after
+* DNS, an oversized body, or a timeout.
+*
+* This is the framework's SSRF / resource-exhaustion guard surfacing: the
+* request was refused on purpose, not a provider failure. `context`
+* carries the offending `url` / `host` / `reason` for logs.
+*
+* @example
+* if (error instanceof OutboundPolicyError) {
+*   logger.warn("blocked outbound fetch", { context: error.context });
+* }
+*/
+var OutboundPolicyError = class extends AIError {
+	static {
+		this.defaultCategory = "validation";
+	}
+	constructor(message, options) {
+		super("OUTBOUND_POLICY_BLOCKED", message, options);
+		this.name = "OutboundPolicyError";
+	}
+};
+
+//#endregion
+export { OutboundPolicyError };
+//# sourceMappingURL=outbound-policy-error.mjs.map

package/esm/errors/outbound-policy-error.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"outbound-policy-error.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/errors/outbound-policy-error.ts"],"sourcesContent":["import { AIError, type AIErrorOptions } from \"./ai-error\";\nimport type { ErrorCategory } from \"./error-category.type\";\n\n/**\n * A server-side outbound request (attachment fetch, URL skill manifest,\n * RAG loader, …) was blocked by the shared `OutboundPolicy` before it\n * left the process — a disallowed scheme, a host outside the allowlist, a\n * private / loopback / link-local / cloud-metadata address resolved after\n * DNS, an oversized body, or a timeout.\n *\n * This is the framework's SSRF / resource-exhaustion guard surfacing: the\n * request was refused on purpose, not a provider failure. `context`\n * carries the offending `url` / `host` / `reason` for logs.\n *\n * @example\n * if (error instanceof OutboundPolicyError) {\n *   logger.warn(\"blocked outbound fetch\", { context: error.context });\n * }\n */\nexport class OutboundPolicyError extends AIError {\n  public static readonly defaultCategory: ErrorCategory = \"validation\";\n\n  public constructor(message: string, options?: AIErrorOptions) {\n    super(\"OUTBOUND_POLICY_BLOCKED\", message, options);\n    this.name = \"OutboundPolicyError\";\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmBA,IAAa,sBAAb,cAAyC,QAAQ;;yBACS;;CAExD,AAAO,YAAY,SAAiB,SAA0B;EAC5D,MAAM,2BAA2B,SAAS,OAAO;EACjD,KAAK,OAAO;CACd;AACF"}

package/esm/eval/dataset.d.mts

@@ -0,0 +1,28 @@
+import { DatasetContract, DatasetOptions } from "./dataset.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/dataset.d.ts
+/**
+ * Create an immutable evaluation dataset that feeds `agent.eval({ cases })`
+ * directly.
+ *
+ * **Role.** A taggable, filterable, shardable wrapper around a list of
+ * {@link DatasetEntry} rows. `agent.eval` accepts a `DatasetContract` in
+ * place of a raw `EvalCase[]`, reading `.cases` off it.
+ *
+ * Sources (combinable — file entries append after inline `cases`):
+ * - `cases` → inline entries.
+ * - `fromFile` → a JSONL file read once, synchronously, at construction
+ *   (one JSON object per line). A malformed line throws an
+ *   `InvalidRequestError` naming the 1-based line number.
+ *
+ * @example
+ * const ds = dataset({ name: "support", fromFile: "./eval/support.jsonl" });
+ * const smoke = ds.filter((entry) => entry.tags?.includes("smoke"));
+ * const shard = ds.shard(0, 4); // first of four parallel CI shards
+ *
+ * const report = await agent.eval({ cases: ds, scorers: [contains()] });
+ */
+declare function dataset<TOutput = unknown>(options: DatasetOptions<TOutput>): DatasetContract<TOutput>;
+//#endregion
+export { dataset };
+//# sourceMappingURL=dataset.d.mts.map

package/esm/eval/dataset.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/dataset.ts"],"mappings":";;;;;AAkIA;;;;;;;;;;;;;;;;AAE0B;;;iBAFV,OAAA,oBACd,OAAA,EAAS,cAAA,CAAe,OAAA,IACvB,eAAA,CAAgB,OAAA"}

package/esm/eval/dataset.mjs

@@ -0,0 +1,112 @@
+import { InvalidRequestError } from "../errors/invalid-request-error.mjs";
+import "../errors/index.mjs";
+import { readFileSync } from "node:fs";
+
+//#region ../@warlock.js/ai/src/eval/dataset.ts
+/**
+* Parse a JSONL file's contents into {@link DatasetEntry} rows. Blank
+* lines (and trailing whitespace-only lines) are skipped; every other
+* line must be a JSON object. A malformed line throws an
+* `InvalidRequestError` naming the 1-based line number — failing loud at
+* construction, like `SystemPrompt.fromFile`.
+*/
+function parseJsonl(path, contents) {
+	const entries = [];
+	const lines = contents.split(/\r?\n/);
+	for (let index = 0; index < lines.length; index++) {
+		const line = lines[index].trim();
+		if (line === "") continue;
+		let parsed;
+		try {
+			parsed = JSON.parse(line);
+		} catch (error) {
+			throw new InvalidRequestError(`Failed to parse dataset file "${path}" — line ${index + 1} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, {
+				context: {
+					path,
+					line: index + 1
+				},
+				cause: error
+			});
+		}
+		if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new InvalidRequestError(`Failed to parse dataset file "${path}" — line ${index + 1} is not a JSON object`, { context: {
+			path,
+			line: index + 1
+		} });
+		entries.push(parsed);
+	}
+	return entries;
+}
+/**
+* Read a JSONL dataset file once, synchronously, at construction.
+* Mirrors `SystemPrompt.fromFile`: a read failure (missing path,
+* permission denied) throws an `InvalidRequestError` surfacing the
+* underlying cause.
+*/
+function readDatasetFile(path) {
+	let contents;
+	try {
+		contents = readFileSync(path, "utf8");
+	} catch (error) {
+		throw new InvalidRequestError(`Failed to read dataset file "${path}" — ${error instanceof Error ? error.message : String(error)}`, {
+			context: { path },
+			cause: error
+		});
+	}
+	return parseJsonl(path, contents);
+}
+/**
+* Build an immutable {@link DatasetContract} over the given entries.
+* Shared by the {@link dataset} factory and by `filter` / `shard`, which
+* each return a fresh dataset built from a derived case list.
+*/
+function makeDataset(name, cases) {
+	return {
+		name,
+		cases,
+		filter(predicate) {
+			return makeDataset(name, cases.filter(predicate));
+		},
+		shard(index, total) {
+			if (!Number.isInteger(total) || total <= 0) throw new InvalidRequestError(`dataset.shard: "total" must be a positive integer, received ${total}`, { context: {
+				name,
+				total
+			} });
+			if (!Number.isInteger(index) || index < 0 || index >= total) throw new InvalidRequestError(`dataset.shard: "index" must be an integer in [0, ${total}), received ${index}`, { context: {
+				name,
+				index,
+				total
+			} });
+			return makeDataset(name, cases.filter((_, position) => position % total === index));
+		}
+	};
+}
+/**
+* Create an immutable evaluation dataset that feeds `agent.eval({ cases })`
+* directly.
+*
+* **Role.** A taggable, filterable, shardable wrapper around a list of
+* {@link DatasetEntry} rows. `agent.eval` accepts a `DatasetContract` in
+* place of a raw `EvalCase[]`, reading `.cases` off it.
+*
+* Sources (combinable — file entries append after inline `cases`):
+* - `cases` → inline entries.
+* - `fromFile` → a JSONL file read once, synchronously, at construction
+*   (one JSON object per line). A malformed line throws an
+*   `InvalidRequestError` naming the 1-based line number.
+*
+* @example
+* const ds = dataset({ name: "support", fromFile: "./eval/support.jsonl" });
+* const smoke = ds.filter((entry) => entry.tags?.includes("smoke"));
+* const shard = ds.shard(0, 4); // first of four parallel CI shards
+*
+* const report = await agent.eval({ cases: ds, scorers: [contains()] });
+*/
+function dataset(options) {
+	const cases = [...options.cases ?? []];
+	if (options.fromFile !== void 0) cases.push(...readDatasetFile(options.fromFile));
+	return makeDataset(options.name, cases);
+}
+
+//#endregion
+export { dataset };
+//# sourceMappingURL=dataset.mjs.map

package/esm/eval/dataset.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.mjs","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/dataset.ts"],"sourcesContent":["import { readFileSync } from \"node:fs\";\nimport type { DatasetContract, DatasetEntry, DatasetOptions } from \"./dataset.type\";\nimport { InvalidRequestError } from \"../errors\";\n\n/**\n * Parse a JSONL file's contents into {@link DatasetEntry} rows. Blank\n * lines (and trailing whitespace-only lines) are skipped; every other\n * line must be a JSON object. A malformed line throws an\n * `InvalidRequestError` naming the 1-based line number — failing loud at\n * construction, like `SystemPrompt.fromFile`.\n */\nfunction parseJsonl<TOutput>(path: string, contents: string): DatasetEntry<TOutput>[] {\n  const entries: DatasetEntry<TOutput>[] = [];\n  const lines = contents.split(/\\r?\\n/);\n\n  for (let index = 0; index < lines.length; index++) {\n    const line = lines[index].trim();\n\n    if (line === \"\") {\n      continue;\n    }\n\n    let parsed: unknown;\n\n    try {\n      parsed = JSON.parse(line);\n    } catch (error) {\n      throw new InvalidRequestError(\n        `Failed to parse dataset file \"${path}\" — line ${index + 1} is not valid JSON: ${\n          error instanceof Error ? error.message : String(error)\n        }`,\n        { context: { path, line: index + 1 }, cause: error },\n      );\n    }\n\n    if (parsed === null || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n      throw new InvalidRequestError(\n        `Failed to parse dataset file \"${path}\" — line ${index + 1} is not a JSON object`,\n        { context: { path, line: index + 1 } },\n      );\n    }\n\n    entries.push(parsed as DatasetEntry<TOutput>);\n  }\n\n  return entries;\n}\n\n/**\n * Read a JSONL dataset file once, synchronously, at construction.\n * Mirrors `SystemPrompt.fromFile`: a read failure (missing path,\n * permission denied) throws an `InvalidRequestError` surfacing the\n * underlying cause.\n */\nfunction readDatasetFile<TOutput>(path: string): DatasetEntry<TOutput>[] {\n  let contents: string;\n\n  try {\n    contents = readFileSync(path, \"utf8\");\n  } catch (error) {\n    throw new InvalidRequestError(\n      `Failed to read dataset file \"${path}\" — ${\n        error instanceof Error ? error.message : String(error)\n      }`,\n      { context: { path }, cause: error },\n    );\n  }\n\n  return parseJsonl<TOutput>(path, contents);\n}\n\n/**\n * Build an immutable {@link DatasetContract} over the given entries.\n * Shared by the {@link dataset} factory and by `filter` / `shard`, which\n * each return a fresh dataset built from a derived case list.\n */\nfunction makeDataset<TOutput>(\n  name: string,\n  cases: DatasetEntry<TOutput>[],\n): DatasetContract<TOutput> {\n  return {\n    name,\n    cases,\n    filter(predicate) {\n      return makeDataset(name, cases.filter(predicate));\n    },\n    shard(index, total) {\n      if (!Number.isInteger(total) || total <= 0) {\n        throw new InvalidRequestError(\n          `dataset.shard: \"total\" must be a positive integer, received ${total}`,\n          { context: { name, total } },\n        );\n      }\n\n      if (!Number.isInteger(index) || index < 0 || index >= total) {\n        throw new InvalidRequestError(\n          `dataset.shard: \"index\" must be an integer in [0, ${total}), received ${index}`,\n          { context: { name, index, total } },\n        );\n      }\n\n      return makeDataset(\n        name,\n        cases.filter((_, position) => position % total === index),\n      );\n    },\n  };\n}\n\n/**\n * Create an immutable evaluation dataset that feeds `agent.eval({ cases })`\n * directly.\n *\n * **Role.** A taggable, filterable, shardable wrapper around a list of\n * {@link DatasetEntry} rows. `agent.eval` accepts a `DatasetContract` in\n * place of a raw `EvalCase[]`, reading `.cases` off it.\n *\n * Sources (combinable — file entries append after inline `cases`):\n * - `cases` → inline entries.\n * - `fromFile` → a JSONL file read once, synchronously, at construction\n *   (one JSON object per line). A malformed line throws an\n *   `InvalidRequestError` naming the 1-based line number.\n *\n * @example\n * const ds = dataset({ name: \"support\", fromFile: \"./eval/support.jsonl\" });\n * const smoke = ds.filter((entry) => entry.tags?.includes(\"smoke\"));\n * const shard = ds.shard(0, 4); // first of four parallel CI shards\n *\n * const report = await agent.eval({ cases: ds, scorers: [contains()] });\n */\nexport function dataset<TOutput = unknown>(\n  options: DatasetOptions<TOutput>,\n): DatasetContract<TOutput> {\n  const cases: DatasetEntry<TOutput>[] = [...(options.cases ?? [])];\n\n  if (options.fromFile !== undefined) {\n    cases.push(...readDatasetFile<TOutput>(options.fromFile));\n  }\n\n  return makeDataset(options.name, cases);\n}\n"],"mappings":";;;;;;;;;;;;AAWA,SAAS,WAAoB,MAAc,UAA2C;CACpF,MAAM,UAAmC,CAAC;CAC1C,MAAM,QAAQ,SAAS,MAAM,OAAO;CAEpC,KAAK,IAAI,QAAQ,GAAG,QAAQ,MAAM,QAAQ,SAAS;EACjD,MAAM,OAAO,MAAM,MAAM,CAAC,KAAK;EAE/B,IAAI,SAAS,IACX;EAGF,IAAI;EAEJ,IAAI;GACF,SAAS,KAAK,MAAM,IAAI;EAC1B,SAAS,OAAO;GACd,MAAM,IAAI,oBACR,iCAAiC,KAAK,WAAW,QAAQ,EAAE,sBACzD,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAEvD;IAAE,SAAS;KAAE;KAAM,MAAM,QAAQ;IAAE;IAAG,OAAO;GAAM,CACrD;EACF;EAEA,IAAI,WAAW,QAAQ,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GACvE,MAAM,IAAI,oBACR,iCAAiC,KAAK,WAAW,QAAQ,EAAE,wBAC3D,EAAE,SAAS;GAAE;GAAM,MAAM,QAAQ;EAAE,EAAE,CACvC;EAGF,QAAQ,KAAK,MAA+B;CAC9C;CAEA,OAAO;AACT;;;;;;;AAQA,SAAS,gBAAyB,MAAuC;CACvE,IAAI;CAEJ,IAAI;EACF,WAAW,aAAa,MAAM,MAAM;CACtC,SAAS,OAAO;EACd,MAAM,IAAI,oBACR,gCAAgC,KAAK,MACnC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAEvD;GAAE,SAAS,EAAE,KAAK;GAAG,OAAO;EAAM,CACpC;CACF;CAEA,OAAO,WAAoB,MAAM,QAAQ;AAC3C;;;;;;AAOA,SAAS,YACP,MACA,OAC0B;CAC1B,OAAO;EACL;EACA;EACA,OAAO,WAAW;GAChB,OAAO,YAAY,MAAM,MAAM,OAAO,SAAS,CAAC;EAClD;EACA,MAAM,OAAO,OAAO;GAClB,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,SAAS,GACvC,MAAM,IAAI,oBACR,+DAA+D,SAC/D,EAAE,SAAS;IAAE;IAAM;GAAM,EAAE,CAC7B;GAGF,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,KAAK,SAAS,OACpD,MAAM,IAAI,oBACR,oDAAoD,MAAM,cAAc,SACxE,EAAE,SAAS;IAAE;IAAM;IAAO;GAAM,EAAE,CACpC;GAGF,OAAO,YACL,MACA,MAAM,QAAQ,GAAG,aAAa,WAAW,UAAU,KAAK,CAC1D;EACF;CACF;AACF;;;;;;;;;;;;;;;;;;;;;;AAuBA,SAAgB,QACd,SAC0B;CAC1B,MAAM,QAAiC,CAAC,GAAI,QAAQ,SAAS,CAAC,CAAE;CAEhE,IAAI,QAAQ,aAAa,QACvB,MAAM,KAAK,GAAG,gBAAyB,QAAQ,QAAQ,CAAC;CAG1D,OAAO,YAAY,QAAQ,MAAM,KAAK;AACxC"}

package/esm/eval/dataset.type.d.mts

@@ -0,0 +1,53 @@
+import { EvalCase } from "../contracts/agent/eval.type.mjs";
+
+//#region ../@warlock.js/ai/src/eval/dataset.type.d.ts
+/**
+ * A single dataset row. A superset of {@link EvalCase} — every dataset
+ * entry is a valid eval case, plus optional `tags` for filtering and
+ * sharding a large suite.
+ */
+type DatasetEntry<TOutput = unknown> = EvalCase<TOutput> & {
+  /** Optional tags for filtering / sharding a large dataset. */tags?: string[];
+};
+/**
+ * An immutable, filterable, shardable collection of {@link DatasetEntry}
+ * rows that feeds `agent.eval({ cases })` directly. Built via the
+ * {@link dataset} factory — never `new`.
+ */
+type DatasetContract<TOutput = unknown> = {
+  /** Stable identifier for the dataset — surfaced for logging / artifacts. */readonly name: string; /** All entries (after any filter / shard applied at construction). */
+  readonly cases: DatasetEntry<TOutput>[];
+  /**
+   * Narrow by an arbitrary predicate (typically a tag check) — returns a
+   * new dataset sharing nothing mutable with this one.
+   */
+  filter(predicate: (entry: DatasetEntry<TOutput>) => boolean): DatasetContract<TOutput>;
+  /**
+   * Deterministic shard `index`-of-`total` for parallel CI jobs. Every
+   * entry lands in exactly one shard (round-robin by position), so the
+   * union of all `total` shards reproduces the full case list with no
+   * gaps or overlaps.
+   */
+  shard(index: number, total: number): DatasetContract<TOutput>;
+};
+/**
+ * Options for the {@link dataset} factory. Supply `cases` inline, or
+ * `fromFile` to load a JSONL file (one JSON-encoded {@link DatasetEntry}
+ * per line) synchronously at construction — mirroring
+ * `SystemPrompt.fromFile`'s one-shot sync read. Both may be combined;
+ * file entries are appended after inline `cases`.
+ */
+type DatasetOptions<TOutput = unknown> = {
+  /** Stable identifier for the dataset. */name: string; /** Inline entries. */
+  cases?: DatasetEntry<TOutput>[];
+  /**
+   * Path to a JSONL file read once, synchronously, at construction. Each
+   * non-blank line must be a JSON object matching {@link DatasetEntry}; a
+   * malformed line throws an `InvalidRequestError` naming the 1-based line
+   * number.
+   */
+  fromFile?: string;
+};
+//#endregion
+export { DatasetContract, DatasetEntry, DatasetOptions };
+//# sourceMappingURL=dataset.type.d.mts.map

package/esm/eval/dataset.type.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.type.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/dataset.type.ts"],"mappings":";;;;;AAOA;;;KAAY,YAAA,sBAAkC,QAAQ,CAAC,OAAA;EAA9B,8DAEvB,IAAA;AAAA;;;AAAI;AAQN;;KAAY,eAAA;EAImB,qFAFpB,IAAA,UAQyB;EAAA,SANzB,KAAA,EAAO,YAAA,CAAa,OAAA;EAOV;;;;EAFnB,MAAA,CACE,SAAA,GAAY,KAAA,EAAO,YAAA,CAAa,OAAA,gBAC/B,eAAA,CAAgB,OAAA;EAOiC;;;;;;EAApD,KAAA,CAAM,KAAA,UAAe,KAAA,WAAgB,eAAA,CAAgB,OAAA;AAAA;;;;;;;;KAU3C,cAAA;EAV2B,yCAYrC,IAAA,UAZ4D;EAc5D,KAAA,GAAQ,YAAY,CAAC,OAAA;EAJX;;;;;;EAWV,QAAA;AAAA"}

package/esm/eval/eval-runner.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-runner.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"mappings":";;;;;;AAmIA;;;;;;;iBAAsB,OAAA,UACpB,KAAA,EAAO,aAAA,CAAc,OAAA,GACrB,OAAA,EAAS,WAAA,CAAY,OAAA,IACpB,OAAA,CAAQ,UAAA,CAAW,OAAA"}
+{"version":3,"file":"eval-runner.d.mts","names":[],"sources":["../../../../../../../@warlock.js/ai/src/eval/eval-runner.ts"],"mappings":";;;;;;AAqJA;;;;;;;iBAAsB,OAAA,UACpB,KAAA,EAAO,aAAA,CAAc,OAAA,GACrB,OAAA,EAAS,WAAA,CAAY,OAAA,IACpB,OAAA,CAAQ,UAAA,CAAW,OAAA"}

package/esm/eval/eval-runner.mjs

@@ -1,9 +1,20 @@
 import { AgentExecutionError } from "../errors/agent-execution-error.mjs";
 import "../errors/index.mjs";
 import { judge } from "./judge-scorer.mjs";
+import { diff } from "./regression.mjs";
 import { log } from "@warlock.js/logger";
 
 //#region ../@warlock.js/ai/src/eval/eval-runner.ts
+/**
+* Narrow `EvalOptions.cases` to the underlying `EvalCase[]`. A
+* `DatasetContract` is identified structurally by its `cases` property
+* (an array carried alongside `name` / `filter` / `shard`); a raw
+* `EvalCase[]` is used as-is.
+*/
+function resolveCases(cases) {
+	if (Array.isArray(cases)) return cases;
+	return cases.cases;
+}
 const LOG_MODULE = "ai.eval";
 const DEFAULT_PASS_THRESHOLD = .5;
 /**
@@ -86,8 +97,9 @@ async function runCase(agent, evalCase, options, passThreshold) {
 async function runEval(agent, options) {
 	const passThreshold = options.passThreshold ?? DEFAULT_PASS_THRESHOLD;
 	const start = performance.now();
+	const suiteCases = resolveCases(options.cases);
 	const cases = [];
-	for (const evalCase of options.cases) {
+	for (const evalCase of suiteCases) {
 		const caseResult = await runCase(agent, evalCase, options, passThreshold);
 		cases.push(caseResult);
 		if (!caseResult.passed && options.onFailure) try {
@@ -103,7 +115,7 @@ async function runEval(agent, options) {
 	const passedCount = cases.filter((entry) => entry.passed).length;
 	const total = cases.length;
 	const meanScore = total > 0 ? cases.reduce((sum, entry) => sum + entry.score, 0) / total : 0;
-	return {
+	const report = {
 		agentName: agent.name,
 		total,
 		passedCount,
@@ -114,6 +126,8 @@ async function runEval(agent, options) {
 		cases,
 		duration: performance.now() - start
 	};
+	if (options.baseline) report.regression = diff(report, options.baseline, options.tolerance);
+	return report;
 }
 
 //#endregion