@workglow/eval 0.3.48 → 0.4.0

package/dist/commands/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AA2C7C,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,EACrC,eAAe,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACnC,IAAI,CA2CN"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOzC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AA2C7C,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,EACrC,eAAe,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACnC,IAAI,CA2CN"}

package/dist/evals/EvalSweepTask.d.ts

@@ -0,0 +1,47 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import type { IExecuteContext } from "@workglow/task-graph";
+import { Task } from "@workglow/task-graph";
+import type { DataPortSchema } from "@workglow/util/schema";
+import type { DatasetRowRecord, EvalStores } from "../storage";
+import { type SweepOptions } from "./runner";
+export interface EvalSweepTaskOutput {
+    readonly run_id: string;
+    readonly rows: number;
+    readonly models: number;
+}
+/** An `interface` carries no index signature; the mapped form is what `DataPorts` accepts. */
+type Ports<T> = {
+    [K in keyof T]: T[K];
+};
+/**
+ * Runs one sweep as a task, so the run has a graph.
+ *
+ * The sweep is a nested loop over models and rows, and each row is its own
+ * small workflow — so putting every row through the CLI seam would report one
+ * RUN per row: fifteen hundred of them for a 500-row sweep of three models,
+ * which is not a graph anyone can read. One task for the sweep is the honest
+ * unit: it is what the operator started, and its progress is what they want to
+ * watch.
+ *
+ * The stores, rows and options are passed as construction state rather than as
+ * ports. They are live handles and parsed dataset records, not values a form
+ * could supply or a cache could serialize.
+ */
+export declare class EvalSweepTask extends Task<Record<string, never>, Ports<EvalSweepTaskOutput>> {
+    #private;
+    static readonly type = "EvalSweepTask";
+    static readonly category = "Eval";
+    static readonly title = "Run eval sweep";
+    static readonly cacheable = false;
+    static inputSchema(): DataPortSchema;
+    static outputSchema(): DataPortSchema;
+    /** Hands the task the live handles a port cannot carry. */
+    withSweep(stores: EvalStores, rows: readonly DatasetRowRecord[], options: SweepOptions): this;
+    execute(_input: Record<string, never>, context: IExecuteContext): Promise<Ports<EvalSweepTaskOutput>>;
+}
+export {};
+//# sourceMappingURL=EvalSweepTask.d.ts.map

package/dist/evals/EvalSweepTask.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalSweepTask.d.ts","sourceRoot":"","sources":["../../src/evals/EvalSweepTask.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAY,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAmBvD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,8FAA8F;AAC9F,KAAK,KAAK,CAAC,CAAC,IAAI;KAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAEzC;;;;;;;;;;;;;GAaG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;;IACxF,gBAAyB,IAAI,mBAAmB;IAChD,gBAAyB,QAAQ,UAAU;IAC3C,gBAAyB,KAAK,oBAAoB;IAClD,gBAAyB,SAAS,SAAS;IAE3C,OAAgB,WAAW,IAAI,cAAc,CAE5C;IACD,OAAgB,YAAY,IAAI,cAAc,CAE7C;IAMD,2DAA2D;IAC3D,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,gBAAgB,EAAE,EAAE,OAAO,EAAE,YAAY,GAAG,IAAI,CAK5F;IAEc,OAAO,CACpB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAC7B,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CA2BrC;CACF"}

package/dist/evals/runner.d.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import type { ModelPricing } from "@workglow/ai";
-import type { StreamEvent, Usage } from "@workglow/task-graph";
+import type { IExecuteContext, StreamEvent, Usage } from "@workglow/task-graph";
 import type { EvalKind } from "../models";
 import type { DatasetRowRecord, EvalStores } from "../storage";
 import type { ColumnOptions, DatasetContext } from "./types";
@@ -22,6 +22,12 @@ export interface SweepOptions {
      * pays nothing for it.
      */
     readonly onStreamChunk?: ((event: StreamEvent) => void) | undefined;
+    /**
+     * The sweep task's own context. Given one, each row's workflow is owned for
+     * its duration, so the run is a single task graph the progress surfaces can
+     * see into rather than a loop of invisible standalone runs.
+     */
+    readonly owner?: IExecuteContext | undefined;
 }
 /**
  * Map one run's token accounting onto the result row's columns.

package/dist/evals/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE/D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAI/D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAe,MAAM,SAAS,CAAC;AAE1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,UAAU,CAAC,EAChB,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,KAAK,IAAI,CAAC,GAC7F,SAAS,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;CACrE;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,KAAK,GAAG,SAAS,EACxB,OAAO,EAAE,YAAY,GAAG,SAAS,GAChC;IACD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAWA;AAoBD;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,SAAS,gBAAgB,EAAE,EACjC,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CA6EjB"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAEhF,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAI/D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAe,MAAM,SAAS,CAAC;AAE1E,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,QAAQ,CAAC,UAAU,CAAC,EAChB,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,KAAK,IAAI,CAAC,GAC7F,SAAS,CAAC;IACd;;;;OAIG;IACH,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IACpE;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,GAAG,SAAS,CAAC;CAC9C;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,KAAK,GAAG,SAAS,EACxB,OAAO,EAAE,YAAY,GAAG,SAAS,GAChC;IACD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAWA;AAoBD;;;;;GAKG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,SAAS,gBAAgB,EAAE,EACjC,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,MAAM,CAAC,CAmFjB"}

package/dist/evals/streamSubscribe.d.ts

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 import type { StreamEvent, Workflow } from "@workglow/task-graph";
+import type { RowOwner } from "./types";
 /**
  * Run a workflow, forwarding its `stream_chunk` events to `onStreamChunk` for
  * the duration of the run only.
@@ -11,5 +12,5 @@ import type { StreamEvent, Workflow } from "@workglow/task-graph";
  * When no listener is given, nothing is subscribed at all — a 500-row sweep
  * without `--stream` pays nothing for it.
  */
-export declare function runWithStreamChunks<Output>(workflow: Workflow, onStreamChunk: ((event: StreamEvent) => void) | undefined): Promise<Output>;
+export declare function runWithStreamChunks<Output>(workflow: Workflow, onStreamChunk: ((event: StreamEvent) => void) | undefined, owner?: RowOwner | undefined): Promise<Output>;
 //# sourceMappingURL=streamSubscribe.d.ts.map

package/dist/evals/streamSubscribe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streamSubscribe.d.ts","sourceRoot":"","sources":["../../src/evals/streamSubscribe.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAc,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAE9E;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAC9C,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,GACxD,OAAO,CAAC,MAAM,CAAC,CAUjB"}
+{"version":3,"file":"streamSubscribe.d.ts","sourceRoot":"","sources":["../../src/evals/streamSubscribe.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAc,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC9E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAExC;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAC9C,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,CAAC,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,GAAG,SAAS,EACzD,KAAK,CAAC,EAAE,QAAQ,GAAG,SAAS,GAC3B,OAAO,CAAC,MAAM,CAAC,CAgBjB"}

package/dist/evals/types.d.ts

@@ -3,7 +3,7 @@
  * Copyright 2026 Steven Roussey <sroussey@gmail.com>
  * SPDX-License-Identifier: Apache-2.0
  */
-import type { StreamEvent, Usage } from "@workglow/task-graph";
+import type { IExecuteContext, StreamEvent, Usage } from "@workglow/task-graph";
 import type { DatasetRow, LabelNames } from "../hf/types";
 /** What one workflow execution produced for one dataset row. */
 export interface RowPrediction {
@@ -20,7 +20,21 @@ export interface RowPrediction {
  * the life of the call only — `--stream` is the only caller that passes it, so
  * an ordinary sweep never pays for the subscription.
  */
-export type RowExecutor = (row: DatasetRow, onStreamChunk?: (event: StreamEvent) => void) => Promise<RowPrediction>;
+/**
+ * How a row's work joins the run's single task graph.
+ *
+ * Each row builds its own small workflow. Run standalone those are invisible:
+ * the run is one graph, and anything executed outside it is work no progress
+ * surface can see. Handing the executor the sweep task's context lets it OWN
+ * that workflow for the row's duration, so the row appears as a live child and
+ * leaves when it finishes.
+ */
+export interface RowOwner {
+    readonly context: IExecuteContext;
+    /** Row label, e.g. `claude-haiku-4-5 · row 12`. */
+    readonly title: string;
+}
+export type RowExecutor = (row: DatasetRow, onStreamChunk?: (event: StreamEvent) => void, owner?: RowOwner) => Promise<RowPrediction>;
 export interface ColumnOptions {
     readonly textColumn: string;
     /** classify: gold label column. */

package/dist/evals/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE1D,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;CACpC;AAED;;;;;;GAMG;AACH,MAAM,MAAM,WAAW,GAAG,CACxB,GAAG,EAAE,UAAU,EACf,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,KACzC,OAAO,CAAC,aAAa,CAAC,CAAC;AAE5B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oFAAoF;IACpF,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD,uDAAuD;IACvD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CACjC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/evals/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAChF,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE1D,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5C,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;CACpC;AAED;;;;;;GAMG;AACH;;;;;;;;GAQG;AACH,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAClC,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,GAAG,EAAE,UAAU,EACf,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,EAC5C,KAAK,CAAC,EAAE,QAAQ,KACb,OAAO,CAAC,aAAa,CAAC,CAAC;AAE5B,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oFAAoF;IACpF,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD,uDAAuD;IACvD,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CACjC"}

package/dist/test/evalSweepTask.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};
+//# sourceMappingURL=evalSweepTask.test.d.ts.map

package/dist/test/evalSweepTask.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalSweepTask.test.d.ts","sourceRoot":"","sources":["../../src/test/evalSweepTask.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/workglow-eval.js

@@ -2,6 +2,7 @@
 // @bun
 
 // src/workglow-eval.ts
+import { registerWebCommand } from "@workglow/cli";
 import { program } from "commander";
 
 // src/hf/auth.ts
@@ -698,6 +699,13 @@ function registerReportCommand(program, openStores) {
   });
 }
 
+// src/commands/run.ts
+import { withCli } from "@workglow/cli";
+import { Workflow as Workflow4 } from "@workglow/task-graph";
+
+// src/evals/EvalSweepTask.ts
+import { Task } from "@workglow/task-graph";
+
 // src/evals/runner.ts
 import { estimateCost } from "@workglow/ai";
 
@@ -974,13 +982,17 @@ ${fence}`;
 }
 
 // src/evals/streamSubscribe.ts
-async function runWithStreamChunks(workflow, onStreamChunk) {
+async function runWithStreamChunks(workflow, onStreamChunk, owner) {
   const listener = onStreamChunk ? (_taskId, event) => onStreamChunk(event) : undefined;
   if (listener)
     workflow.on("stream_chunk", listener);
+  if (owner)
+    owner.context.own(workflow, { title: owner.title });
   try {
     return await workflow.run();
   } finally {
+    if (owner)
+      owner.context.disown(workflow);
     if (listener)
       workflow.off("stream_chunk", listener);
   }
@@ -1025,7 +1037,7 @@ function makeClassifyExecutor(model, options, context) {
     required: ["label"],
     additionalProperties: false
   };
-  return async (row, onStreamChunk) => {
+  return async (row, onStreamChunk, owner) => {
     const text = String(row[options.textColumn] ?? "");
     const workflow = new Workflow;
     workflow.structuredGeneration({
@@ -1035,7 +1047,7 @@ function makeClassifyExecutor(model, options, context) {
       temperature: 0,
       maxTokens: 256
     });
-    const output = await runWithStreamChunks(workflow, onStreamChunk);
+    const output = await runWithStreamChunks(workflow, onStreamChunk, owner);
     return {
       expected: expectedLabel(row, options.labelColumn, context.labelNames, labels),
       predicted: String(output.object?.label ?? ""),
@@ -1112,7 +1124,7 @@ function makeExtractExecutor(model, options, context) {
   const instruction = options.instruction ?? DEFAULT_INSTRUCTION;
   const explicitFields = options.fields && options.fields.length > 0 ? resolveExtractionFields([], options.keyField, options.fields) : undefined;
   const explicitSchema = explicitFields ? buildItemsSchema(explicitFields, options.keyField) : undefined;
-  return async (row, onStreamChunk) => {
+  return async (row, onStreamChunk, owner) => {
     const text = String(row[options.textColumn] ?? "");
     const expectedRows = parseExpectedRows(row[options.expectedColumn], options.expectedColumn);
     const fields = explicitFields ?? resolveExtractionFields(expectedRows, options.keyField, undefined);
@@ -1125,7 +1137,7 @@ function makeExtractExecutor(model, options, context) {
       temperature: 0,
       maxTokens: 2048
     });
-    const output = await runWithStreamChunks(workflow, onStreamChunk);
+    const output = await runWithStreamChunks(workflow, onStreamChunk, owner);
     const items = Array.isArray(output.object?.items) ? output.object.items : [];
     return {
       expected: JSON.stringify(expectedRows),
@@ -1144,12 +1156,12 @@ function makeSimilarityExecutor(model, options, context) {
       throw new Error(`dataset has no column "${column}" (columns: ${context.columns.join(", ")}) \u2014 ` + `set --text-column/--pair-column/--score-column`);
     }
   }
-  return async (row, onStreamChunk) => {
+  return async (row, onStreamChunk, owner) => {
     const a = String(row[options.textColumn] ?? "");
     const b = String(row[options.pairColumn] ?? "");
     const workflow = new Workflow3;
     workflow.textEmbedding({ model, text: [a, b] });
-    const output = await runWithStreamChunks(workflow, onStreamChunk);
+    const output = await runWithStreamChunks(workflow, onStreamChunk, owner);
     const vectors = Array.isArray(output.vector) ? output.vector : [output.vector];
     if (vectors.length < 2) {
       throw new Error(`expected 2 embeddings, got ${vectors.length}`);
@@ -1230,7 +1242,7 @@ async function runSweep(stores, rows, options) {
         outcome = failedOutcome(parseError ?? "invalid stored row");
       } else {
         try {
-          const prediction = await executor(row, options.onStreamChunk);
+          const prediction = await executor(row, options.onStreamChunk, options.owner ? { context: options.owner, title: `${modelId} \xB7 row ${record.row_index}` } : undefined);
           outcomeUsage = prediction.usage;
           outcome = {
             ok: 1,
@@ -1270,6 +1282,61 @@ function failedOutcome(error) {
   };
 }
 
+// src/evals/EvalSweepTask.ts
+var INPUT = {
+  type: "object",
+  properties: {},
+  additionalProperties: false
+};
+var OUTPUT = {
+  type: "object",
+  properties: {
+    run_id: { type: "string", title: "Run id" },
+    rows: { type: "number", title: "Dataset rows" },
+    models: { type: "number", title: "Models swept" }
+  },
+  required: ["run_id"],
+  additionalProperties: false
+};
+
+class EvalSweepTask extends Task {
+  static type = "EvalSweepTask";
+  static category = "Eval";
+  static title = "Run eval sweep";
+  static cacheable = false;
+  static inputSchema() {
+    return INPUT;
+  }
+  static outputSchema() {
+    return OUTPUT;
+  }
+  #stores;
+  #rows = [];
+  #options;
+  withSweep(stores, rows, options) {
+    this.#stores = stores;
+    this.#rows = rows;
+    this.#options = options;
+    return this;
+  }
+  async execute(_input, context) {
+    const stores = this.#stores;
+    const options = this.#options;
+    if (!stores || !options)
+      throw new Error("EvalSweepTask.withSweep was never called");
+    await context.updateProgress(0, `${this.#rows.length} rows \xD7 ${options.models.length} model(s)`);
+    const run_id = await runSweep(stores, this.#rows, {
+      ...options,
+      owner: context,
+      onProgress: (done, total, model, ok, usage) => {
+        options.onProgress?.(done, total, model, ok, usage);
+        context.updateProgress(total > 0 ? done / total * 100 : 0, `${model} \xB7 ${done}/${total}`);
+      }
+    });
+    return { run_id, rows: this.#rows.length, models: options.models.length };
+  }
+}
+
 // src/report/liveTally.ts
 import { estimateCost as estimateCost2, formatCost, formatUsage } from "@workglow/ai";
 import { mergeUsage } from "@workglow/task-graph";
@@ -1386,7 +1453,7 @@ async function runEval(kind, flags, stores) {
     if (event.type === "text-delta")
       process.stderr.write(event.textDelta);
   } : undefined;
-  const runId = await runSweep(stores, limited, {
+  const sweep = new EvalSweepTask({ title: `${kind} \xB7 ${flags.dataset}` }).withSweep(stores, limited, {
     kind,
     dataset: flags.dataset,
     split: flags.split,
@@ -1400,6 +1467,10 @@ async function runEval(kind, flags, stores) {
     },
     onStreamChunk
   });
+  const workflow = new Workflow4;
+  workflow.pipe(sweep);
+  const output = await withCli(workflow, { interactive: false }).run();
+  const runId = output.run_id;
   console.error(`run ${runId} complete
 `);
   await printReport(stores, runId, flags.format);
@@ -1546,9 +1617,10 @@ var stores;
 var openStores = () => stores ??= createSqliteStores(config);
 var providers;
 var ensureProviders = () => providers ??= registerEvalProviders(config);
-program.version("2.0.0").description("Workglow eval example \u2014 pull HuggingFace datasets into storage, run task workflows " + "across models, and score the stored results");
+program.name("workglow-eval").version("2.0.0").description("Workglow eval example \u2014 pull HuggingFace datasets into storage, run task workflows " + "across models, and score the stored results");
 registerDatasetCommand(program, openStores);
 registerRunCommand(program, openStores, ensureProviders);
 registerReportCommand(program, openStores);
+registerWebCommand(program);
 await program.parseAsync(process.argv);
 process.exit(process.exitCode ?? 0);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@workglow/eval",
   "type": "module",
-  "version": "0.3.48",
+  "version": "0.4.0",
   "repository": {
     "type": "git",
     "url": "https://github.com/workglow-dev/libs.git",
@@ -33,18 +33,19 @@
     "src/**/*.md"
   ],
   "dependencies": {
-    "@workglow/ai": "0.3.48",
-    "@workglow/anthropic": "0.3.48",
-    "@workglow/deepseek": "0.3.48",
-    "@workglow/google-gemini": "0.3.48",
-    "@workglow/huggingface-transformers": "0.3.48",
-    "@workglow/node-llama-cpp": "0.3.48",
-    "@workglow/openai": "0.3.48",
-    "@workglow/sqlite": "0.3.48",
-    "@workglow/storage": "0.3.48",
-    "@workglow/task-graph": "0.3.48",
-    "@workglow/util": "0.3.48",
-    "@workglow/xai": "0.3.48",
+    "@workglow/ai": "0.4.0",
+    "@workglow/anthropic": "0.4.0",
+    "@workglow/cli": "0.4.0",
+    "@workglow/deepseek": "0.4.0",
+    "@workglow/google-gemini": "0.4.0",
+    "@workglow/huggingface-transformers": "0.4.0",
+    "@workglow/node-llama-cpp": "0.4.0",
+    "@workglow/openai": "0.4.0",
+    "@workglow/sqlite": "0.4.0",
+    "@workglow/storage": "0.4.0",
+    "@workglow/task-graph": "0.4.0",
+    "@workglow/util": "0.4.0",
+    "@workglow/xai": "0.4.0",
     "@huggingface/transformers": "^4.2.0",
     "commander": "^15.0.0",
     "hyparquet": "^1.28.2",