@dxos/pipeline 0.0.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/lib/index.mjs +174 -0
  2. package/dist/lib/index.mjs.map +1 -0
  3. package/dist/lib/testing.mjs +125 -0
  4. package/dist/lib/testing.mjs.map +1 -0
  5. package/dist/types/src/Pipeline.d.ts +10 -0
  6. package/dist/types/src/Pipeline.d.ts.map +1 -1
  7. package/dist/types/src/Progress.d.ts +17 -0
  8. package/dist/types/src/Progress.d.ts.map +1 -0
  9. package/dist/types/src/ProgressReporter.d.ts +19 -0
  10. package/dist/types/src/ProgressReporter.d.ts.map +1 -0
  11. package/dist/types/src/Stage.d.ts +12 -0
  12. package/dist/types/src/Stage.d.ts.map +1 -1
  13. package/dist/types/src/index.d.ts +2 -0
  14. package/dist/types/src/index.d.ts.map +1 -1
  15. package/dist/types/src/testing/benchmark.d.ts +40 -0
  16. package/dist/types/src/testing/benchmark.d.ts.map +1 -0
  17. package/dist/types/src/testing/benchmark.test.d.ts +2 -0
  18. package/dist/types/src/testing/benchmark.test.d.ts.map +1 -0
  19. package/dist/types/src/testing/index.d.ts +3 -0
  20. package/dist/types/src/testing/index.d.ts.map +1 -1
  21. package/dist/types/src/testing/instrument.d.ts +18 -0
  22. package/dist/types/src/testing/instrument.d.ts.map +1 -0
  23. package/dist/types/src/testing/metrics.d.ts +33 -0
  24. package/dist/types/src/testing/metrics.d.ts.map +1 -0
  25. package/dist/types/tsconfig.tsbuildinfo +1 -1
  26. package/package.json +17 -13
  27. package/src/Pipeline.test.ts +74 -1
  28. package/src/Pipeline.ts +46 -0
  29. package/src/Progress.ts +25 -0
  30. package/src/ProgressReporter.ts +59 -0
  31. package/src/Stage.ts +28 -0
  32. package/src/index.ts +2 -0
  33. package/src/testing/benchmark.test.ts +103 -0
  34. package/src/testing/benchmark.ts +113 -0
  35. package/src/testing/index.ts +3 -0
  36. package/src/testing/instrument.ts +57 -0
  37. package/src/testing/metrics.ts +68 -0
  38. package/dist/lib/neutral/chunk-J5LGTIGS.mjs +0 -10
  39. package/dist/lib/neutral/chunk-J5LGTIGS.mjs.map +0 -7
  40. package/dist/lib/neutral/index.mjs +0 -67
  41. package/dist/lib/neutral/index.mjs.map +0 -7
  42. package/dist/lib/neutral/meta.json +0 -1
  43. package/dist/lib/neutral/testing/index.mjs +0 -18
  44. package/dist/lib/neutral/testing/index.mjs.map +0 -7
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/pipeline",
3
- "version": "0.0.0",
3
+ "version": "0.11.0",
4
4
  "description": "Generic back-pressured streaming pipeline (source → stages → sink).",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -14,14 +14,14 @@
14
14
  "type": "module",
15
15
  "exports": {
16
16
  ".": {
17
- "default": "./dist/lib/neutral/index.mjs",
18
17
  "source": "./src/index.ts",
19
- "types": "./dist/types/src/index.d.ts"
18
+ "types": "./dist/types/src/index.d.ts",
19
+ "import": "./dist/lib/index.mjs"
20
20
  },
21
21
  "./testing": {
22
- "default": "./dist/lib/neutral/testing/index.mjs",
23
22
  "source": "./src/testing/index.ts",
24
- "types": "./dist/types/src/testing/index.d.ts"
23
+ "types": "./dist/types/src/testing/index.d.ts",
24
+ "import": "./dist/lib/testing.mjs"
25
25
  }
26
26
  },
27
27
  "types": "dist/types/src/index.d.ts",
@@ -29,17 +29,21 @@
29
29
  "dist",
30
30
  "src"
31
31
  ],
32
- "scripts": {},
33
- "dependencies": {},
32
+ "dependencies": {
33
+ "@dxos/log": "0.11.0",
34
+ "@dxos/progress": "0.11.0",
35
+ "@dxos/invariant": "0.11.0"
36
+ },
34
37
  "devDependencies": {
35
- "@dxos/effect": "workspace:*",
36
- "effect": "catalog:",
37
- "vitest": "catalog:"
38
+ "effect": "3.21.4",
39
+ "vitest": "4.1.10",
40
+ "@dxos/effect": "0.11.0"
38
41
  },
39
42
  "peerDependencies": {
40
- "effect": "catalog:"
43
+ "effect": "3.21.4"
41
44
  },
42
45
  "publishConfig": {
43
46
  "access": "public"
44
- }
45
- }
47
+ },
48
+ "scripts": {}
49
+ }
@@ -2,13 +2,17 @@
2
2
  // Copyright 2026 DXOS.org
3
3
  //
4
4
 
5
+ import { describe, it, test } from '@effect/vitest';
6
+ import * as Cause from 'effect/Cause';
5
7
  import * as Context from 'effect/Context';
6
8
  import * as Effect from 'effect/Effect';
9
+ import * as Exit from 'effect/Exit';
10
+ import * as Fiber from 'effect/Fiber';
7
11
  import * as Layer from 'effect/Layer';
8
12
  import * as Stream from 'effect/Stream';
9
- import { describe, test } from 'vitest';
10
13
 
11
14
  import { EffectEx } from '@dxos/effect';
15
+ import { invariant } from '@dxos/invariant';
12
16
 
13
17
  import * as Pipeline from './Pipeline';
14
18
  import * as Stage from './Stage';
@@ -115,4 +119,73 @@ describe('Pipeline.run overflow', () => {
115
119
  expect(runs.length).toBeLessThan(4);
116
120
  expect(runs.at(-1)).toBe(4);
117
121
  });
122
+
123
+ it.live(
124
+ 'abort',
125
+ Effect.fnUntraced(function* ({ expect }) {
126
+ const controller = new AbortController();
127
+ const { sink } = captureSink<number>();
128
+ const pipeline = yield* Effect.fork(
129
+ Stream.fromIterable(Array.from({ length: 100 }, (_, index) => index)).pipe(
130
+ Stage.map('id', (n) => Effect.succeed(n)),
131
+ Stage.map('sleep', (n) => Effect.sleep('10 millis').pipe(Effect.as(n)), {
132
+ overflow: 'suspend',
133
+ bufferSize: 4,
134
+ }),
135
+ // Stage.map('log', (n) =>
136
+ // Effect.sync(() => {
137
+ // console.log('item', n);
138
+ // return n;
139
+ // }),
140
+ // ),
141
+ Pipeline.run({ sink, overflow: 'suspend', bufferSize: 4 }),
142
+ Pipeline.abortWith(controller.signal),
143
+ Effect.withSpan('pipeline'),
144
+ Effect.exit,
145
+ ),
146
+ );
147
+
148
+ const aborter = yield* Effect.fork(
149
+ Effect.gen(function* () {
150
+ yield* Effect.sleep('500 millis');
151
+ console.log('aborting');
152
+ controller.abort();
153
+ }),
154
+ );
155
+
156
+ yield* Fiber.join(aborter);
157
+ const result = yield* Fiber.join(pipeline);
158
+ invariant(Exit.isFailure(result), 'pipeline should fail');
159
+ invariant(Cause.isInterrupted(result.cause), 'pipeline should be interrupted');
160
+ }),
161
+ );
162
+
163
+ it.live(
164
+ 'abortWith runs onCancel on abort',
165
+ Effect.fnUntraced(function* ({ expect }) {
166
+ const controller = new AbortController();
167
+ let cancelled = false;
168
+ const { sink } = captureSink<number>();
169
+ const pipeline = yield* Effect.fork(
170
+ Stream.fromIterable(Array.from({ length: 100 }, (_, index) => index)).pipe(
171
+ Stage.map('sleep', (n) => Effect.sleep('10 millis').pipe(Effect.as(n)), {
172
+ overflow: 'suspend',
173
+ bufferSize: 4,
174
+ }),
175
+ Pipeline.run({ sink, overflow: 'suspend', bufferSize: 4 }),
176
+ Pipeline.abortWith(
177
+ controller.signal,
178
+ Effect.sync(() => {
179
+ cancelled = true;
180
+ }),
181
+ ),
182
+ Effect.exit,
183
+ ),
184
+ );
185
+
186
+ controller.abort();
187
+ yield* Fiber.join(pipeline);
188
+ expect(cancelled).toBe(true);
189
+ }),
190
+ );
118
191
  });
package/src/Pipeline.ts CHANGED
@@ -8,6 +8,8 @@ import * as Effect from 'effect/Effect';
8
8
  import * as Function from 'effect/Function';
9
9
  import * as Stream from 'effect/Stream';
10
10
 
11
+ import { log } from '@dxos/log';
12
+
11
13
  import * as Stage from './Stage';
12
14
 
13
15
  /**
@@ -52,3 +54,47 @@ export const run: {
52
54
  Stream.runForEach(options.sink),
53
55
  ),
54
56
  );
57
+
58
+ /**
59
+ * Abort a pipeline with an `AbortSignal`.
60
+ *
61
+ * Races the wrapped effect against the signal; losing the race resolves as a genuine fiber
62
+ * interrupt, which skips every subsequent `yield*`/pipe step in the wrapped effect (an interrupt is
63
+ * not a failure — `Effect.tapError` and friends never see it). `onCancel` runs via `Effect.onInterrupt`
64
+ * so every caller gets cancellation cleanup (e.g. resetting a progress monitor stuck at its last
65
+ * `running` snapshot) without each one having to remember to add its own `Effect.onInterrupt`.
66
+ */
67
+ export const abortWith =
68
+ (
69
+ signal: AbortSignal,
70
+ onCancel: Effect.Effect<void> = Effect.void,
71
+ ): (<A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>) =>
72
+ (effect) =>
73
+ effect.pipe(
74
+ Effect.raceFirst(
75
+ Effect.async<never>((resume) => {
76
+ if (signal.aborted) {
77
+ resume(Effect.interrupt);
78
+ return;
79
+ }
80
+
81
+ const onAbort = () => {
82
+ resume(
83
+ Effect.gen(function* () {
84
+ log.info('aborting pipeline', {
85
+ span: yield* Effect.currentSpan.pipe(
86
+ Effect.map((span) => span.name),
87
+ Effect.catchAll((_) => Effect.succeed(undefined)),
88
+ ),
89
+ });
90
+ return yield* Effect.interrupt;
91
+ }),
92
+ );
93
+ };
94
+
95
+ signal.addEventListener('abort', onAbort, { once: true });
96
+ return Effect.sync(() => signal.removeEventListener('abort', onAbort));
97
+ }),
98
+ ),
99
+ Effect.onInterrupt(() => onCancel),
100
+ );
@@ -0,0 +1,25 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ // @import-as-namespace
6
+
7
+ import * as Context from 'effect/Context';
8
+ import * as Layer from 'effect/Layer';
9
+
10
+ import { Progress as ProgressCore } from '@dxos/progress';
11
+
12
+ // The registry primitives now live in @dxos/progress so the pipeline (this Effect service) and the
13
+ // app-toolkit ProgressRegistry capability share one implementation.
14
+ export type TaskStatus = ProgressCore.TaskStatus;
15
+ export type TaskProgress = ProgressCore.TaskProgress;
16
+ export type ProgressSnapshot = ProgressCore.ProgressSnapshot;
17
+ export type TaskHandle = ProgressCore.TaskHandle;
18
+ export type ProgressApi = ProgressCore.ProgressApi;
19
+ export const make = ProgressCore.make;
20
+ export const deriveEta = ProgressCore.deriveEta;
21
+
22
+ export class Progress extends Context.Tag('@dxos/pipeline/Progress')<Progress, ProgressCore.ProgressApi>() {
23
+ /** A fresh in-memory registry. */
24
+ static layer: Layer.Layer<Progress> = Layer.sync(Progress, () => ProgressCore.make());
25
+ }
@@ -0,0 +1,59 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ // @import-as-namespace
6
+
7
+ import type * as Duration from 'effect/Duration';
8
+ import * as Effect from 'effect/Effect';
9
+ import * as Layer from 'effect/Layer';
10
+
11
+ import { log } from '@dxos/log';
12
+
13
+ import * as Progress from './Progress';
14
+
15
+ /** A destination for progress snapshots — a log line, a file, an app surface, an EDGE stream, … */
16
+ export type ProgressSink = (snapshot: Progress.ProgressSnapshot) => Effect.Effect<void>;
17
+
18
+ /** One compact info-log line per snapshot (works everywhere — node, browser, EDGE). */
19
+ export const logSink: ProgressSink = (snapshot) =>
20
+ Effect.sync(() =>
21
+ log.info('progress', {
22
+ tasks: snapshot.tasks.map(
23
+ (task) => `${task.name} ${task.current}${task.total !== undefined ? `/${task.total}` : ''} ${task.status}`,
24
+ ),
25
+ }),
26
+ );
27
+
28
+ /**
29
+ * A reporter layer: subscribes to the {@link Progress} registry and flushes the latest snapshot to
30
+ * `sink` at most once per `throttle` (coalescing bursts), plus a final flush when the scope closes.
31
+ * Reactive consumers (a browser panel) can instead `Progress.subscribe` directly for instant
32
+ * updates; this layer is for rate-limited sinks (file, log, EDGE).
33
+ */
34
+ export const layer = (options: {
35
+ readonly sink: ProgressSink;
36
+ readonly throttle?: Duration.DurationInput;
37
+ }): Layer.Layer<never, never, Progress.Progress> =>
38
+ Layer.scopedDiscard(
39
+ Effect.gen(function* () {
40
+ const progress = yield* Progress.Progress;
41
+ const throttle = options.throttle ?? '2 seconds';
42
+
43
+ let latest: Progress.ProgressSnapshot | undefined;
44
+ const unsubscribe = progress.subscribe((snapshot) => {
45
+ latest = snapshot;
46
+ });
47
+ yield* Effect.addFinalizer(() => Effect.sync(unsubscribe));
48
+
49
+ const flush = Effect.suspend(() => {
50
+ const snapshot = latest;
51
+ latest = undefined;
52
+ return snapshot ? options.sink(snapshot) : Effect.void;
53
+ });
54
+
55
+ // Periodic throttled flush while the scope is open; a final flush captures the terminal state.
56
+ yield* Effect.forkScoped(Effect.forever(flush.pipe(Effect.delay(throttle))));
57
+ yield* Effect.addFinalizer(() => options.sink(progress.snapshot()).pipe(Effect.ignore));
58
+ }),
59
+ );
package/src/Stage.ts CHANGED
@@ -4,9 +4,12 @@
4
4
 
5
5
  // @import-as-namespace
6
6
 
7
+ import * as Cause from 'effect/Cause';
7
8
  import * as Effect from 'effect/Effect';
8
9
  import * as Stream from 'effect/Stream';
9
10
 
11
+ import * as Progress from './Progress';
12
+
10
13
  /**
11
14
  * Overflow policy applied when a consumer cannot keep pace, mapped onto `Stream.buffer` strategies.
12
15
  * - `suspend`: back pressure — never drop (default; correct for email/document processing).
@@ -126,3 +129,28 @@ export const filter =
126
129
  <In>(id: string, pred: (item: In) => boolean): Stage<In, In> =>
127
130
  <E0, R0>(self: Stream.Stream<In, E0, R0>) =>
128
131
  self.pipe(Stream.filter(pred));
132
+
133
+ /**
134
+ * A pass-through stage that reports live progress to the {@link Progress} registry: it registers the
135
+ * task (with an optional item `total`), advances the item index as each item flows through, marks the
136
+ * task done only when the stream ends successfully, and marks it failed if the stream errors.
137
+ * Progress is thus an artifact of the pipeline itself — the same stage works in the app (feeding a
138
+ * reactive panel) and in tests (feeding a file sink).
139
+ */
140
+ export const track =
141
+ <In>(name: string, options: { total?: number; label?: string } = {}): Stage<In, In, never, Progress.Progress> =>
142
+ <E0, R0>(self: Stream.Stream<In, E0, R0>) =>
143
+ Stream.unwrap(
144
+ Effect.map(Progress.Progress, (progress) => {
145
+ const handle = progress.task(name, options);
146
+ // `done` only on successful completion (a concatenated terminal effect, unreached on error);
147
+ // `fail` on the error path. A finalizer/`ensuring` would report done even for a failed stream.
148
+ return Stream.concat(
149
+ self.pipe(
150
+ Stream.tap(() => Effect.sync(() => handle.advance())),
151
+ Stream.tapErrorCause((cause) => Effect.sync(() => handle.fail(Cause.pretty(cause)))),
152
+ ),
153
+ Stream.execute(Effect.sync(() => handle.done())),
154
+ );
155
+ }),
156
+ );
package/src/index.ts CHANGED
@@ -3,4 +3,6 @@
3
3
  //
4
4
 
5
5
  export * as Pipeline from './Pipeline';
6
+ export * as Progress from './Progress';
7
+ export * as ProgressReporter from './ProgressReporter';
6
8
  export * as Stage from './Stage';
@@ -0,0 +1,103 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as Effect from 'effect/Effect';
6
+ import * as Stream from 'effect/Stream';
7
+ import { describe, test } from 'vitest';
8
+
9
+ import { EffectEx } from '@dxos/effect';
10
+
11
+ import * as Pipeline from '../Pipeline';
12
+ import * as Stage from '../Stage';
13
+ import { renderBenchmark, runBenchmark } from './benchmark';
14
+ import { captureSink } from './capture';
15
+ import { instrument } from './instrument';
16
+ import { useMetrics } from './metrics';
17
+
18
+ describe('benchmark framework', () => {
19
+ test('instrument records per-stage counters and latency, even when items are dropped', async ({ expect }) => {
20
+ // A drop stage (out < in): odds are filtered, kept evens sleep a known amount. Because a
21
+ // dropped input has no output to attribute, `evens.ms` must reflect only the two kept items'
22
+ // ~10ms each — a FIFO pairing would mis-attribute a dropped item's entry time instead.
23
+ const evens = Stage.map('evens', (n: number) =>
24
+ n % 2 === 0 ? Effect.sleep('10 millis').pipe(Effect.as(n)) : Effect.succeed(undefined),
25
+ );
26
+
27
+ const result = await EffectEx.runPromise(
28
+ runBenchmark({
29
+ variants: [{ name: 'default', config: {} }],
30
+ program: () => {
31
+ const { sink, items } = captureSink<number>();
32
+ return Stream.fromIterable([1, 2, 3, 4])
33
+ .pipe(instrument('evens', evens), Pipeline.run({ sink }))
34
+ .pipe(Effect.as(items));
35
+ },
36
+ }),
37
+ );
38
+
39
+ const { metrics } = result.variants[0];
40
+ expect(metrics['evens.in']).toBe(4);
41
+ expect(metrics['evens.out']).toBe(2);
42
+ expect(metrics['evens.errors'] ?? 0).toBe(0);
43
+ // Two kept items sleeping ~10ms each; a generous lower bound avoids CI flakiness but still fails
44
+ // if latency were dropped or attributed to the wrong (instantaneous) entry.
45
+ expect(metrics['evens.ms']).toBeGreaterThanOrEqual(15);
46
+ expect(typeof metrics['total.ms']).toBe('number');
47
+ });
48
+
49
+ test('stages emit custom metrics; evaluator adds effectful domain metrics', async ({ expect }) => {
50
+ // Stage opts into Metrics to count a domain event.
51
+ const flagLarge = Stage.map('flag', (n: number) =>
52
+ (n >= 3 ? useMetrics((metrics) => metrics.inc('large')) : Effect.void).pipe(Effect.as(n)),
53
+ );
54
+
55
+ const result = await EffectEx.runPromise(
56
+ runBenchmark({
57
+ variants: [{ name: 'v1', config: { threshold: 3 } }],
58
+ program: () => {
59
+ const { sink, items } = captureSink<number>();
60
+ return Stream.fromIterable([1, 2, 3, 4, 5]).pipe(flagLarge, Pipeline.run({ sink })).pipe(Effect.as(items));
61
+ },
62
+ evaluate: (_config, items) => Effect.succeed({ count: items.length, sum: items.reduce((a, b) => a + b, 0) }),
63
+ }),
64
+ );
65
+
66
+ const { metrics } = result.variants[0];
67
+ expect(metrics.large).toBe(3); // 3, 4, 5
68
+ expect(metrics.count).toBe(5);
69
+ expect(metrics.sum).toBe(15);
70
+ });
71
+
72
+ test('runs multiple variants and a failing variant is isolated', async ({ expect }) => {
73
+ const result = await EffectEx.runPromise(
74
+ runBenchmark<{ fail: boolean }, number, Error, never>({
75
+ variants: [
76
+ { name: 'ok', config: { fail: false } },
77
+ { name: 'bad', config: { fail: true } },
78
+ ],
79
+ program: (config) => (config.fail ? Effect.fail(new Error('boom')) : Effect.succeed(42)),
80
+ evaluate: (_config, output) => Effect.succeed({ answer: output }),
81
+ }),
82
+ );
83
+
84
+ expect(result.variants.map((variant) => variant.variant)).toEqual(['ok', 'bad']);
85
+ expect(result.variants[0].metrics.answer).toBe(42);
86
+ expect(String(result.variants[1].metrics.error)).toContain('boom');
87
+ // The failing variant does not abort the comparison.
88
+ expect(result.variants).toHaveLength(2);
89
+ });
90
+
91
+ test('renderBenchmark tabulates metrics as rows and variants as columns', async ({ expect }) => {
92
+ const table = renderBenchmark({
93
+ variants: [
94
+ { variant: 'gpt-oss', metrics: { 'total.ms': 1200, 'facts': 8 } },
95
+ { variant: 'llama', metrics: { 'total.ms': 400, 'facts': 3 } },
96
+ ],
97
+ });
98
+ expect(table).toContain('metric');
99
+ expect(table).toContain('gpt-oss');
100
+ expect(table).toContain('llama');
101
+ expect(table).toMatch(/facts\s+8\s+3/);
102
+ });
103
+ });
@@ -0,0 +1,113 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as Cause from 'effect/Cause';
6
+ import * as Duration from 'effect/Duration';
7
+ import * as Effect from 'effect/Effect';
8
+ import * as Layer from 'effect/Layer';
9
+
10
+ import { Metrics, type MetricsSnapshot, type MetricValue, makeMetrics } from './metrics';
11
+
12
+ /** A named configuration to benchmark (e.g. a model). */
13
+ export type BenchmarkVariant<Config> = {
14
+ readonly name: string;
15
+ readonly config: Config;
16
+ };
17
+
18
+ /** One variant's merged metrics: `total.ms`, per-stage `instrument` counters, and evaluator output. */
19
+ export type VariantResult = {
20
+ readonly variant: string;
21
+ readonly metrics: MetricsSnapshot;
22
+ };
23
+
24
+ export type BenchmarkResult = {
25
+ readonly variants: readonly VariantResult[];
26
+ };
27
+
28
+ export type BenchmarkOptions<Config, Output, E, R> = {
29
+ readonly variants: readonly BenchmarkVariant<Config>[];
30
+ /**
31
+ * Run the (instrumented) pipeline for one variant and return whatever the evaluator needs
32
+ * (e.g. captured sink outputs). Stage counters and any `Metrics` writes are collected
33
+ * automatically. `Metrics` is provided by the runner.
34
+ */
35
+ readonly program: (config: Config) => Effect.Effect<Output, E, R | Metrics>;
36
+ /**
37
+ * Compute domain metrics for the variant. Effectful, so it can query services (fact store, ECHO
38
+ * db) that the pipeline populated. Its returned map is merged over the auto-collected metrics.
39
+ */
40
+ readonly evaluate?: (config: Config, output: Output) => Effect.Effect<MetricsSnapshot, E, R | Metrics>;
41
+ };
42
+
43
+ /**
44
+ * Run a pipeline across variants (e.g. models) and collect a comparable metrics row per variant:
45
+ * total wall-clock, per-stage {@link instrument} counters, and the evaluator's domain metrics. A
46
+ * variant whose program or evaluator fails is recorded with an `error` (plus whatever metrics were
47
+ * captured before it failed) rather than aborting the whole comparison; an interruption
48
+ * (cancellation) is re-raised so the whole run stops promptly.
49
+ */
50
+ export const runBenchmark = <Config, Output, E, R>(
51
+ options: BenchmarkOptions<Config, Output, E, R>,
52
+ ): Effect.Effect<BenchmarkResult, E, Exclude<R, Metrics>> =>
53
+ Effect.gen(function* () {
54
+ const variants: VariantResult[] = [];
55
+ for (const variant of options.variants) {
56
+ const metrics = makeMetrics();
57
+ const layer = Layer.succeed(Metrics, metrics);
58
+ const row = yield* Effect.gen(function* () {
59
+ const heapBefore = heapUsedBytes();
60
+ const [duration, output] = yield* Effect.timed(options.program(variant.config));
61
+ const heapAfter = heapUsedBytes();
62
+ const domain = options.evaluate ? yield* options.evaluate(variant.config, output) : {};
63
+ const snapshot = yield* metrics.snapshot;
64
+ return {
65
+ 'total.ms': Math.round(Duration.toMillis(duration)),
66
+ // Heap growth across the run (Node only; approximate — subject to GC, and for out-of-process
67
+ // model back-ends like Ollama it reflects harness overhead, not the model's own memory).
68
+ ...(heapBefore !== undefined && heapAfter !== undefined
69
+ ? { 'heap.mb': Math.round(((heapAfter - heapBefore) / (1024 * 1024)) * 100) / 100 }
70
+ : {}),
71
+ ...snapshot,
72
+ ...domain,
73
+ } satisfies MetricsSnapshot;
74
+ }).pipe(
75
+ Effect.provide(layer),
76
+ Effect.catchAllCause((cause) =>
77
+ // A cancelled benchmark must abort, not be recorded as an `error` row and iterated past;
78
+ // only genuine failures/defects degrade to a metrics row.
79
+ Cause.isInterruptedOnly(cause)
80
+ ? Effect.failCause(cause)
81
+ : metrics.snapshot.pipe(
82
+ Effect.map(
83
+ (snapshot) => ({ ...snapshot, error: Cause.pretty(cause).split('\n')[0] }) satisfies MetricsSnapshot,
84
+ ),
85
+ ),
86
+ ),
87
+ );
88
+ variants.push({ variant: variant.name, metrics: row });
89
+ }
90
+ return { variants };
91
+ });
92
+
93
+ /** Node process heap usage in bytes, or `undefined` off Node (browser) so the metric is simply omitted. */
94
+ const heapUsedBytes = (): number | undefined =>
95
+ typeof process !== 'undefined' && typeof process.memoryUsage === 'function'
96
+ ? process.memoryUsage().heapUsed
97
+ : undefined;
98
+
99
+ const format = (value: MetricValue | undefined): string =>
100
+ value === undefined ? '' : typeof value === 'number' ? String(Math.round(value * 100) / 100) : value;
101
+
102
+ /** Render a benchmark result as a fixed-width metric × variant table (metrics as rows). */
103
+ export const renderBenchmark = (result: BenchmarkResult): string => {
104
+ const metricNames = [...new Set(result.variants.flatMap((variant) => Object.keys(variant.metrics)))].sort();
105
+ const header = ['metric', ...result.variants.map((variant) => variant.variant)];
106
+ const rows = metricNames.map((metric) => [
107
+ metric,
108
+ ...result.variants.map((variant) => format(variant.metrics[metric])),
109
+ ]);
110
+ const widths = header.map((_cell, column) => Math.max(...[header, ...rows].map((row) => row[column].length)));
111
+ const line = (row: readonly string[]) => row.map((cell, column) => cell.padEnd(widths[column])).join(' ');
112
+ return [line(header), widths.map((width) => '-'.repeat(width)).join(' '), ...rows.map(line)].join('\n');
113
+ };
@@ -2,4 +2,7 @@
2
2
  // Copyright 2026 DXOS.org
3
3
  //
4
4
 
5
+ export * from './benchmark';
5
6
  export * from './capture';
7
+ export * from './instrument';
8
+ export * from './metrics';
@@ -0,0 +1,57 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as Clock from 'effect/Clock';
6
+ import * as Effect from 'effect/Effect';
7
+ import * as Stream from 'effect/Stream';
8
+
9
+ import * as Stage from '../Stage';
10
+ import { Metrics, useMetrics } from './metrics';
11
+
12
+ /**
13
+ * Wrap a {@link Stage.Stage} to auto-record per-stage metrics — `${name}.in`, `${name}.out`,
14
+ * `${name}.errors` (counts) and `${name}.ms` (summed processing latency) — through the ambient
15
+ * {@link Metrics} service, with no changes to the stage itself. Compose it in place of the stage:
16
+ * `source.pipe(instrument('extract', s), …)`. The wrapped stage additionally requires `Metrics` in
17
+ * its environment (provided at the edge by the benchmark runner).
18
+ *
19
+ * Latency is the wall-clock between the most recent input entering and an output leaving the stage.
20
+ * For a sequential stage (`Stage.map`, the common case) the item just emitted is the last one pulled
21
+ * in, so `${name}.ms` is the true per-item processing time — including drop/filter stages, whose
22
+ * discarded inputs simply never produce an output to attribute. It is only approximate for a stage
23
+ * that reorders or processes items concurrently (concurrency > 1), where the "most recent input" is
24
+ * no longer the one being emitted. Divide `${name}.ms` by `${name}.out` for the average.
25
+ */
26
+ export const instrument =
27
+ <In, Out, E, R>(name: string, stage: Stage.Stage<In, Out, E, R>): Stage.Stage<In, Out, E, R | Metrics> =>
28
+ (self) => {
29
+ // Timestamp of the most recently ingested item. A sequential stage emits the item it last pulled,
30
+ // so pairing an output with this (rather than FIFO-oldest) stays correct when inputs are dropped.
31
+ let lastIn: number | undefined;
32
+ return stage(
33
+ self.pipe(
34
+ Stream.tap(() =>
35
+ Clock.currentTimeMillis.pipe(
36
+ Effect.flatMap((now) =>
37
+ useMetrics((metrics) => metrics.inc(`${name}.in`)).pipe(
38
+ Effect.zipRight(Effect.sync(() => void (lastIn = now))),
39
+ ),
40
+ ),
41
+ ),
42
+ ),
43
+ ),
44
+ ).pipe(
45
+ Stream.tap(() =>
46
+ Clock.currentTimeMillis.pipe(
47
+ Effect.flatMap((now) => {
48
+ const elapsed = lastIn === undefined ? 0 : now - lastIn;
49
+ return useMetrics((metrics) => metrics.inc(`${name}.out`)).pipe(
50
+ Effect.zipRight(useMetrics((metrics) => metrics.inc(`${name}.ms`, elapsed))),
51
+ );
52
+ }),
53
+ ),
54
+ ),
55
+ Stream.tapError(() => useMetrics((metrics) => metrics.inc(`${name}.errors`))),
56
+ );
57
+ };