@dxos/pipeline 0.0.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/neutral/index.mjs +58 -2
- package/dist/lib/neutral/index.mjs.map +4 -4
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/lib/neutral/testing/index.mjs +124 -3
- package/dist/lib/neutral/testing/index.mjs.map +4 -4
- package/dist/types/src/Progress.d.ts +17 -0
- package/dist/types/src/Progress.d.ts.map +1 -0
- package/dist/types/src/ProgressReporter.d.ts +19 -0
- package/dist/types/src/ProgressReporter.d.ts.map +1 -0
- package/dist/types/src/Stage.d.ts +12 -0
- package/dist/types/src/Stage.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +2 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/testing/benchmark.d.ts +40 -0
- package/dist/types/src/testing/benchmark.d.ts.map +1 -0
- package/dist/types/src/testing/benchmark.test.d.ts +2 -0
- package/dist/types/src/testing/benchmark.test.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +3 -0
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/instrument.d.ts +18 -0
- package/dist/types/src/testing/instrument.d.ts.map +1 -0
- package/dist/types/src/testing/metrics.d.ts +33 -0
- package/dist/types/src/testing/metrics.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +12 -9
- package/src/Progress.ts +25 -0
- package/src/ProgressReporter.ts +59 -0
- package/src/Stage.ts +28 -0
- package/src/index.ts +2 -0
- package/src/testing/benchmark.test.ts +103 -0
- package/src/testing/benchmark.ts +113 -0
- package/src/testing/index.ts +3 -0
- package/src/testing/instrument.ts +57 -0
- package/src/testing/metrics.ts +68 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/pipeline",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.0",
|
|
4
4
|
"description": "Generic back-pressured streaming pipeline (source → stages → sink).",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -29,17 +29,20 @@
|
|
|
29
29
|
"dist",
|
|
30
30
|
"src"
|
|
31
31
|
],
|
|
32
|
-
"
|
|
33
|
-
|
|
32
|
+
"dependencies": {
|
|
33
|
+
"@dxos/log": "0.10.0",
|
|
34
|
+
"@dxos/progress": "0.9.0"
|
|
35
|
+
},
|
|
34
36
|
"devDependencies": {
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
37
|
+
"effect": "3.21.4",
|
|
38
|
+
"vitest": "4.1.8",
|
|
39
|
+
"@dxos/effect": "0.10.0"
|
|
38
40
|
},
|
|
39
41
|
"peerDependencies": {
|
|
40
|
-
"effect": "
|
|
42
|
+
"effect": "3.21.4"
|
|
41
43
|
},
|
|
42
44
|
"publishConfig": {
|
|
43
45
|
"access": "public"
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
+
},
|
|
47
|
+
"scripts": {}
|
|
48
|
+
}
|
package/src/Progress.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// @import-as-namespace
|
|
6
|
+
|
|
7
|
+
import * as Context from 'effect/Context';
|
|
8
|
+
import * as Layer from 'effect/Layer';
|
|
9
|
+
|
|
10
|
+
import { Progress as ProgressCore } from '@dxos/progress';
|
|
11
|
+
|
|
12
|
+
// The registry primitives now live in @dxos/progress so the pipeline (this Effect service) and the
|
|
13
|
+
// app-toolkit ProgressRegistry capability share one implementation.
|
|
14
|
+
export type TaskStatus = ProgressCore.TaskStatus;
|
|
15
|
+
export type TaskProgress = ProgressCore.TaskProgress;
|
|
16
|
+
export type ProgressSnapshot = ProgressCore.ProgressSnapshot;
|
|
17
|
+
export type TaskHandle = ProgressCore.TaskHandle;
|
|
18
|
+
export type ProgressApi = ProgressCore.ProgressApi;
|
|
19
|
+
export const make = ProgressCore.make;
|
|
20
|
+
export const deriveEta = ProgressCore.deriveEta;
|
|
21
|
+
|
|
22
|
+
export class Progress extends Context.Tag('@dxos/pipeline/Progress')<Progress, ProgressCore.ProgressApi>() {
|
|
23
|
+
/** A fresh in-memory registry. */
|
|
24
|
+
static layer: Layer.Layer<Progress> = Layer.sync(Progress, () => ProgressCore.make());
|
|
25
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// @import-as-namespace
|
|
6
|
+
|
|
7
|
+
import type * as Duration from 'effect/Duration';
|
|
8
|
+
import * as Effect from 'effect/Effect';
|
|
9
|
+
import * as Layer from 'effect/Layer';
|
|
10
|
+
|
|
11
|
+
import { log } from '@dxos/log';
|
|
12
|
+
|
|
13
|
+
import * as Progress from './Progress';
|
|
14
|
+
|
|
15
|
+
/** A destination for progress snapshots — a log line, a file, an app surface, an EDGE stream, … */
|
|
16
|
+
export type ProgressSink = (snapshot: Progress.ProgressSnapshot) => Effect.Effect<void>;
|
|
17
|
+
|
|
18
|
+
/** One compact info-log line per snapshot (works everywhere — node, browser, EDGE). */
|
|
19
|
+
export const logSink: ProgressSink = (snapshot) =>
|
|
20
|
+
Effect.sync(() =>
|
|
21
|
+
log.info('progress', {
|
|
22
|
+
tasks: snapshot.tasks.map(
|
|
23
|
+
(task) => `${task.name} ${task.current}${task.total !== undefined ? `/${task.total}` : ''} ${task.status}`,
|
|
24
|
+
),
|
|
25
|
+
}),
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* A reporter layer: subscribes to the {@link Progress} registry and flushes the latest snapshot to
|
|
30
|
+
* `sink` at most once per `throttle` (coalescing bursts), plus a final flush when the scope closes.
|
|
31
|
+
* Reactive consumers (a browser panel) can instead `Progress.subscribe` directly for instant
|
|
32
|
+
* updates; this layer is for rate-limited sinks (file, log, EDGE).
|
|
33
|
+
*/
|
|
34
|
+
export const layer = (options: {
|
|
35
|
+
readonly sink: ProgressSink;
|
|
36
|
+
readonly throttle?: Duration.DurationInput;
|
|
37
|
+
}): Layer.Layer<never, never, Progress.Progress> =>
|
|
38
|
+
Layer.scopedDiscard(
|
|
39
|
+
Effect.gen(function* () {
|
|
40
|
+
const progress = yield* Progress.Progress;
|
|
41
|
+
const throttle = options.throttle ?? '2 seconds';
|
|
42
|
+
|
|
43
|
+
let latest: Progress.ProgressSnapshot | undefined;
|
|
44
|
+
const unsubscribe = progress.subscribe((snapshot) => {
|
|
45
|
+
latest = snapshot;
|
|
46
|
+
});
|
|
47
|
+
yield* Effect.addFinalizer(() => Effect.sync(unsubscribe));
|
|
48
|
+
|
|
49
|
+
const flush = Effect.suspend(() => {
|
|
50
|
+
const snapshot = latest;
|
|
51
|
+
latest = undefined;
|
|
52
|
+
return snapshot ? options.sink(snapshot) : Effect.void;
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
// Periodic throttled flush while the scope is open; a final flush captures the terminal state.
|
|
56
|
+
yield* Effect.forkScoped(Effect.forever(flush.pipe(Effect.delay(throttle))));
|
|
57
|
+
yield* Effect.addFinalizer(() => options.sink(progress.snapshot()).pipe(Effect.ignore));
|
|
58
|
+
}),
|
|
59
|
+
);
|
package/src/Stage.ts
CHANGED
|
@@ -4,9 +4,12 @@
|
|
|
4
4
|
|
|
5
5
|
// @import-as-namespace
|
|
6
6
|
|
|
7
|
+
import * as Cause from 'effect/Cause';
|
|
7
8
|
import * as Effect from 'effect/Effect';
|
|
8
9
|
import * as Stream from 'effect/Stream';
|
|
9
10
|
|
|
11
|
+
import * as Progress from './Progress';
|
|
12
|
+
|
|
10
13
|
/**
|
|
11
14
|
* Overflow policy applied when a consumer cannot keep pace, mapped onto `Stream.buffer` strategies.
|
|
12
15
|
* - `suspend`: back pressure — never drop (default; correct for email/document processing).
|
|
@@ -126,3 +129,28 @@ export const filter =
|
|
|
126
129
|
<In>(id: string, pred: (item: In) => boolean): Stage<In, In> =>
|
|
127
130
|
<E0, R0>(self: Stream.Stream<In, E0, R0>) =>
|
|
128
131
|
self.pipe(Stream.filter(pred));
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* A pass-through stage that reports live progress to the {@link Progress} registry: it registers the
|
|
135
|
+
* task (with an optional item `total`), advances the item index as each item flows through, marks the
|
|
136
|
+
* task done only when the stream ends successfully, and marks it failed if the stream errors.
|
|
137
|
+
* Progress is thus an artifact of the pipeline itself — the same stage works in the app (feeding a
|
|
138
|
+
* reactive panel) and in tests (feeding a file sink).
|
|
139
|
+
*/
|
|
140
|
+
export const track =
|
|
141
|
+
<In>(name: string, options: { total?: number; label?: string } = {}): Stage<In, In, never, Progress.Progress> =>
|
|
142
|
+
<E0, R0>(self: Stream.Stream<In, E0, R0>) =>
|
|
143
|
+
Stream.unwrap(
|
|
144
|
+
Effect.map(Progress.Progress, (progress) => {
|
|
145
|
+
const handle = progress.task(name, options);
|
|
146
|
+
// `done` only on successful completion (a concatenated terminal effect, unreached on error);
|
|
147
|
+
// `fail` on the error path. A finalizer/`ensuring` would report done even for a failed stream.
|
|
148
|
+
return Stream.concat(
|
|
149
|
+
self.pipe(
|
|
150
|
+
Stream.tap(() => Effect.sync(() => handle.advance())),
|
|
151
|
+
Stream.tapErrorCause((cause) => Effect.sync(() => handle.fail(Cause.pretty(cause)))),
|
|
152
|
+
),
|
|
153
|
+
Stream.execute(Effect.sync(() => handle.done())),
|
|
154
|
+
);
|
|
155
|
+
}),
|
|
156
|
+
);
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import * as Stream from 'effect/Stream';
|
|
7
|
+
import { describe, test } from 'vitest';
|
|
8
|
+
|
|
9
|
+
import { EffectEx } from '@dxos/effect';
|
|
10
|
+
|
|
11
|
+
import * as Pipeline from '../Pipeline';
|
|
12
|
+
import * as Stage from '../Stage';
|
|
13
|
+
import { renderBenchmark, runBenchmark } from './benchmark';
|
|
14
|
+
import { captureSink } from './capture';
|
|
15
|
+
import { instrument } from './instrument';
|
|
16
|
+
import { useMetrics } from './metrics';
|
|
17
|
+
|
|
18
|
+
describe('benchmark framework', () => {
|
|
19
|
+
test('instrument records per-stage counters and latency, even when items are dropped', async ({ expect }) => {
|
|
20
|
+
// A drop stage (out < in): odds are filtered, kept evens sleep a known amount. Because a
|
|
21
|
+
// dropped input has no output to attribute, `evens.ms` must reflect only the two kept items'
|
|
22
|
+
// ~10ms each — a FIFO pairing would mis-attribute a dropped item's entry time instead.
|
|
23
|
+
const evens = Stage.map('evens', (n: number) =>
|
|
24
|
+
n % 2 === 0 ? Effect.sleep('10 millis').pipe(Effect.as(n)) : Effect.succeed(undefined),
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
const result = await EffectEx.runPromise(
|
|
28
|
+
runBenchmark({
|
|
29
|
+
variants: [{ name: 'default', config: {} }],
|
|
30
|
+
program: () => {
|
|
31
|
+
const { sink, items } = captureSink<number>();
|
|
32
|
+
return Stream.fromIterable([1, 2, 3, 4])
|
|
33
|
+
.pipe(instrument('evens', evens), Pipeline.run({ sink }))
|
|
34
|
+
.pipe(Effect.as(items));
|
|
35
|
+
},
|
|
36
|
+
}),
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
const { metrics } = result.variants[0];
|
|
40
|
+
expect(metrics['evens.in']).toBe(4);
|
|
41
|
+
expect(metrics['evens.out']).toBe(2);
|
|
42
|
+
expect(metrics['evens.errors'] ?? 0).toBe(0);
|
|
43
|
+
// Two kept items sleeping ~10ms each; a generous lower bound avoids CI flakiness but still fails
|
|
44
|
+
// if latency were dropped or attributed to the wrong (instantaneous) entry.
|
|
45
|
+
expect(metrics['evens.ms']).toBeGreaterThanOrEqual(15);
|
|
46
|
+
expect(typeof metrics['total.ms']).toBe('number');
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test('stages emit custom metrics; evaluator adds effectful domain metrics', async ({ expect }) => {
|
|
50
|
+
// Stage opts into Metrics to count a domain event.
|
|
51
|
+
const flagLarge = Stage.map('flag', (n: number) =>
|
|
52
|
+
(n >= 3 ? useMetrics((metrics) => metrics.inc('large')) : Effect.void).pipe(Effect.as(n)),
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
const result = await EffectEx.runPromise(
|
|
56
|
+
runBenchmark({
|
|
57
|
+
variants: [{ name: 'v1', config: { threshold: 3 } }],
|
|
58
|
+
program: () => {
|
|
59
|
+
const { sink, items } = captureSink<number>();
|
|
60
|
+
return Stream.fromIterable([1, 2, 3, 4, 5]).pipe(flagLarge, Pipeline.run({ sink })).pipe(Effect.as(items));
|
|
61
|
+
},
|
|
62
|
+
evaluate: (_config, items) => Effect.succeed({ count: items.length, sum: items.reduce((a, b) => a + b, 0) }),
|
|
63
|
+
}),
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
const { metrics } = result.variants[0];
|
|
67
|
+
expect(metrics.large).toBe(3); // 3, 4, 5
|
|
68
|
+
expect(metrics.count).toBe(5);
|
|
69
|
+
expect(metrics.sum).toBe(15);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test('runs multiple variants and a failing variant is isolated', async ({ expect }) => {
|
|
73
|
+
const result = await EffectEx.runPromise(
|
|
74
|
+
runBenchmark<{ fail: boolean }, number, Error, never>({
|
|
75
|
+
variants: [
|
|
76
|
+
{ name: 'ok', config: { fail: false } },
|
|
77
|
+
{ name: 'bad', config: { fail: true } },
|
|
78
|
+
],
|
|
79
|
+
program: (config) => (config.fail ? Effect.fail(new Error('boom')) : Effect.succeed(42)),
|
|
80
|
+
evaluate: (_config, output) => Effect.succeed({ answer: output }),
|
|
81
|
+
}),
|
|
82
|
+
);
|
|
83
|
+
|
|
84
|
+
expect(result.variants.map((variant) => variant.variant)).toEqual(['ok', 'bad']);
|
|
85
|
+
expect(result.variants[0].metrics.answer).toBe(42);
|
|
86
|
+
expect(String(result.variants[1].metrics.error)).toContain('boom');
|
|
87
|
+
// The failing variant does not abort the comparison.
|
|
88
|
+
expect(result.variants).toHaveLength(2);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test('renderBenchmark tabulates metrics as rows and variants as columns', async ({ expect }) => {
|
|
92
|
+
const table = renderBenchmark({
|
|
93
|
+
variants: [
|
|
94
|
+
{ variant: 'gpt-oss', metrics: { 'total.ms': 1200, 'facts': 8 } },
|
|
95
|
+
{ variant: 'llama', metrics: { 'total.ms': 400, 'facts': 3 } },
|
|
96
|
+
],
|
|
97
|
+
});
|
|
98
|
+
expect(table).toContain('metric');
|
|
99
|
+
expect(table).toContain('gpt-oss');
|
|
100
|
+
expect(table).toContain('llama');
|
|
101
|
+
expect(table).toMatch(/facts\s+8\s+3/);
|
|
102
|
+
});
|
|
103
|
+
});
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Cause from 'effect/Cause';
|
|
6
|
+
import * as Duration from 'effect/Duration';
|
|
7
|
+
import * as Effect from 'effect/Effect';
|
|
8
|
+
import * as Layer from 'effect/Layer';
|
|
9
|
+
|
|
10
|
+
import { Metrics, type MetricsSnapshot, type MetricValue, makeMetrics } from './metrics';
|
|
11
|
+
|
|
12
|
+
/** A named configuration to benchmark (e.g. a model). */
|
|
13
|
+
export type BenchmarkVariant<Config> = {
|
|
14
|
+
readonly name: string;
|
|
15
|
+
readonly config: Config;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/** One variant's merged metrics: `total.ms`, per-stage `instrument` counters, and evaluator output. */
|
|
19
|
+
export type VariantResult = {
|
|
20
|
+
readonly variant: string;
|
|
21
|
+
readonly metrics: MetricsSnapshot;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export type BenchmarkResult = {
|
|
25
|
+
readonly variants: readonly VariantResult[];
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type BenchmarkOptions<Config, Output, E, R> = {
|
|
29
|
+
readonly variants: readonly BenchmarkVariant<Config>[];
|
|
30
|
+
/**
|
|
31
|
+
* Run the (instrumented) pipeline for one variant and return whatever the evaluator needs
|
|
32
|
+
* (e.g. captured sink outputs). Stage counters and any `Metrics` writes are collected
|
|
33
|
+
* automatically. `Metrics` is provided by the runner.
|
|
34
|
+
*/
|
|
35
|
+
readonly program: (config: Config) => Effect.Effect<Output, E, R | Metrics>;
|
|
36
|
+
/**
|
|
37
|
+
* Compute domain metrics for the variant. Effectful, so it can query services (fact store, ECHO
|
|
38
|
+
* db) that the pipeline populated. Its returned map is merged over the auto-collected metrics.
|
|
39
|
+
*/
|
|
40
|
+
readonly evaluate?: (config: Config, output: Output) => Effect.Effect<MetricsSnapshot, E, R | Metrics>;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Run a pipeline across variants (e.g. models) and collect a comparable metrics row per variant:
|
|
45
|
+
* total wall-clock, per-stage {@link instrument} counters, and the evaluator's domain metrics. A
|
|
46
|
+
* variant whose program or evaluator fails is recorded with an `error` (plus whatever metrics were
|
|
47
|
+
* captured before it failed) rather than aborting the whole comparison; an interruption
|
|
48
|
+
* (cancellation) is re-raised so the whole run stops promptly.
|
|
49
|
+
*/
|
|
50
|
+
export const runBenchmark = <Config, Output, E, R>(
|
|
51
|
+
options: BenchmarkOptions<Config, Output, E, R>,
|
|
52
|
+
): Effect.Effect<BenchmarkResult, E, Exclude<R, Metrics>> =>
|
|
53
|
+
Effect.gen(function* () {
|
|
54
|
+
const variants: VariantResult[] = [];
|
|
55
|
+
for (const variant of options.variants) {
|
|
56
|
+
const metrics = makeMetrics();
|
|
57
|
+
const layer = Layer.succeed(Metrics, metrics);
|
|
58
|
+
const row = yield* Effect.gen(function* () {
|
|
59
|
+
const heapBefore = heapUsedBytes();
|
|
60
|
+
const [duration, output] = yield* Effect.timed(options.program(variant.config));
|
|
61
|
+
const heapAfter = heapUsedBytes();
|
|
62
|
+
const domain = options.evaluate ? yield* options.evaluate(variant.config, output) : {};
|
|
63
|
+
const snapshot = yield* metrics.snapshot;
|
|
64
|
+
return {
|
|
65
|
+
'total.ms': Math.round(Duration.toMillis(duration)),
|
|
66
|
+
// Heap growth across the run (Node only; approximate — subject to GC, and for out-of-process
|
|
67
|
+
// model back-ends like Ollama it reflects harness overhead, not the model's own memory).
|
|
68
|
+
...(heapBefore !== undefined && heapAfter !== undefined
|
|
69
|
+
? { 'heap.mb': Math.round(((heapAfter - heapBefore) / (1024 * 1024)) * 100) / 100 }
|
|
70
|
+
: {}),
|
|
71
|
+
...snapshot,
|
|
72
|
+
...domain,
|
|
73
|
+
} satisfies MetricsSnapshot;
|
|
74
|
+
}).pipe(
|
|
75
|
+
Effect.provide(layer),
|
|
76
|
+
Effect.catchAllCause((cause) =>
|
|
77
|
+
// A cancelled benchmark must abort, not be recorded as an `error` row and iterated past;
|
|
78
|
+
// only genuine failures/defects degrade to a metrics row.
|
|
79
|
+
Cause.isInterruptedOnly(cause)
|
|
80
|
+
? Effect.failCause(cause)
|
|
81
|
+
: metrics.snapshot.pipe(
|
|
82
|
+
Effect.map(
|
|
83
|
+
(snapshot) => ({ ...snapshot, error: Cause.pretty(cause).split('\n')[0] }) satisfies MetricsSnapshot,
|
|
84
|
+
),
|
|
85
|
+
),
|
|
86
|
+
),
|
|
87
|
+
);
|
|
88
|
+
variants.push({ variant: variant.name, metrics: row });
|
|
89
|
+
}
|
|
90
|
+
return { variants };
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
/** Node process heap usage in bytes, or `undefined` off Node (browser) so the metric is simply omitted. */
|
|
94
|
+
const heapUsedBytes = (): number | undefined =>
|
|
95
|
+
typeof process !== 'undefined' && typeof process.memoryUsage === 'function'
|
|
96
|
+
? process.memoryUsage().heapUsed
|
|
97
|
+
: undefined;
|
|
98
|
+
|
|
99
|
+
const format = (value: MetricValue | undefined): string =>
|
|
100
|
+
value === undefined ? '' : typeof value === 'number' ? String(Math.round(value * 100) / 100) : value;
|
|
101
|
+
|
|
102
|
+
/** Render a benchmark result as a fixed-width metric × variant table (metrics as rows). */
|
|
103
|
+
export const renderBenchmark = (result: BenchmarkResult): string => {
|
|
104
|
+
const metricNames = [...new Set(result.variants.flatMap((variant) => Object.keys(variant.metrics)))].sort();
|
|
105
|
+
const header = ['metric', ...result.variants.map((variant) => variant.variant)];
|
|
106
|
+
const rows = metricNames.map((metric) => [
|
|
107
|
+
metric,
|
|
108
|
+
...result.variants.map((variant) => format(variant.metrics[metric])),
|
|
109
|
+
]);
|
|
110
|
+
const widths = header.map((_cell, column) => Math.max(...[header, ...rows].map((row) => row[column].length)));
|
|
111
|
+
const line = (row: readonly string[]) => row.map((cell, column) => cell.padEnd(widths[column])).join(' ');
|
|
112
|
+
return [line(header), widths.map((width) => '-'.repeat(width)).join(' '), ...rows.map(line)].join('\n');
|
|
113
|
+
};
|
package/src/testing/index.ts
CHANGED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Clock from 'effect/Clock';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import * as Stream from 'effect/Stream';
|
|
8
|
+
|
|
9
|
+
import * as Stage from '../Stage';
|
|
10
|
+
import { Metrics, useMetrics } from './metrics';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Wrap a {@link Stage.Stage} to auto-record per-stage metrics — `${name}.in`, `${name}.out`,
|
|
14
|
+
* `${name}.errors` (counts) and `${name}.ms` (summed processing latency) — through the ambient
|
|
15
|
+
* {@link Metrics} service, with no changes to the stage itself. Compose it in place of the stage:
|
|
16
|
+
* `source.pipe(instrument('extract', s), …)`. The wrapped stage additionally requires `Metrics` in
|
|
17
|
+
* its environment (provided at the edge by the benchmark runner).
|
|
18
|
+
*
|
|
19
|
+
* Latency is the wall-clock between the most recent input entering and an output leaving the stage.
|
|
20
|
+
* For a sequential stage (`Stage.map`, the common case) the item just emitted is the last one pulled
|
|
21
|
+
* in, so `${name}.ms` is the true per-item processing time — including drop/filter stages, whose
|
|
22
|
+
* discarded inputs simply never produce an output to attribute. It is only approximate for a stage
|
|
23
|
+
* that reorders or processes items concurrently (concurrency > 1), where the "most recent input" is
|
|
24
|
+
* no longer the one being emitted. Divide `${name}.ms` by `${name}.out` for the average.
|
|
25
|
+
*/
|
|
26
|
+
export const instrument =
|
|
27
|
+
<In, Out, E, R>(name: string, stage: Stage.Stage<In, Out, E, R>): Stage.Stage<In, Out, E, R | Metrics> =>
|
|
28
|
+
(self) => {
|
|
29
|
+
// Timestamp of the most recently ingested item. A sequential stage emits the item it last pulled,
|
|
30
|
+
// so pairing an output with this (rather than FIFO-oldest) stays correct when inputs are dropped.
|
|
31
|
+
let lastIn: number | undefined;
|
|
32
|
+
return stage(
|
|
33
|
+
self.pipe(
|
|
34
|
+
Stream.tap(() =>
|
|
35
|
+
Clock.currentTimeMillis.pipe(
|
|
36
|
+
Effect.flatMap((now) =>
|
|
37
|
+
useMetrics((metrics) => metrics.inc(`${name}.in`)).pipe(
|
|
38
|
+
Effect.zipRight(Effect.sync(() => void (lastIn = now))),
|
|
39
|
+
),
|
|
40
|
+
),
|
|
41
|
+
),
|
|
42
|
+
),
|
|
43
|
+
),
|
|
44
|
+
).pipe(
|
|
45
|
+
Stream.tap(() =>
|
|
46
|
+
Clock.currentTimeMillis.pipe(
|
|
47
|
+
Effect.flatMap((now) => {
|
|
48
|
+
const elapsed = lastIn === undefined ? 0 : now - lastIn;
|
|
49
|
+
return useMetrics((metrics) => metrics.inc(`${name}.out`)).pipe(
|
|
50
|
+
Effect.zipRight(useMetrics((metrics) => metrics.inc(`${name}.ms`, elapsed))),
|
|
51
|
+
);
|
|
52
|
+
}),
|
|
53
|
+
),
|
|
54
|
+
),
|
|
55
|
+
Stream.tapError(() => useMetrics((metrics) => metrics.inc(`${name}.errors`))),
|
|
56
|
+
);
|
|
57
|
+
};
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Context from 'effect/Context';
|
|
6
|
+
import * as Duration from 'effect/Duration';
|
|
7
|
+
import * as Effect from 'effect/Effect';
|
|
8
|
+
import * as Layer from 'effect/Layer';
|
|
9
|
+
|
|
10
|
+
/** A single reported metric: a running number (counter/sum) or a label. */
|
|
11
|
+
export type MetricValue = number | string;
|
|
12
|
+
|
|
13
|
+
/** A flat name → value map, as merged into a benchmark row. */
|
|
14
|
+
export type MetricsSnapshot = Record<string, MetricValue>;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Metrics sink a benchmark run writes to. Stages opt in to emit domain metrics (`inc`/`record`/
|
|
18
|
+
* `time`); the {@link instrument} wrapper writes per-stage counters through the same service. One
|
|
19
|
+
* instance per benchmark variant, provided at the edge via {@link Metrics.layer}.
|
|
20
|
+
*/
|
|
21
|
+
export interface MetricsApi {
|
|
22
|
+
/** Add `by` (default 1) to a numeric counter. */
|
|
23
|
+
readonly inc: (name: string, by?: number) => Effect.Effect<void>;
|
|
24
|
+
/** Set a value (last write wins) — for labels or gauges. */
|
|
25
|
+
readonly record: (name: string, value: MetricValue) => Effect.Effect<void>;
|
|
26
|
+
/** Time an effect: adds its duration (ms) to `${name}.ms` and increments `${name}.count`. */
|
|
27
|
+
readonly time: <A, E, R>(name: string, effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
|
|
28
|
+
/** Current values. */
|
|
29
|
+
readonly snapshot: Effect.Effect<MetricsSnapshot>;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export class Metrics extends Context.Tag('@dxos/pipeline/testing/Metrics')<Metrics, MetricsApi>() {
|
|
33
|
+
/** A fresh, in-memory metrics instance. */
|
|
34
|
+
static layer: Layer.Layer<Metrics> = Layer.sync(Metrics, () => makeMetrics());
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Construct a standalone metrics sink (also usable directly in tests). */
|
|
38
|
+
export const makeMetrics = (): MetricsApi => {
|
|
39
|
+
const values = new Map<string, MetricValue>();
|
|
40
|
+
const addNumber = (name: string, by: number) => {
|
|
41
|
+
const current = values.get(name);
|
|
42
|
+
// A name used as both a string label (record) and a counter (inc/time) is a caller bug; fail
|
|
43
|
+
// loudly rather than silently discarding the label and counting from zero.
|
|
44
|
+
if (typeof current === 'string') {
|
|
45
|
+
throw new Error(`Metric '${name}' already holds a string label; cannot increment it.`);
|
|
46
|
+
}
|
|
47
|
+
values.set(name, (current ?? 0) + by);
|
|
48
|
+
};
|
|
49
|
+
const api: MetricsApi = {
|
|
50
|
+
inc: (name, by = 1) => Effect.sync(() => addNumber(name, by)),
|
|
51
|
+
record: (name, value) => Effect.sync(() => void values.set(name, value)),
|
|
52
|
+
time: (name, effect) =>
|
|
53
|
+
Effect.timed(effect).pipe(
|
|
54
|
+
Effect.map(([duration, result]) => {
|
|
55
|
+
addNumber(`${name}.ms`, Duration.toMillis(duration));
|
|
56
|
+
addNumber(`${name}.count`, 1);
|
|
57
|
+
return result;
|
|
58
|
+
}),
|
|
59
|
+
),
|
|
60
|
+
snapshot: Effect.sync(() => Object.fromEntries(values)),
|
|
61
|
+
};
|
|
62
|
+
return api;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** Read the ambient {@link Metrics} service and apply `f` to it. */
|
|
66
|
+
export const useMetrics = <A, E, R>(
|
|
67
|
+
f: (metrics: MetricsApi) => Effect.Effect<A, E, R>,
|
|
68
|
+
): Effect.Effect<A, E, R | Metrics> => Effect.flatMap(Metrics, f);
|