@pikku/core 0.12.148 → 0.12.151

package/dist/testing/queue-trace-conformance.d.ts

@@ -0,0 +1,26 @@
+import type { QueueService } from '../wirings/queue/queue.types.js';
+/** A real queue adapter, as this suite needs it. */
+export type QueueTraceAdapter = {
+    service: QueueService;
+    /** Registers workers for what `registerQueue` declared. */
+    startWorkers: () => Promise<void>;
+    /** Stops workers and closes every connection. */
+    stop: () => Promise<void>;
+    /** Publishes `payload` as it stands, as a producer older than tracing did. */
+    publishRaw: (queueName: string, payload: unknown) => Promise<void>;
+    /** The trace travels in the payload, so a worker unwraps what it is given. */
+    envelope?: boolean;
+};
+/**
+ * Runs each outcome × inbound × timing as one job through a real adapter, in one
+ * batch so it stays fast against a broker, then jobs in the formats of older and
+ * newer producers. Asserts on what the worker saw, so a wire format that loses
+ * or changes the trace fails here on the adapter that has it.
+ *
+ * @param name - The adapter, for the test names.
+ * @param create - Connects the adapter.
+ * @param options.settleMs - How long a batch may take, retries included.
+ */
+export declare const defineQueueTraceConformance: (name: string, create: () => Promise<QueueTraceAdapter>, { settleMs }?: {
+    settleMs?: number;
+}) => void;

package/dist/testing/queue-trace-conformance.js

@@ -0,0 +1,259 @@
+import assert from 'node:assert/strict';
+import { after, before, describe, test } from 'node:test';
+import { ConflictError } from '../errors/errors.js';
+import { AbandonedError } from '../function/abort-scope.js';
+import { addFunction } from '../function/function-runner.js';
+import { pikkuState, resetPikkuState } from '../pikku-state.js';
+import { parseTraceparent } from '../trace.js';
+import { SignedQueueService } from '../wirings/queue/signed-queue-service.js';
+const OUTCOMES = [
+    'success',
+    'throw-pikku-error',
+    'throw-plain-error',
+    'job-failed',
+    'job-discarded',
+    'abandoned',
+];
+const INBOUNDS = ['none', 'valid', 'invalid', 'forged'];
+const TIMINGS = [
+    'immediate',
+    'delayed',
+    'retried',
+    'duplicate',
+    'concurrent',
+];
+const hex = (length) => Array.from({ length }, () => Math.floor(Math.random() * 16).toString(16)).join('');
+const inboundFor = (inbound) => {
+    switch (inbound) {
+        case 'none':
+            return undefined;
+        case 'valid':
+            return `00-1${hex(31)}-2${hex(15)}-01`;
+        case 'invalid':
+            return `not-a-traceparent-${hex(4)}`;
+        case 'forged':
+            return `00-${'0'.repeat(32)}-${hex(16)}-01`;
+    }
+};
+const expectedAttempts = (outcome, timing) => {
+    if (timing === 'duplicate')
+        return 2;
+    if (timing !== 'retried')
+        return 1;
+    return outcome === 'job-discarded' ? 1 : 2;
+};
+const SEEDED_SECRETS = {
+    getSecret: async () => ({ reveal: () => 'x'.repeat(64) }),
+    hasSecret: async () => true,
+};
+/**
+ * Runs each outcome × inbound × timing as one job through a real adapter, in one
+ * batch so it stays fast against a broker, then jobs in the formats of older and
+ * newer producers. Asserts on what the worker saw, so a wire format that loses
+ * or changes the trace fails here on the adapter that has it.
+ *
+ * @param name - The adapter, for the test names.
+ * @param create - Connects the adapter.
+ * @param options.settleMs - How long a batch may take, retries included.
+ */
+export const defineQueueTraceConformance = (name, create, { settleMs = 30_000 } = {}) => {
+    describe(`${name}: a job keeps its trace`, () => {
+        const queue = `trace_conformance_${hex(8)}`;
+        const attempts = new Map();
+        const logs = [];
+        let adapter;
+        const sent = new Map();
+        const logger = (scope) => {
+            const write = (level) => () => void logs.push({ level, scope });
+            return {
+                info: write('info'),
+                warn: write('warn'),
+                debug: write('debug'),
+                error: write('error'),
+                scope: (traceId) => logger(traceId),
+            };
+        };
+        const worker = async (_services, data, wire) => {
+            const key = String(data?.id ?? data?.data?.id ?? data);
+            const seen = attempts.get(key) ?? [];
+            seen.push({ trace: wire.trace, data });
+            attempts.set(key, seen);
+            if (data?.failFirst && seen.length === 1 && data.outcome === 'success') {
+                throw new Error('flaky');
+            }
+            switch (data?.outcome) {
+                case 'throw-pikku-error':
+                    throw new ConflictError('refused');
+                case 'throw-plain-error':
+                    throw new Error('boom');
+                case 'job-failed':
+                    return wire.queue.fail('no');
+                case 'job-discarded':
+                    return wire.queue.discard('no');
+                case 'abandoned':
+                    throw new AbandonedError('the caller left');
+            }
+        };
+        before(async () => {
+            resetPikkuState();
+            pikkuState(null, 'package', 'singletonServices', {
+                logger: logger(),
+                config: {},
+                secrets: SEEDED_SECRETS,
+            });
+            const funcId = `queue_${queue}`;
+            pikkuState(null, 'queue', 'meta')[queue] = {
+                pikkuFuncId: funcId,
+                name: queue,
+            };
+            pikkuState(null, 'function', 'meta')[funcId] = {
+                pikkuFuncId: funcId,
+                inputSchemaName: null,
+                outputSchemaName: null,
+                sessionless: true,
+            };
+            addFunction(funcId, { func: worker, auth: false });
+            pikkuState(null, 'queue', 'registrations').set(queue, {
+                name: queue,
+                func: { func: worker, auth: false },
+            });
+            adapter = await create();
+            await adapter.startWorkers();
+            const adds = [];
+            for (const outcome of OUTCOMES) {
+                for (const inbound of INBOUNDS) {
+                    for (const timing of TIMINGS) {
+                        for (const copy of timing === 'concurrent' ? [0, 1] : [0]) {
+                            const id = hex(16);
+                            const label = `${outcome} / ${inbound} / ${timing}${copy ? ' #2' : ''}`;
+                            const traceparent = inboundFor(inbound);
+                            const job = {
+                                id,
+                                outcome,
+                                failFirst: timing === 'retried',
+                            };
+                            sent.set(id, {
+                                label,
+                                traceparent,
+                                inbound,
+                                expected: expectedAttempts(outcome, timing),
+                            });
+                            const options = {
+                                traceparent,
+                                attempts: timing === 'retried' ? 2 : 1,
+                                backoff: { type: 'fixed', delay: 10 },
+                                ...(timing === 'delayed' ? { delay: 50 } : {}),
+                            };
+                            adds.push(adapter.service.add(queue, job, options));
+                            if (timing === 'duplicate') {
+                                adds.push(adapter.service.add(queue, job, options));
+                            }
+                        }
+                    }
+                }
+            }
+            await Promise.all(adds);
+            const deadline = Date.now() + settleMs;
+            const pending = () => [...sent].filter(([id, { expected }]) => (attempts.get(id)?.length ?? 0) < expected);
+            while (pending().length && Date.now() < deadline) {
+                await new Promise((resolve) => setTimeout(resolve, 50));
+            }
+        });
+        after(async () => {
+            await adapter.stop();
+        });
+        test('every job ran as often as it should', () => {
+            const short = [...sent]
+                .filter(([id, { expected }]) => (attempts.get(id)?.length ?? 0) !== expected)
+                .map(([id, { label }]) => `${label}: ${attempts.get(id)?.length ?? 0}`);
+            assert.deepEqual(short, []);
+        });
+        for (const outcome of OUTCOMES) {
+            for (const inbound of INBOUNDS) {
+                for (const timing of TIMINGS) {
+                    test(`${outcome} / ${inbound} / ${timing}`, () => {
+                        const jobs = [...sent].filter(([, { label }]) => label.startsWith(`${outcome} / ${inbound} / ${timing}`));
+                        assert.ok(jobs.length > 0);
+                        for (const [id, { traceparent }] of jobs) {
+                            const seen = (attempts.get(id) ?? []).map(({ trace }) => trace);
+                            assert.ok(seen.length > 0, 'the job never ran');
+                            const carried = parseTraceparent(traceparent);
+                            for (const trace of seen) {
+                                assert.match(trace.traceId, /^[0-9a-f]{32}$/);
+                                assert.match(trace.spanId, /^[0-9a-f]{16}$/);
+                                if (carried) {
+                                    assert.equal(trace.traceId, carried.traceId, 'trace changed');
+                                    assert.equal(trace.parentSpanId, carried.parentSpanId);
+                                }
+                                else {
+                                    assert.equal(trace.parentSpanId, undefined, 'not a new trace');
+                                    if (traceparent) {
+                                        assert.ok(!traceparent.includes(trace.traceId), 'adopted');
+                                    }
+                                }
+                            }
+                            const spans = seen.map(({ spanId }) => spanId);
+                            assert.equal(new Set(spans).size, spans.length, 'a span repeats');
+                        }
+                    });
+                }
+            }
+        }
+        test('error logs from running a job carry its trace', () => {
+            const traces = new Set([...attempts.values()].flat().map(({ trace }) => trace.traceId));
+            const failures = logs.filter(({ level, scope }) => level === 'error' && scope);
+            assert.ok(failures.length > 0, 'no failure was logged');
+            for (const { scope } of failures) {
+                assert.ok(traces.has(scope), `a log carries a stray trace ${scope}`);
+            }
+        });
+        describe('jobs from producers of every age', () => {
+            const run = async (id, publish) => {
+                await publish();
+                const deadline = Date.now() + settleMs;
+                while (!attempts.get(id)?.length && Date.now() < deadline) {
+                    await new Promise((resolve) => setTimeout(resolve, 20));
+                }
+                const [attempt] = attempts.get(id) ?? [];
+                assert.ok(attempt, 'the job never ran');
+                return attempt;
+            };
+            test('old format: no trace at all', async () => {
+                const data = { id: hex(16), outcome: 'success' };
+                const { trace, data: seen } = await run(data.id, () => adapter.publishRaw(queue, data));
+                assert.deepEqual(seen, data);
+                assert.equal(trace.parentSpanId, undefined);
+            });
+            test('new format: a trace alongside a signed identity', async () => {
+                const data = { id: hex(16), outcome: 'success' };
+                const traceparent = inboundFor('valid');
+                const signed = new SignedQueueService(adapter.service, SEEDED_SECRETS, logger());
+                const { trace, data: seen } = await run(data.id, () => signed.add(queue, data, {
+                    traceparent,
+                    pikkuUserId: 'user-1',
+                }));
+                assert.deepEqual(seen, data);
+                assert.equal(trace.traceId, parseTraceparent(traceparent).traceId);
+            });
+            test('data shaped like the envelope, with and without a trace', async () => {
+                for (const traceparent of [undefined, inboundFor('valid')]) {
+                    const id = hex(16);
+                    const shaped = { __pikkuTraceparent: 'mine', data: { id } };
+                    const key = `shaped-${id}`;
+                    const { data } = await run(key, () => adapter.service.add(queue, { ...shaped, id: key }, {
+                        traceparent,
+                    }));
+                    assert.deepEqual(data, { ...shaped, id: key });
+                }
+            });
+            test('a poison envelope starts a new trace and keeps the data', async () => {
+                const id = hex(16);
+                const inner = { id, outcome: 'success' };
+                const poison = { __pikkuTraceparent: 'garbage', data: inner };
+                const { trace, data } = await run(id, () => adapter.publishRaw(queue, poison));
+                assert.equal(trace.parentSpanId, undefined);
+                assert.deepEqual(data, adapter.envelope ? inner : poison);
+            });
+        });
+    });
+};

package/dist/testing/service-tests/channel-store-tests.js

@@ -20,6 +20,17 @@ export const defineChannelStoreTests = (name, channelStore) => {
             assert.deepEqual(result.openingData, { foo: 'bar' });
             assert.equal(result.pikkuUserId, undefined);
         });
+        test('keeps the traceparent the channel was added with', async () => {
+            const traceparent = `00-${'a'.repeat(32)}-${'b'.repeat(16)}-01`;
+            await store.addChannel({
+                channelId: 'ch-traced',
+                channelName: 'test-channel',
+                traceparent,
+            });
+            assert.equal((await store.getChannel('ch-traced')).traceparent, traceparent);
+            assert.equal((await store.getChannel('ch-1')).traceparent, undefined);
+            await store.removeChannels(['ch-traced']);
+        });
         test('setPikkuUserId', async () => {
             await store.setPikkuUserId('ch-1', 'user-1');
             const result = await store.getChannel('ch-1');

package/dist/testing/service-tests/queued-workflow-harness.d.ts

@@ -32,11 +32,23 @@ export declare class QueuedWorkflowHarness {
         queue: string;
         step?: string;
     }>;
+    /** The `traceparent` of every job added, in order. */
+    jobTraceparents: Array<string | undefined>;
+    /** The wire trace of every step and compensation invoked, in order. */
+    traces: Array<{
+        rpc: string;
+        trace: any;
+    }>;
     ws: PikkuWorkflowService;
     /** Return true to drop the job instead of running it, simulating a worker that never ran. */
     dropWhen?: (job: Job) => boolean;
     /** Make `queue.add` throw for a job, simulating the broker being unreachable. */
     rejectAddWhen?: (queue: string, data: any) => boolean;
+    /** Carries a job in an adapter's wire format, as it would reach a worker. */
+    transport?: (data: any, options?: any) => {
+        data: any;
+        traceparent?: string;
+    };
     constructor(service: PikkuWorkflowService);
     readonly rpc: {
         rpcWithWire: (rpcName: string, data: any, wire: any) => Promise<any>;
@@ -44,7 +56,12 @@ export declare class QueuedWorkflowHarness {
     register(name: string, handler: Handler): void;
     defineDsl(name: string, body: (workflow: any, input: any) => Promise<any>): void;
     defineGraph(name: string, entry: string, nodes: Record<string, any>): void;
-    start(name: string, input?: any): Promise<string>;
+    start(name: string, input?: any, wire?: {
+        type: string;
+        traceparent?: string;
+    }, options?: {
+        inline?: boolean;
+    }): Promise<string>;
     /** Run jobs until the queue is empty. */
     pump(limit?: number): Promise<void>;
     run(runId: string): Promise<import("../../wirings/workflow/workflow.types.js").WorkflowRun>;

package/dist/testing/service-tests/queued-workflow-harness.js

@@ -12,11 +12,17 @@ export class QueuedWorkflowHarness {
     recovering = [];
     queue = [];
     jobsRun = [];
+    /** The `traceparent` of every job added, in order. */
+    jobTraceparents = [];
+    /** The wire trace of every step and compensation invoked, in order. */
+    traces = [];
     ws;
     /** Return true to drop the job instead of running it, simulating a worker that never ran. */
     dropWhen;
     /** Make `queue.add` throw for a job, simulating the broker being unreachable. */
     rejectAddWhen;
+    /** Carries a job in an adapter's wire format, as it would reach a worker. */
+    transport;
     constructor(service) {
         resetPikkuState();
         this.ws = service;
@@ -25,7 +31,16 @@ export class QueuedWorkflowHarness {
                 if (this.rejectAddWhen?.(queue, data)) {
                     throw new Error('broker unreachable');
                 }
-                this.queue.push({ queue, data, attempts: options?.attempts ?? 1 });
+                const carried = this.transport?.(data, options) ?? {
+                    data,
+                    traceparent: options?.traceparent,
+                };
+                this.jobTraceparents.push(carried.traceparent);
+                this.queue.push({
+                    queue,
+                    data: carried.data,
+                    attempts: options?.attempts ?? 1,
+                });
             },
         };
         pikkuState(null, 'package', 'singletonServices', {
@@ -36,6 +51,7 @@ export class QueuedWorkflowHarness {
     }
     rpc = {
         rpcWithWire: async (rpcName, data, wire) => {
+            this.traces.push({ rpc: rpcName, trace: wire.trace });
             const isCompensation = rpcName.endsWith(':compensate');
             const base = isCompensation
                 ? rpcName.slice(0, -':compensate'.length)
@@ -97,8 +113,8 @@ export class QueuedWorkflowHarness {
             ])),
         };
     }
-    async start(name, input = {}) {
-        const { runId } = await this.ws.startWorkflow(name, input, { type: 'test' }, this.rpc);
+    async start(name, input = {}, wire = { type: 'test' }, options) {
+        const { runId } = await this.ws.startWorkflow(name, input, wire, this.rpc, options);
         await this.pump();
         return runId;
     }

package/dist/testing/service-tests/workflow-service-tests.js

@@ -19,6 +19,18 @@ export const defineWorkflowServiceTests = (name, workflowService) => {
             assert.deepEqual(run.input, { key: 'value' });
             assert.equal(run.graphHash, 'hash-1');
         });
+        test('a run keeps the traceparent it was created with', async () => {
+            const traceparent = `00-${'a'.repeat(32)}-${'b'.repeat(16)}-01`;
+            const runId = await service.createRun('traced-workflow', {}, false, 'hash-trace', { ...wire, traceparent });
+            const run = await service.getRun(runId);
+            assert.equal(run?.wire?.traceparent, traceparent);
+            assert.equal(run?.wire?.type, wire.type);
+        });
+        test('a run created without a trace is given one', async () => {
+            const runId = await service.createRun('untraced-workflow', {}, false, 'hash-untraced', wire);
+            const run = await service.getRun(runId);
+            assert.match(run?.wire?.traceparent ?? '', /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/);
+        });
         test('updateRunStatus', async () => {
             const runId = await service.createRun('status-workflow', {}, false, 'hash-2', wire);
             await service.updateRunStatus(runId, 'completed', { result: 'done' });

package/dist/testing/trace-matrix/assertions.d.ts

@@ -0,0 +1,3 @@
+import { type TraceChain } from './chain-harness.js';
+/** What every generated test asserts about a chain, once it has run. */
+export declare const assertChain: (chain: TraceChain) => Promise<void>;

package/dist/testing/trace-matrix/assertions.js

@@ -0,0 +1,190 @@
+import assert from 'node:assert/strict';
+import { parseTraceparent } from '../../trace.js';
+import { inboundContinues, } from './chain-harness.js';
+import { outcomeError } from './workflow-cases.js';
+const TRACE_ID = /^[0-9a-f]{32}$/;
+const SPAN_ID = /^[0-9a-f]{16}$/;
+const assertWellFormed = (trace, label) => {
+    assert.match(trace.traceId, TRACE_ID, label);
+    assert.match(trace.spanId, SPAN_ID, label);
+};
+const assertUniqueSpans = (traces, label) => {
+    const spans = traces.map((trace) => trace.spanId);
+    assert.equal(new Set(spans).size, spans.length, `${label}: spans repeat`);
+};
+/** Steps and compensations of one run, from the harness's record. */
+const stepTraces = (chain, { index }) => chain.h.traces
+    .filter(({ rpc }) => new RegExp(`-${index}(:compensate)?$`).test(rpc))
+    .map(({ trace }) => trace);
+const assertEntry = (chain, instance) => {
+    const { start } = chain.combo;
+    const inbound = chain.inbound;
+    const sent = parseTraceparent(instance.sent);
+    const continues = inboundContinues(start, inbound) && !chain.meshGuardsEntry;
+    for (const entry of instance.entries) {
+        assertWellFormed(entry, 'entry');
+        if (continues) {
+            assert.equal(entry.traceId, sent.traceId, 'the trace was not continued');
+            // Parsing names the sender's span as the parent of what comes next.
+            const parent = instance.caller?.spanId ?? sent.parentSpanId;
+            assert.equal(entry.parentSpanId, parent, 'wrong parent span');
+            assert.notEqual(entry.spanId, parent, 'no new span');
+        }
+        else {
+            if (sent) {
+                assert.notEqual(entry.traceId, sent.traceId, 'adopted an untrusted id');
+            }
+            // These hold a root span of their own and call the function from it.
+            if (!['channel', 'agent-tool', 'trigger'].includes(start)) {
+                assert.equal(entry.parentSpanId, undefined, 'a root has no parent');
+            }
+        }
+    }
+    assertUniqueSpans(instance.entries, 'entry');
+    // A job with no valid trace has none to keep: each delivery starts one, as
+    // `runQueueJob` documents. A trace a job does carry survives every delivery.
+    if (continues) {
+        const ids = new Set(instance.entries.map((entry) => entry.traceId));
+        assert.ok(ids.size <= 1, 'redelivery changed the trace');
+    }
+};
+const assertHops = (chain, instance) => {
+    const entries = instance.entries;
+    const parents = new Map(entries.map((entry) => [entry.spanId, entry]));
+    for (const relay of instance.relays) {
+        const parent = parents.get(relay.parentSpanId);
+        assert.ok(parent, `a relay hangs off ${relay.parentSpanId}`);
+        parents.set(relay.spanId, parent);
+    }
+    for (const target of instance.targets) {
+        assertWellFormed(target, 'target');
+        const parent = parents.get(target.parentSpanId);
+        assert.ok(parent, `wrong parent: ${target.parentSpanId} is none of ${[...parents.keys()]}`);
+        assert.equal(target.traceId, parent.traceId, 'the hop changed the trace');
+        assert.ok(!parents.has(target.spanId), 'the hop kept the span');
+    }
+    assertUniqueSpans(instance.targets, 'target');
+    for (const outgoing of instance.outgoing) {
+        const parsed = parseTraceparent(outgoing);
+        assert.ok(parsed, `invalid outgoing traceparent ${outgoing}`);
+        assert.ok(entries.some((entry) => entry.traceId === parsed.traceId), 'the wire carries another trace');
+    }
+};
+const assertRun = async (chain, instance) => {
+    if (!instance.runId)
+        return;
+    const run = await chain.h.run(instance.runId);
+    const stored = parseTraceparent(run.wire.traceparent);
+    assert.ok(stored, 'the run lost its stored trace');
+    // A redelivered entry starts a run per delivery; each hangs off its own.
+    const starter = (span) => instance.entries.find((entry) => entry.spanId === span);
+    const entry = starter(stored.parentSpanId);
+    assert.ok(entry, 'the run stored another span');
+    assert.equal(stored.traceId, entry.traceId, 'the run stored another trace');
+    const steps = stepTraces(chain, instance);
+    for (const step of steps) {
+        assert.ok(step, 'a step ran with no trace');
+        const parent = starter(step.parentSpanId);
+        assert.ok(parent, 'steps hang off the start');
+        assert.equal(step.traceId, parent.traceId, 'a step left the trace');
+    }
+    assertUniqueSpans(steps, 'step');
+    for (const job of chain.h.jobTraceparents) {
+        const parsed = parseTraceparent(job);
+        assert.ok(parsed, 'a workflow job lost its trace');
+    }
+};
+/** The instance's trace id, wherever the chain got far enough to show it. */
+const traceOf = (chain, instance) => instance.entries[0]?.traceId ??
+    instance.errorId ??
+    chain.logs.find(({ level, scope }) => level === 'error' && scope)?.scope;
+const assertFailure = (chain, instance) => {
+    const { start, hop, outcome } = chain.combo;
+    const traceId = traceOf(chain, instance);
+    assert.ok(traceId, 'the chain never showed a trace');
+    if (['http', 'channel', 'mcp'].includes(start) && instance.callerRefused) {
+        assert.ok(instance.errorId, 'the caller was given no errorId');
+    }
+    if (instance.errorId) {
+        assert.equal(instance.errorId, traceId, 'errorId is not the trace id');
+    }
+    if (hop === 'mesh' && instance.calleeErrorId && outcome !== 'auth-mesh-jwt') {
+        assert.ok(instance.entries.some(({ traceId: id }) => id === instance.calleeErrorId), 'the callee errorId differs');
+    }
+};
+/**
+ * Every trace id the chain owns. A job refused before its function runs shows
+ * no entry, so what it owns is the trace it carried, or else the one each
+ * delivery started and logged under.
+ */
+const tracesOf = (chain) => {
+    const carried = (instance) => inboundContinues(chain.combo.start, chain.inbound) && !chain.meshGuardsEntry
+        ? [parseTraceparent(instance.sent)?.traceId]
+        : [];
+    const refused = chain.instances.some(({ entries }) => !entries.length);
+    return new Set(chain.instances
+        .flatMap((instance) => [
+        traceOf(chain, instance),
+        ...carried(instance),
+        ...instance.entries.map((entry) => entry.traceId),
+    ])
+        .concat(refused ? chain.logs.map(({ scope }) => scope) : []));
+};
+const assertLogs = (chain) => {
+    const traces = tracesOf(chain);
+    for (const { level, scope, message } of chain.logs) {
+        if (level === 'error') {
+            assert.ok(scope !== undefined, `an error logged with no trace: ${JSON.stringify(message)?.slice(0, 120)}`);
+        }
+        if (scope !== undefined && chain.combo.outcome !== 'auth-mesh-jwt') {
+            assert.ok(traces.has(scope), `a log carries a stray trace ${scope}`);
+        }
+    }
+};
+const assertTelemetry = (chain) => {
+    const seen = tracesOf(chain);
+    for (const { message } of chain.logs) {
+        const record = message;
+        if (!record?.__pikku_telemetry)
+            continue;
+        assert.ok(seen.has(record.traceId), `telemetry carries ${record.traceId}, not a trace of this chain`);
+    }
+};
+/** Chains running at once must not share or swap anything. */
+const assertIsolated = (chain) => {
+    if (chain.instances.length < 2)
+        return;
+    const owners = new Map();
+    const own = (traceId, index) => {
+        const owner = owners.get(traceId) ?? index;
+        assert.equal(owner, index, 'two concurrent starts share a trace');
+        owners.set(traceId, index);
+    };
+    for (const instance of chain.instances) {
+        const { index, entries, targets, outgoing } = instance;
+        own(traceOf(chain, instance), index);
+        for (const { traceId } of [...entries, ...targets])
+            own(traceId, index);
+        for (const hop of outgoing)
+            own(parseTraceparent(hop)?.traceId, index);
+    }
+};
+/** What every generated test asserts about a chain, once it has run. */
+export const assertChain = async (chain) => {
+    for (const instance of chain.instances) {
+        assertEntry(chain, instance);
+        assertHops(chain, instance);
+        await assertRun(chain, instance);
+        if (outcomeError(chain.combo.outcome) || instance.failure) {
+            assertFailure(chain, instance);
+        }
+        if (chain.timing === 'job-after-parent') {
+            for (const startedAt of instance.targetStartedAt) {
+                assert.ok(startedAt >= instance.entryFinishedAt, 'ran before the parent finished');
+            }
+        }
+    }
+    assertLogs(chain);
+    assertTelemetry(chain);
+    assertIsolated(chain);
+};