@fedify/fedify 2.4.0-dev.1564 → 2.4.0-dev.1570

package/dist/federation/tasks/tasks.test.mjs

@@ -0,0 +1,1228 @@
+import { Temporal } from "@js-temporal/polyfill";
+import "urlpattern-polyfill";
+globalThis.addEventListener = () => {};
+import { t as MemoryKvStore } from "../../kv-CIvR3Non.mjs";
+import { a as numberSchema, i as makeSchema, n as baseOptions, o as stringSchema, t as MockQueue } from "../../testing-Dy-WhqpN.mjs";
+import { o as createFederation } from "../../middleware-CjHqy-1i.mjs";
+import { r as createFederationBuilder } from "../../builder-CJnWbgbj.mjs";
+import { t as TaskCodec } from "../../codec-nfBU4Up4.mjs";
+import { t as InProcessMessageQueue } from "../../mq-03vS-C9P.mjs";
+import { W3CTraceContextPropagator } from "@opentelemetry/core";
+import { Note } from "@fedify/vocab";
+import { SpanStatusCode, propagation } from "@opentelemetry/api";
+import { delay } from "es-toolkit";
+import { createTestMeterProvider, createTestTracerProvider, mockDocumentLoader, test } from "@fedify/fixture";
+import { deepStrictEqual, ok, rejects, strictEqual, throws } from "node:assert/strict";
+//#region src/federation/tasks/tasks.test.ts
+const envelopeSchema = makeSchema((data) => typeof data === "object" && data != null && data.note instanceof Note && typeof data.title === "string");
+const makeTaskMessage = async (taskName, data, overrides = {}) => ({
+	type: "task",
+	id: crypto.randomUUID(),
+	baseUrl: "https://example.com/",
+	taskName,
+	data: await new TaskCodec({ contextLoader: mockDocumentLoader }).serialize(data),
+	started: Temporal.Now.instant().toString(),
+	attempt: 0,
+	traceContext: {},
+	...overrides
+});
+async function waitFor(predicate, timeoutMs) {
+	const started = Date.now();
+	while (!predicate()) {
+		await delay(50);
+		if (Date.now() - started > timeoutMs) throw new Error("Timeout");
+	}
+}
+test("defineTask()", async (t) => {
+	await t.step("returns a handle carrying name and schema", () => {
+		const task = createFederation({
+			...baseOptions,
+			queue: { task: new MockQueue() }
+		}).defineTask("greet", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		strictEqual(task.name, "greet");
+		strictEqual(task.schema, stringSchema);
+	});
+	await t.step("throws on a duplicate name", () => {
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: new MockQueue() }
+		});
+		federation.defineTask("dup", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		throws(() => federation.defineTask("dup", {
+			schema: stringSchema,
+			handler: () => {}
+		}), {
+			name: "TypeError",
+			message: /already defined/
+		});
+	});
+	await t.step("accepts names that collide with Object.prototype", () => {
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: new MockQueue() }
+		});
+		for (const name of [
+			"constructor",
+			"toString",
+			"hasOwnProperty"
+		]) {
+			strictEqual(federation.defineTask(name, {
+				schema: stringSchema,
+				handler: () => {}
+			}).name, name);
+			strictEqual(federation.taskDefinitions.get(name)?.name, name);
+		}
+		throws(() => federation.defineTask("toString", {
+			schema: stringSchema,
+			handler: () => {}
+		}), {
+			name: "TypeError",
+			message: /already defined/
+		});
+	});
+	await t.step("build() clones the task registry", async () => {
+		const builder = createFederationBuilder();
+		builder.defineTask("first", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const f1 = await builder.build({
+			...baseOptions,
+			queue: { task: new MockQueue() }
+		});
+		builder.defineTask("second", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const f2 = await builder.build({
+			...baseOptions,
+			queue: { task: new MockQueue() }
+		});
+		deepStrictEqual([...f1.taskDefinitions.keys()], ["first"]);
+		deepStrictEqual([...f2.taskDefinitions.keys()], ["first", "second"]);
+		f1.defineTask("third", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		deepStrictEqual([...f2.taskDefinitions.keys()], ["first", "second"]);
+	});
+});
+test("task type-level guards", () => {});
+test("Context.enqueueTask() end-to-end", async (t) => {
+	await t.step("round-trips a typed payload to the handler", async () => {
+		const queue = new InProcessMessageQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		const received = [];
+		const task = federation.defineTask("notify", {
+			schema: envelopeSchema,
+			handler: (ctx, data) => {
+				received.push({
+					ctx,
+					data
+				});
+			}
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, {
+			note: new Note({ content: "Hello, world!" }),
+			title: "greeting"
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, {
+			signal: controller.signal,
+			queue: "task"
+		});
+		try {
+			await waitFor(() => received.length > 0, 15e3);
+		} finally {
+			controller.abort();
+			await listening;
+		}
+		strictEqual(received.length, 1);
+		const { ctx: handlerCtx, data } = received[0];
+		ok(data.note instanceof Note);
+		strictEqual(data.note.content?.toString(), "Hello, world!");
+		strictEqual(data.title, "greeting");
+		strictEqual(handlerCtx.origin, "https://example.com");
+	});
+});
+test("task queue routing", async (t) => {
+	await t.step("prefers the per-task queue", async () => {
+		const taskQueue = new MockQueue();
+		const perTaskQueue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: taskQueue }
+		});
+		const task = federation.defineTask("isolated", {
+			schema: stringSchema,
+			handler: () => {},
+			queue: perTaskQueue
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "data");
+		strictEqual(perTaskQueue.enqueued.length, 1);
+		strictEqual(taskQueue.enqueued.length, 0);
+	});
+	await t.step("falls back to the outbox queue by default", async () => {
+		const outboxQueue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { outbox: outboxQueue }
+		});
+		const task = federation.defineTask("fallback", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "data");
+		strictEqual(outboxQueue.enqueued.length, 1);
+	});
+	await t.step("taskQueueResolution: \"strict\" throws at enqueue instead", async () => {
+		const outboxQueue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { outbox: outboxQueue },
+			taskQueueResolution: "strict"
+		});
+		const task = federation.defineTask("strict", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+		await rejects(() => ctx.enqueueTask(task, "data"), {
+			name: "TypeError",
+			message: /No message queue is configured/
+		});
+		strictEqual(outboxQueue.enqueued.length, 0);
+	});
+	await t.step("throws when no queue is configured at all", async () => {
+		const federation = createFederation({ ...baseOptions });
+		const task = federation.defineTask("queueless", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+		await rejects(() => ctx.enqueueTask(task, "data"), {
+			name: "TypeError",
+			message: /No message queue is configured/
+		});
+	});
+});
+test("startQueue() task worker", async (t) => {
+	await t.step("starts only the task worker for queue: \"task\"", async () => {
+		const inbox = new MockQueue();
+		const outbox = new MockQueue();
+		const fanout = new MockQueue();
+		const taskQueue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: {
+				inbox,
+				outbox,
+				fanout,
+				task: taskQueue
+			}
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, {
+			signal: controller.signal,
+			queue: "task"
+		});
+		strictEqual(taskQueue.listenCount, 1);
+		strictEqual(inbox.listenCount, 0);
+		strictEqual(outbox.listenCount, 0);
+		strictEqual(fanout.listenCount, 0);
+		controller.abort();
+		await listening;
+	});
+	await t.step("starts the worker for a task-only deployment", async () => {
+		const taskQueue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: taskQueue }
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, { signal: controller.signal });
+		strictEqual(taskQueue.listenCount, 1);
+		controller.abort();
+		await listening;
+	});
+	await t.step("does not double-listen on a shared queue", async () => {
+		const shared = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: {
+				outbox: shared,
+				task: shared
+			}
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, { signal: controller.signal });
+		strictEqual(shared.listenCount, 1);
+		controller.abort();
+		await listening;
+	});
+	await t.step("starts a worker for a dedicated per-task queue", async () => {
+		const taskQueue = new MockQueue();
+		const dedicated = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: taskQueue }
+		});
+		federation.defineTask("dedicated", {
+			schema: stringSchema,
+			handler: () => {},
+			queue: dedicated
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, { signal: controller.signal });
+		strictEqual(taskQueue.listenCount, 1);
+		strictEqual(dedicated.listenCount, 1);
+		controller.abort();
+		await listening;
+	});
+	await t.step("starts a per-task queue even without a federation queue", async () => {
+		const dedicated = new MockQueue();
+		const federation = createFederation({ ...baseOptions });
+		federation.defineTask("dedicated", {
+			schema: stringSchema,
+			handler: () => {},
+			queue: dedicated
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, { signal: controller.signal });
+		strictEqual(dedicated.listenCount, 1);
+		controller.abort();
+		await listening;
+	});
+	await t.step("does not listen twice on a per-task queue shared with a standard queue", async () => {
+		const shared = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: shared }
+		});
+		federation.defineTask("reuses-task-queue", {
+			schema: stringSchema,
+			handler: () => {},
+			queue: shared
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, { signal: controller.signal });
+		strictEqual(shared.listenCount, 1);
+		controller.abort();
+		await listening;
+	});
+	await t.step("routes an enqueued task on a dedicated queue to its handler", async () => {
+		const dedicated = new MockQueue();
+		const federation = createFederation({ ...baseOptions });
+		let received;
+		const task = federation.defineTask("dedicated-end-to-end", {
+			schema: stringSchema,
+			handler: (_ctx, data) => {
+				received = data;
+			},
+			queue: dedicated
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "payload");
+		strictEqual(dedicated.enqueued.length, 1);
+		await federation.processQueuedTask(void 0, dedicated.enqueued[0].message);
+		strictEqual(received, "payload");
+	});
+	await t.step("queue: \"task\" starts the outbox fallback when no task queue is set", async () => {
+		const inbox = new MockQueue();
+		const outbox = new MockQueue();
+		const fanout = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: {
+				inbox,
+				outbox,
+				fanout
+			}
+		});
+		federation.defineTask("fallback", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, {
+			signal: controller.signal,
+			queue: "task"
+		});
+		strictEqual(outbox.listenCount, 1);
+		strictEqual(inbox.listenCount, 0);
+		strictEqual(fanout.listenCount, 0);
+		controller.abort();
+		await listening;
+	});
+	await t.step("queue: \"task\" starts a task queue shared with the outbox once", async () => {
+		const inbox = new MockQueue();
+		const shared = new MockQueue();
+		const fanout = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: {
+				inbox,
+				outbox: shared,
+				fanout,
+				task: shared
+			}
+		});
+		const controller = new AbortController();
+		const listening = federation.startQueue(void 0, {
+			signal: controller.signal,
+			queue: "task"
+		});
+		strictEqual(shared.listenCount, 1);
+		strictEqual(inbox.listenCount, 0);
+		strictEqual(fanout.listenCount, 0);
+		controller.abort();
+		await listening;
+	});
+});
+test("MockQueue.listen() resolves on a pre-aborted signal", async () => {
+	const queue = new MockQueue();
+	const controller = new AbortController();
+	controller.abort();
+	let timer;
+	const settled = await Promise.race([queue.listen(() => {}, { signal: controller.signal }).then(() => "resolved"), new Promise((resolve) => {
+		timer = setTimeout(() => resolve("pending"), 50);
+	})]);
+	clearTimeout(timer);
+	strictEqual(settled, "resolved");
+});
+test("processQueuedTask() task dispatch", async (t) => {
+	await t.step("drops an unknown task with a warning", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		const message = await makeTaskMessage("never-defined", "data");
+		await federation.processQueuedTask(void 0, message);
+		strictEqual(queue.enqueued.length, 0);
+	});
+	await t.step("drops an undecodable payload without retry", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		let called = 0;
+		federation.defineTask("broken-wire", {
+			schema: stringSchema,
+			handler: () => {
+				called++;
+			}
+		});
+		const message = await makeTaskMessage("broken-wire", "data");
+		await federation.processQueuedTask(void 0, {
+			...message,
+			data: "garbage that is not devalue"
+		});
+		strictEqual(called, 0);
+		strictEqual(queue.enqueued.length, 0);
+	});
+	await t.step("drops a drifted payload without retry", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		let called = 0;
+		federation.defineTask("drifted", {
+			schema: numberSchema,
+			handler: () => {
+				called++;
+			}
+		});
+		const message = await makeTaskMessage("drifted", "stringly-typed");
+		await federation.processQueuedTask(void 0, message);
+		strictEqual(called, 0);
+		strictEqual(queue.enqueued.length, 0);
+	});
+	await t.step("re-enqueues with attempt + 1 when the handler throws", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		const errors = [];
+		federation.defineTask("flaky", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			},
+			retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 }),
+			onError: (_ctx, error, data) => {
+				errors.push([error, data]);
+			}
+		});
+		const message = await makeTaskMessage("flaky", "data", { orderingKey: "k" });
+		await federation.processQueuedTask(void 0, message);
+		strictEqual(queue.enqueued.length, 1);
+		const retry = queue.enqueued[0];
+		strictEqual(retry.message.attempt, 1);
+		strictEqual(retry.message.taskName, "flaky");
+		strictEqual(retry.message.orderingKey, "k");
+		strictEqual(retry.options?.orderingKey, "k");
+		ok(retry.options?.delay instanceof Temporal.Duration);
+		strictEqual(errors.length, 1);
+		deepStrictEqual(errors[0], [/* @__PURE__ */ new Error("boom"), "data"]);
+	});
+	await t.step("gives up when the retry policy returns null", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("hopeless", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			},
+			retryPolicy: () => null
+		});
+		const message = await makeTaskMessage("hopeless", "data");
+		await federation.processQueuedTask(void 0, message);
+		strictEqual(queue.enqueued.length, 0);
+	});
+	await t.step("per-task retryPolicy overrides the federation default", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue },
+			taskRetryPolicy: () => null
+		});
+		federation.defineTask("override", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			},
+			retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 })
+		});
+		federation.defineTask("default", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			}
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("override", "data"));
+		strictEqual(queue.enqueued.length, 1);
+		await federation.processQueuedTask(void 0, await makeTaskMessage("default", "data"));
+		strictEqual(queue.enqueued.length, 1);
+	});
+	await t.step("still retries when message.started is malformed", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("bad-started", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			},
+			retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 })
+		});
+		const message = await makeTaskMessage("bad-started", "data", { started: "not-an-instant" });
+		await federation.processQueuedTask(void 0, message);
+		strictEqual(queue.enqueued.length, 1);
+		strictEqual(queue.enqueued[0].message.attempt, 1);
+	});
+	await t.step("rethrows on a nativeRetrial queue", async () => {
+		const queue = new MockQueue({ nativeRetrial: true });
+		const federation = createFederation({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("native", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			}
+		});
+		const message = await makeTaskMessage("native", "data");
+		await rejects(() => federation.processQueuedTask(void 0, message), { message: /boom/ });
+		strictEqual(queue.enqueued.length, 0);
+	});
+});
+/**
+* A {@link KvStore} that delegates to an in-memory store but deliberately
+* omits `cas`, so that `kv.cas == null`.  This drives the deduplication
+* fallback branches that fire when no conditional-write primitive exists.
+*/
+var CaslessKvStore = class {
+	inner = new MemoryKvStore();
+	get(key) {
+		return this.inner.get(key);
+	}
+	set(key, value, options) {
+		return this.inner.set(key, value, options);
+	}
+	delete(key) {
+		return this.inner.delete(key);
+	}
+	list(prefix) {
+		return this.inner.list(prefix);
+	}
+};
+async function collectKeys(kv, prefix) {
+	const keys = [];
+	for await (const { key } of kv.list(prefix)) keys.push(key);
+	return keys;
+}
+const TASK_DEDUP_PREFIX = ["_fedify", "taskDeduplication"];
+const ACTIVITY_IDEMPOTENCE_PREFIX = ["_fedify", "activityIdempotence"];
+test("task deduplication", async (t) => {
+	await t.step("forwards the key to a nativeDeduplication queue without writing KV", async () => {
+		const queue = new MockQueue({ nativeDeduplication: true });
+		const kv = new MemoryKvStore();
+		const federation = createFederation({
+			...baseOptions,
+			kv,
+			queue: { task: queue }
+		});
+		const task = federation.defineTask("native-dedup", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "payload", { deduplicationKey: "k" });
+		strictEqual(queue.enqueued.length, 1);
+		strictEqual(queue.enqueued[0].options?.deduplicationKey, "k");
+		strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 0);
+	});
+	await t.step("skips a second enqueue with the same key within the TTL", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			kv: new MemoryKvStore(),
+			queue: { task: queue }
+		});
+		const task = federation.defineTask("kv-dedup", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+		await ctx.enqueueTask(task, "first", { deduplicationKey: "k" });
+		await ctx.enqueueTask(task, "second", { deduplicationKey: "k" });
+		strictEqual(queue.enqueued.length, 1);
+		strictEqual(queue.enqueued[0].message.taskName, "kv-dedup");
+		strictEqual(queue.enqueued[0].options?.deduplicationKey, void 0);
+	});
+	await t.step("re-enqueues with the same key after the TTL expires", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			kv: new MemoryKvStore(),
+			queue: { task: queue },
+			taskDeduplicationTtl: { milliseconds: 100 }
+		});
+		const task = federation.defineTask("kv-dedup-ttl", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+		await ctx.enqueueTask(task, "first", { deduplicationKey: "k" });
+		strictEqual(queue.enqueued.length, 1);
+		await delay(300);
+		await ctx.enqueueTask(task, "second", { deduplicationKey: "k" });
+		strictEqual(queue.enqueued.length, 2);
+	});
+	await t.step("rejects with TypeError when fallback is \"closed\" and no cas exists", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			kv: new CaslessKvStore(),
+			queue: { task: queue },
+			taskDeduplicationFallback: "closed"
+		});
+		const task = federation.defineTask("closed-fallback", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+		await rejects(() => ctx.enqueueTask(task, "payload", { deduplicationKey: "k" }), { name: "TypeError" });
+		strictEqual(queue.enqueued.length, 0);
+	});
+	await t.step("proceeds when fallback is \"open\" and no cas exists", async () => {
+		const queue = new MockQueue();
+		const federation = createFederation({
+			...baseOptions,
+			kv: new CaslessKvStore(),
+			queue: { task: queue },
+			taskDeduplicationFallback: "open"
+		});
+		const task = federation.defineTask("open-fallback", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "payload", { deduplicationKey: "k" });
+		strictEqual(queue.enqueued.length, 1);
+		strictEqual(queue.enqueued[0].options?.deduplicationKey, void 0);
+	});
+	await t.step("writes only under taskDeduplication, never activityIdempotence", async () => {
+		const queue = new MockQueue();
+		const kv = new MemoryKvStore();
+		const federation = createFederation({
+			...baseOptions,
+			kv,
+			queue: { task: queue }
+		});
+		const task = federation.defineTask("prefix-isolation", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "payload", { deduplicationKey: "k" });
+		strictEqual((await collectKeys(kv, TASK_DEDUP_PREFIX)).length, 1);
+		strictEqual((await collectKeys(kv, ACTIVITY_IDEMPOTENCE_PREFIX)).length, 0);
+	});
+	await t.step("applies one batch-level key to enqueueTaskMany", async () => {
+		const queue = new MockQueue({ supportsEnqueueMany: true });
+		const federation = createFederation({
+			...baseOptions,
+			kv: new MemoryKvStore(),
+			queue: { task: queue }
+		});
+		const task = federation.defineTask("batch-dedup", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+		await ctx.enqueueTaskMany(task, [
+			"a",
+			"b",
+			"c"
+		], { deduplicationKey: "batch" });
+		await ctx.enqueueTaskMany(task, [
+			"a",
+			"b",
+			"c"
+		], { deduplicationKey: "batch" });
+		strictEqual(queue.enqueuedMany.length, 1);
+		strictEqual(queue.enqueuedMany[0].messages.length, 3);
+	});
+});
+test("task deduplication validates every payload before reserving the key", async () => {
+	const queue = new MockQueue();
+	const kv = new MemoryKvStore();
+	const federation = createFederation({
+		...baseOptions,
+		kv,
+		queue: { task: queue }
+	});
+	const task = federation.defineTask("dedup-validation", {
+		schema: stringSchema,
+		handler: () => {}
+	});
+	const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+	await rejects(() => ctx.enqueueTask(task, 123, { deduplicationKey: "k" }));
+	strictEqual(queue.enqueued.length, 0);
+	deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []);
+	await ctx.enqueueTask(task, "valid", { deduplicationKey: "k" });
+	strictEqual(queue.enqueued.length, 1);
+	deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), [[...TASK_DEDUP_PREFIX, "k"]]);
+	await ctx.enqueueTask(task, "duplicate", { deduplicationKey: "k" });
+	strictEqual(queue.enqueued.length, 1);
+});
+test("native task batch deduplication is one enqueueMany operation per call", async () => {
+	class NativeBatchDeduplicatingQueue {
+		nativeDeduplication = true;
+		#seen = /* @__PURE__ */ new Set();
+		attempts = [];
+		accepted = [];
+		enqueue() {
+			throw new Error("A multi-item native batch must use enqueueMany().");
+		}
+		enqueueMany(messages, options) {
+			const key = options?.deduplicationKey;
+			if (key == null) throw new TypeError("Native batch enqueue requires a deduplication key.");
+			this.attempts.push({
+				messages,
+				options
+			});
+			if (this.#seen.has(key)) return Promise.resolve();
+			this.#seen.add(key);
+			this.accepted.push({
+				messages,
+				options
+			});
+			return Promise.resolve();
+		}
+		listen() {
+			return Promise.resolve();
+		}
+	}
+	const queue = new NativeBatchDeduplicatingQueue();
+	const kv = new MemoryKvStore();
+	const federation = createFederation({
+		...baseOptions,
+		kv,
+		queue: { task: queue }
+	});
+	const task = federation.defineTask("native-batch-dedup", {
+		schema: stringSchema,
+		handler: () => {}
+	});
+	const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+	await ctx.enqueueTaskMany(task, [
+		"a1",
+		"a2",
+		"a3"
+	], { deduplicationKey: "batch-a" });
+	await ctx.enqueueTaskMany(task, [
+		"duplicate1",
+		"duplicate2",
+		"duplicate3"
+	], { deduplicationKey: "batch-a" });
+	await ctx.enqueueTaskMany(task, [
+		"b1",
+		"b2",
+		"b3"
+	], { deduplicationKey: "batch-b" });
+	strictEqual(queue.attempts.length, 3);
+	deepStrictEqual(queue.attempts.map(({ messages }) => messages.length), [
+		3,
+		3,
+		3
+	]);
+	deepStrictEqual(queue.attempts.map(({ options }) => options?.deduplicationKey), [
+		"batch-a",
+		"batch-a",
+		"batch-b"
+	]);
+	strictEqual(queue.accepted.length, 2);
+	deepStrictEqual(queue.accepted.map(({ messages }) => messages.length), [3, 3]);
+	deepStrictEqual(queue.accepted.map(({ options }) => options?.deduplicationKey), ["batch-a", "batch-b"]);
+	deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []);
+});
+test("native task batch deduplication rejects without enqueueMany", async () => {
+	const queue = new MockQueue({ nativeDeduplication: true });
+	const kv = new MemoryKvStore();
+	const federation = createFederation({
+		...baseOptions,
+		kv,
+		queue: { task: queue }
+	});
+	const task = federation.defineTask("native-batch-without-enqueue-many", {
+		schema: stringSchema,
+		handler: () => {}
+	});
+	const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+	await rejects(() => ctx.enqueueTaskMany(task, [
+		"a",
+		"b",
+		"c"
+	], { deduplicationKey: "batch" }), {
+		name: "TypeError",
+		message: /enqueueMany/
+	});
+	strictEqual(queue.enqueued.length, 0);
+	strictEqual(queue.enqueuedMany.length, 0);
+	deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []);
+	await ctx.enqueueTaskMany(task, ["single"], { deduplicationKey: "single" });
+	strictEqual(queue.enqueued.length, 1);
+	strictEqual(queue.enqueued[0].options?.deduplicationKey, "single");
+});
+test("deduplication - native batch capability errors precede payload validation", async () => {
+	let validationCalls = 0;
+	const schema = makeSchema((data) => {
+		validationCalls++;
+		return typeof data === "string";
+	});
+	const queue = new MockQueue({ nativeDeduplication: true });
+	const kv = new MemoryKvStore();
+	const federation = createFederation({
+		...baseOptions,
+		kv,
+		queue: { task: queue }
+	});
+	const task = federation.defineTask("native-batch-capability-order", {
+		schema,
+		handler: () => {}
+	});
+	const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+	let caught;
+	try {
+		await ctx.enqueueTaskMany(task, [
+			1,
+			2,
+			3
+		], { deduplicationKey: "batch" });
+	} catch (error) {
+		caught = error;
+	}
+	strictEqual(validationCalls, 0);
+	ok(caught instanceof TypeError);
+	ok(caught.message.includes("enqueueMany"));
+	strictEqual(queue.enqueued.length, 0);
+	strictEqual(queue.enqueuedMany.length, 0);
+	deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []);
+});
+test("closed deduplication fallback errors precede payload validation", async () => {
+	let validationCalls = 0;
+	const schema = makeSchema((data) => {
+		validationCalls++;
+		return typeof data === "string";
+	});
+	const queue = new MockQueue();
+	const kv = new CaslessKvStore();
+	const federation = createFederation({
+		...baseOptions,
+		kv,
+		queue: { task: queue },
+		taskDeduplicationFallback: "closed"
+	});
+	const task = federation.defineTask("closed-fallback-order", {
+		schema,
+		handler: () => {}
+	});
+	const ctx = federation.createContext(new URL("https://example.com/"), void 0);
+	let caught;
+	try {
+		await ctx.enqueueTask(task, 1, { deduplicationKey: "k" });
+	} catch (error) {
+		caught = error;
+	}
+	strictEqual(validationCalls, 0);
+	ok(caught instanceof TypeError);
+	ok(caught.message.includes("conditional write"));
+	strictEqual(queue.enqueued.length, 0);
+	strictEqual(queue.enqueuedMany.length, 0);
+	deepStrictEqual(await collectKeys(kv, TASK_DEDUP_PREFIX), []);
+});
+/** Wires test telemetry doubles into a fresh federation for the suite below. */
+const instrument = (options) => {
+	const [meterProvider, recorder] = createTestMeterProvider();
+	const [tracerProvider, exporter] = createTestTracerProvider();
+	return {
+		federation: createFederation({
+			...options,
+			meterProvider,
+			tracerProvider
+		}),
+		recorder,
+		exporter
+	};
+};
+test("task observability", async (t) => {
+	await t.step("opens a fedify.task span carrying name and attempt on success", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("sendDigest", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("sendDigest", "payload"));
+		const spans = exporter.getSpans("fedify.task");
+		strictEqual(spans.length, 1);
+		strictEqual(spans[0].attributes["fedify.task.name"], "sendDigest");
+		strictEqual(spans[0].attributes["fedify.task.attempt"], 0);
+		strictEqual(spans[0].attributes["fedify.task.failure_reason"], void 0);
+		strictEqual(spans[0].status.code, SpanStatusCode.UNSET);
+		const started = recorder.getMeasurements("fedify.queue.task.started");
+		strictEqual(started.length, 1);
+		strictEqual(started[0].attributes["fedify.queue.role"], "task");
+		strictEqual(started[0].attributes["fedify.task.name"], "sendDigest");
+		const completed = recorder.getMeasurements("fedify.queue.task.completed");
+		strictEqual(completed.length, 1);
+		strictEqual(completed[0].attributes["fedify.queue.role"], "task");
+		strictEqual(completed[0].attributes["fedify.task.name"], "sendDigest");
+		strictEqual(completed[0].attributes["fedify.queue.task.result"], "completed");
+		strictEqual(completed[0].attributes["fedify.task.failure_reason"], void 0);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.failed").length, 0);
+	});
+	await t.step("inherits the parent trace context from the enqueue site", async () => {
+		const traceId = "0af7651916cd43dd8448eb211c80319c";
+		const spanId = "b7ad6b7169203331";
+		propagation.setGlobalPropagator(new W3CTraceContextPropagator());
+		try {
+			const queue = new MockQueue();
+			const { federation, exporter } = instrument({
+				...baseOptions,
+				queue: { task: queue }
+			});
+			federation.defineTask("traced", {
+				schema: stringSchema,
+				handler: () => {}
+			});
+			await federation.processQueuedTask(void 0, await makeTaskMessage("traced", "payload", { traceContext: { traceparent: `00-${traceId}-${spanId}-01` } }));
+			const span = exporter.getSpans("fedify.task")[0];
+			ok(span != null);
+			strictEqual(span.spanContext().traceId, traceId);
+			strictEqual(span.parentSpanContext?.spanId, spanId);
+		} finally {
+			propagation.disable();
+		}
+	});
+	await t.step("attributes a deserialization failure and drops without retry", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		let called = 0;
+		federation.defineTask("decode-me", {
+			schema: stringSchema,
+			handler: () => {
+				called++;
+			}
+		});
+		const message = await makeTaskMessage("decode-me", "payload");
+		await federation.processQueuedTask(void 0, {
+			...message,
+			data: "garbage that is not devalue"
+		});
+		strictEqual(called, 0);
+		strictEqual(queue.enqueued.length, 0);
+		const failed = recorder.getMeasurements("fedify.queue.task.failed");
+		strictEqual(failed.length, 1);
+		strictEqual(failed[0].attributes["fedify.queue.role"], "task");
+		strictEqual(failed[0].attributes["fedify.task.failure_reason"], "deserialization");
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], "deserialization");
+		strictEqual(span.status.code, SpanStatusCode.ERROR);
+	});
+	await t.step("attributes a validation failure and drops without retry", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		let called = 0;
+		federation.defineTask("strict-shape", {
+			schema: numberSchema,
+			handler: () => {
+				called++;
+			}
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("strict-shape", "not a number"));
+		strictEqual(called, 0);
+		strictEqual(queue.enqueued.length, 0);
+		const failed = recorder.getMeasurements("fedify.queue.task.failed");
+		strictEqual(failed.length, 1);
+		strictEqual(failed[0].attributes["fedify.task.failure_reason"], "validation");
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], "validation");
+		strictEqual(span.status.code, SpanStatusCode.ERROR);
+	});
+	await t.step("attributes an unknown task and drops without retry", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("never-registered", "payload"));
+		strictEqual(queue.enqueued.length, 0);
+		const failed = recorder.getMeasurements("fedify.queue.task.failed");
+		strictEqual(failed.length, 1);
+		strictEqual(failed[0].attributes["fedify.task.failure_reason"], "unknown_task");
+		strictEqual(failed[0].attributes["fedify.queue.backend"], "MockQueue");
+		strictEqual(failed[0].attributes["fedify.task.name"], void 0);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.started")[0].attributes["fedify.task.name"], void 0);
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.name"], "never-registered");
+		strictEqual(span.attributes["fedify.task.failure_reason"], "unknown_task");
+		strictEqual(span.status.code, SpanStatusCode.ERROR);
+	});
+	await t.step("attributes a handler failure on a terminal give-up", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("explode", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			},
+			retryPolicy: () => null
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("explode", "payload"));
+		strictEqual(queue.enqueued.length, 0);
+		const failed = recorder.getMeasurements("fedify.queue.task.failed");
+		strictEqual(failed.length, 1);
+		strictEqual(failed[0].attributes["fedify.task.failure_reason"], "handler");
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], "handler");
+		strictEqual(span.status.code, SpanStatusCode.ERROR);
+	});
+	await t.step("reports the resolved outbox queue as the backend on fallback", async () => {
+		class FallbackOutboxQueue extends MockQueue {}
+		const outboxQueue = new FallbackOutboxQueue();
+		const { federation, recorder } = instrument({
+			...baseOptions,
+			queue: { outbox: outboxQueue }
+		});
+		federation.defineTask("fallback", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("fallback", "payload"));
+		const completed = recorder.getMeasurements("fedify.queue.task.completed");
+		strictEqual(completed.length, 1);
+		strictEqual(completed[0].attributes["fedify.queue.backend"], "FallbackOutboxQueue");
+	});
+	await t.step("records an enqueue measurement with role task", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		const task = federation.defineTask("enqueue-me", {
+			schema: stringSchema,
+			handler: () => {}
+		});
+		await federation.createContext(new URL("https://example.com/"), void 0).enqueueTask(task, "payload");
+		const enqueued = recorder.getMeasurements("fedify.queue.task.enqueued");
+		strictEqual(enqueued.length, 1);
+		strictEqual(enqueued[0].attributes["fedify.queue.role"], "task");
+		strictEqual(enqueued[0].attributes["fedify.task.name"], "enqueue-me");
+		strictEqual(enqueued[0].attributes["fedify.queue.task.attempt"], 0);
+	});
+	await t.step("records the retry re-enqueue with role task and a bumped attempt", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("retry-me", {
+			schema: stringSchema,
+			handler: () => {
+				throw new Error("boom");
+			},
+			retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 })
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("retry-me", "payload"));
+		strictEqual(queue.enqueued.length, 1);
+		strictEqual(queue.enqueued[0].message.attempt, 1);
+		const enqueued = recorder.getMeasurements("fedify.queue.task.enqueued");
+		strictEqual(enqueued.length, 1);
+		strictEqual(enqueued[0].attributes["fedify.queue.role"], "task");
+		strictEqual(enqueued[0].attributes["fedify.task.name"], "retry-me");
+		strictEqual(enqueued[0].attributes["fedify.queue.task.attempt"], 1);
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], void 0);
+		strictEqual(span.status.code, SpanStatusCode.UNSET);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.failed").length, 0);
+		const completed = recorder.getMeasurements("fedify.queue.task.completed");
+		strictEqual(completed.length, 1);
+		strictEqual(completed[0].attributes["fedify.task.failure_reason"], void 0);
+	});
+	await t.step("attributes retry_enqueue when the retry re-enqueue itself fails", async () => {
+		class RejectingQueue {
+			enqueue() {
+				return Promise.reject(/* @__PURE__ */ new Error("backend unavailable"));
+			}
+			listen() {
+				return Promise.resolve();
+			}
+		}
+		const queue = new RejectingQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		const handlerError = /* @__PURE__ */ new Error("boom");
+		federation.defineTask("retry-enqueue-fails", {
+			schema: stringSchema,
+			handler: () => {
+				throw handlerError;
+			},
+			retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 })
+		});
+		const message = await makeTaskMessage("retry-enqueue-fails", "payload");
+		await rejects(() => federation.processQueuedTask(void 0, message), (e) => e instanceof Error && e.cause === handlerError);
+		const failed = recorder.getMeasurements("fedify.queue.task.failed");
+		strictEqual(failed.length, 1);
+		strictEqual(failed[0].attributes["fedify.task.failure_reason"], "retry_enqueue");
+		strictEqual(recorder.getMeasurements("fedify.queue.task.enqueued").length, 0);
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], "retry_enqueue");
+		strictEqual(span.status.code, SpanStatusCode.ERROR);
+		ok(span.status.message?.includes("boom"));
+	});
+	await t.step("records an abort as aborted, without a failure reason or error status", async () => {
+		const queue = new MockQueue({ nativeRetrial: true });
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("aborts", {
+			schema: stringSchema,
+			handler: () => {
+				throw globalThis.Object.assign(/* @__PURE__ */ new Error("shutting down"), { name: "AbortError" });
+			}
+		});
+		const message = await makeTaskMessage("aborts", "payload");
+		await rejects(() => federation.processQueuedTask(void 0, message), { name: "AbortError" });
+		const span = exporter.getSpans("fedify.task")[0];
+		ok(span != null);
+		strictEqual(span.attributes["fedify.task.failure_reason"], void 0);
+		strictEqual(span.status.code, SpanStatusCode.UNSET);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.failed").length, 0);
+		const durations = recorder.getMeasurements("fedify.queue.task.duration");
+		strictEqual(durations.length, 1);
+		strictEqual(durations[0].attributes["fedify.queue.task.result"], "aborted");
+	});
+	await t.step("on a non-native queue an aborted handler that gives up is aborted", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("aborts-give-up", {
+			schema: stringSchema,
+			handler: () => {
+				throw globalThis.Object.assign(/* @__PURE__ */ new Error("shutting down"), { name: "AbortError" });
+			},
+			retryPolicy: () => null
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("aborts-give-up", "payload"));
+		strictEqual(queue.enqueued.length, 0);
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], void 0);
+		strictEqual(span.status.code, SpanStatusCode.UNSET);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.failed").length, 0);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.completed").length, 0);
+		const durations = recorder.getMeasurements("fedify.queue.task.duration");
+		strictEqual(durations.length, 1);
+		strictEqual(durations[0].attributes["fedify.queue.task.result"], "aborted");
+	});
+	await t.step("on a non-native queue an aborted handler is retried, not failed", async () => {
+		const queue = new MockQueue();
+		const { federation, recorder, exporter } = instrument({
+			...baseOptions,
+			queue: { task: queue }
+		});
+		federation.defineTask("aborts-soft", {
+			schema: stringSchema,
+			handler: () => {
+				throw globalThis.Object.assign(/* @__PURE__ */ new Error("shutting down"), { name: "AbortError" });
+			},
+			retryPolicy: () => Temporal.Duration.from({ milliseconds: 1 })
+		});
+		await federation.processQueuedTask(void 0, await makeTaskMessage("aborts-soft", "payload"));
+		strictEqual(queue.enqueued.length, 1);
+		strictEqual(queue.enqueued[0].message.attempt, 1);
+		const span = exporter.getSpans("fedify.task")[0];
+		strictEqual(span.attributes["fedify.task.failure_reason"], void 0);
+		strictEqual(span.status.code, SpanStatusCode.UNSET);
+		strictEqual(recorder.getMeasurements("fedify.queue.task.failed").length, 0);
+		const completed = recorder.getMeasurements("fedify.queue.task.completed");
+		strictEqual(completed.length, 1);
+		strictEqual(completed[0].attributes["fedify.task.failure_reason"], void 0);
+	});
+});
+//#endregion
+export {};