@fedify/backfill 2.3.12-dev.2341 → 2.3.12-dev.2346

package/README.md

@@ -153,7 +153,11 @@ An `interval` string requires the global `Temporal` API or a polyfill.
 
 If the seed has no context, or its context resolves to a non-collection,
 context strategies yield nothing.  Loader failures are skipped unless
-traversal is aborted.
+traversal is aborted.  If the context collection cannot be loaded, later
+configured strategies can still run.  Failed loads consume a request, and
+later strategies can use embedded data even when the request budget is
+exhausted.  Configuration errors, such as an invalid `interval`, still
+propagate.
 
 Dereferenced documents are cached in memory for one `backfill()` traversal.
 Applications that need persistent or shared caching can provide it through

package/dist/backfill.test.cjs

@@ -114,7 +114,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -289,7 +289,7 @@ async function* getCollectionItems(context, collection, options, budget, skipIds
 	};
 	const pageOptions = {
 		documentLoader: async (url) => {
-			const object = await loadObject(context, new URL(url), options, budget, true);
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
 			if (object == null) throw new Error(`Collection page not found: ${url}`);
 			return {
 				contextUrl: null,
@@ -345,7 +345,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -368,7 +368,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -380,12 +380,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }
@@ -1757,4 +1762,132 @@ async function collect(context, note, options = {}) {
 		maxDepth: 1
 	})).map((item) => item.id?.href), [a.id?.href, b.id?.href]);
 });
+(0, node_test.describe)("context loader failures", () => {
+	const contextId = new URL("https://example.com/thread");
+	const parent = new _fedify_vocab.Note({ id: new URL("https://example.com/parent") });
+	for (const synchronous of [false, true]) {
+		const failure = synchronous ? "synchronous throw" : "promise rejection";
+		const fail = () => {
+			const error = /* @__PURE__ */ new Error("Context collection unavailable");
+			if (synchronous) throw error;
+			return Promise.reject(error);
+		};
+		(0, node_test.default)(`${failure} allows reply-tree to yield an embedded parent`, async () => {
+			const seed = new _fedify_vocab.Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			let requests = 0;
+			const items = await collect({ documentLoader: () => {
+				requests++;
+				return fail();
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			});
+			(0, node_assert_strict.deepStrictEqual)(items.map((item) => item.object), [parent]);
+			(0, node_assert_strict.strictEqual)(items[0].strategy, "reply-tree");
+			(0, node_assert_strict.strictEqual)(requests, 1);
+		});
+		(0, node_test.default)(`${failure} finishes the default strategy without items`, async () => {
+			const seed = new _fedify_vocab.Note({ contexts: [contextId] });
+			(0, node_assert_strict.deepStrictEqual)(await collect({ documentLoader: fail }, seed), []);
+		});
+		(0, node_test.default)(`${failure} consumes the request budget`, async () => {
+			const seed = new _fedify_vocab.Note({
+				contexts: [contextId],
+				replyTarget: parent.id
+			});
+			const requests = [];
+			(0, node_assert_strict.deepStrictEqual)(await collect({ documentLoader: (url) => {
+				requests.push(url.href);
+				if (url.href === contextId.href) return fail();
+				return Promise.resolve(parent);
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			}), []);
+			(0, node_assert_strict.deepStrictEqual)(requests, [contextId.href]);
+		});
+		(0, node_test.default)(`${failure} propagates the cancellation reason`, async () => {
+			const controller = new AbortController();
+			const reason = /* @__PURE__ */ new Error("Stop backfill");
+			const seed = new _fedify_vocab.Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			const yielded = [];
+			await (0, node_assert_strict.rejects)(async () => {
+				for await (const item of backfill({ documentLoader: (_url, options) => {
+					(0, node_assert_strict.strictEqual)(options?.signal, controller.signal);
+					controller.abort(reason);
+					return fail();
+				} }, seed, {
+					strategies: ["context-auto", "reply-tree"],
+					signal: controller.signal
+				})) yielded.push(item);
+			}, (error) => error === reason);
+			(0, node_assert_strict.deepStrictEqual)(yielded, []);
+		});
+	}
+	(0, node_test.default)("a failed context load is not cached for later reply-tree loads", async () => {
+		const seed = new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: contextId
+		});
+		let requests = 0;
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: (url) => {
+			(0, node_assert_strict.strictEqual)(url.href, contextId.href);
+			requests++;
+			if (requests === 1) return Promise.reject(/* @__PURE__ */ new Error("Temporary failure"));
+			return Promise.resolve(parent);
+		} }, seed, {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 2
+		})).map((item) => item.object.id?.href), [parent.id?.href]);
+		(0, node_assert_strict.strictEqual)(requests, 2);
+	});
+	(0, node_test.default)("an exhausted budget still allows embedded reply-tree data", async () => {
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: () => {
+			throw new Error("No requests allowed");
+		} }, new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 0
+		})).map((item) => item.object), [parent]);
+	});
+	(0, node_test.default)("interval callback errors propagate before the loader is called", async () => {
+		const reason = /* @__PURE__ */ new Error("Invalid interval configuration");
+		let requests = 0;
+		await (0, node_assert_strict.rejects)(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: () => {
+				throw reason;
+			}
+		}), (error) => error === reason);
+		(0, node_assert_strict.strictEqual)(requests, 0);
+	});
+	(0, node_test.default)("invalid interval strings still propagate", async () => {
+		let requests = 0;
+		await (0, node_assert_strict.rejects)(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: "not a duration"
+		}));
+		(0, node_assert_strict.strictEqual)(requests, 0);
+	});
+});
 //#endregion

package/dist/backfill.test.js

@@ -91,7 +91,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -266,7 +266,7 @@ async function* getCollectionItems(context, collection, options, budget, skipIds
 	};
 	const pageOptions = {
 		documentLoader: async (url) => {
-			const object = await loadObject(context, new URL(url), options, budget, true);
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
 			if (object == null) throw new Error(`Collection page not found: ${url}`);
 			return {
 				contextUrl: null,
@@ -322,7 +322,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -345,7 +345,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -357,12 +357,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }
@@ -1734,5 +1739,133 @@ test("embedded pages without IDs follow next without loader calls", async () =>
 		maxDepth: 1
 	})).map((item) => item.id?.href), [a.id?.href, b.id?.href]);
 });
+describe("context loader failures", () => {
+	const contextId = new URL("https://example.com/thread");
+	const parent = new Note({ id: new URL("https://example.com/parent") });
+	for (const synchronous of [false, true]) {
+		const failure = synchronous ? "synchronous throw" : "promise rejection";
+		const fail = () => {
+			const error = /* @__PURE__ */ new Error("Context collection unavailable");
+			if (synchronous) throw error;
+			return Promise.reject(error);
+		};
+		test(`${failure} allows reply-tree to yield an embedded parent`, async () => {
+			const seed = new Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			let requests = 0;
+			const items = await collect({ documentLoader: () => {
+				requests++;
+				return fail();
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			});
+			deepStrictEqual(items.map((item) => item.object), [parent]);
+			strictEqual(items[0].strategy, "reply-tree");
+			strictEqual(requests, 1);
+		});
+		test(`${failure} finishes the default strategy without items`, async () => {
+			const seed = new Note({ contexts: [contextId] });
+			deepStrictEqual(await collect({ documentLoader: fail }, seed), []);
+		});
+		test(`${failure} consumes the request budget`, async () => {
+			const seed = new Note({
+				contexts: [contextId],
+				replyTarget: parent.id
+			});
+			const requests = [];
+			deepStrictEqual(await collect({ documentLoader: (url) => {
+				requests.push(url.href);
+				if (url.href === contextId.href) return fail();
+				return Promise.resolve(parent);
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			}), []);
+			deepStrictEqual(requests, [contextId.href]);
+		});
+		test(`${failure} propagates the cancellation reason`, async () => {
+			const controller = new AbortController();
+			const reason = /* @__PURE__ */ new Error("Stop backfill");
+			const seed = new Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			const yielded = [];
+			await rejects(async () => {
+				for await (const item of backfill({ documentLoader: (_url, options) => {
+					strictEqual(options?.signal, controller.signal);
+					controller.abort(reason);
+					return fail();
+				} }, seed, {
+					strategies: ["context-auto", "reply-tree"],
+					signal: controller.signal
+				})) yielded.push(item);
+			}, (error) => error === reason);
+			deepStrictEqual(yielded, []);
+		});
+	}
+	test("a failed context load is not cached for later reply-tree loads", async () => {
+		const seed = new Note({
+			contexts: [contextId],
+			replyTarget: contextId
+		});
+		let requests = 0;
+		deepStrictEqual((await collect({ documentLoader: (url) => {
+			strictEqual(url.href, contextId.href);
+			requests++;
+			if (requests === 1) return Promise.reject(/* @__PURE__ */ new Error("Temporary failure"));
+			return Promise.resolve(parent);
+		} }, seed, {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 2
+		})).map((item) => item.object.id?.href), [parent.id?.href]);
+		strictEqual(requests, 2);
+	});
+	test("an exhausted budget still allows embedded reply-tree data", async () => {
+		deepStrictEqual((await collect({ documentLoader: () => {
+			throw new Error("No requests allowed");
+		} }, new Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 0
+		})).map((item) => item.object), [parent]);
+	});
+	test("interval callback errors propagate before the loader is called", async () => {
+		const reason = /* @__PURE__ */ new Error("Invalid interval configuration");
+		let requests = 0;
+		await rejects(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: () => {
+				throw reason;
+			}
+		}), (error) => error === reason);
+		strictEqual(requests, 0);
+	});
+	test("invalid interval strings still propagate", async () => {
+		let requests = 0;
+		await rejects(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: "not a duration"
+		}));
+		strictEqual(requests, 0);
+	});
+});
 //#endregion
 export {};

package/dist/mod.cjs

@@ -90,7 +90,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -265,7 +265,7 @@ async function* getCollectionItems(context, collection, options, budget, skipIds
 	};
 	const pageOptions = {
 		documentLoader: async (url) => {
-			const object = await loadObject(context, new URL(url), options, budget, true);
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
 			if (object == null) throw new Error(`Collection page not found: ${url}`);
 			return {
 				contextUrl: null,
@@ -321,7 +321,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -344,7 +344,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -356,12 +356,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }

package/dist/mod.js

@@ -89,7 +89,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -264,7 +264,7 @@ async function* getCollectionItems(context, collection, options, budget, skipIds
 	};
 	const pageOptions = {
 		documentLoader: async (url) => {
-			const object = await loadObject(context, new URL(url), options, budget, true);
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
 			if (object == null) throw new Error(`Collection page not found: ${url}`);
 			return {
 				contextUrl: null,
@@ -320,7 +320,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -343,7 +343,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -355,12 +355,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fedify/backfill",
-  "version": "2.3.12-dev.2341+eb343ebf",
+  "version": "2.3.12-dev.2346+e087854c",
   "description": "ActivityPub backfill support for Fedify",
   "keywords": [
     "Fedify",
@@ -49,7 +49,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@fedify/vocab": "2.3.12-dev.2341+eb343ebf"
+    "@fedify/vocab": "2.3.12-dev.2346+e087854c"
   },
   "devDependencies": {
     "tsdown": "^0.22.0",