@fedify/backfill 2.5.0-dev.2364 → 2.5.0-dev.2378

package/README.md

@@ -65,6 +65,14 @@ for await (
 The seed object itself is not yielded.  If it appears in the discovered
 collection, it is skipped by ID.
 
+Context and `replies` collections support inline items and pagination through
+`first` and successive `next` links.  A collection page used as the starting
+collection also follows `next`.  Pages are loaded as needed, and traversal
+stops on repeated page links.  Pagination does not increase reply-tree depth.
+If a page cannot be loaded, items already yielded are retained and later
+configured strategies can continue unless cancellation or traversal limits
+stop them.
+
 Configured strategies run in order.  They share `maxItems`, `maxRequests`,
 abort state, and object ID deduplication; if two strategies discover the same
 object, the first strategy keeps its `BackfillItem` metadata.
@@ -133,7 +141,7 @@ All configured strategies share the same traversal controls:
  -  `maxItems` limits the number of yielded objects.  Skipped duplicates do
     not count.
  -  `maxRequests` limits calls to `documentLoader`.  Embedded objects and
-    collections do not count.
+    collections, including embedded pages, do not count.
  -  `maxDepth` limits reply-tree traversal and defaults to 10.  It does not
     limit context collection items.
  -  `interval` adds a delay between loader requests.  Its callback receives
@@ -145,7 +153,11 @@ An `interval` string requires the global `Temporal` API or a polyfill.
 
 If the seed has no context, or its context resolves to a non-collection,
 context strategies yield nothing.  Loader failures are skipped unless
-traversal is aborted.
+traversal is aborted.  If the context collection cannot be loaded, later
+configured strategies can still run.  Failed loads consume a request, and
+later strategies can use embedded data even when the request budget is
+exhausted.  Configuration errors, such as an invalid `interval`, still
+propagate.
 
 Dereferenced documents are cached in memory for one `backfill()` traversal.
 Applications that need persistent or shared caching can provide it through

package/dist/backfill.test.cjs

@@ -114,7 +114,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -281,12 +281,49 @@ async function* getContextBackfillItems(context, object, strategy, options, budg
 	}
 }
 async function* getCollectionItems(context, collection, options, budget, skipIds) {
-	yield* collection.getItems({
+	const itemOptions = {
 		documentLoader: async (url) => {
 			return await loadCollectionItemDocument(context, url, options, budget, skipIds);
 		},
 		crossOrigin: "trust"
-	});
+	};
+	const pageOptions = {
+		documentLoader: async (url) => {
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
+			if (object == null) throw new Error(`Collection page not found: ${url}`);
+			return {
+				contextUrl: null,
+				documentUrl: url,
+				document: await object.toJsonLd()
+			};
+		},
+		crossOrigin: "trust"
+	};
+	const visitedIds = /* @__PURE__ */ new Set();
+	const visitedPages = /* @__PURE__ */ new WeakSet();
+	let current = collection;
+	while (current != null) {
+		budget.signal?.throwIfAborted();
+		if (visitedPages.has(current)) return;
+		if (current.id != null) {
+			if (visitedIds.has(current.id.href)) return;
+			visitedIds.add(current.id.href);
+		}
+		visitedPages.add(current);
+		yield* current.getItems(itemOptions);
+		const page = current instanceof _fedify_vocab.CollectionPage || current instanceof _fedify_vocab.OrderedCollectionPage ? current : null;
+		const pageId = page == null ? current.firstId : page.nextId;
+		if (pageId != null && visitedIds.has(pageId.href)) return;
+		try {
+			budget.signal?.throwIfAborted();
+			current = page == null ? await current.getFirst(pageOptions) : await page.getNext(pageOptions);
+		} catch (error) {
+			if (error instanceof MaxRequestsExceeded) throw error;
+			budget.signal?.throwIfAborted();
+			return;
+		}
+		if (pageId != null && pageId.href !== current?.id?.href) visitedIds.add(pageId.href);
+	}
 }
 async function getCreateActivityObject(context, object, options, budget) {
 	if (!(object instanceof _fedify_vocab.Create)) return null;
@@ -308,7 +345,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -331,7 +368,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -343,12 +380,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }
@@ -1560,4 +1602,292 @@ async function collect(context, note, options = {}) {
 		(0, node_assert_strict.deepStrictEqual)(iterations, [0, 1]);
 	});
 });
+(0, node_test.describe)("collection pagination", () => {
+	const iri = (name) => new URL(`https://example.com/${name}`);
+	for (const ordered of [false, true]) for (const replies of [false, true]) for (const embedded of [false, true]) (0, node_test.default)(`${ordered ? "ordered" : "unordered"} ${replies ? "replies" : "context"} with ${embedded ? "embedded" : "linked"} pages`, async () => {
+		const CollectionType = ordered ? _fedify_vocab.OrderedCollection : _fedify_vocab.Collection;
+		const PageType = ordered ? _fedify_vocab.OrderedCollectionPage : _fedify_vocab.CollectionPage;
+		const a = new _fedify_vocab.Note({ id: iri("a") });
+		const b = new _fedify_vocab.Note({ id: iri("b") });
+		const second = new PageType({
+			id: iri("page-2"),
+			items: [b]
+		});
+		const first = new PageType({
+			id: iri("page-1"),
+			items: [a],
+			next: embedded ? second : second.id
+		});
+		const collection = new CollectionType({
+			id: iri("thread"),
+			first: embedded ? first : first.id
+		});
+		const seed = new _fedify_vocab.Note({
+			id: iri("seed"),
+			contexts: replies ? [] : [iri("thread")],
+			replies: replies ? collection : null
+		});
+		const documents = /* @__PURE__ */ new Map([
+			[iri("thread").href, collection],
+			[iri("page-1").href, first],
+			[iri("page-2").href, second]
+		]);
+		const calls = [];
+		const items = await collect({ documentLoader: (url) => {
+			calls.push(url.href);
+			return Promise.resolve(documents.get(url.href) ?? null);
+		} }, seed, {
+			strategies: replies ? ["reply-tree"] : ["context-auto"],
+			maxDepth: 1,
+			maxRequests: embedded ? replies ? 0 : 1 : 3
+		});
+		(0, node_assert_strict.deepStrictEqual)(items.map((item) => item.id?.href), [a.id?.href, b.id?.href]);
+		(0, node_assert_strict.deepStrictEqual)(items.map((item) => item.depth), replies ? [1, 1] : [0, 0]);
+		(0, node_assert_strict.strictEqual)(calls.length, embedded ? replies ? 0 : 1 : replies ? 2 : 3);
+	});
+	(0, node_test.default)("starting page follows next and stops before loading a repeated page", async () => {
+		const first = new _fedify_vocab.OrderedCollectionPage({
+			id: iri("page-1"),
+			items: [new _fedify_vocab.Note({ id: iri("a") })],
+			next: iri("page-2")
+		});
+		const second = new _fedify_vocab.OrderedCollectionPage({
+			id: iri("page-2"),
+			items: [new _fedify_vocab.Note({ id: iri("b") })],
+			next: first.id
+		});
+		const calls = [];
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: (url) => {
+			calls.push(url.href);
+			return Promise.resolve(url.href === first.id?.href ? first : second);
+		} }, new _fedify_vocab.Note({ contexts: [first.id] }))).map((item) => item.id?.href), [iri("a").href, iri("b").href]);
+		(0, node_assert_strict.deepStrictEqual)(calls, [first.id?.href, second.id?.href]);
+	});
+	for (const limit of ["maxItems", "maxRequests"]) (0, node_test.default)(`${limit} stops between pages`, async () => {
+		const first = new _fedify_vocab.CollectionPage({
+			id: iri("page-1"),
+			items: [new _fedify_vocab.Note({ id: iri("a") })],
+			next: iri("page-2")
+		});
+		const calls = [];
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: (url) => {
+			calls.push(url.href);
+			return Promise.resolve(first);
+		} }, new _fedify_vocab.Note({
+			contexts: [first.id],
+			replies: new _fedify_vocab.Collection({ items: [new _fedify_vocab.Note({ id: iri("fallback") })] })
+		}), {
+			[limit]: 1,
+			strategies: ["context-auto", "reply-tree"]
+		})).map((item) => item.id?.href), [iri("a").href]);
+		(0, node_assert_strict.deepStrictEqual)(calls, [first.id?.href]);
+	});
+	for (const failure of [
+		"missing",
+		"throw",
+		"invalid"
+	]) (0, node_test.default)(`${failure} page retains items and allows the next strategy`, async () => {
+		const seed = new _fedify_vocab.Note({
+			contexts: [iri("thread")],
+			replies: new _fedify_vocab.Collection({ items: [new _fedify_vocab.Note({ id: iri("reply") })] })
+		});
+		const first = new _fedify_vocab.CollectionPage({
+			items: [new _fedify_vocab.Note({ id: iri("a") })],
+			next: iri("broken")
+		});
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: (url) => {
+			if (url.href === iri("thread").href) return Promise.resolve(new _fedify_vocab.Collection({ first }));
+			if (failure === "throw") throw new Error("page failed");
+			return Promise.resolve(failure === "missing" ? null : new _fedify_vocab.Note({}));
+		} }, seed, { strategies: ["context-auto", "reply-tree"] })).map((item) => item.id?.href), [iri("a").href, iri("reply").href]);
+	});
+	(0, node_test.default)("inline items precede page items", async () => {
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: () => Promise.resolve(new _fedify_vocab.Collection({
+			items: [new _fedify_vocab.Note({ id: iri("inline") })],
+			first: new _fedify_vocab.CollectionPage({ items: [new _fedify_vocab.Note({ id: iri("paged") })] })
+		})) }, new _fedify_vocab.Note({ contexts: [iri("thread")] }))).map((item) => item.id?.href), [iri("inline").href, iri("paged").href]);
+	});
+	(0, node_test.default)("cancellation during a page load prevents later strategies", async () => {
+		const controller = new AbortController();
+		const reason = /* @__PURE__ */ new Error("cancelled");
+		await (0, node_assert_strict.rejects)(collect({ documentLoader: (url, options) => {
+			(0, node_assert_strict.strictEqual)(options?.signal, controller.signal);
+			if (url.href === iri("thread").href) return Promise.resolve(new _fedify_vocab.Collection({ first: iri("page") }));
+			controller.abort(reason);
+			throw reason;
+		} }, new _fedify_vocab.Note({
+			contexts: [iri("thread")],
+			replies: new _fedify_vocab.Collection({ items: [new _fedify_vocab.Note({ id: iri("fallback") })] })
+		}), {
+			signal: controller.signal,
+			strategies: ["context-auto", "reply-tree"]
+		}), (error) => error === reason);
+	});
+});
+(0, node_test.default)("page loads share the cache and interval across strategies", async () => {
+	const iri = (name) => new URL(`https://example.com/${name}`);
+	const collection = new _fedify_vocab.Collection({ first: iri("page") });
+	const page = new _fedify_vocab.CollectionPage({ items: [new _fedify_vocab.Note({ id: iri("a") })] });
+	const seed = new _fedify_vocab.Note({
+		contexts: [iri("thread")],
+		replies: new _fedify_vocab.Collection({ first: iri("page") })
+	});
+	const calls = [];
+	const intervals = [];
+	(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: (url) => {
+		calls.push(url.href);
+		return Promise.resolve(url.href === iri("thread").href ? collection : page);
+	} }, seed, {
+		strategies: ["context-auto", "reply-tree"],
+		maxRequests: 2,
+		interval: (count) => {
+			intervals.push(count);
+			return { milliseconds: 0 };
+		}
+	})).map((item) => item.id?.href), [iri("a").href]);
+	(0, node_assert_strict.deepStrictEqual)(calls, [iri("thread").href, iri("page").href]);
+	(0, node_assert_strict.deepStrictEqual)(intervals, [0, 1]);
+});
+(0, node_test.default)("embedded pages without IDs follow next without loader calls", async () => {
+	const a = new _fedify_vocab.Note({ id: new URL("https://example.com/a") });
+	const b = new _fedify_vocab.Note({ id: new URL("https://example.com/b") });
+	(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: () => {
+		throw new Error("embedded pages must not be fetched");
+	} }, new _fedify_vocab.Note({ replies: new _fedify_vocab.Collection({ first: new _fedify_vocab.CollectionPage({
+		items: [a],
+		next: new _fedify_vocab.CollectionPage({ items: [b] })
+	}) }) }), {
+		strategies: ["reply-tree"],
+		maxRequests: 0,
+		maxDepth: 1
+	})).map((item) => item.id?.href), [a.id?.href, b.id?.href]);
+});
+(0, node_test.describe)("context loader failures", () => {
+	const contextId = new URL("https://example.com/thread");
+	const parent = new _fedify_vocab.Note({ id: new URL("https://example.com/parent") });
+	for (const synchronous of [false, true]) {
+		const failure = synchronous ? "synchronous throw" : "promise rejection";
+		const fail = () => {
+			const error = /* @__PURE__ */ new Error("Context collection unavailable");
+			if (synchronous) throw error;
+			return Promise.reject(error);
+		};
+		(0, node_test.default)(`${failure} allows reply-tree to yield an embedded parent`, async () => {
+			const seed = new _fedify_vocab.Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			let requests = 0;
+			const items = await collect({ documentLoader: () => {
+				requests++;
+				return fail();
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			});
+			(0, node_assert_strict.deepStrictEqual)(items.map((item) => item.object), [parent]);
+			(0, node_assert_strict.strictEqual)(items[0].strategy, "reply-tree");
+			(0, node_assert_strict.strictEqual)(requests, 1);
+		});
+		(0, node_test.default)(`${failure} finishes the default strategy without items`, async () => {
+			const seed = new _fedify_vocab.Note({ contexts: [contextId] });
+			(0, node_assert_strict.deepStrictEqual)(await collect({ documentLoader: fail }, seed), []);
+		});
+		(0, node_test.default)(`${failure} consumes the request budget`, async () => {
+			const seed = new _fedify_vocab.Note({
+				contexts: [contextId],
+				replyTarget: parent.id
+			});
+			const requests = [];
+			(0, node_assert_strict.deepStrictEqual)(await collect({ documentLoader: (url) => {
+				requests.push(url.href);
+				if (url.href === contextId.href) return fail();
+				return Promise.resolve(parent);
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			}), []);
+			(0, node_assert_strict.deepStrictEqual)(requests, [contextId.href]);
+		});
+		(0, node_test.default)(`${failure} propagates the cancellation reason`, async () => {
+			const controller = new AbortController();
+			const reason = /* @__PURE__ */ new Error("Stop backfill");
+			const seed = new _fedify_vocab.Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			const yielded = [];
+			await (0, node_assert_strict.rejects)(async () => {
+				for await (const item of backfill({ documentLoader: (_url, options) => {
+					(0, node_assert_strict.strictEqual)(options?.signal, controller.signal);
+					controller.abort(reason);
+					return fail();
+				} }, seed, {
+					strategies: ["context-auto", "reply-tree"],
+					signal: controller.signal
+				})) yielded.push(item);
+			}, (error) => error === reason);
+			(0, node_assert_strict.deepStrictEqual)(yielded, []);
+		});
+	}
+	(0, node_test.default)("a failed context load is not cached for later reply-tree loads", async () => {
+		const seed = new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: contextId
+		});
+		let requests = 0;
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: (url) => {
+			(0, node_assert_strict.strictEqual)(url.href, contextId.href);
+			requests++;
+			if (requests === 1) return Promise.reject(/* @__PURE__ */ new Error("Temporary failure"));
+			return Promise.resolve(parent);
+		} }, seed, {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 2
+		})).map((item) => item.object.id?.href), [parent.id?.href]);
+		(0, node_assert_strict.strictEqual)(requests, 2);
+	});
+	(0, node_test.default)("an exhausted budget still allows embedded reply-tree data", async () => {
+		(0, node_assert_strict.deepStrictEqual)((await collect({ documentLoader: () => {
+			throw new Error("No requests allowed");
+		} }, new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 0
+		})).map((item) => item.object), [parent]);
+	});
+	(0, node_test.default)("interval callback errors propagate before the loader is called", async () => {
+		const reason = /* @__PURE__ */ new Error("Invalid interval configuration");
+		let requests = 0;
+		await (0, node_assert_strict.rejects)(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: () => {
+				throw reason;
+			}
+		}), (error) => error === reason);
+		(0, node_assert_strict.strictEqual)(requests, 0);
+	});
+	(0, node_test.default)("invalid interval strings still propagate", async () => {
+		let requests = 0;
+		await (0, node_assert_strict.rejects)(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new _fedify_vocab.Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: "not a duration"
+		}));
+		(0, node_assert_strict.strictEqual)(requests, 0);
+	});
+});
 //#endregion

package/dist/backfill.test.js

@@ -91,7 +91,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -258,12 +258,49 @@ async function* getContextBackfillItems(context, object, strategy, options, budg
 	}
 }
 async function* getCollectionItems(context, collection, options, budget, skipIds) {
-	yield* collection.getItems({
+	const itemOptions = {
 		documentLoader: async (url) => {
 			return await loadCollectionItemDocument(context, url, options, budget, skipIds);
 		},
 		crossOrigin: "trust"
-	});
+	};
+	const pageOptions = {
+		documentLoader: async (url) => {
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
+			if (object == null) throw new Error(`Collection page not found: ${url}`);
+			return {
+				contextUrl: null,
+				documentUrl: url,
+				document: await object.toJsonLd()
+			};
+		},
+		crossOrigin: "trust"
+	};
+	const visitedIds = /* @__PURE__ */ new Set();
+	const visitedPages = /* @__PURE__ */ new WeakSet();
+	let current = collection;
+	while (current != null) {
+		budget.signal?.throwIfAborted();
+		if (visitedPages.has(current)) return;
+		if (current.id != null) {
+			if (visitedIds.has(current.id.href)) return;
+			visitedIds.add(current.id.href);
+		}
+		visitedPages.add(current);
+		yield* current.getItems(itemOptions);
+		const page = current instanceof CollectionPage || current instanceof OrderedCollectionPage ? current : null;
+		const pageId = page == null ? current.firstId : page.nextId;
+		if (pageId != null && visitedIds.has(pageId.href)) return;
+		try {
+			budget.signal?.throwIfAborted();
+			current = page == null ? await current.getFirst(pageOptions) : await page.getNext(pageOptions);
+		} catch (error) {
+			if (error instanceof MaxRequestsExceeded) throw error;
+			budget.signal?.throwIfAborted();
+			return;
+		}
+		if (pageId != null && pageId.href !== current?.id?.href) visitedIds.add(pageId.href);
+	}
 }
 async function getCreateActivityObject(context, object, options, budget) {
 	if (!(object instanceof Create)) return null;
@@ -285,7 +322,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -308,7 +345,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -320,12 +357,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }
@@ -1537,5 +1579,293 @@ describe("backfill", () => {
 		deepStrictEqual(iterations, [0, 1]);
 	});
 });
+describe("collection pagination", () => {
+	const iri = (name) => new URL(`https://example.com/${name}`);
+	for (const ordered of [false, true]) for (const replies of [false, true]) for (const embedded of [false, true]) test(`${ordered ? "ordered" : "unordered"} ${replies ? "replies" : "context"} with ${embedded ? "embedded" : "linked"} pages`, async () => {
+		const CollectionType = ordered ? OrderedCollection : Collection;
+		const PageType = ordered ? OrderedCollectionPage : CollectionPage;
+		const a = new Note({ id: iri("a") });
+		const b = new Note({ id: iri("b") });
+		const second = new PageType({
+			id: iri("page-2"),
+			items: [b]
+		});
+		const first = new PageType({
+			id: iri("page-1"),
+			items: [a],
+			next: embedded ? second : second.id
+		});
+		const collection = new CollectionType({
+			id: iri("thread"),
+			first: embedded ? first : first.id
+		});
+		const seed = new Note({
+			id: iri("seed"),
+			contexts: replies ? [] : [iri("thread")],
+			replies: replies ? collection : null
+		});
+		const documents = /* @__PURE__ */ new Map([
+			[iri("thread").href, collection],
+			[iri("page-1").href, first],
+			[iri("page-2").href, second]
+		]);
+		const calls = [];
+		const items = await collect({ documentLoader: (url) => {
+			calls.push(url.href);
+			return Promise.resolve(documents.get(url.href) ?? null);
+		} }, seed, {
+			strategies: replies ? ["reply-tree"] : ["context-auto"],
+			maxDepth: 1,
+			maxRequests: embedded ? replies ? 0 : 1 : 3
+		});
+		deepStrictEqual(items.map((item) => item.id?.href), [a.id?.href, b.id?.href]);
+		deepStrictEqual(items.map((item) => item.depth), replies ? [1, 1] : [0, 0]);
+		strictEqual(calls.length, embedded ? replies ? 0 : 1 : replies ? 2 : 3);
+	});
+	test("starting page follows next and stops before loading a repeated page", async () => {
+		const first = new OrderedCollectionPage({
+			id: iri("page-1"),
+			items: [new Note({ id: iri("a") })],
+			next: iri("page-2")
+		});
+		const second = new OrderedCollectionPage({
+			id: iri("page-2"),
+			items: [new Note({ id: iri("b") })],
+			next: first.id
+		});
+		const calls = [];
+		deepStrictEqual((await collect({ documentLoader: (url) => {
+			calls.push(url.href);
+			return Promise.resolve(url.href === first.id?.href ? first : second);
+		} }, new Note({ contexts: [first.id] }))).map((item) => item.id?.href), [iri("a").href, iri("b").href]);
+		deepStrictEqual(calls, [first.id?.href, second.id?.href]);
+	});
+	for (const limit of ["maxItems", "maxRequests"]) test(`${limit} stops between pages`, async () => {
+		const first = new CollectionPage({
+			id: iri("page-1"),
+			items: [new Note({ id: iri("a") })],
+			next: iri("page-2")
+		});
+		const calls = [];
+		deepStrictEqual((await collect({ documentLoader: (url) => {
+			calls.push(url.href);
+			return Promise.resolve(first);
+		} }, new Note({
+			contexts: [first.id],
+			replies: new Collection({ items: [new Note({ id: iri("fallback") })] })
+		}), {
+			[limit]: 1,
+			strategies: ["context-auto", "reply-tree"]
+		})).map((item) => item.id?.href), [iri("a").href]);
+		deepStrictEqual(calls, [first.id?.href]);
+	});
+	for (const failure of [
+		"missing",
+		"throw",
+		"invalid"
+	]) test(`${failure} page retains items and allows the next strategy`, async () => {
+		const seed = new Note({
+			contexts: [iri("thread")],
+			replies: new Collection({ items: [new Note({ id: iri("reply") })] })
+		});
+		const first = new CollectionPage({
+			items: [new Note({ id: iri("a") })],
+			next: iri("broken")
+		});
+		deepStrictEqual((await collect({ documentLoader: (url) => {
+			if (url.href === iri("thread").href) return Promise.resolve(new Collection({ first }));
+			if (failure === "throw") throw new Error("page failed");
+			return Promise.resolve(failure === "missing" ? null : new Note({}));
+		} }, seed, { strategies: ["context-auto", "reply-tree"] })).map((item) => item.id?.href), [iri("a").href, iri("reply").href]);
+	});
+	test("inline items precede page items", async () => {
+		deepStrictEqual((await collect({ documentLoader: () => Promise.resolve(new Collection({
+			items: [new Note({ id: iri("inline") })],
+			first: new CollectionPage({ items: [new Note({ id: iri("paged") })] })
+		})) }, new Note({ contexts: [iri("thread")] }))).map((item) => item.id?.href), [iri("inline").href, iri("paged").href]);
+	});
+	test("cancellation during a page load prevents later strategies", async () => {
+		const controller = new AbortController();
+		const reason = /* @__PURE__ */ new Error("cancelled");
+		await rejects(collect({ documentLoader: (url, options) => {
+			strictEqual(options?.signal, controller.signal);
+			if (url.href === iri("thread").href) return Promise.resolve(new Collection({ first: iri("page") }));
+			controller.abort(reason);
+			throw reason;
+		} }, new Note({
+			contexts: [iri("thread")],
+			replies: new Collection({ items: [new Note({ id: iri("fallback") })] })
+		}), {
+			signal: controller.signal,
+			strategies: ["context-auto", "reply-tree"]
+		}), (error) => error === reason);
+	});
+});
+test("page loads share the cache and interval across strategies", async () => {
+	const iri = (name) => new URL(`https://example.com/${name}`);
+	const collection = new Collection({ first: iri("page") });
+	const page = new CollectionPage({ items: [new Note({ id: iri("a") })] });
+	const seed = new Note({
+		contexts: [iri("thread")],
+		replies: new Collection({ first: iri("page") })
+	});
+	const calls = [];
+	const intervals = [];
+	deepStrictEqual((await collect({ documentLoader: (url) => {
+		calls.push(url.href);
+		return Promise.resolve(url.href === iri("thread").href ? collection : page);
+	} }, seed, {
+		strategies: ["context-auto", "reply-tree"],
+		maxRequests: 2,
+		interval: (count) => {
+			intervals.push(count);
+			return { milliseconds: 0 };
+		}
+	})).map((item) => item.id?.href), [iri("a").href]);
+	deepStrictEqual(calls, [iri("thread").href, iri("page").href]);
+	deepStrictEqual(intervals, [0, 1]);
+});
+test("embedded pages without IDs follow next without loader calls", async () => {
+	const a = new Note({ id: new URL("https://example.com/a") });
+	const b = new Note({ id: new URL("https://example.com/b") });
+	deepStrictEqual((await collect({ documentLoader: () => {
+		throw new Error("embedded pages must not be fetched");
+	} }, new Note({ replies: new Collection({ first: new CollectionPage({
+		items: [a],
+		next: new CollectionPage({ items: [b] })
+	}) }) }), {
+		strategies: ["reply-tree"],
+		maxRequests: 0,
+		maxDepth: 1
+	})).map((item) => item.id?.href), [a.id?.href, b.id?.href]);
+});
+describe("context loader failures", () => {
+	const contextId = new URL("https://example.com/thread");
+	const parent = new Note({ id: new URL("https://example.com/parent") });
+	for (const synchronous of [false, true]) {
+		const failure = synchronous ? "synchronous throw" : "promise rejection";
+		const fail = () => {
+			const error = /* @__PURE__ */ new Error("Context collection unavailable");
+			if (synchronous) throw error;
+			return Promise.reject(error);
+		};
+		test(`${failure} allows reply-tree to yield an embedded parent`, async () => {
+			const seed = new Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			let requests = 0;
+			const items = await collect({ documentLoader: () => {
+				requests++;
+				return fail();
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			});
+			deepStrictEqual(items.map((item) => item.object), [parent]);
+			strictEqual(items[0].strategy, "reply-tree");
+			strictEqual(requests, 1);
+		});
+		test(`${failure} finishes the default strategy without items`, async () => {
+			const seed = new Note({ contexts: [contextId] });
+			deepStrictEqual(await collect({ documentLoader: fail }, seed), []);
+		});
+		test(`${failure} consumes the request budget`, async () => {
+			const seed = new Note({
+				contexts: [contextId],
+				replyTarget: parent.id
+			});
+			const requests = [];
+			deepStrictEqual(await collect({ documentLoader: (url) => {
+				requests.push(url.href);
+				if (url.href === contextId.href) return fail();
+				return Promise.resolve(parent);
+			} }, seed, {
+				strategies: ["context-auto", "reply-tree"],
+				maxRequests: 1
+			}), []);
+			deepStrictEqual(requests, [contextId.href]);
+		});
+		test(`${failure} propagates the cancellation reason`, async () => {
+			const controller = new AbortController();
+			const reason = /* @__PURE__ */ new Error("Stop backfill");
+			const seed = new Note({
+				contexts: [contextId],
+				replyTarget: parent
+			});
+			const yielded = [];
+			await rejects(async () => {
+				for await (const item of backfill({ documentLoader: (_url, options) => {
+					strictEqual(options?.signal, controller.signal);
+					controller.abort(reason);
+					return fail();
+				} }, seed, {
+					strategies: ["context-auto", "reply-tree"],
+					signal: controller.signal
+				})) yielded.push(item);
+			}, (error) => error === reason);
+			deepStrictEqual(yielded, []);
+		});
+	}
+	test("a failed context load is not cached for later reply-tree loads", async () => {
+		const seed = new Note({
+			contexts: [contextId],
+			replyTarget: contextId
+		});
+		let requests = 0;
+		deepStrictEqual((await collect({ documentLoader: (url) => {
+			strictEqual(url.href, contextId.href);
+			requests++;
+			if (requests === 1) return Promise.reject(/* @__PURE__ */ new Error("Temporary failure"));
+			return Promise.resolve(parent);
+		} }, seed, {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 2
+		})).map((item) => item.object.id?.href), [parent.id?.href]);
+		strictEqual(requests, 2);
+	});
+	test("an exhausted budget still allows embedded reply-tree data", async () => {
+		deepStrictEqual((await collect({ documentLoader: () => {
+			throw new Error("No requests allowed");
+		} }, new Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			maxRequests: 0
+		})).map((item) => item.object), [parent]);
+	});
+	test("interval callback errors propagate before the loader is called", async () => {
+		const reason = /* @__PURE__ */ new Error("Invalid interval configuration");
+		let requests = 0;
+		await rejects(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: () => {
+				throw reason;
+			}
+		}), (error) => error === reason);
+		strictEqual(requests, 0);
+	});
+	test("invalid interval strings still propagate", async () => {
+		let requests = 0;
+		await rejects(collect({ documentLoader: () => {
+			requests++;
+			return Promise.resolve(null);
+		} }, new Note({
+			contexts: [contextId],
+			replyTarget: parent
+		}), {
+			strategies: ["context-auto", "reply-tree"],
+			interval: "not a duration"
+		}));
+		strictEqual(requests, 0);
+	});
+});
 //#endregion
 export {};

package/dist/mod.cjs

@@ -90,7 +90,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -257,12 +257,49 @@ async function* getContextBackfillItems(context, object, strategy, options, budg
 	}
 }
 async function* getCollectionItems(context, collection, options, budget, skipIds) {
-	yield* collection.getItems({
+	const itemOptions = {
 		documentLoader: async (url) => {
 			return await loadCollectionItemDocument(context, url, options, budget, skipIds);
 		},
 		crossOrigin: "trust"
-	});
+	};
+	const pageOptions = {
+		documentLoader: async (url) => {
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
+			if (object == null) throw new Error(`Collection page not found: ${url}`);
+			return {
+				contextUrl: null,
+				documentUrl: url,
+				document: await object.toJsonLd()
+			};
+		},
+		crossOrigin: "trust"
+	};
+	const visitedIds = /* @__PURE__ */ new Set();
+	const visitedPages = /* @__PURE__ */ new WeakSet();
+	let current = collection;
+	while (current != null) {
+		budget.signal?.throwIfAborted();
+		if (visitedPages.has(current)) return;
+		if (current.id != null) {
+			if (visitedIds.has(current.id.href)) return;
+			visitedIds.add(current.id.href);
+		}
+		visitedPages.add(current);
+		yield* current.getItems(itemOptions);
+		const page = current instanceof _fedify_vocab.CollectionPage || current instanceof _fedify_vocab.OrderedCollectionPage ? current : null;
+		const pageId = page == null ? current.firstId : page.nextId;
+		if (pageId != null && visitedIds.has(pageId.href)) return;
+		try {
+			budget.signal?.throwIfAborted();
+			current = page == null ? await current.getFirst(pageOptions) : await page.getNext(pageOptions);
+		} catch (error) {
+			if (error instanceof MaxRequestsExceeded) throw error;
+			budget.signal?.throwIfAborted();
+			return;
+		}
+		if (pageId != null && pageId.href !== current?.id?.href) visitedIds.add(pageId.href);
+	}
 }
 async function getCreateActivityObject(context, object, options, budget) {
 	if (!(object instanceof _fedify_vocab.Create)) return null;
@@ -284,7 +321,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -307,7 +344,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -319,12 +356,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }

package/dist/mod.js

@@ -89,7 +89,7 @@ function currentContextGroupHasAuto(strategies) {
 async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
 	const contextId = note.contextIds[0];
 	if (contextId == null) return;
-	const collection = await loadObject(context, contextId, options, budget);
+	const collection = await loadObject(context, contextId, options, budget, { skipLoaderErrors: true });
 	if (!isCollection(collection)) return;
 	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
 		object: item.object,
@@ -256,12 +256,49 @@ async function* getContextBackfillItems(context, object, strategy, options, budg
 	}
 }
 async function* getCollectionItems(context, collection, options, budget, skipIds) {
-	yield* collection.getItems({
+	const itemOptions = {
 		documentLoader: async (url) => {
 			return await loadCollectionItemDocument(context, url, options, budget, skipIds);
 		},
 		crossOrigin: "trust"
-	});
+	};
+	const pageOptions = {
+		documentLoader: async (url) => {
+			const object = await loadObject(context, new URL(url), options, budget, { throwOnBudgetExceeded: true });
+			if (object == null) throw new Error(`Collection page not found: ${url}`);
+			return {
+				contextUrl: null,
+				documentUrl: url,
+				document: await object.toJsonLd()
+			};
+		},
+		crossOrigin: "trust"
+	};
+	const visitedIds = /* @__PURE__ */ new Set();
+	const visitedPages = /* @__PURE__ */ new WeakSet();
+	let current = collection;
+	while (current != null) {
+		budget.signal?.throwIfAborted();
+		if (visitedPages.has(current)) return;
+		if (current.id != null) {
+			if (visitedIds.has(current.id.href)) return;
+			visitedIds.add(current.id.href);
+		}
+		visitedPages.add(current);
+		yield* current.getItems(itemOptions);
+		const page = current instanceof CollectionPage || current instanceof OrderedCollectionPage ? current : null;
+		const pageId = page == null ? current.firstId : page.nextId;
+		if (pageId != null && visitedIds.has(pageId.href)) return;
+		try {
+			budget.signal?.throwIfAborted();
+			current = page == null ? await current.getFirst(pageOptions) : await page.getNext(pageOptions);
+		} catch (error) {
+			if (error instanceof MaxRequestsExceeded) throw error;
+			budget.signal?.throwIfAborted();
+			return;
+		}
+		if (pageId != null && pageId.href !== current?.id?.href) visitedIds.add(pageId.href);
+	}
 }
 async function getCreateActivityObject(context, object, options, budget) {
 	if (!(object instanceof Create)) return null;
@@ -283,7 +320,7 @@ async function loadCollectionItemDocument(context, url, options, budget, skipIds
 	try {
 		const iri = new URL(url);
 		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
-		object = await loadObject(context, iri, options, budget, true);
+		object = await loadObject(context, iri, options, budget, { throwOnBudgetExceeded: true });
 	} catch (error) {
 		if (error instanceof MaxRequestsExceeded) throw error;
 		budget.signal?.throwIfAborted();
@@ -306,7 +343,7 @@ function skippedCollectionItemDocument(url) {
 		}
 	};
 }
-async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+async function loadObject(context, iri, options, budget, { throwOnBudgetExceeded = false, skipLoaderErrors = false } = {}) {
 	budget.signal?.throwIfAborted();
 	const cacheKey = iri.href;
 	const cached = budget.documents.get(cacheKey);
@@ -318,12 +355,17 @@ async function loadObject(context, iri, options, budget, throwOnBudgetExceeded =
 	await waitForInterval(options, budget);
 	budget.signal?.throwIfAborted();
 	budget.requestCount++;
-	const document = context.documentLoader(iri, { signal: budget.signal });
-	budget.documents.set(cacheKey, document);
+	let document;
 	try {
+		document = context.documentLoader(iri, { signal: budget.signal });
+		budget.documents.set(cacheKey, document);
 		return await document;
 	} catch (error) {
 		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		if (skipLoaderErrors) {
+			budget.signal?.throwIfAborted();
+			return null;
+		}
 		throw error;
 	}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fedify/backfill",
-  "version": "2.5.0-dev.2364+4d9b5f4d",
+  "version": "2.5.0-dev.2378+fea9c751",
   "description": "ActivityPub backfill support for Fedify",
   "keywords": [
     "Fedify",
@@ -46,7 +46,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@fedify/vocab": "2.5.0-dev.2364+4d9b5f4d"
+    "@fedify/vocab": "2.5.0-dev.2378+fea9c751"
   },
   "devDependencies": {
     "tsdown": "^0.22.0",