@fedify/backfill 2.3.0-dev.0

package/dist/mod.js

@@ -0,0 +1,364 @@
+import { Activity, Collection, CollectionPage, Create, Object as Object$1, OrderedCollection, OrderedCollectionPage } from "@fedify/vocab";
+//#region src/backfill.ts
+const defaultStrategies = ["context-auto"];
+const DEFAULT_MAX_DEPTH = 10;
+/**
+* Thrown when backfill traversal exceeds the configured request budget.
+*
+* @since 2.3.0
+*/
+var MaxRequestsExceeded = class extends Error {};
+/**
+* Backfills post-like objects related to a seed object.
+*
+* The seed object is not yielded by default, but its ID is treated as already
+* seen so it will not be yielded again if the collection contains it.
+*
+* @since 2.3.0
+*/
+async function* backfill(context, note, options = {}) {
+	if (options.maxItems != null && options.maxItems <= 0) return;
+	const strategies = normalizeStrategies(options.strategies);
+	if (strategies.length < 1) return;
+	const budget = {
+		signal: options.signal,
+		requestCount: 0,
+		documents: /* @__PURE__ */ new Map()
+	};
+	const seenIds = /* @__PURE__ */ new Set();
+	if (note.id != null) seenIds.add(note.id.href);
+	let yielded = 0;
+	try {
+		for (let i = 0; i < strategies.length; i++) {
+			const strategy = strategies[i];
+			let items;
+			if (isContextStrategy(strategy)) {
+				const contextStrategies = [strategy];
+				while (true) {
+					const nextStrategy = strategies[i + 1];
+					if (nextStrategy == null || !isContextStrategy(nextStrategy)) break;
+					contextStrategies.push(nextStrategy);
+					i++;
+				}
+				items = getContextStrategyItems(context, note, contextStrategies, options, budget, seenIds);
+			} else items = getStrategyItems(context, note, strategy, options, budget, seenIds);
+			for await (const item of items) {
+				const id = item.object.id ?? void 0;
+				if (id != null) {
+					if (seenIds.has(id.href)) continue;
+					seenIds.add(id.href);
+				}
+				options.signal?.throwIfAborted();
+				yield {
+					object: item.object,
+					id,
+					strategy: item.strategy,
+					origin: item.origin,
+					depth: item.depth
+				};
+				yielded++;
+				if (options.maxItems != null && yielded >= options.maxItems) return;
+			}
+		}
+	} catch (error) {
+		if (error instanceof MaxRequestsExceeded) return;
+		throw error;
+	}
+}
+function normalizeStrategies(strategies = defaultStrategies) {
+	const normalized = [];
+	for (const strategy of strategies) if (strategy === "context-auto") {
+		for (let i = normalized.length - 1; i >= 0 && isContextStrategy(normalized[i]); i--) normalized.splice(i, 1);
+		if (!normalized.includes(strategy)) normalized.push(strategy);
+	} else if (isContextStrategy(strategy)) {
+		if (!currentContextGroupHasAuto(normalized) && !normalized.includes(strategy)) normalized.push(strategy);
+	} else if (!normalized.includes(strategy)) normalized.push(strategy);
+	return normalized;
+}
+function isContextStrategy(strategy) {
+	return strategy === "context-objects" || strategy === "context-activities" || strategy === "context-auto";
+}
+function currentContextGroupHasAuto(strategies) {
+	for (let i = strategies.length - 1; i >= 0; i--) {
+		const strategy = strategies[i];
+		if (!isContextStrategy(strategy)) return false;
+		if (strategy === "context-auto") return true;
+	}
+	return false;
+}
+async function* getContextStrategyItems(context, note, strategies, options, budget, seenIds) {
+	const contextId = note.contextIds[0];
+	if (contextId == null) return;
+	const collection = await loadObject(context, contextId, options, budget);
+	if (!isCollection(collection)) return;
+	for await (const object of getCollectionItems(context, collection, options, budget, seenIds)) for (const strategy of strategies) for await (const item of getContextBackfillItems(context, object, strategy, options, budget)) yield {
+		object: item.object,
+		strategy: item.strategy,
+		origin: "collection",
+		depth: 0
+	};
+}
+async function* getStrategyItems(context, note, strategy, options, budget, seenIds) {
+	if (isContextStrategy(strategy)) yield* getContextStrategyItems(context, note, [strategy], options, budget, seenIds);
+	else if (strategy === "reply-tree") yield* getReplyTreeItems(context, note, options, budget);
+}
+async function* getReplyTreeItems(context, note, options, budget) {
+	const visitedObjectIds = /* @__PURE__ */ new Set();
+	const visitedObjects = /* @__PURE__ */ new WeakSet();
+	const visitedCollectionIds = /* @__PURE__ */ new Set();
+	const visitedCollections = /* @__PURE__ */ new WeakSet();
+	if (note.id != null) visitedObjectIds.add(note.id.href);
+	visitedObjects.add(note);
+	const ancestors = [];
+	for await (const item of getReplyAncestors(context, note, options, budget, {
+		depth: 1,
+		visitedObjectIds,
+		visitedObjects,
+		visitedCollectionIds,
+		visitedCollections
+	})) {
+		ancestors.push({
+			object: item.object,
+			depth: item.depth
+		});
+		yield item;
+	}
+	for (const ancestor of ancestors.toReversed()) yield* getReplyDescendants(context, ancestor.object, options, budget, {
+		depth: ancestor.depth + 1,
+		visitedObjectIds,
+		visitedObjects,
+		visitedCollectionIds,
+		visitedCollections
+	});
+	yield* getReplyDescendants(context, note, options, budget, {
+		depth: 1,
+		visitedObjectIds,
+		visitedObjects,
+		visitedCollectionIds,
+		visitedCollections
+	});
+}
+async function* getReplyAncestors(context, object, options, budget, traversal) {
+	if (traversal.depth > (options.maxDepth ?? DEFAULT_MAX_DEPTH)) return;
+	for await (const target of getReplyTargets(context, object, options, budget)) {
+		if (!isContextPostObject(target)) continue;
+		if (!visitReplyTreeObject(target, traversal)) continue;
+		yield {
+			object: target,
+			strategy: "reply-tree",
+			origin: "in-reply-to",
+			depth: traversal.depth
+		};
+		yield* getReplyAncestors(context, target, options, budget, {
+			depth: traversal.depth + 1,
+			visitedObjectIds: traversal.visitedObjectIds,
+			visitedObjects: traversal.visitedObjects,
+			visitedCollectionIds: traversal.visitedCollectionIds,
+			visitedCollections: traversal.visitedCollections
+		});
+	}
+}
+async function* getReplyDescendants(context, object, options, budget, traversal) {
+	if (traversal.depth > (options.maxDepth ?? DEFAULT_MAX_DEPTH)) return;
+	const repliesId = object.repliesId;
+	if (repliesId != null && traversal.visitedCollectionIds.has(repliesId.href)) return;
+	const replies = await getRepliesCollection(context, object, options, budget);
+	if (replies == null) return;
+	const unvisited = visitReplyTreeCollection(replies, traversal);
+	if (repliesId != null) traversal.visitedCollectionIds.add(repliesId.href);
+	if (!unvisited) return;
+	for await (const reply of getCollectionItems(context, replies, options, budget, traversal.visitedObjectIds)) {
+		if (!isContextPostObject(reply)) continue;
+		if (!visitReplyTreeObject(reply, traversal)) continue;
+		yield {
+			object: reply,
+			strategy: "reply-tree",
+			origin: "replies",
+			depth: traversal.depth
+		};
+		yield* getReplyDescendants(context, reply, options, budget, {
+			depth: traversal.depth + 1,
+			visitedObjectIds: traversal.visitedObjectIds,
+			visitedObjects: traversal.visitedObjects,
+			visitedCollectionIds: traversal.visitedCollectionIds,
+			visitedCollections: traversal.visitedCollections
+		});
+	}
+}
+async function* getReplyTargets(context, object, options, budget) {
+	try {
+		yield* object.getReplyTargets({
+			documentLoader: async (url) => {
+				return await loadCollectionItemDocument(context, url, options, budget);
+			},
+			crossOrigin: "trust"
+		});
+	} catch (error) {
+		if (error instanceof MaxRequestsExceeded) throw error;
+		budget.signal?.throwIfAborted();
+	}
+}
+async function getRepliesCollection(context, object, options, budget) {
+	try {
+		return await object.getReplies({
+			documentLoader: async (url) => {
+				return await loadCollectionItemDocument(context, url, options, budget);
+			},
+			crossOrigin: "trust"
+		});
+	} catch (error) {
+		if (error instanceof MaxRequestsExceeded) throw error;
+		budget.signal?.throwIfAborted();
+		return null;
+	}
+}
+function visitReplyTreeObject(object, traversal) {
+	if (object.id != null) {
+		if (traversal.visitedObjectIds.has(object.id.href)) return false;
+		traversal.visitedObjectIds.add(object.id.href);
+	} else if (traversal.visitedObjects.has(object)) return false;
+	traversal.visitedObjects.add(object);
+	return true;
+}
+function visitReplyTreeCollection(collection, traversal) {
+	if (collection.id != null) return visitReplyTreeCollectionId(collection.id, traversal);
+	else if (traversal.visitedCollections.has(collection)) return false;
+	traversal.visitedCollections.add(collection);
+	return true;
+}
+function visitReplyTreeCollectionId(id, traversal) {
+	if (traversal.visitedCollectionIds.has(id.href)) return false;
+	traversal.visitedCollectionIds.add(id.href);
+	return true;
+}
+async function* getContextBackfillItems(context, object, strategy, options, budget) {
+	if (strategy === "context-objects" && isContextPostObject(object)) yield {
+		object,
+		strategy
+	};
+	else if (strategy === "context-activities") {
+		const activityObject = await getCreateActivityObject(context, object, options, budget);
+		if (activityObject != null && isContextPostObject(activityObject)) yield {
+			object: activityObject,
+			strategy
+		};
+	} else if (strategy === "context-auto") {
+		if (object instanceof Activity) {
+			const activityObject = await getCreateActivityObject(context, object, options, budget);
+			if (activityObject != null && isContextPostObject(activityObject)) yield {
+				object: activityObject,
+				strategy
+			};
+		} else if (isContextPostObject(object)) yield {
+			object,
+			strategy
+		};
+	}
+}
+async function* getCollectionItems(context, collection, options, budget, skipIds) {
+	yield* collection.getItems({
+		documentLoader: async (url) => {
+			return await loadCollectionItemDocument(context, url, options, budget, skipIds);
+		},
+		crossOrigin: "trust"
+	});
+}
+async function getCreateActivityObject(context, object, options, budget) {
+	if (!(object instanceof Create)) return null;
+	try {
+		return await object.getObject({
+			documentLoader: async (url) => {
+				return await loadCollectionItemDocument(context, url, options, budget);
+			},
+			crossOrigin: "trust"
+		});
+	} catch (error) {
+		if (error instanceof MaxRequestsExceeded) throw error;
+		budget.signal?.throwIfAborted();
+		return null;
+	}
+}
+async function loadCollectionItemDocument(context, url, options, budget, skipIds) {
+	let object;
+	try {
+		const iri = new URL(url);
+		if (skipIds?.has(iri.href)) return skippedCollectionItemDocument(url);
+		object = await loadObject(context, iri, options, budget, true);
+	} catch (error) {
+		if (error instanceof MaxRequestsExceeded) throw error;
+		budget.signal?.throwIfAborted();
+		return skippedCollectionItemDocument(url);
+	}
+	if (object == null) return skippedCollectionItemDocument(url);
+	return {
+		contextUrl: null,
+		documentUrl: url,
+		document: await object.toJsonLd()
+	};
+}
+function skippedCollectionItemDocument(url) {
+	return {
+		contextUrl: null,
+		documentUrl: url,
+		document: {
+			"@context": "https://www.w3.org/ns/activitystreams",
+			type: "Activity"
+		}
+	};
+}
+async function loadObject(context, iri, options, budget, throwOnBudgetExceeded = false) {
+	budget.signal?.throwIfAborted();
+	const cacheKey = iri.href;
+	const cached = budget.documents.get(cacheKey);
+	if (cached != null) return await cached;
+	if (options.maxRequests != null && budget.requestCount >= options.maxRequests) {
+		if (throwOnBudgetExceeded) throw new MaxRequestsExceeded();
+		return null;
+	}
+	await waitForInterval(options, budget);
+	budget.signal?.throwIfAborted();
+	budget.requestCount++;
+	const document = context.documentLoader(iri, { signal: budget.signal });
+	budget.documents.set(cacheKey, document);
+	try {
+		return await document;
+	} catch (error) {
+		if (budget.documents.get(cacheKey) === document) budget.documents.delete(cacheKey);
+		throw error;
+	}
+}
+async function waitForInterval(options, budget) {
+	if (options.interval == null) return;
+	const milliseconds = durationToMilliseconds(typeof options.interval === "function" ? options.interval(budget.requestCount) : options.interval);
+	if (milliseconds <= 0) return;
+	await new Promise((resolve, reject) => {
+		if (budget.signal?.aborted) {
+			reject(budget.signal.reason);
+			return;
+		}
+		const timeout = setTimeout(() => {
+			budget.signal?.removeEventListener("abort", onAbort);
+			resolve();
+		}, milliseconds);
+		const onAbort = () => {
+			clearTimeout(timeout);
+			reject(budget.signal?.reason);
+		};
+		budget.signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}
+function durationToMilliseconds(duration) {
+	if (typeof duration === "string") {
+		if (typeof Temporal === "undefined") throw new TypeError("Temporal is not globally available; pass interval as a Temporal.DurationLike object instead of a string, or provide a Temporal polyfill.");
+		return Temporal.Duration.from(duration).total({ unit: "milliseconds" });
+	}
+	return (duration.milliseconds ?? 0) + (duration.seconds ?? 0) * 1e3 + (duration.minutes ?? 0) * 60 * 1e3 + (duration.hours ?? 0) * 60 * 60 * 1e3 + (duration.days ?? 0) * 24 * 60 * 60 * 1e3;
+}
+function isCollection(object) {
+	return object instanceof Collection || object instanceof OrderedCollection || object instanceof CollectionPage || object instanceof OrderedCollectionPage;
+}
+function isContextPostObject(object) {
+	return object instanceof Object$1 && !(object instanceof Activity) && !isCollection(object);
+}
+//#endregion
+export { MaxRequestsExceeded, backfill };

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "@fedify/backfill",
+  "version": "2.3.0-dev.0",
+  "description": "ActivityPub backfill support for Fedify",
+  "keywords": [
+    "Fedify",
+    "ActivityPub",
+    "Fediverse",
+    "Backfill"
+  ],
+  "author": {
+    "name": "Jiwon Kwon",
+    "email": "work@kwonjiwon.org"
+  },
+  "homepage": "https://fedify.dev/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fedify-dev/fedify.git",
+    "directory": "packages/backfill"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/fedify-dev/fedify/issues"
+  },
+  "funding": [
+    "https://opencollective.com/fedify",
+    "https://github.com/sponsors/dahlia"
+  ],
+  "type": "module",
+  "main": "./dist/mod.cjs",
+  "module": "./dist/mod.js",
+  "types": "./dist/mod.d.ts",
+  "exports": {
+    ".": {
+      "types": {
+        "import": "./dist/mod.d.ts",
+        "require": "./dist/mod.d.cts",
+        "default": "./dist/mod.d.ts"
+      },
+      "import": "./dist/mod.js",
+      "require": "./dist/mod.cjs",
+      "default": "./dist/mod.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist/",
+    "package.json",
+    "README.md"
+  ],
+  "dependencies": {
+    "@fedify/vocab": "2.3.0"
+  },
+  "devDependencies": {
+    "tsdown": "^0.22.0",
+    "typescript": "^6.0.0"
+  },
+  "scripts": {
+    "build:self": "tsdown",
+    "build": "pnpm --filter @fedify/backfill... run build:self",
+    "prepublish": "pnpm build",
+    "pretest": "pnpm build",
+    "test": "cd dist/ && node --test",
+    "pretest:bun": "pnpm build",
+    "test:bun": "cd dist/ && bun test --timeout 60000"
+  }
+}