@chrisburnell/eleventy-cache-webmentions 2.3.2 → 2.3.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chrisburnell/eleventy-cache-webmentions",
-  "version": "2.3.2",
+  "version": "2.3.3",
   "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.",
   "license": "MIT",
   "repository": {
@@ -44,9 +44,13 @@
       "require": "./eleventy-cache-webmentions.cjs"
     }
   },
+  "files": [
+    "eleventy-cache-webmentions.js",
+    "eleventy-cache-webmentions.cjs",
+    "src/core.cjs"
+  ],
   "scripts": {
-    "build": "esbuild eleventy-cache-webmentions.js --minify-whitespace --minify-syntax --outfile=eleventy-cache-webmentions.min.js && esbuild eleventy-cache-webmentions.cjs --minify-whitespace --minify-syntax --outfile=eleventy-cache-webmentions.min.cjs",
-    "lint": "eslint eleventy-cache-webmentions.js",
+    "lint": "eslint .",
     "test": "node --test"
   },
   "keywords": [
@@ -65,11 +69,9 @@
     "sanitize-html": "^2.17.0"
   },
   "devDependencies": {
-    "esbuild": "^0.27.1",
     "eslint": "^9.39.1",
     "eslint-plugin-jsdoc": "^61.4.1",
-    "globals": "^16.5.0",
-    "nock": "^14.0.10"
+    "globals": "^16.5.0"
   },
   "type": "module"
 }

package/src/core.cjs

@@ -0,0 +1,689 @@
+/**
+ * @typedef {object} AllowedHTML
+ * @property {Array<string>} [allowedTags]
+ * @property {{[attribute: string]: Array<string>}} [allowedAttributes]
+ */
+
+/**
+ * @typedef {object} OptionsDefaults
+ * @property {boolean} refresh
+ * @property {string} duration
+ * @property {string} uniqueKey
+ * @property {string} [cacheDirectory]
+ * @property {AllowedHTML} allowedHTML
+ * @property {Array<string>} allowlist
+ * @property {Array<string>} blocklist
+ * @property {{[key: string]: string}} urlReplacements
+ * @property {number} maximumHtmlLength
+ * @property {string} maximumHtmlText
+ */
+
+/**
+ * @typedef {object} OptionsUserInput
+ * @property {string} domain
+ * @property {string} feed
+ * @property {string} key
+ */
+
+/**
+ * @typedef {OptionsDefaults & OptionsUserInput} Options
+ */
+
+/**
+ * @typedef {object} Webmention
+ * @property {string} [source]
+ * @property {string} [url]
+ * @property {string} [target]
+ * @property {string} [published]
+ * @property {string} [contentSanitized]
+ * @property {object} [content]
+ * @property {string} [content.html]
+ * @property {string} [content.value]
+ * @property {object} [data]
+ * @property {string} [data.title]
+ * @property {string} [data.url]
+ * @property {string} [data.published]
+ * @property {string} [data.content]
+ * @property {string} [type]
+ * @property {object} [activity]
+ * @property {string} [activity.type]
+ * @property {boolean} [verified]
+ * @property {string} ["wm-property"]
+ * @property {string} ["wm-received"]
+ * @property {string} ["wm-source"]
+ * @property {string} ["wm-target"]
+ * @property {string} ["verified_date"]
+ */
+
+/**
+ * @typedef {"bookmark-of"|"like-of"|"repost-of"|"mention-of"|"in-reply-to"} WebmentionType
+ */
+
+/**
+ * @param {object} deps
+ * @param {typeof import("@11ty/eleventy-fetch").AssetCache} deps.AssetCache
+ * @param {typeof import("node:util").styleText} deps.styleText
+ * @param {typeof import("sanitize-html")} deps.sanitizeHTML
+ * @returns {object}
+ */
+module.exports = ({ AssetCache, styleText, sanitizeHTML }) => {
+	/**
+	 * @type {OptionsDefaults}
+	 */
+	const defaults = {
+		refresh: false,
+		duration: "1d",
+		uniqueKey: "webmentions",
+		cacheDirectory: undefined,
+		allowedHTML: {
+			allowedTags: ["a", "b", "em", "i", "strong"],
+			allowedAttributes: {
+				a: ["href"],
+			},
+		},
+		allowlist: [],
+		blocklist: [],
+		urlReplacements: {},
+		maximumHtmlLength: 1000,
+		maximumHtmlText: "mentioned this in",
+	};
+
+	/**
+	 * @param {string} url
+	 * @param {string} domain
+	 * @returns {string}
+	 */
+	const absoluteURL = (url, domain) => {
+		try {
+			return new URL(url, domain).toString();
+		} catch (error) {
+			console.error(
+				`Trying to convert ${styleText(
+					"bold",
+					url,
+				)} to be an absolute url with base ${styleText(
+					"bold",
+					domain,
+				)} and failed.`,
+				error,
+			);
+			return url;
+		}
+	};
+
+	/**
+	 * @param {string} url
+	 * @returns {string}
+	 */
+	const baseURL = (url) => {
+		let hashSplit = url.split("#");
+		let queryparamSplit = hashSplit[0].split("?");
+		return queryparamSplit[0];
+	};
+
+	/**
+	 * @param {string} url
+	 * @param {{[key: string]: string}} [urlReplacements]
+	 * @returns {string}
+	 */
+	const fixURL = (url, urlReplacements) => {
+		return Object.entries(urlReplacements).reduce(
+			(accumulator, [key, value]) => accumulator.replaceAll(key, value),
+			url,
+		);
+	};
+
+	/**
+	 * @param {string} url
+	 * @returns {string}
+	 */
+	const hostname = (url) => {
+		if (typeof url === "string" && url.includes("//")) {
+			const urlObject = new URL(url);
+			return urlObject.hostname;
+		}
+		return url;
+	};
+
+	/**
+	 * @param {string|number|Date} date
+	 * @returns {number}
+	 */
+	const epoch = (date) => {
+		return new Date(date).getTime();
+	};
+
+	/**
+	 * @param {Array<Webmention>} webmentions
+	 * @returns {Array<Webmention>}
+	 */
+	const removeDuplicates = (webmentions) => {
+		return [
+			...webmentions
+				.reduce((map, webmention) => {
+					const key =
+						webmention === null || webmention === undefined
+							? webmention
+							: getUniqueKey(webmention);
+					if (!map.has(key)) {
+						map.set(key, webmention);
+					}
+					return map;
+				}, new Map())
+				.values(),
+		];
+	};
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string|undefined}
+	 */
+	const getPublished = (webmention) => {
+		return (
+			webmention?.["data"]?.["published"] ||
+			webmention["published"] ||
+			webmention["wm-received"] ||
+			webmention["verified_date"]
+		);
+	};
+	const getWebmentionPublished = getPublished;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string|undefined}
+	 */
+	const getReceived = (webmention) => {
+		return (
+			webmention["wm-received"] ||
+			webmention["verified_date"] ||
+			webmention["published"] ||
+			webmention?.["data"]?.["published"]
+		);
+	};
+	const getWebmentionReceived = getReceived;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string}
+	 */
+	const getContent = (webmention) => {
+		return (
+			webmention?.["contentSanitized"] ||
+			webmention?.["content"]?.["html"] ||
+			webmention?.["content"]?.["value"] ||
+			webmention?.["content"] ||
+			webmention?.["data"]?.["content"] ||
+			""
+		);
+	};
+	const getWebmentionContent = getContent;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string|undefined}
+	 */
+	const getSource = (webmention) => {
+		return (
+			webmention["wm-source"] ||
+			webmention["source"] ||
+			webmention?.["data"]?.["url"] ||
+			webmention["url"]
+		);
+	};
+	const getWebmentionSource = getSource;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string|undefined}
+	 */
+	const getURL = (webmention) => {
+		return (
+			webmention?.["data"]?.["url"] ||
+			webmention["url"] ||
+			webmention["wm-source"] ||
+			webmention["source"]
+		);
+	};
+	const getWebmentionURL = getURL;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string|undefined}
+	 */
+	const getTarget = (webmention) => {
+		return webmention["wm-target"] || webmention["target"];
+	};
+	const getWebmentionTarget = getTarget;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string}
+	 */
+	const getUniqueKey = (webmention) => {
+		return JSON.stringify([getSource(webmention), getTarget(webmention)]);
+	};
+	const getWebmentionUniqueKey = getUniqueKey;
+
+	/**
+	 * @param {Webmention} webmention
+	 * @returns {string|undefined}
+	 */
+	const getType = (webmention) => {
+		return (
+			webmention["wm-property"] ||
+			webmention?.["activity"]?.["type"] ||
+			webmention["type"]
+		);
+	};
+	const getWebmentionType = getType;
+
+	/**
+	 * @param {Array<Webmention>} webmentions
+	 * @param {WebmentionType|Array<WebmentionType>} types
+	 * @returns {Array<Webmention>}
+	 */
+	const getByTypes = (webmentions, types) => {
+		return webmentions.filter((webmention) => {
+			if (typeof types === "string") {
+				return types === getType(webmention);
+			}
+			return types.includes(getType(webmention));
+		});
+	};
+	const getByType = getByTypes;
+	const getWebmentionsByTypes = getByTypes;
+	const getWebmentionsByType = getByTypes;
+
+	/**
+	 * @param {Array<Webmention>} webmentions
+	 * @param {Array<string>} blocklist
+	 * @returns {Array<Webmention>}
+	 */
+	const processBlocklist = (webmentions, blocklist) => {
+		return webmentions.filter((webmention) => {
+			let url = getSource(webmention);
+			let source = getSource(webmention);
+			for (let blocklistURL of blocklist) {
+				if (
+					url.includes(blocklistURL.replace(/\/?$/, "/")) ||
+					source.includes(blocklistURL.replace(/\/?$/, "/"))
+				) {
+					return false;
+				}
+			}
+			return true;
+		});
+	};
+	const processWebmentionBlocklist = processBlocklist;
+	const processWebmentionsBlocklist = processBlocklist;
+
+	/**
+	 * @param {Array<Webmention>} webmentions
+	 * @param {Array<string>} allowlist
+	 * @returns {Array<Webmention>}
+	 */
+	const processAllowlist = (webmentions, allowlist) => {
+		return webmentions.filter((webmention) => {
+			let url = getSource(webmention);
+			let source = getSource(webmention);
+			for (let allowlistURL of allowlist) {
+				if (
+					url.includes(allowlistURL.replace(/\/?$/, "/")) ||
+					source.includes(allowlistURL.replace(/\/?$/, "/"))
+				) {
+					return true;
+				}
+			}
+			return false;
+		});
+	};
+	const processWebmentionAllowlist = processAllowlist;
+	const processWebmentionsAllowlist = processAllowlist;
+
+	/**
+	 * @param {Options} options
+	 * @param {Array<Webmention>} webmentions
+	 * @param {string} url
+	 * @returns {Promise<{found: number, webmentions: Array<Webmention>}>}
+	 */
+	const fetchWebmentions = async (options, webmentions, url) => {
+		return await fetch(url)
+			.then(async (response) => {
+				if (!response.ok) {
+					return Promise.reject(response);
+				}
+
+				const feed = await response.json();
+
+				if (!(options.key in feed)) {
+					console.log(
+						`${styleText("gray", `[${hostname(options.domain)}]`)} ${
+							options.key
+						} was not found as a key in the response from ${styleText(
+							"bold",
+							hostname(options.feed),
+						)}!`,
+					);
+					return Promise.reject(response);
+				}
+
+				// Fetched Webmentions replace cached ones with the same
+				// source / target pair
+				const fetchedKeys = new Set(feed[options.key].map(getUniqueKey));
+				webmentions = [
+					...feed[options.key],
+					...webmentions.filter((wm) => !fetchedKeys.has(getUniqueKey(wm))),
+				];
+				// Remove any remaining duplicates by source / target pair
+				webmentions = removeDuplicates(webmentions);
+				// Process the blocklist, if it has any entries
+				if (options.blocklist.length) {
+					webmentions = processBlocklist(webmentions, options.blocklist);
+				}
+				// Process the allowlist, if it has any entries
+				if (options.allowlist.length) {
+					webmentions = processAllowlist(webmentions, options.allowlist);
+				}
+				// Sort webmentions by received date for getting most recent Webmention on subsequent requests
+				webmentions = webmentions.sort((a, b) => {
+					return epoch(getReceived(b)) - epoch(getReceived(a));
+				});
+
+				return {
+					found: feed[options.key].length,
+					webmentions: webmentions,
+				};
+			})
+			.catch((error) => {
+				console.warn(
+					`${styleText(
+						"gray",
+						`[${hostname(options.domain)}]`,
+					)} Something went wrong with your Webmention request to ${styleText(
+						"bold",
+						hostname(options.feed),
+					)}!`,
+				);
+				console.warn(error instanceof Error ? error.message : error);
+
+				return {
+					found: 0,
+					webmentions: webmentions,
+				};
+			});
+	};
+
+	/**
+	 * @param {Options} options
+	 * @returns {Promise<Array<Webmention>>}
+	 */
+	const retrieveWebmentions = async (options) => {
+		if (!options.domain) {
+			throw new Error(
+				"`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.",
+			);
+		}
+
+		if (!options.feed) {
+			throw new Error(
+				"`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.",
+			);
+		}
+
+		if (!options.key) {
+			throw new Error(
+				"`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.",
+			);
+		}
+
+		let asset = new AssetCache(
+			options.uniqueKey || `webmentions-${hostname(options.domain)}`,
+			options.cacheDirectory,
+		);
+
+		let webmentions = [];
+
+		// Unless specifically getting fresh Webmentions, if there is a cached file
+		// at all, grab its contents now
+		if (asset.isCacheValid("*") && !options.refresh) {
+			webmentions = await asset.getCachedValue();
+		}
+
+		// Get the number of cached Webmentions for diffing against fetched
+		// Webmentions later
+		const webmentionsCachedLength = webmentions.length;
+
+		// If there is a cached file but it is outside of expiry, fetch fresh
+		// results since the most recent Webmention
+		if (!asset.isCacheValid(options.refresh ? "0s" : options.duration)) {
+			const performanceStart = process.hrtime();
+			// Get the received date of the most recent Webmention, if it exists
+			const since = webmentions.length ? getReceived(webmentions[0]) : false;
+			// Build the URL for the fetch request
+			const url = `${options.feed}${
+				since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""
+			}`;
+
+			// If using webmention.io or paginate or per-page in URL
+			const urlObject = new URL(url);
+			if (
+				url.includes("https://webmention.io") ||
+				options.paginate ||
+				urlObject.searchParams.has("per-page")
+			) {
+				const perPage = Number(urlObject.searchParams.get("per-page")) || 1000;
+				urlObject.searchParams.delete("per-page");
+				// Start on page 0, to increment per subsequent request
+				let page = 0;
+				// Loop until a break condition is hit
+				while (true) {
+					const urlPaginated =
+						urlObject.href + `&per-page=${perPage}&page=${page}`;
+					const fetched = await fetchWebmentions(
+						options,
+						webmentions,
+						urlPaginated,
+					);
+
+					// Page has no Webmentions or an error occurred → break
+					if (fetched.found === 0) {
+						break;
+					}
+
+					webmentions = fetched.webmentions;
+
+					// If there are less Webmentions found than should be in each
+					// page → break
+					if (fetched.found < perPage) {
+						break;
+					}
+
+					// Increment page
+					page += 1;
+					// Throttle next request
+					await new Promise((resolve) => setTimeout(resolve, 1000));
+				}
+			} else {
+				const fetched = await fetchWebmentions(options, webmentions, url);
+				webmentions = fetched.webmentions;
+			}
+
+			// Process the blocklist, if it has any entries
+			if (options.blocklist.length) {
+				webmentions = processBlocklist(webmentions, options.blocklist);
+			}
+
+			// Process the allowlist, if it has any entries
+			if (options.allowlist.length) {
+				webmentions = processAllowlist(webmentions, options.allowlist);
+			}
+
+			await asset.save(webmentions, "json");
+
+			const performance = process.hrtime(performanceStart);
+
+			// Add a console message with the number of fetched and processed Webmentions, if any
+			if (webmentionsCachedLength < webmentions.length) {
+				console.log(
+					`${styleText(
+						"gray",
+						`[${hostname(options.domain)}]`,
+					)} ${styleText(
+						"bold",
+						String(webmentions.length - webmentionsCachedLength),
+					)} new Webmentions fetched into cache in ${styleText(
+						"bold",
+						(performance[0] + performance[1] / 1e9).toFixed(3) + " seconds",
+					)}.`,
+				);
+			}
+		}
+
+		return webmentions;
+	};
+
+	/**
+	 * @param {Options} options
+	 * @returns {Promise<{[key: string]: Array<Webmention>}>}
+	 */
+	const webmentionsByURL = async (options) => {
+		let rawWebmentions = await retrieveWebmentions(options);
+
+		// Fix local URLs based on urlReplacements and sort Webmentions into groups
+		// by target base URL
+		const webmentions = {};
+		rawWebmentions.forEach((webmention) => {
+			const target = getTarget(webmention);
+			if (!target) {
+				return;
+			}
+
+			let url = baseURL(
+				fixURL(target.replace(/\/?$/, "/"), options.urlReplacements),
+			);
+
+			if (!webmentions[url]) {
+				webmentions[url] = [];
+			}
+
+			webmentions[url].push(webmention);
+		});
+
+		return webmentions;
+	};
+	const webmentionsByUrl = webmentionsByURL;
+	const filteredWebmentions = webmentionsByURL;
+
+	/**
+	 * @param {Options} options
+	 * @param {string} url
+	 * @param {WebmentionType|Array<WebmentionType>} [types]
+	 * @returns {Promise<Array<Webmention>>}
+	 */
+	const getWebmentions = async (options, url, types = []) => {
+		const webmentions = await webmentionsByURL(options);
+		url = absoluteURL(url, options.domain);
+
+		if (!url || !webmentions || !webmentions[url]) {
+			return [];
+		}
+
+		return webmentions[url]
+			// Filter webmentions by allowed response post types
+			.filter((entry) => {
+				return typeof types === "object" && Object.keys(types).length
+					? types.includes(getType(entry))
+					: typeof types === "string"
+						? types === getType(entry)
+						: true;
+			})
+			// Sanitize content of webmentions against HTML limit
+			.map((entry) => {
+				const html = getContent(entry);
+
+				if (html.length) {
+					entry.contentSanitized = sanitizeHTML(html, options.allowedHTML);
+					if (html.length > options.maximumHtmlLength) {
+						entry.contentSanitized = `${
+							options.maximumHtmlText
+						} <a href="${getSource(entry)}">${getSource(entry)}</a>`;
+					}
+				}
+
+				return entry;
+			})
+			// Sort by published
+			.sort((a, b) => {
+				return epoch(getPublished(a)) - epoch(getPublished(b));
+			});
+	};
+
+	/**
+	 * @param {object} eleventyConfig
+	 * @param {Options} [options]
+	 */
+	const eleventyCacheWebmentions = async (eleventyConfig, options = {}) => {
+		options = Object.assign({}, defaults, options);
+
+		const byURL = await webmentionsByURL(options);
+		const all = Object.values(byURL).reduce(
+			(array, webmentions) => [...array, ...webmentions],
+			[],
+		);
+
+		// Global Data
+		eleventyConfig.addGlobalData("webmentionsDefaults", defaults);
+		eleventyConfig.addGlobalData("webmentionsOptions", options);
+		eleventyConfig.addGlobalData("webmentionsByURL", byURL);
+		eleventyConfig.addGlobalData("webmentionsByUrl", byURL);
+		eleventyConfig.addGlobalData("webmentionsAll", all);
+
+		// Universal Filters
+		eleventyConfig.addFilter("getWebmentionsByType", getByTypes);
+		eleventyConfig.addFilter("getWebmentionsByTypes", getByTypes);
+		eleventyConfig.addFilter("getWebmentionPublished", getPublished);
+		eleventyConfig.addFilter("getWebmentionReceived", getReceived);
+		eleventyConfig.addFilter("getWebmentionContent", getContent);
+		eleventyConfig.addFilter("getWebmentionSource", getSource);
+		eleventyConfig.addFilter("getWebmentionURL", getURL);
+		eleventyConfig.addFilter("getWebmentionTarget", getTarget);
+		eleventyConfig.addFilter("getWebmentionType", getType);
+	};
+
+	return {
+		defaults,
+		getPublished,
+		getWebmentionPublished,
+		getReceived,
+		getWebmentionReceived,
+		getContent,
+		getWebmentionContent,
+		getSource,
+		getWebmentionSource,
+		getURL,
+		getWebmentionURL,
+		getTarget,
+		getWebmentionTarget,
+		getUniqueKey,
+		getWebmentionUniqueKey,
+		getType,
+		getWebmentionType,
+		getByTypes,
+		getByType,
+		getWebmentionsByTypes,
+		getWebmentionsByType,
+		processBlocklist,
+		processWebmentionBlocklist,
+		processWebmentionsBlocklist,
+		processAllowlist,
+		processWebmentionAllowlist,
+		processWebmentionsAllowlist,
+		fetchWebmentions,
+		retrieveWebmentions,
+		webmentionsByURL,
+		webmentionsByUrl,
+		filteredWebmentions,
+		getWebmentions,
+		eleventyCacheWebmentions,
+	};
+};