@chrisburnell/eleventy-cache-webmentions 2.1.2 → 2.1.3-beta.2

package/README.md

@@ -205,9 +205,6 @@ webmentions.forEach((webmention) => {
 <summary>Liquid / Nunjucks</summary>
 
 ```twig
-{# get Array of Webmentions for a given URL #}
-{% set webmentions = ("https://example.com" + page.url) | getWebmentions %}
-
 {# filter Webmentions by their response type #}
 {{ set responses = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) }}
 
@@ -227,7 +224,7 @@ webmentions.forEach((webmention) => {
 
 </details>
 
-## Attach Webmentions to Pages using Directory Data
+### Attach Webmentions to Pages using Directory Data
 
 Using [Eleventy’s Data Cascade](https://www.11ty.dev/docs/data-cascade/), you can attach Webmentions to each page by using [Directory Specific Data Files](https://www.11ty.dev/docs/data-template-dir/).
 
@@ -281,18 +278,7 @@ module.exports = (eleventyConfig) => {
 }
 ```
 
-## Without Directory Data
-
-If you would rather get Webmentions for a given page directly from a Layout/Include/Page itself, you can do so using the Filter, `getWebmentions`:
-
-```twig
-{% set webmentions = ("https://example.com" + page.url) | getWebmentions %}
-{% for webmention in webmentions %}
-    ...
-{% endfor %}
-```
-
-## Get specific types of Webmentions
+### Get specific types of Webmentions
 
 Instead of getting all the Webmentions for a given page, you may want to grab only certain types of Webmentions. This is useful if you want to display different types of Webmentions separately, e.g.:
 
@@ -304,7 +290,7 @@ Instead of getting all the Webmentions for a given page, you may want to grab on
 {% set replies = webmentions | getWebmentionsByTypes(['mention-of', 'in-reply-to']) %}
 ```
 
-## Get all Webmentions at once
+### Get all Webmentions at once
 
 If you need it, the plugin also makes available an Object containing your cached Webmentions organised in key:value pairs, where each key is a full URL on your website and its value is an Array of Webmentions sent to that URL:
 

package/eleventy-cache-webmentions.js

@@ -1,71 +1,108 @@
-const fetch = require("node-fetch")
-const sanitizeHTML = require("sanitize-html")
-const uniqBy = require("lodash/uniqBy")
-const { AssetCache } = require("@11ty/eleventy-fetch")
+const fetch = require("node-fetch");
+const sanitizeHTML = require("sanitize-html");
+const uniqBy = require("lodash/uniqBy");
+const { AssetCache } = require("@11ty/eleventy-fetch");
 
 const absoluteURL = (url, domain) => {
 	try {
-		return new URL(url, domain).toString()
+		return new URL(url, domain).toString();
 	} catch (e) {
-		console.log(`Trying to convert ${url} to be an absolute url with base ${domain} and failed.`)
-		return url
+		console.log(
+			`Trying to convert ${url} to be an absolute url with base ${domain} and failed.`,
+		);
+		return url;
 	}
-}
+};
 
 const baseUrl = (url) => {
-	let hashSplit = url.split("#")
-	let queryparamSplit = hashSplit[0].split("?")
-	return queryparamSplit[0]
-}
+	let hashSplit = url.split("#");
+	let queryparamSplit = hashSplit[0].split("?");
+	return queryparamSplit[0];
+};
 
 const fixUrl = (url, urlReplacements) => {
-	return Object.entries(urlReplacements).reduce((accumulator, [key, value]) => {
-		const regex = new RegExp(key, "g")
-		return accumulator.replace(regex, value)
-	}, url)
-}
+	return Object.entries(urlReplacements).reduce(
+		(accumulator, [key, value]) => {
+			const regex = new RegExp(key, "g");
+			return accumulator.replace(regex, value);
+		},
+		url,
+	);
+};
 
 const hostname = (value) => {
 	if (typeof value === "string" && value.includes("//")) {
-		const urlObject = new URL(value)
-		return urlObject.hostname
+		const urlObject = new URL(value);
+		return urlObject.hostname;
 	}
-	return value
-}
+	return value;
+};
 
 const epoch = (value) => {
-	return new Date(value).getTime()
-}
+	return new Date(value).getTime();
+};
 
 const getPublished = (webmention) => {
-	return webmention?.["data"]?.["published"] || webmention["published"] || webmention["wm-received"] || webmention["verified_date"]
-}
+	return (
+		webmention?.["data"]?.["published"] ||
+		webmention["published"] ||
+		webmention["wm-received"] ||
+		webmention["verified_date"]
+	);
+};
 
 const getReceived = (webmention) => {
-	return webmention["wm-received"] || webmention["verified_date"] || webmention?.["data"]?.["published"] || webmention["published"]
-}
+	return (
+		webmention["wm-received"] ||
+		webmention["verified_date"] ||
+		webmention?.["data"]?.["published"] ||
+		webmention["published"]
+	);
+};
 
 const getContent = (webmention) => {
-	return webmention?.["contentSanitized"] || webmention?.["content"]?.["html"] || webmention?.["content"]?.["value"] || webmention?.["content"] || webmention?.["data"]?.["content"] || ""
-}
+	return (
+		webmention?.["contentSanitized"] ||
+		webmention?.["content"]?.["html"] ||
+		webmention?.["content"]?.["value"] ||
+		webmention?.["content"] ||
+		webmention?.["data"]?.["content"] ||
+		""
+	);
+};
 
 const getSource = (webmention) => {
-	return webmention?.["data"]?.["url"] || webmention["url"] || webmention["wm-source"] || webmention["source"]
-}
+	return (
+		webmention?.["data"]?.["url"] ||
+		webmention["url"] ||
+		webmention["wm-source"] ||
+		webmention["source"]
+	);
+};
 
 const getTarget = (webmention) => {
-	return webmention["wm-target"] || webmention["target"]
-}
+	return webmention["wm-target"] || webmention["target"];
+};
 
 const getType = (webmention) => {
-	return webmention["wm-property"] || webmention?.["activity"]?.["type"] || webmention["type"]
-}
+	return (
+		webmention["wm-property"] ||
+		webmention?.["activity"]?.["type"] ||
+		webmention["type"]
+	);
+};
+
+const getByType = (webmentions, allowedType) => {
+	return webmentions.filter((webmention) => {
+		return allowedTypes === getType(webmention);
+	});
+};
 
 const getByTypes = (webmentions, allowedTypes) => {
 	return webmentions.filter((webmention) => {
-		return allowedTypes.includes(getType(webmention))
-	})
-}
+		return allowedTypes.includes(getType(webmention));
+	});
+};
 
 const defaults = {
 	duration: "1d",
@@ -81,259 +118,315 @@ const defaults = {
 	urlReplacements: {},
 	maximumHtmlLength: 1000,
 	maximumHtmlText: "mentioned this in",
-}
+};
 
 const performFetch = async (options, webmentions, url) => {
 	return await fetch(url)
 		.then(async (response) => {
-			if (response.ok) {
-				const feed = await response.json()
-				if (feed[options.key].length) {
-					// Combine newly-fetched Webmentions with cached Webmentions
-					webmentions = feed[options.key].concat(webmentions)
-					// Remove duplicates by source URL
-					webmentions = uniqBy([...feed[options.key], ...webmentions], (webmention) => {
-						return getSource(webmention)
-					})
-					// Process the blocklist, if it has any entries
-					if (options.blocklist.length) {
-						webmentions = webmentions.filter((webmention) => {
-							let sourceUrl = getSource(webmention)
-							for (let url of options.blocklist) {
-								if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
-									return false
-								}
-							}
-							return true
-						})
+			if (!response.ok) {
+				return Promise.reject(response);
+			}
+
+			const feed = await response.json();
+
+			if (!options.key in feed) {
+				console.log(
+					`[${hostname(options.domain)}] ${
+						options.key
+					} was not found as a key in the response from ${hostname(
+						options.feed,
+					)}!`,
+				);
+				return Promise.reject(response);
+			}
+
+			// Combine newly-fetched Webmentions with cached Webmentions
+			webmentions = feed[options.key].concat(webmentions);
+			// Remove duplicates by source URL
+			webmentions = uniqBy(
+				[...feed[options.key], ...webmentions],
+				(webmention) => {
+					return getSource(webmention);
+				},
+			);
+			// Process the blocklist, if it has any entries
+			if (options.blocklist.length) {
+				webmentions = webmentions.filter((webmention) => {
+					let sourceUrl = getSource(webmention);
+					for (let url of options.blocklist) {
+						if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
+							return false;
+						}
 					}
-					// Process the allowlist, if it has any entries
-					if (options.allowlist.length) {
-						webmentions = webmentions.filter((webmention) => {
-							let sourceUrl = getSource(webmention)
-							for (let url of options.allowlist) {
-								if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
-									return true
-								}
-							}
-							return false
-						})
+					return true;
+				});
+			}
+			// Process the allowlist, if it has any entries
+			if (options.allowlist.length) {
+				webmentions = webmentions.filter((webmention) => {
+					let sourceUrl = getSource(webmention);
+					for (let url of options.allowlist) {
+						if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
+							return true;
+						}
 					}
-					// Sort webmentions by received date for getting most recent Webmention on subsequent requests
-					webmentions = webmentions.sort((a, b) => {
-						return epoch(getReceived(b)) - epoch(getReceived(a))
-					})
-				}
-				return {
-					found: feed[options.key].length,
-					filtered: webmentions,
-				}
+					return false;
+				});
 			}
-			return Promise.reject(response)
+			// Sort webmentions by received date for getting most recent Webmention on subsequent requests
+			webmentions = webmentions.sort((a, b) => {
+				return epoch(getReceived(b)) - epoch(getReceived(a));
+			});
+
+			return {
+				found: feed[options.key].length,
+				filtered: webmentions,
+			};
 		})
 		.catch((error) => {
-			console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error)
-		})
-}
+			console.log(
+				`[${hostname(
+					options.domain,
+				)}] Something went wrong with your request to ${hostname(
+					options.feed,
+				)}!`,
+				error,
+			);
+		});
+};
 
 const fetchWebmentions = async (options) => {
 	if (!options.domain) {
-		throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
+		throw new Error(
+			"`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.",
+		);
 	}
 
 	if (!options.feed) {
-		throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
+		throw new Error(
+			"`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.",
+		);
 	}
 
 	if (!options.key) {
-		throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
+		throw new Error(
+			"`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.",
+		);
 	}
 
-	let asset = new AssetCache(options.uniqueKey, options.directory)
-	asset.ensureDir()
+	let asset = new AssetCache(options.uniqueKey, options.directory);
+	asset.ensureDir();
 
-	let webmentions = []
+	let webmentions = [];
 
 	// If there is a cached file at all, grab its contents now
 	if (asset.isCacheValid("9001y")) {
-		webmentions = await asset.getCachedValue()
+		webmentions = await asset.getCachedValue();
 	}
 
 	// Get the number of cached Webmentions for diffing against fetched
 	// Webmentions later
-	const webmentionsCachedLength = webmentions.length
+	const webmentionsCachedLength = webmentions.length;
 
 	// If there is a cached file but it is outside of expiry, fetch fresh
 	// results since the most recent Webmention
-	if (!asset.isCacheValid(options.duration)) {
+	if (!asset.isCacheValid("1s")) {
 		// Get the received date of the most recent Webmention, if it exists
-		const since = webmentions.length ? getReceived(webmentions[0]) : false
+		const since = webmentions.length ? getReceived(webmentions[0]) : false;
 		// Build the URL for the fetch request
-		const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}`
+		const url = `${options.feed}${
+			since
+				? `${options.feed.includes("?") ? "&" : "?"}since=${since}`
+				: ""
+		}`;
 
 		// If using webmention.io, loop through pages until no results found
 		if (url.includes("https://webmention.io")) {
 			// Start on page 0, to increment per subsequent request
-			let page = 0
+			let page = 0;
 			// Loop until a break condition is hit
 			while (true) {
-				const perPage = Number(new URL(url).searchParams.get("per-page")) || 1000
-				const urlPaginated = url + `&page=${page}`
-				const fetched = await performFetch(options, webmentions, urlPaginated)
+				const perPage =
+					Number(new URL(url).searchParams.get("per-page")) || 1000;
+				const urlPaginated = url + `&per-page=${perPage}&page=${page}`;
+				const fetched = await performFetch(
+					options,
+					webmentions,
+					urlPaginated,
+				);
 
 				// An error occurred during fetching paged results → break
 				if (!fetched && !fetched.found && !fetched.filtered) {
-					break
+					break;
 				}
 
 				// Page has no Webmentions → break
 				if (fetched.found === 0) {
-					break
+					break;
 				}
 
-				webmentions = fetched.filtered
+				webmentions = fetched.filtered;
 
 				// If there are less Webmentions found than should be in each
 				// page → break
 				if (fetched.found < perPage) {
-					break
+					break;
 				}
 
 				// Increment page
-				page += 1
+				page += 1;
 				// Throttle next request
-				await new Promise(resolve => setTimeout(resolve, 1000))
+				await new Promise((resolve) => setTimeout(resolve, 1000));
 			}
 		} else {
-			const fetched = await performFetch(options, webmentions, url)
-			webmentions = fetched.filtered
+			const fetched = await performFetch(options, webmentions, url);
+			webmentions = fetched.filtered;
 		}
 
-		await asset.save(webmentions, "json")
+		await asset.save(webmentions, "json");
 
 		// Add a console message with the number of fetched and processed Webmentions, if any
 		if (webmentionsCachedLength < webmentions.length) {
-			console.log(`[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`)
+			console.log(
+				`[${hostname(options.domain)}] ${
+					webmentions.length - webmentionsCachedLength
+				} new Webmentions fetched into cache.`,
+			);
 		}
 	}
 
-	return webmentions
-}
+	return webmentions;
+};
 
-let filtered = {}
+let filtered = {};
 const filteredWebmentions = async (options) => {
 	if (Object.entries(filtered).length) {
-		return filtered
+		return filtered;
 	}
 
-	let rawWebmentions = await fetchWebmentions(options)
+	let rawWebmentions = await fetchWebmentions(options);
 
 	// Fix local URLs based on urlReplacements and sort Webmentions into groups
 	// by target base URL
 	rawWebmentions.forEach((webmention) => {
-		let url = baseUrl(fixUrl(getTarget(webmention).replace(/\/?$/, "/"), options.urlReplacements))
+		let url = baseUrl(
+			fixUrl(
+				getTarget(webmention).replace(/\/?$/, "/"),
+				options.urlReplacements,
+			),
+		);
 
 		if (!filtered[url]) {
-			filtered[url] = []
+			filtered[url] = [];
 		}
 
-		filtered[url].push(webmention)
-	})
+		filtered[url].push(webmention);
+	});
 
-	return filtered
-}
+	return filtered;
+};
 
 const getWebmentions = async (options, url, allowedTypes = {}) => {
-	const webmentions = await filteredWebmentions(options)
-	url = absoluteURL(url, options.domain)
+	const webmentions = await filteredWebmentions(options);
+	url = absoluteURL(url, options.domain);
 
 	if (!url || !webmentions || !webmentions[url]) {
-		return []
+		return [];
 	}
 
 	return (
 		webmentions[url]
 			// Filter webmentions by allowed response post types
 			.filter((entry) => {
-				return typeof allowedTypes === "object" && Object.keys(allowedTypes).length ? allowedTypes.includes(getType(entry)) : true
+				return typeof allowedTypes === "object" &&
+					Object.keys(allowedTypes).length
+					? allowedTypes.includes(getType(entry))
+					: true;
 			})
 			// Sanitize content of webmentions against HTML limit
 			.map((entry) => {
-				const html = getContent(entry)
+				const html = getContent(entry);
 
 				if (html.length) {
-					entry.contentSanitized = sanitizeHTML(html, options.allowedHTML)
+					entry.contentSanitized = sanitizeHTML(
+						html,
+						options.allowedHTML,
+					);
 					if (html.length > options.maximumHtmlLength) {
-						entry.contentSanitized = `${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`
+						entry.contentSanitized = `${
+							options.maximumHtmlText
+						} <a href="${getSource(entry)}">${getSource(
+							entry,
+						)}</a>`;
 					}
 				}
 
-				return entry
+				return entry;
 			})
 			// Sort by published
 			.sort((a, b) => {
-				return epoch(getPublished(a)) - epoch(getPublished(b))
+				return epoch(getPublished(a)) - epoch(getPublished(b));
 			})
-	)
-}
-
-const getWebmentionsFilter = async (options, url, allowedTypes, callback) => {
-	if (typeof callback !== "function") {
-		callback = allowedTypes
-		allowedTypes = {}
-	}
-	const webmentions = await getWebmentions(options, url, allowedTypes)
-	callback(null, webmentions)
-}
+	);
+};
 
 const eleventyCacheWebmentions = (eleventyConfig, options = {}) => {
-	options = Object.assign(defaults, options)
+	options = Object.assign(defaults, options);
 
 	// Global Data
-	eleventyConfig.addGlobalData("webmentionsDefaults", defaults)
-	const filtered = async () => await filteredWebmentions(options)
-	eleventyConfig.addGlobalData("webmentionsByUrl", filtered)
-	const unfiltered = async () => await filteredWebmentions(options).then((filtered) => Object.values(filtered).reduce((array, webmentions) => [...array, ...webmentions], []))
-	eleventyConfig.addGlobalData("webmentionsAll", unfiltered)
+	eleventyConfig.addGlobalData("webmentionsDefaults", defaults);
+	eleventyConfig.addGlobalData("webmentionsOptions", options);
+	const filtered = async () => await filteredWebmentions(options);
+	eleventyConfig.addGlobalData("webmentionsByUrl", filtered);
+	const unfiltered = async () =>
+		await filteredWebmentions(options).then((filtered) =>
+			Object.values(filtered).reduce(
+				(array, webmentions) => [...array, ...webmentions],
+				[],
+			),
+		);
+	eleventyConfig.addGlobalData("webmentionsAll", unfiltered);
 
 	// Liquid Filters
-	eleventyConfig.addLiquidFilter("getWebmentions", getWebmentionsFilter)
-	eleventyConfig.addLiquidFilter("getWebmentionsByTypes", getByTypes)
-	eleventyConfig.addLiquidFilter("getWebmentionPublished", getPublished)
-	eleventyConfig.addLiquidFilter("getWebmentionReceived", getReceived)
-	eleventyConfig.addLiquidFilter("getWebmentionContent", getContent)
-	eleventyConfig.addLiquidFilter("getWebmentionSource", getSource)
-	eleventyConfig.addLiquidFilter("getWebmentionTarget", getTarget)
-	eleventyConfig.addLiquidFilter("getWebmentionType", getType)
+	eleventyConfig.addLiquidFilter("getWebmentionsByType", getByType);
+	eleventyConfig.addLiquidFilter("getWebmentionsByTypes", getByTypes);
+	eleventyConfig.addLiquidFilter("getWebmentionPublished", getPublished);
+	eleventyConfig.addLiquidFilter("getWebmentionReceived", getReceived);
+	eleventyConfig.addLiquidFilter("getWebmentionContent", getContent);
+	eleventyConfig.addLiquidFilter("getWebmentionSource", getSource);
+	eleventyConfig.addLiquidFilter("getWebmentionTarget", getTarget);
+	eleventyConfig.addLiquidFilter("getWebmentionType", getType);
 
 	// Nunjucks Filters
-	eleventyConfig.addNunjucksAsyncFilter("getWebmentions", getWebmentionsFilter)
-	eleventyConfig.addNunjucksFilter("getWebmentionsByTypes", getByTypes)
-	eleventyConfig.addNunjucksFilter("getWebmentionPublished", getPublished)
-	eleventyConfig.addNunjucksFilter("getWebmentionReceived", getReceived)
-	eleventyConfig.addNunjucksFilter("getWebmentionContent", getContent)
-	eleventyConfig.addNunjucksFilter("getWebmentionSource", getSource)
-	eleventyConfig.addNunjucksFilter("getWebmentionTarget", getTarget)
-	eleventyConfig.addNunjucksFilter("getWebmentionType", getType)
-}
-
-module.exports = eleventyCacheWebmentions
-module.exports.defaults = defaults
-module.exports.filteredWebmentions = filteredWebmentions
-module.exports.webmentionsByUrl = filteredWebmentions
-module.exports.fetchWebmentions = fetchWebmentions
-module.exports.getWebmentions = getWebmentions
-module.exports.getByTypes = getByTypes
-module.exports.getWebmentionsByTypes = getByTypes
-module.exports.getPublished = getPublished
-module.exports.getWebmentionPublished = getPublished
-module.exports.getReceived = getReceived
-module.exports.getWebmentionReceived = getReceived
-module.exports.getContent = getContent
-module.exports.getWebmentionContent = getContent
-module.exports.getSource = getSource
-module.exports.getWebmentionSource = getSource
-module.exports.getTarget = getTarget
-module.exports.getWebmentionTarget = getTarget
-module.exports.getType = getType
-module.exports.getWebmentionType = getType
+	eleventyConfig.addNunjucksFilter("getWebmentionsByType", getByType);
+	eleventyConfig.addNunjucksFilter("getWebmentionsByTypes", getByTypes);
+	eleventyConfig.addNunjucksFilter("getWebmentionPublished", getPublished);
+	eleventyConfig.addNunjucksFilter("getWebmentionReceived", getReceived);
+	eleventyConfig.addNunjucksFilter("getWebmentionContent", getContent);
+	eleventyConfig.addNunjucksFilter("getWebmentionSource", getSource);
+	eleventyConfig.addNunjucksFilter("getWebmentionTarget", getTarget);
+	eleventyConfig.addNunjucksFilter("getWebmentionType", getType);
+};
+
+module.exports = eleventyCacheWebmentions;
+module.exports.defaults = defaults;
+module.exports.filteredWebmentions = filteredWebmentions;
+module.exports.webmentionsByUrl = filteredWebmentions;
+module.exports.fetchWebmentions = fetchWebmentions;
+module.exports.getWebmentions = getWebmentions;
+module.exports.getByType = getByType;
+module.exports.getWebmentionsByType = getByType;
+module.exports.getByTypes = getByTypes;
+module.exports.getWebmentionsByTypes = getByTypes;
+module.exports.getPublished = getPublished;
+module.exports.getWebmentionPublished = getPublished;
+module.exports.getReceived = getReceived;
+module.exports.getWebmentionReceived = getReceived;
+module.exports.getContent = getContent;
+module.exports.getWebmentionContent = getContent;
+module.exports.getSource = getSource;
+module.exports.getWebmentionSource = getSource;
+module.exports.getTarget = getTarget;
+module.exports.getWebmentionTarget = getTarget;
+module.exports.getType = getType;
+module.exports.getWebmentionType = getType;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chrisburnell/eleventy-cache-webmentions",
-  "version": "2.1.2",
+  "version": "2.1.3-beta.2",
   "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.",
   "license": "MIT",
   "repository": {