@chrisburnell/eleventy-cache-webmentions 0.1.12 → 1.0.0-beta.1

package/eleventy-cache-webmentions.js

@@ -2,11 +2,7 @@ const fetch = require("node-fetch")
 const sanitizeHTML = require("sanitize-html")
 const uniqBy = require("lodash/uniqBy")
 const { AssetCache } = require("@11ty/eleventy-fetch")
-
-// Load .env variables with dotenv
-require("dotenv").config()
-
-const TOKEN = process.env.WEBMENTION_IO_TOKEN
+const { defaultsDeep } = require("lodash")
 
 const absoluteURL = (url, domain) => {
 	try {
@@ -42,153 +38,177 @@ const epoch = (value) => {
 	return new Date(value).getTime()
 }
 
-module.exports = (eleventyConfig, options = {}) => {
-	options = Object.assign(
-		{
-			key: "webmentions",
-			allowedHTML: {
-				allowedTags: ["b", "i", "em", "strong", "a"],
-				allowedAttributes: {
-					a: ["href"],
-				},
-			},
-			urlReplacements: {},
-			maximumHtmlLength: 2000,
-			maximumHtmlText: "mentioned this in",
+const getPublished = (webmention) => {
+	return webmention["wm-received"] || webmention?.["data"]["published"] || webmention["verified_date"] || webmention["published"]
+}
+
+const getContent = (webmention) => {
+	return webmention?.["content"]["html"] || webmention["content"] || webmention?.["data"]["content"]
+}
+
+const getSource = (webmention) => {
+	return webmention["url"] || webmention?.["data"]["url"] || webmention["wm-source"] || webmention["source"]
+}
+
+const getTarget = (webmention) => {
+	return webmention["wm-target"] || webmention["target"]
+}
+
+const getType = (webmention) => {
+	return webmention["wm-property"] || webmention?.["activity"]["type"] || webmention["type"]
+}
+
+const defaults = {
+	uniqueKey: "webmentions",
+	allowedHTML: {
+		allowedTags: ["b", "i", "em", "strong", "a"],
+		allowedAttributes: {
+			a: ["href"],
 		},
-		options
-	)
+	},
+	urlReplacements: {},
+	maximumHtmlLength: 2000,
+	maximumHtmlText: "mentioned this in",
+}
 
-	const fetchWebmentions = async () => {
-		let asset = new AssetCache(options.uniqueKey || options.key)
-		asset.ensureDir()
+const fetchWebmentions = async (options) => {
+	let asset = new AssetCache(options.uniqueKey)
+	asset.ensureDir()
 
-		let webmentions = {
-			type: "feed",
-			name: "Webmentions",
-			children: [],
-		}
+	let webmentions = []
 
-		// If there is a cached file at all, grab its contents now
-		if (asset.isCacheValid("9001y")) {
-			webmentions = await asset.getCachedValue()
-		}
+	// If there is a cached file at all, grab its contents now
+	if (asset.isCacheValid("9001y")) {
+		webmentions = await asset.getCachedValue()
+	}
 
-		// If there is a cached file but it is outside of expiry, fetch fresh
-		// results since the most recent
-		if (!asset.isCacheValid(options.duration)) {
-			const since = webmentions.children.length ? webmentions.children[0]["wm-received"] : false
-			const url = `https://webmention.io/api/mentions.jf2?domain=${hostname(options.domain)}&token=${TOKEN}&per-page=9001${since ? `&since=${since}` : ``}`
-			await fetch(url)
-				.then(async (response) => {
-					if (response.ok) {
-						const feed = await response.json()
-						if (feed.children.length) {
-							console.log(`[${hostname(options.domain)}] ${feed.children.length} new Webmentions fetched into cache.`)
-						}
-						webmentions.children = [...feed.children, ...webmentions.children]
-						await asset.save(webmentions, "json")
-						return webmentions
+	// If there is a cached file but it is outside of expiry, fetch fresh
+	// results since the most recent webmention
+	if (!asset.isCacheValid(options.duration)) {
+		const since = webmentions.length ? getPublished(webmentions[0]) : false
+		const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}`
+		await fetch(url)
+			.then(async (response) => {
+				if (response.ok) {
+					const feed = await response.json()
+					if (feed[options.key].length) {
+						webmentions = uniqBy([...feed[options.key], ...webmentions], (entry) => {
+							return getSource(entry)
+						})
+						console.log(`[${hostname(options.domain)}] ${feed[options.key].length} new Webmentions fetched into cache.`)
 					}
-					return Promise.reject(response)
-				})
-				.catch((error) => {
-					console.log(`[${hostname(options.domain)}] Something went wrong with your request to webmention.io!`, error)
-				})
+					await asset.save(webmentions, "json")
+					return webmentions
+				}
+				return Promise.reject(response)
+			})
+			.catch((error) => {
+				console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error)
+			})
+	}
+
+	return webmentions
+}
+
+const filteredWebmentions = async (options) => {
+	const rawWebmentions = await fetchWebmentions(options)
+	let webmentions = {}
+
+	// Sort Webmentions into groups by target
+	rawWebmentions.forEach((webmention) => {
+		// Get the target of the Webmention and fix it up
+		let url = baseUrl(fixUrl(getTarget(webmention).replace(/\/?$/, "/"), options.urlReplacements))
+
+		if (!webmentions[url]) {
+			webmentions[url] = []
 		}
 
-		return webmentions
+		webmentions[url].push(webmention)
+	})
+
+	// Sort Webmentions in groups by url and remove duplicates by `url`
+	for (let url in webmentions) {
+		webmentions[url] = uniqBy(webmentions[url], (entry) => {
+			return getSource(entry)
+		})
 	}
 
-	const filteredWebmentions = async () => {
-		const rawWebmentions = await fetchWebmentions()
-		let webmentions = {}
+	return webmentions
+}
 
-		// Sort Webmentions into groups by `wm-target`
-		rawWebmentions.children.forEach((webmention) => {
-			// Get the target of the Webmention and fix it up
-			let url = baseUrl(fixUrl(webmention["wm-target"].replace(/\/?$/, "/"), options.urlReplacements))
+const getWebmentions = async (options, url, allowedTypes = {}) => {
+	const webmentions = await filteredWebmentions(options)
+	url = absoluteURL(url, options.domain)
 
-			if (!webmentions[url]) {
-				webmentions[url] = []
-			}
+	if (!url || !webmentions || !webmentions[url]) {
+		return []
+	}
 
-			webmentions[url].push(webmention)
+	const results = webmentions[url]
+		// filter webmentions by allowed response post types
+		.filter((entry) => {
+			return typeof allowedTypes === "object" && Object.keys(allowedTypes).length ? allowedTypes.includes(getType(entry)) : true
+		})
+		// sanitize content of webmentions against HTML limit
+		.map((entry) => {
+			if (!("content" in entry) || !("data" in entry)) {
+				return entry
+			}
+			const html = getContent(entry)
+			if (html && html.length > options.maximumHtmlLength) {
+				entry.content = `${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`
+			} else if (Object.keys(options.allowedHTML).length) {
+				entry.content = sanitizeHTML(html, options.allowedHTML)
+			} else {
+				entry.content = html
+			}
+			return entry
+		})
+		// sort by published
+		.sort((a, b) => {
+			return epoch(getPublished(a)) - epoch(getPublished(b))
 		})
 
-		// Sort Webmentions in groups by url and remove duplicates by `url`
-		for (let url in webmentions) {
-			webmentions[url] = uniqBy(webmentions[url], (item) => {
-				return item["url"]
-			})
-		}
+	return results
+}
 
-		return webmentions
+const getWebmentionsFilter = async (options, url, allowedTypes, callback) => {
+	if (typeof callback !== "function") {
+		callback = allowedTypes
+		allowedTypes = {}
 	}
+	const webmentions = await getWebmentions(options, url, allowedTypes)
+	callback(null, webmentions)
+}
 
-	const getWebmentions = async (url, allowedTypes = {}) => {
-		const webmentions = await filteredWebmentions()
-		url = absoluteURL(url, options.domain)
-
-		if (!url || !webmentions || !webmentions[url]) {
-			return []
-		}
+module.exports = (eleventyConfig, options = {}) => {
+	options = Object.assign(defaults, options)
 
-		const results = webmentions[url]
-			// filter webmentions by allowed response post types
-			.filter((entry) => {
-				return typeof allowedTypes === "object" && Object.keys(allowedTypes).length ? allowedTypes.includes(entry["wm-property"]) : true
-			})
-			// remove webmentions without an author name
-			.filter((entry) => {
-				const { author } = entry
-				return !!author && !!author.name
-			})
-			// sanitize content of webmentions and check against HTML limit
-			.map((entry) => {
-				if (!("content" in entry)) {
-					return entry
-				}
-				const { html, text } = entry.content
-				if (html && html.length > options.maximumHtmlLength) {
-					entry.content.value = `${options.maximumHtmlText} <a href="${entry["wm-source"]}">${entry["wm-source"]}</a>`
-				} else if (Object.keys(options.allowedHTML).length) {
-					entry.content.value = sanitizeHTML(html || text, options.allowedHTML)
-				} else {
-					entry.content.value = html || text
-				}
-				return entry
-			})
-			// sort by published/wm-received
-			.sort((a, b) => {
-				return epoch(a.published || a["wm-received"]) - epoch(b.published || b["wm-received"])
-			})
+	if (!options.domain) {
+		throw new Error("domain is a required field when adding the plugin to your eleventyConfig using addPlugin. See https://chrisburnell.com/eleventy-cache-webmentions/#installation for more information.")
+	}
 
-		return results
+	if (!options.feed) {
+		throw new Error("feed is a required field when adding the plugin to your eleventyConfig using addPlugin. See https://chrisburnell.com/eleventy-cache-webmentions/#installation for more information.")
 	}
 
-	const getWebmentionsFilter = async (url, allowedTypes, callback) => {
-		if (typeof callback !== "function") {
-			callback = allowedTypes
-			allowedTypes = {}
-		}
-		const webmentions = await getWebmentions(url, allowedTypes)
-		callback(null, webmentions)
+	if (!options.key) {
+		throw new Error("key is a required field when adding the plugin to your eleventyConfig using addPlugin. See https://chrisburnell.com/eleventy-cache-webmentions/#installation for more information.")
 	}
 
-	if (eleventyConfig && options) {
-		if (!options.domain) {
-			throw new Error("domain is a required option to be passed when adding the plugin to your eleventyConfig using addPlugin.")
-		}
+	if (eleventyConfig) {
+		// Global Data
+		eleventyConfig.addGlobalData("webmentions", filteredWebmentions(options))
 
-		// Liquid/Nunjucks Filters
+		// Liquid Filter
 		eleventyConfig.addLiquidFilter("getWebmentions", getWebmentionsFilter)
+
+		// Nunjucks Filter
 		eleventyConfig.addNunjucksAsyncFilter("getWebmentions", getWebmentionsFilter)
+	}
 
-		// Eleventy Data
-		eleventyConfig.addGlobalData("webmentions", filteredWebmentions)
-	} else {
-		// JavaScript
-		return filteredWebmentions
+	return {
+		webmentions: filteredWebmentions(options),
+		getWebmentions: getWebmentions,
 	}
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@chrisburnell/eleventy-cache-webmentions",
-  "version": "0.1.12",
-  "description": "Cache webmentions from webmention.io using eleventy-fetch.",
+  "version": "1.0.0-beta.1",
+  "description": "Fetch and cache webmentions using eleventy-fetch.",
   "main": "eleventy-cache-webmentions.js",
   "author": "Chris Burnell <me@chrisburnell.com>",
   "license": "MIT",
@@ -17,7 +17,6 @@
   },
   "dependencies": {
     "@11ty/eleventy-fetch": "^3.0.0",
-    "dotenv": "^10.0.0",
     "lodash": "^4.17.21",
     "node-fetch": "2.6.5",
     "sanitize-html": "^2.5.2"