@chrisburnell/eleventy-cache-webmentions 2.0.6 → 2.1.0-beta.2

package/eleventy-cache-webmentions.js

@@ -83,6 +83,59 @@ const defaults = {
 	maximumHtmlText: "mentioned this in",
 }
 
+const performFetch = async (options, webmentions, url) => {
+	return await fetch(url)
+		.then(async (response) => {
+			if (response.ok) {
+				const feed = await response.json()
+				if (feed[options.key].length) {
+					// Combine newly-fetched Webmentions with cached Webmentions
+					webmentions = feed[options.key].concat(webmentions)
+					// Remove duplicates by source URL
+					webmentions = uniqBy([...feed[options.key], ...webmentions], (webmention) => {
+						return getSource(webmention)
+					})
+					// Process the blocklist, if it has any entries
+					if (options.blocklist.length) {
+						webmentions = webmentions.filter((webmention) => {
+							let sourceUrl = getSource(webmention)
+							for (let url of options.blocklist) {
+								if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
+									return false
+								}
+							}
+							return true
+						})
+					}
+					// Process the allowlist, if it has any entries
+					if (options.allowlist.length) {
+						webmentions = webmentions.filter((webmention) => {
+							let sourceUrl = getSource(webmention)
+							for (let url of options.allowlist) {
+								if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
+									return true
+								}
+							}
+							return false
+						})
+					}
+					// Sort webmentions by received date for getting most recent Webmention on subsequent requests
+					webmentions = webmentions.sort((a, b) => {
+						return epoch(getReceived(b)) - epoch(getReceived(a))
+					})
+				}
+				return {
+					found: feed[options.key].length,
+					filtered: webmentions,
+				}
+			}
+			return Promise.reject(response)
+		})
+		.catch((error) => {
+			console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error)
+		})
+}
+
 const fetchWebmentions = async (options) => {
 	if (!options.domain) {
 		throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.")
@@ -117,58 +170,51 @@ const fetchWebmentions = async (options) => {
 		const since = webmentions.length ? getReceived(webmentions[0]) : false
 		// Build the URL for the fetch request
 		const url = `${options.feed}${since ? `${options.feed.includes("?") ? "&" : "?"}since=${since}` : ""}`
-		await fetch(url)
-			.then(async (response) => {
-				if (response.ok) {
-					const feed = await response.json()
-					if (feed[options.key].length) {
-						// Combine newly-fetched Webmentions with cached Webmentions
-						webmentions = feed[options.key].concat(webmentions)
-						// Remove duplicates by source URL
-						webmentions = uniqBy([...feed[options.key], ...webmentions], (webmention) => {
-							return getSource(webmention)
-						})
-						// Process the blocklist, if it has any entries
-						if (options.blocklist.length) {
-							webmentions = webmentions.filter((webmention) => {
-								let sourceUrl = getSource(webmention)
-								for (let url of options.blocklist) {
-									if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
-										return false
-									}
-								}
-								return true
-							})
-						}
-						// Process the allowlist, if it has any entries
-						if (options.allowlist.length) {
-							webmentions = webmentions.filter((webmention) => {
-								let sourceUrl = getSource(webmention)
-								for (let url of options.allowlist) {
-									if (sourceUrl.includes(url.replace(/\/?$/, "/"))) {
-										return true
-									}
-								}
-								return false
-							})
-						}
-						// Sort webmentions by received date for getting most recent Webmention on subsequent requests
-						webmentions = webmentions.sort((a, b) => {
-							return epoch(getReceived(b)) - epoch(getReceived(a))
-						})
-						// Add a console message with the number of fetched and processed Webmentions, if any
-						if (webmentionsCachedLength < webmentions.length) {
-							console.log(`[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`)
-						}
-					}
-					await asset.save(webmentions, "json")
-					return webmentions
+
+		// If using webmention.io, loop through pages until no results found
+		if (url.includes("https://webmention.io")) {
+			// Start on page 0, to increment per subsequent request
+			let page = 0
+			// Loop until a break condition is hit
+			while (true) {
+				const perPage = Number(new URL(url).searchParams.get("per-page")) || 1000
+				const urlPaginated = url + `&page=${page}`
+				const fetched = await performFetch(options, webmentions, urlPaginated)
+
+				// An error occurred during fetching paged results → break
+				if (!fetched && !fetched.found && !fetched.filtered) {
+					break
 				}
-				return Promise.reject(response)
-			})
-			.catch((error) => {
-				console.log(`[${hostname(options.domain)}] Something went wrong with your request to ${hostname(options.feed)}!`, error)
-			})
+
+				// Page has no Webmentions → break
+				if (fetched.found === 0) {
+					break
+				}
+
+				webmentions = fetched.filtered
+
+				// If there are less Webmentions found than should be in each
+				// page → break
+				if (fetched.found < perPage) {
+					break
+				}
+
+				// Increment page
+				page += 1
+				// Throttle next request
+				await new Promise(resolve => setTimeout(resolve, 1000))
+			}
+		} else {
+			const fetched = await performFetch(options, webmentions, url)
+			webmentions = fetched.filtered
+		}
+
+		await asset.save(webmentions, "json")
+
+		// Add a console message with the number of fetched and processed Webmentions, if any
+		if (webmentionsCachedLength < webmentions.length) {
+			console.log(`[${hostname(options.domain)}] ${webmentions.length - webmentionsCachedLength} new Webmentions fetched into cache.`)
+		}
 	}
 
 	return webmentions

package/package.json

@@ -1,16 +1,32 @@
 {
   "name": "@chrisburnell/eleventy-cache-webmentions",
-  "version": "2.0.6",
+  "version": "2.1.0-beta.2",
   "description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.",
-  "author": "Chris Burnell <me@chrisburnell.com>",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git@github.com:chrisburnell/eleventy-cache-webmentions.git"
   },
+  "homepage": "https://chrisburnell.com/eleventy-cache-webmentions/",
   "bugs": {
     "url": "https://github.com/chrisburnell/eleventy-cache-webmentions/issues"
   },
+  "author": "Chris Burnell <me@chrisburnell.com>",
+  "contributors": [
+    {
+      "name": "Chris Burnell",
+      "email": "me@chrisburnell.com",
+      "url": "https://chrisburnell.com",
+      "mastodon": "@chrisburnell@repc.co"
+    }
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "eleventy-cache-webmentions.js",
+  "scripts": {
+    "lint": "eslint eleventy-cache-webmentions.js"
+  },
   "keywords": [
     "eleventy",
     "eleventy-plugin",
@@ -19,17 +35,13 @@
     "js",
     "webmention"
   ],
-  "devDependencies": {
-    "eslint": "^9.2.0"
-  },
   "dependencies": {
     "@11ty/eleventy-fetch": "^4.0.1",
     "lodash": "^4.17.21",
     "node-fetch": "2.6.7",
     "sanitize-html": "^2.13.0"
   },
-  "main": "eleventy-cache-webmentions.js",
-  "scripts": {
-    "lint": "eslint eleventy-cache-webmentions.js"
+  "devDependencies": {
+    "eslint": "^9.2.0"
   }
 }