@chrisburnell/eleventy-cache-webmentions 2.2.7 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"cachedAt":
|
|
1
|
+
{"cachedAt":1772443890877,"type":"json","metadata":{}}
|
|
@@ -327,7 +327,7 @@ const fetchWebmentions = async (options, webmentions, url) => {
|
|
|
327
327
|
|
|
328
328
|
if (!(options.key in feed)) {
|
|
329
329
|
console.log(
|
|
330
|
-
`${styleText("
|
|
330
|
+
`${styleText("gray", `[${hostname(options.domain)}]`)} ${
|
|
331
331
|
options.key
|
|
332
332
|
} was not found as a key in the response from ${styleText(
|
|
333
333
|
"bold",
|
|
@@ -337,9 +337,17 @@ const fetchWebmentions = async (options, webmentions, url) => {
|
|
|
337
337
|
return Promise.reject(response);
|
|
338
338
|
}
|
|
339
339
|
|
|
340
|
-
//
|
|
341
|
-
|
|
342
|
-
|
|
340
|
+
// Fetched Webmentions replace cached ones with the same source URL
|
|
341
|
+
const fetchedSources = new Set(
|
|
342
|
+
feed[options.key].map((wm) => getSource(wm)),
|
|
343
|
+
);
|
|
344
|
+
webmentions = [
|
|
345
|
+
...feed[options.key],
|
|
346
|
+
...webmentions.filter(
|
|
347
|
+
(wm) => !fetchedSources.has(getSource(wm)),
|
|
348
|
+
),
|
|
349
|
+
];
|
|
350
|
+
// Remove any remaining duplicates by source URL
|
|
343
351
|
webmentions = removeDuplicates(webmentions);
|
|
344
352
|
// Process the blocklist, if it has any entries
|
|
345
353
|
if (options.blocklist.length) {
|
|
@@ -362,7 +370,7 @@ const fetchWebmentions = async (options, webmentions, url) => {
|
|
|
362
370
|
.catch((error) => {
|
|
363
371
|
console.warn(
|
|
364
372
|
`${styleText(
|
|
365
|
-
"
|
|
373
|
+
"gray",
|
|
366
374
|
`[${hostname(options.domain)}]`,
|
|
367
375
|
)} Something went wrong with your Webmention request to ${styleText(
|
|
368
376
|
"bold",
|
|
@@ -495,7 +503,7 @@ const retrieveWebmentions = async (options) => {
|
|
|
495
503
|
if (webmentionsCachedLength < webmentions.length) {
|
|
496
504
|
console.log(
|
|
497
505
|
`${styleText(
|
|
498
|
-
"
|
|
506
|
+
"gray",
|
|
499
507
|
`[${hostname(options.domain)}]`,
|
|
500
508
|
)} ${styleText(
|
|
501
509
|
"bold",
|
|
@@ -567,8 +575,8 @@ const getWebmentions = async (options, url, types = []) => {
|
|
|
567
575
|
return typeof types === "object" && Object.keys(types).length
|
|
568
576
|
? types.includes(getType(entry))
|
|
569
577
|
: typeof types === "string"
|
|
570
|
-
|
|
571
|
-
|
|
578
|
+
? types === getType(entry)
|
|
579
|
+
: true;
|
|
572
580
|
})
|
|
573
581
|
// Sanitize content of webmentions against HTML limit
|
|
574
582
|
.map((entry) => {
|
|
@@ -341,7 +341,7 @@ export const fetchWebmentions = async (options, webmentions, url) => {
|
|
|
341
341
|
|
|
342
342
|
if (!(options.key in feed)) {
|
|
343
343
|
console.log(
|
|
344
|
-
`${styleText("
|
|
344
|
+
`${styleText("gray", `[${hostname(options.domain)}]`)} ${
|
|
345
345
|
options.key
|
|
346
346
|
} was not found as a key in the response from ${styleText(
|
|
347
347
|
"bold",
|
|
@@ -351,9 +351,17 @@ export const fetchWebmentions = async (options, webmentions, url) => {
|
|
|
351
351
|
return Promise.reject(response);
|
|
352
352
|
}
|
|
353
353
|
|
|
354
|
-
//
|
|
355
|
-
|
|
356
|
-
|
|
354
|
+
// Fetched Webmentions replace cached ones with the same source URL
|
|
355
|
+
const fetchedSources = new Set(
|
|
356
|
+
feed[options.key].map((wm) => getSource(wm)),
|
|
357
|
+
);
|
|
358
|
+
webmentions = [
|
|
359
|
+
...feed[options.key],
|
|
360
|
+
...webmentions.filter(
|
|
361
|
+
(wm) => !fetchedSources.has(getSource(wm)),
|
|
362
|
+
),
|
|
363
|
+
];
|
|
364
|
+
// Remove any remaining duplicates by source URL
|
|
357
365
|
webmentions = removeDuplicates(webmentions);
|
|
358
366
|
// Process the blocklist, if it has any entries
|
|
359
367
|
if (options.blocklist.length) {
|
|
@@ -376,7 +384,7 @@ export const fetchWebmentions = async (options, webmentions, url) => {
|
|
|
376
384
|
.catch((error) => {
|
|
377
385
|
console.warn(
|
|
378
386
|
`${styleText(
|
|
379
|
-
"
|
|
387
|
+
"gray",
|
|
380
388
|
`[${hostname(options.domain)}]`,
|
|
381
389
|
)} Something went wrong with your Webmention request to ${styleText(
|
|
382
390
|
"bold",
|
|
@@ -509,7 +517,7 @@ export const retrieveWebmentions = async (options) => {
|
|
|
509
517
|
if (webmentionsCachedLength < webmentions.length) {
|
|
510
518
|
console.log(
|
|
511
519
|
`${styleText(
|
|
512
|
-
"
|
|
520
|
+
"gray",
|
|
513
521
|
`[${hostname(options.domain)}]`,
|
|
514
522
|
)} ${styleText(
|
|
515
523
|
"bold",
|
|
@@ -583,8 +591,8 @@ export const getWebmentions = async (options, url, types = []) => {
|
|
|
583
591
|
return typeof types === "object" && Object.keys(types).length
|
|
584
592
|
? types.includes(getType(entry))
|
|
585
593
|
: typeof types === "string"
|
|
586
|
-
|
|
587
|
-
|
|
594
|
+
? types === getType(entry)
|
|
595
|
+
: true;
|
|
588
596
|
})
|
|
589
597
|
// Sanitize content of webmentions against HTML limit
|
|
590
598
|
.map((entry) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
const{AssetCache}=require("@11ty/eleventy-fetch"),{styleText}=require("node:util"),sanitizeHTML=require("sanitize-html"),defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"},absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getSource(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()],getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getTarget=webmention=>webmention["wm-target"]||webmention.target,getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();return options.key in feed?(webmentions=feed[options.key].concat(webmentions),webmentions=removeDuplicates(webmentions),options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),webmentions=webmentions.sort((a,b)=>epoch(getReceived(b))-epoch(getReceived(a))),{found:feed[options.key].length,webmentions}):(console.log(`${styleText("grey",`[${hostname(options.domain)}]`)} ${options.key} was not found as a key in the response from ${styleText("bold",hostname(options.feed))}!`),Promise.reject(response))}).catch(error=>(console.warn(`${styleText("grey",`[${hostname(options.domain)}]`)} Something went wrong with your Webmention request to ${styleText("bold",hostname(options.feed))}!`),console.warn(error instanceof Error?error.message:error),{found:0,webmentions})),retrieveWebmentions=async options=>{if(!options.domain)throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.feed)throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.key)throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");let asset=new AssetCache(options.uniqueKey||`webmentions-${hostname(options.domain)}`,options.cacheDirectory),webmentions=[];asset.isCacheValid("9001y")&&!options.refresh&&(webmentions=await asset.getCachedValue());const webmentionsCachedLength=webmentions.length;if(!asset.isCacheValid(options.refresh?"0s":options.duration)){const performanceStart=process.hrtime(),since=webmentions.length?getReceived(webmentions[0]):!1,url=`${options.feed}${since?`${options.feed.includes("?")?"&":"?"}since=${since}`:""}`;if(url.includes("https://webmention.io")){const urlObject=new URL(url),perPage=Number(urlObject.searchParams.get("per-page"))||1e3;urlObject.searchParams.delete("per-page");let page=0;for(;;){const urlPaginated=urlObject.href+`&per-page=${perPage}&page=${page}`,fetched=await fetchWebmentions(options,webmentions,urlPaginated);if(!fetched&&!fetched.found&&!fetched.webmentions||fetched.found===0||(webmentions=fetched.webmentions,fetched.found<perPage))break;page+=1,await new Promise(resolve=>setTimeout(resolve,1e3))}}else webmentions=(await fetchWebmentions(options,webmentions,url)).webmentions;options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),await asset.save(webmentions,"json");const performance=process.hrtime(performanceStart);webmentionsCachedLength<webmentions.length&&console.log(`${styleText("grey",`[${hostname(options.domain)}]`)} ${styleText("bold",String(webmentions.length-webmentionsCachedLength))} new Webmentions fetched into cache in ${styleText("bold",(performance[0]+performance[1]/1e9).toFixed(3)+" seconds")}.`)}return webmentions},WEBMENTIONS={},webmentionsByURL=async options=>(Object.keys(WEBMENTIONS).length||(await retrieveWebmentions(options)).forEach(webmention=>{let url=baseURL(fixURL(getTarget(webmention).replace(/\/?$/,"/"),options.urlReplacements));WEBMENTIONS[url]||(WEBMENTIONS[url]=[]),WEBMENTIONS[url].push(webmention)}),WEBMENTIONS),getWebmentions=async(options,url,types=[])=>{const webmentions=await webmentionsByURL(options);return url=absoluteURL(url,options.domain),!url||!webmentions||!webmentions[url]?[]:webmentions[url].filter(entry=>typeof types=="object"&&Object.keys(types).length?types.includes(getType(entry)):typeof types=="string"?types===getType(entry):!0).map(entry=>{const html=getContent(entry);return html.length&&(entry.contentSanitized=sanitizeHTML(html,options.allowedHTML),html.length>options.maximumHtmlLength&&(entry.contentSanitized=`${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`)),entry}).sort((a,b)=>epoch(getPublished(a))-epoch(getPublished(b)))},eleventyCacheWebmentions=async(eleventyConfig,options={})=>{options=Object.assign(defaults,options);const byURL=await webmentionsByURL(options),all=Object.values(byURL).reduce((array,webmentions)=>[...array,...webmentions],[]);eleventyConfig.addGlobalData("webmentionsDefaults",defaults),eleventyConfig.addGlobalData("webmentionsOptions",options),eleventyConfig.addGlobalData("webmentionsByURL",byURL),eleventyConfig.addGlobalData("webmentionsByUrl",byURL),eleventyConfig.addGlobalData("webmentionsAll",all),eleventyConfig.addLiquidFilter("getWebmentionsByType",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionPublished",getPublished),eleventyConfig.addLiquidFilter("getWebmentionReceived",getReceived),eleventyConfig.addLiquidFilter("getWebmentionContent",getContent),eleventyConfig.addLiquidFilter("getWebmentionSource",getSource),eleventyConfig.addLiquidFilter("getWebmentionURL",getURL),eleventyConfig.addLiquidFilter("getWebmentionTarget",getTarget),eleventyConfig.addLiquidFilter("getWebmentionType",getType),eleventyConfig.addNunjucksFilter("getWebmentionsByType",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionPublished",getPublished),eleventyConfig.addNunjucksFilter("getWebmentionReceived",getReceived),eleventyConfig.addNunjucksFilter("getWebmentionContent",getContent),eleventyConfig.addNunjucksFilter("getWebmentionSource",getSource),eleventyConfig.addNunjucksFilter("getWebmentionURL",getURL),eleventyConfig.addNunjucksFilter("getWebmentionTarget",getTarget),eleventyConfig.addNunjucksFilter("getWebmentionType",getType)};module.exports=eleventyCacheWebmentions,module.exports.defaults=defaults,module.exports.getPublished=getPublished,module.exports.getWebmentionPublished=getPublished,module.exports.getReceived=getReceived,module.exports.getWebmentionReceived=getReceived,module.exports.getContent=getContent,module.exports.getWebmentionContent=getContent,module.exports.getSource=getSource,module.exports.getWebmentionSource=getSource,module.exports.getURL=getURL,module.exports.getWebmentionURL=getURL,module.exports.getTarget=getTarget,module.exports.getWebmentionTarget=getTarget,module.exports.getType=getType,module.exports.getWebmentionType=getType,module.exports.getByTypes=getByTypes,module.exports.getByType=getByTypes,module.exports.getWebmentionsByTypes=getByTypes,module.exports.getWebmentionsByType=getByTypes,module.exports.processBlocklist=processBlocklist,module.exports.processWebmentionBlocklist=processBlocklist,module.exports.processWebmentionsBlocklist=processBlocklist,module.exports.processAllowlist=processAllowlist,module.exports.processWebmentionAllowlist=processAllowlist,module.exports.processWebmentionsAllowlist=processAllowlist,module.exports.fetchWebmentions=fetchWebmentions,module.exports.retrieveWebmentions=retrieveWebmentions,module.exports.webmentionsByURL=webmentionsByURL,module.exports.webmentionsByUrl=webmentionsByURL,module.exports.filteredWebmentions=webmentionsByURL,module.exports.getWebmentions=getWebmentions;
|
|
1
|
+
const{AssetCache}=require("@11ty/eleventy-fetch"),{styleText}=require("node:util"),sanitizeHTML=require("sanitize-html"),defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"},absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getSource(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()],getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getTarget=webmention=>webmention["wm-target"]||webmention.target,getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();if(!(options.key in feed))return console.log(`${styleText("gray",`[${hostname(options.domain)}]`)} ${options.key} was not found as a key in the response from ${styleText("bold",hostname(options.feed))}!`),Promise.reject(response);const fetchedSources=new Set(feed[options.key].map(wm=>getSource(wm)));return webmentions=[...feed[options.key],...webmentions.filter(wm=>!fetchedSources.has(getSource(wm)))],webmentions=removeDuplicates(webmentions),options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),webmentions=webmentions.sort((a,b)=>epoch(getReceived(b))-epoch(getReceived(a))),{found:feed[options.key].length,webmentions}}).catch(error=>(console.warn(`${styleText("gray",`[${hostname(options.domain)}]`)} Something went wrong with your Webmention request to ${styleText("bold",hostname(options.feed))}!`),console.warn(error instanceof Error?error.message:error),{found:0,webmentions})),retrieveWebmentions=async options=>{if(!options.domain)throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.feed)throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.key)throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");let asset=new AssetCache(options.uniqueKey||`webmentions-${hostname(options.domain)}`,options.cacheDirectory),webmentions=[];asset.isCacheValid("9001y")&&!options.refresh&&(webmentions=await asset.getCachedValue());const webmentionsCachedLength=webmentions.length;if(!asset.isCacheValid(options.refresh?"0s":options.duration)){const performanceStart=process.hrtime(),since=webmentions.length?getReceived(webmentions[0]):!1,url=`${options.feed}${since?`${options.feed.includes("?")?"&":"?"}since=${since}`:""}`;if(url.includes("https://webmention.io")){const urlObject=new URL(url),perPage=Number(urlObject.searchParams.get("per-page"))||1e3;urlObject.searchParams.delete("per-page");let page=0;for(;;){const urlPaginated=urlObject.href+`&per-page=${perPage}&page=${page}`,fetched=await fetchWebmentions(options,webmentions,urlPaginated);if(!fetched&&!fetched.found&&!fetched.webmentions||fetched.found===0||(webmentions=fetched.webmentions,fetched.found<perPage))break;page+=1,await new Promise(resolve=>setTimeout(resolve,1e3))}}else webmentions=(await fetchWebmentions(options,webmentions,url)).webmentions;options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),await asset.save(webmentions,"json");const performance=process.hrtime(performanceStart);webmentionsCachedLength<webmentions.length&&console.log(`${styleText("gray",`[${hostname(options.domain)}]`)} ${styleText("bold",String(webmentions.length-webmentionsCachedLength))} new Webmentions fetched into cache in ${styleText("bold",(performance[0]+performance[1]/1e9).toFixed(3)+" seconds")}.`)}return webmentions},WEBMENTIONS={},webmentionsByURL=async options=>(Object.keys(WEBMENTIONS).length||(await retrieveWebmentions(options)).forEach(webmention=>{let url=baseURL(fixURL(getTarget(webmention).replace(/\/?$/,"/"),options.urlReplacements));WEBMENTIONS[url]||(WEBMENTIONS[url]=[]),WEBMENTIONS[url].push(webmention)}),WEBMENTIONS),getWebmentions=async(options,url,types=[])=>{const webmentions=await webmentionsByURL(options);return url=absoluteURL(url,options.domain),!url||!webmentions||!webmentions[url]?[]:webmentions[url].filter(entry=>typeof types=="object"&&Object.keys(types).length?types.includes(getType(entry)):typeof types=="string"?types===getType(entry):!0).map(entry=>{const html=getContent(entry);return html.length&&(entry.contentSanitized=sanitizeHTML(html,options.allowedHTML),html.length>options.maximumHtmlLength&&(entry.contentSanitized=`${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`)),entry}).sort((a,b)=>epoch(getPublished(a))-epoch(getPublished(b)))},eleventyCacheWebmentions=async(eleventyConfig,options={})=>{options=Object.assign(defaults,options);const byURL=await webmentionsByURL(options),all=Object.values(byURL).reduce((array,webmentions)=>[...array,...webmentions],[]);eleventyConfig.addGlobalData("webmentionsDefaults",defaults),eleventyConfig.addGlobalData("webmentionsOptions",options),eleventyConfig.addGlobalData("webmentionsByURL",byURL),eleventyConfig.addGlobalData("webmentionsByUrl",byURL),eleventyConfig.addGlobalData("webmentionsAll",all),eleventyConfig.addLiquidFilter("getWebmentionsByType",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionPublished",getPublished),eleventyConfig.addLiquidFilter("getWebmentionReceived",getReceived),eleventyConfig.addLiquidFilter("getWebmentionContent",getContent),eleventyConfig.addLiquidFilter("getWebmentionSource",getSource),eleventyConfig.addLiquidFilter("getWebmentionURL",getURL),eleventyConfig.addLiquidFilter("getWebmentionTarget",getTarget),eleventyConfig.addLiquidFilter("getWebmentionType",getType),eleventyConfig.addNunjucksFilter("getWebmentionsByType",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionPublished",getPublished),eleventyConfig.addNunjucksFilter("getWebmentionReceived",getReceived),eleventyConfig.addNunjucksFilter("getWebmentionContent",getContent),eleventyConfig.addNunjucksFilter("getWebmentionSource",getSource),eleventyConfig.addNunjucksFilter("getWebmentionURL",getURL),eleventyConfig.addNunjucksFilter("getWebmentionTarget",getTarget),eleventyConfig.addNunjucksFilter("getWebmentionType",getType)};module.exports=eleventyCacheWebmentions,module.exports.defaults=defaults,module.exports.getPublished=getPublished,module.exports.getWebmentionPublished=getPublished,module.exports.getReceived=getReceived,module.exports.getWebmentionReceived=getReceived,module.exports.getContent=getContent,module.exports.getWebmentionContent=getContent,module.exports.getSource=getSource,module.exports.getWebmentionSource=getSource,module.exports.getURL=getURL,module.exports.getWebmentionURL=getURL,module.exports.getTarget=getTarget,module.exports.getWebmentionTarget=getTarget,module.exports.getType=getType,module.exports.getWebmentionType=getType,module.exports.getByTypes=getByTypes,module.exports.getByType=getByTypes,module.exports.getWebmentionsByTypes=getByTypes,module.exports.getWebmentionsByType=getByTypes,module.exports.processBlocklist=processBlocklist,module.exports.processWebmentionBlocklist=processBlocklist,module.exports.processWebmentionsBlocklist=processBlocklist,module.exports.processAllowlist=processAllowlist,module.exports.processWebmentionAllowlist=processAllowlist,module.exports.processWebmentionsAllowlist=processAllowlist,module.exports.fetchWebmentions=fetchWebmentions,module.exports.retrieveWebmentions=retrieveWebmentions,module.exports.webmentionsByURL=webmentionsByURL,module.exports.webmentionsByUrl=webmentionsByURL,module.exports.filteredWebmentions=webmentionsByURL,module.exports.getWebmentions=getWebmentions;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{AssetCache}from"@11ty/eleventy-fetch";import{styleText}from"node:util";import sanitizeHTML from"sanitize-html";export const defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"};const absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getSource(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()];export const getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getWebmentionPublished=getPublished,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getWebmentionReceived=getReceived,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getWebmentionContent=getContent,getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getWebmentionSource=getSource,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getWebmentionURL=getURL,getTarget=webmention=>webmention["wm-target"]||webmention.target,getWebmentionTarget=getTarget,getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getWebmentionType=getType,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),getByType=getByTypes,getWebmentionsByTypes=getByTypes,getWebmentionsByType=getByTypes,processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processWebmentionBlocklist=processBlocklist,processWebmentionsBlocklist=processBlocklist,processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),processWebmentionAllowlist=processAllowlist,processWebmentionsAllowlist=processAllowlist,fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();return options.key in feed
|
|
1
|
+
import{AssetCache}from"@11ty/eleventy-fetch";import{styleText}from"node:util";import sanitizeHTML from"sanitize-html";export const defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"};const absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getSource(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()];export const getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getWebmentionPublished=getPublished,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getWebmentionReceived=getReceived,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getWebmentionContent=getContent,getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getWebmentionSource=getSource,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getWebmentionURL=getURL,getTarget=webmention=>webmention["wm-target"]||webmention.target,getWebmentionTarget=getTarget,getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getWebmentionType=getType,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),getByType=getByTypes,getWebmentionsByTypes=getByTypes,getWebmentionsByType=getByTypes,processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processWebmentionBlocklist=processBlocklist,processWebmentionsBlocklist=processBlocklist,processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),processWebmentionAllowlist=processAllowlist,processWebmentionsAllowlist=processAllowlist,fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();if(!(options.key in feed))return console.log(`${styleText("gray",`[${hostname(options.domain)}]`)} ${options.key} was not found as a key in the response from ${styleText("bold",hostname(options.feed))}!`),Promise.reject(response);const fetchedSources=new Set(feed[options.key].map(wm=>getSource(wm)));return webmentions=[...feed[options.key],...webmentions.filter(wm=>!fetchedSources.has(getSource(wm)))],webmentions=removeDuplicates(webmentions),options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),webmentions=webmentions.sort((a,b)=>epoch(getReceived(b))-epoch(getReceived(a))),{found:feed[options.key].length,webmentions}}).catch(error=>(console.warn(`${styleText("gray",`[${hostname(options.domain)}]`)} Something went wrong with your Webmention request to ${styleText("bold",hostname(options.feed))}!`),console.warn(error instanceof Error?error.message:error),{found:0,webmentions})),retrieveWebmentions=async options=>{if(!options.domain)throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.feed)throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.key)throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");let asset=new AssetCache(options.uniqueKey||`webmentions-${hostname(options.domain)}`,options.cacheDirectory),webmentions=[];asset.isCacheValid("9001y")&&!options.refresh&&(webmentions=await asset.getCachedValue());const webmentionsCachedLength=webmentions.length;if(!asset.isCacheValid(options.refresh?"0s":options.duration)){const performanceStart=process.hrtime(),since=webmentions.length?getReceived(webmentions[0]):!1,url=`${options.feed}${since?`${options.feed.includes("?")?"&":"?"}since=${since}`:""}`;if(url.includes("https://webmention.io")){const urlObject=new URL(url),perPage=Number(urlObject.searchParams.get("per-page"))||1e3;urlObject.searchParams.delete("per-page");let page=0;for(;;){const urlPaginated=urlObject.href+`&per-page=${perPage}&page=${page}`,fetched=await fetchWebmentions(options,webmentions,urlPaginated);if(!fetched&&!fetched.found&&!fetched.webmentions||fetched.found===0||(webmentions=fetched.webmentions,fetched.found<perPage))break;page+=1,await new Promise(resolve=>setTimeout(resolve,1e3))}}else webmentions=(await fetchWebmentions(options,webmentions,url)).webmentions;options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),await asset.save(webmentions,"json");const performance=process.hrtime(performanceStart);webmentionsCachedLength<webmentions.length&&console.log(`${styleText("gray",`[${hostname(options.domain)}]`)} ${styleText("bold",String(webmentions.length-webmentionsCachedLength))} new Webmentions fetched into cache in ${styleText("bold",(performance[0]+performance[1]/1e9).toFixed(3)+" seconds")}.`)}return webmentions};const WEBMENTIONS={};export const webmentionsByURL=async options=>(Object.keys(WEBMENTIONS).length||(await retrieveWebmentions(options)).forEach(webmention=>{let url=baseURL(fixURL(getTarget(webmention).replace(/\/?$/,"/"),options.urlReplacements));WEBMENTIONS[url]||(WEBMENTIONS[url]=[]),WEBMENTIONS[url].push(webmention)}),WEBMENTIONS),webmentionsByUrl=webmentionsByURL,filteredWebmentions=webmentionsByURL,getWebmentions=async(options,url,types=[])=>{const webmentions=await webmentionsByURL(options);return url=absoluteURL(url,options.domain),!url||!webmentions||!webmentions[url]?[]:webmentions[url].filter(entry=>typeof types=="object"&&Object.keys(types).length?types.includes(getType(entry)):typeof types=="string"?types===getType(entry):!0).map(entry=>{const html=getContent(entry);return html.length&&(entry.contentSanitized=sanitizeHTML(html,options.allowedHTML),html.length>options.maximumHtmlLength&&(entry.contentSanitized=`${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`)),entry}).sort((a,b)=>epoch(getPublished(a))-epoch(getPublished(b)))},eleventyCacheWebmentions=async(eleventyConfig,options={})=>{options=Object.assign(defaults,options);const byURL=await webmentionsByURL(options),all=Object.values(byURL).reduce((array,webmentions)=>[...array,...webmentions],[]);eleventyConfig.addGlobalData("webmentionsDefaults",defaults),eleventyConfig.addGlobalData("webmentionsOptions",options),eleventyConfig.addGlobalData("webmentionsByURL",byURL),eleventyConfig.addGlobalData("webmentionsByUrl",byURL),eleventyConfig.addGlobalData("webmentionsAll",all),eleventyConfig.addLiquidFilter("getWebmentionsByType",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionPublished",getPublished),eleventyConfig.addLiquidFilter("getWebmentionReceived",getReceived),eleventyConfig.addLiquidFilter("getWebmentionContent",getContent),eleventyConfig.addLiquidFilter("getWebmentionSource",getSource),eleventyConfig.addLiquidFilter("getWebmentionURL",getURL),eleventyConfig.addLiquidFilter("getWebmentionTarget",getTarget),eleventyConfig.addLiquidFilter("getWebmentionType",getType),eleventyConfig.addNunjucksFilter("getWebmentionsByType",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionPublished",getPublished),eleventyConfig.addNunjucksFilter("getWebmentionReceived",getReceived),eleventyConfig.addNunjucksFilter("getWebmentionContent",getContent),eleventyConfig.addNunjucksFilter("getWebmentionSource",getSource),eleventyConfig.addNunjucksFilter("getWebmentionURL",getURL),eleventyConfig.addNunjucksFilter("getWebmentionTarget",getTarget),eleventyConfig.addNunjucksFilter("getWebmentionType",getType)};export default eleventyCacheWebmentions;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chrisburnell/eleventy-cache-webmentions",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.1",
|
|
4
4
|
"description": "Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|