@smart-cloud/publisher-exporter 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +4 -4
- package/dist/deploy.js +4 -4
- package/dist/invalidate.js +3 -3
- package/dist/queue-runner.js +2 -3
- package/package.json +10 -9
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import S from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import bt from'crypto';import Ct from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var gt=".deploy-plan.json",ht="deploy-plan.json";function ft(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Ae(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ie(e){return [...new Set(e.map(Ae).filter(t=>t!==null))]}function mt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ie(t.changedFiles||[]),deletedFiles:ie(t.deletedFiles||[]),rewriteTargets:ie(t.rewriteTargets||[])}}function wt(e){let t=f.join(f.resolve(e.outputDir),gt),r=ft();return r?[f.join(r,ht),t]:[t]}function V(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return Ae(n)}async function ke(e,t){let r=mt(e,t);for(let s of wt(e))await S.mkdir(f.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Te(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function yt(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&f.extname(i)===""&&(i+="/"),i==="/"?"":i}function ae(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function St(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,o={};if(typeof i.targetOrigin=="string"){let a=Te(i.targetOrigin);a&&(o.targetOrigin=a);}let u=ae(i.extraReplacements);if(Object.keys(u).length>0&&(o.extraReplacements=u),i.s3&&typeof i.s3=="object"){let a={},c=i.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},c=i.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(l=>String(l??"").trim()).filter(l=>l.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[n]=o;}return t}function oe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function vt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Me(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,oe(s,r))}async function ue(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await S.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=Te(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=yt(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=ae(r.extraReplacements),r.postCrawlCopyMap=ae(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=St(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=vt(),n=s?f.resolve(s,".."):"";return n?(r.outputDir=Me(n,r.outputDir,"export"),r.logDir=Me(n,r.logDir,"logs")):(f.isAbsolute(r.outputDir)||(r.outputDir=oe(r.outputDir,"export")),f.isAbsolute(r.logDir)||(r.logDir=oe(r.logDir,"logs"))),r}function le(e,t=10){return bt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function ce(e){await S.mkdir(f.dirname(e),{recursive:true});}function $(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function F(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function Q(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function G(e){return e.replace(/\//g,"\\/")}function Le(e){return e.replace(/\//g,"\\\\/")}function K(e,t,r){if(!t)return;e[t]=r;let s=G(t),n=G(r);e[s]=n;let i=Le(t),o=Le(r);e[i]=o;let u=Q(t),a=Q(r);e[u]=a;let c=Q(s),l=Q(n);e[c]=l;let p=Q(i),d=Q(o);e[p]=d;}function de(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function De(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ie(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var kt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Mt=new Set([".html",".htm"]),Oe="WPSuite.io Static Publisher",Tt=Oe.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Lt(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=f.extname(r).toLowerCase();return Mt.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Dt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Tt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Oe}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {chromium}from'playwright';import y from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import {getConfig}from'@smart-cloud/wpsuite-core';import Dt from'crypto';import Et from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var mt=".deploy-plan.json",wt="deploy-plan.json";function Pt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ae(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ie(e){return [...new Set(e.map(Ae).filter(t=>t!==null))]}function St(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ie(t.changedFiles||[]),deletedFiles:ie(t.deletedFiles||[]),rewriteTargets:ie(t.rewriteTargets||[])}}function yt(e){let t=h.join(h.resolve(e.outputDir),mt),s=Pt();return s?[h.join(s,wt),t]:[t]}function V(e,t){let s=h.resolve(e),r=h.resolve(t),n=h.relative(s,r).replace(/\\/g,"/");return Ae(n)}async function Me(e,t){let s=St(e,t);for(let r of yt(e))await y.mkdir(h.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(s,null,2),"utf8");}function Le(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Rt(e){let t=String(e??"").trim();if(!t)return "";let s=t;if(/^https?:\/\//i.test(s))try{s=new URL(s).pathname;}catch{return ""}if(s=s.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!s)return "";let r=s.endsWith("/"),n=s.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return r&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function oe(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,s])=>[t.trim(),String(s??"")]).filter(([t])=>t.length>0))}function Ct(e){if(!e||typeof e!="object")return {};let t={};for(let[s,r]of Object.entries(e)){let n=s.trim();if(!n||!r||typeof r!="object")continue;let i=r,o={};if(typeof i.targetOrigin=="string"){let a=Le(i.targetOrigin);a&&(o.targetOrigin=a);}let l=oe(i.extraReplacements);if(Object.keys(l).length>0&&(o.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},c=i.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},c=i.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(u=>String(u??"").trim()).filter(u=>u.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[n]=o;}return t}function Ie(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function xt(e,t){if(!e||typeof e!="object")return null;let s=e,r=s.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(s.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(s.id??`${r}-${t+1}`).trim();if(!i)return null;let o=String(s.deploymentProfile??"").trim(),l=String(s.url??"").trim();return {id:i,enabled:s.enabled!==false,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:s.crawlMode==="incremental"?"incremental":"full"}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&o?{deploymentProfile:o}:{},...l?{url:l}:{}}}function De(e){let t=e&&typeof e=="object"?e:{},s=Array.isArray(t.rules)?t.rules.map((r,n)=>xt(r,n)).filter(r=>!!r):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:s}}function kt(e){if(!e||typeof e!="object")return;let t=e,s={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(s[n]=i.trim());}let r=Number(t.lastUpdate??0);return Number.isFinite(r)&&r>0&&(s.lastUpdate=Math.floor(r)),s.subscriber=t.subscriber===true,Object.keys(s).length>0?s:void 0}function ue(e){let t=e&&typeof e=="object"?e:{},s=kt(t.siteSettings),r={subscriber:t.subscriber===true||s?.subscriber===true};for(let i of ["accountId","siteId","apiBase","nonce","uploadUrl"]){let o=t[i];typeof o=="string"&&o.trim()!==""&&(r[i]=o.trim());}s&&(r.siteSettings=s);let n=Ie(t.subscriptionType);return n&&(r.subscriptionType=n),r}function At(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Mt(e){let t=ue(e.wpsuite),s={...t.siteSettings??{},accountId:String(t.siteSettings?.accountId??t.accountId??"").trim()||void 0,siteId:String(t.siteSettings?.siteId??t.siteId??"").trim()||void 0,subscriber:t.siteSettings?.subscriber===true||t.subscriber===true},r=String(t.accountId??s.accountId??"").trim(),n=String(t.siteId??s.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t,...r?{accountId:r}:{},...n?{siteId:n}:{},siteSettings:s},!r||!n||!i)return false;At(e.sourceOrigin);let o=globalThis,a={...o.WpSuite??{},siteSettings:s,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),t.nonce&&(a.nonce=t.nonce),o.WpSuite=a,true}async function Tt(e){if(!Mt(e))return null;try{let t=await getConfig("publisher");return t&&typeof t=="object"?t:null}catch{return null}}function Lt(e,t){let s=Ct(t?.deploymentProfiles),r=String(t?.defaultDeploymentProfile??"").trim(),n=s[r]?r:"",i=Ie(t?.subscriptionType),o=ue(e.wpsuite);return {...e,scheduler:De(t?.scheduler),deploymentProfiles:s,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...o,subscriber:o.subscriber===true||!!i,...i?{subscriptionType:i}:{}}}}function ae(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let n=r.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function It(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Te(e,t,s){let r=String(t||"").trim();return r&&h.isAbsolute(r)?h.resolve(r):h.resolve(e,ae(r,s))}async function le(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",s=await y.readFile(t,"utf8"),r=JSON.parse(s);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=Le(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=Rt(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=oe(r.extraReplacements),r.postCrawlCopyMap=oe(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.wpsuite=ue(r.wpsuite),r.scheduler=De(void 0),r.deploymentProfiles={},r.defaultDeploymentProfile="",r.deploymentTargetOverride=String(r.deploymentTargetOverride??"").trim();let n=Lt(r,await Tt(r)),i=It(),o=i?h.resolve(i,".."):"";return o?(n.outputDir=Te(o,n.outputDir,"export"),n.logDir=Te(o,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=ae(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=ae(n.logDir,"logs"))),n}function ce(e,t=10){return Dt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function de(e){await y.mkdir(h.dirname(e),{recursive:true});}function U(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function j(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function Q(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function G(e){return e.replace(/\//g,"\\/")}function Oe(e){return e.replace(/\//g,"\\\\/")}function K(e,t,s){if(!t)return;e[t]=s;let r=G(t),n=G(s);e[r]=n;let i=Oe(t),o=Oe(s);e[i]=o;let l=Q(t),a=Q(s);e[l]=a;let c=Q(r),u=Q(n);e[c]=u;let p=Q(i),d=Q(o);e[p]=d;}function pe(e,t){let s=t.startsWith("/")?t:`/${t}`,r=(e||"").replace(/\/$/,"");return !r||r==="."||r==="/"?s:`${r}${s}`}function Ee(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Fe(e,t,s){if(!t)return s;let r=h.dirname(h.resolve(t)),n=h.resolve(e,s.replace(/^\/+/,"")),i=h.relative(r,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var $t=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Ut=new Set([".html",".htm"]),$e="WPSuite.io Static Publisher",Qt=$e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Nt(e,t,s){if(e.wpsuite?.subscriber===true||!s)return false;let r=h.extname(s).toLowerCase();return Ut.has(r)?/<head\b|<html\b|<!doctype html/i.test(t):false}function qt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Qt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${$e}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let r=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Nt(t,r));}updateCurrentEvent(t,r,s){let n=Wt(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(N[this.level]>=N.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){qt("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),N[this.level]>=N.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),N[this.level]>=N.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Bt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Jt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),zt=new Set(["",".html",".htm"]),Vt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Gt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Kt(e){let t=[],r="full",s="full",n=false,i=false;for(let o=0;o<e.length;o++){let u=e[o];if(u==="--retry-timeouts")r="retry-timeouts",n=true;else if(u==="--resume")n=true;else if(u==="--resume-rewrite")n=true,i=true;else if(u==="--crawl-mode"){let a=e[++o];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(u.startsWith("--crawl-mode="))s=u.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(u==="--incremental")s="incremental",n=true;else if(u==="--url"){let a=e[++o];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(u.startsWith("--url="))r="single-url",n=true,t.push(u.slice(6));else if(u==="--urls"){let a=e[++o];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Se(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Xt(){let e=Se();return e?f.resolve(e,".."):""}function Zt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Yt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Xt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Se(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Zt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function Ye(e){let t=Se();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function et(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function er(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function j(e,t,r){let s=V(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Qe(e,t,r){let s=V(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function Y(e,t){e.rewriteTargets.add(f.resolve(t));}async function tr(e){try{let t=await S.readFile(Ye(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return et()}function qe(e){return JSON.parse(JSON.stringify(e))}async function rr(e,t){let r=Ye(e);await S.mkdir(f.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}function ee(e){if(!e)return null;let t=e.trim();return t||null}function sr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],o=t[n];!i||!o||i===o||r.add(n);}return r}async function nr(e,t,r,s){let n=[...new Set(r.map(u=>u.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let o=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let u=await t.post(o,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!u.ok())return s.warn(`Change token lookup failed with HTTP ${u.status()}`,{endpoint:o,status:u.status()}),new Map;let a=await u.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let c=new Map;for(let l of a.items)!l||typeof l.url!="string"||c.set(l.url,l);return c}catch(u){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:o,error:String(u)}),new Map}}async function fe(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let o=(await nr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,o),o}async function ir(e,t,r,s,n,i){if(it(e,n))return {action:"render",changeToken:null};if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await fe(e,t,s,n,i):null};let o=s.manifest.pages[n];if(!o)return {action:"render",changeToken:await fe(e,t,s,n,i)};let u=await fe(e,t,s,n,i);if(u?.supported&&u.token)return o.changeToken===u.token?{action:"reuse",changeToken:u}:{action:"render",changeToken:u};let a=ee(r.sitemapLastmodByPage[n]);return a&&o.sitemapLastmod&&o.sitemapLastmod===a?{action:"reuse",changeToken:u}:{action:"render",changeToken:u}}async function ar(e,t,r,s,n){let i=Ne(r.previousManifest),o=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Qe(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(m)});}delete r.manifest.pages[d],o++;}let u=Ne(r.manifest,r.runId),a=new Set,c=new Set;for(let d of u){for(let h of H(e,d))a.add(h);let g=r.manifest.assets[d]?.outputPath;g&&c.add(g);let m=te(e,t.assetMap,d);m&&c.add(m);}for(let d of Object.keys(r.manifest.assets))u.has(d)||delete r.manifest.assets[d];let l=0,p=0;for(let d of i){let g=new Set,m=r.previousManifest.assets[d]?.outputPath;m&&g.add(m);let h=te(e,t.assetMap,d);if(!u.has(d)&&h)g.add(h);else if(!u.has(d)){let y=be(e,d);g.add(y.originalFilePath),y.hashedFilePath&&g.add(y.hashedFilePath);}for(let y of g)if(!c.has(y)){Qe(n,e,y);try{await S.unlink(y),l++;}catch(v){(v.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:y,error:String(v)});}}for(let y of H(e,d))a.has(y)||Object.prototype.hasOwnProperty.call(t.assetMap,y)&&(delete t.assetMap[y],p++);}(o>0||l>0||p>0)&&s.info(`Incremental cleanup removed ${o} stale pages, ${l} stale assets, ${p} stale asset mappings`,{removedPages:o,removedAssets:l,removedAssetMappings:p,phase:"incremental-cleanup"});}function Ne(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function _e(e,t,r){for(let s of t)for(let n of H(e,s))if(r.has(n))return true;return false}function or(e,t,r,s,n){let i=new Set(r.rewriteTargets),o=sr(s,n);if(o.size===0)return [...i];for(let u of Object.values(t.manifest.pages))_e(e,u.discoveredAssets,o)&&i.add(f.resolve(u.outputPath));for(let[u,a]of Object.entries(t.manifest.assets)){a.isText&&_e(e,a.discoveredAssets,o)&&i.add(f.resolve(a.outputPath));for(let c of H(e,u))if(o.has(c)){j(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function H(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function ur(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function tt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function lr(e,t,r){for(let s of ur(e,r)){let n=t[s];if(!n)continue;let i=tt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function cr(e,t,r){for(let s of H(e,r)){let n=t[s];if(!n)continue;let i=tt(e,n);if(i)return i}return null}function te(e,t,r){let s=cr(e,t,r);return s||be(e,r).originalFilePath}function dr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function We(e){let t=dr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function pr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(W(e,s)));for(let s of t.doneAssets){let n=te(e,t.assetMap,s);!n||!$(n)||r.add(f.resolve(n));}return [...r]}var re="re:",Z=new Map;function gr(e){if(!e.startsWith(re))return null;let t=Z.get(e);if(t!==void 0)return t||null;let r=e.slice(re.length).trim();if(!r)return Z.set(e,false),null;try{let s=new RegExp(r);return Z.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Z.set(e,false),null}}function ve(e,t){let r=gr(t);return r?r.test(e):t.startsWith(re)?false:e.startsWith(t)}function hr(e,t){return t.find(r=>ve(e,r))}function rt(e,t){let r=hr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(re)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function fr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function mr(e,t){return fr(e).has(t.hostname)}function wr(e){let t=F(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function st(e){let t=F(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function se(e){return Gt.some(t=>e.pathname.startsWith(t))}function x(e,t,r=e.sourceOrigin,s,n="url"){let i=wr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(st(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let o=new URL(i,r);if(!["http:","https:"].includes(o.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!mr(e,o))return s?.ignore(n,o.toString(),"host not allowed",r),null;let u=rt(e,o);if(u)return s?.reject(n,o.toString(),u,r),null;let a=new URL(e.sourceOrigin);return o.protocol=a.protocol,o.host=a.host,o.hash="",o}catch{return s?.reject(n,t,"invalid URL",r),null}}function nt(e){return Jt.has(f.extname(e.pathname).toLowerCase())}function z(e){return se(e)?false:zt.has(f.extname(e.pathname).toLowerCase())}function ne(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function R(e,t){return ne(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:nt(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>ve(t.pathname,r)):false}function W(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function Pr(e){if(!e.generated404RequestPath)return null;try{return new URL(e.generated404RequestPath,e.sourceOrigin).toString()}catch{return null}}function it(e,t){let r=Pr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function be(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,o=f.join(e.outputDir,i),u=de(e.targetOrigin,i),a=`${u}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:o,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:n};let c=f.extname(s),p=`${c?s.slice(0,-c.length):s}.${le(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:o,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:f.join(e.outputDir,p),hashedPublicUrl:de(e.targetOrigin,p)}}function yr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),u=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{u.push(new URL(e.targetOrigin).origin);}catch{}for(let a of u)try{let c=new URL(a);t[c.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function at(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Sr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function ut(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function vr(e,t){let r=ut(t),s=Sr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function br(e,t){if(!t)return;let r=ut(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function me(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Vt.has(r)||Bt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function we(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Rr(e){return le(e.toString("base64"),40)}function Cr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function xr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function ye(e,t,r,s,n,i,o){let u=be(e,r),a=Rr(s),c=u.originalFilePath,l=u.originalPublicUrlWithSearch;if(u.preferQueryHashed&&u.hashedFilePath&&u.hashedPublicUrl&&Cr(n,u.originalFilePath,a)){let m=lr(e,t,r);m?(c=m.outputPath,l=m.publicUrl):(c=u.hashedFilePath,l=u.hashedPublicUrl);}yr(e,t,r,l,u.originalPublicUrl);let p=$(c)?Buffer.from(he(s.toString("utf8"),e,t,c),"utf8"):s,d=await at(c),g=false;return (d===null||!d.equals(p))&&(await ce(c),await S.writeFile(c,p),g=true),xr(n,u.originalFilePath,a),await br(c,o),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:c,publicUrl:l,wroteFile:g}}function B(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=x(e,r,i,s,"page");if(!o)return;if(!z(o)||nt(o)){if(R(e,o)){s.info(`Seeded/non-page URL queued as asset: ${o.toString()}`,{url:o.toString(),source:n}),A(e,t,o.toString(),s,n);return}s.reject("page",o.toString(),se(o)?"asset/internal path cannot be page":"not page-like",n);return}let u=o.toString();!t.donePages.has(u)&&!t.queuedPages.has(u)&&(t.queuedPages.add(u),t.pageQueue.push(u),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function A(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=x(e,r,i,s,"asset");if(!o)return;if(!R(e,o)){s.reject("asset",o.toString(),"not a safe asset path/prefix",n);return}let u=o.toString();!t.doneAssets.has(u)&&!t.queuedAssets.has(u)&&(t.queuedAssets.add(u),t.assetQueue.push(u),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Re(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=x(e,r,i,s,"sitemap");if(!o)return;if(!ne(o)){s.reject("sitemap",o.toString(),"not sitemap-like",n);return}let u=o.toString();!t.doneSitemaps.has(u)&&!t.queuedSitemaps.has(u)&&(t.queuedSitemaps.add(u),t.sitemapQueue.push(u),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Ar(e){let t=F(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:ee(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function kr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Mr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Pe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function lt(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),o=f.join(t,n.name);if(n.isDirectory()){r.push(...await lt(i,o));continue}await S.copyFile(i,o),r.push(o);}return r}async function Tr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,o])=>({sourcePath:i.trim(),prefix:String(o||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let o=i.sourcePath,u=i.prefix;if(!o||!u){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:o,prefixPath:u});continue}let a=Yt(o);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:o,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:u});continue}let c=a.resolvedPath,l=Pe(u);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:o,prefixPath:u});continue}let p;try{p=await S.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:o,sourceAbs:c,prefixPath:u});continue}let d=f.resolve(e.outputDir,l);if(p.isDirectory()){n.push(...await lt(c,d)),s++,t.progress(`Copied static directory to export: ${c} -> /${l}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${l}`,copiedItems:s});continue}let g=f.basename(c),m=u.endsWith("/")?f.join(d,g):d;await S.mkdir(f.dirname(m),{recursive:true}),await S.copyFile(c,m),n.push(m),s++,t.progress(`Copied static file to export: ${c} -> /${Pe(u.endsWith("/")?`${u}${g}`:u)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${Pe(u.endsWith("/")?`${u}${g}`:u)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function J(e,t,r,s,n){let i=new Set,o=F(r).replace(/\\\//g,"/");for(let c of Mr(o)){let l=x(e,c,t,s,`${n}:css-url`);l&&R(e,l)&&i.add(l.toString());}for(let c of o.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let l=x(e,c[1],t,s,`${n}:css-import`);l&&R(e,l)&&i.add(l.toString());}for(let c of o.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let l=x(e,c[1],t,s,`${n}:xml-stylesheet`);l&&R(e,l)&&i.add(l.toString());}let u="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${u}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${u}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of o.matchAll(a)){let l=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(st(l)){s.ignore(`${n}:serialized-url`,l,"looks like JavaScript/code fragment",t);continue}let p=x(e,l,t,s,`${n}:serialized-url`);p&&R(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Lr(e,t,r,s){let n=new Set,i=F(r);for(let o of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let u=o[1],a=x(e,u,t,s,"page-link");a&&(rt(e,a)||z(a)&&!se(a)&&!R(e,a)&&n.add(a.toString()));}return [...n]}async function He(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let o=await i.body(),u=i.url()||n;await ye(e,r.assetMap,u,o,r,s);let a=o.toString("utf8");for(let c of J(e,u,a,s,"sitemap"))A(e,r,c,s,u);for(let c of Ar(a)){let l=x(e,c.loc,u,s,"sitemap-loc");l&&(ne(l)?Re(e,r,l.toString(),s,u):R(e,l)?A(e,r,l.toString(),s,u):z(l)?(c.lastmod&&(r.sitemapLastmodByPage[l.toString()]=c.lastmod),B(e,r,l.toString(),s,u)):s.reject("sitemap-loc",l.toString(),"not page/sitemap/asset-like",u));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Dr(e,t,r,s,n,i,o){let u=x(e,s,e.sourceOrigin,n,"asset-fetch");if(!u||!R(e,u))return;let a=u.toString();if(r.doneAssets.has(a))return;let c=i.enabled?i.previousManifest.assets[a]:void 0,l=(p,d={})=>{if(!c)return false;i.manifest.assets[a]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let g of c.discoveredAssets||[])A(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),we(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=te(e,r.assetMap,a),d=p?await ot(p):null;if(p&&d)try{let w=await t.head(a,{timeout:6e4}),b=w.headers();if(w.ok()&&vr(d,b)){let L=await at(p);if(L!==null){let D=w.url()||a,E=await ye(e,r.assetMap,D,L,r,n,b);E.wroteFile&&(j(o,e,E.outputPath),$(E.outputPath)&&Y(o,E.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:b["last-modified"]||"",contentLength:b["content-length"]||"",phase:"download-assets"}),we(r,n);let U=me(u,b),C=c?.discoveredAssets?[...c.discoveredAssets]:[];if(U||c?.isText){let dt=L.toString("utf8"),Ce=J(e,D,dt,n,"asset:cached");(Ce.length>0||!c)&&(C=Ce);for(let pt of C)A(e,r,pt,n,D);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:E.outputPath,isText:U||!!c?.isText,discoveredAssets:C,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(l(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let m=await g.body(),h=g.url()||a,y=g.headers(),v=await ye(e,r.assetMap,h,m,r,n,y);v.wroteFile&&(j(o,e,v.outputPath),$(v.outputPath)&&Y(o,v.outputPath)),r.doneAssets.add(a),we(r,n);let P=[];if(me(u,y)){let w=m.toString("utf8");P=J(e,h,w,n,`asset:${f.extname(u.pathname).toLowerCase()||y["content-type"]||"unknown"}`);for(let b of P)A(e,r,b,n,h);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:v.outputPath,isText:me(u,y),discoveredAssets:P,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(p){if(l("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function Ir(e,t,r,s,n,i){let o=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),u=Array.from({length:o},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Dr(e,t,r,a,s,n,i));}});await Promise.all(u);}async function Er(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Or(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Er(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Fr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function jr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ve(t,r))}async function $r(e,t,r,s,n=true){let i=new Set;if(n){let u=await t.evaluate(()=>{let a=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let m of c){let h=d.getAttribute(m);h&&a.add(h);}for(let m of p){let h=d.getAttribute(m);h&&h.split(",").forEach(y=>{let v=y.trim().split(/\s+/)[0];v&&a.add(v);});}let g=d.getAttribute("style");g&&l.add(g);}),{attrs:[...a],styles:[...l]}});for(let a of u.attrs)for(let c of kr(a)){let l=x(e,c,r,s,"dom-attr");l&&R(e,l)&&i.add(l.toString());}for(let a of u.styles)for(let c of J(e,r,a,s,"dom-style"))i.add(c);}let o=await t.content();for(let u of J(e,r,o,s,"page-html"))i.add(u);return [...i]}async function Be(e,t,r,s,n){let i=W(e,t);await ce(i),await S.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Je(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function ze(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Ur(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Qr(e,t,r,s,n,i,o){let u=await chromium.launch({headless:true}),a=await u.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await u.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Ur(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let p=new URL(l),d=it(e,l),g=jr(e,p.pathname);if(!z(p)||R(e,p)||se(p)){if(R(e,p)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),A(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let m=await ir(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let P=i.manifest.pages[l];if(m.action==="reuse"&&P){for(let w of P.discoveredAssets)A(e,t,w,r,l);for(let w of P.discoveredPages)B(e,t,w,r,l);P.lastSeenRunId=i.runId,P.sitemapLastmod=ee(t.sitemapLastmodByPage[l])??P.sitemapLastmod,m.changeToken?.supported&&(P.changeToken=m.changeToken.token,P.tokenSource=m.changeToken.tokenSource??P.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Je(t)&&ze(t,r);continue}}let h=await(g?c.newPage():a.newPage()),y=!1,v=new Set;await h.route("**/*",async P=>{let w=P.request();if(w.isNavigationRequest()&&w.frame()===h.mainFrame())try{let b=new URL(w.url()),L=new URL(l);if(b.hash="",L.hash="",b.href===L.href){if(y){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:w.url()}),await P.abort("aborted");return}y=!0;}}catch{}await P.continue();}),h.on("response",P=>{try{let w=x(e,P.url(),e.sourceOrigin,void 0,"network-response");w&&R(e,w)&&(v.add(w.toString()),A(e,t,w.toString(),r,l));}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let P=null,w=null;try{if(P=await h.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),P&&P.ok())try{w=await P.text();}catch{w=null;}}catch(C){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(C)});}if(d){let C=P?.status()??null;if(C!==404){r.error(`Generated 404 request path returned unexpected status for ${l}`,{url:l,expectedStatus:404,status:C,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${l}`,{url:l,status:C,outputPath:W(e,l)});}else if(P&&!P.ok()){r.skip("page",l,P.status()),await h.close();continue}g?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Fr(e,h)):await Or(e,h),t.stats.pagesRendered++,Je(t)&&ze(t,r);let b=await $r(e,h,l,r,!g);for(let C of b)A(e,t,C,r,l);let L=[...new Set([...b,...v])],D=w??await h.content(),E=s!=="single-url"&&!d?Lr(e,l,D,r):[];if(s!=="single-url"&&!d)for(let C of E)B(e,t,C,r,l);await Be(e,l,D,t,r);let U=W(e,l);j(o,e,U),Y(o,U),i.trackManifest&&(i.manifest.pages[l]={url:l,outputPath:U,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:ee(t.sitemapLastmodByPage[l])??null,discoveredPages:E,discoveredAssets:L,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(P){try{let w=await h.content();if(w&&w.trim()){await Be(e,l,w,t,r);let b=W(e,l);j(o,e,b),Y(o,b),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(P)});}}catch(w){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(w)});}r.error(`Failed page ${l}`,{url:l,error:String(P)});}finally{await h.close();}}}finally{await c.close(),await a.close(),await u.close();}}async function qr(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ve(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Nr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function _r(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function ct(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ge(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Wr(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ge(s)-Ge(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Ke(e){let t=await ct(f.join(e,"job.json"));for(let r of Wr(e,t)){let s=await qr(r);if(s.length>0)return s}return []}async function Hr(e){let t=f.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let o=f.join(t,i);if(!(await S.stat(o).catch(()=>null))?.isDirectory())continue;let a=await ct(f.join(o,"job.json"));if(!Nr(a))continue;let c=_r(a);c>=n&&(n=c,s=o);}return s}async function Br(e){let t=await Hr(e),r=t!==""?await Ke(t):await Ke(e.logDir),s=new Set;for(let n of r)if(Ve(n.error)||Ve(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Jr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function zr(e){try{let t=await S.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Vr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Xe(e,t,r,s,n){let i=x(e,r,e.sourceOrigin,s,"manual-url");i&&(ne(i)?Re(e,t,i.toString(),s,n):R(e,i)?A(e,t,i.toString(),s,n):z(i)?B(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Gr(){let e=await ue(),t=Kt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Br(e)),t.mode==="single-url"&&(r=await Jr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.mode==="full"&&!t.resumeRewrite,i=t.preserveOutput?await zr(e):{},o=s?await tr(e):et(),u={enabled:s,trackManifest:n,manifest:qe(o),previousManifest:qe(o),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Vr(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...i},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},c=er();t.preserveOutput||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let l=new _(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let h=await chromium.launch({headless:true}),y=await h.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),v=y.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(w=>Re(e,a,w,l)),await He(e,v,a,l),e.seedPaths.forEach(w=>Xe(e,a,w,l,"seed-path")),e.generated404RequestPath&&B(e,a,e.generated404RequestPath,l,"generated-404-request-path")):(r.forEach(w=>Xe(e,a,w,l,"cli")),await He(e,v,a,l)),l.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),l.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let P=Array.from({length:Math.max(1,e.concurrency)},()=>Qr(e,a,l,t.mode,v,u,c));if(await Promise.all(P),l.endMark("render-pages",{pages:a.donePages.size}),l.mark("download-assets"),await Ir(e,v,a,l,u,c),l.endMark("download-assets",{assets:a.doneAssets.size}),u.enabled&&e.maxPages===0&&await ar(e,a,u,l,c),t.mode==="full"){let w=await Tr(e,l);for(let b of w)j(c,e,b);}}finally{await y.close(),await h.close();}}u.trackManifest&&(u.manifest.updatedAt=new Date().toISOString(),await rr(e,u.manifest)),await S.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8"),JSON.stringify(i,null,2)!==JSON.stringify(a.assetMap,null,2)&&j(c,e,f.join(e.outputDir,"asset-map.json"));let p=t.mode==="single-url"?pr(e,a):u.enabled&&!t.resumeRewrite?or(e,u,c,i,a.assetMap):void 0;l.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${p?.length??0}`:"Rewriting text files...";l.progress(d,{phase:"rewrite-text",index:0,totalFiles:p?.length,changedTextFiles:0}),await We(d);let g=Date.now(),m=await Fe(e,a.assetMap,{files:p,previousAssetMap:i,onProgress:async({index:h,totalFiles:y,changedTextFiles:v,file:P,changed:w})=>{w&&j(c,e,f.join(e.outputDir,P)),l.checkpoint(`Rewriting text file ${h}/${y}`,{phase:"rewrite-text",index:h,totalFiles:y,changedTextFiles:v,file:P});let b=Date.now();if(!(h===1||h===y||b-g>=5e3))return;g=b;let D=`Rewriting text files: ${h}/${y}`;l.progress(D,{phase:"rewrite-text",index:h,totalFiles:y,changedTextFiles:v,file:P}),await We(D);}});l.endMark("rewrite-text",{changedTextFiles:m}),await ke(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...c.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...c.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((p||[]).map(h=>V(e.outputDir,f.isAbsolute(h)?h:f.join(e.outputDir,h))).filter(h=>h!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${m} text files.`:`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:m}),await l.flush();}Gr().catch(async e=>{console.error(e);try{let r=(await ue().catch(()=>null))?.logDir??"logs",s=new _(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${r} ${t}`)}function Wt(e){let t=j(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function _t(e,t){let s=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,G(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!s.length)return e;let r=i=>{let o=i;for(let[l,a]of s)o=o.split(l).join(a);return o},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,l,a)=>`${o}${r(l)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,l,a,c)=>Wt(a)?`${o}${r(a)}${c}`:i),n}function Ht(e,t){if(!t)return null;let s=h.dirname(h.resolve(t)),r=h.resolve(e.outputDir),n=h.relative(s,r).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function zt(e,t,s){let r=Ht(t,s);if(!r)return e;let n=e;for(let i of $t){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");n=n.replace(l,`${r}${i}`).replace(a,`${r}${i}`);let c=i.replace(/\//g,"\\/"),u=r.replace(/\//g,"\\/"),p=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),f=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${u}${c}`).replace(g,`${u}${c}`).replace(f,`${u}${c}`);}return n}function he(e,t,s){if(e.urlRewriteMode==="absolute")return t;let r=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=r?r.pathname:Ee(t,e.targetOrigin),i=r?`${r.search}${r.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Fe(e.outputDir,s,n)}${i}`}function ge(e,t,s,r,n){K(e,s,he(t,r,n));}function Bt(e,t,s,r,n){if(!s||s===r)return;K(e,s,r);let i=he(t,s,n),o=he(t,r,n);K(e,i,o);}function fe(e,t,s,r,n={}){let i=e,o={};ge(o,t,t.sourceOrigin,t.targetOrigin,r);for(let[a,c]of Object.entries(t.extraReplacements))ge(o,t,a,c,r);for(let[a,c]of Object.entries(s))ge(o,t,a,c,r);for(let[a,c]of Object.entries(n)){let u=s[a];!u||u===c||Bt(o,t,c,u,r);}let l=Object.entries(o).sort((a,c)=>c[0].length-a[0].length);for(let[a,c]of l)i=i.split(a).join(c);return t.urlRewriteMode!=="absolute"&&(i=zt(i,t,r)),i=_t(i,l.map(([,a])=>a)),Nt(t,i,r)&&(i=qt(i)),i}async function Jt(e,t,s,r){let n=h.join(e.outputDir,r),i=h.extname(n).toLowerCase(),o;try{o=await y.readFile(n,"utf8");}catch(c){if(c.code!=="ENOENT")throw c;return {changed:false}}let a=fe(o,e,t,i===".js"||i===".mjs"?void 0:n,s);return a!==o?(await y.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function Vt(e,t){return await new Promise((s,r)=>{let n=a=>{if(l(),a?.error){r(new Error(a.error));return}s({changed:!!a?.changed});},i=a=>{l(),r(a);},o=a=>{l(),a!==0&&r(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",n),e.off("error",i),e.off("exit",o);};e.on("message",n),e.on("error",i),e.on("exit",o),e.postMessage({file:t});})}async function Ue(e,t,s={}){let n=(s.files?[...new Set(s.files)].map(d=>h.resolve(d)).filter(d=>d.startsWith(h.resolve(e.outputDir))).map(d=>h.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Et(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&U(d)),i=0;if(n.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(o,n.length,availableParallelism())),a=0,c=0,u=null;if(l===1){for(let d of n){let g=await Jt(e,t,s.previousAssetMap??{},d);g.changed&&(i+=1),c+=1,await s.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:d,changed:g.changed});}return i}let p=Array.from({length:l},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:s.previousAssetMap??{}}});try{for(;!u;){let g=a;if(g>=n.length)return;a+=1;let f=n[g],w=await Vt(d,f);w.changed&&(i+=1),c+=1,await s.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:f,changed:w.changed});}}catch(g){throw u=g instanceof Error?g:new Error(String(g)),u}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var q={error:0,warn:1,info:2,debug:3},Qe=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",Y=Qe?h.join(Qe,"current-progress.json"):"",Ne=Promise.resolve();function qe(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function Gt(e,t){let s={...e},r=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let o=s[n];if(typeof o=="number"&&typeof i=="number"&&r.has(n)){s[n]=Math.max(o,i);continue}s[n]=i;}return s}function Kt(e,t,s){Y&&(Ne=Ne.then(async()=>{let r=new Date,n=r.toISOString();await y.mkdir(h.dirname(Y),{recursive:true});let i=await y.readFile(Y,"utf8").then(w=>JSON.parse(w)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},l=qe(e),a=i?.currentStep||qe(i?.source||""),c=i?.startedAt||n,u={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==l&&i?.stepStartedAt){let w=Math.max(0,Math.round((r.getTime()-new Date(i.stepStartedAt).getTime())/1e3));u[a]=(u[a]??0)+w,p=n;}let d=Math.max(0,Math.round((r.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((r.getTime()-new Date(c).getTime())/1e3)),f={checkedAt:n,source:e,message:t,details:Gt(o,s??{}),startedAt:c,currentStep:l,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:u};await y.writeFile(Y,JSON.stringify(f,null,2),"utf8");}).catch(()=>{}));}async function Yt(e,t){await y.mkdir(h.dirname(e),{recursive:true}),await y.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function Xt(e,t){let s=typeof t?.phase=="string"?t.phase.trim():"";return s||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function Zt(e,t,s){return {checkedAt:new Date().toISOString(),currentStep:Xt(e,s),level:e,message:t,details:s??{}}}var W=class{constructor(t,s="info"){this.logDir=t;this.level=typeof s=="boolean"?s?"debug":"info":s,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return h.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return h.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return h.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return h.join(this.logDir,"ignored.jsonl")}get skippedPath(){return h.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return h.join(this.logDir,"errors.jsonl")}get timingsPath(){return h.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(s=>{this.writeError=s instanceof Error?s:new Error(String(s));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await y.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,s){this.enqueueTask(()=>Yt(t,s));}updateCurrentEvent(t,s,r){let n=Zt(t,s,r);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,s,r){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:s,...r||{}});this.enqueueLine(n),this.updateCurrentEvent(t,s,r),t==="error"?console.error(s,r||""):t==="warn"?console.warn(s,r||""):(q[this.level]>=q.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(s);}info(t,s){this.push("info",t,s);}progress(t,s){Kt("crawl.log.jsonl",t,s),this.updateCurrentEvent("progress",t,s);}checkpoint(t,s){this.updateCurrentEvent("checkpoint",t,s);}page(t,s){this.push("page",t,s);}sitemap(t,s){this.push("sitemap",t,s);}asset(t,s){this.push("asset",t,s);}warn(t,s){this.push("warn",t,s);}error(t,s){this.enqueueJsonLine(this.errorsPath,{message:t,...s||{}}),this.push("error",t,s);}summary(t,s){this.push("summary",t,s);}mark(t){this.marks.set(t,Date.now());}endMark(t,s){let r=this.marks.get(t);if(!r)return;let n=Date.now()-r,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...s||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,s,r,n){let i={kind:t,url:s,reason:r,source:n};this.enqueueJsonLine(this.rejectedPath,i),q[this.level]>=q.debug&&this.push("reject",`Rejected ${t} ${s}: ${r}`,i);}ignore(t,s,r,n){let i={kind:t,url:s,reason:r,source:n};this.enqueueJsonLine(this.ignoredPath,i),q[this.level]>=q.debug&&this.push("ignore",`Ignored ${t} ${s}: ${r}`,i);}skip(t,s,r,n){let i={kind:t,url:s,status:r,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${s}: HTTP ${r}`,i);}async flush(){let t=Date.now()-this.startedAt,s={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,s),await this.writeQueue,this.writeError)throw this.writeError}};var tr=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],rr=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),sr=new Set(["",".html",".htm"]),nr=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),ir=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function or(e){let t=[],s="full",r="full",n=false,i=false;for(let o=0;o<e.length;o++){let l=e[o];if(l==="--retry-timeouts")s="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let a=e[++o];if(!a)throw new Error("--crawl-mode requires a value");r=a==="incremental"?"incremental":"full",n||=r==="incremental";}else if(l.startsWith("--crawl-mode="))r=l.slice(13)==="incremental"?"incremental":"full",n||=r==="incremental";else if(l==="--incremental")r="incremental",n=true;else if(l==="--url"){let a=e[++o];if(!a)throw new Error("--url requires a value");s="single-url",n=true,t.push(a);}else if(l.startsWith("--url="))s="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let a=e[++o];if(!a)throw new Error("--urls requires a file path");s="single-url",n=true,t.push(`@${a}`);}}return {mode:s,crawlMode:r,urls:t,preserveOutput:n,resumeRewrite:i}}function be(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function ar(){let e=be();return e?h.resolve(e,".."):""}function ur(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?h.resolve(e):""}function lr(e){let t=e.trim(),s=[{alias:"@storage-root",root:ar(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:be(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:ur(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let r of s){if(t!==r.alias&&!t.startsWith(`${r.alias}/`))continue;if(!r.root)return {resolvedPath:null,alias:r.alias,requiredEnv:r.requiredEnv};let n=t.slice(r.alias.length).replace(/^\/+/,"");return {resolvedPath:n?h.resolve(r.root,n):r.root,alias:r.alias}}return {resolvedPath:h.resolve(t)}}function rt(e){let t=be();return t?h.join(t,"crawl-manifest.json"):h.join(e.outputDir,".crawl-manifest.json")}function st(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function cr(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function $(e,t,s){let r=V(t.outputDir,s);r&&(e.changedFiles.add(r),e.deletedFiles.delete(r));}function We(e,t,s){let r=V(t.outputDir,s);r&&(e.deletedFiles.add(r),e.changedFiles.delete(r));}function Z(e,t){e.rewriteTargets.add(h.resolve(t));}async function dr(e){try{let t=await y.readFile(rt(e),"utf8"),s=JSON.parse(t);if(s&&typeof s=="object"&&s.pages&&typeof s.pages=="object"){if(s.schemaVersion===2&&s.assets&&typeof s.assets=="object")return s;if(s.schemaVersion===1)return {schemaVersion:2,updatedAt:String(s.updatedAt||""),pages:s.pages,assets:{}}}}catch{}return st()}function _e(e){return JSON.parse(JSON.stringify(e))}async function pr(e,t){let s=rt(e);await y.mkdir(h.dirname(s),{recursive:true}),await y.writeFile(s,JSON.stringify(t,null,2),"utf8");}function ee(e){if(!e)return null;let t=e.trim();return t||null}function gr(e,t){let s=new Set,r=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of r){let i=e[n],o=t[n];!i||!o||i===o||s.add(n);}return s}async function hr(e,t,s,r){let n=[...new Set(s.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.nonce||"").trim();if(n.length===0||!i)return new Map;let o=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(o,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-wp-nonce":i},data:JSON.stringify({urls:n})});if(!l.ok())return r.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:o,status:l.status()}),new Map;let a=await l.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let c=new Map;for(let u of a.items)!u||typeof u.url!="string"||c.set(u.url,u);return c}catch(l){return r.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:o,error:String(l)}),new Map}}async function me(e,t,s,r,n){if(!s.enabled&&!s.trackManifest)return null;if(s.changeTokenCache.has(r))return s.changeTokenCache.get(r)??null;let o=(await hr(e,t,[r],n)).get(r)??null;return s.changeTokenCache.set(r,o),o}async function fr(e,t,s,r,n,i){if(ut(e,n))return {action:"render",changeToken:null};if(!r.enabled)return {action:"render",changeToken:r.trackManifest?await me(e,t,r,n,i):null};let o=r.manifest.pages[n];if(!o)return {action:"render",changeToken:await me(e,t,r,n,i)};let l=await me(e,t,r,n,i);if(l?.supported&&l.token)return o.changeToken===l.token?{action:"reuse",changeToken:l}:{action:"render",changeToken:l};let a=ee(s.sitemapLastmodByPage[n]);return a&&o.sitemapLastmod&&o.sitemapLastmod===a?{action:"reuse",changeToken:l}:{action:"render",changeToken:l}}async function mr(e,t,s,r,n){let i=He(s.previousManifest),o=0;for(let[d,g]of Object.entries(s.manifest.pages))if(!s.seenPages.has(d)){We(n,e,g.outputPath);try{await y.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&r.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(f)});}delete s.manifest.pages[d],o++;}let l=He(s.manifest,s.runId),a=new Set,c=new Set;for(let d of l){for(let w of H(e,d))a.add(w);let g=s.manifest.assets[d]?.outputPath;g&&c.add(g);let f=te(e,t.assetMap,d);f&&c.add(f);}for(let d of Object.keys(s.manifest.assets))l.has(d)||delete s.manifest.assets[d];let u=0,p=0;for(let d of i){let g=new Set,f=s.previousManifest.assets[d]?.outputPath;f&&g.add(f);let w=te(e,t.assetMap,d);if(!l.has(d)&&w)g.add(w);else if(!l.has(d)){let b=Re(e,d);g.add(b.originalFilePath),b.hashedFilePath&&g.add(b.hashedFilePath);}for(let b of g)if(!c.has(b)){We(n,e,b);try{await y.unlink(b),u++;}catch(S){(S.code||"")!=="ENOENT"&&r.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:b,error:String(S)});}}for(let b of H(e,d))a.has(b)||Object.prototype.hasOwnProperty.call(t.assetMap,b)&&(delete t.assetMap[b],p++);}(o>0||u>0||p>0)&&r.info(`Incremental cleanup removed ${o} stale pages, ${u} stale assets, ${p} stale asset mappings`,{removedPages:o,removedAssets:u,removedAssetMappings:p,phase:"incremental-cleanup"});}function He(e,t){let s=new Set;for(let r of Object.values(e.pages))for(let n of r.discoveredAssets||[]){let i=n.trim();i&&s.add(i);}for(let[r,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=r.trim();i&&s.add(i);}return s}function ze(e,t,s){for(let r of t)for(let n of H(e,r))if(s.has(n))return true;return false}function wr(e,t,s,r,n){let i=new Set(s.rewriteTargets),o=gr(r,n);if(o.size===0)return [...i];for(let l of Object.values(t.manifest.pages))ze(e,l.discoveredAssets,o)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&ze(e,a.discoveredAssets,o)&&i.add(h.resolve(a.outputPath));for(let c of H(e,l))if(o.has(c)){$(s,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function H(e,t){let s=new Set([t]);try{let r=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{s.add(new URL(o).origin+r.pathname+r.search);}catch{}s.add(r.pathname+r.search),s.add(r.pathname);}catch{}return [...s]}function Pr(e,t){let s=new Set([t]);try{let r=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{s.add(new URL(o).origin+r.pathname+r.search);}catch{}s.add(r.pathname+r.search);}catch{}return [...s]}function nt(e,t){let s=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let r=new URL(t,s);return h.join(e.outputDir,decodeURIComponent(r.pathname))}catch{return null}}function Sr(e,t,s){for(let r of Pr(e,s)){let n=t[r];if(!n)continue;let i=nt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function yr(e,t,s){for(let r of H(e,s)){let n=t[r];if(!n)continue;let i=nt(e,n);if(i)return i}return null}function te(e,t,s){let r=yr(e,t,s);return r||Re(e,s).originalFilePath}function br(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function Be(e){let t=br();if(t)try{let s=await y.readFile(t,"utf8"),r=JSON.parse(s);if(!r||typeof r!="object")return;await y.writeFile(t,JSON.stringify({...r,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function vr(e,t){let s=new Set;for(let r of t.donePages)s.add(h.resolve(_(e,r)));for(let r of t.doneAssets){let n=te(e,t.assetMap,r);!n||!U(n)||s.add(h.resolve(n));}return [...s]}var re="re:",X=new Map;function Rr(e){if(!e.startsWith(re))return null;let t=X.get(e);if(t!==void 0)return t||null;let s=e.slice(re.length).trim();if(!s)return X.set(e,false),null;try{let r=new RegExp(s);return X.set(e,r),r}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),X.set(e,false),null}}function ve(e,t){let s=Rr(t);return s?s.test(e):t.startsWith(re)?false:e.startsWith(t)}function Cr(e,t){return t.find(s=>ve(e,s))}function it(e,t){let s=Cr(t.pathname,e.blockedPathPrefixes);if(s)return s.startsWith(re)?`blocked path regex: ${s}`:`blocked path prefix: ${s}`;let r=e.blockedSearchFragments.find(n=>t.search.includes(n));return r?`blocked search fragment: ${r}`:null}function xr(e){let t=new URL(e.sourceOrigin),s=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{s.add(new URL(e.targetOrigin).hostname);}catch{}for(let r of Object.keys(e.extraReplacements||{}))try{s.add(new URL(r).hostname);}catch{}return s}function kr(e,t){return xr(e).has(t.hostname)}function Ar(e){let t=j(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function ot(e){let t=j(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function se(e){return ir.some(t=>e.pathname.startsWith(t))}function k(e,t,s=e.sourceOrigin,r,n="url"){let i=Ar(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return r?.ignore(n,t,"empty or unsupported scheme",s),null;if(ot(i))return r?.ignore(n,t,"looks like JavaScript/code fragment",s),null;try{let o=new URL(i,s);if(!["http:","https:"].includes(o.protocol))return r?.ignore(n,t,"unsupported protocol: ",s),null;if(!kr(e,o))return r?.ignore(n,o.toString(),"host not allowed",s),null;let l=it(e,o);if(l)return r?.reject(n,o.toString(),l,s),null;let a=new URL(e.sourceOrigin);return o.protocol=a.protocol,o.host=a.host,o.hash="",o}catch{return r?.reject(n,t,"invalid URL",s),null}}function at(e){return rr.has(h.extname(e.pathname).toLowerCase())}function J(e){return se(e)?false:sr.has(h.extname(e.pathname).toLowerCase())}function ne(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return ne(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:at(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(s=>ve(t.pathname,s)):false}function _(e,t){let s=new URL(t),r=decodeURIComponent(s.pathname);return r.endsWith("/")?r+="index.html":h.extname(r)||(r+="/index.html"),h.join(e.outputDir,r)}function Mr(e){if(!e.generated404RequestPath)return null;try{return new URL(e.generated404RequestPath,e.sourceOrigin).toString()}catch{return null}}function ut(e,t){let s=Mr(e);if(!s)return false;try{return new URL(t,e.sourceOrigin).toString()===s}catch{return false}}function Re(e,t){let s=new URL(t),r=decodeURIComponent(s.pathname);r.endsWith("/")&&(r+="index.html");let n=!!s.search,i=r,o=h.join(e.outputDir,i),l=pe(e.targetOrigin,i),a=`${l}${s.search}`;if(!n)return {originalPathname:i,originalFilePath:o,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let c=h.extname(r),p=`${c?r.slice(0,-c.length):r}.${ce(s.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:o,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:pe(e.targetOrigin,p)}}function Tr(e,t,s,r,n){t[s]=r;try{let i=new URL(s),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let c=new URL(a);t[c.origin+i.pathname+i.search]=r;}catch{}try{let a=(()=>{let c=new URL(r,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(r,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function lt(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ct(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Lr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let s=Number.parseInt(t,10);return Number.isFinite(s)&&s>=0?s:null}function dt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let s=Date.parse(t);return Number.isFinite(s)?s:null}function Ir(e,t){let s=dt(t),r=Lr(t),n=e?.mtime.getTime();return s!==null&&Math.abs((n??0)-s)>=1e3||r!==null&&e?.size!==r?false:s!==null||r!==null}async function Dr(e,t){if(!t)return;let s=dt(t);if(s===null)return;let r=await ct(e);if(!r)return;let n=new Date(s);await y.utimes(e,r.atime,n).catch(()=>{});}function we(e,t){let s=h.extname(e.pathname).toLowerCase(),r=t["content-type"]||"";return nr.has(s)||tr.some(n=>r.includes(n))&&!r.startsWith("image/")&&!r.includes("font")}function Pe(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Or(e){return ce(e.toString("base64"),40)}function Er(e,t,s){let r=e.assetVariantDigestsByOriginalPath.get(t);if(!r||r.size===0)return false;for(let n of r)if(n!==s)return true;return false}function Fr(e,t,s){let r=e.assetVariantDigestsByOriginalPath.get(t);r||(r=new Set,e.assetVariantDigestsByOriginalPath.set(t,r)),r.add(s);}async function ye(e,t,s,r,n,i,o){let l=Re(e,s),a=Or(r),c=l.originalFilePath,u=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&Er(n,l.originalFilePath,a)){let f=Sr(e,t,s);f?(c=f.outputPath,u=f.publicUrl):(c=l.hashedFilePath,u=l.hashedPublicUrl);}Tr(e,t,s,u,l.originalPublicUrl);let p=U(c)?Buffer.from(fe(r.toString("utf8"),e,t,c),"utf8"):r,d=await lt(c),g=false;return (d===null||!d.equals(p))&&(await de(c),await y.writeFile(c,p),g=true),Fr(n,l.originalFilePath,a),await Dr(c,o),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:c,publicUrl:u,wroteFile:g}}function z(e,t,s,r,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=k(e,s,i,r,"page");if(!o)return;if(!J(o)||at(o)){if(C(e,o)){r.info(`Seeded/non-page URL queued as asset: ${o.toString()}`,{url:o.toString(),source:n}),M(e,t,o.toString(),r,n);return}r.reject("page",o.toString(),se(o)?"asset/internal path cannot be page":"not page-like",n);return}let l=o.toString();!t.donePages.has(l)&&!t.queuedPages.has(l)&&(t.queuedPages.add(l),t.pageQueue.push(l),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&r.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function M(e,t,s,r,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=k(e,s,i,r,"asset");if(!o)return;if(!C(e,o)){r.reject("asset",o.toString(),"not a safe asset path/prefix",n);return}let l=o.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&r.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Ce(e,t,s,r,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=k(e,s,i,r,"sitemap");if(!o)return;if(!ne(o)){r.reject("sitemap",o.toString(),"not sitemap-like",n);return}let l=o.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&r.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function jr(e){let t=j(e),s=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(r=>r[1]);return s.length>0?s.map(r=>{let n=r.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=r.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:ee(i?.[1]??null)}}).filter(r=>!!r):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(r=>({loc:r[1].trim(),lastmod:null})).filter(r=>r.loc)}function $r(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Ur(e){let t=[],s=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let r of e.matchAll(s)){let n=(r[1]||r[2]||r[3]||"").trim();n&&t.push(n);}return t}function Se(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function pt(e,t){let s=[];await y.mkdir(t,{recursive:true});let r=await y.readdir(e,{withFileTypes:true});for(let n of r){let i=h.join(e,n.name),o=h.join(t,n.name);if(n.isDirectory()){s.push(...await pt(i,o));continue}await y.copyFile(i,o),s.push(o);}return s}async function Qr(e,t){let s=Object.entries(e.postCrawlCopyMap||{}).map(([i,o])=>({sourcePath:i.trim(),prefix:String(o||"").trim()}));if(s.length===0)return [];let r=0,n=[];t.mark("copy-extra-paths");for(let i of s){let o=i.sourcePath,l=i.prefix;if(!o||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:o,prefixPath:l});continue}let a=lr(o);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:o,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let c=a.resolvedPath,u=Se(l);if(!u){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:o,prefixPath:l});continue}let p;try{p=await y.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:o,sourceAbs:c,prefixPath:l});continue}let d=h.resolve(e.outputDir,u);if(p.isDirectory()){n.push(...await pt(c,d)),r++,t.progress(`Copied static directory to export: ${c} -> /${u}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${u}`,copiedItems:r});continue}let g=h.basename(c),f=l.endsWith("/")?h.join(d,g):d;await y.mkdir(h.dirname(f),{recursive:true}),await y.copyFile(c,f),n.push(f),r++,t.progress(`Copied static file to export: ${c} -> /${Se(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${Se(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:r});}return t.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:r}),n}function B(e,t,s,r,n){let i=new Set,o=j(s).replace(/\\\//g,"/");for(let c of Ur(o)){let u=k(e,c,t,r,`${n}:css-url`);u&&C(e,u)&&i.add(u.toString());}for(let c of o.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let u=k(e,c[1],t,r,`${n}:css-import`);u&&C(e,u)&&i.add(u.toString());}for(let c of o.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let u=k(e,c[1],t,r,`${n}:xml-stylesheet`);u&&C(e,u)&&i.add(u.toString());}let l="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of o.matchAll(a)){let u=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(ot(u)){r.ignore(`${n}:serialized-url`,u,"looks like JavaScript/code fragment",t);continue}let p=k(e,u,t,r,`${n}:serialized-url`);p&&C(e,p)?i.add(p.toString()):p&&r.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Nr(e,t,s,r){let n=new Set,i=j(s);for(let o of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=o[1],a=k(e,l,t,r,"page-link");a&&(it(e,a)||J(a)&&!se(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function Je(e,t,s,r){for(;s.sitemapQueue.length>0;){let n=s.sitemapQueue.shift();if(s.queuedSitemaps.delete(n),!s.doneSitemaps.has(n)){s.doneSitemaps.add(n),r.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){r.skip("sitemap",n,i.status());continue}let o=await i.body(),l=i.url()||n;await ye(e,s.assetMap,l,o,s,r);let a=o.toString("utf8");for(let c of B(e,l,a,r,"sitemap"))M(e,s,c,r,l);for(let c of jr(a)){let u=k(e,c.loc,l,r,"sitemap-loc");u&&(ne(u)?Ce(e,s,u.toString(),r,l):C(e,u)?M(e,s,u.toString(),r,l):J(u)?(c.lastmod&&(s.sitemapLastmodByPage[u.toString()]=c.lastmod),z(e,s,u.toString(),r,l)):r.reject("sitemap-loc",u.toString(),"not page/sitemap/asset-like",l));}}catch(i){r.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function qr(e,t,s,r,n,i,o){let l=k(e,r,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(s.doneAssets.has(a))return;let c=i.enabled?i.previousManifest.assets[a]:void 0,u=(p,d={})=>{if(!c)return false;i.manifest.assets[a]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},s.doneAssets.add(a);for(let g of c.discoveredAssets||[])M(e,s,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),Pe(s,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=te(e,s.assetMap,a),d=p?await ct(p):null;if(p&&d)try{let P=await t.head(a,{timeout:6e4}),v=P.headers();if(P.ok()&&Ir(d,v)){let R=await lt(p);if(R!==null){let A=P.url()||a,D=await ye(e,s.assetMap,A,R,s,n,v);D.wroteFile&&($(o,e,D.outputPath),U(D.outputPath)&&Z(o,D.outputPath)),s.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:v["last-modified"]||"",contentLength:v["content-length"]||"",phase:"download-assets"}),Pe(s,n);let O=we(l,v),x=c?.discoveredAssets?[...c.discoveredAssets]:[];if(O||c?.isText){let ht=R.toString("utf8"),xe=B(e,A,ht,n,"asset:cached");(xe.length>0||!c)&&(x=xe);for(let ft of x)M(e,s,ft,n,A);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:D.outputPath,isText:O||!!c?.isText,discoveredAssets:x,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(u(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),w=g.url()||a,b=g.headers(),S=await ye(e,s.assetMap,w,f,s,n,b);S.wroteFile&&($(o,e,S.outputPath),U(S.outputPath)&&Z(o,S.outputPath)),s.doneAssets.add(a),Pe(s,n);let m=[];if(we(l,b)){let P=f.toString("utf8");m=B(e,w,P,n,`asset:${h.extname(l.pathname).toLowerCase()||b["content-type"]||"unknown"}`);for(let v of m)M(e,s,v,n,w);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:S.outputPath,isText:we(l,b),discoveredAssets:m,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(p){if(u("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function Wr(e,t,s,r,n,i){let o=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:o},async()=>{for(;s.assetQueue.length>0;){let a=s.assetQueue.shift();a&&(s.queuedAssets.delete(a),await qr(e,t,s,a,r,n,i));}});await Promise.all(l);}async function _r(e){await e.evaluate(async()=>{await new Promise(t=>{let s=0,r=700,n=window.setInterval(()=>{window.scrollBy(0,r),s+=r,s>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Hr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await _r(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function zr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Br(e,t){return e.noJavaScriptRenderPathPrefixes.some(s=>ve(t,s))}async function Jr(e,t,s,r,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],u=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of c){let w=d.getAttribute(f);w&&a.add(w);}for(let f of p){let w=d.getAttribute(f);w&&w.split(",").forEach(b=>{let S=b.trim().split(/\s+/)[0];S&&a.add(S);});}let g=d.getAttribute("style");g&&u.add(g);}),{attrs:[...a],styles:[...u]}});for(let a of l.attrs)for(let c of $r(a)){let u=k(e,c,s,r,"dom-attr");u&&C(e,u)&&i.add(u.toString());}for(let a of l.styles)for(let c of B(e,s,a,r,"dom-style"))i.add(c);}let o=await t.content();for(let l of B(e,s,o,r,"page-html"))i.add(l);return [...i]}async function Ve(e,t,s,r,n){let i=_(e,t);await de(i),await y.writeFile(i,s,"utf8"),r.stats.pagesSaved++,n&&(r.stats.pagesSaved%10===0||r.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${r.donePages.size}, rendered ${r.stats.pagesRendered}, saved ${r.stats.pagesSaved}, discovered ${r.stats.pagesQueued}.`,{donePages:r.donePages.size,pagesRendered:r.stats.pagesRendered,pagesSaved:r.stats.pagesSaved,pagesQueued:r.stats.pagesQueued,pageQueue:r.pageQueue.length,phase:"save-pages"});}function Ge(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ke(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Vr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=s=>{if(s==null||String(s)==="")return true;try{let r=new URL(String(s),window.location.href),n=new URL(window.location.href);return r.hash="",n.hash="",r.href===n.href}catch{return false}};try{let s=window.location.assign.bind(window.location),r=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return s(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return r(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Gr(e,t,s,r,n,i,o){let l=await chromium.launch({headless:true}),a=await l.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await l.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Vr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let u=t.pageQueue.shift();if(!u)break;if(t.queuedPages.delete(u),t.donePages.has(u))continue;t.donePages.add(u),i.enabled&&i.seenPages.add(u);let p=new URL(u),d=ut(e,u),g=Br(e,p.pathname);if(!J(p)||C(e,p)||se(p)){if(C(e,p)){s.info(`Worker redirected non-page URL to asset queue: ${u}`,{url:u,source:"worker-guard"}),M(e,t,u,s,"worker-guard");continue}s.reject("page",u,"guard rejected non-page URL before rendering","worker");continue}let f=await fr(e,n,t,i,u,s);if(i.enabled&&r!=="single-url"){let m=i.manifest.pages[u];if(f.action==="reuse"&&m){for(let P of m.discoveredAssets)M(e,t,P,s,u);for(let P of m.discoveredPages)z(e,t,P,s,u);m.lastSeenRunId=i.runId,m.sitemapLastmod=ee(t.sitemapLastmodByPage[u])??m.sitemapLastmod,f.changeToken?.supported&&(m.changeToken=f.changeToken.token,m.tokenSource=f.changeToken.tokenSource??m.tokenSource),s.info(`Incremental reuse skipped unchanged page ${u}`,{url:u,mode:"incremental",reason:f.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Ge(t)&&Ke(t,s);continue}}let w=await(g?c.newPage():a.newPage()),b=!1,S=new Set;await w.route("**/*",async m=>{let P=m.request();if(P.isNavigationRequest()&&P.frame()===w.mainFrame())try{let v=new URL(P.url()),R=new URL(u);if(v.hash="",R.hash="",v.href===R.href){if(b){s.warn(`Blocked same-page navigation/reload for ${u}`,{url:u,requestUrl:P.url()}),await m.abort("aborted");return}b=!0;}}catch{}await m.continue();}),w.on("response",m=>{try{let P=k(e,m.url(),e.sourceOrigin,void 0,"network-response");P&&C(e,P)&&(S.add(P.toString()),M(e,t,P.toString(),s,u));}catch{}}),s.page(`Rendering ${u}`,{url:u});try{let m=null,P=null;try{if(m=await w.goto(u,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{P=await m.text();}catch{P=null;}}catch(x){s.warn(`Navigation issue for ${u}; saving current DOM if available`,{url:u,error:String(x)});}if(d){let x=m?.status()??null;if(x!==404){s.error(`Generated 404 request path returned unexpected status for ${u}`,{url:u,expectedStatus:404,status:x,requestPath:e.generated404RequestPath});continue}s.info(`Capturing rendered 404 page for ${u}`,{url:u,status:x,outputPath:_(e,u)});}else if(m&&!m.ok()){s.skip("page",u,m.status()),await w.close();continue}g?(s.info(`Rendering without JS execution for ${u}`,{url:u,mode:"no-js"}),await zr(e,w)):await Hr(e,w),t.stats.pagesRendered++,Ge(t)&&Ke(t,s);let v=await Jr(e,w,u,s,!g);for(let x of v)M(e,t,x,s,u);let R=[...new Set([...v,...S])],A=P??await w.content(),D=r!=="single-url"&&!d?Nr(e,u,A,s):[];if(r!=="single-url"&&!d)for(let x of D)z(e,t,x,s,u);await Ve(e,u,A,t,s);let O=_(e,u);$(o,e,O),Z(o,O),i.trackManifest&&(i.manifest.pages[u]={url:u,outputPath:O,changeToken:f.changeToken?.supported?f.changeToken.token:null,tokenSource:f.changeToken?.supported?f.changeToken.tokenSource??null:null,sitemapLastmod:ee(t.sitemapLastmodByPage[u])??null,discoveredPages:D,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let P=await w.content();if(P&&P.trim()){await Ve(e,u,P,t,s);let v=_(e,u);$(o,e,v),Z(o,v),s.warn(`Saved partial DOM for ${u}`,{url:u,error:String(m)});}}catch(P){s.error(`Could not save partial DOM for ${u}`,{url:u,error:String(P)});}s.error(`Failed page ${u}`,{url:u,error:String(m)});}finally{await w.close();}}}finally{await c.close(),await a.close(),await l.close();}}async function Kr(e){try{let t=await y.readFile(e),r=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!r)return [];try{let n=JSON.parse(r);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return r.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ye(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Yr(e){let t=String(e?.job?.command||"").trim(),s=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:s!=="incremental"}function Xr(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let s of t){let r=Date.parse(String(s||""));if(Number.isFinite(r))return r}return 0}async function gt(e){try{let t=await y.readFile(e,"utf8"),s=JSON.parse(t);return s&&typeof s=="object"&&!Array.isArray(s)?s:null}catch{return null}}function Xe(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Zr(e,t){let s=Array.isArray(t?.artifacts)?t.artifacts.filter(r=>String(r?.role||"").trim()==="errors"&&String(r?.storedFileName||"").trim()!=="").sort((r,n)=>Xe(r)-Xe(n)).map(r=>h.join(e,String(r.storedFileName))):[];return [...new Set([...s,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function Ze(e){let t=await gt(h.join(e,"job.json"));for(let s of Zr(e,t)){let r=await Kr(s);if(r.length>0)return r}return []}async function es(e){let t=h.join(e.logDir,"archive"),s;try{s=await y.readdir(t);}catch{return ""}let r="",n=0;for(let i of s){let o=h.join(t,i);if(!(await y.stat(o).catch(()=>null))?.isDirectory())continue;let a=await gt(h.join(o,"job.json"));if(!Yr(a))continue;let c=Xr(a);c>=n&&(n=c,r=o);}return r}async function ts(e){let t=await es(e),s=t!==""?await Ze(t):await Ze(e.logDir),r=new Set;for(let n of s)if(Ye(n.error)||Ye(n.message)){let i=n.url;typeof i=="string"&&i&&r.add(i);}return [...r]}async function rs(e){let t=[];for(let s of e)if(s.startsWith("@")){let r=s.slice(1),n=await y.readFile(r,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(s);return t}async function ss(e){try{let t=await y.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ns(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async s=>{try{await y.unlink(h.join(e,s));}catch(r){if(r.code!=="ENOENT")throw r}}));}function et(e,t,s,r,n){let i=k(e,s,e.sourceOrigin,r,"manual-url");i&&(ne(i)?Ce(e,t,i.toString(),r,n):C(e,i)?M(e,t,i.toString(),r,n):J(i)?z(e,t,i.toString(),r,n):r.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function is(){let e=await le(),t=or(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let s=[];t.mode==="retry-timeouts"&&(s=await ts(e)),t.mode==="single-url"&&(s=await rs(t.urls));let r=t.crawlMode==="incremental"&&t.mode==="full",n=r&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,o=t.mode!=="full"||t.resumeRewrite||n,l=o?await ss(e):{},a=n?await dr(e):st(),c={enabled:n,trackManifest:i,manifest:_e(a),previousManifest:_e(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await ns(e.logDir);let u={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=cr();o||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let d=new W(e.logDir,e.logLevel);if(r&&!n&&d.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)d.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let S=await chromium.launch({headless:true}),m=await S.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),P=m.request;try{if(d.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(R=>Ce(e,u,R,d)),await Je(e,P,u,d),e.seedPaths.forEach(R=>et(e,u,R,d,"seed-path")),e.generated404RequestPath&&z(e,u,e.generated404RequestPath,d,"generated-404-request-path")):(s.forEach(R=>et(e,u,R,d,"cli")),await Je(e,P,u,d)),d.endMark("discovery",{pages:u.pageQueue.length,assets:u.assetQueue.length,sitemaps:u.sitemapQueue.length}),d.summary(`Queued ${u.pageQueue.length} pages, ${u.assetQueue.length} assets, ${u.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:u.pageQueue.length,queuedAssets:u.assetQueue.length,queuedSitemaps:u.sitemapQueue.length}),t.mode==="retry-timeouts"&&u.pageQueue.length===0&&u.assetQueue.length===0&&u.sitemapQueue.length===0){d.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await d.flush();return}d.mark("render-pages");let v=Array.from({length:Math.max(1,e.concurrency)},()=>Gr(e,u,d,t.mode,P,c,p));if(await Promise.all(v),d.endMark("render-pages",{pages:u.donePages.size}),d.mark("download-assets"),await Wr(e,P,u,d,c,p),d.endMark("download-assets",{assets:u.doneAssets.size}),c.enabled&&e.maxPages===0&&await mr(e,u,c,d,p),t.mode==="full"){let R=await Qr(e,d);for(let A of R)$(p,e,A);}}finally{await m.close(),await S.close();}}c.trackManifest&&(c.manifest.updatedAt=new Date().toISOString(),await pr(e,c.manifest)),await y.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(u.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(u.assetMap,null,2)&&$(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?vr(e,u):c.enabled&&!t.resumeRewrite?wr(e,c,p,l,u.assetMap):void 0;d.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";d.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await Be(f);let w=Date.now(),b=await Ue(e,u.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:S,totalFiles:m,changedTextFiles:P,file:v,changed:R})=>{R&&$(p,e,h.join(e.outputDir,v)),d.checkpoint(`Rewriting text file ${S}/${m}`,{phase:"rewrite-text",index:S,totalFiles:m,changedTextFiles:P,file:v});let A=Date.now();if(!(S===1||S===m||A-w>=5e3))return;w=A;let O=`Rewriting text files: ${S}/${m}`;d.progress(O,{phase:"rewrite-text",index:S,totalFiles:m,changedTextFiles:P,file:v}),await Be(O);}});d.endMark("rewrite-text",{changedTextFiles:b}),await Me(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(c.enabled&&!t.resumeRewrite),changedFiles:c.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:c.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:c.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(S=>V(e.outputDir,h.isAbsolute(S)?S:h.join(e.outputDir,S))).filter(S=>S!==null))].sort():[]}),d.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${b} text files.`:`Done. Rendered ${u.stats.pagesRendered} pages, processed ${u.doneSitemaps.size} sitemaps, downloaded ${u.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...u.stats,pagesRendered:u.stats.pagesRendered,donePages:u.donePages.size,doneSitemaps:u.doneSitemaps.size,doneAssets:u.doneAssets.size,changedTextFiles:b}),await d.flush();}is().catch(async e=>{console.error(e);try{let s=(await le().catch(()=>null))?.logDir??"logs",r=new W(s,"debug");r.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await r.flush();}catch{}process.exit(1);});
|
package/dist/deploy.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {S3Client,ListObjectsV2Command,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {createHash}from'crypto';import nt from'fast-glob';import $ from'fs/promises';import S from'path';import $t from'mime-types';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ge(t){let r=t.trim();return r==="."?".":r.replace(/\/$/,"")}function Je(t){let r=String(t??"").trim();if(!r)return "";let e=r;if(/^https?:\/\//i.test(e))try{e=new URL(e).pathname;}catch{return ""}if(e=e.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!e)return "";let n=e.endsWith("/"),s=e.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(s.length===0)return "";let i=`/${s.join("/")}`;return n&&S.extname(i)===""&&(i+="/"),i==="/"?"":i}function Q(t){return !t||typeof t!="object"?{}:Object.fromEntries(Object.entries(t).map(([r,e])=>[r.trim(),String(e??"")]).filter(([r])=>r.length>0))}function Ke(t){if(!t||typeof t!="object")return {};let r={};for(let[e,n]of Object.entries(t)){let s=e.trim();if(!s||!n||typeof n!="object")continue;let i=n,o={};if(typeof i.targetOrigin=="string"){let a=ge(i.targetOrigin);a&&(o.targetOrigin=a);}let u=Q(i.extraReplacements);if(Object.keys(u).length>0&&(o.extraReplacements=u),i.s3&&typeof i.s3=="object"){let a={},l=i.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},l=i.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}r[s]=o;}return r}function V(t,r){let n=String(t||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return r;let s=n.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return s.length>0?s.join("/"):r}function Be(){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return t.trim()?S.resolve(t):""}function ue(t,r,e){let n=String(r||"").trim();return n&&S.isAbsolute(n)?S.resolve(n):S.resolve(t,V(n,e))}async function G(){let t=process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await $.readFile(t,"utf8"),e=JSON.parse(r);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=ge(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=Je(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=Q(e.extraReplacements),e.postCrawlCopyMap=Q(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.scheduler||={enabled:false,timezone:"UTC",rules:[]},e.scheduler.enabled??=false,e.scheduler.timezone||="UTC",e.scheduler.rules||=[],e.deploymentProfiles=Ke(e.deploymentProfiles),e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.defaultDeploymentProfile=String(e.defaultDeploymentProfile??"").trim(),e.defaultDeploymentProfile&&!e.deploymentProfiles[e.defaultDeploymentProfile]&&(e.defaultDeploymentProfile="");let n=Be(),s=n?S.resolve(n,".."):"";return s?(e.outputDir=ue(s,e.outputDir,"export"),e.logDir=ue(s,e.logDir,"logs")):(S.isAbsolute(e.outputDir)||(e.outputDir=V(e.outputDir,"export")),S.isAbsolute(e.logDir)||(e.logDir=V(e.logDir,"logs"))),e}var qe=".deploy-plan.json",We="deploy-plan.json";function Qe(){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return t.trim()?S.resolve(t):""}function Ve(t){let r=String(t||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !r||r==="."||r.startsWith("../")||r===".."?null:r}function Z(t){return [...new Set(t.map(Ve).filter(r=>r!==null))]}function Ge(t,r){return {schemaVersion:1,generatedAt:String(r.generatedAt||new Date().toISOString()),outputDir:S.resolve(t.outputDir),runMode:r.runMode,crawlMode:r.crawlMode,fullSyncRequired:!!r.fullSyncRequired,changedFiles:Z(r.changedFiles||[]),deletedFiles:Z(r.deletedFiles||[]),rewriteTargets:Z(r.rewriteTargets||[])}}function Ze(t){if(!t||typeof t!="object")return false;let r=t;return r.schemaVersion===1&&typeof r.generatedAt=="string"&&typeof r.outputDir=="string"&&(r.runMode==="full"||r.runMode==="retry-timeouts"||r.runMode==="single-url")&&(r.crawlMode==="full"||r.crawlMode==="incremental")&&typeof r.fullSyncRequired=="boolean"&&Array.isArray(r.changedFiles)&&Array.isArray(r.deletedFiles)&&Array.isArray(r.rewriteTargets)}function pe(t){let r=S.join(S.resolve(t.outputDir),qe),e=Qe();return e?[S.join(e,We),r]:[r]}async function de(t){let r=S.resolve(t.outputDir);for(let e of pe(t))try{let n=await $.readFile(e,"utf8"),s=JSON.parse(n);if(!Ze(s)||S.resolve(s.outputDir)!==r)continue;return Ge(t,s)}catch{}return null}async function fe(t){for(let r of pe(t))try{await $.unlink(r);}catch(e){if(e.code!=="ENOENT")throw e}}function me(t){let r="";for(let n=0;n<t.length;n++){let s=t[n];if(s==="--profile"){let i=(t[n+1]||"").trim();if(!i)throw new Error("Missing value for --profile.");r=i,n+=1;}else if(s.startsWith("--profile=")){let i=s.slice(10).trim();if(!i)throw new Error("Missing value for --profile.");r=i;}}let e=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return r||e||null}function Ye(t,r){return {...t,...r??{}}}function Xe(t,r){return {...t,...r??{},invalidationPaths:[...(r?.invalidationPaths??t.invalidationPaths)||[]]}}function he(t,r){let e=String(r??t.defaultDeploymentProfile??"").trim();if(!e)return {name:null,profile:null,config:t};let n=t.deploymentProfiles?.[e];if(!n)throw new Error(`Unknown deployment profile "${e}". Check deploymentProfiles in publisher.config.json.`);return {name:e,profile:n,config:{...t,targetOrigin:n.targetOrigin??t.targetOrigin,s3:Ye(t.s3,n.s3),cloudFront:Xe(t.cloudFront,n.cloudFront)}}}var we={error:0,warn:1,info:2,debug:3},ye=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",H=ye?S.join(ye,"current-progress.json"):"",Pe=Promise.resolve();function be(t){return t.includes("crawl")?"crawl":t.includes("deploy")?"deploy":t.includes("invalidate")?"invalidate":t}function et(t,r){let e={...t},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[s,i]of Object.entries(r)){let o=e[s];if(typeof o=="number"&&typeof i=="number"&&n.has(s)){e[s]=Math.max(o,i);continue}e[s]=i;}return e}function tt(t,r,e){H&&(Pe=Pe.then(async()=>{let n=new Date,s=n.toISOString();await $.mkdir(S.dirname(H),{recursive:true});let i=await $.readFile(H,"utf8").then(b=>JSON.parse(b)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},u=be(t),a=i?.currentStep||be(i?.source||""),l=i?.startedAt||s,g={...i?.stepDurationsSec??{}},m=i?.stepStartedAt||s;if(a&&a!==u&&i?.stepStartedAt){let b=Math.max(0,Math.round((n.getTime()-new Date(i.stepStartedAt).getTime())/1e3));g[a]=(g[a]??0)+b,m=s;}let c=Math.max(0,Math.round((n.getTime()-new Date(m).getTime())/1e3)),w=Math.max(0,Math.round((n.getTime()-new Date(l).getTime())/1e3)),P={checkedAt:s,source:t,message:r,details:et(o,e??{}),startedAt:l,currentStep:u,stepStartedAt:m,stepElapsedSec:c,totalElapsedSec:w,stepDurationsSec:g};await $.writeFile(H,JSON.stringify(P,null,2),"utf8");}).catch(()=>{}));}async function rt(t,r){await $.mkdir(S.dirname(t),{recursive:true}),await $.appendFile(t,`${JSON.stringify(r)}
|
|
2
|
-
`,"utf8");}var A=class{constructor(r,
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(r
|
|
1
|
+
import {S3Client,ListObjectsV2Command,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {createHash}from'crypto';import ft from'fast-glob';import D from'fs/promises';import v from'path';import jt from'mime-types';import {getConfig}from'@smart-cloud/wpsuite-core';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ge(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Ke(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let n=r.endsWith("/"),s=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(s.length===0)return "";let i=`/${s.join("/")}`;return n&&v.extname(i)===""&&(i+="/"),i==="/"?"":i}function Q(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Qe(e){if(!e||typeof e!="object")return {};let t={};for(let[r,n]of Object.entries(e)){let s=r.trim();if(!s||!n||typeof n!="object")continue;let i=n,o={};if(typeof i.targetOrigin=="string"){let a=ge(i.targetOrigin);a&&(o.targetOrigin=a);}let u=Q(i.extraReplacements);if(Object.keys(u).length>0&&(o.extraReplacements=u),i.s3&&typeof i.s3=="object"){let a={},l=i.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},l=i.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[s]=o;}return t}function pe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Ve(e,t){if(!e||typeof e!="object")return null;let r=e,n=r.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let s=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(s)||s<1)return null;let i=String(r.id??`${n}-${t+1}`).trim();if(!i)return null;let o=String(r.deploymentProfile??"").trim(),u=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:n,intervalMinutes:s,...n==="publish"||n==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&o?{deploymentProfile:o}:{},...u?{url:u}:{}}}function de(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((n,s)=>Ve(n,s)).filter(n=>!!n):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Ge(e){if(!e||typeof e!="object")return;let t=e,r={};for(let s of ["accountId","siteId"]){let i=t[s];typeof i=="string"&&i.trim()!==""&&(r[s]=i.trim());}let n=Number(t.lastUpdate??0);return Number.isFinite(n)&&n>0&&(r.lastUpdate=Math.floor(n)),r.subscriber=t.subscriber===true,Object.keys(r).length>0?r:void 0}function G(e){let t=e&&typeof e=="object"?e:{},r=Ge(t.siteSettings),n={subscriber:t.subscriber===true||r?.subscriber===true};for(let i of ["accountId","siteId","apiBase","nonce","uploadUrl"]){let o=t[i];typeof o=="string"&&o.trim()!==""&&(n[i]=o.trim());}r&&(n.siteSettings=r);let s=pe(t.subscriptionType);return s&&(n.subscriptionType=s),n}function Ye(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Ze(e){let t=G(e.wpsuite),r={...t.siteSettings??{},accountId:String(t.siteSettings?.accountId??t.accountId??"").trim()||void 0,siteId:String(t.siteSettings?.siteId??t.siteId??"").trim()||void 0,subscriber:t.siteSettings?.subscriber===true||t.subscriber===true},n=String(t.accountId??r.accountId??"").trim(),s=String(t.siteId??r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t,...n?{accountId:n}:{},...s?{siteId:s}:{},siteSettings:r},!n||!s||!i)return false;Ye(e.sourceOrigin);let o=globalThis,a={...o.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),t.nonce&&(a.nonce=t.nonce),o.WpSuite=a,true}async function Xe(e){if(!Ze(e))return null;try{let t=await getConfig("publisher");return t&&typeof t=="object"?t:null}catch{return null}}function et(e,t){let r=Qe(t?.deploymentProfiles),n=String(t?.defaultDeploymentProfile??"").trim(),s=r[n]?n:"",i=pe(t?.subscriptionType),o=G(e.wpsuite);return {...e,scheduler:de(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:s,...i?{subscriptionType:i}:{},wpsuite:{...o,subscriber:o.subscriber===true||!!i,...i?{subscriptionType:i}:{}}}}function V(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let s=n.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return s.length>0?s.join("/"):t}function tt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?v.resolve(e):""}function ce(e,t,r){let n=String(t||"").trim();return n&&v.isAbsolute(n)?v.resolve(n):v.resolve(e,V(n,r))}async function Y(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await D.readFile(t,"utf8"),n=JSON.parse(r);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=ge(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=Ke(n.generated404RequestPath),n.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=Q(n.extraReplacements),n.postCrawlCopyMap=Q(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=G(n.wpsuite),n.scheduler=de(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let s=et(n,await Xe(n)),i=tt(),o=i?v.resolve(i,".."):"";return o?(s.outputDir=ce(o,s.outputDir,"export"),s.logDir=ce(o,s.logDir,"logs")):(v.isAbsolute(s.outputDir)||(s.outputDir=V(s.outputDir,"export")),v.isAbsolute(s.logDir)||(s.logDir=V(s.logDir,"logs"))),s}var rt=".deploy-plan.json",nt="deploy-plan.json";function it(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?v.resolve(e):""}function st(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function Z(e){return [...new Set(e.map(st).filter(t=>t!==null))]}function ot(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:v.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:Z(t.changedFiles||[]),deletedFiles:Z(t.deletedFiles||[]),rewriteTargets:Z(t.rewriteTargets||[])}}function at(e){if(!e||typeof e!="object")return false;let t=e;return t.schemaVersion===1&&typeof t.generatedAt=="string"&&typeof t.outputDir=="string"&&(t.runMode==="full"||t.runMode==="retry-timeouts"||t.runMode==="single-url")&&(t.crawlMode==="full"||t.crawlMode==="incremental")&&typeof t.fullSyncRequired=="boolean"&&Array.isArray(t.changedFiles)&&Array.isArray(t.deletedFiles)&&Array.isArray(t.rewriteTargets)}function me(e){let t=v.join(v.resolve(e.outputDir),rt),r=it();return r?[v.join(r,nt),t]:[t]}async function he(e){let t=v.resolve(e.outputDir);for(let r of me(e))try{let n=await D.readFile(r,"utf8"),s=JSON.parse(n);if(!at(s)||v.resolve(s.outputDir)!==t)continue;return ot(e,s)}catch{}return null}async function we(e){for(let t of me(e))try{await D.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function lt(e){return typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!==""}function ye(e){let t="";for(let n=0;n<e.length;n++){let s=e[n];if(s==="--profile"){let i=(e[n+1]||"").trim();if(!i)throw new Error("Missing value for --profile.");t=i,n+=1;}else if(s.startsWith("--profile=")){let i=s.slice(10).trim();if(!i)throw new Error("Missing value for --profile.");t=i;}}let r=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||r||null}function ut(e,t){return {...e,...t??{}}}function ct(e,t){return {...e,...t??{},invalidationPaths:[...(t?.invalidationPaths??e.invalidationPaths)||[]]}}function be(e,t){let r=String(t??e.deploymentTargetOverride??"").trim();if(!lt(e)){if(r)throw new Error(`Deployment target "${r}" requires an active WPSuite subscription and remote publisher config.`);return {name:null,profile:null,config:e}}let n=String(r||e.defaultDeploymentProfile||"").trim();if(!n)return {name:null,profile:null,config:e};let s=e.deploymentProfiles?.[n];if(!s)throw new Error(`Unknown deployment profile "${n}". Check the linked WPSuite publisher configuration.`);return {name:n,profile:s,config:{...e,targetOrigin:s.targetOrigin??e.targetOrigin,s3:ut(e.s3,s.s3),cloudFront:ct(e.cloudFront,s.cloudFront)}}}var Pe={error:0,warn:1,info:2,debug:3},Se=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",W=Se?v.join(Se,"current-progress.json"):"",ve=Promise.resolve();function xe(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function gt(e,t){let r={...e},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[s,i]of Object.entries(t)){let o=r[s];if(typeof o=="number"&&typeof i=="number"&&n.has(s)){r[s]=Math.max(o,i);continue}r[s]=i;}return r}function pt(e,t,r){W&&(ve=ve.then(async()=>{let n=new Date,s=n.toISOString();await D.mkdir(v.dirname(W),{recursive:true});let i=await D.readFile(W,"utf8").then(P=>JSON.parse(P)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},u=xe(e),a=i?.currentStep||xe(i?.source||""),l=i?.startedAt||s,c={...i?.stepDurationsSec??{}},m=i?.stepStartedAt||s;if(a&&a!==u&&i?.stepStartedAt){let P=Math.max(0,Math.round((n.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+P,m=s;}let g=Math.max(0,Math.round((n.getTime()-new Date(m).getTime())/1e3)),w=Math.max(0,Math.round((n.getTime()-new Date(l).getTime())/1e3)),b={checkedAt:s,source:e,message:t,details:gt(o,r??{}),startedAt:l,currentStep:u,stepStartedAt:m,stepElapsedSec:g,totalElapsedSec:w,stepDurationsSec:c};await D.writeFile(W,JSON.stringify(b,null,2),"utf8");}).catch(()=>{}));}async function dt(e,t){await D.mkdir(v.dirname(e),{recursive:true}),await D.appendFile(e,`${JSON.stringify(t)}
|
|
2
|
+
`,"utf8");}var A=class{constructor(t,r,n="info"){this.logDir=t;this.logFile=r;this.level=n;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return v.join(this.logDir,this.logFile)}get errorsPath(){return v.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await D.mkdir(this.logDir,{recursive:true}),await D.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await D.appendFile(this.logPath,`${t}
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>dt(t,r));}accepts(t){return Pe[t]<=Pe[this.level]}push(t,r,n){if(!this.accepts(t))return;let s=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...n||{}});this.enqueueLine(s),t==="error"?console.error(`[ERROR] ${r}`,n?JSON.stringify(n):""):t==="warn"?console.warn(`[WARN] ${r}`,n?JSON.stringify(n):""):console.log(`[${t.toUpperCase().padEnd(5)}] ${r}`);}info(t,r){this.push("info",t,r);}progress(t,r){pt(this.logFile,t,r);}debug(t,r){this.push("debug",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let n=this.marks.get(t);if(!n)return;let s=Date.now()-n,i=Number((s/1e3).toFixed(2));this.push("info",`Timing ${t}: ${i}s`,{name:t,ms:s,seconds:i,...r||{}});}async flush(){let t=Date.now()-this.startedAt,r=Number((t/1e3).toFixed(2));if(this.push("info",`Total time: ${r}s`,{name:"total",ms:t,seconds:r}),await this.writeQueue,this.writeError)throw this.writeError}};function De(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(v.extname(e).toLowerCase())}function $e(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function k(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function B(e){return e.replace(/\//g,"\\/")}function Re(e){return e.replace(/\//g,"\\\\/")}function H(e,t,r){if(!t)return;e[t]=r;let n=B(t),s=B(r);e[n]=s;let i=Re(t),o=Re(r);e[i]=o;let u=k(t),a=k(r);e[u]=a;let l=k(n),c=k(s);e[l]=c;let m=k(i),g=k(o);e[m]=g;}function Ce(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ee(e,t,r){if(!t)return r;let n=v.dirname(v.resolve(t)),s=v.resolve(e,r.replace(/^\/+/,"")),i=v.relative(n,s).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var wt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],yt=new Set([".html",".htm"]),Me="WPSuite.io Static Publisher",bt=Me.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Pt(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let n=v.extname(r).toLowerCase();return yt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function St(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${bt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Me}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let n=e.includes(`\r
|
|
4
4
|
`)?`\r
|
|
5
5
|
`:`
|
|
6
|
-
`;return t.replace(/<head\b[^>]*>/i,s=>`${s}${n} ${r}`)}function ct(t){let r=xe(t).trim();return r.startsWith("{")||r.startsWith("[")||r.includes("\\/")||/"@context"|"@type"/.test(r)}function pt(t,r){let e=[...new Set(r)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!e.length)return t;let n=i=>{let o=i;for(let[u,a]of e)o=o.split(u).join(a);return o},s=t.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,u,a)=>`${o}${n(u)}${a}`);return s=s.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,u,a,l)=>ct(a)?`${o}${n(a)}${l}`:i),s}function dt(t,r){if(!r)return null;let e=S.dirname(S.resolve(r)),n=S.resolve(t.outputDir),s=S.relative(e,n).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s.endsWith("/")?s:`${s}/`):"./"}function ft(t,r,e){let n=dt(r,e);if(!n)return t;let s=t;for(let i of ot){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");s=s.replace(u,`${n}${i}`).replace(a,`${n}${i}`);let l=i.replace(/\//g,"\\/"),g=n.replace(/\//g,"\\/"),m=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}\\\\/)+${m}`,"g"),w=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${m}`,"g"),P=new RegExp(`(?<![\\.\\\\])\\/${m}`,"g");s=s.replace(c,`${g}${l}`).replace(w,`${g}${l}`).replace(P,`${g}${l}`);}return s}function X(t,r,e){if(t.urlRewriteMode==="absolute")return r;let n=(()=>{try{return new URL(r,t.targetOrigin==="."?"https://relative.invalid":t.targetOrigin)}catch{return null}})(),s=n?n.pathname:De(r,t.targetOrigin),i=n?`${n.search}${n.hash}`:"";return t.urlRewriteMode==="root-relative"?`${s}${i}`:`${$e(t.outputDir,e,s)}${i}`}function Y(t,r,e,n,s){K(t,e,X(r,n,s));}function mt(t,r,e,n,s){if(!e||e===n)return;K(t,e,n);let i=X(r,e,s),o=X(r,n,s);K(t,i,o);}function ht(t,r,e,n,s={}){let i=t,o={};Y(o,r,r.sourceOrigin,r.targetOrigin,n);for(let[a,l]of Object.entries(r.extraReplacements))Y(o,r,a,l,n);for(let[a,l]of Object.entries(e))Y(o,r,a,l,n);for(let[a,l]of Object.entries(s)){let g=e[a];!g||g===l||mt(o,r,l,g,n);}let u=Object.entries(o).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of u)i=i.split(a).join(l);return r.urlRewriteMode!=="absolute"&&(i=ft(i,r,n)),i=pt(i,u.map(([,a])=>a)),ut(r,i,n)&&(i=gt(i)),i}async function wt(t,r,e,n){let s=S.join(t.outputDir,n),i=S.extname(s).toLowerCase(),o;try{o=await $.readFile(s,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=ht(o,t,r,i===".js"||i===".mjs"?void 0:s,e);return a!==o?(await $.writeFile(s,a,"utf8"),{changed:true}):{changed:false}}async function yt(t,r){return await new Promise((e,n)=>{let s=a=>{if(u(),a?.error){n(new Error(a.error));return}e({changed:!!a?.changed});},i=a=>{u(),n(a);},o=a=>{u(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},u=()=>{t.off("message",s),t.off("error",i),t.off("exit",o);};t.on("message",s),t.on("error",i),t.on("exit",o),t.postMessage({file:r});})}async function Ee(t,r,e={}){let s=(e.files?[...new Set(e.files)].map(c=>S.resolve(c)).filter(c=>c.startsWith(S.resolve(t.outputDir))).map(c=>S.relative(t.outputDir,c).replace(/\\/g,"/")).filter(c=>c.length>0):await nt(["**/*"],{cwd:t.outputDir,onlyFiles:true,dot:true})).filter(c=>c!=="asset-map.json"&&Se(c)),i=0;if(s.length===0)return 0;let o=Math.max(1,Number(t.rewriteConcurrency||t.assetDownloadConcurrency||1)),u=Math.max(1,Math.min(o,s.length,availableParallelism())),a=0,l=0,g=null;if(u===1){for(let c of s){let w=await wt(t,r,e.previousAssetMap??{},c);w.changed&&(i+=1),l+=1,await e.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:c,changed:w.changed});}return i}let m=Array.from({length:u},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:t,assetMap:r,previousAssetMap:e.previousAssetMap??{}}});try{for(;!g;){let w=a;if(w>=s.length)return;a+=1;let P=s[w],b=await yt(c,P);b.changed&&(i+=1),l+=1,await e.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:P,changed:b.changed});}}catch(w){throw g=w instanceof Error?w:new Error(String(w)),g}finally{await c.terminate().catch(()=>{});}});return await Promise.all(m),i}var Le="wpsuite-sha256",Fe="wpsuite-normalized-sha256",Rt=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,oe=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;function Ct(t){return t&&(oe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{8,}$/.test(t)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(t)?"__alnum__":t)}function re(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:Ct(r)).join("")}function Et(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:oe.test(r)?"__uuid__":/^[0-9a-f]{6,}$/i.test(r)?"__hex__":/^[0-9]{6,}$/.test(r)?"__num__":/^[a-z0-9_-]{6,}$/i.test(r)?"__id__":r).join("")}function B(t,r){return Array.isArray(t)?t.map(e=>B(e,r)):t&&typeof t=="object"?Object.fromEntries(Object.entries(t).map(([e,n])=>[e,B(n,e)])):typeof t!="string"?t:r&&Rt.test(r)?Et(t):oe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)||/^[0-9]{8,}$/.test(t)?re(t):t}function Mt(t){return t.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function Lt(t,r){let e=r.trim();if((e.startsWith("{")||e.startsWith("[")||e.startsWith('"'))&&t.startsWith("data-"))try{let s=JSON.parse(e);return JSON.stringify(B(s))}catch{}if(t.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(e))try{let s=Buffer.from(e,"base64").toString("utf8"),i=JSON.parse(s),o=JSON.stringify(B(i));return Buffer.from(o,"utf8").toString("base64")}catch{}return r}function Ft(t,r){let e=t.toLowerCase();return e==="class"?r.split(/(\s+)/).map((n,s)=>s%2===1?n:re(n)).join(""):e==="id"||e==="for"||e==="aria-controls"||e==="aria-labelledby"||e==="aria-describedby"||e==="aria-owns"||e==="aria-activedescendant"||e==="name"?re(r):e==="href"||e==="src"?Mt(r):Lt(e,r)}function kt(t){return t.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(r,e,n)=>{if(e.startsWith("/"))return r;let s=n.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(i,o,u,a)=>{let l=Ft(o,a);return ` ${o}=${u}${l}${u}`});return `<${e}${s}>`})}function ee(t){let r=S.extname(t).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(r)||["robots.txt","sitemap.xml"].includes(S.basename(t))}function te(t,r){return [t.replace(/^\/+|\/+$/g,""),r.replace(/^\/+/,"")].filter(Boolean).join("/")}async function jt(t,r,e,n){let s=new Map,i,o=0;do{o++;let u=await t.send(new ListObjectsV2Command({Bucket:r,Prefix:e||void 0,ContinuationToken:i}));for(let a of u.Contents||[]){if(!a.Key)continue;let l=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),g=Number(a.Size??0);s.set(a.Key,{etag:l,size:g});}i=u.NextContinuationToken,n.debug(`Listed S3 keys page ${o}: ${s.size} keys so far`);}while(i);return s}async function It(t,r,e,n){if(n.has(e))return n.get(e)??null;try{let s=await t.send(new HeadObjectCommand({Bucket:r,Key:e})),i={etag:(s.ETag||"").replace(/^"|"$/g,"").toLowerCase(),size:Number(s.ContentLength??0)};return n.set(e,i),i}catch{return n.set(e,null),null}}async function ke(t,r,e,n){let s=[...new Set(Array.from(e).filter(Boolean))];if(s.length===0)return n.info("No stale S3 objects to delete."),[];n.info(`Deleting ${s.length} stale S3 object(s)\u2026`);for(let i=0;i<s.length;i+=1e3){let o=s.slice(i,i+1e3);if(o.length!==0){await t.send(new DeleteObjectsCommand({Bucket:r,Delete:{Objects:o.map(u=>({Key:u}))}})),n.info(`Deleted batch of ${o.length} object(s).`);for(let u of o)n.debug(` deleted: ${u}`);}}return s}async function Ot(t,r,e,n,s){return ke(t,r,Array.from(e).filter(i=>!n.has(i)),s)}function Tt(t){return createHash("md5").update(t).digest("hex")}function Me(t){return createHash("sha256").update(t).digest("hex")}function ne(t){return createHash("sha256").update(kt(t.toString("utf8"))).digest("hex")}async function _t(t,r,e,n){if(n.has(e))return n.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[Le]||"").trim().toLowerCase()||null;return n.set(e,o),o}catch{return n.set(e,null),null}}async function At(t,r,e,n,s){if(n.has(e))return n.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[Fe]||"").trim().toLowerCase();if(o)return n.set(e,o),o}catch{return n.set(e,null),null}if(s.has(e)){let i=s.get(e)??null;return n.set(e,i),i}try{let o=await(await t.send(new GetObjectCommand({Bucket:r,Key:e}))).Body?.transformToByteArray(),u=o?ne(Buffer.from(o)):null;return s.set(e,u),n.set(e,u),u}catch{return s.set(e,null),n.set(e,null),null}}function Nt(t){return t.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function zt(t,r){let e=t,n=Object.entries(r).sort((s,i)=>i[0].length-s[0].length);for(let[s,i]of n)s&&(e=e.split(s).join(i));return e}async function Ut(t,r,e,n){let s=S.join(t,"asset-map.json"),i;try{i=await $.readFile(s,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(i);if(!o||typeof o!="object")return;let u=Object.fromEntries(Object.entries(o).map(([a,l])=>{let g=String(l??"");return r.urlRewriteMode==="absolute"&&r.targetOrigin&&e.targetOrigin&&r.targetOrigin!==e.targetOrigin&&(g=g.split(r.targetOrigin).join(e.targetOrigin)),g=zt(g,n),[a,g]}));await $.writeFile(s,JSON.stringify(u,null,2),"utf8");}async function Ht(t,r,e,n,s){let i=t.targetOrigin!==r.targetOrigin,o=Object.keys(n).length>0;if(!e||!i&&!o)return {outputDir:t.outputDir,cleanup:async()=>{}};if(i&&t.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${e}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let u=S.join(S.dirname(t.outputDir),`.deploy-profile-${Nt(e)}-${Date.now()}`);s.info(`Preparing deployment profile "${e}" from existing crawl output...`),await $.rm(u,{recursive:true,force:true}),await $.cp(t.outputDir,u,{recursive:true});let a={...t,sourceOrigin:t.targetOrigin,targetOrigin:r.targetOrigin,extraReplacements:n,outputDir:u},l=await Ee(a,{});return await Ut(u,t,r,n),s.info(`Prepared deployment profile "${e}" in ${u} (${l} rewritten text file(s)).`),{outputDir:u,cleanup:async()=>{await $.rm(u,{recursive:true,force:true});}}}async function Jt(){let t=await G(),r=me(process.argv.slice(2)),e=he(t,r),n=S.resolve(e.config.logDir||"logs"),s=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",i=s?S.join(S.resolve(s),"deploy-diff.json"):"",o=new A(n,"deploy.log.jsonl",e.config.logLevel??"info"),u=e.profile?.extraReplacements??{},a=async()=>{};try{let l=await Ht(t,e.config,e.name,u,o);a=l.cleanup;let g={...e.config,outputDir:l.outputDir},m=g.s3SyncMode??"sdk-upload-delete",c=await de(t),w=l.outputDir!==t.outputDir&&(t.targetOrigin!==e.config.targetOrigin||Object.keys(u).length>0),P=!!c&&!c.fullSyncRequired&&!w;e.name&&o.info(`Using deployment profile: ${e.name}`),o.info(`Starting deploy \u2014 s3://${g.s3.bucket}/${g.s3.prefix||""} (region: ${g.s3.region})`),o.info(`S3 sync mode: ${m}`),c&&c.fullSyncRequired?o.info("Deploy plan requests a full sync; scanning the whole output tree."):c&&w?o.info("Ignoring incremental deploy plan because deployment-profile rewrites require a full sync."):P&&o.info(`Using incremental deploy plan: ${c.changedFiles.length} changed file(s), ${c.deletedFiles.length} deleted file(s).`),o.mark("deploy");let b=new S3Client({region:g.s3.region}),p=P?(await Promise.all(c.changedFiles.map(async f=>{try{return (await $.stat(S.join(g.outputDir,f))).isFile()?f:null}catch{return null}}))).filter(f=>f!==null):await nt(["**/*"],{cwd:g.outputDir,onlyFiles:!0,dot:!0}),je=P?new Set:new Set(p.map(f=>te(g.s3.prefix,f)));P?o.info(`Using ${p.length} planned changed file(s) from deploy plan for upload.`):o.info(`Found ${p.length} file(s) in output directory "${g.outputDir}".`);let v=0,y=0,E=0,q=[],k=[],W=[],z=[],j=new Map,Ie=new Map,Oe=new Map,Te=new Map,_e=new Map;P||(o.info("Listing existing S3 objects to detect unchanged files..."),j=await jt(b,g.s3.bucket,g.s3.prefix,o),o.info(`Found ${j.size} existing key(s) in S3.`));let Ae=async f=>j.has(f)?j.get(f)??null:P?It(b,g.s3.bucket,f,Ie):null;for(let f of p){let ae=S.join(g.outputDir,f),d=te(g.s3.prefix,f),I=$t.lookup(f)||"application/octet-stream",Ne=ee(f)?g.s3.htmlCacheControl:g.s3.assetCacheControl,h=v+y+E+1;p.length<=200||h<=5||h%10===0||h===p.length?(o.progress(`Uploading [${h}/${p.length}] ${d} (${I})`,{file:f,key:d,index:h,totalFiles:p.length,contentType:I,mode:m}),o.info(`Uploading [${h}/${p.length}] ${d} (${I})`)):o.debug(`Uploading [${h}/${p.length}] ${d} (${I})`);try{let x=await $.readFile(ae),O=await Ae(d),T=null,L=null;if(O){let ze=x.byteLength;if(O.size===ze){let U=O.etag.includes("-");if(O.etag!==""&&!U){let Ue=Tt(x);if(O.etag===Ue){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:E,mode:m,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}T=Me(x);let le=await _t(b,g.s3.bucket,d,Oe);if(le&&le===T){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:E,mode:m,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}if(ee(f)){L=ne(x);let U=await At(b,g.s3.bucket,d,Te,_e);if(U&&U===L){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped semantically unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:E,mode:m,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped semantically unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}}T=T??Me(x),L=L??(ee(f)?ne(x):null),await new Upload({client:b,params:{Bucket:g.s3.bucket,Key:d,Body:x,ContentType:I,CacheControl:Ne,Metadata:{[Le]:T,...L?{[Fe]:L}:{}}}}).done(),v++,q.push(d),(v%25===0||v===p.length)&&(o.progress(`Uploaded ${v}/${p.length} file(s)\u2026`,{uploaded:v,skipped:y,failed:E,totalFiles:p.length,mode:m}),o.info(`Uploaded ${v}/${p.length} file(s)\u2026`));}catch(x){E++,W.push(d),o.error(`Failed to upload ${d}: ${x.message}`,{key:d,error:String(x)});}}o.info(`Upload complete: ${v} uploaded, ${y} skipped unchanged, ${E} failed.`),m==="sdk-upload-delete"?P?z=await ke(b,g.s3.bucket,c.deletedFiles.map(f=>te(g.s3.prefix,f)),o):z=await Ot(b,g.s3.bucket,j.keys(),je,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion."),i&&(await $.mkdir(S.dirname(i),{recursive:!0}),await $.writeFile(i,JSON.stringify({generatedAt:new Date().toISOString(),mode:m,summary:{totalFiles:p.length,uploaded:q.length,skipped:k.length,failed:W.length,deleted:z.length,...e.name?{profile:e.name}:{}},uploadedKeys:q,skippedKeys:k,failedKeys:W,deletedKeys:z},null,2),"utf8")),await fe(t),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}Jt().catch(async t=>{console.error(t);try{let e=(await G().catch(()=>null))?.logDir??"logs",n=new A(e,"deploy.log.jsonl","debug");n.error(`Unhandled error: ${t instanceof Error?t.message:String(t)}`,t instanceof Error?{stack:t.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
|
6
|
+
`;return e.replace(/<head\b[^>]*>/i,s=>`${s}${n} ${t}`)}function vt(e){let t=$e(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function xt(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,B(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!r.length)return e;let n=i=>{let o=i;for(let[u,a]of r)o=o.split(u).join(a);return o},s=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,u,a)=>`${o}${n(u)}${a}`);return s=s.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,u,a,l)=>vt(a)?`${o}${n(a)}${l}`:i),s}function Rt(e,t){if(!t)return null;let r=v.dirname(v.resolve(t)),n=v.resolve(e.outputDir),s=v.relative(r,n).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s.endsWith("/")?s:`${s}/`):"./"}function Dt(e,t,r){let n=Rt(t,r);if(!n)return e;let s=e;for(let i of wt){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");s=s.replace(u,`${n}${i}`).replace(a,`${n}${i}`);let l=i.replace(/\//g,"\\/"),c=n.replace(/\//g,"\\/"),m=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),g=new RegExp(`(?:\\.{1,}\\\\/)+${m}`,"g"),w=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${m}`,"g"),b=new RegExp(`(?<![\\.\\\\])\\/${m}`,"g");s=s.replace(g,`${c}${l}`).replace(w,`${c}${l}`).replace(b,`${c}${l}`);}return s}function ee(e,t,r){if(e.urlRewriteMode==="absolute")return t;let n=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),s=n?n.pathname:Ce(t,e.targetOrigin),i=n?`${n.search}${n.hash}`:"";return e.urlRewriteMode==="root-relative"?`${s}${i}`:`${Ee(e.outputDir,r,s)}${i}`}function X(e,t,r,n,s){H(e,r,ee(t,n,s));}function $t(e,t,r,n,s){if(!r||r===n)return;H(e,r,n);let i=ee(t,r,s),o=ee(t,n,s);H(e,i,o);}function Ct(e,t,r,n,s={}){let i=e,o={};X(o,t,t.sourceOrigin,t.targetOrigin,n);for(let[a,l]of Object.entries(t.extraReplacements))X(o,t,a,l,n);for(let[a,l]of Object.entries(r))X(o,t,a,l,n);for(let[a,l]of Object.entries(s)){let c=r[a];!c||c===l||$t(o,t,l,c,n);}let u=Object.entries(o).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of u)i=i.split(a).join(l);return t.urlRewriteMode!=="absolute"&&(i=Dt(i,t,n)),i=xt(i,u.map(([,a])=>a)),Pt(t,i,n)&&(i=St(i)),i}async function Et(e,t,r,n){let s=v.join(e.outputDir,n),i=v.extname(s).toLowerCase(),o;try{o=await D.readFile(s,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=Ct(o,e,t,i===".js"||i===".mjs"?void 0:s,r);return a!==o?(await D.writeFile(s,a,"utf8"),{changed:true}):{changed:false}}async function Lt(e,t){return await new Promise((r,n)=>{let s=a=>{if(u(),a?.error){n(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{u(),n(a);},o=a=>{u(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},u=()=>{e.off("message",s),e.off("error",i),e.off("exit",o);};e.on("message",s),e.on("error",i),e.on("exit",o),e.postMessage({file:t});})}async function ke(e,t,r={}){let s=(r.files?[...new Set(r.files)].map(g=>v.resolve(g)).filter(g=>g.startsWith(v.resolve(e.outputDir))).map(g=>v.relative(e.outputDir,g).replace(/\\/g,"/")).filter(g=>g.length>0):await ft(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(g=>g!=="asset-map.json"&&De(g)),i=0;if(s.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),u=Math.max(1,Math.min(o,s.length,availableParallelism())),a=0,l=0,c=null;if(u===1){for(let g of s){let w=await Et(e,t,r.previousAssetMap??{},g);w.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:g,changed:w.changed});}return i}let m=Array.from({length:u},async()=>{let g=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let w=a;if(w>=s.length)return;a+=1;let b=s[w],P=await Lt(g,b);P.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:b,changed:P.changed});}}catch(w){throw c=w instanceof Error?w:new Error(String(w)),c}finally{await g.terminate().catch(()=>{});}});return await Promise.all(m),i}var Te="wpsuite-sha256",Fe="wpsuite-normalized-sha256",_t=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,ae=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;function At(e){return e&&(ae.test(e)?"__uuid__":/^[0-9a-f]{6,}$/i.test(e)?"__hex__":/^[0-9]{8,}$/.test(e)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(e)?"__alnum__":e)}function ne(e){return e.split(/([_:.\-/=?&#]+)/g).map((t,r)=>r%2===1?t:At(t)).join("")}function Nt(e){return e.split(/([_:.\-/=?&#]+)/g).map((t,r)=>r%2===1?t:ae.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{6,}$/.test(t)?"__num__":/^[a-z0-9_-]{6,}$/i.test(t)?"__id__":t).join("")}function q(e,t){return Array.isArray(e)?e.map(r=>q(r,t)):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([r,n])=>[r,q(n,r)])):typeof e!="string"?e:t&&_t.test(t)?Nt(e):ae.test(e)?"__uuid__":/^[0-9a-f]{6,}$/i.test(e)||/^[0-9]{8,}$/.test(e)?ne(e):e}function Ut(e){return e.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function zt(e,t){let r=t.trim();if((r.startsWith("{")||r.startsWith("[")||r.startsWith('"'))&&e.startsWith("data-"))try{let s=JSON.parse(r);return JSON.stringify(q(s))}catch{}if(e.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(r))try{let s=Buffer.from(r,"base64").toString("utf8"),i=JSON.parse(s),o=JSON.stringify(q(i));return Buffer.from(o,"utf8").toString("base64")}catch{}return t}function Wt(e,t){let r=e.toLowerCase();return r==="class"?t.split(/(\s+)/).map((n,s)=>s%2===1?n:ne(n)).join(""):r==="id"||r==="for"||r==="aria-controls"||r==="aria-labelledby"||r==="aria-describedby"||r==="aria-owns"||r==="aria-activedescendant"||r==="name"?ne(t):r==="href"||r==="src"?Ut(t):zt(r,t)}function Bt(e){return e.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(t,r,n)=>{if(r.startsWith("/"))return t;let s=n.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(i,o,u,a)=>{let l=Wt(o,a);return ` ${o}=${u}${l}${u}`});return `<${r}${s}>`})}function te(e){let t=v.extname(e).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(t)||["robots.txt","sitemap.xml"].includes(v.basename(e))}function re(e,t){return [e.replace(/^\/+|\/+$/g,""),t.replace(/^\/+/,"")].filter(Boolean).join("/")}async function Ht(e,t,r,n){let s=new Map,i,o=0;do{o++;let u=await e.send(new ListObjectsV2Command({Bucket:t,Prefix:r||void 0,ContinuationToken:i}));for(let a of u.Contents||[]){if(!a.Key)continue;let l=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),c=Number(a.Size??0);s.set(a.Key,{etag:l,size:c});}i=u.NextContinuationToken,n.debug(`Listed S3 keys page ${o}: ${s.size} keys so far`);}while(i);return s}async function qt(e,t,r,n){if(n.has(r))return n.get(r)??null;try{let s=await e.send(new HeadObjectCommand({Bucket:t,Key:r})),i={etag:(s.ETag||"").replace(/^"|"$/g,"").toLowerCase(),size:Number(s.ContentLength??0)};return n.set(r,i),i}catch{return n.set(r,null),null}}async function Oe(e,t,r,n){let s=[...new Set(Array.from(r).filter(Boolean))];if(s.length===0)return n.info("No stale S3 objects to delete."),[];n.info(`Deleting ${s.length} stale S3 object(s)\u2026`);for(let i=0;i<s.length;i+=1e3){let o=s.slice(i,i+1e3);if(o.length!==0){await e.send(new DeleteObjectsCommand({Bucket:t,Delete:{Objects:o.map(u=>({Key:u}))}})),n.info(`Deleted batch of ${o.length} object(s).`);for(let u of o)n.debug(` deleted: ${u}`);}}return s}async function Jt(e,t,r,n,s){return Oe(e,t,Array.from(r).filter(i=>!n.has(i)),s)}function Kt(e){return createHash("md5").update(e).digest("hex")}function Ie(e){return createHash("sha256").update(e).digest("hex")}function ie(e){return createHash("sha256").update(Bt(e.toString("utf8"))).digest("hex")}async function Qt(e,t,r,n){if(n.has(r))return n.get(r)??null;try{let o=((await e.send(new HeadObjectCommand({Bucket:t,Key:r}))).Metadata?.[Te]||"").trim().toLowerCase()||null;return n.set(r,o),o}catch{return n.set(r,null),null}}async function Vt(e,t,r,n,s){if(n.has(r))return n.get(r)??null;try{let o=((await e.send(new HeadObjectCommand({Bucket:t,Key:r}))).Metadata?.[Fe]||"").trim().toLowerCase();if(o)return n.set(r,o),o}catch{return n.set(r,null),null}if(s.has(r)){let i=s.get(r)??null;return n.set(r,i),i}try{let o=await(await e.send(new GetObjectCommand({Bucket:t,Key:r}))).Body?.transformToByteArray(),u=o?ie(Buffer.from(o)):null;return s.set(r,u),n.set(r,u),u}catch{return s.set(r,null),n.set(r,null),null}}function Gt(e){return e.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function Yt(e,t){let r=e,n=Object.entries(t).sort((s,i)=>i[0].length-s[0].length);for(let[s,i]of n)s&&(r=r.split(s).join(i));return r}async function Zt(e,t,r,n){let s=v.join(e,"asset-map.json"),i;try{i=await D.readFile(s,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(i);if(!o||typeof o!="object")return;let u=Object.fromEntries(Object.entries(o).map(([a,l])=>{let c=String(l??"");return t.urlRewriteMode==="absolute"&&t.targetOrigin&&r.targetOrigin&&t.targetOrigin!==r.targetOrigin&&(c=c.split(t.targetOrigin).join(r.targetOrigin)),c=Yt(c,n),[a,c]}));await D.writeFile(s,JSON.stringify(u,null,2),"utf8");}async function Xt(e,t,r,n,s){let i=e.targetOrigin!==t.targetOrigin,o=Object.keys(n).length>0;if(!r||!i&&!o)return {outputDir:e.outputDir,cleanup:async()=>{}};if(i&&e.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${r}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let u=v.join(v.dirname(e.outputDir),`.deploy-profile-${Gt(r)}-${Date.now()}`);s.info(`Preparing deployment profile "${r}" from existing crawl output...`),await D.rm(u,{recursive:true,force:true}),await D.cp(e.outputDir,u,{recursive:true});let a={...e,sourceOrigin:e.targetOrigin,targetOrigin:t.targetOrigin,extraReplacements:n,outputDir:u},l=await ke(a,{});return await Zt(u,e,t,n),s.info(`Prepared deployment profile "${r}" in ${u} (${l} rewritten text file(s)).`),{outputDir:u,cleanup:async()=>{await D.rm(u,{recursive:true,force:true});}}}async function er(){let e=await Y(),t=ye(process.argv.slice(2)),r=be(e,t),n=v.resolve(r.config.logDir||"logs"),s=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",i=s?v.join(v.resolve(s),"deploy-diff.json"):"",o=new A(n,"deploy.log.jsonl",r.config.logLevel??"info"),u=r.profile?.extraReplacements??{},a=async()=>{};try{let l=await Xt(e,r.config,r.name,u,o);a=l.cleanup;let c={...r.config,outputDir:l.outputDir},m=c.s3SyncMode??"sdk-upload-delete",g=await he(e),w=l.outputDir!==e.outputDir&&(e.targetOrigin!==r.config.targetOrigin||Object.keys(u).length>0),b=!!g&&!g.fullSyncRequired&&!w;r.name&&o.info(`Using deployment profile: ${r.name}`),o.info(`Starting deploy \u2014 s3://${c.s3.bucket}/${c.s3.prefix||""} (region: ${c.s3.region})`),o.info(`S3 sync mode: ${m}`),g&&g.fullSyncRequired?o.info("Deploy plan requests a full sync; scanning the whole output tree."):g&&w?o.info("Ignoring incremental deploy plan because deployment-profile rewrites require a full sync."):b&&o.info(`Using incremental deploy plan: ${g.changedFiles.length} changed file(s), ${g.deletedFiles.length} deleted file(s).`),o.mark("deploy");let P=new S3Client({region:c.s3.region}),p=b?(await Promise.all(g.changedFiles.map(async f=>{try{return (await D.stat(v.join(c.outputDir,f))).isFile()?f:null}catch{return null}}))).filter(f=>f!==null):await ft(["**/*"],{cwd:c.outputDir,onlyFiles:!0,dot:!0}),je=b?new Set:new Set(p.map(f=>re(c.s3.prefix,f)));b?o.info(`Using ${p.length} planned changed file(s) from deploy plan for upload.`):o.info(`Found ${p.length} file(s) in output directory "${c.outputDir}".`);let S=0,y=0,E=0,J=[],I=[],K=[],U=[],T=new Map,_e=new Map,Ae=new Map,Ne=new Map,Ue=new Map;b||(o.info("Listing existing S3 objects to detect unchanged files..."),T=await Ht(P,c.s3.bucket,c.s3.prefix,o),o.info(`Found ${T.size} existing key(s) in S3.`));let ze=async f=>T.has(f)?T.get(f)??null:b?qt(P,c.s3.bucket,f,_e):null;for(let f of p){let le=v.join(c.outputDir,f),d=re(c.s3.prefix,f),F=jt.lookup(f)||"application/octet-stream",We=te(f)?c.s3.htmlCacheControl:c.s3.assetCacheControl,h=S+y+E+1;p.length<=200||h<=5||h%10===0||h===p.length?(o.progress(`Uploading [${h}/${p.length}] ${d} (${F})`,{file:f,key:d,index:h,totalFiles:p.length,contentType:F,mode:m}),o.info(`Uploading [${h}/${p.length}] ${d} (${F})`)):o.debug(`Uploading [${h}/${p.length}] ${d} (${F})`);try{let x=await D.readFile(le),O=await ze(d),j=null,M=null;if(O){let Be=x.byteLength;if(O.size===Be){let z=O.etag.includes("-");if(O.etag!==""&&!z){let He=Kt(x);if(O.etag===He){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:S,skipped:y,failed:E,mode:m,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),I.push(d);continue}}j=Ie(x);let ue=await Qt(P,c.s3.bucket,d,Ae);if(ue&&ue===j){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:S,skipped:y,failed:E,mode:m,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),I.push(d);continue}}if(te(f)){M=ie(x);let z=await Vt(P,c.s3.bucket,d,Ne,Ue);if(z&&z===M){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped semantically unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:S,skipped:y,failed:E,mode:m,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped semantically unchanged [${h}/${p.length}] ${d}`),I.push(d);continue}}}j=j??Ie(x),M=M??(te(f)?ie(x):null),await new Upload({client:P,params:{Bucket:c.s3.bucket,Key:d,Body:x,ContentType:F,CacheControl:We,Metadata:{[Te]:j,...M?{[Fe]:M}:{}}}}).done(),S++,J.push(d),(S%25===0||S===p.length)&&(o.progress(`Uploaded ${S}/${p.length} file(s)\u2026`,{uploaded:S,skipped:y,failed:E,totalFiles:p.length,mode:m}),o.info(`Uploaded ${S}/${p.length} file(s)\u2026`));}catch(x){E++,K.push(d),o.error(`Failed to upload ${d}: ${x.message}`,{key:d,error:String(x)});}}o.info(`Upload complete: ${S} uploaded, ${y} skipped unchanged, ${E} failed.`),m==="sdk-upload-delete"?b?U=await Oe(P,c.s3.bucket,g.deletedFiles.map(f=>re(c.s3.prefix,f)),o):U=await Jt(P,c.s3.bucket,T.keys(),je,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion."),i&&(await D.mkdir(v.dirname(i),{recursive:!0}),await D.writeFile(i,JSON.stringify({generatedAt:new Date().toISOString(),mode:m,summary:{totalFiles:p.length,uploaded:J.length,skipped:I.length,failed:K.length,deleted:U.length,...r.name?{profile:r.name}:{}},uploadedKeys:J,skippedKeys:I,failedKeys:K,deletedKeys:U},null,2),"utf8")),await we(e),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}er().catch(async e=>{console.error(e);try{let r=(await Y().catch(()=>null))?.logDir??"logs",n=new A(r,"deploy.log.jsonl","debug");n.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
package/dist/invalidate.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {CloudFrontClient,CreateInvalidationCommand}from'@aws-sdk/client-cloudfront';import c from'fs/promises';import g from'path';function S(i){let t=i.trim();return t==="."?".":t.replace(/\/$/,"")}function T(i){let t=String(i??"").trim();if(!t)return "";let e=t;if(/^https?:\/\//i.test(e))try{e=new URL(e).pathname;}catch{return ""}if(e=e.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!e)return "";let n=e.endsWith("/"),o=e.replace(/^\/+/,"").split("/").map(l=>l.replace(/[^A-Za-z0-9._-]/g,"")).filter(l=>l.length>0&&l!=="."&&l!=="..");if(o.length===0)return "";let r=`/${o.join("/")}`;return n&&g.extname(r)===""&&(r+="/"),r==="/"?"":r}function P(i){return !i||typeof i!="object"?{}:Object.fromEntries(Object.entries(i).map(([t,e])=>[t.trim(),String(e??"")]).filter(([t])=>t.length>0))}function O(i){if(!i||typeof i!="object")return {};let t={};for(let[e,n]of Object.entries(i)){let o=e.trim();if(!o||!n||typeof n!="object")continue;let r=n,l={};if(typeof r.targetOrigin=="string"){let s=S(r.targetOrigin);s&&(l.targetOrigin=s);}let d=P(r.extraReplacements);if(Object.keys(d).length>0&&(l.extraReplacements=d),r.s3&&typeof r.s3=="object"){let s={},a=r.s3;typeof a.bucket=="string"&&(s.bucket=a.bucket.trim()),typeof a.prefix=="string"&&(s.prefix=a.prefix.trim()),typeof a.region=="string"&&(s.region=a.region.trim()),typeof a.htmlCacheControl=="string"&&(s.htmlCacheControl=a.htmlCacheControl.trim()),typeof a.assetCacheControl=="string"&&(s.assetCacheControl=a.assetCacheControl.trim()),Object.keys(s).length>0&&(l.s3=s);}if(r.cloudFront&&typeof r.cloudFront=="object"){let s={},a=r.cloudFront;typeof a.distributionId=="string"&&(s.distributionId=a.distributionId.trim()),Array.isArray(a.invalidationPaths)&&(s.invalidationPaths=a.invalidationPaths.map(u=>String(u??"").trim()).filter(u=>u.length>0)),Object.keys(s).length>0&&(l.cloudFront=s);}t[o]=l;}return t}function w(i,t){let n=String(i||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let o=n.split("/").map(r=>r.trim()).filter(r=>r.length>0&&r!=="."&&r!=="..");return o.length>0?o.join("/"):t}function M(){let i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return i.trim()?g.resolve(i):""}function b(i,t,e){let n=String(t||"").trim();return n&&g.isAbsolute(n)?g.resolve(n):g.resolve(i,w(n,e))}async function y(){let i=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await c.readFile(i,"utf8"),e=JSON.parse(t);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=S(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=T(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=P(e.extraReplacements),e.postCrawlCopyMap=P(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.scheduler||={enabled:false,timezone:"UTC",rules:[]},e.scheduler.enabled??=false,e.scheduler.timezone||="UTC",e.scheduler.rules||=[],e.deploymentProfiles=O(e.deploymentProfiles),e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.defaultDeploymentProfile=String(e.defaultDeploymentProfile??"").trim(),e.defaultDeploymentProfile&&!e.deploymentProfiles[e.defaultDeploymentProfile]&&(e.defaultDeploymentProfile="");let n=M(),o=n?g.resolve(n,".."):"";return o?(e.outputDir=b(o,e.outputDir,"export"),e.logDir=b(o,e.logDir,"logs")):(g.isAbsolute(e.outputDir)||(e.outputDir=w(e.outputDir,"export")),g.isAbsolute(e.logDir)||(e.logDir=w(e.logDir,"logs"))),e}function D(i){let t="";for(let n=0;n<i.length;n++){let o=i[n];if(o==="--profile"){let r=(i[n+1]||"").trim();if(!r)throw new Error("Missing value for --profile.");t=r,n+=1;}else if(o.startsWith("--profile=")){let r=o.slice(10).trim();if(!r)throw new Error("Missing value for --profile.");t=r;}}let e=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||e||null}function $(i,t){return {...i,...t??{}}}function A(i,t){return {...i,...t??{},invalidationPaths:[...(t?.invalidationPaths??i.invalidationPaths)||[]]}}function C(i,t){let e=String(t??i.defaultDeploymentProfile??"").trim();if(!e)return {name:null,profile:null,config:i};let n=i.deploymentProfiles?.[e];if(!n)throw new Error(`Unknown deployment profile "${e}". Check deploymentProfiles in publisher.config.json.`);return {name:e,profile:n,config:{...i,targetOrigin:n.targetOrigin??i.targetOrigin,s3:$(i.s3,n.s3),cloudFront:A(i.cloudFront,n.cloudFront)}}}var L={error:0,warn:1,info:2,debug:3},I=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",f=I?g.join(I,"current-progress.json"):"",R=Promise.resolve();function F(i){return i.includes("crawl")?"crawl":i.includes("deploy")?"deploy":i.includes("invalidate")?"invalidate":i}function N(i,t){let e={...i},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[o,r]of Object.entries(t)){let l=e[o];if(typeof l=="number"&&typeof r=="number"&&n.has(o)){e[o]=Math.max(l,r);continue}e[o]=r;}return e}function q(i,t,e){f&&(R=R.then(async()=>{let n=new Date,o=n.toISOString();await c.mkdir(g.dirname(f),{recursive:true});let r=await c.readFile(f,"utf8").then(v=>JSON.parse(v)).catch(()=>null),l=r&&typeof r.details=="object"&&r.details?r.details:{},d=F(i),s=r?.currentStep||F(r?.source||""),a=r?.startedAt||o,u={...r?.stepDurationsSec??{}},p=r?.stepStartedAt||o;if(s&&s!==d&&r?.stepStartedAt){let v=Math.max(0,Math.round((n.getTime()-new Date(r.stepStartedAt).getTime())/1e3));u[s]=(u[s]??0)+v,p=o;}let k=Math.max(0,Math.round((n.getTime()-new Date(p).getTime())/1e3)),E=Math.max(0,Math.round((n.getTime()-new Date(a).getTime())/1e3)),x={checkedAt:o,source:i,message:t,details:N(l,e??{}),startedAt:a,currentStep:d,stepStartedAt:p,stepElapsedSec:k,totalElapsedSec:E,stepDurationsSec:u};await c.writeFile(f,JSON.stringify(x,null,2),"utf8");}).catch(()=>{}));}async function _(i,t){await c.mkdir(g.dirname(i),{recursive:true}),await c.appendFile(i,`${JSON.stringify(t)}
|
|
2
|
-
`,"utf8");}var h=class{constructor(t,e
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(t,
|
|
1
|
+
import {CloudFrontClient,CreateInvalidationCommand}from'@aws-sdk/client-cloudfront';import p from'fs/promises';import c from'path';import {getConfig}from'@smart-cloud/wpsuite-core';function C(i){let t=i.trim();return t==="."?".":t.replace(/\/$/,"")}function N(i){let t=String(i??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let e=r.endsWith("/"),o=r.replace(/^\/+/,"").split("/").map(s=>s.replace(/[^A-Za-z0-9._-]/g,"")).filter(s=>s.length>0&&s!=="."&&s!=="..");if(o.length===0)return "";let n=`/${o.join("/")}`;return e&&c.extname(n)===""&&(n+="/"),n==="/"?"":n}function P(i){return !i||typeof i!="object"?{}:Object.fromEntries(Object.entries(i).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function $(i){if(!i||typeof i!="object")return {};let t={};for(let[r,e]of Object.entries(i)){let o=r.trim();if(!o||!e||typeof e!="object")continue;let n=e,s={};if(typeof n.targetOrigin=="string"){let a=C(n.targetOrigin);a&&(s.targetOrigin=a);}let u=P(n.extraReplacements);if(Object.keys(u).length>0&&(s.extraReplacements=u),n.s3&&typeof n.s3=="object"){let a={},l=n.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(s.s3=a);}if(n.cloudFront&&typeof n.cloudFront=="object"){let a={},l=n.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(a).length>0&&(s.cloudFront=a);}t[o]=s;}return t}function I(i){return i==="PROFESSIONAL"||i==="AGENCY"?i:void 0}function W(i,t){if(!i||typeof i!="object")return null;let r=i,e=r.command;if(e!=="publish"&&e!=="crawl"&&e!=="deploy"&&e!=="invalidate"&&e!=="retry-timeouts"&&e!=="url")return null;let o=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(o)||o<1)return null;let n=String(r.id??`${e}-${t+1}`).trim();if(!n)return null;let s=String(r.deploymentProfile??"").trim(),u=String(r.url??"").trim();return {id:n,enabled:r.enabled!==false,command:e,intervalMinutes:o,...e==="publish"||e==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(e==="publish"||e==="deploy"||e==="invalidate")&&s?{deploymentProfile:s}:{},...u?{url:u}:{}}}function D(i){let t=i&&typeof i=="object"?i:{},r=Array.isArray(t.rules)?t.rules.map((e,o)=>W(e,o)).filter(e=>!!e):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function U(i){if(!i||typeof i!="object")return;let t=i,r={};for(let o of ["accountId","siteId"]){let n=t[o];typeof n=="string"&&n.trim()!==""&&(r[o]=n.trim());}let e=Number(t.lastUpdate??0);return Number.isFinite(e)&&e>0&&(r.lastUpdate=Math.floor(e)),r.subscriber=t.subscriber===true,Object.keys(r).length>0?r:void 0}function w(i){let t=i&&typeof i=="object"?i:{},r=U(t.siteSettings),e={subscriber:t.subscriber===true||r?.subscriber===true};for(let n of ["accountId","siteId","apiBase","nonce","uploadUrl"]){let s=t[n];typeof s=="string"&&s.trim()!==""&&(e[n]=s.trim());}r&&(e.siteSettings=r);let o=I(t.subscriptionType);return o&&(e.subscriptionType=o),e}function q(i){try{let t=new URL(i);globalThis.location=t;}catch{}}function _(i){let t=w(i.wpsuite),r={...t.siteSettings??{},accountId:String(t.siteSettings?.accountId??t.accountId??"").trim()||void 0,siteId:String(t.siteSettings?.siteId??t.siteId??"").trim()||void 0,subscriber:t.siteSettings?.subscriber===true||t.subscriber===true},e=String(t.accountId??r.accountId??"").trim(),o=String(t.siteId??r.siteId??"").trim(),n=String(t.uploadUrl??"").trim();if(i.wpsuite={...t,...e?{accountId:e}:{},...o?{siteId:o}:{},siteSettings:r},!e||!o||!n)return false;q(i.sourceOrigin);let s=globalThis,a={...s.WpSuite??{},siteSettings:r,uploadUrl:n};return t.apiBase&&(a.apiBase=t.apiBase),t.nonce&&(a.nonce=t.nonce),s.WpSuite=a,true}async function J(i){if(!_(i))return null;try{let t=await getConfig("publisher");return t&&typeof t=="object"?t:null}catch{return null}}function z(i,t){let r=$(t?.deploymentProfiles),e=String(t?.defaultDeploymentProfile??"").trim(),o=r[e]?e:"",n=I(t?.subscriptionType),s=w(i.wpsuite);return {...i,scheduler:D(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:o,...n?{subscriptionType:n}:{},wpsuite:{...s,subscriber:s.subscriber===true||!!n,...n?{subscriptionType:n}:{}}}}function v(i,t){let e=String(i||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!e)return t;let o=e.split("/").map(n=>n.trim()).filter(n=>n.length>0&&n!=="."&&n!=="..");return o.length>0?o.join("/"):t}function Q(){let i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return i.trim()?c.resolve(i):""}function y(i,t,r){let e=String(t||"").trim();return e&&c.isAbsolute(e)?c.resolve(e):c.resolve(i,v(e,r))}async function S(i){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await p.readFile(t,"utf8"),e=JSON.parse(r);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=C(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=N(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=P(e.extraReplacements),e.postCrawlCopyMap=P(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.wpsuite=w(e.wpsuite),e.scheduler=D(void 0),e.deploymentProfiles={},e.defaultDeploymentProfile="",e.deploymentTargetOverride=String(e.deploymentTargetOverride??"").trim();let o=z(e,await J(e)),n=Q(),s=n?c.resolve(n,".."):"";return s?(o.outputDir=y(s,o.outputDir,"export"),o.logDir=y(s,o.logDir,"logs")):(c.isAbsolute(o.outputDir)||(o.outputDir=v(o.outputDir,"export")),c.isAbsolute(o.logDir)||(o.logDir=v(o.logDir,"logs"))),o}function B(i){return typeof i.subscriptionType=="string"&&i.subscriptionType.trim()!==""}function L(i){let t="";for(let e=0;e<i.length;e++){let o=i[e];if(o==="--profile"){let n=(i[e+1]||"").trim();if(!n)throw new Error("Missing value for --profile.");t=n,e+=1;}else if(o.startsWith("--profile=")){let n=o.slice(10).trim();if(!n)throw new Error("Missing value for --profile.");t=n;}}let r=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||r||null}function H(i,t){return {...i,...t??{}}}function G(i,t){return {...i,...t??{},invalidationPaths:[...(t?.invalidationPaths??i.invalidationPaths)||[]]}}function R(i,t){let r=String(t??i.deploymentTargetOverride??"").trim();if(!B(i)){if(r)throw new Error(`Deployment target "${r}" requires an active WPSuite subscription and remote publisher config.`);return {name:null,profile:null,config:i}}let e=String(r||i.defaultDeploymentProfile||"").trim();if(!e)return {name:null,profile:null,config:i};let o=i.deploymentProfiles?.[e];if(!o)throw new Error(`Unknown deployment profile "${e}". Check the linked WPSuite publisher configuration.`);return {name:e,profile:o,config:{...i,targetOrigin:o.targetOrigin??i.targetOrigin,s3:H(i.s3,o.s3),cloudFront:G(i.cloudFront,o.cloudFront)}}}var k={error:0,warn:1,info:2,debug:3},F=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",f=F?c.join(F,"current-progress.json"):"",T=Promise.resolve();function x(i){return i.includes("crawl")?"crawl":i.includes("deploy")?"deploy":i.includes("invalidate")?"invalidate":i}function Y(i,t){let r={...i},e=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[o,n]of Object.entries(t)){let s=r[o];if(typeof s=="number"&&typeof n=="number"&&e.has(o)){r[o]=Math.max(s,n);continue}r[o]=n;}return r}function K(i,t,r){f&&(T=T.then(async()=>{let e=new Date,o=e.toISOString();await p.mkdir(c.dirname(f),{recursive:true});let n=await p.readFile(f,"utf8").then(b=>JSON.parse(b)).catch(()=>null),s=n&&typeof n.details=="object"&&n.details?n.details:{},u=x(i),a=n?.currentStep||x(n?.source||""),l=n?.startedAt||o,g={...n?.stepDurationsSec??{}},d=n?.stepStartedAt||o;if(a&&a!==u&&n?.stepStartedAt){let b=Math.max(0,Math.round((e.getTime()-new Date(n.stepStartedAt).getTime())/1e3));g[a]=(g[a]??0)+b,d=o;}let E=Math.max(0,Math.round((e.getTime()-new Date(d).getTime())/1e3)),O=Math.max(0,Math.round((e.getTime()-new Date(l).getTime())/1e3)),j={checkedAt:o,source:i,message:t,details:Y(s,r??{}),startedAt:l,currentStep:u,stepStartedAt:d,stepElapsedSec:E,totalElapsedSec:O,stepDurationsSec:g};await p.writeFile(f,JSON.stringify(j,null,2),"utf8");}).catch(()=>{}));}async function V(i,t){await p.mkdir(c.dirname(i),{recursive:true}),await p.appendFile(i,`${JSON.stringify(t)}
|
|
2
|
+
`,"utf8");}var h=class{constructor(t,r,e="info"){this.logDir=t;this.logFile=r;this.level=e;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return c.join(this.logDir,this.logFile)}get errorsPath(){return c.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await p.mkdir(this.logDir,{recursive:true}),await p.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await p.appendFile(this.logPath,`${t}
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>V(t,r));}accepts(t){return k[t]<=k[this.level]}push(t,r,e){if(!this.accepts(t))return;let o=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...e||{}});this.enqueueLine(o),t==="error"?console.error(`[ERROR] ${r}`,e?JSON.stringify(e):""):t==="warn"?console.warn(`[WARN] ${r}`,e?JSON.stringify(e):""):console.log(`[${t.toUpperCase().padEnd(5)}] ${r}`);}info(t,r){this.push("info",t,r);}progress(t,r){K(this.logFile,t,r);}debug(t,r){this.push("debug",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let e=this.marks.get(t);if(!e)return;let o=Date.now()-e,n=Number((o/1e3).toFixed(2));this.push("info",`Timing ${t}: ${n}s`,{name:t,ms:o,seconds:n,...r||{}});}async flush(){let t=Date.now()-this.startedAt,r=Number((t/1e3).toFixed(2));if(this.push("info",`Total time: ${r}s`,{name:"total",ms:t,seconds:r}),await this.writeQueue,this.writeError)throw this.writeError}};async function ee(){let i=await S(),t=L(process.argv.slice(2)),r=R(i,t),e=r.config,o=e.logDir||"logs",n=new h(o,"invalidate.log.jsonl",e.logLevel??"info");if(r.name&&n.info(`Using deployment profile: ${r.name}`),!e.cloudFront.distributionId){n.info("No CloudFront distributionId configured; skipping invalidation."),await n.flush();return}let s=e.cloudFront.invalidationPaths.length>0?e.cloudFront.invalidationPaths:["/*"];n.info(`Starting CloudFront invalidation for distribution ${e.cloudFront.distributionId}`),n.info(`Invalidation paths (${s.length}): ${s.join(", ")}`),n.mark("invalidate");let u=new CloudFrontClient({region:"us-east-1"}),a=`smartcloud-static-publisher-${Date.now()}`;n.debug(`CallerReference: ${a}`);let l=await u.send(new CreateInvalidationCommand({DistributionId:e.cloudFront.distributionId,InvalidationBatch:{CallerReference:a,Paths:{Quantity:s.length,Items:s}}})),g=l.Invalidation?.Id??"(unknown)",d=l.Invalidation?.Status??"(unknown)";n.info(`CloudFront invalidation created. ID: ${g} Status: ${d}`),n.debug(`Full invalidation response: ${JSON.stringify(l.Invalidation)}`),n.endMark("invalidate"),await n.flush();}ee().catch(async i=>{console.error(i);try{let r=(await S().catch(()=>null))?.logDir??"logs",e=new h(r,"invalidate.log.jsonl","debug");e.error(`Unhandled error: ${i instanceof Error?i.message:String(i)}`,i instanceof Error?{stack:i.stack}:void 0),await e.flush();}catch{}process.exit(1);});
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import g from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import d from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';var J=class extends Error{request;step;constructor(t,r){super(`Stop requested during ${r}.`),this.name="StopRequestedError",this.request=t,this.step=r;}},V="queue-mutation.lock",Y=5e3,Z=3e4;function ee(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",r=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),n=process.env.PUBLISHER_CONFIG||"",o=1;for(let u=0;u<e.length;u++){let s=e[u];s==="--runtime-dir"?t=e[++u]||"":s.startsWith("--runtime-dir=")?t=s.slice(14):s==="--exporter-dir"?r=e[++u]||r:s.startsWith("--exporter-dir=")?r=s.slice(15):s==="--config"?n=e[++u]||"":s.startsWith("--config=")?n=s.slice(9):s==="--max-jobs"?o=Number.parseInt(e[++u]||"1",10):s.startsWith("--max-jobs=")&&(o=Number.parseInt(s.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let a=d.resolve(t),i=n?d.resolve(n):d.join(a,"config.json");return {runtimeDir:a,exporterDir:d.resolve(r),configPath:i,maxJobs:Number.isFinite(o)&&o>0?o:1}}async function S(e,t){try{let r=await g.readFile(e,"utf8");return JSON.parse(r)}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="ENOENT"){let o=r instanceof Error?r.message:String(r);console.warn(`[queue-runner] failed to read JSON ${e}: ${o}`);}return t}}async function y(e,t){await g.mkdir(d.dirname(e),{recursive:true}),await g.writeFile(e,JSON.stringify(t,null,2),"utf8");}function B(e){return d.join(e,V)}async function te(e){await new Promise(t=>setTimeout(t,e));}async function re(e){let t=B(e),r=Date.now()+Y;for(;;)try{await g.mkdir(d.dirname(t),{recursive:!0}),await g.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="EEXIST")throw n;let a=await g.stat(t).catch(()=>null);if(a&&Date.now()-a.mtimeMs>Z){await g.unlink(t).catch(()=>{});continue}if(Date.now()>=r)throw new Error("Timed out acquiring queue mutation lock.",{cause:n});await te(50);}}async function ne(e){await g.unlink(B(e)).catch(()=>{});}async function R(e,t){await re(e);try{return await t()}finally{await ne(e);}}function ie(e){return d.join(e,"queue-runner-heartbeat.json")}function x(e){return d.join(e,"current-progress.json")}function T(e){return d.join(e,"scheduler-state.json")}function oe(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let o=n.split("/").map(a=>a.trim()).filter(a=>a.length>0&&a!=="."&&a!=="..");return o.length>0?o.join("/"):t}async function L(e){let t=await S(e.configPath||"",null),r=d.resolve(e.runtimeDir,".."),o=(typeof t?.logDir=="string"?t.logDir:"").trim();return o&&d.isAbsolute(o)?d.resolve(o):d.resolve(r,oe(o,"logs"))}function C(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function se(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function ue(e){return [se(e.endedAt||e.startedAt),C(e.command||"job","job"),C(e.id||"job","job"),C(e.status||"finished","finished")].join("-")}function ae(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function ce(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function de(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function k(e,t,r){let n=`${r}.gz`,o=d.join(t,n);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(o));let[a,i]=await Promise.all([g.stat(e),g.stat(o)]);return {role:ce(r),originalFileName:r,storedFileName:n,compressed:true,compression:"gzip",contentType:de(n),originalSize:a.size,storedSize:i.size}}async function M(e,t){let r=await L(t),n=new Date().toISOString(),o=ue(e),a=d.join(r,"archive",o),i=[],u=[];await g.mkdir(a,{recursive:true});try{let p=await g.readdir(r,{withFileTypes:!0});for(let m of p){if(!m.isFile()||!ae(m.name))continue;let l=await k(d.join(r,m.name),a,m.name);u.push(l),i.push(l.storedFileName);}}catch(p){if((p&&typeof p=="object"&&"code"in p?String(p.code||""):"")!=="ENOENT")throw p}let s=x(t.runtimeDir);if(existsSync(s)){let p=await k(s,a,"current-progress.json");u.push(p),i.push(p.storedFileName);}let c=[...i,"job.json"];return await y(d.join(a,"job.json"),{manifestVersion:1,archivedAt:n,archiveKey:o,archiveDir:a,logDir:r,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:u,job:e}),{archiveKey:o,archiveDir:a,archiveCreatedAt:n,archivedFiles:c,artifacts:u}}function le(e){return d.join(e,"audit-events.jsonl")}function D(e){return d.join(e,"stop-request.json")}async function O(e){try{let t=await g.readFile(D(e),"utf8"),r=JSON.parse(t);return r&&typeof r=="object"?r:null}catch{return null}}async function Q(e,t){let r=await O(e);if(!r)return null;let n=String(r.targetJobId||"").trim();return n&&n!==t?null:r}async function E(e,t){if(!t){await g.unlink(D(e)).catch(()=>{});return}let r=await O(e);if(!r)return;let n=String(r.targetJobId||"").trim();(!n||n===t)&&await g.unlink(D(e)).catch(()=>{});}async function P(e,t){try{let r={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await g.mkdir(e,{recursive:!0}),await g.appendFile(le(e),`${JSON.stringify(r)}
|
|
2
|
-
`,"utf8");}catch{}}async function w(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await y(ie(e.runtimeDir),n);}catch{}}async function I(e){try{await g.unlink(x(e));}catch{}}function me(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let o=(e.id||`${r}-${t+1}`).trim();if(!o)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",s=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:o,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:s}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function pe(e){let t=await S(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((o,a)=>me(o,a)).filter(o=>!!o):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ge(e,t){let r=(t.url||"").trim(),n=t.crawlMode||"full",o=(t.deploymentProfile||"").trim();return e.some(a=>a.command===t.command&&(a.url||"").trim()===r&&(a.crawlMode||"full")===n&&(a.deploymentProfile||"").trim()===o&&(a.status===void 0||a.status==="queued"||a.status==="running"))}async function fe(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await pe(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(s){let c=s instanceof Error?s.message:String(s);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=d.join(e.runtimeDir,"queue.json"),o=d.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,u=[];await R(e.runtimeDir,async()=>{let s=await S(n,[]),c=await S(o,null),p=await S(T(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...s];c&&c.status==="running"&&m.unshift(c);for(let l of r.rules){if(!l.enabled)continue;let f=l.intervalMinutes*60*1e3,h=Math.floor(a/f),A=p.lastEnqueuedBucketByRuleId[l.id]??-1;if(h<=A||ge(m,{command:l.command,url:l.url,crawlMode:l.crawlMode,deploymentProfile:l.deploymentProfile}))continue;let q={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};s.push(q),m.push(q),u.push({job:q,rule:l}),p.lastEnqueuedBucketByRuleId[l.id]=h,i+=1;}i>0&&(await y(n,s),await y(T(e.runtimeDir),p));});for(let s of u)await P(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:s.job.id,command:s.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:s.rule.id,intervalMinutes:s.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:s.rule.deploymentProfile||"",url:s.rule.url||""}});return i}async function he(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await g.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function we(e){try{await g.unlink(e);}catch{}}function ye(e,t){let r=[d.join(e,"dist",`${t}.js`),d.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${d.join(e,"dist")} or ${e}`)}async function b(e,t,r,n,o,a,i){if(i){let c=await Q(t,i);if(c)throw new J(c,r)}let u=ye(e,r),s={...process.env,...a??{}};return s.STATIC_PUBLISHER_RUNTIME_DIR=t,o&&(s.PUBLISHER_CONFIG=o),await new Promise((c,p)=>{let m=null,l=null,f=null,h=false,A=spawn(process.execPath,[u,...n],{cwd:e,env:s,stdio:"inherit"}),q=()=>{l&&clearInterval(l),f&&clearTimeout(f);},$=async()=>{if(!(!i||m||h)){h=true;try{let v=await Q(t,i);if(!v)return;m=v,A.kill("SIGTERM"),f=setTimeout(()=>{A.kill("SIGKILL");},1e4);}finally{h=false;}}};i&&(l=setInterval(()=>{$();},1e3)),A.on("error",v=>{q(),p(v);}),A.on("close",(v,j)=>{if(q(),m){p(new J(m,r));return}v===0?c():p(new Error(`${r} exited with code ${v??-1}${j?` (signal ${j})`:""}`));});}),0}function Se(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function be(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let o=(await S(t.configPath||"",null))?.wpsuite;return o&&(o.accountId||o.siteId||o.siteKey)?o:null}async function ve(e,t){return await be(e,t)}function U(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function Ae(e,t){let r=await S(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let o=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(o)return o;let a=await L(e).catch(()=>"");if(a){let i=await S(d.join(a,"current-crawl-event.json"),null),u=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(u)return u}return t}function qe(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function Pe(e){let t=d.join(e.runtimeDir,"queue.json"),r=d.join(e.runtimeDir,"current-run.json"),n=await R(e.runtimeDir,async()=>{let o=await S(r,null);if(!o||o.status!=="running"&&o.status!=="queued")return null;let a=await S(t,[]),i=Array.isArray(a)?a.filter(s=>s?.id!==o.id):[],u=U(o);return await y(t,[u,...i]),await y(r,null),u});n&&(await E(e.runtimeDir,n.id),await P(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function Ie(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let u=await ve(e,t);await r.assertActiveSubscriptionForIdentity(u);}let o=Se(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await b(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,o,e.id),await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await b(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await b(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="crawl")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await b(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,o,e.id),{exitCode:0};if(e.command==="deploy")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await b(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="invalidate")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await b(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await b(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,o,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await b(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,o,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof J?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function W(e){return async function(r=process.argv.slice(2)){let n=ee(r),o=d.join(n.runtimeDir,"export.lock"),a=d.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await w(n,"starting",{message:"queue-runner starting"});try{await he(o);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await w(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await Pe(n);let i=await fe(n,e),u=0;for(let s=0;s<n.maxJobs;s++){let c=await Re(n,e);if(c==="none"||(u+=1,c==="stopped"))break}if(u===0)await I(n.runtimeDir),await w(n,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let s=await S(a,null),c=String(s?.status||"").trim();await w(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:s?.id,lastJobCommand:s?.command,lastJobStatus:s?.status,lastJobExitCode:s?.exitCode,lastJobError:s?.error,...c==="stopped"?{currentStep:s?.stoppedStep,stopRequestedAt:s?.stopRequestedAt,stopRequestedByLogin:s?.stopRequestedByLogin,stopRequestedMode:s?.stopMode,lastStoppedStep:s?.stoppedStep,message:s?.stopMode==="requeue"?`Job stopped during ${s?.stoppedStep||s?.command||"active step"} and requeued.`:`Job stopped during ${s?.stoppedStep||s?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await w(n,"error",{message:u}),await P(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await we(o);}}}async function Je(e,t,r){let n=d.join(e.runtimeDir,"queue.json"),o=d.join(e.runtimeDir,"current-run.json");await R(e.runtimeDir,async()=>{let a=await S(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],u=U(t),s=qe(t,r);s?u.resumeFromStep=s:delete u.resumeFromStep,await y(n,[u,...i]),await y(o,null);});}async function Re(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json"),o=d.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await R(e.runtimeDir,async()=>{let m=await S(r,[]);if(!Array.isArray(m)||m.length===0)return null;let l={...m[0],status:"running",startedAt:a};return await y(r,m.slice(1)),await y(n,l),l});if(!i)return "none";await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await w(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await Ie(i,e,t),s=new Date().toISOString();if(u.stopped){let m=await Ae(e,u.stoppedStep||i.command),l=u.stopRequest?.mode==="requeue"?"requeue":"stop",f={...i,status:"stopped",endedAt:s,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:l,stoppedStep:m};l==="requeue"?await Je(e,i,m):await y(n,null);try{let h=await M(f,e);f.logArchiveDir=h.archiveDir,f.logArchiveCreatedAt=h.archiveCreatedAt,f.logArchiveFileCount=h.archivedFiles.length;}catch(h){f.logArchiveError=h instanceof Error?h.message:String(h);}return await y(o,f),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:l==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:s,deploymentProfile:i.deploymentProfile||"",stopMode:l,stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveKey:f.logArchiveError||!f.logArchiveDir?"":d.basename(f.logArchiveDir),logArchiveDir:f.logArchiveDir||"",logArchiveCreatedAt:f.logArchiveCreatedAt||"",logArchiveFileCount:f.logArchiveFileCount??0,logArchiveError:f.logArchiveError||""}}),"stopped"}let c={...i,status:u.exitCode===0?"success":"failed",endedAt:s,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await y(n,null),await E(e.runtimeDir,i.id);try{let m=await M(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await y(o,c);let p=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await P(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:p,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":d.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Ee(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function xe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Ee(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,u=setTimeout(()=>i.abort(),12e3);try{let s=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!s.ok)throw new Error(`Subscription check failed with HTTP ${s.status}`);let c=await s.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(u);}}var je={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:xe},_=W(je),Te=_,Ke=_;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Te().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
-
export{Ke as default,Te as main,_ as runQueueRunner};
|
|
1
|
+
import {pathToFileURL}from'url';import m from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import l from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';import {getConfig}from'@smart-cloud/wpsuite-core';function Q(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function re(e){let t=String(e??"").trim();if(!t)return "";let n=t;if(/^https?:\/\//i.test(n))try{n=new URL(n).pathname;}catch{return ""}if(n=n.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!n)return "";let r=n.endsWith("/"),i=n.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(i.length===0)return "";let s=`/${i.join("/")}`;return r&&l.extname(s)===""&&(s+="/"),s==="/"?"":s}function x(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,n])=>[t.trim(),String(n??"")]).filter(([t])=>t.length>0))}function ne(e){if(!e||typeof e!="object")return {};let t={};for(let[n,r]of Object.entries(e)){let i=n.trim();if(!i||!r||typeof r!="object")continue;let s=r,o={};if(typeof s.targetOrigin=="string"){let u=Q(s.targetOrigin);u&&(o.targetOrigin=u);}let a=x(s.extraReplacements);if(Object.keys(a).length>0&&(o.extraReplacements=a),s.s3&&typeof s.s3=="object"){let u={},c=s.s3;typeof c.bucket=="string"&&(u.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(u.prefix=c.prefix.trim()),typeof c.region=="string"&&(u.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(u.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(u.assetCacheControl=c.assetCacheControl.trim()),Object.keys(u).length>0&&(o.s3=u);}if(s.cloudFront&&typeof s.cloudFront=="object"){let u={},c=s.cloudFront;typeof c.distributionId=="string"&&(u.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(u.invalidationPaths=c.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(u).length>0&&(o.cloudFront=u);}t[i]=o;}return t}function W(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function ie(e,t){if(!e||typeof e!="object")return null;let n=e,r=n.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let i=Number.parseInt(String(n.intervalMinutes??"0"),10);if(!Number.isFinite(i)||i<1)return null;let s=String(n.id??`${r}-${t+1}`).trim();if(!s)return null;let o=String(n.deploymentProfile??"").trim(),a=String(n.url??"").trim();return {id:s,enabled:n.enabled!==false,command:r,intervalMinutes:i,...r==="publish"||r==="crawl"?{crawlMode:n.crawlMode==="incremental"?"incremental":"full"}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&o?{deploymentProfile:o}:{},...a?{url:a}:{}}}function U(e){let t=e&&typeof e=="object"?e:{},n=Array.isArray(t.rules)?t.rules.map((r,i)=>ie(r,i)).filter(r=>!!r):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:n}}function oe(e){if(!e||typeof e!="object")return;let t=e,n={};for(let i of ["accountId","siteId"]){let s=t[i];typeof s=="string"&&s.trim()!==""&&(n[i]=s.trim());}let r=Number(t.lastUpdate??0);return Number.isFinite(r)&&r>0&&(n.lastUpdate=Math.floor(r)),n.subscriber=t.subscriber===true,Object.keys(n).length>0?n:void 0}function k(e){let t=e&&typeof e=="object"?e:{},n=oe(t.siteSettings),r={subscriber:t.subscriber===true||n?.subscriber===true};for(let s of ["accountId","siteId","apiBase","nonce","uploadUrl"]){let o=t[s];typeof o=="string"&&o.trim()!==""&&(r[s]=o.trim());}n&&(r.siteSettings=n);let i=W(t.subscriptionType);return i&&(r.subscriptionType=i),r}function se(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function ue(e){let t=k(e.wpsuite),n={...t.siteSettings??{},accountId:String(t.siteSettings?.accountId??t.accountId??"").trim()||void 0,siteId:String(t.siteSettings?.siteId??t.siteId??"").trim()||void 0,subscriber:t.siteSettings?.subscriber===true||t.subscriber===true},r=String(t.accountId??n.accountId??"").trim(),i=String(t.siteId??n.siteId??"").trim(),s=String(t.uploadUrl??"").trim();if(e.wpsuite={...t,...r?{accountId:r}:{},...i?{siteId:i}:{},siteSettings:n},!r||!i||!s)return false;se(e.sourceOrigin);let o=globalThis,u={...o.WpSuite??{},siteSettings:n,uploadUrl:s};return t.apiBase&&(u.apiBase=t.apiBase),t.nonce&&(u.nonce=t.nonce),o.WpSuite=u,true}async function ae(e){if(!ue(e))return null;try{let t=await getConfig("publisher");return t&&typeof t=="object"?t:null}catch{return null}}function ce(e,t){let n=ne(t?.deploymentProfiles),r=String(t?.defaultDeploymentProfile??"").trim(),i=n[r]?r:"",s=W(t?.subscriptionType),o=k(e.wpsuite);return {...e,scheduler:U(t?.scheduler),deploymentProfiles:n,defaultDeploymentProfile:i,...s?{subscriptionType:s}:{},wpsuite:{...o,subscriber:o.subscriber===true||!!s,...s?{subscriptionType:s}:{}}}}function J(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let i=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return i.length>0?i.join("/"):t}function le(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?l.resolve(e):""}function L(e,t,n){let r=String(t||"").trim();return r&&l.isAbsolute(r)?l.resolve(r):l.resolve(e,J(r,n))}async function E(e){let t=String(e??"").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await m.readFile(t,"utf8"),r=JSON.parse(n);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=Q(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=re(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=x(r.extraReplacements),r.postCrawlCopyMap=x(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.wpsuite=k(r.wpsuite),r.scheduler=U(void 0),r.deploymentProfiles={},r.defaultDeploymentProfile="",r.deploymentTargetOverride=String(r.deploymentTargetOverride??"").trim();let i=ce(r,await ae(r)),s=le(),o=s?l.resolve(s,".."):"";return o?(i.outputDir=L(o,i.outputDir,"export"),i.logDir=L(o,i.logDir,"logs")):(l.isAbsolute(i.outputDir)||(i.outputDir=J(i.outputDir,"export")),l.isAbsolute(i.logDir)||(i.logDir=J(i.logDir,"logs"))),i}var D=class extends Error{request;step;constructor(t,n){super(`Stop requested during ${n}.`),this.name="StopRequestedError",this.request=t,this.step=n;}},be="queue-mutation.lock",ye=5e3,Se=3e4;function we(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",n=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),r=process.env.PUBLISHER_CONFIG||"",i=1;for(let a=0;a<e.length;a++){let u=e[a];u==="--runtime-dir"?t=e[++a]||"":u.startsWith("--runtime-dir=")?t=u.slice(14):u==="--exporter-dir"?n=e[++a]||n:u.startsWith("--exporter-dir=")?n=u.slice(15):u==="--config"?r=e[++a]||"":u.startsWith("--config=")?r=u.slice(9):u==="--max-jobs"?i=Number.parseInt(e[++a]||"1",10):u.startsWith("--max-jobs=")&&(i=Number.parseInt(u.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let s=l.resolve(t),o=r?l.resolve(r):l.join(s,"config.json");return {runtimeDir:s,exporterDir:l.resolve(n),configPath:o,maxJobs:Number.isFinite(i)&&i>0?i:1}}async function S(e,t){try{let n=await m.readFile(e,"utf8");return JSON.parse(n)}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="ENOENT"){let i=n instanceof Error?n.message:String(n);console.warn(`[queue-runner] failed to read JSON ${e}: ${i}`);}return t}}async function y(e,t){await m.mkdir(l.dirname(e),{recursive:true}),await m.writeFile(e,JSON.stringify(t,null,2),"utf8");}function H(e){return l.join(e,be)}async function Pe(e){await new Promise(t=>setTimeout(t,e));}async function ve(e){let t=H(e),n=Date.now()+ye;for(;;)try{await m.mkdir(l.dirname(t),{recursive:!0}),await m.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="EEXIST")throw r;let s=await m.stat(t).catch(()=>null);if(s&&Date.now()-s.mtimeMs>Se){await m.unlink(t).catch(()=>{});continue}if(Date.now()>=n)throw new Error("Timed out acquiring queue mutation lock.",{cause:r});await Pe(50);}}async function Ce(e){await m.unlink(H(e)).catch(()=>{});}async function q(e,t){await ve(e);try{return await t()}finally{await Ce(e);}}function Re(e){return l.join(e,"queue-runner-heartbeat.json")}function F(e){return l.join(e,"current-progress.json")}function B(e){return l.join(e,"scheduler-state.json")}function Ae(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let i=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return i.length>0?i.join("/"):t}async function K(e){let t=await S(e.configPath||"",null),n=l.resolve(e.runtimeDir,".."),i=(typeof t?.logDir=="string"?t.logDir:"").trim();return i&&l.isAbsolute(i)?l.resolve(i):l.resolve(n,Ae(i,"logs"))}function T(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function Ie(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function De(e){return [Ie(e.endedAt||e.startedAt),T(e.command||"job","job"),T(e.id||"job","job"),T(e.status||"finished","finished")].join("-")}function qe(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function xe(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function Je(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function N(e,t,n){let r=`${n}.gz`,i=l.join(t,r);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(i));let[s,o]=await Promise.all([m.stat(e),m.stat(i)]);return {role:xe(n),originalFileName:n,storedFileName:r,compressed:true,compression:"gzip",contentType:Je(r),originalSize:s.size,storedSize:o.size}}async function z(e,t){let n=await K(t),r=new Date().toISOString(),i=De(e),s=l.join(n,"archive",i),o=[],a=[];await m.mkdir(s,{recursive:true});try{let g=await m.readdir(n,{withFileTypes:!0});for(let p of g){if(!p.isFile()||!qe(p.name))continue;let d=await N(l.join(n,p.name),s,p.name);a.push(d),o.push(d.storedFileName);}}catch(g){if((g&&typeof g=="object"&&"code"in g?String(g.code||""):"")!=="ENOENT")throw g}let u=F(t.runtimeDir);if(existsSync(u)){let g=await N(u,s,"current-progress.json");a.push(g),o.push(g.storedFileName);}let c=[...o,"job.json"];return await y(l.join(s,"job.json"),{manifestVersion:1,archivedAt:r,archiveKey:i,archiveDir:s,logDir:n,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:a,job:e}),{archiveKey:i,archiveDir:s,archiveCreatedAt:r,archivedFiles:c,artifacts:a}}function ke(e){return l.join(e,"audit-events.jsonl")}function j(e){return l.join(e,"stop-request.json")}async function G(e){try{let t=await m.readFile(j(e),"utf8"),n=JSON.parse(t);return n&&typeof n=="object"?n:null}catch{return null}}async function _(e,t){let n=await G(e);if(!n)return null;let r=String(n.targetJobId||"").trim();return r&&r!==t?null:n}async function O(e,t){if(!t){await m.unlink(j(e)).catch(()=>{});return}let n=await G(e);if(!n)return;let r=String(n.targetJobId||"").trim();(!r||r===t)&&await m.unlink(j(e)).catch(()=>{});}async function A(e,t){try{let n={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await m.mkdir(e,{recursive:!0}),await m.appendFile(ke(e),`${JSON.stringify(n)}
|
|
2
|
+
`,"utf8");}catch{}}async function b(e,t,n){let r={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...n??{}};try{await y(Re(e.runtimeDir),r);}catch{}}async function I(e){try{await m.unlink(F(e));}catch{}}function Ee(e,t){let n=e.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let r=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(r)||r<1)return null;let i=(e.id||`${n}-${t+1}`).trim();if(!i)return null;let s=e.enabled!==false,o=typeof e.url=="string"?e.url.trim():"",a=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",u=(n==="publish"||n==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return n==="url"&&!o?null:{id:i,enabled:s,command:n,intervalMinutes:r,...n==="publish"||n==="crawl"?{crawlMode:u}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&a?{deploymentProfile:a}:{},...o?{url:o}:{}}}async function Te(e){let t=await E(e.configPath).catch(()=>null),n=t?.scheduler,r=Array.isArray(n?.rules)?n.rules.map((i,s)=>Ee(i,s)).filter(i=>!!i):[];return {enabled:!!n?.enabled,timezone:typeof n?.timezone=="string"&&n.timezone.trim()?n.timezone.trim():"UTC",rules:r,wpsuite:t?.wpsuite}}function je(e,t){let n=(t.url||"").trim(),r=t.crawlMode||"full",i=(t.deploymentProfile||"").trim();return e.some(s=>s.command===t.command&&(s.url||"").trim()===n&&(s.crawlMode||"full")===r&&(s.deploymentProfile||"").trim()===i&&(s.status===void 0||s.status==="queued"||s.status==="running"))}async function Oe(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let n=await Te(e);if(!n.enabled||n.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(n.wpsuite??null);}catch(u){let c=u instanceof Error?u.message:String(u);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let r=l.join(e.runtimeDir,"queue.json"),i=l.join(e.runtimeDir,"current-run.json"),s=Date.now(),o=0,a=[];await q(e.runtimeDir,async()=>{let u=await S(r,[]),c=await S(i,null),g=await S(B(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),p=[...u];c&&c.status==="running"&&p.unshift(c);for(let d of n.rules){if(!d.enabled)continue;let f=d.intervalMinutes*60*1e3,h=Math.floor(s/f),C=g.lastEnqueuedBucketByRuleId[d.id]??-1;if(h<=C||je(p,{command:d.command,url:d.url,crawlMode:d.crawlMode,deploymentProfile:d.deploymentProfile}))continue;let R={id:randomUUID(),command:d.command,...(d.command==="publish"||d.command==="crawl")&&d.crawlMode?{crawlMode:d.crawlMode}:{},...(d.command==="publish"||d.command==="deploy"||d.command==="invalidate")&&d.deploymentProfile?{deploymentProfile:d.deploymentProfile}:{},...d.url?{url:d.url}:{},enqueueSource:"scheduler",...n.wpsuite?{wpsuite:n.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};u.push(R),p.push(R),a.push({job:R,rule:d}),g.lastEnqueuedBucketByRuleId[d.id]=h,o+=1;}o>0&&(await y(r,u),await y(B(e.runtimeDir),g));});for(let u of a)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:u.job.id,command:u.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:u.rule.id,intervalMinutes:u.rule.intervalMinutes,timezone:n.timezone,deploymentProfile:u.rule.deploymentProfile||"",url:u.rule.url||""}});return o}async function Fe(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await m.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function Me(e){try{await m.unlink(e);}catch{}}function Le(e,t){let n=[l.join(e,"dist",`${t}.js`),l.join(e,`${t}.js`)];for(let r of n)if(existsSync(r))return r;throw new Error(`Cannot find ${t}.js in ${l.join(e,"dist")} or ${e}`)}async function P(e,t,n,r,i,s,o){if(o){let c=await _(t,o);if(c)throw new D(c,n)}let a=Le(e,n),u={...process.env,...s??{}};return u.STATIC_PUBLISHER_RUNTIME_DIR=t,i&&(u.PUBLISHER_CONFIG=i),await new Promise((c,g)=>{let p=null,d=null,f=null,h=false,C=spawn(process.execPath,[a,...r],{cwd:e,env:u,stdio:"inherit"}),R=()=>{d&&clearInterval(d),f&&clearTimeout(f);},X=async()=>{if(!(!o||p||h)){h=true;try{let v=await _(t,o);if(!v)return;p=v,C.kill("SIGTERM"),f=setTimeout(()=>{C.kill("SIGKILL");},1e4);}finally{h=false;}}};o&&(d=setInterval(()=>{X();},1e3)),C.on("error",v=>{R(),g(v);}),C.on("close",(v,M)=>{if(R(),p){g(new D(p,n));return}v===0?c():g(new Error(`${n} exited with code ${v??-1}${M?` (signal ${M})`:""}`));});}),0}function Qe(e){let t=e.awsTempCreds;if(!t)return {};let n={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(n.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(n.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(n.AWS_SESSION_TOKEN=t.sessionToken.trim()),n}async function We(e,t){let r=(await E(t.configPath).catch(()=>null))?.wpsuite;if(r&&(r.accountId||r.siteId||r.nonce||r.uploadUrl||r.subscriptionType))return r;let i=e.wpsuite;return i&&(i.accountId||i.siteId||i.nonce||i.uploadUrl||i.subscriptionType)?i:null}async function Ue(e,t){return await We(e,t)}function Y(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function Be(e,t){let n=await S(F(e.runtimeDir),null),r=n&&n.details&&typeof n.details.phase=="string"?n.details.phase.trim():"";if(r)return r;let i=n&&typeof n.currentStep=="string"?n.currentStep.trim():"";if(i)return i;let s=await K(e).catch(()=>"");if(s){let o=await S(l.join(s,"current-crawl-event.json"),null),a=o&&typeof o.currentStep=="string"?o.currentStep.trim():"";if(a)return a}return t}function Ne(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function ze(e){let t=l.join(e.runtimeDir,"queue.json"),n=l.join(e.runtimeDir,"current-run.json"),r=await q(e.runtimeDir,async()=>{let i=await S(n,null);if(!i||i.status!=="running"&&i.status!=="queued")return null;let s=await S(t,[]),o=Array.isArray(s)?s.filter(u=>u?.id!==i.id):[],a=Y(i);return await y(t,[a,...o]),await y(n,null),a});r&&(await O(e.runtimeDir,r.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:r.id,command:r.command,message:"Recovered stale current-run entry back into queue."}));}async function _e(e,t,n){try{if((n.shouldEnforceSubscriptionOnJobExecution?.(e)??n.enforceSubscriptionOnJobExecution)&&n.assertActiveSubscriptionForIdentity){let a=await Ue(e,t);await n.assertActiveSubscriptionForIdentity(a);}let i=Qe(e),s=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],o=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await P(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,i,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await P(t.exporterDir,t.runtimeDir,"deploy",o,t.configPath,i,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await P(t.exporterDir,t.runtimeDir,"invalidate",o,t.configPath,i,e.id),{exitCode:0};if(e.command==="crawl")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await P(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,i,e.id),{exitCode:0};if(e.command==="deploy")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await P(t.exporterDir,t.runtimeDir,"deploy",o,t.configPath,i,e.id),{exitCode:0};if(e.command==="invalidate")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await P(t.exporterDir,t.runtimeDir,"invalidate",o,t.configPath,i,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await P(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,i,e.id),{exitCode:0};if(e.command==="url"){let a=(e.url||"").trim();return a?(await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await P(t.exporterDir,t.runtimeDir,"crawl",["--url",a],t.configPath,i,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(r){return r instanceof D?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:r.request,stoppedStep:r.step}:{exitCode:1,error:r instanceof Error?r.message:String(r)}}}function Z(e){return async function(n=process.argv.slice(2)){let r=we(n),i=l.join(r.runtimeDir,"export.lock"),s=l.join(r.runtimeDir,"last-run.json");await I(r.runtimeDir),await b(r,"starting",{message:"queue-runner starting"});try{await Fe(i);}catch(o){if((o&&typeof o=="object"&&"code"in o?String(o.code||""):"")!=="EEXIST")throw o;console.log("[queue-runner] lock active, skipping"),await b(r,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await ze(r);let o=await Oe(r,e),a=0;for(let u=0;u<r.maxJobs;u++){let c=await He(r,e);if(c==="none"||(a+=1,c==="stopped"))break}if(a===0)await I(r.runtimeDir),await b(r,"idle",{processedJobs:a,schedulerEnqueued:o,message:"no queued jobs"});else {await I(r.runtimeDir);let u=await S(s,null),c=String(u?.status||"").trim();await b(r,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:a,schedulerEnqueued:o,lastJobId:u?.id,lastJobCommand:u?.command,lastJobStatus:u?.status,lastJobExitCode:u?.exitCode,lastJobError:u?.error,...c==="stopped"?{currentStep:u?.stoppedStep,stopRequestedAt:u?.stopRequestedAt,stopRequestedByLogin:u?.stopRequestedByLogin,stopRequestedMode:u?.stopMode,lastStoppedStep:u?.stoppedStep,message:u?.stopMode==="requeue"?`Job stopped during ${u?.stoppedStep||u?.command||"active step"} and requeued.`:`Job stopped during ${u?.stoppedStep||u?.command||"active step"} and left out of queue.`}:{}});}}catch(o){let a=o instanceof Error?o.message:String(o);throw await b(r,"error",{message:a}),await A(r.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:a}}),o}finally{await Me(i);}}}async function $e(e,t,n){let r=l.join(e.runtimeDir,"queue.json"),i=l.join(e.runtimeDir,"current-run.json");await q(e.runtimeDir,async()=>{let s=await S(r,[]),o=Array.isArray(s)?s.filter(c=>c?.id!==t.id):[],a=Y(t),u=Ne(t,n);u?a.resumeFromStep=u:delete a.resumeFromStep,await y(r,[a,...o]),await y(i,null);});}async function He(e,t){let n=l.join(e.runtimeDir,"queue.json"),r=l.join(e.runtimeDir,"current-run.json"),i=l.join(e.runtimeDir,"last-run.json"),s=new Date().toISOString(),o=await q(e.runtimeDir,async()=>{let p=await S(n,[]);if(!Array.isArray(p)||p.length===0)return null;let d={...p[0],status:"running",startedAt:s};return await y(n,p.slice(1)),await y(r,d),d});if(!o)return "none";await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:o.id,command:o.command,message:"Job execution started.",details:{createdAt:o.createdAt||"",startedAt:s,createdBy:o.createdBy??null,deploymentProfile:o.deploymentProfile||"",queuedWithTempAwsCreds:!!o.awsTempCreds}}),await b(e,"running",{currentJobId:o.id,currentJobCommand:o.command,currentStep:o.command});let a=await _e(o,e,t),u=new Date().toISOString();if(a.stopped){let p=await Be(e,a.stoppedStep||o.command),d=a.stopRequest?.mode==="requeue"?"requeue":"stop",f={...o,status:"stopped",endedAt:u,exitCode:a.exitCode,...a.error?{error:a.error}:{},stopRequestedAt:a.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof a.stopRequest?.requestedByUserId=="number"?a.stopRequest.requestedByUserId:null,stopRequestedByLogin:a.stopRequest?.requestedByLogin||"",stopMode:d,stoppedStep:p};d==="requeue"?await $e(e,o,p):await y(r,null);try{let h=await z(f,e);f.logArchiveDir=h.archiveDir,f.logArchiveCreatedAt=h.archiveCreatedAt,f.logArchiveFileCount=h.archivedFiles.length;}catch(h){f.logArchiveError=h instanceof Error?h.message:String(h);}return await y(i,f),await O(e.runtimeDir,o.id),await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:o.id,command:o.command,message:d==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:o.startedAt||"",endedAt:u,deploymentProfile:o.deploymentProfile||"",stopMode:d,stopRequestedAt:a.stopRequest?.requestedAt||"",stopRequestedByLogin:a.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof a.stopRequest?.requestedByUserId=="number"?a.stopRequest.requestedByUserId:null,stoppedStep:p,logArchiveKey:f.logArchiveError||!f.logArchiveDir?"":l.basename(f.logArchiveDir),logArchiveDir:f.logArchiveDir||"",logArchiveCreatedAt:f.logArchiveCreatedAt||"",logArchiveFileCount:f.logArchiveFileCount??0,logArchiveError:f.logArchiveError||""}}),"stopped"}let c={...o,status:a.exitCode===0?"success":"failed",endedAt:u,exitCode:a.exitCode,...a.error?{error:a.error}:{}};await y(r,null),await O(e.runtimeDir,o.id);try{let p=await z(c,e);c.logArchiveDir=p.archiveDir,c.logArchiveCreatedAt=p.archiveCreatedAt,c.logArchiveFileCount=p.archivedFiles.length;}catch(p){c.logArchiveError=p instanceof Error?p.message:String(p);}await y(i,c);let g=o.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(o.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:o.startedAt||"",endedAt:c.endedAt||"",durationSec:g,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":l.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}async function Ge(e){if(!e)throw new Error("Missing remote WPSuite publisher state. Refresh the runtime config before running subscription-gated jobs.");if(e.subscriptionType!=="PROFESSIONAL"&&e.subscriptionType!=="AGENCY")throw new Error("This job requires an active WPSuite publisher subscription in the remote site configuration.")}var Ye={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0,assertActiveSubscriptionForIdentity:Ge},V=Z(Ye),Ze=V,gt=V;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Ze().catch(e=>{console.error(e),process.exit(1);});export{gt as default,Ze as main,V as runQueueRunner};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|
|
@@ -39,10 +39,11 @@
|
|
|
39
39
|
"release:npm": "WPSUITE_PREMIUM=true npm run build && npm publish --access=public"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@smart-cloud/publisher-core": "^1.1.
|
|
43
|
-
"@
|
|
44
|
-
"@aws-sdk/client-
|
|
45
|
-
"@aws-sdk/
|
|
42
|
+
"@smart-cloud/publisher-core": "^1.1.1",
|
|
43
|
+
"@smart-cloud/wpsuite-core": "^2.3.1",
|
|
44
|
+
"@aws-sdk/client-cloudfront": "^3.1057.0",
|
|
45
|
+
"@aws-sdk/client-s3": "^3.1057.0",
|
|
46
|
+
"@aws-sdk/lib-storage": "^3.1057.0",
|
|
46
47
|
"fast-glob": "^3.3.3",
|
|
47
48
|
"mime-types": "^3.0.2",
|
|
48
49
|
"playwright": "^1.60.0"
|
|
@@ -51,13 +52,13 @@
|
|
|
51
52
|
"@eslint/js": "^10.0.1",
|
|
52
53
|
"@types/mime-types": "^3.0.1",
|
|
53
54
|
"@types/node": "^25.9.1",
|
|
54
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
55
|
-
"@typescript-eslint/parser": "^8.
|
|
55
|
+
"@typescript-eslint/eslint-plugin": "^8.60.0",
|
|
56
|
+
"@typescript-eslint/parser": "^8.60.0",
|
|
56
57
|
"esbuild": "^0.28.0",
|
|
57
|
-
"eslint": "^10.4.
|
|
58
|
+
"eslint": "^10.4.1",
|
|
58
59
|
"globals": "^17.6.0",
|
|
59
60
|
"tsup": "^8.5.1",
|
|
60
61
|
"typescript": "^6.0.3",
|
|
61
|
-
"typescript-eslint": "^8.
|
|
62
|
+
"typescript-eslint": "^8.60.0"
|
|
62
63
|
}
|
|
63
64
|
}
|