@smart-cloud/publisher-exporter 1.0.10 → 1.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/crawl.js +1 -1
  2. package/package.json +1 -1
package/dist/crawl.js CHANGED
@@ -3,4 +3,4 @@ import {chromium}from'playwright';import y from'fs/promises';import f from'path'
3
3
  `:`
4
4
  `;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function kt(e){let t=E(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Mt(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,H(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[o,a]of r)u=u.split(o).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,o,a)=>`${u}${s(o)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,o,a,l)=>kt(a)?`${u}${s(a)}${l}`:i),n}function Tt(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Lt(e,t,r){let s=Tt(t,r);if(!s)return e;let n=e;for(let i of bt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),o=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(o,`${s}${i}`).replace(a,`${s}${i}`);let l=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),m=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${m}`,"g"),h=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${m}`,"g"),p=new RegExp(`(?<![\\.\\\\])\\/${m}`,"g");n=n.replace(d,`${c}${l}`).replace(h,`${c}${l}`).replace(p,`${c}${l}`);}return n}function ce(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:Me(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Te(e.outputDir,r,n)}${i}`}function le(e,t,r,s,n){J(e,r,ce(t,s,n));}function Et(e,t,r,s,n){if(!r||r===s)return;J(e,r,s);let i=ce(t,r,n),u=ce(t,s,n);J(e,i,u);}function Dt(e,t,r,s,n={}){let i=e,u={};le(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,l]of Object.entries(t.extraReplacements))le(u,t,a,l,s);for(let[a,l]of Object.entries(r))le(u,t,a,l,s);for(let[a,l]of Object.entries(n)){let c=r[a];!c||c===l||Et(u,t,l,c,s);}let o=Object.entries(u).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of o)i=i.split(a).join(l);return t.urlRewriteMode!=="absolute"&&(i=Lt(i,t,s)),i=Mt(i,o.map(([,a])=>a)),xt(t,i,s)&&(i=At(i)),i}async function It(e,t,r,s){let n=f.join(e.outputDir,s),i=f.extname(n).toLowerCase(),u;try{u=await y.readFile(n,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=Dt(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await y.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function Ft(e,t){return await new Promise((r,s)=>{let n=a=>{if(o(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{o(),s(a);},u=a=>{o(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},o=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function De(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>f.resolve(d)).filter(d=>d.startsWith(f.resolve(e.outputDir))).map(d=>f.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await yt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&$(d)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),o=Math.max(1,Math.min(u,n.length,availableParallelism())),a=0,l=0,c=null;if(o===1){for(let d of n){let h=await It(e,t,r.previousAssetMap??{},d);h.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:d,changed:h.changed});}return i}let m=Array.from({length:o},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let h=a;if(h>=n.length)return;a+=1;let p=n[h],P=await Ft(d,p);P.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:p,changed:P.changed});}}catch(h){throw c=h instanceof Error?h:new Error(String(h)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(m),i}var U={error:0,warn:1,info:2,debug:3},Ie=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",z=Ie?f.join(Ie,"current-progress.json"):"",Fe=Promise.resolve();function je(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function jt(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function Ot(e,t,r){z&&(Fe=Fe.then(async()=>{let s=new Date,n=s.toISOString();await y.mkdir(f.dirname(z),{recursive:true});let i=await y.readFile(z,"utf8").then(P=>JSON.parse(P)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},o=je(e),a=i?.currentStep||je(i?.source||""),l=i?.startedAt||n,c={...i?.stepDurationsSec??{}},m=i?.stepStartedAt||n;if(a&&a!==o&&i?.stepStartedAt){let P=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+P,m=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(m).getTime())/1e3)),h=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),p={checkedAt:n,source:e,message:t,details:jt(u,r??{}),startedAt:l,currentStep:o,stepStartedAt:m,stepElapsedSec:d,totalElapsedSec:h,stepDurationsSec:c};await y.writeFile(z,JSON.stringify(p,null,2),"utf8");}).catch(()=>{}));}async function $t(e,t){await y.mkdir(f.dirname(e),{recursive:true}),await y.appendFile(e,`${JSON.stringify(t)}
5
5
  `,"utf8");}function Ut(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function Qt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:Ut(e,r),level:e,message:t,details:r??{}}}var Q=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await y.appendFile(this.logPath,`${t}
6
- `,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>$t(t,r));}updateCurrentEvent(t,r,s){let n=Qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ot("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var qt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],_t=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Wt=new Set(["",".html",".htm"]),Ht=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Jt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function zt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let o=e[u];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Bt(){let e=me();return e?f.resolve(e,".."):""}function Vt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Gt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Bt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:me(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Vt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function Ye(e){let t=me();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function Ze(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Kt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function D(e,t,r){let s=W(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Oe(e,t,r){let s=W(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function V(e,t){e.rewriteTargets.add(f.resolve(t));}async function Xt(e){try{let t=await y.readFile(Ye(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return Ze()}function $e(e){return JSON.parse(JSON.stringify(e))}async function Yt(e,t){let r=Ye(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function Zt(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function er(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:u,status:o.status()}),new Map;let a=await o.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let c of a.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(o)}),new Map}}async function Ue(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await er(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function tr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await Ue(e,t,s,n,i)};let o=await Ue(e,t,s,n,i);if(o?.supported&&o.token)return u.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let a=G(r.sitemapLastmodByPage[n]);return a&&u.sitemapLastmod&&u.sitemapLastmod===a?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function rr(e,t,r,s,n){let i=Qe(r.previousManifest),u=0;for(let[d,h]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Oe(n,e,h.outputPath);try{await y.unlink(h.outputPath);}catch(p){(p.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:h.outputPath,error:String(p)});}delete r.manifest.pages[d],u++;}let o=Qe(r.manifest,r.runId),a=new Set,l=new Set;for(let d of o){for(let P of N(e,d))a.add(P);let h=r.manifest.assets[d]?.outputPath;h&&l.add(h);let p=K(e,t.assetMap,d);p&&l.add(p);}for(let d of Object.keys(r.manifest.assets))o.has(d)||delete r.manifest.assets[d];let c=0,m=0;for(let d of i){let h=new Set,p=r.previousManifest.assets[d]?.outputPath;p&&h.add(p);let P=K(e,t.assetMap,d);if(!o.has(d)&&P)h.add(P);else if(!o.has(d)){let g=we(e,d);h.add(g.originalFilePath),g.hashedFilePath&&h.add(g.hashedFilePath);}for(let g of h)if(!l.has(g)){Oe(n,e,g);try{await y.unlink(g),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:g,error:String(w)});}}for(let g of N(e,d))a.has(g)||Object.prototype.hasOwnProperty.call(t.assetMap,g)&&(delete t.assetMap[g],m++);}(u>0||c>0||m>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${m} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:m,phase:"incremental-cleanup"});}function Qe(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function Ne(e,t,r){for(let s of t)for(let n of N(e,s))if(r.has(n))return true;return false}function sr(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=Zt(s,n);if(u.size===0)return [...i];for(let o of Object.values(t.manifest.pages))Ne(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let[o,a]of Object.entries(t.manifest.assets)){a.isText&&Ne(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let l of N(e,o))if(u.has(l)){D(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function N(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function nr(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ir(e,t,r){for(let s of N(e,r)){let n=t[s];if(!n)continue;let i=nr(e,n);if(i)return i}return null}function K(e,t,r){let s=ir(e,t,r);return s||we(e,r).originalFilePath}function or(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function qe(e){let t=or();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function ar(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Y(e,s)));for(let s of t.doneAssets){let n=K(e,t.assetMap,s);!n||!$(n)||r.add(f.resolve(n));}return [...r]}var X="re:",B=new Map;function ur(e){if(!e.startsWith(X))return null;let t=B.get(e);if(t!==void 0)return t||null;let r=e.slice(X.length).trim();if(!r)return B.set(e,false),null;try{let s=new RegExp(r);return B.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),B.set(e,false),null}}function fe(e,t){let r=ur(t);return r?r.test(e):t.startsWith(X)?false:e.startsWith(t)}function lr(e,t){return t.find(r=>fe(e,r))}function et(e,t){let r=lr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(X)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function cr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function dr(e,t){return cr(e).has(t.hostname)}function pr(e){let t=E(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function tt(e){let t=E(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function ee(e){return Jt.some(t=>e.pathname.startsWith(t))}function R(e,t,r=e.sourceOrigin,s,n="url"){let i=pr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(tt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!dr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let o=et(e,u);if(o)return s?.reject(n,u.toString(),o,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function rt(e){return _t.has(f.extname(e.pathname).toLowerCase())}function _(e){return ee(e)?false:Wt.has(f.extname(e.pathname).toLowerCase())}function te(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function b(e,t){return te(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:rt(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>fe(t.pathname,r)):false}function Y(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function we(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),o=ue(e.targetOrigin,i),a=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=f.extname(s),m=`${l?s.slice(0,-l.length):s}.${ke(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:m,hashedFilePath:f.join(e.outputDir,m),hashedPublicUrl:ue(e.targetOrigin,m)}}function gr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let a of o)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function ge(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function st(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function hr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function nt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function mr(e,t){let r=nt(t),s=hr(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function fr(e,t){if(!t)return;let r=nt(t);if(r===null)return;let s=await st(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function de(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Ht.has(r)||qt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function _e(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function he(e,t,r,s,n,i,u){let o=we(e,r),a=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let d=await ge(o.originalFilePath);d!==null&&!d.equals(s)&&(a=o.hashedFilePath,l=o.hashedPublicUrl);}gr(e,t,r,l,o.originalPublicUrl);let c=await ge(a),m=false;return (c===null||!c.equals(s))&&(await ae(a),await y.writeFile(a,s),m=true),await fr(a,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:l,wroteFile:m}}function Z(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"page");if(!u)return;if(!_(u)||rt(u)){if(b(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),T(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),ee(u)?"asset/internal path cannot be page":"not page-like",n);return}let o=u.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function T(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"asset");if(!u)return;if(!b(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let o=u.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Pe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"sitemap");if(!u)return;if(!te(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let o=u.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function wr(e){let t=E(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Pr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function yr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function pe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function it(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await it(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Sr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,o=i.prefix;if(!u||!o){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:o});continue}let a=Gt(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:o});continue}let l=a.resolvedPath,c=pe(o);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:o});continue}let m;try{m=await y.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:l,prefixPath:o});continue}let d=f.resolve(e.outputDir,c);if(m.isDirectory()){n.push(...await it(l,d)),s++,t.progress(`Copied static directory to export: ${l} -> /${c}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${c}`,copiedItems:s});continue}let h=f.basename(l),p=o.endsWith("/")?f.join(d,h):d;await y.mkdir(f.dirname(p),{recursive:true}),await y.copyFile(l,p),n.push(p),s++,t.progress(`Copied static file to export: ${l} -> /${pe(o.endsWith("/")?`${o}${h}`:o)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${pe(o.endsWith("/")?`${o}${h}`:o)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function q(e,t,r,s,n){let i=new Set,u=E(r).replace(/\\\//g,"/");for(let l of yr(u)){let c=R(e,l,t,s,`${n}:css-url`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=R(e,l[1],t,s,`${n}:css-import`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=R(e,l[1],t,s,`${n}:xml-stylesheet`);c&&b(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of u.matchAll(a)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(tt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let m=R(e,c,t,s,`${n}:serialized-url`);m&&b(e,m)?i.add(m.toString()):m&&s.ignore("asset",m.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function vr(e,t,r,s){let n=new Set,i=E(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=u[1],a=R(e,o,t,s,"page-link");a&&(et(e,a)||_(a)&&!ee(a)&&!b(e,a)&&n.add(a.toString()));}return [...n]}async function We(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),o=i.url()||n;await he(e,r.assetMap,o,u,r,s);let a=u.toString("utf8");for(let l of q(e,o,a,s,"sitemap"))T(e,r,l,s,o);for(let l of wr(a)){let c=R(e,l.loc,o,s,"sitemap-loc");c&&(te(c)?Pe(e,r,c.toString(),s,o):b(e,c)?T(e,r,c.toString(),s,o):_(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),Z(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function br(e,t,r,s,n,i,u){let o=R(e,s,e.sourceOrigin,n,"asset-fetch");if(!o||!b(e,o))return;let a=o.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let l=K(e,r.assetMap,a),c=l?await st(l):null;if(l&&c)try{let w=await t.head(a,{timeout:6e4}),S=w.headers();if(w.ok()&&mr(c,S)){let v=await ge(l);if(v!==null){let I=w.url()||a,C=await he(e,r.assetMap,I,v,r,n,S);C.wroteFile&&(D(u,e,C.outputPath),$(C.outputPath)&&V(u,C.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:l,lastModified:S["last-modified"]||"",contentLength:S["content-length"]||"",phase:"download-assets"}),_e(r,n);let x=i.enabled?i.previousManifest.assets[a]:void 0,ye=de(o,S),re=x?.discoveredAssets?[...x.discoveredAssets]:[];if(ye||x?.isText){let at=v.toString("utf8"),Se=q(e,I,at,n,"asset:cached");(Se.length>0||!x)&&(re=Se);for(let ut of re)T(e,r,ut,n,I);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:C.outputPath,isText:ye||!!x?.isText,discoveredAssets:re,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let m=await t.get(a,{timeout:6e4});if(!m.ok()){n.skip("asset",a,m.status());return}let d=await m.body(),h=m.url()||a,p=m.headers(),P=await he(e,r.assetMap,h,d,r,n,p);P.wroteFile&&(D(u,e,P.outputPath),$(P.outputPath)&&V(u,P.outputPath)),r.doneAssets.add(a),_e(r,n);let g=[];if(de(o,p)){let w=d.toString("utf8");g=q(e,h,w,n,`asset:${f.extname(o.pathname).toLowerCase()||p["content-type"]||"unknown"}`);for(let S of g)T(e,r,S,n,h);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:P.outputPath,isText:de(o,p),discoveredAssets:g,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(l){n.error(`Failed asset ${a}`,{url:a,error:String(l)});}}}async function Rr(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),o=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await br(e,t,r,a,s,n,i));}});await Promise.all(o);}async function Cr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function xr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Cr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ar(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function kr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>fe(t,r))}async function Mr(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,m=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let p of l){let P=d.getAttribute(p);P&&a.add(P);}for(let p of m){let P=d.getAttribute(p);P&&P.split(",").forEach(g=>{let w=g.trim().split(/\s+/)[0];w&&a.add(w);});}let h=d.getAttribute("style");h&&c.add(h);}),{attrs:[...a],styles:[...c]}});for(let a of o.attrs)for(let l of Pr(a)){let c=R(e,l,r,s,"dom-attr");c&&b(e,c)&&i.add(c.toString());}for(let a of o.styles)for(let l of q(e,r,a,s,"dom-style"))i.add(l);}let u=await t.content();for(let o of q(e,r,u,s,"page-html"))i.add(o);return [...i]}async function He(e,t,r,s,n){let i=Y(e,t);await ae(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Je(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function ze(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Tr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Lr(e,t,r,s,n,i,u){let o=await chromium.launch({headless:true}),a=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Tr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let c=t.pageQueue.shift();if(!c)break;if(t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),i.enabled&&i.seenPages.add(c);let m=new URL(c),d=kr(e,m.pathname);if(!_(m)||b(e,m)||ee(m)){if(b(e,m)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),T(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let h=await tr(e,n,t,i,c,r);if(i.enabled&&s!=="single-url"){let g=i.manifest.pages[c];if(h.action==="reuse"&&g){for(let w of g.discoveredAssets)T(e,t,w,r,c);for(let w of g.discoveredPages)Z(e,t,w,r,c);g.lastSeenRunId=i.runId,g.sitemapLastmod=G(t.sitemapLastmodByPage[c])??g.sitemapLastmod,h.changeToken?.supported&&(g.changeToken=h.changeToken.token,g.tokenSource=h.changeToken.tokenSource??g.tokenSource),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:h.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Je(t)&&ze(t,r);continue}}let p=await(d?l.newPage():a.newPage()),P=!1;await p.route("**/*",async g=>{let w=g.request();if(w.isNavigationRequest()&&w.frame()===p.mainFrame())try{let S=new URL(w.url()),v=new URL(c);if(S.hash="",v.hash="",S.href===v.href){if(P){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:w.url()}),await g.abort("aborted");return}P=!0;}}catch{}await g.continue();}),p.on("response",g=>{try{let w=R(e,g.url(),e.sourceOrigin,void 0,"network-response");w&&b(e,w)&&T(e,t,w.toString(),r,c);}catch{}}),r.page(`Rendering ${c}`,{url:c});try{let g=null,w=null;try{if(g=await p.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),g&&g.ok())try{w=await g.text();}catch{w=null;}}catch(x){r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(x)});}if(g&&!g.ok()){r.skip("page",c,g.status()),await p.close();continue}d?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await Ar(e,p)):await xr(e,p),t.stats.pagesRendered++,Je(t)&&ze(t,r);let S=await Mr(e,p,c,r,!d);for(let x of S)T(e,t,x,r,c);let v=w??await p.content(),I=s!=="single-url"?vr(e,c,v,r):[];if(s!=="single-url")for(let x of I)Z(e,t,x,r,c);await He(e,c,v,t,r);let C=Y(e,c);D(u,e,C),V(u,C),i.enabled&&(i.manifest.pages[c]={url:c,outputPath:C,changeToken:h.changeToken?.supported?h.changeToken.token:null,tokenSource:h.changeToken?.supported?h.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[c])??null,discoveredPages:I,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){try{let w=await p.content();if(w&&w.trim()){await He(e,c,w,t,r);let S=Y(e,c);D(u,e,S),V(u,S),r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(g)});}}catch(w){r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(w)});}r.error(`Failed page ${c}`,{url:c,error:String(g)});}finally{await p.close();}}}finally{await l.close(),await a.close(),await o.close();}}async function Er(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Be(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Dr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ir(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function ot(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ve(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Fr(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ve(s)-Ve(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Ge(e){let t=await ot(f.join(e,"job.json"));for(let r of Fr(e,t)){let s=await Er(r);if(s.length>0)return s}return []}async function jr(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let a=await ot(f.join(u,"job.json"));if(!Dr(a))continue;let l=Ir(a);l>=n&&(n=l,s=u);}return s}async function Or(e){let t=await jr(e),r=t!==""?await Ge(t):await Ge(e.logDir),s=new Set;for(let n of r)if(Be(n.error)||Be(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function $r(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Ur(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Qr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ke(e,t,r,s,n){let i=R(e,r,e.sourceOrigin,s,"manual-url");i&&(te(i)?Pe(e,t,i.toString(),s,n):b(e,i)?T(e,t,i.toString(),s,n):_(i)?Z(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Nr(){let e=await oe(),t=zt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Or(e)),t.mode==="single-url"&&(r=await $r(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.preserveOutput?await Ur(e):{},i=s?await Xt(e):Ze(),u={enabled:s,manifest:$e(i),previousManifest:$e(i),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Qr(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...n},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},a=Kt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new Q(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let p=await chromium.launch({headless:true}),P=await p.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),g=P.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>Pe(e,o,S,l)),await We(e,g,o,l),e.seedPaths.forEach(S=>Ke(e,o,S,l,"seed-path"))):(r.forEach(S=>Ke(e,o,S,l,"cli")),await We(e,g,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let w=Array.from({length:Math.max(1,e.concurrency)},()=>Lr(e,o,l,t.mode,g,u,a));if(await Promise.all(w),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Rr(e,g,o,l,u,a),l.endMark("download-assets",{assets:o.doneAssets.size}),u.enabled&&e.maxPages===0&&await rr(e,o,u,l,a),t.mode==="full"){let S=await Sr(e,l);for(let v of S)D(a,e,v);}}finally{await P.close(),await p.close();}}u.enabled&&!t.resumeRewrite&&(u.manifest.updatedAt=new Date().toISOString(),await Yt(e,u.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(n,null,2)!==JSON.stringify(o.assetMap,null,2)&&D(a,e,f.join(e.outputDir,"asset-map.json"));let c=t.mode==="single-url"?ar(e,o):u.enabled&&!t.resumeRewrite?sr(e,u,a,n,o.assetMap):void 0;l.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${c?.length??0}`:"Rewriting text files...";l.progress(m,{phase:"rewrite-text",index:0,totalFiles:c?.length,changedTextFiles:0}),await qe(m);let d=Date.now(),h=await De(e,o.assetMap,{files:c,previousAssetMap:n,onProgress:async({index:p,totalFiles:P,changedTextFiles:g,file:w,changed:S})=>{S&&D(a,e,f.join(e.outputDir,w)),l.checkpoint(`Rewriting text file ${p}/${P}`,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w});let v=Date.now();if(!(p===1||p===P||v-d>=5e3))return;d=v;let C=`Rewriting text files: ${p}/${P}`;l.progress(C,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w}),await qe(C);}});l.endMark("rewrite-text",{changedTextFiles:h}),await Re(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...a.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...a.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((c||[]).map(p=>W(e.outputDir,f.isAbsolute(p)?p:f.join(e.outputDir,p))).filter(p=>p!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${h} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:h}),await l.flush();}Nr().catch(async e=>{console.error(e);try{let r=(await oe().catch(()=>null))?.logDir??"logs",s=new Q(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
6
+ `,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>$t(t,r));}updateCurrentEvent(t,r,s){let n=Qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ot("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var qt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],_t=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Wt=new Set(["",".html",".htm"]),Ht=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Jt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function zt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let o=e[u];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Bt(){let e=me();return e?f.resolve(e,".."):""}function Vt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Gt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Bt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:me(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Vt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function Ye(e){let t=me();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function Ze(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Kt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function D(e,t,r){let s=W(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Oe(e,t,r){let s=W(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function V(e,t){e.rewriteTargets.add(f.resolve(t));}async function Xt(e){try{let t=await y.readFile(Ye(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return Ze()}function $e(e){return JSON.parse(JSON.stringify(e))}async function Yt(e,t){let r=Ye(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function Zt(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function er(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:u,status:o.status()}),new Map;let a=await o.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let c of a.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(o)}),new Map}}async function Ue(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await er(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function tr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await Ue(e,t,s,n,i)};let o=await Ue(e,t,s,n,i);if(o?.supported&&o.token)return u.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let a=G(r.sitemapLastmodByPage[n]);return a&&u.sitemapLastmod&&u.sitemapLastmod===a?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function rr(e,t,r,s,n){let i=Qe(r.previousManifest),u=0;for(let[d,h]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Oe(n,e,h.outputPath);try{await y.unlink(h.outputPath);}catch(p){(p.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:h.outputPath,error:String(p)});}delete r.manifest.pages[d],u++;}let o=Qe(r.manifest,r.runId),a=new Set,l=new Set;for(let d of o){for(let P of N(e,d))a.add(P);let h=r.manifest.assets[d]?.outputPath;h&&l.add(h);let p=K(e,t.assetMap,d);p&&l.add(p);}for(let d of Object.keys(r.manifest.assets))o.has(d)||delete r.manifest.assets[d];let c=0,m=0;for(let d of i){let h=new Set,p=r.previousManifest.assets[d]?.outputPath;p&&h.add(p);let P=K(e,t.assetMap,d);if(!o.has(d)&&P)h.add(P);else if(!o.has(d)){let g=we(e,d);h.add(g.originalFilePath),g.hashedFilePath&&h.add(g.hashedFilePath);}for(let g of h)if(!l.has(g)){Oe(n,e,g);try{await y.unlink(g),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:g,error:String(w)});}}for(let g of N(e,d))a.has(g)||Object.prototype.hasOwnProperty.call(t.assetMap,g)&&(delete t.assetMap[g],m++);}(u>0||c>0||m>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${m} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:m,phase:"incremental-cleanup"});}function Qe(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function Ne(e,t,r){for(let s of t)for(let n of N(e,s))if(r.has(n))return true;return false}function sr(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=Zt(s,n);if(u.size===0)return [...i];for(let o of Object.values(t.manifest.pages))Ne(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let[o,a]of Object.entries(t.manifest.assets)){a.isText&&Ne(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let l of N(e,o))if(u.has(l)){D(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function N(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function nr(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ir(e,t,r){for(let s of N(e,r)){let n=t[s];if(!n)continue;let i=nr(e,n);if(i)return i}return null}function K(e,t,r){let s=ir(e,t,r);return s||we(e,r).originalFilePath}function or(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function qe(e){let t=or();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function ar(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Y(e,s)));for(let s of t.doneAssets){let n=K(e,t.assetMap,s);!n||!$(n)||r.add(f.resolve(n));}return [...r]}var X="re:",B=new Map;function ur(e){if(!e.startsWith(X))return null;let t=B.get(e);if(t!==void 0)return t||null;let r=e.slice(X.length).trim();if(!r)return B.set(e,false),null;try{let s=new RegExp(r);return B.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),B.set(e,false),null}}function fe(e,t){let r=ur(t);return r?r.test(e):t.startsWith(X)?false:e.startsWith(t)}function lr(e,t){return t.find(r=>fe(e,r))}function et(e,t){let r=lr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(X)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function cr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function dr(e,t){return cr(e).has(t.hostname)}function pr(e){let t=E(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function tt(e){let t=E(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function ee(e){return Jt.some(t=>e.pathname.startsWith(t))}function R(e,t,r=e.sourceOrigin,s,n="url"){let i=pr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(tt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!dr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let o=et(e,u);if(o)return s?.reject(n,u.toString(),o,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function rt(e){return _t.has(f.extname(e.pathname).toLowerCase())}function _(e){return ee(e)?false:Wt.has(f.extname(e.pathname).toLowerCase())}function te(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function b(e,t){return te(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:rt(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>fe(t.pathname,r)):false}function Y(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function we(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),o=ue(e.targetOrigin,i),a=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=f.extname(s),m=`${l?s.slice(0,-l.length):s}.${ke(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:m,hashedFilePath:f.join(e.outputDir,m),hashedPublicUrl:ue(e.targetOrigin,m)}}function gr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let a of o)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function ge(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function st(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function hr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function nt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function mr(e,t){let r=nt(t),s=hr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function fr(e,t){if(!t)return;let r=nt(t);if(r===null)return;let s=await st(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function de(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Ht.has(r)||qt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function _e(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function he(e,t,r,s,n,i,u){let o=we(e,r),a=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let d=await ge(o.originalFilePath);d!==null&&!d.equals(s)&&(a=o.hashedFilePath,l=o.hashedPublicUrl);}gr(e,t,r,l,o.originalPublicUrl);let c=await ge(a),m=false;return (c===null||!c.equals(s))&&(await ae(a),await y.writeFile(a,s),m=true),await fr(a,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:l,wroteFile:m}}function Z(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"page");if(!u)return;if(!_(u)||rt(u)){if(b(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),T(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),ee(u)?"asset/internal path cannot be page":"not page-like",n);return}let o=u.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function T(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"asset");if(!u)return;if(!b(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let o=u.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Pe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"sitemap");if(!u)return;if(!te(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let o=u.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function wr(e){let t=E(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Pr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function yr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function pe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function it(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await it(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Sr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,o=i.prefix;if(!u||!o){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:o});continue}let a=Gt(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:o});continue}let l=a.resolvedPath,c=pe(o);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:o});continue}let m;try{m=await y.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:l,prefixPath:o});continue}let d=f.resolve(e.outputDir,c);if(m.isDirectory()){n.push(...await it(l,d)),s++,t.progress(`Copied static directory to export: ${l} -> /${c}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${c}`,copiedItems:s});continue}let h=f.basename(l),p=o.endsWith("/")?f.join(d,h):d;await y.mkdir(f.dirname(p),{recursive:true}),await y.copyFile(l,p),n.push(p),s++,t.progress(`Copied static file to export: ${l} -> /${pe(o.endsWith("/")?`${o}${h}`:o)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${pe(o.endsWith("/")?`${o}${h}`:o)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function q(e,t,r,s,n){let i=new Set,u=E(r).replace(/\\\//g,"/");for(let l of yr(u)){let c=R(e,l,t,s,`${n}:css-url`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=R(e,l[1],t,s,`${n}:css-import`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=R(e,l[1],t,s,`${n}:xml-stylesheet`);c&&b(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of u.matchAll(a)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(tt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let m=R(e,c,t,s,`${n}:serialized-url`);m&&b(e,m)?i.add(m.toString()):m&&s.ignore("asset",m.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function vr(e,t,r,s){let n=new Set,i=E(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=u[1],a=R(e,o,t,s,"page-link");a&&(et(e,a)||_(a)&&!ee(a)&&!b(e,a)&&n.add(a.toString()));}return [...n]}async function We(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),o=i.url()||n;await he(e,r.assetMap,o,u,r,s);let a=u.toString("utf8");for(let l of q(e,o,a,s,"sitemap"))T(e,r,l,s,o);for(let l of wr(a)){let c=R(e,l.loc,o,s,"sitemap-loc");c&&(te(c)?Pe(e,r,c.toString(),s,o):b(e,c)?T(e,r,c.toString(),s,o):_(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),Z(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function br(e,t,r,s,n,i,u){let o=R(e,s,e.sourceOrigin,n,"asset-fetch");if(!o||!b(e,o))return;let a=o.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let l=K(e,r.assetMap,a),c=l?await st(l):null;if(l&&c)try{let w=await t.head(a,{timeout:6e4}),S=w.headers();if(w.ok()&&mr(c,S)){let v=await ge(l);if(v!==null){let I=w.url()||a,C=await he(e,r.assetMap,I,v,r,n,S);C.wroteFile&&(D(u,e,C.outputPath),$(C.outputPath)&&V(u,C.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:l,lastModified:S["last-modified"]||"",contentLength:S["content-length"]||"",phase:"download-assets"}),_e(r,n);let x=i.enabled?i.previousManifest.assets[a]:void 0,ye=de(o,S),re=x?.discoveredAssets?[...x.discoveredAssets]:[];if(ye||x?.isText){let at=v.toString("utf8"),Se=q(e,I,at,n,"asset:cached");(Se.length>0||!x)&&(re=Se);for(let ut of re)T(e,r,ut,n,I);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:C.outputPath,isText:ye||!!x?.isText,discoveredAssets:re,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let m=await t.get(a,{timeout:6e4});if(!m.ok()){n.skip("asset",a,m.status());return}let d=await m.body(),h=m.url()||a,p=m.headers(),P=await he(e,r.assetMap,h,d,r,n,p);P.wroteFile&&(D(u,e,P.outputPath),$(P.outputPath)&&V(u,P.outputPath)),r.doneAssets.add(a),_e(r,n);let g=[];if(de(o,p)){let w=d.toString("utf8");g=q(e,h,w,n,`asset:${f.extname(o.pathname).toLowerCase()||p["content-type"]||"unknown"}`);for(let S of g)T(e,r,S,n,h);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:P.outputPath,isText:de(o,p),discoveredAssets:g,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(l){n.error(`Failed asset ${a}`,{url:a,error:String(l)});}}}async function Rr(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),o=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await br(e,t,r,a,s,n,i));}});await Promise.all(o);}async function Cr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function xr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Cr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ar(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function kr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>fe(t,r))}async function Mr(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,m=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let p of l){let P=d.getAttribute(p);P&&a.add(P);}for(let p of m){let P=d.getAttribute(p);P&&P.split(",").forEach(g=>{let w=g.trim().split(/\s+/)[0];w&&a.add(w);});}let h=d.getAttribute("style");h&&c.add(h);}),{attrs:[...a],styles:[...c]}});for(let a of o.attrs)for(let l of Pr(a)){let c=R(e,l,r,s,"dom-attr");c&&b(e,c)&&i.add(c.toString());}for(let a of o.styles)for(let l of q(e,r,a,s,"dom-style"))i.add(l);}let u=await t.content();for(let o of q(e,r,u,s,"page-html"))i.add(o);return [...i]}async function He(e,t,r,s,n){let i=Y(e,t);await ae(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Je(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function ze(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Tr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Lr(e,t,r,s,n,i,u){let o=await chromium.launch({headless:true}),a=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Tr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let c=t.pageQueue.shift();if(!c)break;if(t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),i.enabled&&i.seenPages.add(c);let m=new URL(c),d=kr(e,m.pathname);if(!_(m)||b(e,m)||ee(m)){if(b(e,m)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),T(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let h=await tr(e,n,t,i,c,r);if(i.enabled&&s!=="single-url"){let g=i.manifest.pages[c];if(h.action==="reuse"&&g){for(let w of g.discoveredAssets)T(e,t,w,r,c);for(let w of g.discoveredPages)Z(e,t,w,r,c);g.lastSeenRunId=i.runId,g.sitemapLastmod=G(t.sitemapLastmodByPage[c])??g.sitemapLastmod,h.changeToken?.supported&&(g.changeToken=h.changeToken.token,g.tokenSource=h.changeToken.tokenSource??g.tokenSource),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:h.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Je(t)&&ze(t,r);continue}}let p=await(d?l.newPage():a.newPage()),P=!1;await p.route("**/*",async g=>{let w=g.request();if(w.isNavigationRequest()&&w.frame()===p.mainFrame())try{let S=new URL(w.url()),v=new URL(c);if(S.hash="",v.hash="",S.href===v.href){if(P){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:w.url()}),await g.abort("aborted");return}P=!0;}}catch{}await g.continue();}),p.on("response",g=>{try{let w=R(e,g.url(),e.sourceOrigin,void 0,"network-response");w&&b(e,w)&&T(e,t,w.toString(),r,c);}catch{}}),r.page(`Rendering ${c}`,{url:c});try{let g=null,w=null;try{if(g=await p.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),g&&g.ok())try{w=await g.text();}catch{w=null;}}catch(x){r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(x)});}if(g&&!g.ok()){r.skip("page",c,g.status()),await p.close();continue}d?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await Ar(e,p)):await xr(e,p),t.stats.pagesRendered++,Je(t)&&ze(t,r);let S=await Mr(e,p,c,r,!d);for(let x of S)T(e,t,x,r,c);let v=w??await p.content(),I=s!=="single-url"?vr(e,c,v,r):[];if(s!=="single-url")for(let x of I)Z(e,t,x,r,c);await He(e,c,v,t,r);let C=Y(e,c);D(u,e,C),V(u,C),i.enabled&&(i.manifest.pages[c]={url:c,outputPath:C,changeToken:h.changeToken?.supported?h.changeToken.token:null,tokenSource:h.changeToken?.supported?h.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[c])??null,discoveredPages:I,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){try{let w=await p.content();if(w&&w.trim()){await He(e,c,w,t,r);let S=Y(e,c);D(u,e,S),V(u,S),r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(g)});}}catch(w){r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(w)});}r.error(`Failed page ${c}`,{url:c,error:String(g)});}finally{await p.close();}}}finally{await l.close(),await a.close(),await o.close();}}async function Er(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Be(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Dr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ir(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function ot(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ve(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Fr(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ve(s)-Ve(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Ge(e){let t=await ot(f.join(e,"job.json"));for(let r of Fr(e,t)){let s=await Er(r);if(s.length>0)return s}return []}async function jr(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let a=await ot(f.join(u,"job.json"));if(!Dr(a))continue;let l=Ir(a);l>=n&&(n=l,s=u);}return s}async function Or(e){let t=await jr(e),r=t!==""?await Ge(t):await Ge(e.logDir),s=new Set;for(let n of r)if(Be(n.error)||Be(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function $r(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Ur(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Qr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ke(e,t,r,s,n){let i=R(e,r,e.sourceOrigin,s,"manual-url");i&&(te(i)?Pe(e,t,i.toString(),s,n):b(e,i)?T(e,t,i.toString(),s,n):_(i)?Z(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Nr(){let e=await oe(),t=zt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Or(e)),t.mode==="single-url"&&(r=await $r(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.preserveOutput?await Ur(e):{},i=s?await Xt(e):Ze(),u={enabled:s,manifest:$e(i),previousManifest:$e(i),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Qr(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...n},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},a=Kt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new Q(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let p=await chromium.launch({headless:true}),P=await p.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),g=P.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>Pe(e,o,S,l)),await We(e,g,o,l),e.seedPaths.forEach(S=>Ke(e,o,S,l,"seed-path"))):(r.forEach(S=>Ke(e,o,S,l,"cli")),await We(e,g,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let w=Array.from({length:Math.max(1,e.concurrency)},()=>Lr(e,o,l,t.mode,g,u,a));if(await Promise.all(w),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Rr(e,g,o,l,u,a),l.endMark("download-assets",{assets:o.doneAssets.size}),u.enabled&&e.maxPages===0&&await rr(e,o,u,l,a),t.mode==="full"){let S=await Sr(e,l);for(let v of S)D(a,e,v);}}finally{await P.close(),await p.close();}}u.enabled&&!t.resumeRewrite&&(u.manifest.updatedAt=new Date().toISOString(),await Yt(e,u.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(n,null,2)!==JSON.stringify(o.assetMap,null,2)&&D(a,e,f.join(e.outputDir,"asset-map.json"));let c=t.mode==="single-url"?ar(e,o):u.enabled&&!t.resumeRewrite?sr(e,u,a,n,o.assetMap):void 0;l.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${c?.length??0}`:"Rewriting text files...";l.progress(m,{phase:"rewrite-text",index:0,totalFiles:c?.length,changedTextFiles:0}),await qe(m);let d=Date.now(),h=await De(e,o.assetMap,{files:c,previousAssetMap:n,onProgress:async({index:p,totalFiles:P,changedTextFiles:g,file:w,changed:S})=>{S&&D(a,e,f.join(e.outputDir,w)),l.checkpoint(`Rewriting text file ${p}/${P}`,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w});let v=Date.now();if(!(p===1||p===P||v-d>=5e3))return;d=v;let C=`Rewriting text files: ${p}/${P}`;l.progress(C,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w}),await qe(C);}});l.endMark("rewrite-text",{changedTextFiles:h}),await Re(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...a.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...a.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((c||[]).map(p=>W(e.outputDir,f.isAbsolute(p)?p:f.join(e.outputDir,p))).filter(p=>p!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${h} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:h}),await l.flush();}Nr().catch(async e=>{console.error(e);try{let r=(await oe().catch(()=>null))?.logDir??"logs",s=new Q(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@smart-cloud/publisher-exporter",
3
- "version": "1.0.10",
3
+ "version": "1.0.11",
4
4
  "license": "MIT",
5
5
  "type": "module",
6
6
  "description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",