@smart-cloud/publisher-exporter 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +3 -3
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import y from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import St from'crypto';import bt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var pt=".deploy-plan.json",gt="deploy-plan.json";function ht(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function xe(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ne(e){return [...new Set(e.map(xe).filter(t=>t!==null))]}function
|
|
1
|
+
import {chromium}from'playwright';import y from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import St from'crypto';import bt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var pt=".deploy-plan.json",gt="deploy-plan.json";function ht(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function xe(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ne(e){return [...new Set(e.map(xe).filter(t=>t!==null))]}function ft(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ne(t.changedFiles||[]),deletedFiles:ne(t.deletedFiles||[]),rewriteTargets:ne(t.rewriteTargets||[])}}function mt(e){let t=f.join(f.resolve(e.outputDir),pt),r=ht();return r?[f.join(r,gt),t]:[t]}function H(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return xe(n)}async function Ae(e,t){let r=ft(e,t);for(let s of mt(e))await y.mkdir(f.dirname(s),{recursive:true}),await y.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Me(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function ie(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Pt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let o=Me(i.targetOrigin);o&&(u.targetOrigin=o);}let a=ie(i.extraReplacements);if(Object.keys(a).length>0&&(u.extraReplacements=a),i.s3&&typeof i.s3=="object"){let o={},c=i.s3;typeof c.bucket=="string"&&(o.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(o.prefix=c.prefix.trim()),typeof c.region=="string"&&(o.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(o.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(o.assetCacheControl=c.assetCacheControl.trim()),Object.keys(o).length>0&&(u.s3=o);}if(i.cloudFront&&typeof i.cloudFront=="object"){let o={},c=i.cloudFront;typeof c.distributionId=="string"&&(o.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(o.invalidationPaths=c.invalidationPaths.map(l=>String(l??"").trim()).filter(l=>l.length>0)),Object.keys(o).length>0&&(u.cloudFront=o);}t[n]=u;}return t}function oe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function yt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function ke(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,oe(s,r))}async function ae(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await y.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=Me(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=ie(r.extraReplacements),r.postCrawlCopyMap=ie(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=Pt(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=yt(),n=s?f.resolve(s,".."):"";return n?(r.outputDir=ke(n,r.outputDir,"export"),r.logDir=ke(n,r.logDir,"logs")):(f.isAbsolute(r.outputDir)||(r.outputDir=oe(r.outputDir,"export")),f.isAbsolute(r.logDir)||(r.logDir=oe(r.logDir,"logs"))),r}function Le(e,t=10){return St.createHash("sha1").update(e).digest("hex").slice(0,t)}async function ue(e){await y.mkdir(f.dirname(e),{recursive:true});}function j(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function D(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function O(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function J(e){return e.replace(/\//g,"\\/")}function Te(e){return e.replace(/\//g,"\\\\/")}function z(e,t,r){if(!t)return;e[t]=r;let s=J(t),n=J(r);e[s]=n;let i=Te(t),u=Te(r);e[i]=u;let a=O(t),o=O(r);e[a]=o;let c=O(s),l=O(n);e[c]=l;let g=O(i),d=O(u);e[g]=d;}function le(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Ie(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function De(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var xt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],At=new Set([".html",".htm"]),Fe="WPSuite.io Static Publisher",kt=Fe.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Mt(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=f.extname(r).toLowerCase();return At.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Tt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${kt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Fe}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Lt(e){let t=D(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function It(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[o
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Lt(e){let t=D(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function It(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[a,o]of r)u=u.split(a).join(o);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,a,o)=>`${u}${s(a)}${o}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,a,o,c)=>Lt(o)?`${u}${s(o)}${c}`:i),n}function Dt(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Et(e,t,r){let s=Dt(t,r);if(!s)return e;let n=e;for(let i of xt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),a=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),o=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(a,`${s}${i}`).replace(o,`${s}${i}`);let c=i.replace(/\//g,"\\/"),l=s.replace(/\//g,"\\/"),g=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${g}`,"g"),p=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${g}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${g}`,"g");n=n.replace(d,`${l}${c}`).replace(p,`${l}${c}`).replace(m,`${l}${c}`);}return n}function de(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:Ie(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${De(e.outputDir,r,n)}${i}`}function ce(e,t,r,s,n){z(e,r,de(t,s,n));}function Ft(e,t,r,s,n){if(!r||r===s)return;z(e,r,s);let i=de(t,r,n),u=de(t,s,n);z(e,i,u);}function pe(e,t,r,s,n={}){let i=e,u={};ce(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[o,c]of Object.entries(t.extraReplacements))ce(u,t,o,c,s);for(let[o,c]of Object.entries(r))ce(u,t,o,c,s);for(let[o,c]of Object.entries(n)){let l=r[o];!l||l===c||Ft(u,t,c,l,s);}let a=Object.entries(u).sort((o,c)=>c[0].length-o[0].length);for(let[o,c]of a)i=i.split(o).join(c);return t.urlRewriteMode!=="absolute"&&(i=Et(i,t,s)),i=It(i,a.map(([,o])=>o)),Mt(t,i,s)&&(i=Tt(i)),i}async function jt(e,t,r,s){let n=f.join(e.outputDir,s),i=f.extname(n).toLowerCase(),u;try{u=await y.readFile(n,"utf8");}catch(c){if(c.code!=="ENOENT")throw c;return {changed:false}}let o=pe(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return o!==u?(await y.writeFile(n,o,"utf8"),{changed:true}):{changed:false}}async function Ot(e,t){return await new Promise((r,s)=>{let n=o=>{if(a(),o?.error){s(new Error(o.error));return}r({changed:!!o?.changed});},i=o=>{a(),s(o);},u=o=>{a(),o!==0&&s(new Error(`Rewrite worker exited with code ${o}.`));},a=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function je(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>f.resolve(d)).filter(d=>d.startsWith(f.resolve(e.outputDir))).map(d=>f.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await bt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&j(d)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),a=Math.max(1,Math.min(u,n.length,availableParallelism())),o=0,c=0,l=null;if(a===1){for(let d of n){let p=await jt(e,t,r.previousAssetMap??{},d);p.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:d,changed:p.changed});}return i}let g=Array.from({length:a},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!l;){let p=o;if(p>=n.length)return;o+=1;let m=n[p],P=await Ot(d,m);P.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:m,changed:P.changed});}}catch(p){throw l=p instanceof Error?p:new Error(String(p)),l}finally{await d.terminate().catch(()=>{});}});return await Promise.all(g),i}var U={error:0,warn:1,info:2,debug:3},Oe=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",B=Oe?f.join(Oe,"current-progress.json"):"",$e=Promise.resolve();function Ue(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function $t(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function Ut(e,t,r){B&&($e=$e.then(async()=>{let s=new Date,n=s.toISOString();await y.mkdir(f.dirname(B),{recursive:true});let i=await y.readFile(B,"utf8").then(P=>JSON.parse(P)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},a=Ue(e),o=i?.currentStep||Ue(i?.source||""),c=i?.startedAt||n,l={...i?.stepDurationsSec??{}},g=i?.stepStartedAt||n;if(o&&o!==a&&i?.stepStartedAt){let P=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));l[o]=(l[o]??0)+P,g=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(g).getTime())/1e3)),p=Math.max(0,Math.round((s.getTime()-new Date(c).getTime())/1e3)),m={checkedAt:n,source:e,message:t,details:$t(u,r??{}),startedAt:c,currentStep:a,stepStartedAt:g,stepElapsedSec:d,totalElapsedSec:p,stepDurationsSec:l};await y.writeFile(B,JSON.stringify(m,null,2),"utf8");}).catch(()=>{}));}async function Qt(e,t){await y.mkdir(f.dirname(e),{recursive:true}),await y.appendFile(e,`${JSON.stringify(t)}
|
|
5
5
|
`,"utf8");}function Nt(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function qt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:Nt(e,r),level:e,message:t,details:r??{}}}var N=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await y.appendFile(this.logPath,`${t}
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Qt(t,r));}updateCurrentEvent(t,r,s){let n=qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Wt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Ht=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Jt=new Set(["",".html",".htm"]),zt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Bt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Vt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let o=e[u];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Pe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Kt(){let e=Pe();return e?f.resolve(e,".."):""}function Gt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Xt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Kt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Pe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Gt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function et(e){let t=Pe();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function tt(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Yt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function E(e,t,r){let s=H(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Qe(e,t,r){let s=H(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function K(e,t){e.rewriteTargets.add(f.resolve(t));}async function Zt(e){try{let t=await y.readFile(et(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return tt()}function Ne(e){return JSON.parse(JSON.stringify(e))}async function er(e,t){let r=et(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function tr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function rr(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:u,status:o.status()}),new Map;let a=await o.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let c of a.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(o)}),new Map}}async function qe(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await rr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function sr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await qe(e,t,s,n,i)};let o=await qe(e,t,s,n,i);if(o?.supported&&o.token)return u.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let a=G(r.sitemapLastmodByPage[n]);return a&&u.sitemapLastmod&&u.sitemapLastmod===a?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function nr(e,t,r,s,n){let i=_e(r.previousManifest),u=0;for(let[d,p]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Qe(n,e,p.outputPath);try{await y.unlink(p.outputPath);}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:p.outputPath,error:String(g)});}delete r.manifest.pages[d],u++;}let o=_e(r.manifest,r.runId),a=new Set,l=new Set;for(let d of o){for(let P of q(e,d))a.add(P);let p=r.manifest.assets[d]?.outputPath;p&&l.add(p);let g=X(e,t.assetMap,d);g&&l.add(g);}for(let d of Object.keys(r.manifest.assets))o.has(d)||delete r.manifest.assets[d];let c=0,h=0;for(let d of i){let p=new Set,g=r.previousManifest.assets[d]?.outputPath;g&&p.add(g);let P=X(e,t.assetMap,d);if(!o.has(d)&&P)p.add(P);else if(!o.has(d)){let m=Se(e,d);p.add(m.originalFilePath),m.hashedFilePath&&p.add(m.hashedFilePath);}for(let m of p)if(!l.has(m)){Qe(n,e,m);try{await y.unlink(m),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:m,error:String(w)});}}for(let m of q(e,d))a.has(m)||Object.prototype.hasOwnProperty.call(t.assetMap,m)&&(delete t.assetMap[m],h++);}(u>0||c>0||h>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${h} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:h,phase:"incremental-cleanup"});}function _e(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function We(e,t,r){for(let s of t)for(let n of q(e,s))if(r.has(n))return true;return false}function ir(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=tr(s,n);if(u.size===0)return [...i];for(let o of Object.values(t.manifest.pages))We(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let[o,a]of Object.entries(t.manifest.assets)){a.isText&&We(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let l of q(e,o))if(u.has(l)){E(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function q(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function or(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ar(e,t,r){for(let s of or(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function ur(e,t,r){for(let s of q(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return i}return null}function X(e,t,r){let s=ur(e,t,r);return s||Se(e,r).originalFilePath}function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function He(e){let t=lr();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function cr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Z(e,s)));for(let s of t.doneAssets){let n=X(e,t.assetMap,s);!n||!j(n)||r.add(f.resolve(n));}return [...r]}var Y="re:",V=new Map;function dr(e){if(!e.startsWith(Y))return null;let t=V.get(e);if(t!==void 0)return t||null;let r=e.slice(Y.length).trim();if(!r)return V.set(e,false),null;try{let s=new RegExp(r);return V.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),V.set(e,false),null}}function ye(e,t){let r=dr(t);return r?r.test(e):t.startsWith(Y)?false:e.startsWith(t)}function pr(e,t){return t.find(r=>ye(e,r))}function st(e,t){let r=pr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Y)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function gr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function hr(e,t){return gr(e).has(t.hostname)}function mr(e){let t=D(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=D(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function te(e){return Bt.some(t=>e.pathname.startsWith(t))}function R(e,t,r=e.sourceOrigin,s,n="url"){let i=mr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!hr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let o=st(e,u);if(o)return s?.reject(n,u.toString(),o,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function it(e){return Ht.has(f.extname(e.pathname).toLowerCase())}function W(e){return te(e)?false:Jt.has(f.extname(e.pathname).toLowerCase())}function re(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function b(e,t){return re(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:it(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>ye(t.pathname,r)):false}function Z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function Se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),o=le(e.targetOrigin,i),a=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=f.extname(s),h=`${l?s.slice(0,-l.length):s}.${Le(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:h,hashedFilePath:f.join(e.outputDir,h),hashedPublicUrl:le(e.targetOrigin,h)}}function fr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let a of o)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function fe(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function wr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function at(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Pr(e,t){let r=at(t),s=wr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function yr(e,t){if(!t)return;let r=at(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function ge(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return zt.has(r)||Wt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function he(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function we(e,t,r,s,n,i,u){let o=Se(e,r),a=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let d=ar(e,t,r);if(d)a=d.outputPath,l=d.publicUrl;else {let p=await fe(o.originalFilePath);if(p!==null&&!p.equals(s)){let g=j(o.originalFilePath)?Buffer.from(pe(s.toString("utf8"),e,t,o.originalFilePath),"utf8"):null;(g===null||!p.equals(g))&&(a=o.hashedFilePath,l=o.hashedPublicUrl);}}}fr(e,t,r,l,o.originalPublicUrl);let c=await fe(a),h=false;return (c===null||!c.equals(s))&&(await ue(a),await y.writeFile(a,s),h=true),await yr(a,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:l,wroteFile:h}}function ee(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"page");if(!u)return;if(!W(u)||it(u)){if(b(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),x(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),te(u)?"asset/internal path cannot be page":"not page-like",n);return}let o=u.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function x(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"asset");if(!u)return;if(!b(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let o=u.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ve(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"sitemap");if(!u)return;if(!re(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let o=u.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Sr(e){let t=D(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function vr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function br(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function me(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function ut(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await ut(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Rr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,o=i.prefix;if(!u||!o){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:o});continue}let a=Xt(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:o});continue}let l=a.resolvedPath,c=me(o);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:o});continue}let h;try{h=await y.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:l,prefixPath:o});continue}let d=f.resolve(e.outputDir,c);if(h.isDirectory()){n.push(...await ut(l,d)),s++,t.progress(`Copied static directory to export: ${l} -> /${c}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${c}`,copiedItems:s});continue}let p=f.basename(l),g=o.endsWith("/")?f.join(d,p):d;await y.mkdir(f.dirname(g),{recursive:true}),await y.copyFile(l,g),n.push(g),s++,t.progress(`Copied static file to export: ${l} -> /${me(o.endsWith("/")?`${o}${p}`:o)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${me(o.endsWith("/")?`${o}${p}`:o)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function _(e,t,r,s,n){let i=new Set,u=D(r).replace(/\\\//g,"/");for(let l of br(u)){let c=R(e,l,t,s,`${n}:css-url`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=R(e,l[1],t,s,`${n}:css-import`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=R(e,l[1],t,s,`${n}:xml-stylesheet`);c&&b(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of u.matchAll(a)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(nt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let h=R(e,c,t,s,`${n}:serialized-url`);h&&b(e,h)?i.add(h.toString()):h&&s.ignore("asset",h.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Cr(e,t,r,s){let n=new Set,i=D(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=u[1],a=R(e,o,t,s,"page-link");a&&(st(e,a)||W(a)&&!te(a)&&!b(e,a)&&n.add(a.toString()));}return [...n]}async function Je(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),o=i.url()||n;await we(e,r.assetMap,o,u,r,s);let a=u.toString("utf8");for(let l of _(e,o,a,s,"sitemap"))x(e,r,l,s,o);for(let l of Sr(a)){let c=R(e,l.loc,o,s,"sitemap-loc");c&&(re(c)?ve(e,r,c.toString(),s,o):b(e,c)?x(e,r,c.toString(),s,o):W(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),ee(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function xr(e,t,r,s,n,i,u){let o=R(e,s,e.sourceOrigin,n,"asset-fetch");if(!o||!b(e,o))return;let a=o.toString();if(r.doneAssets.has(a))return;let l=i.enabled?i.previousManifest.assets[a]:void 0,c=(h,d={})=>{if(!l)return false;i.manifest.assets[a]={...l,discoveredAssets:[...l.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let p of l.discoveredAssets||[])x(e,r,p,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:h,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),he(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let h=X(e,r.assetMap,a),d=h?await ot(h):null;if(h&&d)try{let v=await t.head(a,{timeout:6e4}),C=v.headers();if(v.ok()&&Pr(d,C)){let A=await fe(h);if(A!==null){let L=v.url()||a,Q=await we(e,r.assetMap,L,A,r,n,C);Q.wroteFile&&(E(u,e,Q.outputPath),j(Q.outputPath)&&K(u,Q.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:h,lastModified:C["last-modified"]||"",contentLength:C["content-length"]||"",phase:"download-assets"}),he(r,n);let be=ge(o,C),se=l?.discoveredAssets?[...l.discoveredAssets]:[];if(be||l?.isText){let ct=A.toString("utf8"),Re=_(e,L,ct,n,"asset:cached");(Re.length>0||!l)&&(se=Re);for(let dt of se)x(e,r,dt,n,L);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:Q.outputPath,isText:be||!!l?.isText,discoveredAssets:se,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let p=await t.get(a,{timeout:6e4});if(!p.ok()){if(c(`http-${p.status()}`,{status:p.status()}))return;n.skip("asset",a,p.status());return}let g=await p.body(),P=p.url()||a,m=p.headers(),w=await we(e,r.assetMap,P,g,r,n,m);w.wroteFile&&(E(u,e,w.outputPath),j(w.outputPath)&&K(u,w.outputPath)),r.doneAssets.add(a),he(r,n);let S=[];if(ge(o,m)){let v=g.toString("utf8");S=_(e,P,v,n,`asset:${f.extname(o.pathname).toLowerCase()||m["content-type"]||"unknown"}`);for(let C of S)x(e,r,C,n,P);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:ge(o,m),discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(h){if(c("fetch-error",{error:String(h)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(h)});}}async function Ar(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),o=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await xr(e,t,r,a,s,n,i));}});await Promise.all(o);}async function kr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Mr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await kr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Tr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Lr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ye(t,r))}async function Ir(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,h=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let g of l){let P=d.getAttribute(g);P&&a.add(P);}for(let g of h){let P=d.getAttribute(g);P&&P.split(",").forEach(m=>{let w=m.trim().split(/\s+/)[0];w&&a.add(w);});}let p=d.getAttribute("style");p&&c.add(p);}),{attrs:[...a],styles:[...c]}});for(let a of o.attrs)for(let l of vr(a)){let c=R(e,l,r,s,"dom-attr");c&&b(e,c)&&i.add(c.toString());}for(let a of o.styles)for(let l of _(e,r,a,s,"dom-style"))i.add(l);}let u=await t.content();for(let o of _(e,r,u,s,"page-html"))i.add(o);return [...i]}async function ze(e,t,r,s,n){let i=Z(e,t);await ue(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Be(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ve(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Dr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Er(e,t,r,s,n,i,u){let o=await chromium.launch({headless:true}),a=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Dr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let c=t.pageQueue.shift();if(!c)break;if(t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),i.enabled&&i.seenPages.add(c);let h=new URL(c),d=Lr(e,h.pathname);if(!W(h)||b(e,h)||te(h)){if(b(e,h)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),x(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let p=await sr(e,n,t,i,c,r);if(i.enabled&&s!=="single-url"){let m=i.manifest.pages[c];if(p.action==="reuse"&&m){for(let w of m.discoveredAssets)x(e,t,w,r,c);for(let w of m.discoveredPages)ee(e,t,w,r,c);m.lastSeenRunId=i.runId,m.sitemapLastmod=G(t.sitemapLastmodByPage[c])??m.sitemapLastmod,p.changeToken?.supported&&(m.changeToken=p.changeToken.token,m.tokenSource=p.changeToken.tokenSource??m.tokenSource),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Be(t)&&Ve(t,r);continue}}let g=await(d?l.newPage():a.newPage()),P=!1;await g.route("**/*",async m=>{let w=m.request();if(w.isNavigationRequest()&&w.frame()===g.mainFrame())try{let S=new URL(w.url()),v=new URL(c);if(S.hash="",v.hash="",S.href===v.href){if(P){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:w.url()}),await m.abort("aborted");return}P=!0;}}catch{}await m.continue();}),g.on("response",m=>{try{let w=R(e,m.url(),e.sourceOrigin,void 0,"network-response");w&&b(e,w)&&x(e,t,w.toString(),r,c);}catch{}}),r.page(`Rendering ${c}`,{url:c});try{let m=null,w=null;try{if(m=await g.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{w=await m.text();}catch{w=null;}}catch(L){r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(L)});}if(m&&!m.ok()){r.skip("page",c,m.status()),await g.close();continue}d?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await Tr(e,g)):await Mr(e,g),t.stats.pagesRendered++,Be(t)&&Ve(t,r);let S=await Ir(e,g,c,r,!d);for(let L of S)x(e,t,L,r,c);let v=w??await g.content(),C=s!=="single-url"?Cr(e,c,v,r):[];if(s!=="single-url")for(let L of C)ee(e,t,L,r,c);await ze(e,c,v,t,r);let A=Z(e,c);E(u,e,A),K(u,A),i.enabled&&(i.manifest.pages[c]={url:c,outputPath:A,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[c])??null,discoveredPages:C,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let w=await g.content();if(w&&w.trim()){await ze(e,c,w,t,r);let S=Z(e,c);E(u,e,S),K(u,S),r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(m)});}}catch(w){r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(w)});}r.error(`Failed page ${c}`,{url:c,error:String(m)});}finally{await g.close();}}}finally{await l.close(),await a.close(),await o.close();}}async function Fr(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ke(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function jr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Or(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function lt(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ge(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function $r(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ge(s)-Ge(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Xe(e){let t=await lt(f.join(e,"job.json"));for(let r of $r(e,t)){let s=await Fr(r);if(s.length>0)return s}return []}async function Ur(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let a=await lt(f.join(u,"job.json"));if(!jr(a))continue;let l=Or(a);l>=n&&(n=l,s=u);}return s}async function Qr(e){let t=await Ur(e),r=t!==""?await Xe(t):await Xe(e.logDir),s=new Set;for(let n of r)if(Ke(n.error)||Ke(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function qr(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function _r(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ye(e,t,r,s,n){let i=R(e,r,e.sourceOrigin,s,"manual-url");i&&(re(i)?ve(e,t,i.toString(),s,n):b(e,i)?x(e,t,i.toString(),s,n):W(i)?ee(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Wr(){let e=await ae(),t=Vt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Qr(e)),t.mode==="single-url"&&(r=await Nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.preserveOutput?await qr(e):{},i=s?await Zt(e):tt(),u={enabled:s,manifest:Ne(i),previousManifest:Ne(i),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await _r(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...n},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},a=Yt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new N(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let g=await chromium.launch({headless:true}),P=await g.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),m=P.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>ve(e,o,S,l)),await Je(e,m,o,l),e.seedPaths.forEach(S=>Ye(e,o,S,l,"seed-path"))):(r.forEach(S=>Ye(e,o,S,l,"cli")),await Je(e,m,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let w=Array.from({length:Math.max(1,e.concurrency)},()=>Er(e,o,l,t.mode,m,u,a));if(await Promise.all(w),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Ar(e,m,o,l,u,a),l.endMark("download-assets",{assets:o.doneAssets.size}),u.enabled&&e.maxPages===0&&await nr(e,o,u,l,a),t.mode==="full"){let S=await Rr(e,l);for(let v of S)E(a,e,v);}}finally{await P.close(),await g.close();}}u.enabled&&!t.resumeRewrite&&(u.manifest.updatedAt=new Date().toISOString(),await er(e,u.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(n,null,2)!==JSON.stringify(o.assetMap,null,2)&&E(a,e,f.join(e.outputDir,"asset-map.json"));let c=t.mode==="single-url"?cr(e,o):u.enabled&&!t.resumeRewrite?ir(e,u,a,n,o.assetMap):void 0;l.mark("rewrite-text");let h=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${c?.length??0}`:"Rewriting text files...";l.progress(h,{phase:"rewrite-text",index:0,totalFiles:c?.length,changedTextFiles:0}),await He(h);let d=Date.now(),p=await je(e,o.assetMap,{files:c,previousAssetMap:n,onProgress:async({index:g,totalFiles:P,changedTextFiles:m,file:w,changed:S})=>{S&&E(a,e,f.join(e.outputDir,w)),l.checkpoint(`Rewriting text file ${g}/${P}`,{phase:"rewrite-text",index:g,totalFiles:P,changedTextFiles:m,file:w});let v=Date.now();if(!(g===1||g===P||v-d>=5e3))return;d=v;let A=`Rewriting text files: ${g}/${P}`;l.progress(A,{phase:"rewrite-text",index:g,totalFiles:P,changedTextFiles:m,file:w}),await He(A);}});l.endMark("rewrite-text",{changedTextFiles:p}),await Ae(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...a.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...a.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((c||[]).map(g=>H(e.outputDir,f.isAbsolute(g)?g:f.join(e.outputDir,g))).filter(g=>g!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${p} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:p}),await l.flush();}Wr().catch(async e=>{console.error(e);try{let r=(await ae().catch(()=>null))?.logDir??"logs",s=new N(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Qt(t,r));}updateCurrentEvent(t,r,s){let n=qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Wt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Ht=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Jt=new Set(["",".html",".htm"]),zt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Bt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Vt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let a=e[u];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--resume-rewrite")n=true,i=true;else if(a==="--crawl-mode"){let o=e[++u];if(!o)throw new Error("--crawl-mode requires a value");s=o==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let o=e[++u];if(!o)throw new Error("--url requires a value");r="single-url",n=true,t.push(o);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let o=e[++u];if(!o)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${o}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Pe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Kt(){let e=Pe();return e?f.resolve(e,".."):""}function Gt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Xt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Kt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Pe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Gt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function et(e){let t=Pe();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function tt(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Yt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function E(e,t,r){let s=H(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Qe(e,t,r){let s=H(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function K(e,t){e.rewriteTargets.add(f.resolve(t));}async function Zt(e){try{let t=await y.readFile(et(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return tt()}function Ne(e){return JSON.parse(JSON.stringify(e))}async function er(e,t){let r=et(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function tr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function rr(e,t,r,s){let n=[...new Set(r.map(a=>a.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let a=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!a.ok())return s.warn(`Change token lookup failed with HTTP ${a.status()}`,{endpoint:u,status:a.status()}),new Map;let o=await a.json().catch(()=>null);if(!o||!Array.isArray(o.items))return new Map;let c=new Map;for(let l of o.items)!l||typeof l.url!="string"||c.set(l.url,l);return c}catch(a){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(a)}),new Map}}async function qe(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await rr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function sr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await qe(e,t,s,n,i)};let a=await qe(e,t,s,n,i);if(a?.supported&&a.token)return u.changeToken===a.token?{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=G(r.sitemapLastmodByPage[n]);return o&&u.sitemapLastmod&&u.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function nr(e,t,r,s,n){let i=_e(r.previousManifest),u=0;for(let[d,p]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Qe(n,e,p.outputPath);try{await y.unlink(p.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:p.outputPath,error:String(m)});}delete r.manifest.pages[d],u++;}let a=_e(r.manifest,r.runId),o=new Set,c=new Set;for(let d of a){for(let P of q(e,d))o.add(P);let p=r.manifest.assets[d]?.outputPath;p&&c.add(p);let m=X(e,t.assetMap,d);m&&c.add(m);}for(let d of Object.keys(r.manifest.assets))a.has(d)||delete r.manifest.assets[d];let l=0,g=0;for(let d of i){let p=new Set,m=r.previousManifest.assets[d]?.outputPath;m&&p.add(m);let P=X(e,t.assetMap,d);if(!a.has(d)&&P)p.add(P);else if(!a.has(d)){let h=Se(e,d);p.add(h.originalFilePath),h.hashedFilePath&&p.add(h.hashedFilePath);}for(let h of p)if(!c.has(h)){Qe(n,e,h);try{await y.unlink(h),l++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:h,error:String(w)});}}for(let h of q(e,d))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],g++);}(u>0||l>0||g>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${l} stale assets, ${g} stale asset mappings`,{removedPages:u,removedAssets:l,removedAssetMappings:g,phase:"incremental-cleanup"});}function _e(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function We(e,t,r){for(let s of t)for(let n of q(e,s))if(r.has(n))return true;return false}function ir(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=tr(s,n);if(u.size===0)return [...i];for(let a of Object.values(t.manifest.pages))We(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let[a,o]of Object.entries(t.manifest.assets)){o.isText&&We(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let c of q(e,a))if(u.has(c)){E(r,e,o.outputPath),o.isText&&i.add(f.resolve(o.outputPath));break}}return [...i]}function q(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function or(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ar(e,t,r){for(let s of or(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function ur(e,t,r){for(let s of q(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return i}return null}function X(e,t,r){let s=ur(e,t,r);return s||Se(e,r).originalFilePath}function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function He(e){let t=lr();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function cr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Z(e,s)));for(let s of t.doneAssets){let n=X(e,t.assetMap,s);!n||!j(n)||r.add(f.resolve(n));}return [...r]}var Y="re:",V=new Map;function dr(e){if(!e.startsWith(Y))return null;let t=V.get(e);if(t!==void 0)return t||null;let r=e.slice(Y.length).trim();if(!r)return V.set(e,false),null;try{let s=new RegExp(r);return V.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),V.set(e,false),null}}function ye(e,t){let r=dr(t);return r?r.test(e):t.startsWith(Y)?false:e.startsWith(t)}function pr(e,t){return t.find(r=>ye(e,r))}function st(e,t){let r=pr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Y)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function gr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function hr(e,t){return gr(e).has(t.hostname)}function fr(e){let t=D(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=D(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function te(e){return Bt.some(t=>e.pathname.startsWith(t))}function C(e,t,r=e.sourceOrigin,s,n="url"){let i=fr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!hr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let a=st(e,u);if(a)return s?.reject(n,u.toString(),a,r),null;let o=new URL(e.sourceOrigin);return u.protocol=o.protocol,u.host=o.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function it(e){return Ht.has(f.extname(e.pathname).toLowerCase())}function W(e){return te(e)?false:Jt.has(f.extname(e.pathname).toLowerCase())}function re(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function R(e,t){return re(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:it(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>ye(t.pathname,r)):false}function Z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function Se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),a=le(e.targetOrigin,i),o=`${a}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n};let c=f.extname(s),g=`${c?s.slice(0,-c.length):s}.${Le(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n,hashedPathname:g,hashedFilePath:f.join(e.outputDir,g),hashedPublicUrl:le(e.targetOrigin,g)}}function mr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),a=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{a.push(new URL(e.targetOrigin).origin);}catch{}for(let o of a)try{let c=new URL(o);t[c.origin+i.pathname+i.search]=s;}catch{}try{let o=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=o;}catch{}try{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}try{let o=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}}catch{}}async function me(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function wr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function at(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Pr(e,t){let r=at(t),s=wr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function yr(e,t){if(!t)return;let r=at(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function ge(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return zt.has(r)||Wt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function he(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function we(e,t,r,s,n,i,u){let a=Se(e,r),o=a.originalFilePath,c=a.originalPublicUrlWithSearch;if(a.preferQueryHashed&&a.hashedFilePath&&a.hashedPublicUrl){let d=ar(e,t,r);if(d)o=d.outputPath,c=d.publicUrl;else {let p=await me(a.originalFilePath);if(p!==null&&!p.equals(s)){let m=j(a.originalFilePath)?Buffer.from(pe(s.toString("utf8"),e,t,a.originalFilePath),"utf8"):null;(m===null||!p.equals(m))&&(o=a.hashedFilePath,c=a.hashedPublicUrl);}}}mr(e,t,r,c,a.originalPublicUrl);let l=await me(o),g=false;return (l===null||!l.equals(s))&&(await ue(o),await y.writeFile(o,s),g=true),await yr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function ee(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"page");if(!u)return;if(!W(u)||it(u)){if(R(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),A(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),te(u)?"asset/internal path cannot be page":"not page-like",n);return}let a=u.toString();!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function A(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"asset");if(!u)return;if(!R(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let a=u.toString();!t.doneAssets.has(a)&&!t.queuedAssets.has(a)&&(t.queuedAssets.add(a),t.assetQueue.push(a),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ve(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"sitemap");if(!u)return;if(!re(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let a=u.toString();!t.doneSitemaps.has(a)&&!t.queuedSitemaps.has(a)&&(t.queuedSitemaps.add(a),t.sitemapQueue.push(a),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Sr(e){let t=D(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function vr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function br(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function fe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function ut(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await ut(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Rr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,a=i.prefix;if(!u||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:a});continue}let o=Xt(u);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let c=o.resolvedPath,l=fe(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:a});continue}let g;try{g=await y.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:c,prefixPath:a});continue}let d=f.resolve(e.outputDir,l);if(g.isDirectory()){n.push(...await ut(c,d)),s++,t.progress(`Copied static directory to export: ${c} -> /${l}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${l}`,copiedItems:s});continue}let p=f.basename(c),m=a.endsWith("/")?f.join(d,p):d;await y.mkdir(f.dirname(m),{recursive:true}),await y.copyFile(c,m),n.push(m),s++,t.progress(`Copied static file to export: ${c} -> /${fe(a.endsWith("/")?`${a}${p}`:a)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${fe(a.endsWith("/")?`${a}${p}`:a)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function _(e,t,r,s,n){let i=new Set,u=D(r).replace(/\\\//g,"/");for(let c of br(u)){let l=C(e,c,t,s,`${n}:css-url`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let l=C(e,c[1],t,s,`${n}:css-import`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let l=C(e,c[1],t,s,`${n}:xml-stylesheet`);l&&R(e,l)&&i.add(l.toString());}let a="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",o=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of u.matchAll(o)){let l=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(nt(l)){s.ignore(`${n}:serialized-url`,l,"looks like JavaScript/code fragment",t);continue}let g=C(e,l,t,s,`${n}:serialized-url`);g&&R(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Cr(e,t,r,s){let n=new Set,i=D(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let a=u[1],o=C(e,a,t,s,"page-link");o&&(st(e,o)||W(o)&&!te(o)&&!R(e,o)&&n.add(o.toString()));}return [...n]}async function Je(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),a=i.url()||n;await we(e,r.assetMap,a,u,r,s);let o=u.toString("utf8");for(let c of _(e,a,o,s,"sitemap"))A(e,r,c,s,a);for(let c of Sr(o)){let l=C(e,c.loc,a,s,"sitemap-loc");l&&(re(l)?ve(e,r,l.toString(),s,a):R(e,l)?A(e,r,l.toString(),s,a):W(l)?(c.lastmod&&(r.sitemapLastmodByPage[l.toString()]=c.lastmod),ee(e,r,l.toString(),s,a)):s.reject("sitemap-loc",l.toString(),"not page/sitemap/asset-like",a));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function xr(e,t,r,s,n,i,u){let a=C(e,s,e.sourceOrigin,n,"asset-fetch");if(!a||!R(e,a))return;let o=a.toString();if(r.doneAssets.has(o))return;let c=i.enabled?i.previousManifest.assets[o]:void 0,l=(g,d={})=>{if(!c)return false;i.manifest.assets[o]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(o);for(let p of c.discoveredAssets||[])A(e,r,p,n,o);return n.warn(`Keeping previous incremental asset ${o} after fetch issue.`,{url:o,reason:g,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),he(r,n),true};n.asset(`Fetching asset ${o}`,{url:o});try{let g=X(e,r.assetMap,o),d=g?await ot(g):null;if(g&&d)try{let S=await t.head(o,{timeout:6e4}),b=S.headers();if(S.ok()&&Pr(d,b)){let I=await me(g);if(I!==null){let x=S.url()||o,Q=await we(e,r.assetMap,x,I,r,n,b);Q.wroteFile&&(E(u,e,Q.outputPath),j(Q.outputPath)&&K(u,Q.outputPath)),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:g,lastModified:b["last-modified"]||"",contentLength:b["content-length"]||"",phase:"download-assets"}),he(r,n);let be=ge(a,b),se=c?.discoveredAssets?[...c.discoveredAssets]:[];if(be||c?.isText){let ct=I.toString("utf8"),Re=_(e,x,ct,n,"asset:cached");(Re.length>0||!c)&&(se=Re);for(let dt of se)A(e,r,dt,n,x);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:Q.outputPath,isText:be||!!c?.isText,discoveredAssets:se,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let p=await t.get(o,{timeout:6e4});if(!p.ok()){if(l(`http-${p.status()}`,{status:p.status()}))return;n.skip("asset",o,p.status());return}let m=await p.body(),P=p.url()||o,h=p.headers(),w=await we(e,r.assetMap,P,m,r,n,h);w.wroteFile&&(E(u,e,w.outputPath),j(w.outputPath)&&K(u,w.outputPath)),r.doneAssets.add(o),he(r,n);let v=[];if(ge(a,h)){let S=m.toString("utf8");v=_(e,P,S,n,`asset:${f.extname(a.pathname).toLowerCase()||h["content-type"]||"unknown"}`);for(let b of v)A(e,r,b,n,P);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:w.outputPath,isText:ge(a,h),discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){if(l("fetch-error",{error:String(g)}))return;n.error(`Failed asset ${o}`,{url:o,error:String(g)});}}async function Ar(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),a=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await xr(e,t,r,o,s,n,i));}});await Promise.all(a);}async function kr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Mr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await kr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Tr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Lr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ye(t,r))}async function Ir(e,t,r,s,n=true){let i=new Set;if(n){let a=await t.evaluate(()=>{let o=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,g=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let m of c){let P=d.getAttribute(m);P&&o.add(P);}for(let m of g){let P=d.getAttribute(m);P&&P.split(",").forEach(h=>{let w=h.trim().split(/\s+/)[0];w&&o.add(w);});}let p=d.getAttribute("style");p&&l.add(p);}),{attrs:[...o],styles:[...l]}});for(let o of a.attrs)for(let c of vr(o)){let l=C(e,c,r,s,"dom-attr");l&&R(e,l)&&i.add(l.toString());}for(let o of a.styles)for(let c of _(e,r,o,s,"dom-style"))i.add(c);}let u=await t.content();for(let a of _(e,r,u,s,"page-html"))i.add(a);return [...i]}async function ze(e,t,r,s,n){let i=Z(e,t);await ue(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Be(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ve(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Dr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Er(e,t,r,s,n,i,u){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Dr(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let g=new URL(l),d=Lr(e,g.pathname);if(!W(g)||R(e,g)||te(g)){if(R(e,g)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),A(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let p=await sr(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let h=i.manifest.pages[l];if(p.action==="reuse"&&h){for(let w of h.discoveredAssets)A(e,t,w,r,l);for(let w of h.discoveredPages)ee(e,t,w,r,l);h.lastSeenRunId=i.runId,h.sitemapLastmod=G(t.sitemapLastmodByPage[l])??h.sitemapLastmod,p.changeToken?.supported&&(h.changeToken=p.changeToken.token,h.tokenSource=p.changeToken.tokenSource??h.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Be(t)&&Ve(t,r);continue}}let m=await(d?c.newPage():o.newPage()),P=!1;await m.route("**/*",async h=>{let w=h.request();if(w.isNavigationRequest()&&w.frame()===m.mainFrame())try{let v=new URL(w.url()),S=new URL(l);if(v.hash="",S.hash="",v.href===S.href){if(P){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:w.url()}),await h.abort("aborted");return}P=!0;}}catch{}await h.continue();}),m.on("response",h=>{try{let w=C(e,h.url(),e.sourceOrigin,void 0,"network-response");w&&R(e,w)&&A(e,t,w.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let h=null,w=null;try{if(h=await m.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),h&&h.ok())try{w=await h.text();}catch{w=null;}}catch(x){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(x)});}if(h&&!h.ok()){r.skip("page",l,h.status()),await m.close();continue}d?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Tr(e,m)):await Mr(e,m),t.stats.pagesRendered++,Be(t)&&Ve(t,r);let v=await Ir(e,m,l,r,!d);for(let x of v)A(e,t,x,r,l);let S=w??await m.content(),b=s!=="single-url"?Cr(e,l,S,r):[];if(s!=="single-url")for(let x of b)ee(e,t,x,r,l);await ze(e,l,S,t,r);let I=Z(e,l);E(u,e,I),K(u,I),i.trackManifest&&(i.manifest.pages[l]={url:l,outputPath:I,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[l])??null,discoveredPages:b,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(h){try{let w=await m.content();if(w&&w.trim()){await ze(e,l,w,t,r);let v=Z(e,l);E(u,e,v),K(u,v),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(h)});}}catch(w){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(w)});}r.error(`Failed page ${l}`,{url:l,error:String(h)});}finally{await m.close();}}}finally{await c.close(),await o.close(),await a.close();}}async function Fr(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ke(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function jr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Or(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function lt(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ge(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function $r(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ge(s)-Ge(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Xe(e){let t=await lt(f.join(e,"job.json"));for(let r of $r(e,t)){let s=await Fr(r);if(s.length>0)return s}return []}async function Ur(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let o=await lt(f.join(u,"job.json"));if(!jr(o))continue;let c=Or(o);c>=n&&(n=c,s=u);}return s}async function Qr(e){let t=await Ur(e),r=t!==""?await Xe(t):await Xe(e.logDir),s=new Set;for(let n of r)if(Ke(n.error)||Ke(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function qr(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function _r(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ye(e,t,r,s,n){let i=C(e,r,e.sourceOrigin,s,"manual-url");i&&(re(i)?ve(e,t,i.toString(),s,n):R(e,i)?A(e,t,i.toString(),s,n):W(i)?ee(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Wr(){let e=await ae(),t=Vt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Qr(e)),t.mode==="single-url"&&(r=await Nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.mode==="full"&&!t.resumeRewrite,i=t.preserveOutput?await qr(e):{},u=s?await Zt(e):tt(),a={enabled:s,trackManifest:n,manifest:Ne(u),previousManifest:Ne(u),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await _r(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...i},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},c=Yt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new N(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let P=await chromium.launch({headless:true}),h=await P.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),w=h.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>ve(e,o,S,l)),await Je(e,w,o,l),e.seedPaths.forEach(S=>Ye(e,o,S,l,"seed-path"))):(r.forEach(S=>Ye(e,o,S,l,"cli")),await Je(e,w,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let v=Array.from({length:Math.max(1,e.concurrency)},()=>Er(e,o,l,t.mode,w,a,c));if(await Promise.all(v),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Ar(e,w,o,l,a,c),l.endMark("download-assets",{assets:o.doneAssets.size}),a.enabled&&e.maxPages===0&&await nr(e,o,a,l,c),t.mode==="full"){let S=await Rr(e,l);for(let b of S)E(c,e,b);}}finally{await h.close(),await P.close();}}a.trackManifest&&(a.manifest.updatedAt=new Date().toISOString(),await er(e,a.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(i,null,2)!==JSON.stringify(o.assetMap,null,2)&&E(c,e,f.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?cr(e,o):a.enabled&&!t.resumeRewrite?ir(e,a,c,i,o.assetMap):void 0;l.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";l.progress(d,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await He(d);let p=Date.now(),m=await je(e,o.assetMap,{files:g,previousAssetMap:i,onProgress:async({index:P,totalFiles:h,changedTextFiles:w,file:v,changed:S})=>{S&&E(c,e,f.join(e.outputDir,v)),l.checkpoint(`Rewriting text file ${P}/${h}`,{phase:"rewrite-text",index:P,totalFiles:h,changedTextFiles:w,file:v});let b=Date.now();if(!(P===1||P===h||b-p>=5e3))return;p=b;let x=`Rewriting text files: ${P}/${h}`;l.progress(x,{phase:"rewrite-text",index:P,totalFiles:h,changedTextFiles:w,file:v}),await He(x);}});l.endMark("rewrite-text",{changedTextFiles:m}),await Ae(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(a.enabled&&!t.resumeRewrite),changedFiles:a.enabled&&!t.resumeRewrite?[...c.changedFiles].sort():[],deletedFiles:a.enabled&&!t.resumeRewrite?[...c.deletedFiles].sort():[],rewriteTargets:a.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(P=>H(e.outputDir,f.isAbsolute(P)?P:f.join(e.outputDir,P))).filter(P=>P!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${m} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:m}),await l.flush();}Wr().catch(async e=>{console.error(e);try{let r=(await ae().catch(()=>null))?.logDir??"logs",s=new N(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.13",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|