@smart-cloud/publisher-exporter 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +4 -4
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import y from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import
|
|
1
|
+
import {chromium}from'playwright';import y from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import wt from'crypto';import yt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var lt=".deploy-plan.json",ct="deploy-plan.json";function dt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function be(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function se(e){return [...new Set(e.map(be).filter(t=>t!==null))]}function pt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:se(t.changedFiles||[]),deletedFiles:se(t.deletedFiles||[]),rewriteTargets:se(t.rewriteTargets||[])}}function gt(e){let t=f.join(f.resolve(e.outputDir),lt),r=dt();return r?[f.join(r,ct),t]:[t]}function W(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return be(n)}async function Re(e,t){let r=pt(e,t);for(let s of gt(e))await y.mkdir(f.dirname(s),{recursive:true}),await y.writeFile(s,JSON.stringify(r,null,2),"utf8");}function xe(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function ne(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function mt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=xe(i.targetOrigin);a&&(u.targetOrigin=a);}let o=ne(i.extraReplacements);if(Object.keys(o).length>0&&(u.extraReplacements=o),i.s3&&typeof i.s3=="object"){let a={},l=i.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},l=i.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function ie(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function ft(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Ce(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,ie(s,r))}async function oe(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await y.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=xe(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=ne(r.extraReplacements),r.postCrawlCopyMap=ne(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=mt(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=ft(),n=s?f.resolve(s,".."):"";return n?(r.outputDir=Ce(n,r.outputDir,"export"),r.logDir=Ce(n,r.logDir,"logs")):(f.isAbsolute(r.outputDir)||(r.outputDir=ie(r.outputDir,"export")),f.isAbsolute(r.logDir)||(r.logDir=ie(r.logDir,"logs"))),r}function ke(e,t=10){return wt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function ae(e){await y.mkdir(f.dirname(e),{recursive:true});}function $(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function E(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function j(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function H(e){return e.replace(/\//g,"\\/")}function Ae(e){return e.replace(/\//g,"\\\\/")}function J(e,t,r){if(!t)return;e[t]=r;let s=H(t),n=H(r);e[s]=n;let i=Ae(t),u=Ae(r);e[i]=u;let o=j(t),a=j(r);e[o]=a;let l=j(s),c=j(n);e[l]=c;let m=j(i),d=j(u);e[m]=d;}function ue(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Me(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Te(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var bt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Rt=new Set([".html",".htm"]),Ee="WPSuite.io Static Publisher",Ct=Ee.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function xt(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=f.extname(r).toLowerCase();return Rt.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function At(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Ct}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ee}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Ft(t,r));}updateCurrentEvent(t,r,s){let n=Ot(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){It("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Ut=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Qt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Nt=new Set(["",".html",".htm"]),qt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),_t=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Wt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let o=e[u];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function he(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Ht(){let e=he();return e?f.resolve(e,".."):""}function Jt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function zt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Ht(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:he(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Jt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function Ge(e){let t=he();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function Ke(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Bt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function D(e,t,r){let s=_(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Ie(e,t,r){let s=_(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function V(e,t){e.rewriteTargets.add(f.resolve(t));}async function Vt(e){try{let t=await y.readFile(Ge(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return Ke()}function Fe(e){return JSON.parse(JSON.stringify(e))}async function Gt(e,t){let r=Ge(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function Kt(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function Xt(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:u,status:o.status()}),new Map;let a=await o.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let c of a.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(o)}),new Map}}async function je(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await Xt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function Yt(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await je(e,t,s,n,i)};let o=await je(e,t,s,n,i);if(o?.supported&&o.token)return u.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let a=G(r.sitemapLastmodByPage[n]);return a&&u.sitemapLastmod&&u.sitemapLastmod===a?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function Zt(e,t,r,s,n){let i=Oe(r.previousManifest),u=0;for(let[d,h]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ie(n,e,h.outputPath);try{await y.unlink(h.outputPath);}catch(p){(p.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:h.outputPath,error:String(p)});}delete r.manifest.pages[d],u++;}let o=Oe(r.manifest,r.runId),a=new Set,l=new Set;for(let d of o){for(let P of K(e,d))a.add(P);let h=r.manifest.assets[d]?.outputPath;h&&l.add(h);let p=X(e,t.assetMap,d);p&&l.add(p);}for(let d of Object.keys(r.manifest.assets))o.has(d)||delete r.manifest.assets[d];let c=0,m=0;for(let d of i){let h=new Set,p=r.previousManifest.assets[d]?.outputPath;p&&h.add(p);let P=X(e,t.assetMap,d);if(!o.has(d)&&P)h.add(P);else if(!o.has(d)){let g=fe(e,d);h.add(g.originalFilePath),g.hashedFilePath&&h.add(g.hashedFilePath);}for(let g of h)if(!l.has(g)){Ie(n,e,g);try{await y.unlink(g),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:g,error:String(w)});}}for(let g of K(e,d))a.has(g)||Object.prototype.hasOwnProperty.call(t.assetMap,g)&&(delete t.assetMap[g],m++);}(u>0||c>0||m>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${m} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:m,phase:"incremental-cleanup"});}function Oe(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function $e(e,t,r){for(let s of t)for(let n of K(e,s))if(r.has(n))return true;return false}function er(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=Kt(s,n);if(u.size===0)return [...i];for(let o of Object.values(t.manifest.pages))$e(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let[o,a]of Object.entries(t.manifest.assets)){a.isText&&$e(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let l of K(e,o))if(u.has(l)){D(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function K(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function tr(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function X(e,t,r){let s=t[r];return s?tr(e,s):fe(e,r).originalFilePath}function rr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function Ue(e){let t=rr();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function sr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Z(e,s)));for(let s of t.doneAssets){let n=X(e,t.assetMap,s);!n||!$(n)||r.add(f.resolve(n));}return [...r]}var Y="re:",z=new Map;function nr(e){if(!e.startsWith(Y))return null;let t=z.get(e);if(t!==void 0)return t||null;let r=e.slice(Y.length).trim();if(!r)return z.set(e,false),null;try{let s=new RegExp(r);return z.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),z.set(e,false),null}}function me(e,t){let r=nr(t);return r?r.test(e):t.startsWith(Y)?false:e.startsWith(t)}function ir(e,t){return t.find(r=>me(e,r))}function Xe(e,t){let r=ir(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Y)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function or(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function ar(e,t){return or(e).has(t.hostname)}function ur(e){let t=E(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Ye(e){let t=E(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function te(e){return _t.some(t=>e.pathname.startsWith(t))}function R(e,t,r=e.sourceOrigin,s,n="url"){let i=ur(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Ye(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!ar(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let o=Xe(e,u);if(o)return s?.reject(n,u.toString(),o,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function Ze(e){return Qt.has(f.extname(e.pathname).toLowerCase())}function q(e){return te(e)?false:Nt.has(f.extname(e.pathname).toLowerCase())}function re(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function b(e,t){return re(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Ze(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>me(t.pathname,r)):false}function Z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function fe(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),o=ue(e.targetOrigin,i),a=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=f.extname(s),m=`${l?s.slice(0,-l.length):s}.${Ce(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:m,hashedFilePath:f.join(e.outputDir,m),hashedPublicUrl:ue(e.targetOrigin,m)}}function lr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let a of o)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function pe(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function et(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function cr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function tt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function dr(e,t){let r=tt(t),s=cr(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function pr(e,t){if(!t)return;let r=tt(t);if(r===null)return;let s=await et(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function B(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return qt.has(r)||Ut.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Qe(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ge(e,t,r,s,n,i,u){let o=fe(e,r),a=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let d=await pe(o.originalFilePath);d!==null&&!d.equals(s)&&(a=o.hashedFilePath,l=o.hashedPublicUrl);}lr(e,t,r,l,o.originalPublicUrl);let c=await pe(a),m=false;return (c===null||!c.equals(s))&&(await ae(a),await y.writeFile(a,s),m=true),await pr(a,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:l,wroteFile:m}}function ee(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"page");if(!u)return;if(!q(u)||Ze(u)){if(b(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),M(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),te(u)?"asset/internal path cannot be page":"not page-like",n);return}let o=u.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function M(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"asset");if(!u)return;if(!b(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let o=u.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function we(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"sitemap");if(!u)return;if(!re(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let o=u.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function gr(e){let t=E(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function hr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function mr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function de(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function rt(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await rt(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function fr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,o=i.prefix;if(!u||!o){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:o});continue}let a=zt(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:o});continue}let l=a.resolvedPath,c=de(o);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:o});continue}let m;try{m=await y.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:l,prefixPath:o});continue}let d=f.resolve(e.outputDir,c);if(m.isDirectory()){n.push(...await rt(l,d)),s++,t.progress(`Copied static directory to export: ${l} -> /${c}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${c}`,copiedItems:s});continue}let h=f.basename(l),p=o.endsWith("/")?f.join(d,h):d;await y.mkdir(f.dirname(p),{recursive:true}),await y.copyFile(l,p),n.push(p),s++,t.progress(`Copied static file to export: ${l} -> /${de(o.endsWith("/")?`${o}${h}`:o)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${de(o.endsWith("/")?`${o}${h}`:o)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function N(e,t,r,s,n){let i=new Set,u=E(r).replace(/\\\//g,"/");for(let l of mr(u)){let c=R(e,l,t,s,`${n}:css-url`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=R(e,l[1],t,s,`${n}:css-import`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=R(e,l[1],t,s,`${n}:xml-stylesheet`);c&&b(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of u.matchAll(a)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(Ye(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let m=R(e,c,t,s,`${n}:serialized-url`);m&&b(e,m)?i.add(m.toString()):m&&s.ignore("asset",m.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function wr(e,t,r,s){let n=new Set,i=E(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=u[1],a=R(e,o,t,s,"page-link");a&&(Xe(e,a)||q(a)&&!te(a)&&!b(e,a)&&n.add(a.toString()));}return [...n]}async function Ne(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),o=i.url()||n;await ge(e,r.assetMap,o,u,r,s);let a=u.toString("utf8");for(let l of N(e,o,a,s,"sitemap"))M(e,r,l,s,o);for(let l of gr(a)){let c=R(e,l.loc,o,s,"sitemap-loc");c&&(re(c)?we(e,r,c.toString(),s,o):b(e,c)?M(e,r,c.toString(),s,o):q(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),ee(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Pr(e,t,r,s,n,i,u){let o=R(e,s,e.sourceOrigin,n,"asset-fetch");if(!o||!b(e,o))return;let a=o.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let l=X(e,r.assetMap,a),c=l?await et(l):null;if(l&&c)try{let w=await t.head(a,{timeout:6e4}),S=w.headers();if(w.ok()&&dr(c,S)){let v=await pe(l);if(v!==null){let I=w.url()||a,C=await ge(e,r.assetMap,I,v,r,n,S);C.wroteFile&&(D(u,e,C.outputPath),$(C.outputPath)&&V(u,C.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:l,lastModified:S["last-modified"]||"",contentLength:S["content-length"]||"",phase:"download-assets"}),Qe(r,n);let T=[];if(B(o,S)){let nt=v.toString("utf8");T=N(e,I,nt,n,"asset:cached");for(let it of T)M(e,r,it,n,I);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:C.outputPath,isText:B(o,S),discoveredAssets:T,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let m=await t.get(a,{timeout:6e4});if(!m.ok()){n.skip("asset",a,m.status());return}let d=await m.body(),h=m.url()||a,p=m.headers(),P=await ge(e,r.assetMap,h,d,r,n,p);P.wroteFile&&(D(u,e,P.outputPath),$(P.outputPath)&&V(u,P.outputPath)),r.doneAssets.add(a),Qe(r,n);let g=[];if(B(o,p)){let w=d.toString("utf8");g=N(e,h,w,n,`asset:${f.extname(o.pathname).toLowerCase()||p["content-type"]||"unknown"}`);for(let S of g)M(e,r,S,n,h);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:P.outputPath,isText:B(o,p),discoveredAssets:g,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(l){n.error(`Failed asset ${a}`,{url:a,error:String(l)});}}}async function yr(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),o=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Pr(e,t,r,a,s,n,i));}});await Promise.all(o);}async function Sr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function vr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Sr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function br(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Rr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>me(t,r))}async function Cr(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,m=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let p of l){let P=d.getAttribute(p);P&&a.add(P);}for(let p of m){let P=d.getAttribute(p);P&&P.split(",").forEach(g=>{let w=g.trim().split(/\s+/)[0];w&&a.add(w);});}let h=d.getAttribute("style");h&&c.add(h);}),{attrs:[...a],styles:[...c]}});for(let a of o.attrs)for(let l of hr(a)){let c=R(e,l,r,s,"dom-attr");c&&b(e,c)&&i.add(c.toString());}for(let a of o.styles)for(let l of N(e,r,a,s,"dom-style"))i.add(l);}let u=await t.content();for(let o of N(e,r,u,s,"page-html"))i.add(o);return [...i]}async function qe(e,t,r,s,n){let i=Z(e,t);await ae(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function _e(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function We(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function xr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Ar(e,t,r,s,n,i,u){let o=await chromium.launch({headless:true}),a=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await xr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let c=t.pageQueue.shift();if(!c)break;if(t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),i.enabled&&i.seenPages.add(c);let m=new URL(c),d=Rr(e,m.pathname);if(!q(m)||b(e,m)||te(m)){if(b(e,m)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),M(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let h=await Yt(e,n,t,i,c,r);if(i.enabled&&s!=="single-url"){let g=i.manifest.pages[c];if(h.action==="reuse"&&g){for(let w of g.discoveredAssets)M(e,t,w,r,c);for(let w of g.discoveredPages)ee(e,t,w,r,c);g.lastSeenRunId=i.runId,g.sitemapLastmod=G(t.sitemapLastmodByPage[c])??g.sitemapLastmod,h.changeToken?.supported&&(g.changeToken=h.changeToken.token,g.tokenSource=h.changeToken.tokenSource??g.tokenSource),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:h.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),_e(t)&&We(t,r);continue}}let p=await(d?l.newPage():a.newPage()),P=!1;await p.route("**/*",async g=>{let w=g.request();if(w.isNavigationRequest()&&w.frame()===p.mainFrame())try{let S=new URL(w.url()),v=new URL(c);if(S.hash="",v.hash="",S.href===v.href){if(P){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:w.url()}),await g.abort("aborted");return}P=!0;}}catch{}await g.continue();}),p.on("response",g=>{try{let w=R(e,g.url(),e.sourceOrigin,void 0,"network-response");w&&b(e,w)&&M(e,t,w.toString(),r,c);}catch{}}),r.page(`Rendering ${c}`,{url:c});try{let g=null,w=null;try{if(g=await p.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),g&&g.ok())try{w=await g.text();}catch{w=null;}}catch(T){r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(T)});}if(g&&!g.ok()){r.skip("page",c,g.status()),await p.close();continue}d?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await br(e,p)):await vr(e,p),t.stats.pagesRendered++,_e(t)&&We(t,r);let S=await Cr(e,p,c,r,!d);for(let T of S)M(e,t,T,r,c);let v=w??await p.content(),I=s!=="single-url"?wr(e,c,v,r):[];if(s!=="single-url")for(let T of I)ee(e,t,T,r,c);await qe(e,c,v,t,r);let C=Z(e,c);D(u,e,C),V(u,C),i.enabled&&(i.manifest.pages[c]={url:c,outputPath:C,changeToken:h.changeToken?.supported?h.changeToken.token:null,tokenSource:h.changeToken?.supported?h.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[c])??null,discoveredPages:I,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){try{let w=await p.content();if(w&&w.trim()){await qe(e,c,w,t,r);let S=Z(e,c);D(u,e,S),V(u,S),r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(g)});}}catch(w){r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(w)});}r.error(`Failed page ${c}`,{url:c,error:String(g)});}finally{await p.close();}}}finally{await l.close(),await a.close(),await o.close();}}async function kr(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function He(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Mr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Tr(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function st(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Je(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Lr(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Je(s)-Je(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function ze(e){let t=await st(f.join(e,"job.json"));for(let r of Lr(e,t)){let s=await kr(r);if(s.length>0)return s}return []}async function Er(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let a=await st(f.join(u,"job.json"));if(!Mr(a))continue;let l=Tr(a);l>=n&&(n=l,s=u);}return s}async function Dr(e){let t=await Er(e),r=t!==""?await ze(t):await ze(e.logDir),s=new Set;for(let n of r)if(He(n.error)||He(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Ir(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Fr(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function jr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Be(e,t,r,s,n){let i=R(e,r,e.sourceOrigin,s,"manual-url");i&&(re(i)?we(e,t,i.toString(),s,n):b(e,i)?M(e,t,i.toString(),s,n):q(i)?ee(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Or(){let e=await oe(),t=Wt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Dr(e)),t.mode==="single-url"&&(r=await Ir(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.preserveOutput?await Fr(e):{},i=s?await Vt(e):Ke(),u={enabled:s,manifest:Fe(i),previousManifest:Fe(i),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await jr(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...n},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},a=Bt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new Q(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let p=await chromium.launch({headless:true}),P=await p.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),g=P.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>we(e,o,S,l)),await Ne(e,g,o,l),e.seedPaths.forEach(S=>Be(e,o,S,l,"seed-path"))):(r.forEach(S=>Be(e,o,S,l,"cli")),await Ne(e,g,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let w=Array.from({length:Math.max(1,e.concurrency)},()=>Ar(e,o,l,t.mode,g,u,a));if(await Promise.all(w),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await yr(e,g,o,l,u,a),l.endMark("download-assets",{assets:o.doneAssets.size}),u.enabled&&e.maxPages===0&&await Zt(e,o,u,l,a),t.mode==="full"){let S=await fr(e,l);for(let v of S)D(a,e,v);}}finally{await P.close(),await p.close();}}u.enabled&&!t.resumeRewrite&&(u.manifest.updatedAt=new Date().toISOString(),await Gt(e,u.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(n,null,2)!==JSON.stringify(o.assetMap,null,2)&&D(a,e,f.join(e.outputDir,"asset-map.json"));let c=t.mode==="single-url"?sr(e,o):u.enabled&&!t.resumeRewrite?er(e,u,a,n,o.assetMap):void 0;l.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${c?.length??0}`:"Rewriting text files...";l.progress(m,{phase:"rewrite-text",index:0,totalFiles:c?.length,changedTextFiles:0}),await Ue(m);let d=Date.now(),h=await Te(e,o.assetMap,{files:c,previousAssetMap:n,onProgress:async({index:p,totalFiles:P,changedTextFiles:g,file:w,changed:S})=>{S&&D(a,e,f.join(e.outputDir,w)),l.checkpoint(`Rewriting text file ${p}/${P}`,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w});let v=Date.now();if(!(p===1||p===P||v-d>=5e3))return;d=v;let C=`Rewriting text files: ${p}/${P}`;l.progress(C,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w}),await Ue(C);}});l.endMark("rewrite-text",{changedTextFiles:h}),await Se(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...a.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...a.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((c||[]).map(p=>_(e.outputDir,f.isAbsolute(p)?p:f.join(e.outputDir,p))).filter(p=>p!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${h} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:h}),await l.flush();}Or().catch(async e=>{console.error(e);try{let r=(await oe().catch(()=>null))?.logDir??"logs",s=new Q(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function kt(e){let t=E(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Mt(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,H(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[o,a]of r)u=u.split(o).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,o,a)=>`${u}${s(o)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,o,a,l)=>kt(a)?`${u}${s(a)}${l}`:i),n}function Tt(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Lt(e,t,r){let s=Tt(t,r);if(!s)return e;let n=e;for(let i of bt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),o=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(o,`${s}${i}`).replace(a,`${s}${i}`);let l=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),m=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${m}`,"g"),h=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${m}`,"g"),p=new RegExp(`(?<![\\.\\\\])\\/${m}`,"g");n=n.replace(d,`${c}${l}`).replace(h,`${c}${l}`).replace(p,`${c}${l}`);}return n}function ce(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:Me(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Te(e.outputDir,r,n)}${i}`}function le(e,t,r,s,n){J(e,r,ce(t,s,n));}function Et(e,t,r,s,n){if(!r||r===s)return;J(e,r,s);let i=ce(t,r,n),u=ce(t,s,n);J(e,i,u);}function Dt(e,t,r,s,n={}){let i=e,u={};le(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,l]of Object.entries(t.extraReplacements))le(u,t,a,l,s);for(let[a,l]of Object.entries(r))le(u,t,a,l,s);for(let[a,l]of Object.entries(n)){let c=r[a];!c||c===l||Et(u,t,l,c,s);}let o=Object.entries(u).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of o)i=i.split(a).join(l);return t.urlRewriteMode!=="absolute"&&(i=Lt(i,t,s)),i=Mt(i,o.map(([,a])=>a)),xt(t,i,s)&&(i=At(i)),i}async function It(e,t,r,s){let n=f.join(e.outputDir,s),i=f.extname(n).toLowerCase(),u;try{u=await y.readFile(n,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=Dt(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await y.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function Ft(e,t){return await new Promise((r,s)=>{let n=a=>{if(o(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{o(),s(a);},u=a=>{o(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},o=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function De(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>f.resolve(d)).filter(d=>d.startsWith(f.resolve(e.outputDir))).map(d=>f.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await yt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&$(d)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),o=Math.max(1,Math.min(u,n.length,availableParallelism())),a=0,l=0,c=null;if(o===1){for(let d of n){let h=await It(e,t,r.previousAssetMap??{},d);h.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:d,changed:h.changed});}return i}let m=Array.from({length:o},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let h=a;if(h>=n.length)return;a+=1;let p=n[h],P=await Ft(d,p);P.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:p,changed:P.changed});}}catch(h){throw c=h instanceof Error?h:new Error(String(h)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(m),i}var U={error:0,warn:1,info:2,debug:3},Ie=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",z=Ie?f.join(Ie,"current-progress.json"):"",Fe=Promise.resolve();function je(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function jt(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function Ot(e,t,r){z&&(Fe=Fe.then(async()=>{let s=new Date,n=s.toISOString();await y.mkdir(f.dirname(z),{recursive:true});let i=await y.readFile(z,"utf8").then(P=>JSON.parse(P)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},o=je(e),a=i?.currentStep||je(i?.source||""),l=i?.startedAt||n,c={...i?.stepDurationsSec??{}},m=i?.stepStartedAt||n;if(a&&a!==o&&i?.stepStartedAt){let P=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+P,m=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(m).getTime())/1e3)),h=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),p={checkedAt:n,source:e,message:t,details:jt(u,r??{}),startedAt:l,currentStep:o,stepStartedAt:m,stepElapsedSec:d,totalElapsedSec:h,stepDurationsSec:c};await y.writeFile(z,JSON.stringify(p,null,2),"utf8");}).catch(()=>{}));}async function $t(e,t){await y.mkdir(f.dirname(e),{recursive:true}),await y.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function Ut(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function Qt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:Ut(e,r),level:e,message:t,details:r??{}}}var Q=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await y.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>$t(t,r));}updateCurrentEvent(t,r,s){let n=Qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ot("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var qt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],_t=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Wt=new Set(["",".html",".htm"]),Ht=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Jt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function zt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let o=e[u];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Bt(){let e=me();return e?f.resolve(e,".."):""}function Vt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Gt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Bt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:me(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Vt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function Ye(e){let t=me();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function Ze(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Kt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function D(e,t,r){let s=W(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Oe(e,t,r){let s=W(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function V(e,t){e.rewriteTargets.add(f.resolve(t));}async function Xt(e){try{let t=await y.readFile(Ye(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return Ze()}function $e(e){return JSON.parse(JSON.stringify(e))}async function Yt(e,t){let r=Ye(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function Zt(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function er(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:u,status:o.status()}),new Map;let a=await o.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let c of a.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(o)}),new Map}}async function Ue(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await er(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function tr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await Ue(e,t,s,n,i)};let o=await Ue(e,t,s,n,i);if(o?.supported&&o.token)return u.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let a=G(r.sitemapLastmodByPage[n]);return a&&u.sitemapLastmod&&u.sitemapLastmod===a?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function rr(e,t,r,s,n){let i=Qe(r.previousManifest),u=0;for(let[d,h]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Oe(n,e,h.outputPath);try{await y.unlink(h.outputPath);}catch(p){(p.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:h.outputPath,error:String(p)});}delete r.manifest.pages[d],u++;}let o=Qe(r.manifest,r.runId),a=new Set,l=new Set;for(let d of o){for(let P of N(e,d))a.add(P);let h=r.manifest.assets[d]?.outputPath;h&&l.add(h);let p=K(e,t.assetMap,d);p&&l.add(p);}for(let d of Object.keys(r.manifest.assets))o.has(d)||delete r.manifest.assets[d];let c=0,m=0;for(let d of i){let h=new Set,p=r.previousManifest.assets[d]?.outputPath;p&&h.add(p);let P=K(e,t.assetMap,d);if(!o.has(d)&&P)h.add(P);else if(!o.has(d)){let g=we(e,d);h.add(g.originalFilePath),g.hashedFilePath&&h.add(g.hashedFilePath);}for(let g of h)if(!l.has(g)){Oe(n,e,g);try{await y.unlink(g),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:g,error:String(w)});}}for(let g of N(e,d))a.has(g)||Object.prototype.hasOwnProperty.call(t.assetMap,g)&&(delete t.assetMap[g],m++);}(u>0||c>0||m>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${m} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:m,phase:"incremental-cleanup"});}function Qe(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function Ne(e,t,r){for(let s of t)for(let n of N(e,s))if(r.has(n))return true;return false}function sr(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=Zt(s,n);if(u.size===0)return [...i];for(let o of Object.values(t.manifest.pages))Ne(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let[o,a]of Object.entries(t.manifest.assets)){a.isText&&Ne(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let l of N(e,o))if(u.has(l)){D(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function N(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function nr(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ir(e,t,r){for(let s of N(e,r)){let n=t[s];if(!n)continue;let i=nr(e,n);if(i)return i}return null}function K(e,t,r){let s=ir(e,t,r);return s||we(e,r).originalFilePath}function or(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function qe(e){let t=or();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function ar(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Y(e,s)));for(let s of t.doneAssets){let n=K(e,t.assetMap,s);!n||!$(n)||r.add(f.resolve(n));}return [...r]}var X="re:",B=new Map;function ur(e){if(!e.startsWith(X))return null;let t=B.get(e);if(t!==void 0)return t||null;let r=e.slice(X.length).trim();if(!r)return B.set(e,false),null;try{let s=new RegExp(r);return B.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),B.set(e,false),null}}function fe(e,t){let r=ur(t);return r?r.test(e):t.startsWith(X)?false:e.startsWith(t)}function lr(e,t){return t.find(r=>fe(e,r))}function et(e,t){let r=lr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(X)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function cr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function dr(e,t){return cr(e).has(t.hostname)}function pr(e){let t=E(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function tt(e){let t=E(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function ee(e){return Jt.some(t=>e.pathname.startsWith(t))}function R(e,t,r=e.sourceOrigin,s,n="url"){let i=pr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(tt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!dr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let o=et(e,u);if(o)return s?.reject(n,u.toString(),o,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function rt(e){return _t.has(f.extname(e.pathname).toLowerCase())}function _(e){return ee(e)?false:Wt.has(f.extname(e.pathname).toLowerCase())}function te(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function b(e,t){return te(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:rt(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>fe(t.pathname,r)):false}function Y(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function we(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),o=ue(e.targetOrigin,i),a=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=f.extname(s),m=`${l?s.slice(0,-l.length):s}.${ke(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:o,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:m,hashedFilePath:f.join(e.outputDir,m),hashedPublicUrl:ue(e.targetOrigin,m)}}function gr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let a of o)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function ge(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function st(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function hr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function nt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function mr(e,t){let r=nt(t),s=hr(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function fr(e,t){if(!t)return;let r=nt(t);if(r===null)return;let s=await st(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function de(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Ht.has(r)||qt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function _e(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function he(e,t,r,s,n,i,u){let o=we(e,r),a=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let d=await ge(o.originalFilePath);d!==null&&!d.equals(s)&&(a=o.hashedFilePath,l=o.hashedPublicUrl);}gr(e,t,r,l,o.originalPublicUrl);let c=await ge(a),m=false;return (c===null||!c.equals(s))&&(await ae(a),await y.writeFile(a,s),m=true),await fr(a,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:l,wroteFile:m}}function Z(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"page");if(!u)return;if(!_(u)||rt(u)){if(b(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),T(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),ee(u)?"asset/internal path cannot be page":"not page-like",n);return}let o=u.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function T(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"asset");if(!u)return;if(!b(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let o=u.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Pe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=R(e,r,i,s,"sitemap");if(!u)return;if(!te(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let o=u.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function wr(e){let t=E(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Pr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function yr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function pe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function it(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await it(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Sr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,o=i.prefix;if(!u||!o){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:o});continue}let a=Gt(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:o});continue}let l=a.resolvedPath,c=pe(o);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:o});continue}let m;try{m=await y.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:l,prefixPath:o});continue}let d=f.resolve(e.outputDir,c);if(m.isDirectory()){n.push(...await it(l,d)),s++,t.progress(`Copied static directory to export: ${l} -> /${c}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${c}`,copiedItems:s});continue}let h=f.basename(l),p=o.endsWith("/")?f.join(d,h):d;await y.mkdir(f.dirname(p),{recursive:true}),await y.copyFile(l,p),n.push(p),s++,t.progress(`Copied static file to export: ${l} -> /${pe(o.endsWith("/")?`${o}${h}`:o)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${pe(o.endsWith("/")?`${o}${h}`:o)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function q(e,t,r,s,n){let i=new Set,u=E(r).replace(/\\\//g,"/");for(let l of yr(u)){let c=R(e,l,t,s,`${n}:css-url`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=R(e,l[1],t,s,`${n}:css-import`);c&&b(e,c)&&i.add(c.toString());}for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=R(e,l[1],t,s,`${n}:xml-stylesheet`);c&&b(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of u.matchAll(a)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(tt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let m=R(e,c,t,s,`${n}:serialized-url`);m&&b(e,m)?i.add(m.toString()):m&&s.ignore("asset",m.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function vr(e,t,r,s){let n=new Set,i=E(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=u[1],a=R(e,o,t,s,"page-link");a&&(et(e,a)||_(a)&&!ee(a)&&!b(e,a)&&n.add(a.toString()));}return [...n]}async function We(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),o=i.url()||n;await he(e,r.assetMap,o,u,r,s);let a=u.toString("utf8");for(let l of q(e,o,a,s,"sitemap"))T(e,r,l,s,o);for(let l of wr(a)){let c=R(e,l.loc,o,s,"sitemap-loc");c&&(te(c)?Pe(e,r,c.toString(),s,o):b(e,c)?T(e,r,c.toString(),s,o):_(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),Z(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function br(e,t,r,s,n,i,u){let o=R(e,s,e.sourceOrigin,n,"asset-fetch");if(!o||!b(e,o))return;let a=o.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let l=K(e,r.assetMap,a),c=l?await st(l):null;if(l&&c)try{let w=await t.head(a,{timeout:6e4}),S=w.headers();if(w.ok()&&mr(c,S)){let v=await ge(l);if(v!==null){let I=w.url()||a,C=await he(e,r.assetMap,I,v,r,n,S);C.wroteFile&&(D(u,e,C.outputPath),$(C.outputPath)&&V(u,C.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:l,lastModified:S["last-modified"]||"",contentLength:S["content-length"]||"",phase:"download-assets"}),_e(r,n);let x=i.enabled?i.previousManifest.assets[a]:void 0,ye=de(o,S),re=x?.discoveredAssets?[...x.discoveredAssets]:[];if(ye||x?.isText){let at=v.toString("utf8"),Se=q(e,I,at,n,"asset:cached");(Se.length>0||!x)&&(re=Se);for(let ut of re)T(e,r,ut,n,I);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:C.outputPath,isText:ye||!!x?.isText,discoveredAssets:re,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let m=await t.get(a,{timeout:6e4});if(!m.ok()){n.skip("asset",a,m.status());return}let d=await m.body(),h=m.url()||a,p=m.headers(),P=await he(e,r.assetMap,h,d,r,n,p);P.wroteFile&&(D(u,e,P.outputPath),$(P.outputPath)&&V(u,P.outputPath)),r.doneAssets.add(a),_e(r,n);let g=[];if(de(o,p)){let w=d.toString("utf8");g=q(e,h,w,n,`asset:${f.extname(o.pathname).toLowerCase()||p["content-type"]||"unknown"}`);for(let S of g)T(e,r,S,n,h);}i.enabled&&(i.manifest.assets[a]={url:a,outputPath:P.outputPath,isText:de(o,p),discoveredAssets:g,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(l){n.error(`Failed asset ${a}`,{url:a,error:String(l)});}}}async function Rr(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),o=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await br(e,t,r,a,s,n,i));}});await Promise.all(o);}async function Cr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function xr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Cr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ar(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function kr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>fe(t,r))}async function Mr(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,m=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let p of l){let P=d.getAttribute(p);P&&a.add(P);}for(let p of m){let P=d.getAttribute(p);P&&P.split(",").forEach(g=>{let w=g.trim().split(/\s+/)[0];w&&a.add(w);});}let h=d.getAttribute("style");h&&c.add(h);}),{attrs:[...a],styles:[...c]}});for(let a of o.attrs)for(let l of Pr(a)){let c=R(e,l,r,s,"dom-attr");c&&b(e,c)&&i.add(c.toString());}for(let a of o.styles)for(let l of q(e,r,a,s,"dom-style"))i.add(l);}let u=await t.content();for(let o of q(e,r,u,s,"page-html"))i.add(o);return [...i]}async function He(e,t,r,s,n){let i=Y(e,t);await ae(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Je(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function ze(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Tr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Lr(e,t,r,s,n,i,u){let o=await chromium.launch({headless:true}),a=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Tr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let c=t.pageQueue.shift();if(!c)break;if(t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),i.enabled&&i.seenPages.add(c);let m=new URL(c),d=kr(e,m.pathname);if(!_(m)||b(e,m)||ee(m)){if(b(e,m)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),T(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let h=await tr(e,n,t,i,c,r);if(i.enabled&&s!=="single-url"){let g=i.manifest.pages[c];if(h.action==="reuse"&&g){for(let w of g.discoveredAssets)T(e,t,w,r,c);for(let w of g.discoveredPages)Z(e,t,w,r,c);g.lastSeenRunId=i.runId,g.sitemapLastmod=G(t.sitemapLastmodByPage[c])??g.sitemapLastmod,h.changeToken?.supported&&(g.changeToken=h.changeToken.token,g.tokenSource=h.changeToken.tokenSource??g.tokenSource),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:h.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Je(t)&&ze(t,r);continue}}let p=await(d?l.newPage():a.newPage()),P=!1;await p.route("**/*",async g=>{let w=g.request();if(w.isNavigationRequest()&&w.frame()===p.mainFrame())try{let S=new URL(w.url()),v=new URL(c);if(S.hash="",v.hash="",S.href===v.href){if(P){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:w.url()}),await g.abort("aborted");return}P=!0;}}catch{}await g.continue();}),p.on("response",g=>{try{let w=R(e,g.url(),e.sourceOrigin,void 0,"network-response");w&&b(e,w)&&T(e,t,w.toString(),r,c);}catch{}}),r.page(`Rendering ${c}`,{url:c});try{let g=null,w=null;try{if(g=await p.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),g&&g.ok())try{w=await g.text();}catch{w=null;}}catch(x){r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(x)});}if(g&&!g.ok()){r.skip("page",c,g.status()),await p.close();continue}d?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await Ar(e,p)):await xr(e,p),t.stats.pagesRendered++,Je(t)&&ze(t,r);let S=await Mr(e,p,c,r,!d);for(let x of S)T(e,t,x,r,c);let v=w??await p.content(),I=s!=="single-url"?vr(e,c,v,r):[];if(s!=="single-url")for(let x of I)Z(e,t,x,r,c);await He(e,c,v,t,r);let C=Y(e,c);D(u,e,C),V(u,C),i.enabled&&(i.manifest.pages[c]={url:c,outputPath:C,changeToken:h.changeToken?.supported?h.changeToken.token:null,tokenSource:h.changeToken?.supported?h.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[c])??null,discoveredPages:I,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){try{let w=await p.content();if(w&&w.trim()){await He(e,c,w,t,r);let S=Y(e,c);D(u,e,S),V(u,S),r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(g)});}}catch(w){r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(w)});}r.error(`Failed page ${c}`,{url:c,error:String(g)});}finally{await p.close();}}}finally{await l.close(),await a.close(),await o.close();}}async function Er(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Be(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Dr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ir(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function ot(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ve(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Fr(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ve(s)-Ve(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Ge(e){let t=await ot(f.join(e,"job.json"));for(let r of Fr(e,t)){let s=await Er(r);if(s.length>0)return s}return []}async function jr(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let a=await ot(f.join(u,"job.json"));if(!Dr(a))continue;let l=Ir(a);l>=n&&(n=l,s=u);}return s}async function Or(e){let t=await jr(e),r=t!==""?await Ge(t):await Ge(e.logDir),s=new Set;for(let n of r)if(Be(n.error)||Be(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function $r(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Ur(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Qr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ke(e,t,r,s,n){let i=R(e,r,e.sourceOrigin,s,"manual-url");i&&(te(i)?Pe(e,t,i.toString(),s,n):b(e,i)?T(e,t,i.toString(),s,n):_(i)?Z(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Nr(){let e=await oe(),t=zt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Or(e)),t.mode==="single-url"&&(r=await $r(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.preserveOutput?await Ur(e):{},i=s?await Xt(e):Ze(),u={enabled:s,manifest:$e(i),previousManifest:$e(i),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Qr(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...n},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},a=Kt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new Q(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let p=await chromium.launch({headless:true}),P=await p.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),g=P.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>Pe(e,o,S,l)),await We(e,g,o,l),e.seedPaths.forEach(S=>Ke(e,o,S,l,"seed-path"))):(r.forEach(S=>Ke(e,o,S,l,"cli")),await We(e,g,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let w=Array.from({length:Math.max(1,e.concurrency)},()=>Lr(e,o,l,t.mode,g,u,a));if(await Promise.all(w),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Rr(e,g,o,l,u,a),l.endMark("download-assets",{assets:o.doneAssets.size}),u.enabled&&e.maxPages===0&&await rr(e,o,u,l,a),t.mode==="full"){let S=await Sr(e,l);for(let v of S)D(a,e,v);}}finally{await P.close(),await p.close();}}u.enabled&&!t.resumeRewrite&&(u.manifest.updatedAt=new Date().toISOString(),await Yt(e,u.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(n,null,2)!==JSON.stringify(o.assetMap,null,2)&&D(a,e,f.join(e.outputDir,"asset-map.json"));let c=t.mode==="single-url"?ar(e,o):u.enabled&&!t.resumeRewrite?sr(e,u,a,n,o.assetMap):void 0;l.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${c?.length??0}`:"Rewriting text files...";l.progress(m,{phase:"rewrite-text",index:0,totalFiles:c?.length,changedTextFiles:0}),await qe(m);let d=Date.now(),h=await De(e,o.assetMap,{files:c,previousAssetMap:n,onProgress:async({index:p,totalFiles:P,changedTextFiles:g,file:w,changed:S})=>{S&&D(a,e,f.join(e.outputDir,w)),l.checkpoint(`Rewriting text file ${p}/${P}`,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w});let v=Date.now();if(!(p===1||p===P||v-d>=5e3))return;d=v;let C=`Rewriting text files: ${p}/${P}`;l.progress(C,{phase:"rewrite-text",index:p,totalFiles:P,changedTextFiles:g,file:w}),await qe(C);}});l.endMark("rewrite-text",{changedTextFiles:h}),await Re(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...a.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...a.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((c||[]).map(p=>W(e.outputDir,f.isAbsolute(p)?p:f.join(e.outputDir,p))).filter(p=>p!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${h} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:h}),await l.flush();}Nr().catch(async e=>{console.error(e);try{let r=(await oe().catch(()=>null))?.logDir??"logs",s=new Q(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.9",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|