@smart-cloud/publisher-exporter 1.0.13 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +4 -2
- package/dist/crawl.js +3 -3
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {spawn}from'child_process';import {existsSync}from'fs';import o from'path';import {fileURLToPath}from'url';var c=o.dirname(fileURLToPath(import.meta.url)),i=o.resolve(c,"..");function
|
|
2
|
+
import {spawn}from'child_process';import {existsSync,readFileSync}from'fs';import o from'path';import {fileURLToPath}from'url';var c=o.dirname(fileURLToPath(import.meta.url)),i=o.resolve(c,"..");function f(){try{let r=readFileSync(o.join(i,"package.json"),"utf8"),e=JSON.parse(r);if(typeof e.version=="string"&&e.version.trim())return e.version.trim()}catch{}return "unknown"}function g(){console.log(`publisher-exporter <command> [args]
|
|
3
3
|
|
|
4
4
|
Commands:
|
|
5
5
|
crawl Run the crawl phase
|
|
@@ -9,6 +9,7 @@ Commands:
|
|
|
9
9
|
queue-runner Process queued jobs from runtime JSON
|
|
10
10
|
prune-logs Delete archived log directories older than a retention window
|
|
11
11
|
install-browsers Install Playwright Chromium into the current/shared browsers path
|
|
12
|
+
version Show the installed CLI version
|
|
12
13
|
help Show this help
|
|
13
14
|
|
|
14
15
|
Examples:
|
|
@@ -17,4 +18,5 @@ Examples:
|
|
|
17
18
|
publisher-exporter queue-runner --runtime-dir /srv/site/runtime --max-jobs 1
|
|
18
19
|
publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30
|
|
19
20
|
publisher-exporter install-browsers
|
|
20
|
-
|
|
21
|
+
publisher-exporter --version
|
|
22
|
+
`);}function v(){console.log(f());}function p(r){let e=o.join(c,`${r}.js`);if(!existsSync(e))throw new Error(`Missing CLI entrypoint: ${e}`);return e}function y(){let r=[o.join(i,"node_modules","playwright","cli.js"),o.join(i,"node_modules","playwright-core","cli.js")];for(let e of r)if(existsSync(e))return e;throw new Error("Playwright CLI was not found under this package. Run `npm install` first.")}function t(r,e){return new Promise((n,s)=>{let a=spawn(process.execPath,[r,...e],{cwd:i,env:process.env,stdio:"inherit"});a.on("error",s),a.on("close",(d,l)=>{if(l){s(new Error(`Command terminated by signal ${l}.`));return}n(d??1);});})}async function b(r){for(let e of ["crawl","deploy","invalidate"]){let n=await t(p(e),r);if(n!==0)return n}return 0}async function x(){let[r="help",...e]=process.argv.slice(2);if(["version","--version","-v"].includes(r)){v();return}if(["help","--help","-h"].includes(r)){g();return}if(r==="publish"){process.exitCode=await b(e);return}if(r==="install-browsers"){let n=e.length>0?e:["install","chromium"];process.exitCode=await t(y(),n);return}if(r==="crawl"||r==="deploy"||r==="invalidate"||r==="queue-runner"||r==="prune-logs"){process.exitCode=await t(p(r),e);return}throw new Error(`Unknown command: ${r}`)}x().catch(r=>{console.error(r instanceof Error?r.message:String(r)),process.exit(1);});
|
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import y from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import St from'crypto';import bt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var pt=".deploy-plan.json",gt="deploy-plan.json";function ht(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function
|
|
1
|
+
import {chromium}from'playwright';import y from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import St from'crypto';import bt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var pt=".deploy-plan.json",gt="deploy-plan.json";function ht(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Ae(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ne(e){return [...new Set(e.map(Ae).filter(t=>t!==null))]}function ft(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ne(t.changedFiles||[]),deletedFiles:ne(t.deletedFiles||[]),rewriteTargets:ne(t.rewriteTargets||[])}}function mt(e){let t=f.join(f.resolve(e.outputDir),pt),r=ht();return r?[f.join(r,gt),t]:[t]}function H(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return Ae(n)}async function ke(e,t){let r=ft(e,t);for(let s of mt(e))await y.mkdir(f.dirname(s),{recursive:true}),await y.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Te(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function ie(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Pt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let o=Te(i.targetOrigin);o&&(u.targetOrigin=o);}let a=ie(i.extraReplacements);if(Object.keys(a).length>0&&(u.extraReplacements=a),i.s3&&typeof i.s3=="object"){let o={},c=i.s3;typeof c.bucket=="string"&&(o.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(o.prefix=c.prefix.trim()),typeof c.region=="string"&&(o.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(o.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(o.assetCacheControl=c.assetCacheControl.trim()),Object.keys(o).length>0&&(u.s3=o);}if(i.cloudFront&&typeof i.cloudFront=="object"){let o={},c=i.cloudFront;typeof c.distributionId=="string"&&(o.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(o.invalidationPaths=c.invalidationPaths.map(l=>String(l??"").trim()).filter(l=>l.length>0)),Object.keys(o).length>0&&(u.cloudFront=o);}t[n]=u;}return t}function oe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function yt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Me(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,oe(s,r))}async function ae(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await y.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=Te(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=ie(r.extraReplacements),r.postCrawlCopyMap=ie(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=Pt(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=yt(),n=s?f.resolve(s,".."):"";return n?(r.outputDir=Me(n,r.outputDir,"export"),r.logDir=Me(n,r.logDir,"logs")):(f.isAbsolute(r.outputDir)||(r.outputDir=oe(r.outputDir,"export")),f.isAbsolute(r.logDir)||(r.logDir=oe(r.logDir,"logs"))),r}function Ie(e,t=10){return St.createHash("sha1").update(e).digest("hex").slice(0,t)}async function ue(e){await y.mkdir(f.dirname(e),{recursive:true});}function j(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function D(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function O(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function J(e){return e.replace(/\//g,"\\/")}function Le(e){return e.replace(/\//g,"\\\\/")}function z(e,t,r){if(!t)return;e[t]=r;let s=J(t),n=J(r);e[s]=n;let i=Le(t),u=Le(r);e[i]=u;let a=O(t),o=O(r);e[a]=o;let c=O(s),l=O(n);e[c]=l;let g=O(i),d=O(u);e[g]=d;}function le(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function De(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ee(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var xt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],At=new Set([".html",".htm"]),je="WPSuite.io Static Publisher",kt=je.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Mt(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=f.extname(r).toLowerCase();return At.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Tt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${kt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${je}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Lt(e){let t=D(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function It(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[a,o]of r)u=u.split(a).join(o);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,a,o)=>`${u}${s(a)}${o}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,a,o,c)=>Lt(o)?`${u}${s(o)}${c}`:i),n}function Dt(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Et(e,t,r){let s=Dt(t,r);if(!s)return e;let n=e;for(let i of xt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),a=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),o=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(a,`${s}${i}`).replace(o,`${s}${i}`);let c=i.replace(/\//g,"\\/"),l=s.replace(/\//g,"\\/"),g=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${g}`,"g"),p=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${g}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${g}`,"g");n=n.replace(d,`${l}${c}`).replace(p,`${l}${c}`).replace(m,`${l}${c}`);}return n}function de(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Lt(e){let t=D(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function It(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[a,o]of r)u=u.split(a).join(o);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,a,o)=>`${u}${s(a)}${o}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,a,o,c)=>Lt(o)?`${u}${s(o)}${c}`:i),n}function Dt(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Et(e,t,r){let s=Dt(t,r);if(!s)return e;let n=e;for(let i of xt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),a=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),o=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(a,`${s}${i}`).replace(o,`${s}${i}`);let c=i.replace(/\//g,"\\/"),l=s.replace(/\//g,"\\/"),g=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${g}`,"g"),p=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${g}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${g}`,"g");n=n.replace(d,`${l}${c}`).replace(p,`${l}${c}`).replace(m,`${l}${c}`);}return n}function de(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:De(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Ee(e.outputDir,r,n)}${i}`}function ce(e,t,r,s,n){z(e,r,de(t,s,n));}function Ft(e,t,r,s,n){if(!r||r===s)return;z(e,r,s);let i=de(t,r,n),u=de(t,s,n);z(e,i,u);}function pe(e,t,r,s,n={}){let i=e,u={};ce(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[o,c]of Object.entries(t.extraReplacements))ce(u,t,o,c,s);for(let[o,c]of Object.entries(r))ce(u,t,o,c,s);for(let[o,c]of Object.entries(n)){let l=r[o];!l||l===c||Ft(u,t,c,l,s);}let a=Object.entries(u).sort((o,c)=>c[0].length-o[0].length);for(let[o,c]of a)i=i.split(o).join(c);return t.urlRewriteMode!=="absolute"&&(i=Et(i,t,s)),i=It(i,a.map(([,o])=>o)),Mt(t,i,s)&&(i=Tt(i)),i}async function jt(e,t,r,s){let n=f.join(e.outputDir,s),i=f.extname(n).toLowerCase(),u;try{u=await y.readFile(n,"utf8");}catch(c){if(c.code!=="ENOENT")throw c;return {changed:false}}let o=pe(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return o!==u?(await y.writeFile(n,o,"utf8"),{changed:true}):{changed:false}}async function Ot(e,t){return await new Promise((r,s)=>{let n=o=>{if(a(),o?.error){s(new Error(o.error));return}r({changed:!!o?.changed});},i=o=>{a(),s(o);},u=o=>{a(),o!==0&&s(new Error(`Rewrite worker exited with code ${o}.`));},a=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function Oe(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>f.resolve(d)).filter(d=>d.startsWith(f.resolve(e.outputDir))).map(d=>f.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await bt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&j(d)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),a=Math.max(1,Math.min(u,n.length,availableParallelism())),o=0,c=0,l=null;if(a===1){for(let d of n){let p=await jt(e,t,r.previousAssetMap??{},d);p.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:d,changed:p.changed});}return i}let g=Array.from({length:a},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!l;){let p=o;if(p>=n.length)return;o+=1;let m=n[p],P=await Ot(d,m);P.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:m,changed:P.changed});}}catch(p){throw l=p instanceof Error?p:new Error(String(p)),l}finally{await d.terminate().catch(()=>{});}});return await Promise.all(g),i}var U={error:0,warn:1,info:2,debug:3},$e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",B=$e?f.join($e,"current-progress.json"):"",Ue=Promise.resolve();function Qe(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function $t(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function Ut(e,t,r){B&&(Ue=Ue.then(async()=>{let s=new Date,n=s.toISOString();await y.mkdir(f.dirname(B),{recursive:true});let i=await y.readFile(B,"utf8").then(P=>JSON.parse(P)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},a=Qe(e),o=i?.currentStep||Qe(i?.source||""),c=i?.startedAt||n,l={...i?.stepDurationsSec??{}},g=i?.stepStartedAt||n;if(o&&o!==a&&i?.stepStartedAt){let P=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));l[o]=(l[o]??0)+P,g=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(g).getTime())/1e3)),p=Math.max(0,Math.round((s.getTime()-new Date(c).getTime())/1e3)),m={checkedAt:n,source:e,message:t,details:$t(u,r??{}),startedAt:c,currentStep:a,stepStartedAt:g,stepElapsedSec:d,totalElapsedSec:p,stepDurationsSec:l};await y.writeFile(B,JSON.stringify(m,null,2),"utf8");}).catch(()=>{}));}async function Qt(e,t){await y.mkdir(f.dirname(e),{recursive:true}),await y.appendFile(e,`${JSON.stringify(t)}
|
|
5
5
|
`,"utf8");}function Nt(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function qt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:Nt(e,r),level:e,message:t,details:r??{}}}var N=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await y.appendFile(this.logPath,`${t}
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Qt(t,r));}updateCurrentEvent(t,r,s){let n=qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Wt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Ht=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Jt=new Set(["",".html",".htm"]),zt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Bt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Vt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let a=e[u];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--resume-rewrite")n=true,i=true;else if(a==="--crawl-mode"){let o=e[++u];if(!o)throw new Error("--crawl-mode requires a value");s=o==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let o=e[++u];if(!o)throw new Error("--url requires a value");r="single-url",n=true,t.push(o);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let o=e[++u];if(!o)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${o}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Pe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Kt(){let e=Pe();return e?f.resolve(e,".."):""}function Gt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Xt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Kt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Pe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Gt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function et(e){let t=Pe();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function tt(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Yt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function E(e,t,r){let s=H(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Qe(e,t,r){let s=H(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function K(e,t){e.rewriteTargets.add(f.resolve(t));}async function Zt(e){try{let t=await y.readFile(et(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return tt()}function Ne(e){return JSON.parse(JSON.stringify(e))}async function er(e,t){let r=et(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function tr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function rr(e,t,r,s){let n=[...new Set(r.map(a=>a.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let a=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!a.ok())return s.warn(`Change token lookup failed with HTTP ${a.status()}`,{endpoint:u,status:a.status()}),new Map;let o=await a.json().catch(()=>null);if(!o||!Array.isArray(o.items))return new Map;let c=new Map;for(let l of o.items)!l||typeof l.url!="string"||c.set(l.url,l);return c}catch(a){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(a)}),new Map}}async function qe(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await rr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function sr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await qe(e,t,s,n,i)};let a=await qe(e,t,s,n,i);if(a?.supported&&a.token)return u.changeToken===a.token?{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=G(r.sitemapLastmodByPage[n]);return o&&u.sitemapLastmod&&u.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function nr(e,t,r,s,n){let i=_e(r.previousManifest),u=0;for(let[d,p]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Qe(n,e,p.outputPath);try{await y.unlink(p.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:p.outputPath,error:String(m)});}delete r.manifest.pages[d],u++;}let a=_e(r.manifest,r.runId),o=new Set,c=new Set;for(let d of a){for(let P of q(e,d))o.add(P);let p=r.manifest.assets[d]?.outputPath;p&&c.add(p);let m=X(e,t.assetMap,d);m&&c.add(m);}for(let d of Object.keys(r.manifest.assets))a.has(d)||delete r.manifest.assets[d];let l=0,g=0;for(let d of i){let p=new Set,m=r.previousManifest.assets[d]?.outputPath;m&&p.add(m);let P=X(e,t.assetMap,d);if(!a.has(d)&&P)p.add(P);else if(!a.has(d)){let h=Se(e,d);p.add(h.originalFilePath),h.hashedFilePath&&p.add(h.hashedFilePath);}for(let h of p)if(!c.has(h)){Qe(n,e,h);try{await y.unlink(h),l++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:h,error:String(w)});}}for(let h of q(e,d))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],g++);}(u>0||l>0||g>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${l} stale assets, ${g} stale asset mappings`,{removedPages:u,removedAssets:l,removedAssetMappings:g,phase:"incremental-cleanup"});}function _e(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function We(e,t,r){for(let s of t)for(let n of q(e,s))if(r.has(n))return true;return false}function ir(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=tr(s,n);if(u.size===0)return [...i];for(let a of Object.values(t.manifest.pages))We(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let[a,o]of Object.entries(t.manifest.assets)){o.isText&&We(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let c of q(e,a))if(u.has(c)){E(r,e,o.outputPath),o.isText&&i.add(f.resolve(o.outputPath));break}}return [...i]}function q(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function or(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ar(e,t,r){for(let s of or(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function ur(e,t,r){for(let s of q(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return i}return null}function X(e,t,r){let s=ur(e,t,r);return s||Se(e,r).originalFilePath}function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function He(e){let t=lr();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function cr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Z(e,s)));for(let s of t.doneAssets){let n=X(e,t.assetMap,s);!n||!j(n)||r.add(f.resolve(n));}return [...r]}var Y="re:",V=new Map;function dr(e){if(!e.startsWith(Y))return null;let t=V.get(e);if(t!==void 0)return t||null;let r=e.slice(Y.length).trim();if(!r)return V.set(e,false),null;try{let s=new RegExp(r);return V.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),V.set(e,false),null}}function ye(e,t){let r=dr(t);return r?r.test(e):t.startsWith(Y)?false:e.startsWith(t)}function pr(e,t){return t.find(r=>ye(e,r))}function st(e,t){let r=pr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Y)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function gr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function hr(e,t){return gr(e).has(t.hostname)}function fr(e){let t=D(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=D(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function te(e){return Bt.some(t=>e.pathname.startsWith(t))}function C(e,t,r=e.sourceOrigin,s,n="url"){let i=fr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!hr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let a=st(e,u);if(a)return s?.reject(n,u.toString(),a,r),null;let o=new URL(e.sourceOrigin);return u.protocol=o.protocol,u.host=o.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function it(e){return Ht.has(f.extname(e.pathname).toLowerCase())}function W(e){return te(e)?false:Jt.has(f.extname(e.pathname).toLowerCase())}function re(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function R(e,t){return re(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:it(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>ye(t.pathname,r)):false}function Z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function Se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),a=le(e.targetOrigin,i),o=`${a}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n};let c=f.extname(s),g=`${c?s.slice(0,-c.length):s}.${Le(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n,hashedPathname:g,hashedFilePath:f.join(e.outputDir,g),hashedPublicUrl:le(e.targetOrigin,g)}}function mr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),a=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{a.push(new URL(e.targetOrigin).origin);}catch{}for(let o of a)try{let c=new URL(o);t[c.origin+i.pathname+i.search]=s;}catch{}try{let o=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=o;}catch{}try{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}try{let o=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}}catch{}}async function me(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function wr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function at(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Pr(e,t){let r=at(t),s=wr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function yr(e,t){if(!t)return;let r=at(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function ge(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return zt.has(r)||Wt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function he(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function we(e,t,r,s,n,i,u){let a=Se(e,r),o=a.originalFilePath,c=a.originalPublicUrlWithSearch;if(a.preferQueryHashed&&a.hashedFilePath&&a.hashedPublicUrl){let d=ar(e,t,r);if(d)o=d.outputPath,c=d.publicUrl;else {let p=await me(a.originalFilePath);if(p!==null&&!p.equals(s)){let m=j(a.originalFilePath)?Buffer.from(pe(s.toString("utf8"),e,t,a.originalFilePath),"utf8"):null;(m===null||!p.equals(m))&&(o=a.hashedFilePath,c=a.hashedPublicUrl);}}}mr(e,t,r,c,a.originalPublicUrl);let l=await me(o),g=false;return (l===null||!l.equals(s))&&(await ue(o),await y.writeFile(o,s),g=true),await yr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function ee(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"page");if(!u)return;if(!W(u)||it(u)){if(R(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),A(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),te(u)?"asset/internal path cannot be page":"not page-like",n);return}let a=u.toString();!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function A(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"asset");if(!u)return;if(!R(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let a=u.toString();!t.doneAssets.has(a)&&!t.queuedAssets.has(a)&&(t.queuedAssets.add(a),t.assetQueue.push(a),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ve(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"sitemap");if(!u)return;if(!re(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let a=u.toString();!t.doneSitemaps.has(a)&&!t.queuedSitemaps.has(a)&&(t.queuedSitemaps.add(a),t.sitemapQueue.push(a),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Sr(e){let t=D(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function vr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function br(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function fe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function ut(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await ut(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Rr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,a=i.prefix;if(!u||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:a});continue}let o=Xt(u);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let c=o.resolvedPath,l=fe(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:a});continue}let g;try{g=await y.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:c,prefixPath:a});continue}let d=f.resolve(e.outputDir,l);if(g.isDirectory()){n.push(...await ut(c,d)),s++,t.progress(`Copied static directory to export: ${c} -> /${l}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${l}`,copiedItems:s});continue}let p=f.basename(c),m=a.endsWith("/")?f.join(d,p):d;await y.mkdir(f.dirname(m),{recursive:true}),await y.copyFile(c,m),n.push(m),s++,t.progress(`Copied static file to export: ${c} -> /${fe(a.endsWith("/")?`${a}${p}`:a)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${fe(a.endsWith("/")?`${a}${p}`:a)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function _(e,t,r,s,n){let i=new Set,u=D(r).replace(/\\\//g,"/");for(let c of br(u)){let l=C(e,c,t,s,`${n}:css-url`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let l=C(e,c[1],t,s,`${n}:css-import`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let l=C(e,c[1],t,s,`${n}:xml-stylesheet`);l&&R(e,l)&&i.add(l.toString());}let a="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",o=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of u.matchAll(o)){let l=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(nt(l)){s.ignore(`${n}:serialized-url`,l,"looks like JavaScript/code fragment",t);continue}let g=C(e,l,t,s,`${n}:serialized-url`);g&&R(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Cr(e,t,r,s){let n=new Set,i=D(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let a=u[1],o=C(e,a,t,s,"page-link");o&&(st(e,o)||W(o)&&!te(o)&&!R(e,o)&&n.add(o.toString()));}return [...n]}async function Je(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),a=i.url()||n;await we(e,r.assetMap,a,u,r,s);let o=u.toString("utf8");for(let c of _(e,a,o,s,"sitemap"))A(e,r,c,s,a);for(let c of Sr(o)){let l=C(e,c.loc,a,s,"sitemap-loc");l&&(re(l)?ve(e,r,l.toString(),s,a):R(e,l)?A(e,r,l.toString(),s,a):W(l)?(c.lastmod&&(r.sitemapLastmodByPage[l.toString()]=c.lastmod),ee(e,r,l.toString(),s,a)):s.reject("sitemap-loc",l.toString(),"not page/sitemap/asset-like",a));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function xr(e,t,r,s,n,i,u){let a=C(e,s,e.sourceOrigin,n,"asset-fetch");if(!a||!R(e,a))return;let o=a.toString();if(r.doneAssets.has(o))return;let c=i.enabled?i.previousManifest.assets[o]:void 0,l=(g,d={})=>{if(!c)return false;i.manifest.assets[o]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(o);for(let p of c.discoveredAssets||[])A(e,r,p,n,o);return n.warn(`Keeping previous incremental asset ${o} after fetch issue.`,{url:o,reason:g,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),he(r,n),true};n.asset(`Fetching asset ${o}`,{url:o});try{let g=X(e,r.assetMap,o),d=g?await ot(g):null;if(g&&d)try{let S=await t.head(o,{timeout:6e4}),b=S.headers();if(S.ok()&&Pr(d,b)){let I=await me(g);if(I!==null){let x=S.url()||o,Q=await we(e,r.assetMap,x,I,r,n,b);Q.wroteFile&&(E(u,e,Q.outputPath),j(Q.outputPath)&&K(u,Q.outputPath)),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:g,lastModified:b["last-modified"]||"",contentLength:b["content-length"]||"",phase:"download-assets"}),he(r,n);let be=ge(a,b),se=c?.discoveredAssets?[...c.discoveredAssets]:[];if(be||c?.isText){let ct=I.toString("utf8"),Re=_(e,x,ct,n,"asset:cached");(Re.length>0||!c)&&(se=Re);for(let dt of se)A(e,r,dt,n,x);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:Q.outputPath,isText:be||!!c?.isText,discoveredAssets:se,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let p=await t.get(o,{timeout:6e4});if(!p.ok()){if(l(`http-${p.status()}`,{status:p.status()}))return;n.skip("asset",o,p.status());return}let m=await p.body(),P=p.url()||o,h=p.headers(),w=await we(e,r.assetMap,P,m,r,n,h);w.wroteFile&&(E(u,e,w.outputPath),j(w.outputPath)&&K(u,w.outputPath)),r.doneAssets.add(o),he(r,n);let v=[];if(ge(a,h)){let S=m.toString("utf8");v=_(e,P,S,n,`asset:${f.extname(a.pathname).toLowerCase()||h["content-type"]||"unknown"}`);for(let b of v)A(e,r,b,n,P);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:w.outputPath,isText:ge(a,h),discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){if(l("fetch-error",{error:String(g)}))return;n.error(`Failed asset ${o}`,{url:o,error:String(g)});}}async function Ar(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),a=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await xr(e,t,r,o,s,n,i));}});await Promise.all(a);}async function kr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Mr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await kr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Tr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Lr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ye(t,r))}async function Ir(e,t,r,s,n=true){let i=new Set;if(n){let a=await t.evaluate(()=>{let o=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,g=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let m of c){let P=d.getAttribute(m);P&&o.add(P);}for(let m of g){let P=d.getAttribute(m);P&&P.split(",").forEach(h=>{let w=h.trim().split(/\s+/)[0];w&&o.add(w);});}let p=d.getAttribute("style");p&&l.add(p);}),{attrs:[...o],styles:[...l]}});for(let o of a.attrs)for(let c of vr(o)){let l=C(e,c,r,s,"dom-attr");l&&R(e,l)&&i.add(l.toString());}for(let o of a.styles)for(let c of _(e,r,o,s,"dom-style"))i.add(c);}let u=await t.content();for(let a of _(e,r,u,s,"page-html"))i.add(a);return [...i]}async function ze(e,t,r,s,n){let i=Z(e,t);await ue(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Be(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ve(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Dr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Er(e,t,r,s,n,i,u){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Dr(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let g=new URL(l),d=Lr(e,g.pathname);if(!W(g)||R(e,g)||te(g)){if(R(e,g)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),A(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let p=await sr(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let h=i.manifest.pages[l];if(p.action==="reuse"&&h){for(let w of h.discoveredAssets)A(e,t,w,r,l);for(let w of h.discoveredPages)ee(e,t,w,r,l);h.lastSeenRunId=i.runId,h.sitemapLastmod=G(t.sitemapLastmodByPage[l])??h.sitemapLastmod,p.changeToken?.supported&&(h.changeToken=p.changeToken.token,h.tokenSource=p.changeToken.tokenSource??h.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Be(t)&&Ve(t,r);continue}}let m=await(d?c.newPage():o.newPage()),P=!1;await m.route("**/*",async h=>{let w=h.request();if(w.isNavigationRequest()&&w.frame()===m.mainFrame())try{let v=new URL(w.url()),S=new URL(l);if(v.hash="",S.hash="",v.href===S.href){if(P){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:w.url()}),await h.abort("aborted");return}P=!0;}}catch{}await h.continue();}),m.on("response",h=>{try{let w=C(e,h.url(),e.sourceOrigin,void 0,"network-response");w&&R(e,w)&&A(e,t,w.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let h=null,w=null;try{if(h=await m.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),h&&h.ok())try{w=await h.text();}catch{w=null;}}catch(x){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(x)});}if(h&&!h.ok()){r.skip("page",l,h.status()),await m.close();continue}d?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Tr(e,m)):await Mr(e,m),t.stats.pagesRendered++,Be(t)&&Ve(t,r);let v=await Ir(e,m,l,r,!d);for(let x of v)A(e,t,x,r,l);let S=w??await m.content(),b=s!=="single-url"?Cr(e,l,S,r):[];if(s!=="single-url")for(let x of b)ee(e,t,x,r,l);await ze(e,l,S,t,r);let I=Z(e,l);E(u,e,I),K(u,I),i.trackManifest&&(i.manifest.pages[l]={url:l,outputPath:I,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[l])??null,discoveredPages:b,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(h){try{let w=await m.content();if(w&&w.trim()){await ze(e,l,w,t,r);let v=Z(e,l);E(u,e,v),K(u,v),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(h)});}}catch(w){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(w)});}r.error(`Failed page ${l}`,{url:l,error:String(h)});}finally{await m.close();}}}finally{await c.close(),await o.close(),await a.close();}}async function Fr(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ke(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function jr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Or(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function lt(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ge(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function $r(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ge(s)-Ge(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Xe(e){let t=await lt(f.join(e,"job.json"));for(let r of $r(e,t)){let s=await Fr(r);if(s.length>0)return s}return []}async function Ur(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let o=await lt(f.join(u,"job.json"));if(!jr(o))continue;let c=Or(o);c>=n&&(n=c,s=u);}return s}async function Qr(e){let t=await Ur(e),r=t!==""?await Xe(t):await Xe(e.logDir),s=new Set;for(let n of r)if(Ke(n.error)||Ke(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function qr(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function _r(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ye(e,t,r,s,n){let i=C(e,r,e.sourceOrigin,s,"manual-url");i&&(re(i)?ve(e,t,i.toString(),s,n):R(e,i)?A(e,t,i.toString(),s,n):W(i)?ee(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Wr(){let e=await ae(),t=Vt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Qr(e)),t.mode==="single-url"&&(r=await Nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.mode==="full"&&!t.resumeRewrite,i=t.preserveOutput?await qr(e):{},u=s?await Zt(e):tt(),a={enabled:s,trackManifest:n,manifest:Ne(u),previousManifest:Ne(u),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await _r(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...i},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},c=Yt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new N(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let P=await chromium.launch({headless:true}),h=await P.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),w=h.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>ve(e,o,S,l)),await Je(e,w,o,l),e.seedPaths.forEach(S=>Ye(e,o,S,l,"seed-path"))):(r.forEach(S=>Ye(e,o,S,l,"cli")),await Je(e,w,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let v=Array.from({length:Math.max(1,e.concurrency)},()=>Er(e,o,l,t.mode,w,a,c));if(await Promise.all(v),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Ar(e,w,o,l,a,c),l.endMark("download-assets",{assets:o.doneAssets.size}),a.enabled&&e.maxPages===0&&await nr(e,o,a,l,c),t.mode==="full"){let S=await Rr(e,l);for(let b of S)E(c,e,b);}}finally{await h.close(),await P.close();}}a.trackManifest&&(a.manifest.updatedAt=new Date().toISOString(),await er(e,a.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(i,null,2)!==JSON.stringify(o.assetMap,null,2)&&E(c,e,f.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?cr(e,o):a.enabled&&!t.resumeRewrite?ir(e,a,c,i,o.assetMap):void 0;l.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";l.progress(d,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await He(d);let p=Date.now(),m=await je(e,o.assetMap,{files:g,previousAssetMap:i,onProgress:async({index:P,totalFiles:h,changedTextFiles:w,file:v,changed:S})=>{S&&E(c,e,f.join(e.outputDir,v)),l.checkpoint(`Rewriting text file ${P}/${h}`,{phase:"rewrite-text",index:P,totalFiles:h,changedTextFiles:w,file:v});let b=Date.now();if(!(P===1||P===h||b-p>=5e3))return;p=b;let x=`Rewriting text files: ${P}/${h}`;l.progress(x,{phase:"rewrite-text",index:P,totalFiles:h,changedTextFiles:w,file:v}),await He(x);}});l.endMark("rewrite-text",{changedTextFiles:m}),await Ae(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(a.enabled&&!t.resumeRewrite),changedFiles:a.enabled&&!t.resumeRewrite?[...c.changedFiles].sort():[],deletedFiles:a.enabled&&!t.resumeRewrite?[...c.deletedFiles].sort():[],rewriteTargets:a.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(P=>H(e.outputDir,f.isAbsolute(P)?P:f.join(e.outputDir,P))).filter(P=>P!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${m} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:m}),await l.flush();}Wr().catch(async e=>{console.error(e);try{let r=(await ae().catch(()=>null))?.logDir??"logs",s=new N(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Qt(t,r));}updateCurrentEvent(t,r,s){let n=qt(t,r,s);this.enqueueTask(async()=>{await y.mkdir(this.logDir,{recursive:true}),await y.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Wt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Ht=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Jt=new Set(["",".html",".htm"]),zt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Bt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Vt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let a=e[u];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--resume-rewrite")n=true,i=true;else if(a==="--crawl-mode"){let o=e[++u];if(!o)throw new Error("--crawl-mode requires a value");s=o==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let o=e[++u];if(!o)throw new Error("--url requires a value");r="single-url",n=true,t.push(o);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let o=e[++u];if(!o)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${o}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function ye(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Kt(){let e=ye();return e?f.resolve(e,".."):""}function Gt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Xt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Kt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:ye(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Gt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function et(e){let t=ye();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function tt(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Yt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function E(e,t,r){let s=H(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Ne(e,t,r){let s=H(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function K(e,t){e.rewriteTargets.add(f.resolve(t));}async function Zt(e){try{let t=await y.readFile(et(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return tt()}function qe(e){return JSON.parse(JSON.stringify(e))}async function er(e,t){let r=et(e);await y.mkdir(f.dirname(r),{recursive:true}),await y.writeFile(r,JSON.stringify(t,null,2),"utf8");}function G(e){if(!e)return null;let t=e.trim();return t||null}function tr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function rr(e,t,r,s){let n=[...new Set(r.map(a=>a.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let a=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!a.ok())return s.warn(`Change token lookup failed with HTTP ${a.status()}`,{endpoint:u,status:a.status()}),new Map;let o=await a.json().catch(()=>null);if(!o||!Array.isArray(o.items))return new Map;let c=new Map;for(let l of o.items)!l||typeof l.url!="string"||c.set(l.url,l);return c}catch(a){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(a)}),new Map}}async function ge(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await rr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function sr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await ge(e,t,s,n,i):null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await ge(e,t,s,n,i)};let a=await ge(e,t,s,n,i);if(a?.supported&&a.token)return u.changeToken===a.token?{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=G(r.sitemapLastmodByPage[n]);return o&&u.sitemapLastmod&&u.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function nr(e,t,r,s,n){let i=_e(r.previousManifest),u=0;for(let[d,p]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ne(n,e,p.outputPath);try{await y.unlink(p.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:p.outputPath,error:String(m)});}delete r.manifest.pages[d],u++;}let a=_e(r.manifest,r.runId),o=new Set,c=new Set;for(let d of a){for(let P of q(e,d))o.add(P);let p=r.manifest.assets[d]?.outputPath;p&&c.add(p);let m=X(e,t.assetMap,d);m&&c.add(m);}for(let d of Object.keys(r.manifest.assets))a.has(d)||delete r.manifest.assets[d];let l=0,g=0;for(let d of i){let p=new Set,m=r.previousManifest.assets[d]?.outputPath;m&&p.add(m);let P=X(e,t.assetMap,d);if(!a.has(d)&&P)p.add(P);else if(!a.has(d)){let h=ve(e,d);p.add(h.originalFilePath),h.hashedFilePath&&p.add(h.hashedFilePath);}for(let h of p)if(!c.has(h)){Ne(n,e,h);try{await y.unlink(h),l++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:h,error:String(w)});}}for(let h of q(e,d))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],g++);}(u>0||l>0||g>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${l} stale assets, ${g} stale asset mappings`,{removedPages:u,removedAssets:l,removedAssetMappings:g,phase:"incremental-cleanup"});}function _e(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function We(e,t,r){for(let s of t)for(let n of q(e,s))if(r.has(n))return true;return false}function ir(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=tr(s,n);if(u.size===0)return [...i];for(let a of Object.values(t.manifest.pages))We(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let[a,o]of Object.entries(t.manifest.assets)){o.isText&&We(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let c of q(e,a))if(u.has(c)){E(r,e,o.outputPath),o.isText&&i.add(f.resolve(o.outputPath));break}}return [...i]}function q(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function or(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ar(e,t,r){for(let s of or(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function ur(e,t,r){for(let s of q(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return i}return null}function X(e,t,r){let s=ur(e,t,r);return s||ve(e,r).originalFilePath}function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function He(e){let t=lr();if(t)try{let r=await y.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await y.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function cr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(Z(e,s)));for(let s of t.doneAssets){let n=X(e,t.assetMap,s);!n||!j(n)||r.add(f.resolve(n));}return [...r]}var Y="re:",V=new Map;function dr(e){if(!e.startsWith(Y))return null;let t=V.get(e);if(t!==void 0)return t||null;let r=e.slice(Y.length).trim();if(!r)return V.set(e,false),null;try{let s=new RegExp(r);return V.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),V.set(e,false),null}}function Se(e,t){let r=dr(t);return r?r.test(e):t.startsWith(Y)?false:e.startsWith(t)}function pr(e,t){return t.find(r=>Se(e,r))}function st(e,t){let r=pr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Y)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function gr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function hr(e,t){return gr(e).has(t.hostname)}function fr(e){let t=D(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=D(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function te(e){return Bt.some(t=>e.pathname.startsWith(t))}function C(e,t,r=e.sourceOrigin,s,n="url"){let i=fr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!hr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let a=st(e,u);if(a)return s?.reject(n,u.toString(),a,r),null;let o=new URL(e.sourceOrigin);return u.protocol=o.protocol,u.host=o.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function it(e){return Ht.has(f.extname(e.pathname).toLowerCase())}function W(e){return te(e)?false:Jt.has(f.extname(e.pathname).toLowerCase())}function re(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function R(e,t){return re(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:it(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>Se(t.pathname,r)):false}function Z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function ve(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),a=le(e.targetOrigin,i),o=`${a}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n};let c=f.extname(s),g=`${c?s.slice(0,-c.length):s}.${Ie(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n,hashedPathname:g,hashedFilePath:f.join(e.outputDir,g),hashedPublicUrl:le(e.targetOrigin,g)}}function mr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),a=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{a.push(new URL(e.targetOrigin).origin);}catch{}for(let o of a)try{let c=new URL(o);t[c.origin+i.pathname+i.search]=s;}catch{}try{let o=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=o;}catch{}try{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}try{let o=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}}catch{}}async function we(e){try{return await y.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await y.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function wr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function at(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Pr(e,t){let r=at(t),s=wr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function yr(e,t){if(!t)return;let r=at(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await y.utimes(e,s.atime,n).catch(()=>{});}function he(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return zt.has(r)||Wt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function fe(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function Pe(e,t,r,s,n,i,u){let a=ve(e,r),o=a.originalFilePath,c=a.originalPublicUrlWithSearch;if(a.preferQueryHashed&&a.hashedFilePath&&a.hashedPublicUrl){let d=ar(e,t,r);if(d)o=d.outputPath,c=d.publicUrl;else {let p=await we(a.originalFilePath);if(p!==null&&!p.equals(s)){let m=j(a.originalFilePath)?Buffer.from(pe(s.toString("utf8"),e,t,a.originalFilePath),"utf8"):null;(m===null||!p.equals(m))&&(o=a.hashedFilePath,c=a.hashedPublicUrl);}}}mr(e,t,r,c,a.originalPublicUrl);let l=await we(o),g=false;return (l===null||!l.equals(s))&&(await ue(o),await y.writeFile(o,s),g=true),await yr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function ee(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"page");if(!u)return;if(!W(u)||it(u)){if(R(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),A(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),te(u)?"asset/internal path cannot be page":"not page-like",n);return}let a=u.toString();!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function A(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"asset");if(!u)return;if(!R(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let a=u.toString();!t.doneAssets.has(a)&&!t.queuedAssets.has(a)&&(t.queuedAssets.add(a),t.assetQueue.push(a),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function be(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"sitemap");if(!u)return;if(!re(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let a=u.toString();!t.doneSitemaps.has(a)&&!t.queuedSitemaps.has(a)&&(t.queuedSitemaps.add(a),t.sitemapQueue.push(a),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Sr(e){let t=D(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:G(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function vr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function br(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function me(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function ut(e,t){let r=[];await y.mkdir(t,{recursive:true});let s=await y.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),u=f.join(t,n.name);if(n.isDirectory()){r.push(...await ut(i,u));continue}await y.copyFile(i,u),r.push(u);}return r}async function Rr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,a=i.prefix;if(!u||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:a});continue}let o=Xt(u);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let c=o.resolvedPath,l=me(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:a});continue}let g;try{g=await y.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:c,prefixPath:a});continue}let d=f.resolve(e.outputDir,l);if(g.isDirectory()){n.push(...await ut(c,d)),s++,t.progress(`Copied static directory to export: ${c} -> /${l}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${l}`,copiedItems:s});continue}let p=f.basename(c),m=a.endsWith("/")?f.join(d,p):d;await y.mkdir(f.dirname(m),{recursive:true}),await y.copyFile(c,m),n.push(m),s++,t.progress(`Copied static file to export: ${c} -> /${me(a.endsWith("/")?`${a}${p}`:a)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${me(a.endsWith("/")?`${a}${p}`:a)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function _(e,t,r,s,n){let i=new Set,u=D(r).replace(/\\\//g,"/");for(let c of br(u)){let l=C(e,c,t,s,`${n}:css-url`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let l=C(e,c[1],t,s,`${n}:css-import`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let l=C(e,c[1],t,s,`${n}:xml-stylesheet`);l&&R(e,l)&&i.add(l.toString());}let a="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",o=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of u.matchAll(o)){let l=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(nt(l)){s.ignore(`${n}:serialized-url`,l,"looks like JavaScript/code fragment",t);continue}let g=C(e,l,t,s,`${n}:serialized-url`);g&&R(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Cr(e,t,r,s){let n=new Set,i=D(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let a=u[1],o=C(e,a,t,s,"page-link");o&&(st(e,o)||W(o)&&!te(o)&&!R(e,o)&&n.add(o.toString()));}return [...n]}async function Je(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),a=i.url()||n;await Pe(e,r.assetMap,a,u,r,s);let o=u.toString("utf8");for(let c of _(e,a,o,s,"sitemap"))A(e,r,c,s,a);for(let c of Sr(o)){let l=C(e,c.loc,a,s,"sitemap-loc");l&&(re(l)?be(e,r,l.toString(),s,a):R(e,l)?A(e,r,l.toString(),s,a):W(l)?(c.lastmod&&(r.sitemapLastmodByPage[l.toString()]=c.lastmod),ee(e,r,l.toString(),s,a)):s.reject("sitemap-loc",l.toString(),"not page/sitemap/asset-like",a));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function xr(e,t,r,s,n,i,u){let a=C(e,s,e.sourceOrigin,n,"asset-fetch");if(!a||!R(e,a))return;let o=a.toString();if(r.doneAssets.has(o))return;let c=i.enabled?i.previousManifest.assets[o]:void 0,l=(g,d={})=>{if(!c)return false;i.manifest.assets[o]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(o);for(let p of c.discoveredAssets||[])A(e,r,p,n,o);return n.warn(`Keeping previous incremental asset ${o} after fetch issue.`,{url:o,reason:g,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),fe(r,n),true};n.asset(`Fetching asset ${o}`,{url:o});try{let g=X(e,r.assetMap,o),d=g?await ot(g):null;if(g&&d)try{let S=await t.head(o,{timeout:6e4}),b=S.headers();if(S.ok()&&Pr(d,b)){let I=await we(g);if(I!==null){let x=S.url()||o,Q=await Pe(e,r.assetMap,x,I,r,n,b);Q.wroteFile&&(E(u,e,Q.outputPath),j(Q.outputPath)&&K(u,Q.outputPath)),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:g,lastModified:b["last-modified"]||"",contentLength:b["content-length"]||"",phase:"download-assets"}),fe(r,n);let Re=he(a,b),se=c?.discoveredAssets?[...c.discoveredAssets]:[];if(Re||c?.isText){let ct=I.toString("utf8"),Ce=_(e,x,ct,n,"asset:cached");(Ce.length>0||!c)&&(se=Ce);for(let dt of se)A(e,r,dt,n,x);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:Q.outputPath,isText:Re||!!c?.isText,discoveredAssets:se,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let p=await t.get(o,{timeout:6e4});if(!p.ok()){if(l(`http-${p.status()}`,{status:p.status()}))return;n.skip("asset",o,p.status());return}let m=await p.body(),P=p.url()||o,h=p.headers(),w=await Pe(e,r.assetMap,P,m,r,n,h);w.wroteFile&&(E(u,e,w.outputPath),j(w.outputPath)&&K(u,w.outputPath)),r.doneAssets.add(o),fe(r,n);let v=[];if(he(a,h)){let S=m.toString("utf8");v=_(e,P,S,n,`asset:${f.extname(a.pathname).toLowerCase()||h["content-type"]||"unknown"}`);for(let b of v)A(e,r,b,n,P);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:w.outputPath,isText:he(a,h),discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){if(l("fetch-error",{error:String(g)}))return;n.error(`Failed asset ${o}`,{url:o,error:String(g)});}}async function Ar(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),a=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await xr(e,t,r,o,s,n,i));}});await Promise.all(a);}async function kr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Mr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await kr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Tr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Lr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Se(t,r))}async function Ir(e,t,r,s,n=true){let i=new Set;if(n){let a=await t.evaluate(()=>{let o=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,g=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let m of c){let P=d.getAttribute(m);P&&o.add(P);}for(let m of g){let P=d.getAttribute(m);P&&P.split(",").forEach(h=>{let w=h.trim().split(/\s+/)[0];w&&o.add(w);});}let p=d.getAttribute("style");p&&l.add(p);}),{attrs:[...o],styles:[...l]}});for(let o of a.attrs)for(let c of vr(o)){let l=C(e,c,r,s,"dom-attr");l&&R(e,l)&&i.add(l.toString());}for(let o of a.styles)for(let c of _(e,r,o,s,"dom-style"))i.add(c);}let u=await t.content();for(let a of _(e,r,u,s,"page-html"))i.add(a);return [...i]}async function ze(e,t,r,s,n){let i=Z(e,t);await ue(i),await y.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Be(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ve(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Dr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Er(e,t,r,s,n,i,u){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Dr(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let g=new URL(l),d=Lr(e,g.pathname);if(!W(g)||R(e,g)||te(g)){if(R(e,g)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),A(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let p=await sr(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let h=i.manifest.pages[l];if(p.action==="reuse"&&h){for(let w of h.discoveredAssets)A(e,t,w,r,l);for(let w of h.discoveredPages)ee(e,t,w,r,l);h.lastSeenRunId=i.runId,h.sitemapLastmod=G(t.sitemapLastmodByPage[l])??h.sitemapLastmod,p.changeToken?.supported&&(h.changeToken=p.changeToken.token,h.tokenSource=p.changeToken.tokenSource??h.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Be(t)&&Ve(t,r);continue}}let m=await(d?c.newPage():o.newPage()),P=!1;await m.route("**/*",async h=>{let w=h.request();if(w.isNavigationRequest()&&w.frame()===m.mainFrame())try{let v=new URL(w.url()),S=new URL(l);if(v.hash="",S.hash="",v.href===S.href){if(P){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:w.url()}),await h.abort("aborted");return}P=!0;}}catch{}await h.continue();}),m.on("response",h=>{try{let w=C(e,h.url(),e.sourceOrigin,void 0,"network-response");w&&R(e,w)&&A(e,t,w.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let h=null,w=null;try{if(h=await m.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),h&&h.ok())try{w=await h.text();}catch{w=null;}}catch(x){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(x)});}if(h&&!h.ok()){r.skip("page",l,h.status()),await m.close();continue}d?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Tr(e,m)):await Mr(e,m),t.stats.pagesRendered++,Be(t)&&Ve(t,r);let v=await Ir(e,m,l,r,!d);for(let x of v)A(e,t,x,r,l);let S=w??await m.content(),b=s!=="single-url"?Cr(e,l,S,r):[];if(s!=="single-url")for(let x of b)ee(e,t,x,r,l);await ze(e,l,S,t,r);let I=Z(e,l);E(u,e,I),K(u,I),i.trackManifest&&(i.manifest.pages[l]={url:l,outputPath:I,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:G(t.sitemapLastmodByPage[l])??null,discoveredPages:b,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(h){try{let w=await m.content();if(w&&w.trim()){await ze(e,l,w,t,r);let v=Z(e,l);E(u,e,v),K(u,v),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(h)});}}catch(w){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(w)});}r.error(`Failed page ${l}`,{url:l,error:String(h)});}finally{await m.close();}}}finally{await c.close(),await o.close(),await a.close();}}async function Fr(e){try{let t=await y.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ke(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function jr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Or(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function lt(e){try{let t=await y.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ge(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function $r(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ge(s)-Ge(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Xe(e){let t=await lt(f.join(e,"job.json"));for(let r of $r(e,t)){let s=await Fr(r);if(s.length>0)return s}return []}async function Ur(e){let t=f.join(e.logDir,"archive"),r;try{r=await y.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await y.stat(u).catch(()=>null))?.isDirectory())continue;let o=await lt(f.join(u,"job.json"));if(!jr(o))continue;let c=Or(o);c>=n&&(n=c,s=u);}return s}async function Qr(e){let t=await Ur(e),r=t!==""?await Xe(t):await Xe(e.logDir),s=new Set;for(let n of r)if(Ke(n.error)||Ke(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await y.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function qr(e){try{let t=await y.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function _r(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await y.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await y.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ye(e,t,r,s,n){let i=C(e,r,e.sourceOrigin,s,"manual-url");i&&(re(i)?be(e,t,i.toString(),s,n):R(e,i)?A(e,t,i.toString(),s,n):W(i)?ee(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Wr(){let e=await ae(),t=Vt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Qr(e)),t.mode==="single-url"&&(r=await Nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.mode==="full"&&!t.resumeRewrite,i=t.preserveOutput?await qr(e):{},u=s?await Zt(e):tt(),a={enabled:s,trackManifest:n,manifest:qe(u),previousManifest:qe(u),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await _r(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...i},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},c=Yt();t.preserveOutput||await y.rm(e.outputDir,{recursive:true,force:true}),await y.mkdir(e.outputDir,{recursive:true});let l=new N(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let P=await chromium.launch({headless:true}),h=await P.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),w=h.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(S=>be(e,o,S,l)),await Je(e,w,o,l),e.seedPaths.forEach(S=>Ye(e,o,S,l,"seed-path"))):(r.forEach(S=>Ye(e,o,S,l,"cli")),await Je(e,w,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let v=Array.from({length:Math.max(1,e.concurrency)},()=>Er(e,o,l,t.mode,w,a,c));if(await Promise.all(v),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Ar(e,w,o,l,a,c),l.endMark("download-assets",{assets:o.doneAssets.size}),a.enabled&&e.maxPages===0&&await nr(e,o,a,l,c),t.mode==="full"){let S=await Rr(e,l);for(let b of S)E(c,e,b);}}finally{await h.close(),await P.close();}}a.trackManifest&&(a.manifest.updatedAt=new Date().toISOString(),await er(e,a.manifest)),await y.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(i,null,2)!==JSON.stringify(o.assetMap,null,2)&&E(c,e,f.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?cr(e,o):a.enabled&&!t.resumeRewrite?ir(e,a,c,i,o.assetMap):void 0;l.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";l.progress(d,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await He(d);let p=Date.now(),m=await Oe(e,o.assetMap,{files:g,previousAssetMap:i,onProgress:async({index:P,totalFiles:h,changedTextFiles:w,file:v,changed:S})=>{S&&E(c,e,f.join(e.outputDir,v)),l.checkpoint(`Rewriting text file ${P}/${h}`,{phase:"rewrite-text",index:P,totalFiles:h,changedTextFiles:w,file:v});let b=Date.now();if(!(P===1||P===h||b-p>=5e3))return;p=b;let x=`Rewriting text files: ${P}/${h}`;l.progress(x,{phase:"rewrite-text",index:P,totalFiles:h,changedTextFiles:w,file:v}),await He(x);}});l.endMark("rewrite-text",{changedTextFiles:m}),await ke(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(a.enabled&&!t.resumeRewrite),changedFiles:a.enabled&&!t.resumeRewrite?[...c.changedFiles].sort():[],deletedFiles:a.enabled&&!t.resumeRewrite?[...c.deletedFiles].sort():[],rewriteTargets:a.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(P=>H(e.outputDir,f.isAbsolute(P)?P:f.join(e.outputDir,P))).filter(P=>P!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${m} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:m}),await l.flush();}Wr().catch(async e=>{console.error(e);try{let r=(await ae().catch(()=>null))?.logDir??"logs",s=new N(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.15",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|