@smart-cloud/publisher-exporter 1.1.17 → 1.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +5 -5
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import S from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import Mt from'http';import Lt from'https';import Nt from'crypto';import _t from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Rt=".deploy-plan.json",Ct="deploy-plan.json";function xt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function De(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function pe(e){return [...new Set(e.map(De).filter(t=>t!==null))]}function kt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:pe(t.changedFiles||[]),deletedFiles:pe(t.deletedFiles||[]),rewriteTargets:pe(t.rewriteTargets||[])}}function Tt(e){let t=h.join(h.resolve(e.outputDir),Rt),r=xt();return r?[h.join(r,Ct),t]:[t]}function te(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return De(n)}async function Fe(e,t){let r=kt(e,t);for(let s of Tt(e))await S.mkdir(h.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function je(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function It(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function ge(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ot(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=je(i.targetOrigin);a&&(u.targetOrigin=a);}let l=ge(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function $e(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Dt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Ue(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Dt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Ft(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function fe(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Ft(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(u).length>0&&(l.siteSettings=u);let p=$e(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Et(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function qe(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function jt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function Qe(e,t,r=5){let s=new URL(qe(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Mt:Lt,u=jt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},c=>{let p=[];c.on("data",d=>{p.push(Buffer.isBuffer(d)?d:Buffer.from(d));}),c.on("error",a),c.on("end",()=>{let d=c.statusCode??0,g=c.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(d)){l(Qe(new URL(g,s),{method:d===303?"GET":n},r-1));return}let f=new Headers;for(let[m,P]of Object.entries(c.headers))Array.isArray(P)?f.set(m,P.join(", ")):typeof P=="string"&&f.set(m,P);l(new Response(Buffer.concat(p),{status:d,statusText:c.statusMessage??"",headers:f}));});});o.on("error",a),o.end();})}function $t(e){let t=fe(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Et(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Ut(e){if(!$t(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>qe(i).startsWith(t)?Qe(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function qt(e,t){let r=Ot(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=$e(t?.subscriptionType),u=fe(e.wpsuite);return {...e,scheduler:Ue(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function he(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Qt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ee(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,he(s,r))}async function me(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=je(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=It(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=ge(s.extraReplacements),s.postCrawlCopyMap=ge(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=fe(s.wpsuite),s.scheduler=Ue(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=qt(s,await Ut(s)),i=Qt(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=Ee(u,n.outputDir,"export"),n.logDir=Ee(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=he(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=he(n.logDir,"logs"))),n}function se(e,t=10){return Nt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function we(e){await S.mkdir(h.dirname(e),{recursive:true});}function _(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function U(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function B(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function re(e){return e.replace(/\//g,"\\/")}function Ne(e){return e.replace(/\//g,"\\\\/")}function ne(e,t,r){if(!t)return;e[t]=r;let s=re(t),n=re(r);e[s]=n;let i=Ne(t),u=Ne(r);e[i]=u;let l=B(t),a=B(r);e[l]=a;let o=B(s),c=B(n);e[o]=c;let p=B(i),d=B(u);e[p]=d;}function Pe(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function We(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function _e(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var zt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Jt=new Set([".html",".htm"]),He="WPSuite.io Static Publisher",Gt=He.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Vt(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Jt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Kt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Gt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${He}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import S from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import Mt from'http';import Lt from'https';import Nt from'crypto';import _t from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Rt=".deploy-plan.json",Ct="deploy-plan.json";function kt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ee(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ge(e){return [...new Set(e.map(Ee).filter(t=>t!==null))]}function xt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ge(t.changedFiles||[]),deletedFiles:ge(t.deletedFiles||[]),rewriteTargets:ge(t.rewriteTargets||[])}}function Tt(e){let t=h.join(h.resolve(e.outputDir),Rt),r=kt();return r?[h.join(r,Ct),t]:[t]}function se(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return Ee(n)}async function je(e,t){let r=xt(e,t);for(let s of Tt(e))await S.mkdir(h.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function It(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function he(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ot(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=Ue(i.targetOrigin);a&&(u.targetOrigin=a);}let l=he(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function qe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Dt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Qe(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Dt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Ft(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function me(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Ft(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(u).length>0&&(l.siteSettings=u);let p=qe(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Et(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Ne(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function jt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function We(e,t,r=5){let s=new URL(Ne(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Mt:Lt,u=jt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},c=>{let p=[];c.on("data",d=>{p.push(Buffer.isBuffer(d)?d:Buffer.from(d));}),c.on("error",a),c.on("end",()=>{let d=c.statusCode??0,g=c.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(d)){l(We(new URL(g,s),{method:d===303?"GET":n},r-1));return}let f=new Headers;for(let[m,P]of Object.entries(c.headers))Array.isArray(P)?f.set(m,P.join(", ")):typeof P=="string"&&f.set(m,P);l(new Response(Buffer.concat(p),{status:d,statusText:c.statusMessage??"",headers:f}));});});o.on("error",a),o.end();})}function $t(e){let t=me(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Et(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Ut(e){if(!$t(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>Ne(i).startsWith(t)?We(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function qt(e,t){let r=Ot(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=qe(t?.subscriptionType),u=me(e.wpsuite);return {...e,scheduler:Qe(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function fe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Qt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function $e(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,fe(s,r))}async function we(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Ue(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=It(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=he(s.extraReplacements),s.postCrawlCopyMap=he(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=me(s.wpsuite),s.scheduler=Qe(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=qt(s,await Ut(s)),i=Qt(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=$e(u,n.outputDir,"export"),n.logDir=$e(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=fe(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=fe(n.logDir,"logs"))),n}function ie(e,t=10){return Nt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Pe(e){await S.mkdir(h.dirname(e),{recursive:true});}function _(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function q(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function B(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function ne(e){return e.replace(/\//g,"\\/")}function _e(e){return e.replace(/\//g,"\\\\/")}function oe(e,t,r){if(!t)return;e[t]=r;let s=ne(t),n=ne(r);e[s]=n;let i=_e(t),u=_e(r);e[i]=u;let l=B(t),a=B(r);e[l]=a;let o=B(s),c=B(n);e[o]=c;let p=B(i),d=B(u);e[p]=d;}function Se(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Be(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function He(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var zt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Jt=new Set([".html",".htm"]),Je="WPSuite.io Static Publisher",Gt=Je.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Vt(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Jt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Kt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Gt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Je}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Yt(e){let t=
|
|
5
|
-
`,"utf8");}function ar(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function ur(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:ar(e,r),level:e,message:t,details:r??{}}}var
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>or(t,r));}updateCurrentEvent(t,r,s){let n=ur(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Yt(e){let t=q(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Xt(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,ne(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[l,a]of r)u=u.split(l).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,l,a)=>`${u}${s(l)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,l,a,o)=>Yt(a)?`${u}${s(a)}${o}`:i),n}function Zt(e,t){if(!t)return null;let r=h.dirname(h.resolve(t)),s=h.resolve(e.outputDir),n=h.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function er(e,t,r){let s=Zt(t,r);if(!s)return e;let n=e;for(let i of zt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(l,`${s}${i}`).replace(a,`${s}${i}`);let o=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),p=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),f=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${c}${o}`).replace(g,`${c}${o}`).replace(f,`${c}${o}`);}return n}function ye(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:Be(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${He(e.outputDir,r,n)}${i}`}function be(e,t,r,s,n){oe(e,r,ye(t,s,n));}function tr(e,t,r,s,n){if(!r||r===s)return;oe(e,r,s);let i=ye(t,r,n),u=ye(t,s,n);oe(e,i,u);}function ve(e,t,r,s,n={}){let i=e,u={};be(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,o]of Object.entries(t.extraReplacements))be(u,t,a,o,s);for(let[a,o]of Object.entries(r))be(u,t,a,o,s);for(let[a,o]of Object.entries(n)){let c=r[a];!c||c===o||tr(u,t,o,c,s);}let l=Object.entries(u).sort((a,o)=>o[0].length-a[0].length);for(let[a,o]of l)i=i.split(a).join(o);return t.urlRewriteMode!=="absolute"&&(i=er(i,t,s)),i=Xt(i,l.map(([,a])=>a)),Vt(t,i,s)&&(i=Kt(i)),i}async function rr(e,t,r,s){let n=h.join(e.outputDir,s),i=h.extname(n).toLowerCase(),u;try{u=await S.readFile(n,"utf8");}catch(o){if(o.code!=="ENOENT")throw o;return {changed:false}}let a=ve(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await S.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function sr(e,t){return await new Promise((r,s)=>{let n=a=>{if(l(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{l(),s(a);},u=a=>{l(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function Ge(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>h.resolve(d)).filter(d=>d.startsWith(h.resolve(e.outputDir))).map(d=>h.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await _t(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&_(d)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(u,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:l});let a=0,o=0,c=null;if(l===1){for(let d of n){let g=await rr(e,t,r.previousAssetMap??{},d);g.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:d,changed:g.changed});}return i}let p=Array.from({length:l},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let g=a;if(g>=n.length)return;a+=1;let f=n[g],m=await sr(d,f);m.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:f,changed:m.changed});}}catch(g){throw c=g instanceof Error?g:new Error(String(g)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var Q={error:0,warn:1,info:2,debug:3},Ve=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",ae=Ve?h.join(Ve,"current-progress.json"):"",Ke=Promise.resolve();function Ye(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function nr(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function ir(e,t,r){ae&&(Ke=Ke.then(async()=>{let s=new Date,n=s.toISOString();await S.mkdir(h.dirname(ae),{recursive:true});let i=await S.readFile(ae,"utf8").then(m=>JSON.parse(m)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},l=Ye(e),a=i?.currentStep||Ye(i?.source||""),o=i?.startedAt||n,c={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==l&&i?.stepStartedAt){let m=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+m,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(o).getTime())/1e3)),f={checkedAt:n,source:e,message:t,details:nr(u,r??{}),startedAt:o,currentStep:l,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:c};await S.writeFile(ae,JSON.stringify(f,null,2),"utf8");}).catch(()=>{}));}async function or(e,t){await S.mkdir(h.dirname(e),{recursive:true}),await S.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function ar(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function ur(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:ar(e,r),level:e,message:t,details:r??{}}}var K=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return h.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return h.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return h.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return h.join(this.logDir,"ignored.jsonl")}get skippedPath(){return h.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return h.join(this.logDir,"errors.jsonl")}get timingsPath(){return h.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await S.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>or(t,r));}updateCurrentEvent(t,r,s){let n=ur(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(Q[this.level]>=Q.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}debug(t,r){Q[this.level]<Q.debug||this.push("debug",t,r);}progress(t,r){ir("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),Q[this.level]>=Q.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),Q[this.level]>=Q.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var pr=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],gr=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),hr=new Set(["",".html",".htm"]),fr=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),mr=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function wr(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let l=e[u];if(l==="--retry-timeouts")r="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(l.startsWith("--crawl-mode="))s=l.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(l==="--incremental")s="incremental",n=true;else if(l==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(l.startsWith("--url="))r="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Ae(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Pr(){let e=Ae();return e?h.resolve(e,".."):""}function Sr(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?h.resolve(e):""}function br(e){let t=e.trim(),r=[{alias:"@storage-root",root:Pr(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Ae(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Sr(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?h.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:h.resolve(t)}}function ct(e){let t=Ae();return t?h.join(t,"crawl-manifest.json"):h.join(e.outputDir,".crawl-manifest.json")}function dt(){return {schemaVersion:3,updatedAt:"",pages:{},assets:{}}}function Xe(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function yr(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true||!t}}function vr(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function N(e,t,r){let s=se(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Ze(e,t,r){let s=se(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ce(e,t){e.rewriteTargets.add(h.resolve(t));}async function Rr(e){try{let t=await S.readFile(ct(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3)&&r.assets&&typeof r.assets=="object")return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Xe(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,yr(n)]))};if(r.schemaVersion===1)return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Xe(n)])),assets:{}}}}catch{}return dt()}function xe(e){return JSON.parse(JSON.stringify(e))}async function Cr(e,t){let r=ct(e);await S.mkdir(h.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function J(e,t){if(!t.trackManifest)return;let r=async()=>{let s=xe(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await Cr(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function Y(e){if(!e)return null;let t=e.trim();return t||null}function kr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function xr(e,t,r,s){let n=[...new Set(r.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!l.ok())return s.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:u,status:l.status()}),new Map;let a=await l.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let o=new Map;for(let c of a.items)!c||typeof c.url!="string"||o.set(c.url,c);return o}catch(l){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(l)}),new Map}}async function Re(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await xr(e,t,[s],n)).get(s)??null;return Le(e,s)&&(!u||u.supported!==true||!u.token)&&(u=await Tr(e,t,s,n)),r.changeTokenCache.set(s,u),u}async function Tr(e,t,r,s){try{let n=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(n.status()!==404)return s.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:n.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let i=await n.text().catch(()=>""),u=ie(`${r}
|
|
7
7
|
404
|
|
8
|
-
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Tr(e,t,r,s,n,i){let u=Ae(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await ve(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await ve(e,t,s,n,i)};let a=await ve(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await Ar(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=le(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Ar(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Mr(e,t,r,s,n){let i=et(r.previousManifest),u=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ye(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(f)});}delete r.manifest.pages[d],u++;}let l=et(r.manifest,r.runId),a=new Set,o=new Set;for(let d of l){for(let m of K(e,d))a.add(m);let g=r.manifest.assets[d]?.outputPath;g&&o.add(g);let f=ce(e,t.assetMap,d);f&&o.add(f);}for(let d of Object.keys(r.manifest.assets))l.has(d)||delete r.manifest.assets[d];let c=0,p=0;for(let d of i){let g=new Set,f=r.previousManifest.assets[d]?.outputPath;f&&g.add(f);let m=ce(e,t.assetMap,d);if(!l.has(d)&&m)g.add(m);else if(!l.has(d)){let P=Me(e,d);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){Ye(n,e,P);try{await S.unlink(P),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:P,error:String(w)});}}for(let P of K(e,d))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||c>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:p,phase:"incremental-cleanup"});}function et(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function tt(e,t,r){for(let s of t)for(let n of K(e,s))if(r.has(n))return true;return false}function Lr(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=Cr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))tt(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&tt(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of K(e,l))if(u.has(o)){Q(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function Ir(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function K(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Or(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function pt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function Dr(e,t,r){for(let s of Or(e,r)){let n=t[s];if(!n)continue;let i=pt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Fr(e,t,r){for(let s of K(e,r)){let n=t[s];if(!n)continue;let i=pt(e,n);if(i)return i}return null}function ce(e,t,r){let s=Fr(e,t,r);return s||Me(e,r).originalFilePath}function Er(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function rt(e){let t=Er();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function jr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(z(e,s)));for(let s of t.doneAssets){let n=ce(e,t.assetMap,s);!n||!_(n)||r.add(h.resolve(n));}return [...r]}var de="re:",oe=new Map;function $r(e){if(!e.startsWith(de))return null;let t=oe.get(e);if(t!==void 0)return t||null;let r=e.slice(de.length).trim();if(!r)return oe.set(e,false),null;try{let s=new RegExp(r);return oe.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),oe.set(e,false),null}}function Te(e,t){let r=$r(t);return r?r.test(e):t.startsWith(de)?false:e.startsWith(t)}function Ur(e,t){return t.find(r=>Te(e,r))}function gt(e,t){let r=Ur(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(de)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function qr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Qr(e,t){return qr(e).has(t.hostname)}function Nr(e){let t=U(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function ht(e){let t=U(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function Y(e){return mr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Nr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(ht(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Qr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=gt(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function ft(e){return gr.has(h.extname(e.pathname).toLowerCase())}function J(e){return Y(e)?false:hr.has(h.extname(e.pathname).toLowerCase())}function ee(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return ee(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:ft(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>Te(t.pathname,r)):false}function z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function Wr(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Ae(e,t){let r=Wr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Me(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Pe(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${se(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Pe(e.targetOrigin,p)}}function _r(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function mt(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function wt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Br(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Pt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Hr(e,t){let r=Pt(t),s=Br(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function zr(e,t){if(!t)return;let r=Pt(t);if(r===null)return;let s=await wt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function st(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return fr.has(r)||pr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Re(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Jr(e){return se(e.toString("base64"),40)}function Gr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function Vr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function xe(e,t,r,s,n,i,u){let l=Me(e,r),a=Jr(s),o=l.originalFilePath,c=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&Gr(n,l.originalFilePath,a)){let f=Dr(e,t,r);f?(o=f.outputPath,c=f.publicUrl):(o=l.hashedFilePath,c=l.hashedPublicUrl);}_r(e,t,r,c,l.originalPublicUrl);let p=_(o)?Buffer.from(ye(s.toString("utf8"),e,t,o),"utf8"):s,d=await mt(o),g=false;return (d===null||!d.equals(p))&&(await we(o),await S.writeFile(o,p),g=true),Vr(n,l.originalFilePath,a),await zr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function X(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let d=new URL(p);return ee(d)?{discoveredFrom:p,discoveredFromType:"sitemap"}:C(e,d)?{discoveredFrom:p,discoveredFromType:"asset"}:J(d)&&!Y(d)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!J(l)||ft(l)){if(C(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),L(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),Y(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:c}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:c,outputPath:z(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function L(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!C(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Le(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!ee(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Kr(e){let t=U(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:le(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Yr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Xr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Ce(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function St(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await St(i,u));continue}await S.copyFile(i,u),r.push(u);}return r}async function Zr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,l=i.prefix;if(!u||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:l});continue}let a=br(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let o=a.resolvedPath,c=Ce(l);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:l});continue}let p;try{p=await S.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:o,prefixPath:l});continue}let d=h.resolve(e.outputDir,c);if(p.isDirectory()){n.push(...await St(o,d)),s++,t.progress(`Copied static directory to export: ${o} -> /${c}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${c}`,copiedItems:s});continue}let g=h.basename(o),f=l.endsWith("/")?h.join(d,g):d;await S.mkdir(h.dirname(f),{recursive:true}),await S.copyFile(o,f),n.push(f),s++,t.progress(`Copied static file to export: ${o} -> /${Ce(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${Ce(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function Z(e,t,r,s,n){let i=new Set,u=U(r).replace(/\\\//g,"/");for(let o of Xr(u)){let c=T(e,o,t,s,`${n}:css-url`);c&&C(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,o[1],t,s,`${n}:css-import`);c&&C(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,o[1],t,s,`${n}:xml-stylesheet`);c&&C(e,c)&&i.add(c.toString());}let l="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let o of u.matchAll(a)){let c=o[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${o[0]}`:o[0];if(ht(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=T(e,c,t,s,`${n}:serialized-url`);p&&C(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function es(e,t,r,s){let n=new Set,i=U(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(gt(e,a)||J(a)&&!Y(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function nt(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),l=i.url()||n;await xe(e,r.assetMap,l,u,r,s);let a=u.toString("utf8");for(let o of Z(e,l,a,s,"sitemap"))L(e,r,o,s,l);for(let o of Kr(a)){let c=T(e,o.loc,l,s,"sitemap-loc");c&&(ee(c)?Le(e,r,c.toString(),s,l):C(e,c)?L(e,r,c.toString(),s,l):J(c)?(o.lastmod&&(r.sitemapLastmodByPage[c.toString()]=o.lastmod),X(e,r,c.toString(),s,l)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",l));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function ts(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,c=(p,d={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let g of o.discoveredAssets||[])L(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),Re(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=ce(e,r.assetMap,a),d=p?await wt(p):null;if(p&&d)try{let x=await t.head(a,{timeout:6e4}),M=x.headers();if(x.ok()&&Hr(d,M)){let E=await mt(p);if(E!==null){let N=x.url()||a,A=await xe(e,r.assetMap,N,E,r,n,M);A.wroteFile&&(Q(u,e,A.outputPath),_(A.outputPath)&&ue(u,A.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:M["last-modified"]||"",contentLength:M["content-length"]||"",phase:"download-assets"}),Re(r,n);let b=st(l,M),R=o?.discoveredAssets?[...o.discoveredAssets]:[];if(b||o?.isText){let W=E.toString("utf8"),G=Z(e,N,W,n,"asset:cached");(G.length>0||!o)&&(R=G);for(let vt of R)L(e,r,vt,n,N);}if(i.trackManifest){let W=b||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:A.outputPath,isText:W,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!W};}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(c(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),m=g.url()||a,P=g.headers(),w=await xe(e,r.assetMap,m,f,r,n,P);w.wroteFile&&(Q(u,e,w.outputPath),_(w.outputPath)&&ue(u,w.outputPath)),r.doneAssets.add(a),Re(r,n);let y=st(l,P),v=[];if(y){let x=f.toString("utf8");v=Z(e,m,x,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let M of v)L(e,r,M,n,m);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:y,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!y});}catch(p){if(c("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function rs(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await ts(e,t,r,a,s,n,i));}});await Promise.all(l);}function ss(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function F(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ae(e){return Math.max(15e3,Math.min(6e4,F(e)))}function ns(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function Ie(e){return e.stats.pagesCompleted}function bt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function is(e,t,r){let s=bt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function k(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${ss(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function os(e,t){let r=ns(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let c=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(c,p);i>=d+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function as(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await os(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function us(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function ls(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Te(t,r))}async function cs(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of o){let m=d.getAttribute(f);m&&a.add(m);}for(let f of p){let m=d.getAttribute(f);m&&m.split(",").forEach(P=>{let w=P.trim().split(/\s+/)[0];w&&a.add(w);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...a],styles:[...c]}});for(let a of l.attrs)for(let o of Yr(a)){let c=T(e,o,r,s,"dom-attr");c&&C(e,c)&&i.add(c.toString());}for(let a of l.styles)for(let o of Z(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of Z(e,r,u,s,"page-html"))i.add(l);return [...i]}async function it(e,t,r,s,n){let i=z(e,t);await k(`save page ${t}`,F(e),async()=>{await we(i),await S.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,Ie(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ds(e){let t=Ie(e);return t%5===0||t===1||t===e.stats.pagesQueued}function ps(e,t){let r=Ie(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function gs(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function hs(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await gs(s),s}async function fs(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=bt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let c=null,p=null;try{let d=new URL(o),g=Ae(e,o),f=ls(e,d.pathname);if(!J(d)||C(e,d)||Y(d)){if(C(e,d)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),L(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let m=await Tr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let b=i.manifest.pages[o];if(m.action==="reuse"&&b){for(let R of b.discoveredAssets)L(e,t,R,r,o);for(let R of b.discoveredPages)X(e,t,R,r,o);b.lastSeenRunId=i.runId,b.sitemapLastmod=le(t.sitemapLastmodByPage[o])??b.sitemapLastmod,m.changeToken?.supported&&(b.changeToken=m.changeToken.token,b.tokenSource=m.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}c=await k(`create page context ${o}`,F(e),()=>hs(l,e,f)),p=await k(`create page ${o}`,F(e),()=>c.newPage());let P=!1,w=new Set;await p.route("**/*",async b=>{let R=b.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let W=new URL(R.url()),G=new URL(o);if(W.hash="",G.hash="",W.href===G.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await b.abort("aborted");return}P=!0;}}catch{}await b.continue();}),p.on("response",b=>{try{let R=T(e,b.url(),e.sourceOrigin,void 0,"network-response");R&&C(e,R)&&(w.add(R.toString()),L(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await k(`read response body ${o}`,F(e),()=>y.text());}catch{v=null;}}catch(b){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(b)});}if(g){let b=y?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:b,outputPath:z(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status(),t.pageDiscoverySources.get(o)),await p.close();continue}f?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await k(`wait for rendered page ${o}`,F(e),()=>us(e,p))):await k(`wait for rendered page ${o}`,F(e),()=>as(e,p)),t.stats.pagesRendered++;let x=await k(`extract rendered assets ${o}`,F(e),()=>cs(e,p,o,r,!f));for(let b of x)L(e,t,b,r,o);let M=[...new Set([...x,...w])],E=v??await k(`capture rendered html ${o}`,F(e),()=>p.content()),N=s!=="single-url"&&!g?es(e,o,E,r):[];if(s!=="single-url"&&!g)for(let b of N)X(e,t,b,r,o);await it(e,o,E,t,r);let A=z(e,o);Q(u,e,A),ue(u,A),i.trackManifest&&(i.manifest.pages[o]={url:o,outputPath:A,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:le(t.sitemapLastmodByPage[o])??null,discoveredPages:N,discoveredAssets:M,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1});}catch(d){try{let g=p?await k(`capture partial html ${o}`,F(e),()=>p.content()):"";if(g&&g.trim()){await it(e,o,g,t,r);let f=z(e,o);Q(u,e,f),ue(u,f),r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(d)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(d)});}finally{t.stats.pagesCompleted++,ds(t)&&ps(t,r),p&&await k(`close page ${o}`,ae(e),()=>p.close()).catch(d=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(d)});}),c&&await k(`close page context ${o}`,ae(e),()=>c.close()).catch(d=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(d)});});}}}finally{await k("close worker browser",ae(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),is(l,r,"browser.close timeout")));}}async function ms(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function ot(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function ws(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ps(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function yt(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function at(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Ss(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>at(s)-at(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function ut(e){let t=await yt(h.join(e,"job.json"));for(let r of Ss(e,t)){let s=await ms(r);if(s.length>0)return s}return []}async function bs(e){let t=h.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let a=await yt(h.join(u,"job.json"));if(!ws(a))continue;let o=Ps(a);o>=n&&(n=o,s=u);}return s}async function ys(e){let t=await bs(e),r=t!==""?await ut(t):await ut(e.logDir),s=new Set;for(let n of r)if(ot(n.error)||ot(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function vs(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Rs(e){try{let t=await S.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Cs(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function lt(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(ee(i)?Le(e,t,i.toString(),s,n):C(e,i)?L(e,t,i.toString(),s,n):J(i)?X(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function xs(){let e=await me(),t=wr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await ys(e)),t.mode==="single-url"&&(r=await vs(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await Rs(e):{},a=n?await Rr(e):dt(),o={enabled:n,trackManifest:i,manifest:Xe(a),previousManifest:Xe(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Cs(e.logDir);let c={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=vr();u||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let d=new V(e.logDir,e.logLevel);if(s&&!n&&d.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)d.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(d.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Le(e,c,v,d)),await nt(e,w,c,d),e.seedPaths.forEach(v=>lt(e,c,v,d,"seed-path")),e.generated404RequestPath&&X(e,c,e.generated404RequestPath,d,"generated-404-request-path")):(r.forEach(v=>lt(e,c,v,d,"cli")),await nt(e,w,c,d)),d.endMark("discovery",{pages:c.pageQueue.length,assets:c.assetQueue.length,sitemaps:c.sitemapQueue.length}),d.summary(`Queued ${c.pageQueue.length} pages, ${c.assetQueue.length} assets, ${c.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:c.pageQueue.length,queuedAssets:c.assetQueue.length,queuedSitemaps:c.sitemapQueue.length}),t.mode==="retry-timeouts"&&c.pageQueue.length===0&&c.assetQueue.length===0&&c.sitemapQueue.length===0){d.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await d.flush();return}d.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>fs(e,c,d,t.mode,w,o,p));if(await Promise.all(y),d.endMark("render-pages",{pages:c.stats.pagesCompleted}),d.mark("download-assets"),d.progress("Asset download phase started.",{phase:"download-assets",doneAssets:c.doneAssets.size,assetsQueued:c.stats.assetsQueued,assetQueue:c.assetQueue.length}),await rs(e,w,c,d,o,p),d.endMark("download-assets",{assets:c.doneAssets.size}),o.enabled&&e.maxPages===0&&await Mr(e,c,o,d,p),t.mode==="full"){let v=await Zr(e,d);for(let x of v)Q(p,e,x);}}finally{await k("dispose crawl request context",ae(e),()=>w.dispose()).catch(y=>{d.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&(o.manifest.updatedAt=new Date().toISOString(),await Ze(e,o.manifest)),await S.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(c.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(c.assetMap,null,2)&&Q(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?jr(e,c):o.enabled&&!t.resumeRewrite?Lr(e,o,p,l,c.assetMap):void 0;d.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";d.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await rt(f);let m=Date.now(),P=await ze(e,c.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:x,changed:M})=>{M&&Q(p,e,h.join(e.outputDir,x)),d.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x});let E=Date.now();if(!(w===1||w===y||E-m>=5e3))return;m=E;let A=`Rewriting text files: ${w}/${y}`;d.progress(A,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x}),await rt(A);}});o.trackManifest&&(Ir(o,g),o.manifest.updatedAt=new Date().toISOString(),await Ze(e,o.manifest)),d.endMark("rewrite-text",{changedTextFiles:P}),await Fe(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>te(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),d.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${c.stats.pagesRendered} pages, processed ${c.doneSitemaps.size} sitemaps, downloaded ${c.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...c.stats,pagesRendered:c.stats.pagesRendered,donePages:c.stats.pagesCompleted,claimedPages:c.donePages.size,doneSitemaps:c.doneSitemaps.size,doneAssets:c.doneAssets.size,changedTextFiles:P}),await d.flush();}xs().catch(async e=>{console.error(e);try{let r=(await me().catch(()=>null))?.logDir??"logs",s=new V(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
8
|
+
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Ar(e,t,r,s,n,i){let u=Le(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Re(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Re(e,t,s,n,i)};let a=await Re(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await Mr(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=Y(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Mr(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Lr(e,t,r,s,n){let i=et(r.previousManifest),u=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ze(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(f)});}delete r.manifest.pages[d],u++;}let l=et(r.manifest,r.runId),a=new Set,o=new Set;for(let d of l){for(let m of X(e,d))a.add(m);let g=r.manifest.assets[d]?.outputPath;g&&o.add(g);let f=de(e,t.assetMap,d);f&&o.add(f);}for(let d of Object.keys(r.manifest.assets))l.has(d)||delete r.manifest.assets[d];let c=0,p=0;for(let d of i){let g=new Set,f=r.previousManifest.assets[d]?.outputPath;f&&g.add(f);let m=de(e,t.assetMap,d);if(!l.has(d)&&m)g.add(m);else if(!l.has(d)){let P=Ie(e,d);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){Ze(n,e,P);try{await S.unlink(P),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:P,error:String(w)});}}for(let P of X(e,d))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||c>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:p,phase:"incremental-cleanup"});}function et(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function tt(e,t,r){for(let s of t)for(let n of X(e,s))if(r.has(n))return true;return false}function Ir(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=kr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))tt(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&tt(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of X(e,l))if(u.has(o)){N(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function Or(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function X(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Dr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function pt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function Fr(e,t,r){for(let s of Dr(e,r)){let n=t[s];if(!n)continue;let i=pt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Er(e,t,r){for(let s of X(e,r)){let n=t[s];if(!n)continue;let i=pt(e,n);if(i)return i}return null}function de(e,t,r){let s=Er(e,t,r);return s||Ie(e,r).originalFilePath}function jr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function rt(e){let t=jr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function $r(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(z(e,s)));for(let s of t.doneAssets){let n=de(e,t.assetMap,s);!n||!_(n)||r.add(h.resolve(n));}return [...r]}var pe="re:",ue=new Map;function Ur(e){if(!e.startsWith(pe))return null;let t=ue.get(e);if(t!==void 0)return t||null;let r=e.slice(pe.length).trim();if(!r)return ue.set(e,false),null;try{let s=new RegExp(r);return ue.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),ue.set(e,false),null}}function Me(e,t){let r=Ur(t);return r?r.test(e):t.startsWith(pe)?false:e.startsWith(t)}function qr(e,t){return t.find(r=>Me(e,r))}function gt(e,t){let r=qr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(pe)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Qr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Nr(e,t){return Qr(e).has(t.hostname)}function Wr(e){let t=q(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function ht(e){let t=q(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function Z(e){return mr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Wr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(ht(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Nr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=gt(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function ft(e){return gr.has(h.extname(e.pathname).toLowerCase())}function G(e){return Z(e)?false:hr.has(h.extname(e.pathname).toLowerCase())}function re(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return re(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:ft(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>Me(t.pathname,r)):false}function z(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function _r(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Le(e,t){let r=_r(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Ie(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Se(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${ie(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Se(e.targetOrigin,p)}}function Br(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function mt(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function wt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Hr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Pt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function zr(e,t){let r=Pt(t),s=Hr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function Jr(e,t){if(!t)return;let r=Pt(t);if(r===null)return;let s=await wt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function st(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return fr.has(r)||pr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Gr(e){return ie(e.toString("base64"),40)}function Vr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function Kr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Te(e,t,r,s,n,i,u){let l=Ie(e,r),a=Gr(s),o=l.originalFilePath,c=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&Vr(n,l.originalFilePath,a)){let f=Fr(e,t,r);f?(o=f.outputPath,c=f.publicUrl):(o=l.hashedFilePath,c=l.hashedPublicUrl);}Br(e,t,r,c,l.originalPublicUrl);let p=_(o)?Buffer.from(ve(s.toString("utf8"),e,t,o),"utf8"):s,d=await mt(o),g=false;return (d===null||!d.equals(p))&&(await Pe(o),await S.writeFile(o,p),g=true),Kr(n,l.originalFilePath,a),await Jr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function ee(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let d=new URL(p);return re(d)?{discoveredFrom:p,discoveredFromType:"sitemap"}:C(e,d)?{discoveredFrom:p,discoveredFromType:"asset"}:G(d)&&!Z(d)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!G(l)||ft(l)){if(C(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),L(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),Z(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:c}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:c,outputPath:z(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function L(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!C(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Oe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!re(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Yr(e){let t=q(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:Y(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Xr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Zr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function ke(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function St(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await St(i,u));continue}await S.copyFile(i,u),r.push(u);}return r}async function es(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,l=i.prefix;if(!u||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:l});continue}let a=br(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let o=a.resolvedPath,c=ke(l);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:l});continue}let p;try{p=await S.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:o,prefixPath:l});continue}let d=h.resolve(e.outputDir,c);if(p.isDirectory()){n.push(...await St(o,d)),s++,t.progress(`Copied static directory to export: ${o} -> /${c}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${c}`,copiedItems:s});continue}let g=h.basename(o),f=l.endsWith("/")?h.join(d,g):d;await S.mkdir(h.dirname(f),{recursive:true}),await S.copyFile(o,f),n.push(f),s++,t.progress(`Copied static file to export: ${o} -> /${ke(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${ke(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function te(e,t,r,s,n){let i=new Set,u=q(r).replace(/\\\//g,"/");for(let o of Zr(u)){let c=T(e,o,t,s,`${n}:css-url`);c&&C(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,o[1],t,s,`${n}:css-import`);c&&C(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,o[1],t,s,`${n}:xml-stylesheet`);c&&C(e,c)&&i.add(c.toString());}let l="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let o of u.matchAll(a)){let c=o[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${o[0]}`:o[0];if(ht(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=T(e,c,t,s,`${n}:serialized-url`);p&&C(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function ts(e,t,r,s){let n=new Set,i=q(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(gt(e,a)||G(a)&&!Z(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function nt(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),l=i.url()||n;await Te(e,r.assetMap,l,u,r,s);let a=u.toString("utf8");for(let o of te(e,l,a,s,"sitemap"))L(e,r,o,s,l);for(let o of Yr(a)){let c=T(e,o.loc,l,s,"sitemap-loc");c&&(re(c)?Oe(e,r,c.toString(),s,l):C(e,c)?L(e,r,c.toString(),s,l):G(c)?(o.lastmod&&(r.sitemapLastmodByPage[c.toString()]=o.lastmod),ee(e,r,c.toString(),s,l)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",l));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function rs(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,c=(p,d={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let g of o.discoveredAssets||[])L(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=de(e,r.assetMap,a),d=p?await wt(p):null;if(p&&d)try{let k=await t.head(a,{timeout:6e4}),M=k.headers();if(k.ok()&&zr(d,M)){let E=await mt(p);if(E!==null){let W=k.url()||a,A=await Te(e,r.assetMap,W,E,r,n,M);A.wroteFile&&(N(u,e,A.outputPath),_(A.outputPath)&&ce(u,A.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:M["last-modified"]||"",contentLength:M["content-length"]||"",phase:"download-assets"}),Ce(r,n);let b=st(l,M),R=o?.discoveredAssets?[...o.discoveredAssets]:[];if(b||o?.isText){let $=E.toString("utf8"),V=te(e,W,$,n,"asset:cached");(V.length>0||!o)&&(R=V);for(let vt of R)L(e,r,vt,n,W);}if(i.trackManifest){let $=b||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:A.outputPath,isText:$,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!$},i.enabled&&$&&await J(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(c(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),m=g.url()||a,P=g.headers(),w=await Te(e,r.assetMap,m,f,r,n,P);w.wroteFile&&(N(u,e,w.outputPath),_(w.outputPath)&&ce(u,w.outputPath)),r.doneAssets.add(a),Ce(r,n);let y=st(l,P),v=[];if(y){let k=f.toString("utf8");v=te(e,m,k,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let M of v)L(e,r,M,n,m);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:y,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!y},i.enabled&&y&&await J(e,i));}catch(p){if(c("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function ss(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await rs(e,t,r,a,s,n,i));}});await Promise.all(l);}function ns(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function F(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function le(e){return Math.max(15e3,Math.min(6e4,F(e)))}function is(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function De(e){return e.stats.pagesCompleted}function bt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function os(e,t,r){let s=bt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function x(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${ns(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function as(e,t){let r=is(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let c=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(c,p);i>=d+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function us(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await as(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function ls(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function cs(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Me(t,r))}async function ds(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of o){let m=d.getAttribute(f);m&&a.add(m);}for(let f of p){let m=d.getAttribute(f);m&&m.split(",").forEach(P=>{let w=P.trim().split(/\s+/)[0];w&&a.add(w);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...a],styles:[...c]}});for(let a of l.attrs)for(let o of Xr(a)){let c=T(e,o,r,s,"dom-attr");c&&C(e,c)&&i.add(c.toString());}for(let a of l.styles)for(let o of te(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of te(e,r,u,s,"page-html"))i.add(l);return [...i]}async function it(e,t,r,s,n){let i=z(e,t);await x(`save page ${t}`,F(e),async()=>{await Pe(i),await S.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,De(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ps(e){let t=De(e);return t%5===0||t===1||t===e.stats.pagesQueued}function gs(e,t){let r=De(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function hs(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function fs(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await hs(s),s}async function ms(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=bt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let c=null,p=null;try{let d=new URL(o),g=Le(e,o),f=cs(e,d.pathname);if(!G(d)||C(e,d)||Z(d)){if(C(e,d)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),L(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let m=await Ar(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let b=i.manifest.pages[o];if(m.action==="reuse"&&b){for(let R of b.discoveredAssets)L(e,t,R,r,o);for(let R of b.discoveredPages)ee(e,t,R,r,o);b.lastSeenRunId=i.runId,b.sitemapLastmod=Y(t.sitemapLastmodByPage[o])??b.sitemapLastmod,m.changeToken?.supported&&(b.changeToken=m.changeToken.token,b.tokenSource=m.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}c=await x(`create page context ${o}`,F(e),()=>fs(l,e,f)),p=await x(`create page ${o}`,F(e),()=>c.newPage());let P=!1,w=new Set;await p.route("**/*",async b=>{let R=b.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let $=new URL(R.url()),V=new URL(o);if($.hash="",V.hash="",$.href===V.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await b.abort("aborted");return}P=!0;}}catch{}await b.continue();}),p.on("response",b=>{try{let R=T(e,b.url(),e.sourceOrigin,void 0,"network-response");R&&C(e,R)&&(w.add(R.toString()),L(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await x(`read response body ${o}`,F(e),()=>y.text());}catch{v=null;}}catch(b){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(b)});}if(g){let b=y?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:b,outputPath:z(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status(),t.pageDiscoverySources.get(o)),await p.close();continue}f?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await x(`wait for rendered page ${o}`,F(e),()=>ls(e,p))):await x(`wait for rendered page ${o}`,F(e),()=>us(e,p)),t.stats.pagesRendered++;let k=await x(`extract rendered assets ${o}`,F(e),()=>ds(e,p,o,r,!f));for(let b of k)L(e,t,b,r,o);let M=[...new Set([...k,...w])],E=v??await x(`capture rendered html ${o}`,F(e),()=>p.content()),W=s!=="single-url"&&!g?ts(e,o,E,r):[];if(s!=="single-url"&&!g)for(let b of W)ee(e,t,b,r,o);await it(e,o,E,t,r);let A=z(e,o);N(u,e,A),ce(u,A),i.trackManifest&&(i.manifest.pages[o]={url:o,outputPath:A,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:Y(t.sitemapLastmodByPage[o])??null,discoveredPages:W,discoveredAssets:M,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await J(e,i));}catch(d){try{let g=p?await x(`capture partial html ${o}`,F(e),()=>p.content()):"";if(g&&g.trim()){await it(e,o,g,t,r);let f=z(e,o);if(N(u,e,f),ce(u,f),i.trackManifest){let m=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:f,changeToken:null,tokenSource:null,sitemapLastmod:Y(t.sitemapLastmodByPage[o])??m?.sitemapLastmod??null,discoveredPages:m?.discoveredPages?[...m.discoveredPages]:[],discoveredAssets:m?.discoveredAssets?[...m.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await J(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(d)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(d)});}finally{t.stats.pagesCompleted++,ps(t)&&gs(t,r),p&&await x(`close page ${o}`,le(e),()=>p.close()).catch(d=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(d)});}),c&&await x(`close page context ${o}`,le(e),()=>c.close()).catch(d=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(d)});});}}}finally{await x("close worker browser",le(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),os(l,r,"browser.close timeout")));}}async function ws(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function ot(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Ps(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ss(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function yt(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function at(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function bs(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>at(s)-at(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function ut(e){let t=await yt(h.join(e,"job.json"));for(let r of bs(e,t)){let s=await ws(r);if(s.length>0)return s}return []}async function ys(e){let t=h.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let a=await yt(h.join(u,"job.json"));if(!Ps(a))continue;let o=Ss(a);o>=n&&(n=o,s=u);}return s}async function vs(e){let t=await ys(e),r=t!==""?await ut(t):await ut(e.logDir),s=new Set;for(let n of r)if(ot(n.error)||ot(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Rs(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Cs(e){try{let t=await S.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ks(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function lt(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(re(i)?Oe(e,t,i.toString(),s,n):C(e,i)?L(e,t,i.toString(),s,n):G(i)?ee(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function xs(){let e=await we(),t=wr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await vs(e)),t.mode==="single-url"&&(r=await Rs(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await Cs(e):{},a=n?await Rr(e):dt(),o={enabled:n,trackManifest:i,manifest:xe(a),previousManifest:xe(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await ks(e.logDir);let c={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=vr();u||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let d=new K(e.logDir,e.logLevel);if(s&&!n&&d.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)d.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(d.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Oe(e,c,v,d)),await nt(e,w,c,d),e.seedPaths.forEach(v=>lt(e,c,v,d,"seed-path")),e.generated404RequestPath&&ee(e,c,e.generated404RequestPath,d,"generated-404-request-path")):(r.forEach(v=>lt(e,c,v,d,"cli")),await nt(e,w,c,d)),d.endMark("discovery",{pages:c.pageQueue.length,assets:c.assetQueue.length,sitemaps:c.sitemapQueue.length}),d.summary(`Queued ${c.pageQueue.length} pages, ${c.assetQueue.length} assets, ${c.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:c.pageQueue.length,queuedAssets:c.assetQueue.length,queuedSitemaps:c.sitemapQueue.length}),t.mode==="retry-timeouts"&&c.pageQueue.length===0&&c.assetQueue.length===0&&c.sitemapQueue.length===0){d.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await d.flush();return}d.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>ms(e,c,d,t.mode,w,o,p));if(await Promise.all(y),d.endMark("render-pages",{pages:c.stats.pagesCompleted}),d.mark("download-assets"),d.progress("Asset download phase started.",{phase:"download-assets",doneAssets:c.doneAssets.size,assetsQueued:c.stats.assetsQueued,assetQueue:c.assetQueue.length}),await ss(e,w,c,d,o,p),d.endMark("download-assets",{assets:c.doneAssets.size}),o.enabled&&e.maxPages===0&&await Lr(e,c,o,d,p),t.mode==="full"){let v=await es(e,d);for(let k of v)N(p,e,k);}}finally{await x("dispose crawl request context",le(e),()=>w.dispose()).catch(y=>{d.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&await J(e,o),await S.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(c.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(c.assetMap,null,2)&&N(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?$r(e,c):o.enabled&&!t.resumeRewrite?Ir(e,o,p,l,c.assetMap):void 0;d.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";d.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await rt(f);let m=Date.now(),P=await Ge(e,c.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:k,changed:M})=>{M&&N(p,e,h.join(e.outputDir,k)),d.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:k});let E=Date.now();if(!(w===1||w===y||E-m>=5e3))return;m=E;let A=`Rewriting text files: ${w}/${y}`;d.progress(A,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:k}),await rt(A);}});o.trackManifest&&(Or(o,g),await J(e,o)),d.endMark("rewrite-text",{changedTextFiles:P}),await je(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>se(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),d.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${c.stats.pagesRendered} pages, processed ${c.doneSitemaps.size} sitemaps, downloaded ${c.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...c.stats,pagesRendered:c.stats.pagesRendered,donePages:c.stats.pagesCompleted,claimedPages:c.donePages.size,doneSitemaps:c.doneSitemaps.size,doneAssets:c.doneAssets.size,changedTextFiles:P}),await d.flush();}xs().catch(async e=>{console.error(e);try{let r=(await we().catch(()=>null))?.logDir??"logs",s=new K(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.18",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|