@smart-cloud/publisher-exporter 1.1.28 → 1.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +6 -5
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import b from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import Ot from'http';import Dt from'https';import Ht from'crypto';import Gt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Tt=".deploy-plan.json",At="deploy-plan.json";function Mt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ee(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ge(e){return [...new Set(e.map(Ee).filter(t=>t!==null))]}function Lt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ge(t.changedFiles||[]),deletedFiles:ge(t.deletedFiles||[]),rewriteTargets:ge(t.rewriteTargets||[])}}function It(e){let t=h.join(h.resolve(e.outputDir),Tt),r=Mt();return r?[h.join(r,At),t]:[t]}function ne(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return Ee(n)}async function je(e,t){let r=Lt(e,t);for(let s of It(e))await b.mkdir(h.dirname(s),{recursive:true}),await b.writeFile(s,JSON.stringify(r,null,2),"utf8");}function qe(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Ft(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function he(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Et(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=qe(i.targetOrigin);a&&(u.targetOrigin=a);}let l=he(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function Qe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function jt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Ne(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>jt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Ut(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function me(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Ut(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let d=String(t.uploadUrl??"").trim();d&&(l.uploadUrl=d),Object.keys(u).length>0&&(l.siteSettings=u);let p=Qe(t.subscriptionType);return p&&(l.subscriptionType=p),l}function qt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function We(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Qt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function _e(e,t,r=5){let s=new URL(We(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Ot:Dt,u=Qt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let p=[];d.on("data",c=>{p.push(Buffer.isBuffer(c)?c:Buffer.from(c));}),d.on("error",a),d.on("end",()=>{let c=d.statusCode??0,g=d.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(c)){l(_e(new URL(g,s),{method:c===303?"GET":n},r-1));return}let f=new Headers;for(let[m,P]of Object.entries(d.headers))Array.isArray(P)?f.set(m,P.join(", ")):typeof P=="string"&&f.set(m,P);l(new Response(Buffer.concat(p),{status:c,statusText:d.statusMessage??"",headers:f}));});});o.on("error",a),o.end();})}function Nt(e){let t=me(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;qt(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Wt(e){if(!Nt(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>We(i).startsWith(t)?_e(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function _t(e,t){let r=Et(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=Qe(t?.subscriptionType),u=me(e.wpsuite);return {...e,scheduler:Ne(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function fe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Bt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ue(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,fe(s,r))}async function we(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await b.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=qe(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=Ft(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=he(s.extraReplacements),s.postCrawlCopyMap=he(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=me(s.wpsuite),s.scheduler=Ne(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=_t(s,await Wt(s)),i=Bt(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=Ue(u,n.outputDir,"export"),n.logDir=Ue(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=fe(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=fe(n.logDir,"logs"))),n}function oe(e,t=10){return Ht.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Pe(e){await b.mkdir(h.dirname(e),{recursive:true});}function _(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function j(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function H(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function ie(e){return e.replace(/\//g,"\\/")}function Be(e){return e.replace(/\//g,"\\\\/")}function Jt(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function He(e,t,r){if(!t)return;e[t]=r;let s=ie(t),n=ie(r);e[s]=n;let i=Be(t),u=Be(r);e[i]=u;let l=H(t),a=H(r);e[l]=a;let o=H(s),d=H(n);e[o]=d;let p=H(i),c=H(u);e[p]=c;}function ae(e,t,r){He(e,t,r);let s=Jt(t);s&&s!==t&&He(e,s,r);}function Se(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function ze(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Je(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Yt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Zt=new Set([".html",".htm"]),Ve="WPSuite.io Static Publisher",Xt=Ve.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function er(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function tr(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function rr(e,t,r){return tr(t)?e.replace(new RegExp(`(?<!:)${er(t)}`,"g"),r):e.split(t).join(r)}function sr(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Zt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function nr(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Xt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ve}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import b from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import Dt from'http';import Et from'https';import zt from'crypto';import Vt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var At=".deploy-plan.json",Mt="deploy-plan.json";function Lt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Fe(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ge(e){return [...new Set(e.map(Fe).filter(t=>t!==null))]}function It(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ge(t.changedFiles||[]),deletedFiles:ge(t.deletedFiles||[]),rewriteTargets:ge(t.rewriteTargets||[])}}function $t(e){let t=h.join(h.resolve(e.outputDir),At),r=Lt();return r?[h.join(r,Mt),t]:[t]}function ie(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return Fe(n)}async function je(e,t){let r=It(e,t);for(let s of $t(e))await b.mkdir(h.dirname(s),{recursive:true}),await b.writeFile(s,JSON.stringify(r,null,2),"utf8");}function qe(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Ft(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function he(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function jt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=qe(i.targetOrigin);a&&(u.targetOrigin=a);}let l=he(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function Qe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Ut(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Ne(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Ut(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function qt(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function me(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=qt(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let d=String(t.uploadUrl??"").trim();d&&(l.uploadUrl=d),Object.keys(u).length>0&&(l.siteSettings=u);let p=Qe(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Qt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function We(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Nt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function _e(e,t,r=5){let s=new URL(We(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Dt:Et,u=Nt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let p=[];d.on("data",c=>{p.push(Buffer.isBuffer(c)?c:Buffer.from(c));}),d.on("error",a),d.on("end",()=>{let c=d.statusCode??0,g=d.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(c)){l(_e(new URL(g,s),{method:c===303?"GET":n},r-1));return}let m=new Headers;for(let[f,w]of Object.entries(d.headers))Array.isArray(w)?m.set(f,w.join(", ")):typeof w=="string"&&m.set(f,w);l(new Response(Buffer.concat(p),{status:c,statusText:d.statusMessage??"",headers:m}));});});o.on("error",a),o.end();})}function Wt(e){let t=me(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Qt(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function _t(e){if(!Wt(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>We(i).startsWith(t)?_e(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Bt(e,t){let r=jt(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=Qe(t?.subscriptionType),u=me(e.wpsuite);return {...e,scheduler:Ne(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function fe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Ht(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ue(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,fe(s,r))}async function we(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await b.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=qe(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=Ft(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=he(s.extraReplacements),s.postCrawlCopyMap=he(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=me(s.wpsuite),s.scheduler=Ne(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Bt(s,await _t(s)),i=Ht(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=Ue(u,n.outputDir,"export"),n.logDir=Ue(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=fe(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=fe(n.logDir,"logs"))),n}function Y(e,t=10){return zt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Pe(e){await b.mkdir(h.dirname(e),{recursive:true});}function _(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function j(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function z(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function oe(e){return e.replace(/\//g,"\\/")}function Be(e){return e.replace(/\//g,"\\\\/")}function Gt(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function He(e,t,r){if(!t)return;e[t]=r;let s=oe(t),n=oe(r);e[s]=n;let i=Be(t),u=Be(r);e[i]=u;let l=z(t),a=z(r);e[l]=a;let o=z(s),d=z(n);e[o]=d;let p=z(i),c=z(u);e[p]=c;}function ae(e,t,r){He(e,t,r);let s=Gt(t);s&&s!==t&&He(e,s,r);}function be(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function ze(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Je(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Zt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Xt=new Set([".html",".htm"]),Ve="WPSuite.io Static Publisher",er=Ve.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function tr(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function rr(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function sr(e,t,r){return rr(t)?e.replace(new RegExp(`(?<!:)${tr(t)}`,"g"),r):e.split(t).join(r)}function nr(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Xt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function ir(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${er}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ve}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function or(e){let t=j(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function ar(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,oe(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[l,a]of r)u=u.split(l).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,l,a)=>`${u}${s(l)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,l,a,o)=>or(a)?`${u}${s(a)}${o}`:i),n}function ur(e,t){if(!t)return null;let r=h.dirname(h.resolve(t)),s=h.resolve(e.outputDir),n=h.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function lr(e,t,r){let s=ur(t,r);if(!s)return e;let n=e;for(let i of Zt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(l,`${s}${i}`).replace(a,`${s}${i}`);let o=i.replace(/\//g,"\\/"),d=s.replace(/\//g,"\\/"),p=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(c,`${d}${o}`).replace(g,`${d}${o}`).replace(m,`${d}${o}`);}return n}function ye(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:ze(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Je(e.outputDir,r,n)}${i}`}function Se(e,t,r,s,n){ae(e,r,ye(t,s,n));}function cr(e,t,r,s,n){if(!r||r===s)return;ae(e,r,s);let i=ye(t,r,n),u=ye(t,s,n);ae(e,i,u);}function ve(e,t,r,s,n={}){let i=e,u={};Se(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,o]of Object.entries(t.extraReplacements))Se(u,t,a,o,s);for(let[a,o]of Object.entries(r))Se(u,t,a,o,s);for(let[a,o]of Object.entries(n)){let d=r[a];!d||d===o||cr(u,t,o,d,s);}let l=Object.entries(u).sort((a,o)=>o[0].length-a[0].length);for(let[a,o]of l)i=sr(i,a,o);return t.urlRewriteMode!=="absolute"&&(i=lr(i,t,s)),i=ar(i,l.map(([,a])=>a)),nr(t,i,s)&&(i=ir(i)),i}async function dr(e,t,r,s){let n=h.join(e.outputDir,s),i=h.extname(n).toLowerCase(),u;try{u=await b.readFile(n,"utf8");}catch(o){if(o.code!=="ENOENT")throw o;return {changed:false}}let a=ve(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await b.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function pr(e,t){return await new Promise((r,s)=>{let n=a=>{if(l(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{l(),s(a);},u=a=>{l(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function Ke(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(c=>h.resolve(c)).filter(c=>c.startsWith(h.resolve(e.outputDir))).map(c=>h.relative(e.outputDir,c).replace(/\\/g,"/")).filter(c=>c.length>0):await Vt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(c=>c!=="asset-map.json"&&_(c)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(u,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:l});let a=0,o=0,d=null;if(l===1){for(let c of n){let g=await dr(e,t,r.previousAssetMap??{},c);g.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:c,changed:g.changed});}return i}let p=Array.from({length:l},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!d;){let g=a;if(g>=n.length)return;a+=1;let m=n[g],f=await pr(c,m);f.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:m,changed:f.changed});}}catch(g){throw d=g instanceof Error?g:new Error(String(g)),d}finally{await c.terminate().catch(()=>{});}});return await Promise.all(p),i}var N={error:0,warn:1,info:2,debug:3},Ye=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",ue=Ye?h.join(Ye,"current-progress.json"):"",Ze=Promise.resolve();function Xe(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function gr(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function hr(e,t,r){ue&&(Ze=Ze.then(async()=>{let s=new Date,n=s.toISOString();await b.mkdir(h.dirname(ue),{recursive:true});let i=await b.readFile(ue,"utf8").then(f=>JSON.parse(f)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},l=Xe(e),a=i?.currentStep||Xe(i?.source||""),o=i?.startedAt||n,d={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==l&&i?.stepStartedAt){let f=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));d[a]=(d[a]??0)+f,p=n;}let c=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(o).getTime())/1e3)),m={checkedAt:n,source:e,message:t,details:gr(u,r??{}),startedAt:o,currentStep:l,stepStartedAt:p,stepElapsedSec:c,totalElapsedSec:g,stepDurationsSec:d};await b.writeFile(ue,JSON.stringify(m,null,2),"utf8");}).catch(()=>{}));}async function fr(e,t){await b.mkdir(h.dirname(e),{recursive:true}),await b.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function mr(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function wr(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:mr(e,r),level:e,message:t,details:r??{}}}var Z=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return h.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return h.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return h.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return h.join(this.logDir,"ignored.jsonl")}get skippedPath(){return h.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return h.join(this.logDir,"errors.jsonl")}get timingsPath(){return h.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await b.mkdir(this.logDir,{recursive:true}),await b.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await b.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>fr(t,r));}updateCurrentEvent(t,r,s){let n=wr(t,r,s);this.enqueueTask(async()=>{await b.mkdir(this.logDir,{recursive:true}),await b.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(N[this.level]>=N.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}debug(t,r){N[this.level]<N.debug||this.push("debug",t,r);}progress(t,r){hr("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),N[this.level]>=N.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),N[this.level]>=N.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var yr=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],pt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),gt=new Set(["",".html",".htm"]),vr=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Rr=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Cr(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let l=e[u];if(l==="--retry-timeouts")r="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(l.startsWith("--crawl-mode="))s=l.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(l==="--incremental")s="incremental",n=true;else if(l==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(l.startsWith("--url="))r="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function kr(){let e=Me();return e?h.resolve(e,".."):""}function xr(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?h.resolve(e):""}function Tr(e){let t=e.trim(),r=[{alias:"@storage-root",root:kr(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Me(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:xr(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?h.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:h.resolve(t)}}function ht(e){let t=Me();return t?h.join(t,"crawl-manifest.json"):h.join(e.outputDir,".crawl-manifest.json")}function ft(){return {schemaVersion:3,updatedAt:"",pages:{},assets:{}}}function et(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function Ar(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true||!t}}function xe(e){let t=e?.["content-type"]?.trim();return t||null}function Mr(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function U(e,t,r){let s=ie(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function tt(e,t,r){let s=ie(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ee(e,t){e.rewriteTargets.add(h.resolve(t));}async function Lr(e){try{let t=await b.readFile(ht(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3)&&r.assets&&typeof r.assets=="object")return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,et(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,Ar(n)]))};if(r.schemaVersion===1)return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,et(n)])),assets:{}}}}catch{}return ft()}function Te(e){return JSON.parse(JSON.stringify(e))}async function Ir(e,t){let r=ht(e);await b.mkdir(h.dirname(r),{recursive:true}),await b.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function G(e,t){if(!t.trackManifest)return;let r=async()=>{let s=Te(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await Ir(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function X(e){if(!e)return null;let t=e.trim();return t||null}function $r(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function Or(e,t,r,s){let n=[...new Set(r.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!l.ok())return s.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:u,status:l.status()}),new Map;let a=await l.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let o=new Map;for(let d of a.items)!d||typeof d.url!="string"||o.set(d.url,d);return o}catch(l){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(l)}),new Map}}async function Re(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await Or(e,t,[s],n)).get(s)??null;return Ie(e,s)&&(!u||u.supported!==true||!u.token)&&(u=await Fr(e,t,s,n)),u=Er(e,u),r.changeTokenCache.set(s,u),u}function Dr(e){return `subscriber=${e.wpsuite?.siteSettings?.subscriber===true?"1":"0"}`}function Er(e,t){if(!t?.supported||!t.token)return t;let r=Dr(e);return {...t,token:Y(`${t.token}
|
|
7
|
+
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function Fr(e,t,r,s){try{let n=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(n.status()!==404)return s.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:n.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let i=await n.text().catch(()=>""),u=Y(`${r}
|
|
7
8
|
404
|
|
8
|
-
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Dr(e,t,r,s,n,i){let u=Ie(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Re(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Re(e,t,s,n,i)};let a=await Re(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await Fr(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=Z(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Fr(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Er(e,t,r,s,n){let i=rt(r.previousManifest),u=0;for(let[c,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(c)){tt(n,e,g.outputPath);try{await b.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${c}`,{url:c,outputPath:g.outputPath,error:String(f)});}delete r.manifest.pages[c],u++;}let l=rt(r.manifest,r.runId),a=new Set,o=new Set;for(let c of l){for(let m of ee(e,c))a.add(m);let g=r.manifest.assets[c]?.outputPath;g&&o.add(g);let f=de(e,t.assetMap,c);f&&o.add(f);}for(let c of Object.keys(r.manifest.assets))l.has(c)||delete r.manifest.assets[c];let d=0,p=0;for(let c of i){let g=new Set,f=r.previousManifest.assets[c]?.outputPath;f&&g.add(f);let m=de(e,t.assetMap,c);if(!l.has(c)&&m)g.add(m);else if(!l.has(c)){let P=$e(e,c);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){tt(n,e,P);try{await b.unlink(P),d++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${c}`,{url:c,outputPath:P,error:String(w)});}}for(let P of ee(e,c))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||d>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${d} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:d,removedAssetMappings:p,phase:"incremental-cleanup"});}function rt(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function st(e,t,r){for(let s of t)for(let n of ee(e,s))if(r.has(n))return true;return false}function jr(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=Ir(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))st(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&st(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of ee(e,l))if(u.has(o)){U(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function Ur(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function ee(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function qr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function mt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function Qr(e,t,r){for(let s of qr(e,r)){let n=t[s];if(!n)continue;let i=mt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Nr(e,t,r){for(let s of ee(e,r)){let n=t[s];if(!n)continue;let i=mt(e,n);if(i)return i}return null}function de(e,t,r){let s=Nr(e,t,r);return s||$e(e,r).originalFilePath}function Wr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function nt(e){let t=Wr();if(t)try{let r=await b.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await b.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function _r(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(J(e,s)));for(let s of t.doneAssets){let n=de(e,t.assetMap,s);!n||!_(n)||r.add(h.resolve(n));}return [...r]}var pe="re:",le=new Map;function Br(e){if(!e.startsWith(pe))return null;let t=le.get(e);if(t!==void 0)return t||null;let r=e.slice(pe.length).trim();if(!r)return le.set(e,false),null;try{let s=new RegExp(r);return le.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),le.set(e,false),null}}function Le(e,t){let r=Br(t);return r?r.test(e):t.startsWith(pe)?false:e.startsWith(t)}function Hr(e,t){return t.find(r=>Le(e,r))}function wt(e,t){let r=Hr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(pe)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function zr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Jr(e,t){return zr(e).has(t.hostname)}function Gr(e){let t=j(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Pt(e){let t=j(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function Vr(e){let t=j(e).replace(/\\\//g,"/").trim(),r=h.extname(t).toLowerCase();return pt.has(r)||gt.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function V(e){return vr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Gr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Pt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(Vr(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Jr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=wt(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function St(e){return pt.has(h.extname(e.pathname).toLowerCase())}function B(e){return V(e)?false:gt.has(h.extname(e.pathname).toLowerCase())}function se(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return !!(se(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>Le(t.pathname,r))||St(t)||!B(t)&&!V(t))}function J(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function Kr(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Ie(e,t){let r=Kr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function $e(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Se(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${oe(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Se(e.targetOrigin,p)}}function Yr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function bt(e){try{return await b.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function yt(e){try{return await b.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Zr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function vt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Xr(e,t){let r=vt(t),s=Zr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function es(e,t){if(!t)return;let r=vt(t);if(r===null)return;let s=await yt(e);if(!s)return;let n=new Date(r);await b.utimes(e,s.atime,n).catch(()=>{});}function it(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return yr.has(r)||br.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function ts(e){return oe(e.toString("base64"),40)}function rs(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function ss(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Ae(e,t,r,s,n,i,u){let l=$e(e,r),a=ts(s),o=l.originalFilePath,d=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&rs(n,l.originalFilePath,a)){let f=Qr(e,t,r);f?(o=f.outputPath,d=f.publicUrl):(o=l.hashedFilePath,d=l.hashedPublicUrl);}Yr(e,t,r,d,l.originalPublicUrl);let p=_(o)?Buffer.from(ve(s.toString("utf8"),e,t,o),"utf8"):s,c=await bt(o),g=false;return (c===null||!c.equals(p))&&(await Pe(o),await b.writeFile(o,p),g=true),ss(n,l.originalFilePath,a),await es(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:d,wroteFile:g}}function te(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let c=new URL(p);return se(c)?{discoveredFrom:p,discoveredFromType:"sitemap"}:C(e,c)?{discoveredFrom:p,discoveredFromType:"asset"}:B(c)&&!V(c)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!B(l)||St(l)){if(C(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),L(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),V(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:d}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:d,outputPath:J(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function L(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!C(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Oe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!se(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function ns(e){let t=j(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:Z(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function is(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function os(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function as(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function us(e,t){if(t==="dom-style")return true;let r=(()=>{try{return h.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function xe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Rt(e,t){let r=[];await b.mkdir(t,{recursive:true});let s=await b.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await Rt(i,u));continue}await b.copyFile(i,u),r.push(u);}return r}async function ls(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,l=i.prefix;if(!u||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:l});continue}let a=kr(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let o=a.resolvedPath,d=xe(l);if(!d){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:l});continue}let p;try{p=await b.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:o,prefixPath:l});continue}let c=h.resolve(e.outputDir,d);if(p.isDirectory()){n.push(...await Rt(o,c)),s++,t.progress(`Copied static directory to export: ${o} -> /${d}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${d}`,copiedItems:s});continue}let g=h.basename(o),f=l.endsWith("/")?h.join(c,g):c;await b.mkdir(h.dirname(f),{recursive:true}),await b.copyFile(o,f),n.push(f),s++,t.progress(`Copied static file to export: ${o} -> /${xe(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${xe(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function re(e,t,r,s,n){let i=new Set,u=j(r).replace(/\\\//g,"/"),l=n==="page-html"?as(u):u,a=us(t,n);if(a){for(let p of os(l)){let c=T(e,p,t,s,`${n}:css-url`);c&&C(e,c)&&i.add(c.toString());}for(let p of l.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,p[1],t,s,`${n}:css-import`);c&&C(e,c)&&i.add(c.toString());}}if(a||n==="page-html")for(let p of l.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,p[1],t,s,`${n}:xml-stylesheet`);c&&C(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",d=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let p of u.matchAll(d)){let c=p[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${p[0]}`:p[0];if(Pt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let g=T(e,c,t,s,`${n}:serialized-url`);g&&C(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function cs(e,t,r,s){let n=new Set,i=j(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(wt(e,a)||B(a)&&!V(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function ot(e,t,r,s,n){for(;r.sitemapQueue.length>0;){let i=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(i),!r.doneSitemaps.has(i)){r.doneSitemaps.add(i),s.sitemap(`Fetching sitemap ${i}`,{url:i});try{let u=await t.get(i,{timeout:6e4});if(!u.ok()){s.skip("sitemap",i,u.status());continue}let l=await u.body(),a=u.url()||i,o=await Ae(e,r.assetMap,a,l,r,s);U(n,e,o.outputPath),X(n,o.outputPath);let d=l.toString("utf8");for(let p of re(e,a,d,s,"sitemap"))L(e,r,p,s,a);for(let p of ns(d)){let c=T(e,p.loc,a,s,"sitemap-loc");c&&(se(c)?Oe(e,r,c.toString(),s,a):C(e,c)?L(e,r,c.toString(),s,a):B(c)?(p.lastmod&&(r.sitemapLastmodByPage[c.toString()]=p.lastmod),te(e,r,c.toString(),s,a)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",a));}}catch(u){s.error(`Failed sitemap ${i}`,{url:i,error:String(u)});}}}}async function ds(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,d=(p,c={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},delete i.manifest.pages[a],r.doneAssets.add(a);for(let g of o.discoveredAssets||[])L(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...c,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=de(e,r.assetMap,a),c=p?await yt(p):null;if(p&&c)try{let x=await t.head(a,{timeout:6e4}),A=x.headers();if(x.ok()&&Xr(c,A)){let F=await bt(p);if(F!==null){let W=x.url()||a,M=await Ae(e,r.assetMap,W,F,r,n,A);M.wroteFile&&(U(u,e,M.outputPath),_(M.outputPath)&&X(u,M.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:A["last-modified"]||"",contentLength:A["content-length"]||"",phase:"download-assets"}),Ce(r,n);let S=it(l,A),R=o?.discoveredAssets?[...o.discoveredAssets]:[];if(S||o?.isText){let q=F.toString("utf8"),K=re(e,W,q,n,"asset:cached");(K.length>0||!o)&&(R=K);for(let kt of R)L(e,r,kt,n,W);}if(i.trackManifest){let q=S||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:M.outputPath,isText:q,contentType:ke(A)||o?.contentType||null,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!q},delete i.manifest.pages[a],i.enabled&&q&&await G(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(d(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),m=g.url()||a,P=g.headers(),w=await Ae(e,r.assetMap,m,f,r,n,P);w.wroteFile&&(U(u,e,w.outputPath),_(w.outputPath)&&X(u,w.outputPath)),r.doneAssets.add(a),Ce(r,n);let y=it(l,P),v=[];if(y){let x=f.toString("utf8");v=re(e,m,x,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let A of v)L(e,r,A,n,m);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:y,contentType:ke(P),discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!y},delete i.manifest.pages[a],i.enabled&&y&&await G(e,i));}catch(p){if(d("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function ps(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await ds(e,t,r,a,s,n,i));}});await Promise.all(l);}function gs(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function D(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ce(e){return Math.max(15e3,Math.min(6e4,D(e)))}function hs(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function De(e){return e.stats.pagesCompleted}function Ct(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function fs(e,t,r){let s=Ct(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function k(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${gs(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function ms(e,t){let r=hs(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let d=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,c=Math.max(d,p);i>=c+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function ws(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await ms(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ps(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Ss(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Le(t,r))}async function bs(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(c=>{for(let f of o){let m=c.getAttribute(f);m&&a.add(m);}for(let f of p){let m=c.getAttribute(f);m&&m.split(",").forEach(P=>{let w=P.trim().split(/\s+/)[0];w&&a.add(w);});}let g=c.getAttribute("style");g&&d.add(g);}),{attrs:[...a],styles:[...d]}});for(let a of l.attrs)for(let o of is(a)){let d=T(e,o,r,s,"dom-attr");d&&C(e,d)&&i.add(d.toString());}for(let a of l.styles)for(let o of re(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of re(e,r,u,s,"page-html"))i.add(l);return [...i]}async function at(e,t,r,s,n){let i=J(e,t);await k(`save page ${t}`,D(e),async()=>{await Pe(i),await b.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,De(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ys(e){let t=De(e);return t%5===0||t===1||t===e.stats.pagesQueued}function vs(e,t){let r=De(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Rs(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Cs(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Rs(s),s}async function xs(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=Ct(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let d=null,p=null;try{let c=new URL(o),g=Ie(e,o),f=Ss(e,c.pathname);if(!B(c)||C(e,c)||V(c)){if(C(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),L(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let m=await Dr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let S=i.manifest.pages[o];if(m.action==="reuse"&&S){for(let R of S.discoveredAssets)L(e,t,R,r,o);for(let R of S.discoveredPages)te(e,t,R,r,o);S.lastSeenRunId=i.runId,S.sitemapLastmod=Z(t.sitemapLastmodByPage[o])??S.sitemapLastmod,m.changeToken?.supported&&(S.changeToken=m.changeToken.token,S.tokenSource=m.changeToken.tokenSource??S.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}d=await k(`create page context ${o}`,D(e),()=>Cs(l,e,f)),p=await k(`create page ${o}`,D(e),()=>d.newPage());let P=!1,w=new Set;await p.route("**/*",async S=>{let R=S.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let q=new URL(R.url()),K=new URL(o);if(q.hash="",K.hash="",q.href===K.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await S.abort("aborted");return}P=!0;}}catch{}await S.continue();}),p.on("response",S=>{try{let R=T(e,S.url(),e.sourceOrigin,void 0,"network-response");R&&C(e,R)&&(w.add(R.toString()),L(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await k(`read response body ${o}`,D(e),()=>y.text());}catch{v=null;}}catch(S){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(S)});}if(g){let S=y?.status()??null;if(S!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:S,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:S,outputPath:J(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status(),t.pageDiscoverySources.get(o)),await p.close();continue}f?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await k(`wait for rendered page ${o}`,D(e),()=>Ps(e,p))):await k(`wait for rendered page ${o}`,D(e),()=>ws(e,p)),t.stats.pagesRendered++;let x=await k(`extract rendered assets ${o}`,D(e),()=>bs(e,p,o,r,!f));for(let S of x)L(e,t,S,r,o);let A=[...new Set([...x,...w])],F=v??await k(`capture rendered html ${o}`,D(e),()=>p.content()),W=s!=="single-url"&&!g?cs(e,o,F,r):[];if(s!=="single-url"&&!g)for(let S of W)te(e,t,S,r,o);await at(e,o,F,t,r);let M=J(e,o);if(U(u,e,M),X(u,M),i.trackManifest){let S=y?.headers();i.manifest.pages[o]={url:o,outputPath:M,contentType:ke(S),changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:Z(t.sitemapLastmodByPage[o])??null,discoveredPages:W,discoveredAssets:A,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}}catch(c){try{let g=p?await k(`capture partial html ${o}`,D(e),()=>p.content()):"";if(g&&g.trim()){await at(e,o,g,t,r);let f=J(e,o);if(U(u,e,f),X(u,f),i.trackManifest){let m=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:f,contentType:m?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:Z(t.sitemapLastmodByPage[o])??m?.sitemapLastmod??null,discoveredPages:m?.discoveredPages?[...m.discoveredPages]:[],discoveredAssets:m?.discoveredAssets?[...m.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(c)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(c)});}finally{t.stats.pagesCompleted++,ys(t)&&vs(t,r),p&&await k(`close page ${o}`,ce(e),()=>p.close()).catch(c=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(c)});}),d&&await k(`close page context ${o}`,ce(e),()=>d.close()).catch(c=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(c)});});}}}finally{await k("close worker browser",ce(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),fs(l,r,"browser.close timeout")));}}async function ks(e){try{let t=await b.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function ut(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Ts(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function As(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function xt(e){try{let t=await b.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function lt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Ms(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>lt(s)-lt(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function ct(e){let t=await xt(h.join(e,"job.json"));for(let r of Ms(e,t)){let s=await ks(r);if(s.length>0)return s}return []}async function Ls(e){let t=h.join(e.logDir,"archive"),r;try{r=await b.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await b.stat(u).catch(()=>null))?.isDirectory())continue;let a=await xt(h.join(u,"job.json"));if(!Ts(a))continue;let o=As(a);o>=n&&(n=o,s=u);}return s}async function Is(e){let t=await Ls(e),r=t!==""?await ct(t):await ct(e.logDir),s=new Set;for(let n of r)if(ut(n.error)||ut(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function $s(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await b.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Os(e){try{let t=await b.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Ds(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await b.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await b.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function dt(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(se(i)?Oe(e,t,i.toString(),s,n):C(e,i)?L(e,t,i.toString(),s,n):B(i)?te(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Fs(){let e=await we(),t=Rr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Is(e)),t.mode==="single-url"&&(r=await $s(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await Os(e):{},a=n?await Mr(e):ft(),o={enabled:n,trackManifest:i,manifest:Te(a),previousManifest:Te(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await Ds(e.logDir);let d={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Ar();u||await b.rm(e.outputDir,{recursive:true,force:true}),await b.mkdir(e.outputDir,{recursive:true});let c=new Y(e.logDir,e.logLevel);if(s&&!n&&c.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)c.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(c.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Oe(e,d,v,c)),await ot(e,w,d,c,p),e.seedPaths.forEach(v=>dt(e,d,v,c,"seed-path")),e.generated404RequestPath&&te(e,d,e.generated404RequestPath,c,"generated-404-request-path")):(r.forEach(v=>dt(e,d,v,c,"cli")),await ot(e,w,d,c,p)),c.endMark("discovery",{pages:d.pageQueue.length,assets:d.assetQueue.length,sitemaps:d.sitemapQueue.length}),c.summary(`Queued ${d.pageQueue.length} pages, ${d.assetQueue.length} assets, ${d.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:d.pageQueue.length,queuedAssets:d.assetQueue.length,queuedSitemaps:d.sitemapQueue.length}),t.mode==="retry-timeouts"&&d.pageQueue.length===0&&d.assetQueue.length===0&&d.sitemapQueue.length===0){c.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await c.flush();return}c.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>xs(e,d,c,t.mode,w,o,p));if(await Promise.all(y),c.endMark("render-pages",{pages:d.stats.pagesCompleted}),c.mark("download-assets"),c.progress("Asset download phase started.",{phase:"download-assets",doneAssets:d.doneAssets.size,assetsQueued:d.stats.assetsQueued,assetQueue:d.assetQueue.length}),await ps(e,w,d,c,o,p),c.endMark("download-assets",{assets:d.doneAssets.size}),o.enabled&&e.maxPages===0&&await Er(e,d,o,c,p),t.mode==="full"){let v=await ls(e,c);for(let x of v)U(p,e,x);}}finally{await k("dispose crawl request context",ce(e),()=>w.dispose()).catch(y=>{c.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&await G(e,o),await b.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(d.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(d.assetMap,null,2)&&U(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?_r(e,d):o.enabled&&!t.resumeRewrite?jr(e,o,p,l,d.assetMap):void 0;c.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";c.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await nt(f);let m=Date.now(),P=await Ke(e,d.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:x,changed:A})=>{A&&U(p,e,h.join(e.outputDir,x)),c.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x});let F=Date.now();if(!(w===1||w===y||F-m>=5e3))return;m=F;let M=`Rewriting text files: ${w}/${y}`;c.progress(M,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x}),await nt(M);}});o.trackManifest&&(Ur(o,g),await G(e,o)),c.endMark("rewrite-text",{changedTextFiles:P}),await je(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>ne(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),c.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${d.stats.pagesRendered} pages, processed ${d.doneSitemaps.size} sitemaps, downloaded ${d.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...d.stats,pagesRendered:d.stats.pagesRendered,donePages:d.stats.pagesCompleted,claimedPages:d.donePages.size,doneSitemaps:d.doneSitemaps.size,doneAssets:d.doneAssets.size,changedTextFiles:P}),await c.flush();}Fs().catch(async e=>{console.error(e);try{let r=(await we().catch(()=>null))?.logDir??"logs",s=new Y(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
9
|
+
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function jr(e,t,r,s,n,i){let u=Ie(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Re(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Re(e,t,s,n,i)};let a=await Re(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await Ur(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=X(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Ur(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function qr(e,t,r,s,n){let i=rt(r.previousManifest),u=0;for(let[c,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(c)){tt(n,e,g.outputPath);try{await b.unlink(g.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${c}`,{url:c,outputPath:g.outputPath,error:String(m)});}delete r.manifest.pages[c],u++;}let l=rt(r.manifest,r.runId),a=new Set,o=new Set;for(let c of l){for(let f of te(e,c))a.add(f);let g=r.manifest.assets[c]?.outputPath;g&&o.add(g);let m=de(e,t.assetMap,c);m&&o.add(m);}for(let c of Object.keys(r.manifest.assets))l.has(c)||delete r.manifest.assets[c];let d=0,p=0;for(let c of i){let g=new Set,m=r.previousManifest.assets[c]?.outputPath;m&&g.add(m);let f=de(e,t.assetMap,c);if(!l.has(c)&&f)g.add(f);else if(!l.has(c)){let w=$e(e,c);g.add(w.originalFilePath),w.hashedFilePath&&g.add(w.hashedFilePath);}for(let w of g)if(!o.has(w)){tt(n,e,w);try{await b.unlink(w),d++;}catch(P){(P.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${c}`,{url:c,outputPath:w,error:String(P)});}}for(let w of te(e,c))a.has(w)||Object.prototype.hasOwnProperty.call(t.assetMap,w)&&(delete t.assetMap[w],p++);}(u>0||d>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${d} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:d,removedAssetMappings:p,phase:"incremental-cleanup"});}function rt(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function st(e,t,r){for(let s of t)for(let n of te(e,s))if(r.has(n))return true;return false}function Qr(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=$r(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))st(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&st(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of te(e,l))if(u.has(o)){U(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function Nr(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function te(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Wr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function mt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function _r(e,t,r){for(let s of Wr(e,r)){let n=t[s];if(!n)continue;let i=mt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Br(e,t,r){for(let s of te(e,r)){let n=t[s];if(!n)continue;let i=mt(e,n);if(i)return i}return null}function de(e,t,r){let s=Br(e,t,r);return s||$e(e,r).originalFilePath}function Hr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function nt(e){let t=Hr();if(t)try{let r=await b.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await b.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function zr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(B(e,s)));for(let s of t.doneAssets){let n=de(e,t.assetMap,s);!n||!_(n)||r.add(h.resolve(n));}return [...r]}var pe="re:",le=new Map;function Jr(e){if(!e.startsWith(pe))return null;let t=le.get(e);if(t!==void 0)return t||null;let r=e.slice(pe.length).trim();if(!r)return le.set(e,false),null;try{let s=new RegExp(r);return le.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),le.set(e,false),null}}function Le(e,t){let r=Jr(t);return r?r.test(e):t.startsWith(pe)?false:e.startsWith(t)}function Gr(e,t){return t.find(r=>Le(e,r))}function wt(e,t){let r=Gr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(pe)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Vr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Kr(e,t){return Vr(e).has(t.hostname)}function Yr(e){let t=j(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Pt(e){let t=j(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function Zr(e){let t=j(e).replace(/\\\//g,"/").trim(),r=h.extname(t).toLowerCase();return pt.has(r)||gt.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function V(e){return Rr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Yr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Pt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(Zr(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Kr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=wt(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function bt(e){return pt.has(h.extname(e.pathname).toLowerCase())}function H(e){return V(e)?false:gt.has(h.extname(e.pathname).toLowerCase())}function ne(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return !!(ne(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>Le(t.pathname,r))||bt(t)||!H(t)&&!V(t))}function B(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function Xr(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Ie(e,t){let r=Xr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function $e(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=be(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${Y(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:be(e.targetOrigin,p)}}function es(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function St(e){try{return await b.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function yt(e){try{return await b.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function ts(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function vt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function rs(e,t){let r=vt(t),s=ts(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function ss(e,t){if(!t)return;let r=vt(t);if(r===null)return;let s=await yt(e);if(!s)return;let n=new Date(r);await b.utimes(e,s.atime,n).catch(()=>{});}function it(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return vr.has(r)||yr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function ns(e){return Y(e.toString("base64"),40)}function is(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function os(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Ae(e,t,r,s,n,i,u){let l=$e(e,r),a=ns(s),o=l.originalFilePath,d=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&is(n,l.originalFilePath,a)){let m=_r(e,t,r);m?(o=m.outputPath,d=m.publicUrl):(o=l.hashedFilePath,d=l.hashedPublicUrl);}es(e,t,r,d,l.originalPublicUrl);let p=_(o)?Buffer.from(ve(s.toString("utf8"),e,t,o),"utf8"):s,c=await St(o),g=false;return (c===null||!c.equals(p))&&(await Pe(o),await b.writeFile(o,p),g=true),os(n,l.originalFilePath,a),await ss(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:d,wroteFile:g}}function re(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let c=new URL(p);return ne(c)?{discoveredFrom:p,discoveredFromType:"sitemap"}:C(e,c)?{discoveredFrom:p,discoveredFromType:"asset"}:H(c)&&!V(c)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!H(l)||bt(l)){if(C(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),L(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),V(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:d}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:d,outputPath:B(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function L(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!C(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Oe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!ne(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function as(e){let t=j(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:X(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function us(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function ls(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function cs(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function ds(e,t){if(t==="dom-style")return true;let r=(()=>{try{return h.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function ke(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Rt(e,t,r,s){let n=[];await b.mkdir(t,{recursive:true});let i=await b.readdir(e,{withFileTypes:true});for(let u of i){let l=h.join(e,u.name),a=h.join(t,u.name);if(u.isDirectory()){n.push(...await Rt(l,a,r,s));continue}await Ct(l,a,r,s)&&n.push(a);}return n}async function ps(e,t){try{let[r,s]=await Promise.all([b.stat(e),b.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([b.readFile(e),b.readFile(t)]);return n.equals(i)}catch{return false}}async function Ct(e,t,r,s){let n=h.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await ps(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await b.mkdir(h.dirname(t),{recursive:true}),await b.copyFile(e,t),true)}async function gs(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([l,a])=>({sourcePath:l.trim(),prefix:String(a||"").trim()}));if(s.length===0)return [];let n=0,i=[],u=new Set([...t.donePages].map(l=>h.resolve(B(e,l))));r.mark("copy-extra-paths");for(let l of s){let a=l.sourcePath,o=l.prefix;if(!a||!o){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:o});continue}let d=Tr(a);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:o});continue}let p=d.resolvedPath,c=ke(o);if(!c){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:o});continue}let g;try{g=await b.stat(p);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:p,prefixPath:o});continue}let m=h.resolve(e.outputDir,c);if(g.isDirectory()){let S=await Rt(p,m,u,r);i.push(...S),n+=S.length,S.length>0&&r.progress(`Copied static directory to export: ${p} -> /${c}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${c}`,copiedItems:n,changedFiles:S.length});continue}let f=h.basename(p),w=o.endsWith("/")?h.join(m,f):m;await Ct(p,w,u,r)&&(i.push(w),n++,r.progress(`Copied static file to export: ${p} -> /${ke(o.endsWith("/")?`${o}${f}`:o)}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${ke(o.endsWith("/")?`${o}${f}`:o)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function se(e,t,r,s,n){let i=new Set,u=j(r).replace(/\\\//g,"/"),l=n==="page-html"?cs(u):u,a=ds(t,n);if(a){for(let p of ls(l)){let c=T(e,p,t,s,`${n}:css-url`);c&&C(e,c)&&i.add(c.toString());}for(let p of l.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,p[1],t,s,`${n}:css-import`);c&&C(e,c)&&i.add(c.toString());}}if(a||n==="page-html")for(let p of l.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,p[1],t,s,`${n}:xml-stylesheet`);c&&C(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",d=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let p of u.matchAll(d)){let c=p[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${p[0]}`:p[0];if(Pt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let g=T(e,c,t,s,`${n}:serialized-url`);g&&C(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function hs(e,t,r,s){let n=new Set,i=j(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(wt(e,a)||H(a)&&!V(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function ot(e,t,r,s,n){for(;r.sitemapQueue.length>0;){let i=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(i),!r.doneSitemaps.has(i)){r.doneSitemaps.add(i),s.sitemap(`Fetching sitemap ${i}`,{url:i});try{let u=await t.get(i,{timeout:6e4});if(!u.ok()){s.skip("sitemap",i,u.status());continue}let l=await u.body(),a=u.url()||i,o=await Ae(e,r.assetMap,a,l,r,s);U(n,e,o.outputPath),ee(n,o.outputPath);let d=l.toString("utf8");for(let p of se(e,a,d,s,"sitemap"))L(e,r,p,s,a);for(let p of as(d)){let c=T(e,p.loc,a,s,"sitemap-loc");c&&(ne(c)?Oe(e,r,c.toString(),s,a):C(e,c)?L(e,r,c.toString(),s,a):H(c)?(p.lastmod&&(r.sitemapLastmodByPage[c.toString()]=p.lastmod),re(e,r,c.toString(),s,a)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",a));}}catch(u){s.error(`Failed sitemap ${i}`,{url:i,error:String(u)});}}}}async function fs(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,d=(p,c={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},delete i.manifest.pages[a],r.doneAssets.add(a);for(let g of o.discoveredAssets||[])L(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...c,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=de(e,r.assetMap,a),c=p?await yt(p):null;if(p&&c)try{let k=await t.head(a,{timeout:6e4}),A=k.headers();if(k.ok()&&rs(c,A)){let E=await St(p);if(E!==null){let W=k.url()||a,M=await Ae(e,r.assetMap,W,E,r,n,A);M.wroteFile&&(U(u,e,M.outputPath),_(M.outputPath)&&ee(u,M.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:A["last-modified"]||"",contentLength:A["content-length"]||"",phase:"download-assets"}),Ce(r,n);let y=it(l,A),R=o?.discoveredAssets?[...o.discoveredAssets]:[];if(y||o?.isText){let q=E.toString("utf8"),K=se(e,W,q,n,"asset:cached");(K.length>0||!o)&&(R=K);for(let Tt of R)L(e,r,Tt,n,W);}if(i.trackManifest){let q=y||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:M.outputPath,isText:q,contentType:xe(A)||o?.contentType||null,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!q},delete i.manifest.pages[a],i.enabled&&q&&await G(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(d(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let m=await g.body(),f=g.url()||a,w=g.headers(),P=await Ae(e,r.assetMap,f,m,r,n,w);P.wroteFile&&(U(u,e,P.outputPath),_(P.outputPath)&&ee(u,P.outputPath)),r.doneAssets.add(a),Ce(r,n);let S=it(l,w),v=[];if(S){let k=m.toString("utf8");v=se(e,f,k,n,`asset:${h.extname(l.pathname).toLowerCase()||w["content-type"]||"unknown"}`);for(let A of v)L(e,r,A,n,f);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:P.outputPath,isText:S,contentType:xe(w),discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!S},delete i.manifest.pages[a],i.enabled&&S&&await G(e,i));}catch(p){if(d("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function ms(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await fs(e,t,r,a,s,n,i));}});await Promise.all(l);}function ws(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function D(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ce(e){return Math.max(15e3,Math.min(6e4,D(e)))}function Ps(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function De(e){return e.stats.pagesCompleted}function kt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function bs(e,t,r){let s=kt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function x(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${ws(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function Ss(e,t){let r=Ps(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let d=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,c=Math.max(d,p);i>=c+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function ys(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Ss(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function vs(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Rs(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Le(t,r))}async function Cs(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(c=>{for(let m of o){let f=c.getAttribute(m);f&&a.add(f);}for(let m of p){let f=c.getAttribute(m);f&&f.split(",").forEach(w=>{let P=w.trim().split(/\s+/)[0];P&&a.add(P);});}let g=c.getAttribute("style");g&&d.add(g);}),{attrs:[...a],styles:[...d]}});for(let a of l.attrs)for(let o of us(a)){let d=T(e,o,r,s,"dom-attr");d&&C(e,d)&&i.add(d.toString());}for(let a of l.styles)for(let o of se(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of se(e,r,u,s,"page-html"))i.add(l);return [...i]}async function at(e,t,r,s,n){let i=B(e,t);await x(`save page ${t}`,D(e),async()=>{await Pe(i),await b.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,De(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ks(e){let t=De(e);return t%5===0||t===1||t===e.stats.pagesQueued}function xs(e,t){let r=De(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Ts(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function As(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Ts(s),s}async function Ms(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=kt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let d=null,p=null;try{let c=new URL(o),g=Ie(e,o),m=Rs(e,c.pathname);if(!H(c)||C(e,c)||V(c)){if(C(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),L(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let f=await jr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let y=i.manifest.pages[o];if(f.action==="reuse"&&y){for(let R of y.discoveredAssets)L(e,t,R,r,o);for(let R of y.discoveredPages)re(e,t,R,r,o);y.lastSeenRunId=i.runId,y.sitemapLastmod=X(t.sitemapLastmodByPage[o])??y.sitemapLastmod,f.changeToken?.supported&&(y.changeToken=f.changeToken.token,y.tokenSource=f.changeToken.tokenSource??y.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:f.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}d=await x(`create page context ${o}`,D(e),()=>As(l,e,m)),p=await x(`create page ${o}`,D(e),()=>d.newPage());let w=!1,P=new Set;await p.route("**/*",async y=>{let R=y.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let q=new URL(R.url()),K=new URL(o);if(q.hash="",K.hash="",q.href===K.href){if(w){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await y.abort("aborted");return}w=!0;}}catch{}await y.continue();}),p.on("response",y=>{try{let R=T(e,y.url(),e.sourceOrigin,void 0,"network-response");R&&C(e,R)&&(P.add(R.toString()),L(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let S=null,v=null;try{if(S=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),S&&S.ok())try{v=await x(`read response body ${o}`,D(e),()=>S.text());}catch{v=null;}}catch(y){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(y)});}if(g){let y=S?.status()??null;if(y!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:y,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:y,outputPath:B(e,o)});}else if(S&&!S.ok()){r.skip("page",o,S.status(),t.pageDiscoverySources.get(o)),await p.close();continue}m?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await x(`wait for rendered page ${o}`,D(e),()=>vs(e,p))):await x(`wait for rendered page ${o}`,D(e),()=>ys(e,p)),t.stats.pagesRendered++;let k=await x(`extract rendered assets ${o}`,D(e),()=>Cs(e,p,o,r,!m));for(let y of k)L(e,t,y,r,o);let A=[...new Set([...k,...P])],E=v??await x(`capture rendered html ${o}`,D(e),()=>p.content()),W=s!=="single-url"&&!g?hs(e,o,E,r):[];if(s!=="single-url"&&!g)for(let y of W)re(e,t,y,r,o);await at(e,o,E,t,r);let M=B(e,o);if(U(u,e,M),ee(u,M),i.trackManifest){let y=S?.headers();i.manifest.pages[o]={url:o,outputPath:M,contentType:xe(y),changeToken:f.changeToken?.supported?f.changeToken.token:null,tokenSource:f.changeToken?.supported?f.changeToken.tokenSource??null:null,sitemapLastmod:X(t.sitemapLastmodByPage[o])??null,discoveredPages:W,discoveredAssets:A,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}}catch(c){try{let g=p?await x(`capture partial html ${o}`,D(e),()=>p.content()):"";if(g&&g.trim()){await at(e,o,g,t,r);let m=B(e,o);if(U(u,e,m),ee(u,m),i.trackManifest){let f=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:m,contentType:f?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:X(t.sitemapLastmodByPage[o])??f?.sitemapLastmod??null,discoveredPages:f?.discoveredPages?[...f.discoveredPages]:[],discoveredAssets:f?.discoveredAssets?[...f.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(c)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(c)});}finally{t.stats.pagesCompleted++,ks(t)&&xs(t,r),p&&await x(`close page ${o}`,ce(e),()=>p.close()).catch(c=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(c)});}),d&&await x(`close page context ${o}`,ce(e),()=>d.close()).catch(c=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(c)});});}}}finally{await x("close worker browser",ce(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),bs(l,r,"browser.close timeout")));}}async function Ls(e){try{let t=await b.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function ut(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Is(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function $s(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function xt(e){try{let t=await b.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function lt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Os(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>lt(s)-lt(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function ct(e){let t=await xt(h.join(e,"job.json"));for(let r of Os(e,t)){let s=await Ls(r);if(s.length>0)return s}return []}async function Ds(e){let t=h.join(e.logDir,"archive"),r;try{r=await b.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await b.stat(u).catch(()=>null))?.isDirectory())continue;let a=await xt(h.join(u,"job.json"));if(!Is(a))continue;let o=$s(a);o>=n&&(n=o,s=u);}return s}async function Es(e){let t=await Ds(e),r=t!==""?await ct(t):await ct(e.logDir),s=new Set;for(let n of r)if(ut(n.error)||ut(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Fs(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await b.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function js(e){try{let t=await b.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Us(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await b.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await b.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function dt(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(ne(i)?Oe(e,t,i.toString(),s,n):C(e,i)?L(e,t,i.toString(),s,n):H(i)?re(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function qs(){let e=await we(),t=Cr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Es(e)),t.mode==="single-url"&&(r=await Fs(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=(t.mode==="full"||t.mode==="single-url")&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await js(e):{},a=n||t.mode==="single-url"?await Lr(e):ft(),o={enabled:n,trackManifest:i,manifest:Te(a),previousManifest:Te(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await Us(e.logDir);let d={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Mr();u||await b.rm(e.outputDir,{recursive:true,force:true}),await b.mkdir(e.outputDir,{recursive:true});let c=new Z(e.logDir,e.logLevel);if(s&&!n&&c.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)c.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let P=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(c.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Oe(e,d,v,c)),await ot(e,P,d,c,p),e.seedPaths.forEach(v=>dt(e,d,v,c,"seed-path")),e.generated404RequestPath&&re(e,d,e.generated404RequestPath,c,"generated-404-request-path")):(r.forEach(v=>dt(e,d,v,c,"cli")),await ot(e,P,d,c,p)),c.endMark("discovery",{pages:d.pageQueue.length,assets:d.assetQueue.length,sitemaps:d.sitemapQueue.length}),c.summary(`Queued ${d.pageQueue.length} pages, ${d.assetQueue.length} assets, ${d.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:d.pageQueue.length,queuedAssets:d.assetQueue.length,queuedSitemaps:d.sitemapQueue.length}),t.mode==="retry-timeouts"&&d.pageQueue.length===0&&d.assetQueue.length===0&&d.sitemapQueue.length===0){c.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await c.flush();return}c.mark("render-pages");let S=Array.from({length:Math.max(1,e.concurrency)},()=>Ms(e,d,c,t.mode,P,o,p));if(await Promise.all(S),c.endMark("render-pages",{pages:d.stats.pagesCompleted}),c.mark("download-assets"),c.progress("Asset download phase started.",{phase:"download-assets",doneAssets:d.doneAssets.size,assetsQueued:d.stats.assetsQueued,assetQueue:d.assetQueue.length}),await ms(e,P,d,c,o,p),c.endMark("download-assets",{assets:d.doneAssets.size}),o.enabled&&e.maxPages===0&&await qr(e,d,o,c,p),t.mode==="full"){let v=await gs(e,d,c);for(let k of v)U(p,e,k);}}finally{await x("dispose crawl request context",ce(e),()=>P.dispose()).catch(S=>{c.warn("Timed out while disposing crawl request context",{error:String(S)});});}}o.trackManifest&&await G(e,o),await b.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(d.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(d.assetMap,null,2)&&U(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?zr(e,d):o.enabled&&!t.resumeRewrite?Qr(e,o,p,l,d.assetMap):void 0;c.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";c.progress(m,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await nt(m);let f=Date.now(),w=await Ke(e,d.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:P,totalFiles:S,changedTextFiles:v,file:k,changed:A})=>{A&&U(p,e,h.join(e.outputDir,k)),c.checkpoint(`Rewriting text file ${P}/${S}`,{phase:"rewrite-text",index:P,totalFiles:S,changedTextFiles:v,file:k});let E=Date.now();if(!(P===1||P===S||E-f>=5e3))return;f=E;let M=`Rewriting text files: ${P}/${S}`;c.progress(M,{phase:"rewrite-text",index:P,totalFiles:S,changedTextFiles:v,file:k}),await nt(M);}});o.trackManifest&&(Nr(o,g),await G(e,o)),c.endMark("rewrite-text",{changedTextFiles:w}),await je(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(P=>ie(e.outputDir,h.isAbsolute(P)?P:h.join(e.outputDir,P))).filter(P=>P!==null))].sort():[]}),c.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${w} text files.`:`Done. Rendered ${d.stats.pagesRendered} pages, processed ${d.doneSitemaps.size} sitemaps, downloaded ${d.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...d.stats,pagesRendered:d.stats.pagesRendered,donePages:d.stats.pagesCompleted,claimedPages:d.donePages.size,doneSitemaps:d.doneSitemaps.size,doneAssets:d.doneAssets.size,changedTextFiles:w}),await c.flush();}qs().catch(async e=>{console.error(e);try{let r=(await we().catch(()=>null))?.logDir??"logs",s=new Z(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.30",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|