@smart-cloud/publisher-exporter 1.1.24 → 1.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +5 -5
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import S from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import Lt from'http';import It from'https';import Wt from'crypto';import Ht from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Ct=".deploy-plan.json",xt="deploy-plan.json";function kt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function $e(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ge(e){return [...new Set(e.map($e).filter(t=>t!==null))]}function Tt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ge(t.changedFiles||[]),deletedFiles:ge(t.deletedFiles||[]),rewriteTargets:ge(t.rewriteTargets||[])}}function At(e){let t=h.join(h.resolve(e.outputDir),Ct),r=kt();return r?[h.join(r,xt),t]:[t]}function ne(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return $e(n)}async function Ee(e,t){let r=Tt(e,t);for(let s of At(e))await S.mkdir(h.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Ot(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function he(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Dt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=Ue(i.targetOrigin);a&&(u.targetOrigin=a);}let l=he(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function qe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Ft(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Qe(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Ft(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function $t(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function me(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=$t(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(u).length>0&&(l.siteSettings=u);let p=qe(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Et(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Ne(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function jt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function We(e,t,r=5){let s=new URL(Ne(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Lt:It,u=jt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},c=>{let p=[];c.on("data",d=>{p.push(Buffer.isBuffer(d)?d:Buffer.from(d));}),c.on("error",a),c.on("end",()=>{let d=c.statusCode??0,g=c.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(d)){l(We(new URL(g,s),{method:d===303?"GET":n},r-1));return}let f=new Headers;for(let[m,P]of Object.entries(c.headers))Array.isArray(P)?f.set(m,P.join(", ")):typeof P=="string"&&f.set(m,P);l(new Response(Buffer.concat(p),{status:d,statusText:c.statusMessage??"",headers:f}));});});o.on("error",a),o.end();})}function Ut(e){let t=me(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Et(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function qt(e){if(!Ut(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>Ne(i).startsWith(t)?We(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Qt(e,t){let r=Dt(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=qe(t?.subscriptionType),u=me(e.wpsuite);return {...e,scheduler:Qe(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function fe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Nt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function je(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,fe(s,r))}async function we(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Ue(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=Ot(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=he(s.extraReplacements),s.postCrawlCopyMap=he(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=me(s.wpsuite),s.scheduler=Qe(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Qt(s,await qt(s)),i=Nt(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=je(u,n.outputDir,"export"),n.logDir=je(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=fe(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=fe(n.logDir,"logs"))),n}function oe(e,t=10){return Wt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Pe(e){await S.mkdir(h.dirname(e),{recursive:true});}function _(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function Q(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function H(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function ie(e){return e.replace(/\//g,"\\/")}function _e(e){return e.replace(/\//g,"\\\\/")}function Bt(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Be(e,t,r){if(!t)return;e[t]=r;let s=ie(t),n=ie(r);e[s]=n;let i=_e(t),u=_e(r);e[i]=u;let l=H(t),a=H(r);e[l]=a;let o=H(s),c=H(n);e[o]=c;let p=H(i),d=H(u);e[p]=d;}function ae(e,t,r){Be(e,t,r);let s=Bt(t);s&&s!==t&&Be(e,s,r);}function Se(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function He(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function ze(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Gt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Vt=new Set([".html",".htm"]),Ge="WPSuite.io Static Publisher",Kt=Ge.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Yt(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function Xt(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function Zt(e,t,r){return Xt(t)?e.replace(new RegExp(`(?<!:)${Yt(t)}`,"g"),r):e.split(t).join(r)}function er(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Vt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function tr(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Kt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ge}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import S from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import $t from'http';import Ot from'https';import Bt from'crypto';import Jt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var kt=".deploy-plan.json",Tt="deploy-plan.json";function At(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Fe(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ge(e){return [...new Set(e.map(Fe).filter(t=>t!==null))]}function Mt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ge(t.changedFiles||[]),deletedFiles:ge(t.deletedFiles||[]),rewriteTargets:ge(t.rewriteTargets||[])}}function Lt(e){let t=h.join(h.resolve(e.outputDir),kt),r=At();return r?[h.join(r,Tt),t]:[t]}function ne(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return Fe(n)}async function Ee(e,t){let r=Mt(e,t);for(let s of Lt(e))await S.mkdir(h.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Dt(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function he(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ft(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=Ue(i.targetOrigin);a&&(u.targetOrigin=a);}let l=he(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function qe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Et(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Qe(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Et(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function jt(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function me(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=jt(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let d=String(t.uploadUrl??"").trim();d&&(l.uploadUrl=d),Object.keys(u).length>0&&(l.siteSettings=u);let p=qe(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Ut(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Ne(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function qt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function We(e,t,r=5){let s=new URL(Ne(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?$t:Ot,u=qt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let p=[];d.on("data",c=>{p.push(Buffer.isBuffer(c)?c:Buffer.from(c));}),d.on("error",a),d.on("end",()=>{let c=d.statusCode??0,g=d.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(c)){l(We(new URL(g,s),{method:c===303?"GET":n},r-1));return}let f=new Headers;for(let[m,P]of Object.entries(d.headers))Array.isArray(P)?f.set(m,P.join(", ")):typeof P=="string"&&f.set(m,P);l(new Response(Buffer.concat(p),{status:c,statusText:d.statusMessage??"",headers:f}));});});o.on("error",a),o.end();})}function Qt(e){let t=me(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Ut(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Nt(e){if(!Qt(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>Ne(i).startsWith(t)?We(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Wt(e,t){let r=Ft(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=qe(t?.subscriptionType),u=me(e.wpsuite);return {...e,scheduler:Qe(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function fe(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function _t(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function je(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,fe(s,r))}async function we(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Ue(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=Dt(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=he(s.extraReplacements),s.postCrawlCopyMap=he(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=me(s.wpsuite),s.scheduler=Qe(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Wt(s,await Nt(s)),i=_t(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=je(u,n.outputDir,"export"),n.logDir=je(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=fe(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=fe(n.logDir,"logs"))),n}function oe(e,t=10){return Bt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Pe(e){await S.mkdir(h.dirname(e),{recursive:true});}function _(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function j(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function H(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function ie(e){return e.replace(/\//g,"\\/")}function _e(e){return e.replace(/\//g,"\\\\/")}function zt(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Be(e,t,r){if(!t)return;e[t]=r;let s=ie(t),n=ie(r);e[s]=n;let i=_e(t),u=_e(r);e[i]=u;let l=H(t),a=H(r);e[l]=a;let o=H(s),d=H(n);e[o]=d;let p=H(i),c=H(u);e[p]=c;}function ae(e,t,r){Be(e,t,r);let s=zt(t);s&&s!==t&&Be(e,s,r);}function Se(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function He(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function ze(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Kt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Yt=new Set([".html",".htm"]),Ge="WPSuite.io Static Publisher",Zt=Ge.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Xt(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function er(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function tr(e,t,r){return er(t)?e.replace(new RegExp(`(?<!:)${Xt(t)}`,"g"),r):e.split(t).join(r)}function rr(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Yt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function sr(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Zt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ge}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function nr(e){let t=j(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function ir(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,ie(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[l,a]of r)u=u.split(l).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,l,a)=>`${u}${s(l)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,l,a,o)=>nr(a)?`${u}${s(a)}${o}`:i),n}function or(e,t){if(!t)return null;let r=h.dirname(h.resolve(t)),s=h.resolve(e.outputDir),n=h.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function ar(e,t,r){let s=or(t,r);if(!s)return e;let n=e;for(let i of Kt){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(l,`${s}${i}`).replace(a,`${s}${i}`);let o=i.replace(/\//g,"\\/"),d=s.replace(/\//g,"\\/"),p=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),f=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(c,`${d}${o}`).replace(g,`${d}${o}`).replace(f,`${d}${o}`);}return n}function ye(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:He(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${ze(e.outputDir,r,n)}${i}`}function be(e,t,r,s,n){ae(e,r,ye(t,s,n));}function ur(e,t,r,s,n){if(!r||r===s)return;ae(e,r,s);let i=ye(t,r,n),u=ye(t,s,n);ae(e,i,u);}function ve(e,t,r,s,n={}){let i=e,u={};be(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,o]of Object.entries(t.extraReplacements))be(u,t,a,o,s);for(let[a,o]of Object.entries(r))be(u,t,a,o,s);for(let[a,o]of Object.entries(n)){let d=r[a];!d||d===o||ur(u,t,o,d,s);}let l=Object.entries(u).sort((a,o)=>o[0].length-a[0].length);for(let[a,o]of l)i=tr(i,a,o);return t.urlRewriteMode!=="absolute"&&(i=ar(i,t,s)),i=ir(i,l.map(([,a])=>a)),rr(t,i,s)&&(i=sr(i)),i}async function lr(e,t,r,s){let n=h.join(e.outputDir,s),i=h.extname(n).toLowerCase(),u;try{u=await S.readFile(n,"utf8");}catch(o){if(o.code!=="ENOENT")throw o;return {changed:false}}let a=ve(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await S.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function cr(e,t){return await new Promise((r,s)=>{let n=a=>{if(l(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{l(),s(a);},u=a=>{l(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function Ve(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(c=>h.resolve(c)).filter(c=>c.startsWith(h.resolve(e.outputDir))).map(c=>h.relative(e.outputDir,c).replace(/\\/g,"/")).filter(c=>c.length>0):await Jt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(c=>c!=="asset-map.json"&&_(c)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(u,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:l});let a=0,o=0,d=null;if(l===1){for(let c of n){let g=await lr(e,t,r.previousAssetMap??{},c);g.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:c,changed:g.changed});}return i}let p=Array.from({length:l},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!d;){let g=a;if(g>=n.length)return;a+=1;let f=n[g],m=await cr(c,f);m.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:f,changed:m.changed});}}catch(g){throw d=g instanceof Error?g:new Error(String(g)),d}finally{await c.terminate().catch(()=>{});}});return await Promise.all(p),i}var N={error:0,warn:1,info:2,debug:3},Ke=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",ue=Ke?h.join(Ke,"current-progress.json"):"",Ye=Promise.resolve();function Ze(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function dr(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function pr(e,t,r){ue&&(Ye=Ye.then(async()=>{let s=new Date,n=s.toISOString();await S.mkdir(h.dirname(ue),{recursive:true});let i=await S.readFile(ue,"utf8").then(m=>JSON.parse(m)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},l=Ze(e),a=i?.currentStep||Ze(i?.source||""),o=i?.startedAt||n,d={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==l&&i?.stepStartedAt){let m=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));d[a]=(d[a]??0)+m,p=n;}let c=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(o).getTime())/1e3)),f={checkedAt:n,source:e,message:t,details:dr(u,r??{}),startedAt:o,currentStep:l,stepStartedAt:p,stepElapsedSec:c,totalElapsedSec:g,stepDurationsSec:d};await S.writeFile(ue,JSON.stringify(f,null,2),"utf8");}).catch(()=>{}));}async function gr(e,t){await S.mkdir(h.dirname(e),{recursive:true}),await S.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function hr(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function fr(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:hr(e,r),level:e,message:t,details:r??{}}}var Y=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return h.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return h.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return h.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return h.join(this.logDir,"ignored.jsonl")}get skippedPath(){return h.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return h.join(this.logDir,"errors.jsonl")}get timingsPath(){return h.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await S.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>gr(t,r));}updateCurrentEvent(t,r,s){let n=fr(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(N[this.level]>=N.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}debug(t,r){N[this.level]<N.debug||this.push("debug",t,r);}progress(t,r){pr("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),N[this.level]>=N.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),N[this.level]>=N.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Sr=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],dt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),pt=new Set(["",".html",".htm"]),br=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),yr=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function vr(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let l=e[u];if(l==="--retry-timeouts")r="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(l.startsWith("--crawl-mode="))s=l.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(l==="--incremental")s="incremental",n=true;else if(l==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(l.startsWith("--url="))r="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Ae(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Rr(){let e=Ae();return e?h.resolve(e,".."):""}function Cr(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?h.resolve(e):""}function xr(e){let t=e.trim(),r=[{alias:"@storage-root",root:Rr(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Ae(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Cr(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?h.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:h.resolve(t)}}function gt(e){let t=Ae();return t?h.join(t,"crawl-manifest.json"):h.join(e.outputDir,".crawl-manifest.json")}function ht(){return {schemaVersion:3,updatedAt:"",pages:{},assets:{}}}function Xe(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function kr(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true||!t}}function Tr(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function U(e,t,r){let s=ne(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function et(e,t,r){let s=ne(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function X(e,t){e.rewriteTargets.add(h.resolve(t));}async function Ar(e){try{let t=await S.readFile(gt(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3)&&r.assets&&typeof r.assets=="object")return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Xe(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,kr(n)]))};if(r.schemaVersion===1)return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Xe(n)])),assets:{}}}}catch{}return ht()}function ke(e){return JSON.parse(JSON.stringify(e))}async function Mr(e,t){let r=gt(e);await S.mkdir(h.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function G(e,t){if(!t.trackManifest)return;let r=async()=>{let s=ke(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await Mr(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function Z(e){if(!e)return null;let t=e.trim();return t||null}function Lr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function Ir(e,t,r,s){let n=[...new Set(r.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!l.ok())return s.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:u,status:l.status()}),new Map;let a=await l.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let o=new Map;for(let d of a.items)!d||typeof d.url!="string"||o.set(d.url,d);return o}catch(l){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(l)}),new Map}}async function Re(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await Ir(e,t,[s],n)).get(s)??null;return Le(e,s)&&(!u||u.supported!==true||!u.token)&&(u=await $r(e,t,s,n)),r.changeTokenCache.set(s,u),u}async function $r(e,t,r,s){try{let n=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(n.status()!==404)return s.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:n.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let i=await n.text().catch(()=>""),u=oe(`${r}
|
|
7
7
|
404
|
|
8
|
-
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Dr(e,t,r,s,n,i){let u=Le(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Re(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Re(e,t,s,n,i)};let a=await Re(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await Fr(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=X(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Fr(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function $r(e,t,r,s,n){let i=tt(r.previousManifest),u=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){et(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(f)});}delete r.manifest.pages[d],u++;}let l=tt(r.manifest,r.runId),a=new Set,o=new Set;for(let d of l){for(let m of ee(e,d))a.add(m);let g=r.manifest.assets[d]?.outputPath;g&&o.add(g);let f=de(e,t.assetMap,d);f&&o.add(f);}for(let d of Object.keys(r.manifest.assets))l.has(d)||delete r.manifest.assets[d];let c=0,p=0;for(let d of i){let g=new Set,f=r.previousManifest.assets[d]?.outputPath;f&&g.add(f);let m=de(e,t.assetMap,d);if(!l.has(d)&&m)g.add(m);else if(!l.has(d)){let P=Ie(e,d);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){et(n,e,P);try{await S.unlink(P),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:P,error:String(w)});}}for(let P of ee(e,d))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||c>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:p,phase:"incremental-cleanup"});}function tt(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function rt(e,t,r){for(let s of t)for(let n of ee(e,s))if(r.has(n))return true;return false}function Er(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=Lr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))rt(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&rt(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of ee(e,l))if(u.has(o)){j(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function jr(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function ee(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Ur(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function gt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function qr(e,t,r){for(let s of Ur(e,r)){let n=t[s];if(!n)continue;let i=gt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Qr(e,t,r){for(let s of ee(e,r)){let n=t[s];if(!n)continue;let i=gt(e,n);if(i)return i}return null}function de(e,t,r){let s=Qr(e,t,r);return s||Ie(e,r).originalFilePath}function Nr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function st(e){let t=Nr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Wr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(J(e,s)));for(let s of t.doneAssets){let n=de(e,t.assetMap,s);!n||!_(n)||r.add(h.resolve(n));}return [...r]}var pe="re:",le=new Map;function _r(e){if(!e.startsWith(pe))return null;let t=le.get(e);if(t!==void 0)return t||null;let r=e.slice(pe.length).trim();if(!r)return le.set(e,false),null;try{let s=new RegExp(r);return le.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),le.set(e,false),null}}function Me(e,t){let r=_r(t);return r?r.test(e):t.startsWith(pe)?false:e.startsWith(t)}function Br(e,t){return t.find(r=>Me(e,r))}function ht(e,t){let r=Br(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(pe)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Hr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function zr(e,t){return Hr(e).has(t.hostname)}function Jr(e){let t=Q(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function ft(e){let t=Q(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function V(e){return yr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Jr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(ft(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!zr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=ht(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function mt(e){return Pr.has(h.extname(e.pathname).toLowerCase())}function B(e){return V(e)?false:Sr.has(h.extname(e.pathname).toLowerCase())}function se(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return !!(se(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>Me(t.pathname,r))||mt(t)||!B(t)&&!V(t))}function J(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function Gr(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Le(e,t){let r=Gr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Ie(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Se(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${oe(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Se(e.targetOrigin,p)}}function Vr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function wt(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Pt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Kr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function St(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Yr(e,t){let r=St(t),s=Kr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function Xr(e,t){if(!t)return;let r=St(t);if(r===null)return;let s=await Pt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function nt(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return br.has(r)||wr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Zr(e){return oe(e.toString("base64"),40)}function es(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function ts(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Te(e,t,r,s,n,i,u){let l=Ie(e,r),a=Zr(s),o=l.originalFilePath,c=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&es(n,l.originalFilePath,a)){let f=qr(e,t,r);f?(o=f.outputPath,c=f.publicUrl):(o=l.hashedFilePath,c=l.hashedPublicUrl);}Vr(e,t,r,c,l.originalPublicUrl);let p=_(o)?Buffer.from(ve(s.toString("utf8"),e,t,o),"utf8"):s,d=await wt(o),g=false;return (d===null||!d.equals(p))&&(await Pe(o),await S.writeFile(o,p),g=true),ts(n,l.originalFilePath,a),await Xr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function te(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let d=new URL(p);return se(d)?{discoveredFrom:p,discoveredFromType:"sitemap"}:C(e,d)?{discoveredFrom:p,discoveredFromType:"asset"}:B(d)&&!V(d)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!B(l)||mt(l)){if(C(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),L(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),V(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:c}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:c,outputPath:J(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function L(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!C(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Oe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!se(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function rs(e){let t=Q(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:X(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function ss(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function ns(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function xe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function bt(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await bt(i,u));continue}await S.copyFile(i,u),r.push(u);}return r}async function is(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,l=i.prefix;if(!u||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:l});continue}let a=xr(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let o=a.resolvedPath,c=xe(l);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:l});continue}let p;try{p=await S.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:o,prefixPath:l});continue}let d=h.resolve(e.outputDir,c);if(p.isDirectory()){n.push(...await bt(o,d)),s++,t.progress(`Copied static directory to export: ${o} -> /${c}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${c}`,copiedItems:s});continue}let g=h.basename(o),f=l.endsWith("/")?h.join(d,g):d;await S.mkdir(h.dirname(f),{recursive:true}),await S.copyFile(o,f),n.push(f),s++,t.progress(`Copied static file to export: ${o} -> /${xe(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${xe(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function re(e,t,r,s,n){let i=new Set,u=Q(r).replace(/\\\//g,"/");for(let o of ns(u)){let c=T(e,o,t,s,`${n}:css-url`);c&&C(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,o[1],t,s,`${n}:css-import`);c&&C(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,o[1],t,s,`${n}:xml-stylesheet`);c&&C(e,c)&&i.add(c.toString());}let l="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let o of u.matchAll(a)){let c=o[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${o[0]}`:o[0];if(ft(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=T(e,c,t,s,`${n}:serialized-url`);p&&C(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function os(e,t,r,s){let n=new Set,i=Q(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(ht(e,a)||B(a)&&!V(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function it(e,t,r,s,n){for(;r.sitemapQueue.length>0;){let i=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(i),!r.doneSitemaps.has(i)){r.doneSitemaps.add(i),s.sitemap(`Fetching sitemap ${i}`,{url:i});try{let u=await t.get(i,{timeout:6e4});if(!u.ok()){s.skip("sitemap",i,u.status());continue}let l=await u.body(),a=u.url()||i,o=await Te(e,r.assetMap,a,l,r,s);j(n,e,o.outputPath),Z(n,o.outputPath);let c=l.toString("utf8");for(let p of re(e,a,c,s,"sitemap"))L(e,r,p,s,a);for(let p of rs(c)){let d=T(e,p.loc,a,s,"sitemap-loc");d&&(se(d)?Oe(e,r,d.toString(),s,a):C(e,d)?L(e,r,d.toString(),s,a):B(d)?(p.lastmod&&(r.sitemapLastmodByPage[d.toString()]=p.lastmod),te(e,r,d.toString(),s,a)):s.reject("sitemap-loc",d.toString(),"not page/sitemap/asset-like",a));}}catch(u){s.error(`Failed sitemap ${i}`,{url:i,error:String(u)});}}}}async function as(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,c=(p,d={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let g of o.discoveredAssets||[])L(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=de(e,r.assetMap,a),d=p?await Pt(p):null;if(p&&d)try{let x=await t.head(a,{timeout:6e4}),A=x.headers();if(x.ok()&&Yr(d,A)){let $=await wt(p);if($!==null){let W=x.url()||a,M=await Te(e,r.assetMap,W,$,r,n,A);M.wroteFile&&(j(u,e,M.outputPath),_(M.outputPath)&&Z(u,M.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:A["last-modified"]||"",contentLength:A["content-length"]||"",phase:"download-assets"}),Ce(r,n);let b=nt(l,A),R=o?.discoveredAssets?[...o.discoveredAssets]:[];if(b||o?.isText){let U=$.toString("utf8"),K=re(e,W,U,n,"asset:cached");(K.length>0||!o)&&(R=K);for(let Rt of R)L(e,r,Rt,n,W);}if(i.trackManifest){let U=b||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:M.outputPath,isText:U,contentType:A["content-type"]?.split(";")[0].trim()||o?.contentType||null,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!U},i.enabled&&U&&await G(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(c(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),m=g.url()||a,P=g.headers(),w=await Te(e,r.assetMap,m,f,r,n,P);w.wroteFile&&(j(u,e,w.outputPath),_(w.outputPath)&&Z(u,w.outputPath)),r.doneAssets.add(a),Ce(r,n);let y=nt(l,P),v=[];if(y){let x=f.toString("utf8");v=re(e,m,x,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let A of v)L(e,r,A,n,m);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:y,contentType:P["content-type"]?.split(";")[0].trim()||null,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!y},i.enabled&&y&&await G(e,i));}catch(p){if(c("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function us(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await as(e,t,r,a,s,n,i));}});await Promise.all(l);}function ls(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function F(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ce(e){return Math.max(15e3,Math.min(6e4,F(e)))}function cs(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function De(e){return e.stats.pagesCompleted}function yt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function ds(e,t,r){let s=yt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function k(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${ls(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function ps(e,t){let r=cs(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let c=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(c,p);i>=d+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function gs(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await ps(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function hs(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function fs(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Me(t,r))}async function ms(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of o){let m=d.getAttribute(f);m&&a.add(m);}for(let f of p){let m=d.getAttribute(f);m&&m.split(",").forEach(P=>{let w=P.trim().split(/\s+/)[0];w&&a.add(w);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...a],styles:[...c]}});for(let a of l.attrs)for(let o of ss(a)){let c=T(e,o,r,s,"dom-attr");c&&C(e,c)&&i.add(c.toString());}for(let a of l.styles)for(let o of re(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of re(e,r,u,s,"page-html"))i.add(l);return [...i]}async function ot(e,t,r,s,n){let i=J(e,t);await k(`save page ${t}`,F(e),async()=>{await Pe(i),await S.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,De(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ws(e){let t=De(e);return t%5===0||t===1||t===e.stats.pagesQueued}function Ps(e,t){let r=De(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Ss(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function bs(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Ss(s),s}async function ys(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=yt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let c=null,p=null;try{let d=new URL(o),g=Le(e,o),f=fs(e,d.pathname);if(!B(d)||C(e,d)||V(d)){if(C(e,d)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),L(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let m=await Dr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let b=i.manifest.pages[o];if(m.action==="reuse"&&b){for(let R of b.discoveredAssets)L(e,t,R,r,o);for(let R of b.discoveredPages)te(e,t,R,r,o);b.lastSeenRunId=i.runId,b.sitemapLastmod=X(t.sitemapLastmodByPage[o])??b.sitemapLastmod,m.changeToken?.supported&&(b.changeToken=m.changeToken.token,b.tokenSource=m.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}c=await k(`create page context ${o}`,F(e),()=>bs(l,e,f)),p=await k(`create page ${o}`,F(e),()=>c.newPage());let P=!1,w=new Set;await p.route("**/*",async b=>{let R=b.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let U=new URL(R.url()),K=new URL(o);if(U.hash="",K.hash="",U.href===K.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await b.abort("aborted");return}P=!0;}}catch{}await b.continue();}),p.on("response",b=>{try{let R=T(e,b.url(),e.sourceOrigin,void 0,"network-response");R&&C(e,R)&&(w.add(R.toString()),L(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await k(`read response body ${o}`,F(e),()=>y.text());}catch{v=null;}}catch(b){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(b)});}if(g){let b=y?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:b,outputPath:J(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status(),t.pageDiscoverySources.get(o)),await p.close();continue}f?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await k(`wait for rendered page ${o}`,F(e),()=>hs(e,p))):await k(`wait for rendered page ${o}`,F(e),()=>gs(e,p)),t.stats.pagesRendered++;let x=await k(`extract rendered assets ${o}`,F(e),()=>ms(e,p,o,r,!f));for(let b of x)L(e,t,b,r,o);let A=[...new Set([...x,...w])],$=v??await k(`capture rendered html ${o}`,F(e),()=>p.content()),W=s!=="single-url"&&!g?os(e,o,$,r):[];if(s!=="single-url"&&!g)for(let b of W)te(e,t,b,r,o);await ot(e,o,$,t,r);let M=J(e,o);j(u,e,M),Z(u,M),i.trackManifest&&(i.manifest.pages[o]={url:o,outputPath:M,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:X(t.sitemapLastmodByPage[o])??null,discoveredPages:W,discoveredAssets:A,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i));}catch(d){try{let g=p?await k(`capture partial html ${o}`,F(e),()=>p.content()):"";if(g&&g.trim()){await ot(e,o,g,t,r);let f=J(e,o);if(j(u,e,f),Z(u,f),i.trackManifest){let m=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:f,changeToken:null,tokenSource:null,sitemapLastmod:X(t.sitemapLastmodByPage[o])??m?.sitemapLastmod??null,discoveredPages:m?.discoveredPages?[...m.discoveredPages]:[],discoveredAssets:m?.discoveredAssets?[...m.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(d)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(d)});}finally{t.stats.pagesCompleted++,ws(t)&&Ps(t,r),p&&await k(`close page ${o}`,ce(e),()=>p.close()).catch(d=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(d)});}),c&&await k(`close page context ${o}`,ce(e),()=>c.close()).catch(d=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(d)});});}}}finally{await k("close worker browser",ce(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),ds(l,r,"browser.close timeout")));}}async function vs(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function at(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Rs(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Cs(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function vt(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function ut(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function xs(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>ut(s)-ut(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function lt(e){let t=await vt(h.join(e,"job.json"));for(let r of xs(e,t)){let s=await vs(r);if(s.length>0)return s}return []}async function ks(e){let t=h.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let a=await vt(h.join(u,"job.json"));if(!Rs(a))continue;let o=Cs(a);o>=n&&(n=o,s=u);}return s}async function Ts(e){let t=await ks(e),r=t!==""?await lt(t):await lt(e.logDir),s=new Set;for(let n of r)if(at(n.error)||at(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function As(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Ms(e){try{let t=await S.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Ls(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function ct(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(se(i)?Oe(e,t,i.toString(),s,n):C(e,i)?L(e,t,i.toString(),s,n):B(i)?te(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Is(){let e=await we(),t=vr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Ts(e)),t.mode==="single-url"&&(r=await As(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await Ms(e):{},a=n?await Ar(e):pt(),o={enabled:n,trackManifest:i,manifest:ke(a),previousManifest:ke(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await Ls(e.logDir);let c={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Tr();u||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let d=new Y(e.logDir,e.logLevel);if(s&&!n&&d.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)d.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(d.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Oe(e,c,v,d)),await it(e,w,c,d,p),e.seedPaths.forEach(v=>ct(e,c,v,d,"seed-path")),e.generated404RequestPath&&te(e,c,e.generated404RequestPath,d,"generated-404-request-path")):(r.forEach(v=>ct(e,c,v,d,"cli")),await it(e,w,c,d,p)),d.endMark("discovery",{pages:c.pageQueue.length,assets:c.assetQueue.length,sitemaps:c.sitemapQueue.length}),d.summary(`Queued ${c.pageQueue.length} pages, ${c.assetQueue.length} assets, ${c.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:c.pageQueue.length,queuedAssets:c.assetQueue.length,queuedSitemaps:c.sitemapQueue.length}),t.mode==="retry-timeouts"&&c.pageQueue.length===0&&c.assetQueue.length===0&&c.sitemapQueue.length===0){d.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await d.flush();return}d.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>ys(e,c,d,t.mode,w,o,p));if(await Promise.all(y),d.endMark("render-pages",{pages:c.stats.pagesCompleted}),d.mark("download-assets"),d.progress("Asset download phase started.",{phase:"download-assets",doneAssets:c.doneAssets.size,assetsQueued:c.stats.assetsQueued,assetQueue:c.assetQueue.length}),await us(e,w,c,d,o,p),d.endMark("download-assets",{assets:c.doneAssets.size}),o.enabled&&e.maxPages===0&&await $r(e,c,o,d,p),t.mode==="full"){let v=await is(e,d);for(let x of v)j(p,e,x);}}finally{await k("dispose crawl request context",ce(e),()=>w.dispose()).catch(y=>{d.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&await G(e,o),await S.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(c.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(c.assetMap,null,2)&&j(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?Wr(e,c):o.enabled&&!t.resumeRewrite?Er(e,o,p,l,c.assetMap):void 0;d.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";d.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await st(f);let m=Date.now(),P=await Ve(e,c.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:x,changed:A})=>{A&&j(p,e,h.join(e.outputDir,x)),d.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x});let $=Date.now();if(!(w===1||w===y||$-m>=5e3))return;m=$;let M=`Rewriting text files: ${w}/${y}`;d.progress(M,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x}),await st(M);}});o.trackManifest&&(jr(o,g),await G(e,o)),d.endMark("rewrite-text",{changedTextFiles:P}),await Ee(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>ne(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),d.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${c.stats.pagesRendered} pages, processed ${c.doneSitemaps.size} sitemaps, downloaded ${c.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...c.stats,pagesRendered:c.stats.pagesRendered,donePages:c.stats.pagesCompleted,claimedPages:c.donePages.size,doneSitemaps:c.doneSitemaps.size,doneAssets:c.doneAssets.size,changedTextFiles:P}),await d.flush();}Is().catch(async e=>{console.error(e);try{let r=(await we().catch(()=>null))?.logDir??"logs",s=new Y(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
8
|
+
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Or(e,t,r,s,n,i){let u=Le(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Re(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Re(e,t,s,n,i)};let a=await Re(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await Dr(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=Z(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Dr(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Fr(e,t,r,s,n){let i=tt(r.previousManifest),u=0;for(let[c,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(c)){et(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${c}`,{url:c,outputPath:g.outputPath,error:String(f)});}delete r.manifest.pages[c],u++;}let l=tt(r.manifest,r.runId),a=new Set,o=new Set;for(let c of l){for(let m of ee(e,c))a.add(m);let g=r.manifest.assets[c]?.outputPath;g&&o.add(g);let f=de(e,t.assetMap,c);f&&o.add(f);}for(let c of Object.keys(r.manifest.assets))l.has(c)||delete r.manifest.assets[c];let d=0,p=0;for(let c of i){let g=new Set,f=r.previousManifest.assets[c]?.outputPath;f&&g.add(f);let m=de(e,t.assetMap,c);if(!l.has(c)&&m)g.add(m);else if(!l.has(c)){let P=Ie(e,c);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){et(n,e,P);try{await S.unlink(P),d++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${c}`,{url:c,outputPath:P,error:String(w)});}}for(let P of ee(e,c))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||d>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${d} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:d,removedAssetMappings:p,phase:"incremental-cleanup"});}function tt(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function rt(e,t,r){for(let s of t)for(let n of ee(e,s))if(r.has(n))return true;return false}function Er(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=Lr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))rt(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&rt(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of ee(e,l))if(u.has(o)){U(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function jr(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function ee(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Ur(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function ft(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function qr(e,t,r){for(let s of Ur(e,r)){let n=t[s];if(!n)continue;let i=ft(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Qr(e,t,r){for(let s of ee(e,r)){let n=t[s];if(!n)continue;let i=ft(e,n);if(i)return i}return null}function de(e,t,r){let s=Qr(e,t,r);return s||Ie(e,r).originalFilePath}function Nr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function st(e){let t=Nr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Wr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(J(e,s)));for(let s of t.doneAssets){let n=de(e,t.assetMap,s);!n||!_(n)||r.add(h.resolve(n));}return [...r]}var pe="re:",le=new Map;function _r(e){if(!e.startsWith(pe))return null;let t=le.get(e);if(t!==void 0)return t||null;let r=e.slice(pe.length).trim();if(!r)return le.set(e,false),null;try{let s=new RegExp(r);return le.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),le.set(e,false),null}}function Me(e,t){let r=_r(t);return r?r.test(e):t.startsWith(pe)?false:e.startsWith(t)}function Br(e,t){return t.find(r=>Me(e,r))}function mt(e,t){let r=Br(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(pe)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Hr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function zr(e,t){return Hr(e).has(t.hostname)}function Jr(e){let t=j(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function wt(e){let t=j(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function Gr(e){let t=j(e).replace(/\\\//g,"/").trim(),r=h.extname(t).toLowerCase();return dt.has(r)||pt.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function V(e){return yr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Jr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(wt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(Gr(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!zr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=mt(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function Pt(e){return dt.has(h.extname(e.pathname).toLowerCase())}function B(e){return V(e)?false:pt.has(h.extname(e.pathname).toLowerCase())}function se(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function C(e,t){return !!(se(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>Me(t.pathname,r))||Pt(t)||!B(t)&&!V(t))}function J(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function Vr(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Le(e,t){let r=Vr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Ie(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Se(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${oe(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Se(e.targetOrigin,p)}}function Kr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function St(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function bt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Yr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function yt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Zr(e,t){let r=yt(t),s=Yr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function Xr(e,t){if(!t)return;let r=yt(t);if(r===null)return;let s=await bt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function nt(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return br.has(r)||Sr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function es(e){return oe(e.toString("base64"),40)}function ts(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function rs(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Te(e,t,r,s,n,i,u){let l=Ie(e,r),a=es(s),o=l.originalFilePath,d=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&ts(n,l.originalFilePath,a)){let f=qr(e,t,r);f?(o=f.outputPath,d=f.publicUrl):(o=l.hashedFilePath,d=l.hashedPublicUrl);}Kr(e,t,r,d,l.originalPublicUrl);let p=_(o)?Buffer.from(ve(s.toString("utf8"),e,t,o),"utf8"):s,c=await St(o),g=false;return (c===null||!c.equals(p))&&(await Pe(o),await S.writeFile(o,p),g=true),rs(n,l.originalFilePath,a),await Xr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:d,wroteFile:g}}function te(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let c=new URL(p);return se(c)?{discoveredFrom:p,discoveredFromType:"sitemap"}:C(e,c)?{discoveredFrom:p,discoveredFromType:"asset"}:B(c)&&!V(c)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!B(l)||Pt(l)){if(C(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),L(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),V(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:d}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:d,outputPath:J(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function L(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!C(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function $e(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!se(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function ss(e){let t=j(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:Z(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function ns(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function is(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function os(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function as(e,t){if(t==="dom-style")return true;let r=(()=>{try{return h.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function xe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function vt(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await vt(i,u));continue}await S.copyFile(i,u),r.push(u);}return r}async function us(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,l=i.prefix;if(!u||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:l});continue}let a=xr(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let o=a.resolvedPath,d=xe(l);if(!d){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:l});continue}let p;try{p=await S.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:o,prefixPath:l});continue}let c=h.resolve(e.outputDir,d);if(p.isDirectory()){n.push(...await vt(o,c)),s++,t.progress(`Copied static directory to export: ${o} -> /${d}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${d}`,copiedItems:s});continue}let g=h.basename(o),f=l.endsWith("/")?h.join(c,g):c;await S.mkdir(h.dirname(f),{recursive:true}),await S.copyFile(o,f),n.push(f),s++,t.progress(`Copied static file to export: ${o} -> /${xe(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${xe(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function re(e,t,r,s,n){let i=new Set,u=j(r).replace(/\\\//g,"/"),l=n==="page-html"?os(u):u,a=as(t,n);if(a){for(let p of is(l)){let c=T(e,p,t,s,`${n}:css-url`);c&&C(e,c)&&i.add(c.toString());}for(let p of l.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,p[1],t,s,`${n}:css-import`);c&&C(e,c)&&i.add(c.toString());}}if(a||n==="page-html")for(let p of l.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,p[1],t,s,`${n}:xml-stylesheet`);c&&C(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",d=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let p of u.matchAll(d)){let c=p[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${p[0]}`:p[0];if(wt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let g=T(e,c,t,s,`${n}:serialized-url`);g&&C(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function ls(e,t,r,s){let n=new Set,i=j(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(mt(e,a)||B(a)&&!V(a)&&!C(e,a)&&n.add(a.toString()));}return [...n]}async function it(e,t,r,s,n){for(;r.sitemapQueue.length>0;){let i=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(i),!r.doneSitemaps.has(i)){r.doneSitemaps.add(i),s.sitemap(`Fetching sitemap ${i}`,{url:i});try{let u=await t.get(i,{timeout:6e4});if(!u.ok()){s.skip("sitemap",i,u.status());continue}let l=await u.body(),a=u.url()||i,o=await Te(e,r.assetMap,a,l,r,s);U(n,e,o.outputPath),X(n,o.outputPath);let d=l.toString("utf8");for(let p of re(e,a,d,s,"sitemap"))L(e,r,p,s,a);for(let p of ss(d)){let c=T(e,p.loc,a,s,"sitemap-loc");c&&(se(c)?$e(e,r,c.toString(),s,a):C(e,c)?L(e,r,c.toString(),s,a):B(c)?(p.lastmod&&(r.sitemapLastmodByPage[c.toString()]=p.lastmod),te(e,r,c.toString(),s,a)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",a));}}catch(u){s.error(`Failed sitemap ${i}`,{url:i,error:String(u)});}}}}async function cs(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!C(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,d=(p,c={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},delete i.manifest.pages[a],r.doneAssets.add(a);for(let g of o.discoveredAssets||[])L(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...c,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=de(e,r.assetMap,a),c=p?await bt(p):null;if(p&&c)try{let x=await t.head(a,{timeout:6e4}),A=x.headers();if(x.ok()&&Zr(c,A)){let F=await St(p);if(F!==null){let W=x.url()||a,M=await Te(e,r.assetMap,W,F,r,n,A);M.wroteFile&&(U(u,e,M.outputPath),_(M.outputPath)&&X(u,M.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:A["last-modified"]||"",contentLength:A["content-length"]||"",phase:"download-assets"}),Ce(r,n);let b=nt(l,A),R=o?.discoveredAssets?[...o.discoveredAssets]:[];if(b||o?.isText){let q=F.toString("utf8"),K=re(e,W,q,n,"asset:cached");(K.length>0||!o)&&(R=K);for(let xt of R)L(e,r,xt,n,W);}if(i.trackManifest){let q=b||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:M.outputPath,isText:q,contentType:A["content-type"]?.split(";")[0].trim()||o?.contentType||null,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!q},delete i.manifest.pages[a],i.enabled&&q&&await G(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(d(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),m=g.url()||a,P=g.headers(),w=await Te(e,r.assetMap,m,f,r,n,P);w.wroteFile&&(U(u,e,w.outputPath),_(w.outputPath)&&X(u,w.outputPath)),r.doneAssets.add(a),Ce(r,n);let y=nt(l,P),v=[];if(y){let x=f.toString("utf8");v=re(e,m,x,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let A of v)L(e,r,A,n,m);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:y,contentType:P["content-type"]?.split(";")[0].trim()||null,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!y},delete i.manifest.pages[a],i.enabled&&y&&await G(e,i));}catch(p){if(d("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function ds(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await cs(e,t,r,a,s,n,i));}});await Promise.all(l);}function ps(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function D(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ce(e){return Math.max(15e3,Math.min(6e4,D(e)))}function gs(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function Oe(e){return e.stats.pagesCompleted}function Rt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function hs(e,t,r){let s=Rt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function k(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${ps(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function fs(e,t){let r=gs(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let d=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,c=Math.max(d,p);i>=c+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function ms(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await fs(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function ws(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Ps(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Me(t,r))}async function Ss(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(c=>{for(let f of o){let m=c.getAttribute(f);m&&a.add(m);}for(let f of p){let m=c.getAttribute(f);m&&m.split(",").forEach(P=>{let w=P.trim().split(/\s+/)[0];w&&a.add(w);});}let g=c.getAttribute("style");g&&d.add(g);}),{attrs:[...a],styles:[...d]}});for(let a of l.attrs)for(let o of ns(a)){let d=T(e,o,r,s,"dom-attr");d&&C(e,d)&&i.add(d.toString());}for(let a of l.styles)for(let o of re(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of re(e,r,u,s,"page-html"))i.add(l);return [...i]}async function ot(e,t,r,s,n){let i=J(e,t);await k(`save page ${t}`,D(e),async()=>{await Pe(i),await S.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,Oe(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function bs(e){let t=Oe(e);return t%5===0||t===1||t===e.stats.pagesQueued}function ys(e,t){let r=Oe(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function vs(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Rs(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await vs(s),s}async function Cs(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=Rt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let d=null,p=null;try{let c=new URL(o),g=Le(e,o),f=Ps(e,c.pathname);if(!B(c)||C(e,c)||V(c)){if(C(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),L(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let m=await Or(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let b=i.manifest.pages[o];if(m.action==="reuse"&&b){for(let R of b.discoveredAssets)L(e,t,R,r,o);for(let R of b.discoveredPages)te(e,t,R,r,o);b.lastSeenRunId=i.runId,b.sitemapLastmod=Z(t.sitemapLastmodByPage[o])??b.sitemapLastmod,m.changeToken?.supported&&(b.changeToken=m.changeToken.token,b.tokenSource=m.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}d=await k(`create page context ${o}`,D(e),()=>Rs(l,e,f)),p=await k(`create page ${o}`,D(e),()=>d.newPage());let P=!1,w=new Set;await p.route("**/*",async b=>{let R=b.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let q=new URL(R.url()),K=new URL(o);if(q.hash="",K.hash="",q.href===K.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await b.abort("aborted");return}P=!0;}}catch{}await b.continue();}),p.on("response",b=>{try{let R=T(e,b.url(),e.sourceOrigin,void 0,"network-response");R&&C(e,R)&&(w.add(R.toString()),L(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await k(`read response body ${o}`,D(e),()=>y.text());}catch{v=null;}}catch(b){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(b)});}if(g){let b=y?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:b,outputPath:J(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status(),t.pageDiscoverySources.get(o)),await p.close();continue}f?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await k(`wait for rendered page ${o}`,D(e),()=>ws(e,p))):await k(`wait for rendered page ${o}`,D(e),()=>ms(e,p)),t.stats.pagesRendered++;let x=await k(`extract rendered assets ${o}`,D(e),()=>Ss(e,p,o,r,!f));for(let b of x)L(e,t,b,r,o);let A=[...new Set([...x,...w])],F=v??await k(`capture rendered html ${o}`,D(e),()=>p.content()),W=s!=="single-url"&&!g?ls(e,o,F,r):[];if(s!=="single-url"&&!g)for(let b of W)te(e,t,b,r,o);await ot(e,o,F,t,r);let M=J(e,o);U(u,e,M),X(u,M),i.trackManifest&&(i.manifest.pages[o]={url:o,outputPath:M,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:Z(t.sitemapLastmodByPage[o])??null,discoveredPages:W,discoveredAssets:A,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i));}catch(c){try{let g=p?await k(`capture partial html ${o}`,D(e),()=>p.content()):"";if(g&&g.trim()){await ot(e,o,g,t,r);let f=J(e,o);if(U(u,e,f),X(u,f),i.trackManifest){let m=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:f,changeToken:null,tokenSource:null,sitemapLastmod:Z(t.sitemapLastmodByPage[o])??m?.sitemapLastmod??null,discoveredPages:m?.discoveredPages?[...m.discoveredPages]:[],discoveredAssets:m?.discoveredAssets?[...m.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(c)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(c)});}finally{t.stats.pagesCompleted++,bs(t)&&ys(t,r),p&&await k(`close page ${o}`,ce(e),()=>p.close()).catch(c=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(c)});}),d&&await k(`close page context ${o}`,ce(e),()=>d.close()).catch(c=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(c)});});}}}finally{await k("close worker browser",ce(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),hs(l,r,"browser.close timeout")));}}async function xs(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function at(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function ks(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ts(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function Ct(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function ut(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function As(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>ut(s)-ut(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function lt(e){let t=await Ct(h.join(e,"job.json"));for(let r of As(e,t)){let s=await xs(r);if(s.length>0)return s}return []}async function Ms(e){let t=h.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let a=await Ct(h.join(u,"job.json"));if(!ks(a))continue;let o=Ts(a);o>=n&&(n=o,s=u);}return s}async function Ls(e){let t=await Ms(e),r=t!==""?await lt(t):await lt(e.logDir),s=new Set;for(let n of r)if(at(n.error)||at(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Is(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function $s(e){try{let t=await S.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Os(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function ct(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(se(i)?$e(e,t,i.toString(),s,n):C(e,i)?L(e,t,i.toString(),s,n):B(i)?te(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Ds(){let e=await we(),t=vr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Ls(e)),t.mode==="single-url"&&(r=await Is(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await $s(e):{},a=n?await Ar(e):ht(),o={enabled:n,trackManifest:i,manifest:ke(a),previousManifest:ke(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await Os(e.logDir);let d={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Tr();u||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let c=new Y(e.logDir,e.logLevel);if(s&&!n&&c.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)c.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(c.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>$e(e,d,v,c)),await it(e,w,d,c,p),e.seedPaths.forEach(v=>ct(e,d,v,c,"seed-path")),e.generated404RequestPath&&te(e,d,e.generated404RequestPath,c,"generated-404-request-path")):(r.forEach(v=>ct(e,d,v,c,"cli")),await it(e,w,d,c,p)),c.endMark("discovery",{pages:d.pageQueue.length,assets:d.assetQueue.length,sitemaps:d.sitemapQueue.length}),c.summary(`Queued ${d.pageQueue.length} pages, ${d.assetQueue.length} assets, ${d.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:d.pageQueue.length,queuedAssets:d.assetQueue.length,queuedSitemaps:d.sitemapQueue.length}),t.mode==="retry-timeouts"&&d.pageQueue.length===0&&d.assetQueue.length===0&&d.sitemapQueue.length===0){c.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await c.flush();return}c.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>Cs(e,d,c,t.mode,w,o,p));if(await Promise.all(y),c.endMark("render-pages",{pages:d.stats.pagesCompleted}),c.mark("download-assets"),c.progress("Asset download phase started.",{phase:"download-assets",doneAssets:d.doneAssets.size,assetsQueued:d.stats.assetsQueued,assetQueue:d.assetQueue.length}),await ds(e,w,d,c,o,p),c.endMark("download-assets",{assets:d.doneAssets.size}),o.enabled&&e.maxPages===0&&await Fr(e,d,o,c,p),t.mode==="full"){let v=await us(e,c);for(let x of v)U(p,e,x);}}finally{await k("dispose crawl request context",ce(e),()=>w.dispose()).catch(y=>{c.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&await G(e,o),await S.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(d.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(d.assetMap,null,2)&&U(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?Wr(e,d):o.enabled&&!t.resumeRewrite?Er(e,o,p,l,d.assetMap):void 0;c.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";c.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await st(f);let m=Date.now(),P=await Ve(e,d.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:x,changed:A})=>{A&&U(p,e,h.join(e.outputDir,x)),c.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x});let F=Date.now();if(!(w===1||w===y||F-m>=5e3))return;m=F;let M=`Rewriting text files: ${w}/${y}`;c.progress(M,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:x}),await st(M);}});o.trackManifest&&(jr(o,g),await G(e,o)),c.endMark("rewrite-text",{changedTextFiles:P}),await Ee(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>ne(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),c.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${d.stats.pagesRendered} pages, processed ${d.doneSitemaps.size} sitemaps, downloaded ${d.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...d.stats,pagesRendered:d.stats.pagesRendered,donePages:d.stats.pagesCompleted,claimedPages:d.donePages.size,doneSitemaps:d.doneSitemaps.size,doneAssets:d.doneAssets.size,changedTextFiles:P}),await c.flush();}Ds().catch(async e=>{console.error(e);try{let r=(await we().catch(()=>null))?.logDir??"logs",s=new Y(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.26",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|