@smart-cloud/publisher-exporter 1.1.31 → 1.1.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +6 -6
- package/dist/queue-runner.js +2 -2
- package/package.json +24 -16
package/dist/crawl.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import S from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import jt from'http';import Ft from'https';import Je from'crypto';import Vt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Mt=".deploy-plan.json",It="deploy-plan.json";function Lt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ee(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function fe(e){return [...new Set(e.map(Ee).filter(t=>t!==null))]}function $t(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:fe(t.changedFiles||[]),deletedFiles:fe(t.deletedFiles||[]),rewriteTargets:fe(t.rewriteTargets||[])}}function Dt(e){let t=h.join(h.resolve(e.outputDir),Mt),r=Lt();return r?[h.join(r,It),t]:[t]}function Y(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return Ee(n)}async function Ue(e,t){let r=$t(e,t);for(let s of Dt(e))await S.mkdir(h.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Qe(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Et(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function me(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ut(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=Qe(i.targetOrigin);a&&(u.targetOrigin=a);}let l=me(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function Ne(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function qt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function We(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>qt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Qt(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function Pe(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Qt(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let d=String(t.uploadUrl??"").trim();d&&(l.uploadUrl=d),Object.keys(u).length>0&&(l.siteSettings=u);let p=Ne(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Nt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function _e(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Wt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function Be(e,t,r=5){let s=new URL(_e(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?jt:Ft,u=Wt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let p=[];d.on("data",c=>{p.push(Buffer.isBuffer(c)?c:Buffer.from(c));}),d.on("error",a),d.on("end",()=>{let c=d.statusCode??0,g=d.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(c)){l(Be(new URL(g,s),{method:c===303?"GET":n},r-1));return}let m=new Headers;for(let[f,P]of Object.entries(d.headers))Array.isArray(P)?m.set(f,P.join(", ")):typeof P=="string"&&m.set(f,P);l(new Response(Buffer.concat(p),{status:c,statusText:d.statusMessage??"",headers:m}));});});o.on("error",a),o.end();})}function _t(e){let t=Pe(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Nt(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Bt(e){if(!_t(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>_e(i).startsWith(t)?Be(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Ht(e,t){let r=Ut(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=Ne(t?.subscriptionType),u=Pe(e.wpsuite);return {...e,scheduler:We(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function we(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function zt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function qe(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,we(s,r))}async function be(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Qe(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=Et(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=me(s.extraReplacements),s.postCrawlCopyMap=me(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=Pe(s.wpsuite),s.scheduler=We(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Ht(s,await Bt(s)),i=zt(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=qe(u,n.outputDir,"export"),n.logDir=qe(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=we(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=we(n.logDir,"logs"))),n}function X(e,t=10){return Je.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Jt(e){await S.mkdir(h.dirname(e),{recursive:true});}async function ee(e,t,r){await Jt(e);let s=h.join(h.dirname(e),`.${h.basename(e)}.${process.pid}.${Date.now()}.${Je.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await S.writeFile(s,t,r??"utf8"):await S.writeFile(s,t),await S.rename(s,e);}catch(n){throw await S.unlink(s).catch(()=>{}),n}}function H(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function U(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function J(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function le(e){return e.replace(/\//g,"\\/")}function He(e){return e.replace(/\//g,"\\\\/")}function Gt(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function ze(e,t,r){if(!t)return;e[t]=r;let s=le(t),n=le(r);e[s]=n;let i=He(t),u=He(r);e[i]=u;let l=J(t),a=J(r);e[l]=a;let o=J(s),d=J(n);e[o]=d;let p=J(i),c=J(u);e[p]=c;}function ce(e,t,r){ze(e,t,r);let s=Gt(t);s&&s!==t&&ze(e,s,r);}function Se(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Ge(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ve(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Xt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],er=new Set([".html",".htm"]),Ke="WPSuite.io Static Publisher",tr=Ke.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function rr(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function sr(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function nr(e,t,r){return sr(t)?e.replace(new RegExp(`(?<!:)${rr(t)}`,"g"),r):e.split(t).join(r)}function ir(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return er.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function or(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${tr}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ke}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import S from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import Wt from'http';import _t from'https';import Xe from'crypto';import rr from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var jt=".deploy-plan.json",Et="deploy-plan.json";function Ut(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function _e(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function Se(e){return [...new Set(e.map(_e).filter(t=>t!==null))]}function qt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:Se(t.changedFiles||[]),deletedFiles:Se(t.deletedFiles||[]),rewriteTargets:Se(t.rewriteTargets||[])}}function Qt(e){let t=f.join(f.resolve(e.outputDir),jt),r=Ut();return r?[f.join(r,Et),t]:[t]}function ee(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return _e(n)}async function Be(e,t){let r=qt(e,t);for(let s of Qt(e))await S.mkdir(f.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function ze(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Bt(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&f.extname(i)===""&&(i+="/"),i==="/"?"":i}function be(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ht(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=ze(i.targetOrigin);a&&(u.targetOrigin=a);}let l=be(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function Je(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function zt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function Ge(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>zt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Jt(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function ve(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Jt(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let d=String(t.uploadUrl??"").trim();d&&(l.uploadUrl=d),Object.keys(u).length>0&&(l.siteSettings=u);let p=Je(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Gt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Ve(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Vt(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function Ke(e,t,r=5){let s=new URL(Ve(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Wt:_t,u=Vt(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let p=[];d.on("data",c=>{p.push(Buffer.isBuffer(c)?c:Buffer.from(c));}),d.on("error",a),d.on("end",()=>{let c=d.statusCode??0,g=d.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(c)){l(Ke(new URL(g,s),{method:c===303?"GET":n},r-1));return}let m=new Headers;for(let[w,h]of Object.entries(d.headers))Array.isArray(h)?m.set(w,h.join(", ")):typeof h=="string"&&m.set(w,h);l(new Response(Buffer.concat(p),{status:c,statusText:d.statusMessage??"",headers:m}));});});o.on("error",a),o.end();})}function Kt(e){let t=ve(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Gt(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Yt(e){if(!Kt(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(i,u)=>Ve(i).startsWith(t)?Ke(i,u):r(i,u)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),n=await s("publisher");return n&&typeof n=="object"?n:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Zt(e,t){let r=Ht(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=Je(t?.subscriptionType),u=ve(e.wpsuite);return {...e,scheduler:Ge(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function ye(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Xt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function He(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,ye(s,r))}async function Re(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=ze(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=Bt(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=be(s.extraReplacements),s.postCrawlCopyMap=be(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=ve(s.wpsuite),s.scheduler=Ge(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Zt(s,await Yt(s)),i=Xt(),u=i?f.resolve(i,".."):"";return u?(n.outputDir=He(u,n.outputDir,"export"),n.logDir=He(u,n.logDir,"logs")):(f.isAbsolute(n.outputDir)||(n.outputDir=ye(n.outputDir,"export")),f.isAbsolute(n.logDir)||(n.logDir=ye(n.logDir,"logs"))),n}function re(e,t=10){return Xe.createHash("sha1").update(e).digest("hex").slice(0,t)}async function er(e){await S.mkdir(f.dirname(e),{recursive:true});}async function se(e,t,r){await er(e);let s=f.join(f.dirname(e),`.${f.basename(e)}.${process.pid}.${Date.now()}.${Xe.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await S.writeFile(s,t,r??"utf8"):await S.writeFile(s,t),await S.rename(s,e);}catch(n){throw await S.unlink(s).catch(()=>{}),n}}function z(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function E(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function V(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function pe(e){return e.replace(/\//g,"\\/")}function Ye(e){return e.replace(/\//g,"\\\\/")}function tr(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Ze(e,t,r){if(!t)return;e[t]=r;let s=pe(t),n=pe(r);e[s]=n;let i=Ye(t),u=Ye(r);e[i]=u;let l=V(t),a=V(r);e[l]=a;let o=V(s),d=V(n);e[o]=d;let p=V(i),c=V(u);e[p]=c;}function ge(e,t,r){Ze(e,t,r);let s=tr(t);s&&s!==t&&Ze(e,s,r);}function Ce(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function et(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function tt(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var or=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],ar=new Set([".html",".htm"]),rt="WPSuite.io Static Publisher",ur=rt.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function lr(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function cr(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function dr(e,t,r){return cr(t)?e.replace(new RegExp(`(?<!:)${lr(t)}`,"g"),r):e.split(t).join(r)}function pr(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=f.extname(r).toLowerCase();return ar.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function gr(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${ur}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${rt}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>
|
|
7
|
-
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function fr(e){let t=E(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function hr(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,pe(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[l,a]of r)u=u.split(l).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,l,a)=>`${u}${s(l)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,l,a,o)=>fr(a)?`${u}${s(a)}${o}`:i),n}function mr(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function wr(e,t,r){let s=mr(t,r);if(!s)return e;let n=e;for(let i of or){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(l,`${s}${i}`).replace(a,`${s}${i}`);let o=i.replace(/\//g,"\\/"),d=s.replace(/\//g,"\\/"),p=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(c,`${d}${o}`).replace(g,`${d}${o}`).replace(m,`${d}${o}`);}return n}function xe(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:et(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${tt(e.outputDir,r,n)}${i}`}function ke(e,t,r,s,n){ge(e,r,xe(t,s,n));}function Pr(e,t,r,s,n){if(!r||r===s)return;ge(e,r,s);let i=xe(t,r,n),u=xe(t,s,n);ge(e,i,u);}function Te(e,t,r,s,n={}){let i=e,u={};ke(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,o]of Object.entries(t.extraReplacements))ke(u,t,a,o,s);for(let[a,o]of Object.entries(r))ke(u,t,a,o,s);for(let[a,o]of Object.entries(n)){let d=r[a];!d||d===o||Pr(u,t,o,d,s);}let l=Object.entries(u).sort((a,o)=>o[0].length-a[0].length);for(let[a,o]of l)i=dr(i,a,o);return t.urlRewriteMode!=="absolute"&&(i=wr(i,t,s)),i=hr(i,l.map(([,a])=>a)),pr(t,i,s)&&(i=gr(i)),i}async function Sr(e,t,r,s){let n=f.join(e.outputDir,s),i=f.extname(n).toLowerCase(),u;try{u=await S.readFile(n,"utf8");}catch(o){if(o.code!=="ENOENT")throw o;return {changed:false}}let a=Te(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await se(n,a,"utf8"),{changed:true}):{changed:false}}async function br(e,t){return await new Promise((r,s)=>{let n=a=>{if(l(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{l(),s(a);},u=a=>{l(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function st(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(c=>f.resolve(c)).filter(c=>c.startsWith(f.resolve(e.outputDir))).map(c=>f.relative(e.outputDir,c).replace(/\\/g,"/")).filter(c=>c.length>0):await rr(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(c=>c!=="asset-map.json"&&z(c)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(u,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:l});let a=0,o=0,d=null;if(l===1){for(let c of n){let g=await Sr(e,t,r.previousAssetMap??{},c);g.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:c,changed:g.changed});}return i}let p=Array.from({length:l},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!d;){let g=a;if(g>=n.length)return;a+=1;let m=n[g],w=await br(c,m);w.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:m,changed:w.changed});}}catch(g){throw d=g instanceof Error?g:new Error(String(g)),d}finally{await c.terminate().catch(()=>{});}});return await Promise.all(p),i}var _={error:0,warn:1,info:2,debug:3},nt=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",fe=nt?f.join(nt,"current-progress.json"):"",it=Promise.resolve();function ot(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function yr(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function vr(e,t,r){fe&&(it=it.then(async()=>{let s=new Date,n=s.toISOString();await S.mkdir(f.dirname(fe),{recursive:true});let i=await S.readFile(fe,"utf8").then(w=>JSON.parse(w)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},l=ot(e),a=i?.currentStep||ot(i?.source||""),o=i?.startedAt||n,d={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==l&&i?.stepStartedAt){let w=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));d[a]=(d[a]??0)+w,p=n;}let c=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(o).getTime())/1e3)),m={checkedAt:n,source:e,message:t,details:yr(u,r??{}),startedAt:o,currentStep:l,stepStartedAt:p,stepElapsedSec:c,totalElapsedSec:g,stepDurationsSec:d};await S.writeFile(fe,JSON.stringify(m,null,2),"utf8");}).catch(()=>{}));}async function Rr(e,t){await S.mkdir(f.dirname(e),{recursive:true}),await S.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function Cr(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function kr(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:Cr(e,r),level:e,message:t,details:r??{}}}var ne=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await S.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Rr(t,r));}updateCurrentEvent(t,r,s){let n=kr(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(_[this.level]>=_.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}debug(t,r){_[this.level]<_.debug||this.push("debug",t,r);}progress(t,r){vr("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),_[this.level]>=_.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),_[this.level]>=_.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Mr=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],St=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),bt=new Set(["",".html",".htm"]),Lr=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Ir=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function $r(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let l=e[u];if(l==="--retry-timeouts")r="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(l.startsWith("--crawl-mode="))s=l.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(l==="--incremental")s="incremental",n=true;else if(l==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(l.startsWith("--url="))r="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Oe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Dr(){let e=Oe();return e?f.resolve(e,".."):""}function Or(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Fr(e){let t=e.trim(),r=[{alias:"@storage-root",root:Dr(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Oe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Or(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function yt(e){let t=Oe();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function vt(){return {schemaVersion:3,updatedAt:"",pages:{},assets:{}}}function at(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function jr(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true||!t}}function Ie(e){let t=e?.["content-type"]?.trim();return t||null}function Er(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function q(e,t,r){let s=ee(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function ut(e,t,r){let s=ee(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function oe(e,t){e.rewriteTargets.add(f.resolve(t));}async function Ur(e){try{let t=await S.readFile(yt(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3)&&r.assets&&typeof r.assets=="object")return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,at(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,jr(n)]))};if(r.schemaVersion===1)return {schemaVersion:3,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,at(n)])),assets:{}}}}catch{}return vt()}function $e(e){return JSON.parse(JSON.stringify(e))}async function qr(e,t){let r=yt(e);await S.mkdir(f.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function K(e,t){if(!t.trackManifest)return;let r=async()=>{let s=$e(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await qr(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function ie(e){if(!e)return null;let t=e.trim();return t||null}function Qr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function Nr(e,t,r,s){let n=[...new Set(r.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!l.ok())return s.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:u,status:l.status()}),new Map;let a=await l.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let o=new Map;for(let d of a.items)!d||typeof d.url!="string"||o.set(d.url,d);return o}catch(l){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(l)}),new Map}}async function Ae(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await Nr(e,t,[s],n)).get(s)??null;return Ee(e,s)&&(!u||u.supported!==true||!u.token)&&(u=await Br(e,t,s,n)),u=_r(e,u),r.changeTokenCache.set(s,u),u}function Wr(e){return `subscriber=${e.wpsuite?.siteSettings?.subscriber===true?"1":"0"}`}function _r(e,t){if(!t?.supported||!t.token)return t;let r=Wr(e);return {...t,token:re(`${t.token}
|
|
7
|
+
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function Br(e,t,r,s){try{let n=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(n.status()!==404)return s.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:n.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let i=await n.text().catch(()=>""),u=re(`${r}
|
|
8
8
|
404
|
|
9
|
-
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Ur(e,t,r,s,n,i){let u=$e(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Ce(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Ce(e,t,s,n,i)};let a=await Ce(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await qr(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=re(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function qr(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Qr(e,t,r,s,n){let i=st(r.previousManifest),u=0;for(let[c,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(c)){rt(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${c}`,{url:c,outputPath:g.outputPath,error:String(m)});}delete r.manifest.pages[c],u++;}let l=st(r.manifest,r.runId),a=new Set,o=new Set;for(let c of l){for(let f of ne(e,c))a.add(f);let g=r.manifest.assets[c]?.outputPath;g&&o.add(g);let m=ie(e,t.assetMap,c);m&&o.add(m);}for(let c of Object.keys(r.manifest.assets))l.has(c)||delete r.manifest.assets[c];let d=0,p=0;for(let c of i){let g=new Set,m=r.previousManifest.assets[c]?.outputPath;m&&g.add(m);let f=ie(e,t.assetMap,c);if(!l.has(c)&&f)g.add(f);else if(!l.has(c)){let P=De(e,c);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){rt(n,e,P);try{await S.unlink(P),d++;}catch(C){(C.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${c}`,{url:c,outputPath:P,error:String(C)});}}for(let P of ne(e,c))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||d>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${d} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:d,removedAssetMappings:p,phase:"incremental-cleanup"});}function st(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function nt(e,t,r){for(let s of t)for(let n of ne(e,s))if(r.has(n))return true;return false}function Nr(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(h.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(h.resolve(l.outputPath));let u=Dr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))nt(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&nt(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of ne(e,l))if(u.has(o)){Q(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function Wr(e,t){let r=t?new Set(Array.from(t,n=>h.resolve(n))):null,s=n=>r===null||r.has(h.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function _r(e,t,r){let s=new Set,n=i=>{if(!i)return;let u=Y(e.outputDir,i);u&&s.add(u);};for(let i of t.donePages)n(B(e,i));for(let i of Object.values(r.manifest.pages))i.lastSeenRunId===r.runId&&n(i.outputPath);for(let i of t.doneAssets)n(ie(e,t.assetMap,i));for(let i of Object.values(r.manifest.assets))i.lastSeenRunId===r.runId&&n(i.outputPath);return s}async function Br(e,t,r,s,n){let i=[...s.deletedFiles].sort(),u=_r(e,t,r),l=0;for(let d of [...u].sort()){try{if((await S.stat(h.join(e.outputDir,d))).isFile())continue}catch{}l++,n.warn(`Active output is missing after crawl: ${d}`,{phase:"deploy-plan",file:d,outputPath:h.join(e.outputDir,d)});}if(l>0&&n.warn(`Detected ${l} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:l}),i.length===0)return [];let a=[],o=0;for(let d of i){if(!u.has(d)){a.push(d);continue}o++;let p=h.join(e.outputDir,d),c;try{c=(await S.stat(p)).isFile();}catch{c=false;}n.warn(`Deploy plan protected active output from deletion: ${d}`,{phase:"deploy-plan",file:d,outputPath:p,exists:c});}return o>0&&n.info(`Deploy plan skipped ${o} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:o,deletedFiles:a.length}),a}function ne(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Hr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function wt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function zr(e,t,r){for(let s of Hr(e,r)){let n=t[s];if(!n)continue;let i=wt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Jr(e,t,r){for(let s of ne(e,r)){let n=t[s];if(!n)continue;let i=wt(e,n);if(i)return i}return null}function ie(e,t,r){let s=Jr(e,t,r);return s||De(e,r).originalFilePath}function Gr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function it(e){let t=Gr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Vr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(B(e,s)));for(let s of t.doneAssets){let n=ie(e,t.assetMap,s);!n||!H(n)||r.add(h.resolve(n));}return [...r]}var he="re:",pe=new Map;function Kr(e){if(!e.startsWith(he))return null;let t=pe.get(e);if(t!==void 0)return t||null;let r=e.slice(he.length).trim();if(!r)return pe.set(e,false),null;try{let s=new RegExp(r);return pe.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),pe.set(e,false),null}}function Le(e,t){let r=Kr(t);return r?r.test(e):t.startsWith(he)?false:e.startsWith(t)}function Yr(e,t){return t.find(r=>Le(e,r))}function Pt(e,t){let r=Yr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(he)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Zr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Xr(e,t){return Zr(e).has(t.hostname)}function es(e){let t=U(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function bt(e){let t=U(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function ts(e){let t=U(e).replace(/\\\//g,"/").trim(),r=h.extname(t).toLowerCase();return gt.has(r)||ht.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function V(e){return Cr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=es(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(bt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(ts(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Xr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=Pt(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function St(e){return gt.has(h.extname(e.pathname).toLowerCase())}function z(e){return V(e)?false:ht.has(h.extname(e.pathname).toLowerCase())}function ue(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function x(e,t){return !!(ue(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>Le(t.pathname,r))||St(t)||!z(t)&&!V(t))}function B(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function rs(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&h.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function $e(e,t){let r=rs(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function De(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Se(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${X(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Se(e.targetOrigin,p)}}function ss(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function yt(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function vt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function ns(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Rt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function is(e,t){let r=Rt(t),s=ns(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function os(e,t){if(!t)return;let r=Rt(t);if(r===null)return;let s=await vt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function ot(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Rr.has(r)||vr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function ke(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function as(e){return X(e.toString("base64"),40)}function us(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function ls(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Me(e,t,r,s,n,i,u){let l=De(e,r),a=as(s),o=l.originalFilePath,d=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&us(n,l.originalFilePath,a)){let m=zr(e,t,r);m?(o=m.outputPath,d=m.publicUrl):(o=l.hashedFilePath,d=l.hashedPublicUrl);}ss(e,t,r,d,l.originalPublicUrl);let p=H(o)?Buffer.from(Re(s.toString("utf8"),e,t,o),"utf8"):s,c=await yt(o),g=false;return (c===null||!c.equals(p))&&(await ee(o,p),g=true),i?.debug(`${g?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:o,bytes:p.length,wroteFile:g}),ls(n,l.originalFilePath,a),await os(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:d,wroteFile:g}}function oe(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let c=new URL(p);return ue(c)?{discoveredFrom:p,discoveredFromType:"sitemap"}:x(e,c)?{discoveredFrom:p,discoveredFromType:"asset"}:z(c)&&!V(c)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!z(l)||St(l)){if(x(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),M(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),V(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:d}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:d,outputPath:B(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function M(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!x(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Oe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!ue(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function cs(e){let t=U(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:re(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function ds(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function ps(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function gs(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function hs(e,t){if(t==="dom-style")return true;let r=(()=>{try{return h.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function xe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Ct(e,t,r,s){let n=[];await S.mkdir(t,{recursive:true});let i=await S.readdir(e,{withFileTypes:true});for(let u of i){let l=h.join(e,u.name),a=h.join(t,u.name);if(u.isDirectory()){n.push(...await Ct(l,a,r,s));continue}await kt(l,a,r,s)&&n.push(a);}return n}async function fs(e,t){try{let[r,s]=await Promise.all([S.stat(e),S.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([S.readFile(e),S.readFile(t)]);return n.equals(i)}catch{return false}}async function kt(e,t,r,s){let n=h.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await fs(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await S.mkdir(h.dirname(t),{recursive:true}),await S.copyFile(e,t),true)}async function ms(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([l,a])=>({sourcePath:l.trim(),prefix:String(a||"").trim()}));if(s.length===0)return [];let n=0,i=[],u=new Set([...t.donePages].map(l=>h.resolve(B(e,l))));r.mark("copy-extra-paths");for(let l of s){let a=l.sourcePath,o=l.prefix;if(!a||!o){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:o});continue}let d=Ar(a);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:o});continue}let p=d.resolvedPath,c=xe(o);if(!c){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:o});continue}let g;try{g=await S.stat(p);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:p,prefixPath:o});continue}let m=h.resolve(e.outputDir,c);if(g.isDirectory()){let w=await Ct(p,m,u,r);i.push(...w),n+=w.length,w.length>0&&r.progress(`Copied static directory to export: ${p} -> /${c}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${c}`,copiedItems:n,changedFiles:w.length});continue}let f=h.basename(p),P=o.endsWith("/")?h.join(m,f):m;await kt(p,P,u,r)&&(i.push(P),n++,r.progress(`Copied static file to export: ${p} -> /${xe(o.endsWith("/")?`${o}${f}`:o)}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${xe(o.endsWith("/")?`${o}${f}`:o)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function ae(e,t,r,s,n){let i=new Set,u=U(r).replace(/\\\//g,"/"),l=n==="page-html"?gs(u):u,a=hs(t,n);if(a){for(let p of ps(l)){let c=T(e,p,t,s,`${n}:css-url`);c&&x(e,c)&&i.add(c.toString());}for(let p of l.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,p[1],t,s,`${n}:css-import`);c&&x(e,c)&&i.add(c.toString());}}if(a||n==="page-html")for(let p of l.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,p[1],t,s,`${n}:xml-stylesheet`);c&&x(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",d=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let p of u.matchAll(d)){let c=p[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${p[0]}`:p[0];if(bt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let g=T(e,c,t,s,`${n}:serialized-url`);g&&x(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function ws(e,t,r,s){let n=new Set,i=U(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(Pt(e,a)||z(a)&&!V(a)&&!x(e,a)&&n.add(a.toString()));}return [...n]}async function at(e,t,r,s,n){for(;r.sitemapQueue.length>0;){let i=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(i),!r.doneSitemaps.has(i)){r.doneSitemaps.add(i),s.sitemap(`Fetching sitemap ${i}`,{url:i});try{let u=await t.get(i,{timeout:6e4});if(!u.ok()){s.skip("sitemap",i,u.status());continue}let l=await u.body(),a=u.url()||i,o=await Me(e,r.assetMap,a,l,r,s);Q(n,e,o.outputPath),se(n,o.outputPath);let d=l.toString("utf8");for(let p of ae(e,a,d,s,"sitemap"))M(e,r,p,s,a);for(let p of cs(d)){let c=T(e,p.loc,a,s,"sitemap-loc");c&&(ue(c)?Oe(e,r,c.toString(),s,a):x(e,c)?M(e,r,c.toString(),s,a):z(c)?(p.lastmod&&(r.sitemapLastmodByPage[c.toString()]=p.lastmod),oe(e,r,c.toString(),s,a)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",a));}}catch(u){s.error(`Failed sitemap ${i}`,{url:i,error:String(u)});}}}}async function Ps(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!x(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,d=(p,c={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},delete i.manifest.pages[a],r.doneAssets.add(a);for(let g of o.discoveredAssets||[])M(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...c,phase:"download-assets"}),ke(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=ie(e,r.assetMap,a),c=p?await vt(p):null;if(p&&c)try{let v=await t.head(a,{timeout:6e4}),R=v.headers();if(v.ok()&&is(c,R)){let I=await yt(p);if(I!==null){let j=v.url()||a,L=await Me(e,r.assetMap,j,I,r,n,R);L.wroteFile&&(Q(u,e,L.outputPath),H(L.outputPath)&&se(u,L.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:R["last-modified"]||"",contentLength:R["content-length"]||"",phase:"download-assets"}),ke(r,n);let b=ot(l,R),k=o?.discoveredAssets?[...o.discoveredAssets]:[];if(b||o?.isText){let N=I.toString("utf8"),K=ae(e,j,N,n,"asset:cached");(K.length>0||!o)&&(k=K);for(let At of k)M(e,r,At,n,j);}if(i.trackManifest){let N=b||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:L.outputPath,isText:N,contentType:Te(R)||o?.contentType||null,discoveredAssets:k,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!N},delete i.manifest.pages[a],i.enabled&&N&&await G(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(d(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let m=await g.body(),f=g.url()||a,P=g.headers(),C=await Me(e,r.assetMap,f,m,r,n,P);C.wroteFile&&(Q(u,e,C.outputPath),H(C.outputPath)&&se(u,C.outputPath)),r.doneAssets.add(a),ke(r,n);let w=ot(l,P),y=[];if(w){let v=m.toString("utf8");y=ae(e,f,v,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let R of y)M(e,r,R,n,f);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:C.outputPath,isText:w,contentType:Te(P),discoveredAssets:y,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!w},delete i.manifest.pages[a],i.enabled&&w&&await G(e,i));}catch(p){if(d("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function bs(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Ps(e,t,r,a,s,n,i));}});await Promise.all(l);}function Ss(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function q(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ge(e){return Math.max(15e3,Math.min(6e4,q(e)))}function ys(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function je(e){return e.stats.pagesCompleted}function xt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function vs(e,t,r){let s=xt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function A(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${Ss(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function Rs(e,t){let r=ys(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let d=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,c=Math.max(d,p);i>=c+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function Cs(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Rs(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function ks(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function xs(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Le(t,r))}async function Ts(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(c=>{for(let m of o){let f=c.getAttribute(m);f&&a.add(f);}for(let m of p){let f=c.getAttribute(m);f&&f.split(",").forEach(P=>{let C=P.trim().split(/\s+/)[0];C&&a.add(C);});}let g=c.getAttribute("style");g&&d.add(g);}),{attrs:[...a],styles:[...d]}});for(let a of l.attrs)for(let o of ds(a)){let d=T(e,o,r,s,"dom-attr");d&&x(e,d)&&i.add(d.toString());}for(let a of l.styles)for(let o of ae(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of ae(e,r,u,s,"page-html"))i.add(l);return [...i]}async function ut(e,t,r,s,n){let i=B(e,t);await ee(i,r,"utf8");let u=await S.stat(i);n?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:i,bytes:u.size}),s.stats.pagesSaved++;let l=Math.min(s.stats.pagesQueued,je(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${l}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:l,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function As(e){let t=je(e);return t%5===0||t===1||t===e.stats.pagesQueued}function Ms(e,t){let r=je(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Is(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Ls(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Is(s),s}async function $s(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=xt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let d=null,p=null;try{let c=new URL(o),g=$e(e,o),m=xs(e,c.pathname);if(!z(c)||x(e,c)||V(c)){if(x(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),M(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let f=await Ur(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let b=i.manifest.pages[o];if(f.action==="reuse"&&b){for(let k of b.discoveredAssets)M(e,t,k,r,o);for(let k of b.discoveredPages)oe(e,t,k,r,o);b.lastSeenRunId=i.runId,b.sitemapLastmod=re(t.sitemapLastmodByPage[o])??b.sitemapLastmod,f.changeToken?.supported&&(b.changeToken=f.changeToken.token,b.tokenSource=f.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:f.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}d=await A(`create page context ${o}`,q(e),()=>Ls(l,e,m)),p=await A(`create page ${o}`,q(e),()=>d.newPage());let P=!1,C=new Set;await p.route("**/*",async b=>{let k=b.request();if(k.isNavigationRequest()&&k.frame()===p.mainFrame())try{let N=new URL(k.url()),K=new URL(o);if(N.hash="",K.hash="",N.href===K.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:k.url()}),await b.abort("aborted");return}P=!0;}}catch{}await b.continue();}),p.on("response",b=>{try{let k=T(e,b.url(),e.sourceOrigin,void 0,"network-response");k&&x(e,k)&&(C.add(k.toString()),M(e,t,k.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let w=null,y=null;try{if(w=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),w&&w.ok())try{y=await A(`read response body ${o}`,q(e),()=>w.text());}catch{y=null;}}catch(b){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(b)});}if(g){let b=w?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:b,outputPath:B(e,o)});}else if(w&&!w.ok()){r.skip("page",o,w.status(),t.pageDiscoverySources.get(o)),await p.close();continue}m?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await A(`wait for rendered page ${o}`,q(e),()=>ks(e,p))):await A(`wait for rendered page ${o}`,q(e),()=>Cs(e,p)),t.stats.pagesRendered++;let v=await A(`extract rendered assets ${o}`,q(e),()=>Ts(e,p,o,r,!m));for(let b of v)M(e,t,b,r,o);let R=[...new Set([...v,...C])],I=y??await A(`capture rendered html ${o}`,q(e),()=>p.content()),j=s!=="single-url"&&!g?ws(e,o,I,r):[];if(s!=="single-url"&&!g)for(let b of j)oe(e,t,b,r,o);await ut(e,o,I,t,r);let L=B(e,o);if(Q(u,e,L),se(u,L),i.trackManifest){let b=w?.headers();i.manifest.pages[o]={url:o,outputPath:L,contentType:Te(b),changeToken:f.changeToken?.supported?f.changeToken.token:null,tokenSource:f.changeToken?.supported?f.changeToken.tokenSource??null:null,sitemapLastmod:re(t.sitemapLastmodByPage[o])??null,discoveredPages:j,discoveredAssets:R,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}}catch(c){try{let g=p?await A(`capture partial html ${o}`,q(e),()=>p.content()):"";if(g&&g.trim()){await ut(e,o,g,t,r);let m=B(e,o);if(Q(u,e,m),se(u,m),i.trackManifest){let f=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:m,contentType:f?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:re(t.sitemapLastmodByPage[o])??f?.sitemapLastmod??null,discoveredPages:f?.discoveredPages?[...f.discoveredPages]:[],discoveredAssets:f?.discoveredAssets?[...f.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await G(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(c)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(c)});}finally{t.stats.pagesCompleted++,As(t)&&Ms(t,r),p&&await A(`close page ${o}`,ge(e),()=>p.close()).catch(c=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(c)});}),d&&await A(`close page context ${o}`,ge(e),()=>d.close()).catch(c=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(c)});});}}}finally{await A("close worker browser",ge(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),vs(l,r,"browser.close timeout")));}}async function Ds(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function lt(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Os(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function js(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function Tt(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function ct(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Fs(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>ct(s)-ct(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function dt(e){let t=await Tt(h.join(e,"job.json"));for(let r of Fs(e,t)){let s=await Ds(r);if(s.length>0)return s}return []}async function Es(e){let t=h.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let a=await Tt(h.join(u,"job.json"));if(!Os(a))continue;let o=js(a);o>=n&&(n=o,s=u);}return s}async function Us(e){let t=await Es(e),r=t!==""?await dt(t):await dt(e.logDir),s=new Set;for(let n of r)if(lt(n.error)||lt(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function qs(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Qs(e){try{let t=await S.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Ns(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function pt(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(ue(i)?Oe(e,t,i.toString(),s,n):x(e,i)?M(e,t,i.toString(),s,n):z(i)?oe(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Ws(){let e=await be(),t=kr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Us(e)),t.mode==="single-url"&&(r=await qs(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=(t.mode==="full"||t.mode==="single-url")&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await Qs(e):{},a=n||t.mode==="single-url"?await Lr(e):mt(),o={enabled:n,trackManifest:i,manifest:Ae(a),previousManifest:Ae(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await Ns(e.logDir);let d={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Ir();u||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let c=new te(e.logDir,e.logLevel);if(s&&!n&&c.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)c.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(c.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Oe(e,d,v,c)),await at(e,w,d,c,p),e.seedPaths.forEach(v=>pt(e,d,v,c,"seed-path")),e.generated404RequestPath&&oe(e,d,e.generated404RequestPath,c,"generated-404-request-path")):(r.forEach(v=>pt(e,d,v,c,"cli")),await at(e,w,d,c,p)),c.endMark("discovery",{pages:d.pageQueue.length,assets:d.assetQueue.length,sitemaps:d.sitemapQueue.length}),c.summary(`Queued ${d.pageQueue.length} pages, ${d.assetQueue.length} assets, ${d.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:d.pageQueue.length,queuedAssets:d.assetQueue.length,queuedSitemaps:d.sitemapQueue.length}),t.mode==="retry-timeouts"&&d.pageQueue.length===0&&d.assetQueue.length===0&&d.sitemapQueue.length===0){c.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await c.flush();return}c.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>$s(e,d,c,t.mode,w,o,p));if(await Promise.all(y),c.endMark("render-pages",{pages:d.stats.pagesCompleted}),c.mark("download-assets"),c.progress("Asset download phase started.",{phase:"download-assets",doneAssets:d.doneAssets.size,assetsQueued:d.stats.assetsQueued,assetQueue:d.assetQueue.length}),await bs(e,w,d,c,o,p),c.endMark("download-assets",{assets:d.doneAssets.size}),o.enabled&&e.maxPages===0&&await Qr(e,d,o,c,p),t.mode==="full"){let v=await ms(e,d,c);for(let R of v)Q(p,e,R);}}finally{await A("dispose crawl request context",ge(e),()=>w.dispose()).catch(y=>{c.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&await G(e,o),await S.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(d.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(d.assetMap,null,2)&&Q(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?Vr(e,d):o.enabled&&!t.resumeRewrite?Nr(e,o,p,l,d.assetMap):void 0;c.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";c.progress(m,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await it(m);let f=Date.now(),P=await Ye(e,d.assetMap,{files:g,previousAssetMap:l,onStart:({totalFiles:w,workerCount:y})=>{c.info(`Rewrite phase scanning ${w} text files.`,{phase:"rewrite-text",totalFiles:w,workerCount:y});},onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:R,changed:I})=>{I&&Q(p,e,h.join(e.outputDir,R)),c.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R}),c.debug(`${I?"Rewrote":"Checked unchanged"} text file ${w}/${y}: ${R}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R,changed:I});let j=Date.now();if(!(w===1||w===y||w%50===0||j-f>=5e3))return;f=j;let b=`Rewriting text files: ${w}/${y}`;c.progress(b,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R}),await it(b);}});o.trackManifest&&(Wr(o,g),await G(e,o)),c.endMark("rewrite-text",{changedTextFiles:P});let C=o.enabled&&!t.resumeRewrite?await Br(e,d,o,p,c):[];await Ue(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?C:[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>Y(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),c.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${d.stats.pagesRendered} pages, processed ${d.doneSitemaps.size} sitemaps, downloaded ${d.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...d.stats,pagesRendered:d.stats.pagesRendered,donePages:d.stats.pagesCompleted,claimedPages:d.donePages.size,doneSitemaps:d.doneSitemaps.size,doneAssets:d.doneAssets.size,changedTextFiles:P}),await c.flush();}Ws().catch(async e=>{console.error(e);try{let r=(await be().catch(()=>null))?.logDir??"logs",s=new te(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
9
|
+
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Hr(e,t,r,s,n,i){let u=Ee(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Ae(e,t,s,n,i):null};let l=s.manifest.pages[n];if(!l)return {action:"render",changeToken:await Ae(e,t,s,n,i)};let a=await Ae(e,t,s,n,i);if(a?.supported&&a.token)return l.changeToken===a.token?u&&!await zr(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=ie(r.sitemapLastmodByPage[n]);return o&&l.sitemapLastmod&&l.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function zr(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Jr(e,t,r,s,n){let i=lt(r.previousManifest,void 0,s,"previous-manifest"),u=0;for(let[c,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(c)){ut(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${c}`,{url:c,outputPath:g.outputPath,error:String(m)});}delete r.manifest.pages[c],u++;}let l=lt(r.manifest,r.runId,s,"active-manifest"),a=new Set,o=new Set;for(let c of l){for(let w of ae(e,c))a.add(w);let g=r.manifest.assets[c]?.outputPath;g&&o.add(g);let m=ue(e,t.assetMap,c);m&&o.add(m);}for(let c of Object.keys(r.manifest.assets))l.has(c)||delete r.manifest.assets[c];let d=0,p=0;for(let c of i){let g=new Set,m=r.previousManifest.assets[c]?.outputPath;m&&g.add(m);let w=ue(e,t.assetMap,c);if(!l.has(c)&&w)g.add(w);else if(!l.has(c)){let h=cs(e,c,s,"incremental-cleanup");h&&(g.add(h.originalFilePath),h.hashedFilePath&&g.add(h.hashedFilePath));}for(let h of g)if(!o.has(h)){ut(n,e,h);try{await S.unlink(h),d++;}catch(C){(C.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${c}`,{url:c,outputPath:h,error:String(C)});}}for(let h of ae(e,c))a.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],p++);}(u>0||d>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${d} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:d,removedAssetMappings:p,phase:"incremental-cleanup"});}function lt(e,t,r,s){let n=new Set,i=u=>{let l=u.trim();if(l){if(r){let a=J(l,r,"asset",s??"manifest");if(!a)return;n.add(a.toString());return}n.add(l);}};for(let u of Object.values(e.pages))for(let l of u.discoveredAssets||[])i(l);for(let[u,l]of Object.entries(e.assets||{}))t&&l.lastSeenRunId!==t||i(u);return n}function ct(e,t,r){for(let s of t)for(let n of ae(e,s))if(r.has(n))return true;return false}function Gr(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let l of Object.values(t.manifest.pages))l.rewriteComplete||i.add(f.resolve(l.outputPath));for(let l of Object.values(t.manifest.assets))l.isText&&!l.rewriteComplete&&i.add(f.resolve(l.outputPath));let u=Qr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))ct(e,l.discoveredAssets,u)&&i.add(f.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&ct(e,a.discoveredAssets,u)&&i.add(f.resolve(a.outputPath));for(let o of ae(e,l))if(u.has(o)){q(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function Vr(e,t){let r=t?new Set(Array.from(t,n=>f.resolve(n))):null,s=n=>r===null||r.has(f.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function Kr(e,t,r){let s=new Set,n=i=>{if(!i)return;let u=ee(e.outputDir,i);u&&s.add(u);};for(let i of t.donePages)n(B(e,i));for(let i of Object.values(r.manifest.pages))i.lastSeenRunId===r.runId&&n(i.outputPath);for(let i of t.doneAssets)n(ue(e,t.assetMap,i));for(let i of Object.values(r.manifest.assets))i.lastSeenRunId===r.runId&&n(i.outputPath);return s}async function Yr(e,t,r,s,n){let i=[...s.deletedFiles].sort(),u=Kr(e,t,r),l=0;for(let d of [...u].sort()){try{if((await S.stat(f.join(e.outputDir,d))).isFile())continue}catch{}l++,n.warn(`Active output is missing after crawl: ${d}`,{phase:"deploy-plan",file:d,outputPath:f.join(e.outputDir,d)});}if(l>0&&n.warn(`Detected ${l} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:l}),i.length===0)return [];let a=[],o=0;for(let d of i){if(!u.has(d)){a.push(d);continue}o++;let p=f.join(e.outputDir,d),c;try{c=(await S.stat(p)).isFile();}catch{c=false;}n.warn(`Deploy plan protected active output from deletion: ${d}`,{phase:"deploy-plan",file:d,outputPath:p,exists:c});}return o>0&&n.info(`Deploy plan skipped ${o} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:o,deletedFiles:a.length}),a}function ae(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Zr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function Rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,je(s.pathname))}catch{return null}}function Xr(e,t,r){for(let s of Zr(e,r)){let n=t[s];if(!n)continue;let i=Rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function es(e,t,r){for(let s of ae(e,r)){let n=t[s];if(!n)continue;let i=Rt(e,n);if(i)return i}return null}function ue(e,t,r){let s=es(e,t,r);if(s)return s;try{return Ue(e,r).originalFilePath}catch{return null}}function ts(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function dt(e){let t=ts();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function rs(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(B(e,s)));for(let s of t.doneAssets){let n=ue(e,t.assetMap,s);!n||!z(n)||r.add(f.resolve(n));}return [...r]}var we="re:",he=new Map;function ss(e){if(!e.startsWith(we))return null;let t=he.get(e);if(t!==void 0)return t||null;let r=e.slice(we.length).trim();if(!r)return he.set(e,false),null;try{let s=new RegExp(r);return he.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),he.set(e,false),null}}function Fe(e,t){let r=ss(t);return r?r.test(e):t.startsWith(we)?false:e.startsWith(t)}function ns(e,t){return t.find(r=>Fe(e,r))}function Ct(e,t){let r=ns(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(we)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function is(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function os(e,t){return is(e).has(t.hostname)}function as(e){let t=E(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function kt(e){let t=E(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function us(e){let t=E(e).replace(/\\\//g,"/").trim(),r=f.extname(t).toLowerCase();return St.has(r)||bt.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function Y(e){return Ir.some(t=>e.pathname.startsWith(t))}function A(e,t,r=e.sourceOrigin,s,n="url"){let i=as(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(kt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(us(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(xt(u))return s?.reject(n,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!os(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=Ct(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function xt(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function J(e,t,r,s){try{let n=new URL(e);return xt(n)?(t.reject(r,e,"malformed URL encoding",s),null):n.protocol!=="http:"&&n.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${n.protocol}`,s),null):n}catch{return t.reject(r,e,"invalid URL",s),null}}function Tt(e){return St.has(f.extname(e.pathname).toLowerCase())}function G(e){return Y(e)?false:bt.has(f.extname(e.pathname).toLowerCase())}function de(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function k(e,t){return !!(de(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>Fe(t.pathname,r))||Tt(t)||!G(t)&&!Y(t))}function B(e,t){let r=new URL(t),s=je(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function je(e){try{return decodeURIComponent(e)}catch{return e}}function ls(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&f.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Ee(e,t){let r=ls(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Ue(e,t){let r=new URL(t),s=je(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),l=Ce(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=f.extname(s),p=`${o?s.slice(0,-o.length):s}.${re(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:f.join(e.outputDir,p),hashedPublicUrl:Ce(e.targetOrigin,p)}}function cs(e,t,r,s){let n=J(t,r,"asset",s);return n?Ue(e,n.toString()):null}function ds(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function At(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Mt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function ps(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Lt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function gs(e,t){let r=Lt(t),s=ps(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function fs(e,t){if(!t)return;let r=Lt(t);if(r===null)return;let s=await Mt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function pt(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Lr.has(r)||Mr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Me(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function hs(e){return re(e.toString("base64"),40)}function ms(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function ws(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function De(e,t,r,s,n,i,u){let l=Ue(e,r),a=hs(s),o=l.originalFilePath,d=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&ms(n,l.originalFilePath,a)){let m=Xr(e,t,r);m?(o=m.outputPath,d=m.publicUrl):(o=l.hashedFilePath,d=l.hashedPublicUrl);}ds(e,t,r,d,l.originalPublicUrl);let p=z(o)?Buffer.from(Te(s.toString("utf8"),e,t,o),"utf8"):s,c=await At(o),g=false;return (c===null||!c.equals(p))&&(await se(o,p),g=true),i?.debug(`${g?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:o,bytes:p.length,wroteFile:g}),ws(n,l.originalFilePath,a),await fs(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:d,wroteFile:g}}function le(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let c=new URL(p);return de(c)?{discoveredFrom:p,discoveredFromType:"sitemap"}:k(e,c)?{discoveredFrom:p,discoveredFromType:"asset"}:G(c)&&!Y(c)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=A(e,r,u,s,"page");if(!l)return;if(!G(l)||Tt(l)){if(k(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),I(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),Y(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString(),{discoveredFrom:o,discoveredFromType:d}=i();o&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,o),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:d,outputPath:B(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function I(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=A(e,r,i,s,"asset");if(!u)return;if(!k(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function qe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=A(e,r,i,s,"sitemap");if(!u)return;if(!de(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Ps(e){let t=E(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:ie(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Ss(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function bs(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function ys(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function vs(e,t){if(t==="dom-style")return true;let r=(()=>{try{return f.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function Le(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function It(e,t,r,s){let n=[];await S.mkdir(t,{recursive:true});let i=await S.readdir(e,{withFileTypes:true});for(let u of i){let l=f.join(e,u.name),a=f.join(t,u.name);if(u.isDirectory()){n.push(...await It(l,a,r,s));continue}await $t(l,a,r,s)&&n.push(a);}return n}async function Rs(e,t){try{let[r,s]=await Promise.all([S.stat(e),S.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([S.readFile(e),S.readFile(t)]);return n.equals(i)}catch{return false}}async function $t(e,t,r,s){let n=f.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await Rs(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await S.mkdir(f.dirname(t),{recursive:true}),await S.copyFile(e,t),true)}async function Cs(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([l,a])=>({sourcePath:l.trim(),prefix:String(a||"").trim()}));if(s.length===0)return [];let n=0,i=[],u=new Set([...t.donePages].map(l=>f.resolve(B(e,l))));r.mark("copy-extra-paths");for(let l of s){let a=l.sourcePath,o=l.prefix;if(!a||!o){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:o});continue}let d=Fr(a);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:o});continue}let p=d.resolvedPath,c=Le(o);if(!c){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:o});continue}let g;try{g=await S.stat(p);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:p,prefixPath:o});continue}let m=f.resolve(e.outputDir,c);if(g.isDirectory()){let P=await It(p,m,u,r);i.push(...P),n+=P.length,P.length>0&&r.progress(`Copied static directory to export: ${p} -> /${c}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${c}`,copiedItems:n,changedFiles:P.length});continue}let w=f.basename(p),h=o.endsWith("/")?f.join(m,w):m;await $t(p,h,u,r)&&(i.push(h),n++,r.progress(`Copied static file to export: ${p} -> /${Le(o.endsWith("/")?`${o}${w}`:o)}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${Le(o.endsWith("/")?`${o}${w}`:o)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function ce(e,t,r,s,n){let i=new Set,u=E(r).replace(/\\\//g,"/"),l=n==="page-html"?ys(u):u,a=vs(t,n);if(a){for(let p of bs(l)){let c=A(e,p,t,s,`${n}:css-url`);c&&k(e,c)&&i.add(c.toString());}for(let p of l.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=A(e,p[1],t,s,`${n}:css-import`);c&&k(e,c)&&i.add(c.toString());}}if(a||n==="page-html")for(let p of l.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=A(e,p[1],t,s,`${n}:xml-stylesheet`);c&&k(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",d=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let p of u.matchAll(d)){let c=p[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${p[0]}`:p[0];if(kt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let g=A(e,c,t,s,`${n}:serialized-url`);g&&k(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function ks(e,t,r,s){let n=new Set,i=E(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=A(e,l,t,s,"page-link");a&&(Ct(e,a)||G(a)&&!Y(a)&&!k(e,a)&&n.add(a.toString()));}return [...n]}async function gt(e,t,r,s,n){for(;r.sitemapQueue.length>0;){let i=r.sitemapQueue.shift();r.queuedSitemaps.delete(i);let u=J(i,s,"sitemap","sitemap-queue");if(!u)continue;let l=u.toString();if(!r.doneSitemaps.has(l)){r.doneSitemaps.add(l),s.sitemap(`Fetching sitemap ${l}`,{url:l});try{let a=await t.get(l,{timeout:6e4});if(!a.ok()){s.skip("sitemap",l,a.status());continue}let o=await a.body(),d=a.url()||l,p=J(d,s,"sitemap-response",l);if(!p)continue;let c=p.toString(),g=await De(e,r.assetMap,c,o,r,s);q(n,e,g.outputPath),oe(n,g.outputPath);let m=o.toString("utf8");for(let w of ce(e,c,m,s,"sitemap"))I(e,r,w,s,c);for(let w of Ps(m)){let h=A(e,w.loc,c,s,"sitemap-loc");h&&(de(h)?qe(e,r,h.toString(),s,c):k(e,h)?I(e,r,h.toString(),s,c):G(h)?(w.lastmod&&(r.sitemapLastmodByPage[h.toString()]=w.lastmod),le(e,r,h.toString(),s,c)):s.reject("sitemap-loc",h.toString(),"not page/sitemap/asset-like",c));}}catch(a){s.error(`Failed sitemap ${l}`,{url:l,error:String(a)});}}}}async function xs(e,t,r,s,n,i,u){let l=A(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!k(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,d=(p,c={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},delete i.manifest.pages[a],r.doneAssets.add(a);for(let g of o.discoveredAssets||[])I(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...c,phase:"download-assets"}),Me(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=ue(e,r.assetMap,a),c=p?await Mt(p):null;if(p&&c)try{let L=await t.head(a,{timeout:6e4}),T=L.headers();if(L.ok()&&gs(c,T)){let H=await At(p);if(H!==null){let Q=L.url()||a,b=J(Q,n,"asset-response",a);if(!b)return;let R=b.toString(),N=await De(e,r.assetMap,R,H,r,n,T);N.wroteFile&&(q(u,e,N.outputPath),z(N.outputPath)&&oe(u,N.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:T["last-modified"]||"",contentLength:T["content-length"]||"",phase:"download-assets"}),Me(r,n);let Z=pt(l,T),Pe=o?.discoveredAssets?[...o.discoveredAssets]:[];if(Z||o?.isText){let X=H.toString("utf8"),Ne=ce(e,R,X,n,"asset:cached");(Ne.length>0||!o)&&(Pe=Ne);for(let Ft of Pe)I(e,r,Ft,n,R);}if(i.trackManifest){let X=Z||!!o?.isText;i.manifest.assets[a]={url:a,outputPath:N.outputPath,isText:X,contentType:Ie(T)||o?.contentType||null,discoveredAssets:Pe,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!X},delete i.manifest.pages[a],i.enabled&&X&&await K(e,i);}return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(d(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let m=await g.body(),w=g.url()||a,h=J(w,n,"asset-response",a);if(!h)return;let C=h.toString(),P=g.headers(),y=await De(e,r.assetMap,C,m,r,n,P);y.wroteFile&&(q(u,e,y.outputPath),z(y.outputPath)&&oe(u,y.outputPath)),r.doneAssets.add(a),Me(r,n);let v=pt(l,P),x=[];if(v){let L=m.toString("utf8");x=ce(e,C,L,n,`asset:${f.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let T of x)I(e,r,T,n,C);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:y.outputPath,isText:v,contentType:Ie(P),discoveredAssets:x,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!v},delete i.manifest.pages[a],i.enabled&&v&&await K(e,i));}catch(p){if(d("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function Ts(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await xs(e,t,r,a,s,n,i));}});await Promise.all(l);}function As(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function U(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function me(e){return Math.max(15e3,Math.min(6e4,U(e)))}function Ms(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function Qe(e){return e.stats.pagesCompleted}function Dt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function Ls(e,t,r){let s=Dt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function M(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${As(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function Is(e,t){let r=Ms(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let d=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,c=Math.max(d,p);i>=c+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function $s(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Is(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ds(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Os(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Fe(t,r))}async function Fs(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(c=>{for(let m of o){let w=c.getAttribute(m);w&&a.add(w);}for(let m of p){let w=c.getAttribute(m);w&&w.split(",").forEach(h=>{let C=h.trim().split(/\s+/)[0];C&&a.add(C);});}let g=c.getAttribute("style");g&&d.add(g);}),{attrs:[...a],styles:[...d]}});for(let a of l.attrs)for(let o of Ss(a)){let d=A(e,o,r,s,"dom-attr");d&&k(e,d)&&i.add(d.toString());}for(let a of l.styles)for(let o of ce(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of ce(e,r,u,s,"page-html"))i.add(l);return [...i]}async function ft(e,t,r,s,n){let i=B(e,t);await se(i,r,"utf8");let u=await S.stat(i);n?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:i,bytes:u.size}),s.stats.pagesSaved++;let l=Math.min(s.stats.pagesQueued,Qe(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${l}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:l,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function js(e){let t=Qe(e);return t%5===0||t===1||t===e.stats.pagesQueued}function Es(e,t){let r=Qe(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Us(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function qs(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Us(s),s}async function Qs(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=Dt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;t.queuedPages.delete(o);let d=null,p=null;try{let c=J(o,r,"page","page-queue");if(!c)continue;let g=c.toString();if(o=g,t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let m=Ee(e,o),w=Os(e,c.pathname);if(!G(c)||k(e,c)||Y(c)){if(k(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),I(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let h=await Hr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let b=i.manifest.pages[o];if(h.action==="reuse"&&b){for(let R of b.discoveredAssets)I(e,t,R,r,o);for(let R of b.discoveredPages)le(e,t,R,r,o);b.lastSeenRunId=i.runId,b.sitemapLastmod=ie(t.sitemapLastmodByPage[o])??b.sitemapLastmod,h.changeToken?.supported&&(b.changeToken=h.changeToken.token,b.tokenSource=h.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:h.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}d=await M(`create page context ${o}`,U(e),()=>qs(l,e,w)),p=await M(`create page ${o}`,U(e),()=>d.newPage());let C=!1,P=new Set;await p.route("**/*",async b=>{let R=b.request();if(R.isNavigationRequest()&&R.frame()===p.mainFrame())try{let N=new URL(R.url()),Z=new URL(g);if(N.hash="",Z.hash="",N.href===Z.href){if(C){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:R.url()}),await b.abort("aborted");return}C=!0;}}catch{}await b.continue();}),p.on("response",b=>{try{let R=A(e,b.url(),e.sourceOrigin,r,"network-response");R&&k(e,R)&&(P.add(R.toString()),I(e,t,R.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await M(`read response body ${o}`,U(e),()=>y.text());}catch{v=null;}}catch(b){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(b)});}if(m){let b=y?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:b,outputPath:B(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status(),t.pageDiscoverySources.get(o)),await p.close();continue}w?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await M(`wait for rendered page ${o}`,U(e),()=>Ds(e,p))):await M(`wait for rendered page ${o}`,U(e),()=>$s(e,p)),t.stats.pagesRendered++;let x=await M(`extract rendered assets ${o}`,U(e),()=>Fs(e,p,g,r,!w));for(let b of x)I(e,t,b,r,o);let L=[...new Set([...x,...P])],T=v??await M(`capture rendered html ${o}`,U(e),()=>p.content()),H=s!=="single-url"&&!m?ks(e,o,T,r):[];if(s!=="single-url"&&!m)for(let b of H)le(e,t,b,r,o);await ft(e,o,T,t,r);let Q=B(e,o);if(q(u,e,Q),oe(u,Q),i.trackManifest){let b=y?.headers();i.manifest.pages[o]={url:o,outputPath:Q,contentType:Ie(b),changeToken:h.changeToken?.supported?h.changeToken.token:null,tokenSource:h.changeToken?.supported?h.changeToken.tokenSource??null:null,sitemapLastmod:ie(t.sitemapLastmodByPage[o])??null,discoveredPages:H,discoveredAssets:L,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await K(e,i);}}catch(c){try{let g=p?await M(`capture partial html ${o}`,U(e),()=>p.content()):"";if(g&&g.trim()){await ft(e,o,g,t,r);let m=B(e,o);if(q(u,e,m),oe(u,m),i.trackManifest){let w=i.manifest.pages[o]??i.previousManifest.pages[o];i.manifest.pages[o]={url:o,outputPath:m,contentType:w?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:ie(t.sitemapLastmodByPage[o])??w?.sitemapLastmod??null,discoveredPages:w?.discoveredPages?[...w.discoveredPages]:[],discoveredAssets:w?.discoveredAssets?[...w.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},i.enabled&&await K(e,i);}r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(c)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(c)});}finally{t.stats.pagesCompleted++,js(t)&&Es(t,r),p&&await M(`close page ${o}`,me(e),()=>p.close()).catch(c=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(c)});}),d&&await M(`close page context ${o}`,me(e),()=>d.close()).catch(c=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(c)});});}}}finally{await M("close worker browser",me(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),Ls(l,r,"browser.close timeout")));}}async function Ns(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function ht(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Ws(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function _s(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function Ot(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function mt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Bs(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>mt(s)-mt(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function wt(e){let t=await Ot(f.join(e,"job.json"));for(let r of Bs(e,t)){let s=await Ns(r);if(s.length>0)return s}return []}async function Hs(e){let t=f.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let a=await Ot(f.join(u,"job.json"));if(!Ws(a))continue;let o=_s(a);o>=n&&(n=o,s=u);}return s}async function zs(e){let t=await Hs(e),r=t!==""?await wt(t):await wt(e.logDir),s=new Set;for(let n of r)if(ht(n.error)||ht(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Js(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Gs(e){try{let t=await S.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Vs(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Pt(e,t,r,s,n){let i=A(e,r,e.sourceOrigin,s,"manual-url");i&&(de(i)?qe(e,t,i.toString(),s,n):k(e,i)?I(e,t,i.toString(),s,n):G(i)?le(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Ks(){let e=await Re(),t=$r(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await zs(e)),t.mode==="single-url"&&(r=await Js(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=(t.mode==="full"||t.mode==="single-url")&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await Gs(e):{},a=n||t.mode==="single-url"?await Ur(e):vt(),o={enabled:n,trackManifest:i,manifest:$e(a),previousManifest:$e(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve()};await Vs(e.logDir);let d={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Er();u||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let c=new ne(e.logDir,e.logLevel);if(s&&!n&&c.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)c.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let P=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(c.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>qe(e,d,v,c)),await gt(e,P,d,c,p),e.seedPaths.forEach(v=>Pt(e,d,v,c,"seed-path")),e.generated404RequestPath&&le(e,d,e.generated404RequestPath,c,"generated-404-request-path")):(r.forEach(v=>Pt(e,d,v,c,"cli")),await gt(e,P,d,c,p)),c.endMark("discovery",{pages:d.pageQueue.length,assets:d.assetQueue.length,sitemaps:d.sitemapQueue.length}),c.summary(`Queued ${d.pageQueue.length} pages, ${d.assetQueue.length} assets, ${d.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:d.pageQueue.length,queuedAssets:d.assetQueue.length,queuedSitemaps:d.sitemapQueue.length}),t.mode==="retry-timeouts"&&d.pageQueue.length===0&&d.assetQueue.length===0&&d.sitemapQueue.length===0){c.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await c.flush();return}c.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>Qs(e,d,c,t.mode,P,o,p));if(await Promise.all(y),c.endMark("render-pages",{pages:d.stats.pagesCompleted}),c.mark("download-assets"),c.progress("Asset download phase started.",{phase:"download-assets",doneAssets:d.doneAssets.size,assetsQueued:d.stats.assetsQueued,assetQueue:d.assetQueue.length}),await Ts(e,P,d,c,o,p),c.endMark("download-assets",{assets:d.doneAssets.size}),o.enabled&&e.maxPages===0&&await Jr(e,d,o,c,p),t.mode==="full"){let v=await Cs(e,d,c);for(let x of v)q(p,e,x);}}finally{await M("dispose crawl request context",me(e),()=>P.dispose()).catch(y=>{c.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&await K(e,o),await S.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(d.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(d.assetMap,null,2)&&q(p,e,f.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?rs(e,d):o.enabled&&!t.resumeRewrite?Gr(e,o,p,l,d.assetMap):void 0;c.mark("rewrite-text");let m=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";c.progress(m,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await dt(m);let w=Date.now(),h=await st(e,d.assetMap,{files:g,previousAssetMap:l,onStart:({totalFiles:P,workerCount:y})=>{c.info(`Rewrite phase scanning ${P} text files.`,{phase:"rewrite-text",totalFiles:P,workerCount:y});},onProgress:async({index:P,totalFiles:y,changedTextFiles:v,file:x,changed:L})=>{L&&q(p,e,f.join(e.outputDir,x)),c.checkpoint(`Rewriting text file ${P}/${y}`,{phase:"rewrite-text",index:P,totalFiles:y,changedTextFiles:v,file:x}),c.debug(`${L?"Rewrote":"Checked unchanged"} text file ${P}/${y}: ${x}`,{phase:"rewrite-text",index:P,totalFiles:y,changedTextFiles:v,file:x,changed:L});let T=Date.now();if(!(P===1||P===y||P%50===0||T-w>=5e3))return;w=T;let Q=`Rewriting text files: ${P}/${y}`;c.progress(Q,{phase:"rewrite-text",index:P,totalFiles:y,changedTextFiles:v,file:x}),await dt(Q);}});o.trackManifest&&(Vr(o,g),await K(e,o)),c.endMark("rewrite-text",{changedTextFiles:h});let C=o.enabled&&!t.resumeRewrite?await Yr(e,d,o,p,c):[];await Be(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?C:[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(P=>ee(e.outputDir,f.isAbsolute(P)?P:f.join(e.outputDir,P))).filter(P=>P!==null))].sort():[]}),c.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${h} text files.`:`Done. Rendered ${d.stats.pagesRendered} pages, processed ${d.doneSitemaps.size} sitemaps, downloaded ${d.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...d.stats,pagesRendered:d.stats.pagesRendered,donePages:d.stats.pagesCompleted,claimedPages:d.donePages.size,doneSitemaps:d.doneSitemaps.size,doneAssets:d.doneAssets.size,changedTextFiles:h}),await c.flush();}Ks().catch(async e=>{console.error(e);try{let r=(await Re().catch(()=>null))?.logDir??"logs",s=new ne(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import f from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import p from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';import ne from'http';import ie from'https';function U(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function oe(e){let t=String(e??"").trim();if(!t)return "";let n=t;if(/^https?:\/\//i.test(n))try{n=new URL(n).pathname;}catch{return ""}if(n=n.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!n)return "";let r=n.endsWith("/"),o=n.replace(/^\/+/,"").split("/").map(i=>i.replace(/[^A-Za-z0-9._-]/g,"")).filter(i=>i.length>0&&i!=="."&&i!=="..");if(o.length===0)return "";let s=`/${o.join("/")}`;return r&&p.extname(s)===""&&(s+="/"),s==="/"?"":s}function x(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,n])=>[t.trim(),String(n??"")]).filter(([t])=>t.length>0))}function se(e){if(!e||typeof e!="object")return {};let t={};for(let[n,r]of Object.entries(e)){let o=n.trim();if(!o||!r||typeof r!="object")continue;let s=r,i={};if(typeof s.targetOrigin=="string"){let a=U(s.targetOrigin);a&&(i.targetOrigin=a);}let u=x(s.extraReplacements);if(Object.keys(u).length>0&&(i.extraReplacements=u),s.s3&&typeof s.s3=="object"){let a={},c=s.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(i.s3=a);}if(s.cloudFront&&typeof s.cloudFront=="object"){let a={},c=s.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(a).length>0&&(i.cloudFront=a);}t[o]=i;}return t}function W(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function ue(e,t){if(!e||typeof e!="object")return null;let n=e,r=n.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let o=Number.parseInt(String(n.intervalMinutes??"0"),10);if(!Number.isFinite(o)||o<1)return null;let s=String(n.id??`${r}-${t+1}`).trim();if(!s)return null;let i=String(n.deploymentProfile??"").trim(),u=String(n.url??"").trim();return {id:s,enabled:n.enabled!==false,command:r,intervalMinutes:o,...r==="publish"||r==="crawl"?{crawlMode:n.crawlMode==="incremental"?"incremental":"full"}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&i?{deploymentProfile:i}:{},...u?{url:u}:{}}}function B(e){let t=e&&typeof e=="object"?e:{},n=Array.isArray(t.rules)?t.rules.map((r,o)=>ue(r,o)).filter(r=>!!r):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:n}}function ae(e){if(!e||typeof e!="object")return;let t=e,n={};for(let o of ["accountId","siteId"]){let s=t[o];typeof s=="string"&&s.trim()!==""&&(n[o]=s.trim());}let r=Number(t.lastUpdate??0);return Number.isFinite(r)&&r>0&&(n.lastUpdate=Math.floor(r)),t.subscriber===true&&(n.subscriber=true),Object.keys(n).length>0?n:void 0}function T(e){let t=e&&typeof e=="object"?e:{},n=String(t.accountId??"").trim(),r=String(t.siteId??"").trim(),o=t.subscriber===true,s=ae(t.siteSettings),i={...s??{},...n?{accountId:s?.accountId??n}:{},...r?{siteId:s?.siteId??r}:{},...s?.subscriber===true||o?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let c=String(t.runtimeToken??t.nonce??"").trim();c&&(u.runtimeToken=c);let g=String(t.uploadUrl??"").trim();g&&(u.uploadUrl=g),Object.keys(i).length>0&&(u.siteSettings=i);let d=W(t.subscriptionType);return d&&(u.subscriptionType=d),u}function ce(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Q(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function le(e,t){let n=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[r,o]of new Headers(t.headers).entries())n.set(r,o);return n}async function z(e,t,n=5){let r=new URL(Q(e)),o=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),s=r.protocol==="http:"?ne:ie,i=le(e,t);return await new Promise((u,a)=>{let c=s.request(r,{method:o,headers:Object.fromEntries(i.entries()),...r.protocol==="https:"?{rejectUnauthorized:false}:{}},g=>{let d=[];g.on("data",l=>{d.push(Buffer.isBuffer(l)?l:Buffer.from(l));}),g.on("error",a),g.on("end",()=>{let l=g.statusCode??0,m=g.headers.location;if(n>0&&m&&[301,302,303,307,308].includes(l)){u(z(new URL(m,r),{method:l===303?"GET":o},n-1));return}let h=new Headers;for(let[P,y]of Object.entries(g.headers))Array.isArray(y)?h.set(P,y.join(", ")):typeof y=="string"&&h.set(P,y);u(new Response(Buffer.concat(d),{status:l,statusText:g.statusMessage??"",headers:h}));});});c.on("error",a),c.end();})}function de(e){let t=T(e.wpsuite),n={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},r=String(n.accountId??"").trim(),o=String(n.siteId??"").trim(),s=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...s?{uploadUrl:s}:{},...Object.keys(n).length>0?{siteSettings:n}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!r||!o||!s)return false;ce(e.sourceOrigin);let i=globalThis,a={...i.WpSuite??{},siteSettings:n,uploadUrl:s};return t.apiBase&&(a.apiBase=t.apiBase),i.WpSuite=a,true}async function pe(e){if(!de(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),n=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof n=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(s,i)=>Q(s).startsWith(t)?z(s,i):n(s,i)));let{getConfig:r}=await import('@smart-cloud/wpsuite-core'),o=await r("publisher");return o&&typeof o=="object"?o:null}catch{return null}finally{typeof n=="function"&&(globalThis.fetch=n);}}function ge(e,t){let n=se(t?.deploymentProfiles),r=String(t?.defaultDeploymentProfile??"").trim(),o=n[r]?r:"",s=W(t?.subscriptionType),i=T(e.wpsuite);return {...e,scheduler:B(t?.scheduler),deploymentProfiles:n,defaultDeploymentProfile:o,...s?{subscriptionType:s}:{},wpsuite:{...i,...i.siteSettings||s?{siteSettings:{...i.siteSettings??{},...i.siteSettings?.subscriber===true||s==="PROFESSIONAL"||s==="AGENCY"?{subscriber:true}:{}}}:{},...s?{subscriptionType:s}:{}}}}function J(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let o=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):t}function me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?p.resolve(e):""}function L(e,t,n){let r=String(t||"").trim();return r&&p.isAbsolute(r)?p.resolve(r):p.resolve(e,J(r,n))}async function k(e){let t=String(e??"").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await f.readFile(t,"utf8"),r=JSON.parse(n);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=U(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=oe(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=x(r.extraReplacements),r.postCrawlCopyMap=x(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.wpsuite=T(r.wpsuite),r.scheduler=B(void 0),r.deploymentProfiles={},r.defaultDeploymentProfile="",r.deploymentTargetOverride=String(r.deploymentTargetOverride??"").trim();let o=ge(r,await pe(r)),s=me(),i=s?p.resolve(s,".."):"";return i?(o.outputDir=L(i,o.outputDir,"export"),o.logDir=L(i,o.logDir,"logs")):(p.isAbsolute(o.outputDir)||(o.outputDir=J(o.outputDir,"export")),p.isAbsolute(o.logDir)||(o.logDir=J(o.logDir,"logs"))),o}var I=class extends Error{request;step;constructor(t,n){super(`Stop requested during ${n}.`),this.name="StopRequestedError",this.request=t,this.step=n;}},Pe="queue-mutation.lock",ve=5e3,Ce=3e4;function Re(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",n=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),r=process.env.PUBLISHER_CONFIG||"",o=1;for(let u=0;u<e.length;u++){let a=e[u];a==="--runtime-dir"?t=e[++u]||"":a.startsWith("--runtime-dir=")?t=a.slice(14):a==="--exporter-dir"?n=e[++u]||n:a.startsWith("--exporter-dir=")?n=a.slice(15):a==="--config"?r=e[++u]||"":a.startsWith("--config=")?r=a.slice(9):a==="--max-jobs"?o=Number.parseInt(e[++u]||"1",10):a.startsWith("--max-jobs=")&&(o=Number.parseInt(a.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let s=p.resolve(t),i=r?p.resolve(r):p.join(s,"config.json");return {runtimeDir:s,exporterDir:p.resolve(n),configPath:i,maxJobs:Number.isFinite(o)&&o>0?o:1}}async function w(e,t){try{let n=await f.readFile(e,"utf8");return JSON.parse(n)}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="ENOENT"){let o=n instanceof Error?n.message:String(n);console.warn(`[queue-runner] failed to read JSON ${e}: ${o}`);}return t}}async function S(e,t){await f.mkdir(p.dirname(e),{recursive:true}),await f.writeFile(e,JSON.stringify(t,null,2),"utf8");}function G(e){return p.join(e,Pe)}async function Ae(e){await new Promise(t=>setTimeout(t,e));}async function qe(e){let t=G(e),n=Date.now()+ve;for(;;)try{await f.mkdir(p.dirname(t),{recursive:!0}),await f.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="EEXIST")throw r;let s=await f.stat(t).catch(()=>null);if(s&&Date.now()-s.mtimeMs>Ce){await f.unlink(t).catch(()=>{});continue}if(Date.now()>=n)throw new Error("Timed out acquiring queue mutation lock.",{cause:r});await Ae(50);}}async function Ie(e){await f.unlink(G(e)).catch(()=>{});}async function D(e,t){await qe(e);try{return await t()}finally{await Ie(e);}}function De(e){return p.join(e,"queue-runner-heartbeat.json")}function F(e){return p.join(e,"current-progress.json")}function N(e){return p.join(e,"scheduler-state.json")}function xe(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let o=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):t}async function Y(e){let t=await w(e.configPath||"",null),n=p.resolve(e.runtimeDir,".."),o=(typeof t?.logDir=="string"?t.logDir:"").trim();return o&&p.isAbsolute(o)?p.resolve(o):p.resolve(n,xe(o,"logs"))}function E(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function Je(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function Te(e){return [Je(e.endedAt||e.startedAt),E(e.command||"job","job"),E(e.id||"job","job"),E(e.status||"finished","finished")].join("-")}function ke(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")||e.endsWith(".rejected.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function Ee(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"||e.endsWith(".rejected.jsonl")?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function je(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function _(e,t,n){let r=`${n}.gz`,o=p.join(t,r);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(o));let[s,i]=await Promise.all([f.stat(e),f.stat(o)]);return {role:Ee(n),originalFileName:n,storedFileName:r,compressed:true,compression:"gzip",contentType:je(r),originalSize:s.size,storedSize:i.size}}async function H(e,t){let n=await Y(t),r=new Date().toISOString(),o=Te(e),s=p.join(n,"archive",o),i=[],u=[];await f.mkdir(s,{recursive:true});try{let g=await f.readdir(n,{withFileTypes:!0});for(let d of g){if(!d.isFile()||!ke(d.name))continue;let l=await _(p.join(n,d.name),s,d.name);u.push(l),i.push(l.storedFileName);}}catch(g){if((g&&typeof g=="object"&&"code"in g?String(g.code||""):"")!=="ENOENT")throw g}let a=F(t.runtimeDir);if(existsSync(a)){let g=await _(a,s,"current-progress.json");u.push(g),i.push(g.storedFileName);}let c=[...i,"job.json"];return await S(p.join(s,"job.json"),{manifestVersion:1,archivedAt:r,archiveKey:o,archiveDir:s,logDir:n,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:u,job:e}),{archiveKey:o,archiveDir:s,archiveCreatedAt:r,archivedFiles:c,artifacts:u}}function Oe(e){return p.join(e,"audit-events.jsonl")}function j(e){return p.join(e,"stop-request.json")}async function Z(e){try{let t=await f.readFile(j(e),"utf8"),n=JSON.parse(t);return n&&typeof n=="object"?n:null}catch{return null}}async function $(e,t){let n=await Z(e);if(!n)return null;let r=String(n.targetJobId||"").trim();return r&&r!==t?null:n}async function O(e,t){if(!t){await f.unlink(j(e)).catch(()=>{});return}let n=await Z(e);if(!n)return;let r=String(n.targetJobId||"").trim();(!r||r===t)&&await f.unlink(j(e)).catch(()=>{});}async function A(e,t){try{let n={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await f.mkdir(e,{recursive:!0}),await f.appendFile(Oe(e),`${JSON.stringify(n)}
|
|
2
|
-
`,"utf8");}catch{}}async function b(e,t,n){let r={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...n??{}};try{await S(De(e.runtimeDir),r);}catch{}}async function q(e){try{await f.unlink(F(e));}catch{}}function Fe(e,t){let n=e.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let r=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(r)||r<1)return null;let o=(e.id||`${n}-${t+1}`).trim();if(!o)return null;let s=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",a=(n==="publish"||n==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return n==="url"&&!i?null:{id:o,enabled:s,command:n,intervalMinutes:r,...n==="publish"||n==="crawl"?{crawlMode:a}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function Me(e){let t=await k(e.configPath).catch(()=>null),n=t?.scheduler,r=Array.isArray(n?.rules)?n.rules.map((o,s)=>Fe(o,s)).filter(o=>!!o):[];return {enabled:!!n?.enabled,timezone:typeof n?.timezone=="string"&&n.timezone.trim()?n.timezone.trim():"UTC",rules:r,wpsuite:t?.wpsuite}}function Le(e,t){let n=(t.url||"").trim(),r=t.crawlMode||"full",o=(t.deploymentProfile||"").trim();return e.some(s=>s.command===t.command&&(s.url||"").trim()===n&&(s.crawlMode||"full")===r&&(s.deploymentProfile||"").trim()===o&&(s.status===void 0||s.status==="queued"||s.status==="running"))}async function Ue(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let n=await Me(e);if(!n.enabled||n.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(n.wpsuite??null);}catch(a){let c=a instanceof Error?a.message:String(a);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let r=p.join(e.runtimeDir,"queue.json"),o=p.join(e.runtimeDir,"current-run.json"),s=Date.now(),i=0,u=[];await D(e.runtimeDir,async()=>{let a=await w(r,[]),c=await w(o,null),g=await w(N(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),d=[...a];c&&c.status==="running"&&d.unshift(c);for(let l of n.rules){if(!l.enabled)continue;let m=l.intervalMinutes*60*1e3,h=Math.floor(s/m),P=g.lastEnqueuedBucketByRuleId[l.id]??-1;if(h<=P||Le(d,{command:l.command,url:l.url,crawlMode:l.crawlMode,deploymentProfile:l.deploymentProfile}))continue;let y={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...n.wpsuite?{wpsuite:n.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};a.push(y),d.push(y),u.push({job:y,rule:l}),g.lastEnqueuedBucketByRuleId[l.id]=h,i+=1;}i>0&&(await S(r,a),await S(N(e.runtimeDir),g));});for(let a of u)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:a.job.id,command:a.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:a.rule.id,intervalMinutes:a.rule.intervalMinutes,timezone:n.timezone,deploymentProfile:a.rule.deploymentProfile||"",url:a.rule.url||""}});return i}async function We(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await f.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function Be(e){try{await f.unlink(e);}catch{}}function Qe(e,t){let n=[p.join(e,"dist",`${t}.js`),p.join(e,`${t}.js`)];for(let r of n)if(existsSync(r))return r;throw new Error(`Cannot find ${t}.js in ${p.join(e,"dist")} or ${e}`)}async function C(e,t,n,r,o,s,i){if(i){let c=await $(t,i);if(c)throw new I(c,n)}let u=Qe(e,n),a={...process.env,...s??{}};return a.STATIC_PUBLISHER_RUNTIME_DIR=t,o&&(a.PUBLISHER_CONFIG=o),await new Promise((c,g)=>{let d=null,l=null,m=null,h=false,P=spawn(process.execPath,[u,...r],{cwd:e,env:a,stdio:"inherit"}),y=()=>{l&&clearInterval(l),m&&clearTimeout(m);},te=async()=>{if(!(!i||d||h)){h=true;try{let R=await $(t,i);if(!R)return;d=R,P.kill("SIGTERM"),m=setTimeout(()=>{P.kill("SIGKILL");},1e4);}finally{h=false;}}};i&&(l=setInterval(()=>{te();},1e3)),P.on("error",R=>{y(),g(R);}),P.on("close",(R,M)=>{if(y(),d){g(new I(d,n));return}R===0?c():g(new Error(`${n} exited with code ${R??-1}${M?` (signal ${M})`:""}`));});}),0}function ze(e){let t=e.awsTempCreds;if(!t)return {};let n={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(n.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(n.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(n.AWS_SESSION_TOKEN=t.sessionToken.trim()),n}async function Ne(e,t){let n=i=>{if(!i||typeof i!="object")return null;let u=i,a={...u.siteSettings??{},...String(u.accountId??"").trim()&&!u.siteSettings?.accountId?{accountId:String(u.accountId).trim()}:{},...String(u.siteId??"").trim()&&!u.siteSettings?.siteId?{siteId:String(u.siteId).trim()}:{},...u.siteSettings?.subscriber===true||u.subscriber===true?{subscriber:true}:{}},c={...String(u.apiBase??"").trim()?{apiBase:String(u.apiBase).trim()}:{},...String(u.runtimeToken??u.nonce??"").trim()?{runtimeToken:String(u.runtimeToken??u.nonce).trim()}:{},...String(u.uploadUrl??"").trim()?{uploadUrl:String(u.uploadUrl).trim()}:{},...Object.keys(a).length>0?{siteSettings:a}:{},...u.subscriptionType?{subscriptionType:u.subscriptionType}:{}};return Object.keys(c).length>0?c:null},r=await k(t.configPath).catch(()=>null),o=n(r?.wpsuite);if(o)return o;let s=n(e.wpsuite);return s||null}async function _e(e,t){return await Ne(e,t)}function V(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function He(e,t){let n=await w(F(e.runtimeDir),null),r=n&&n.details&&typeof n.details.phase=="string"?n.details.phase.trim():"";if(r)return r;let o=n&&typeof n.currentStep=="string"?n.currentStep.trim():"";if(o)return o;let s=await Y(e).catch(()=>"");if(s){let i=await w(p.join(s,"current-crawl-event.json"),null),u=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(u)return u}return t}function $e(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function Ke(e){let t=p.join(e.runtimeDir,"queue.json"),n=p.join(e.runtimeDir,"current-run.json"),r=await D(e.runtimeDir,async()=>{let o=await w(n,null);if(!o||o.status!=="running"&&o.status!=="queued")return null;let s=await w(t,[]),i=Array.isArray(s)?s.filter(a=>a?.id!==o.id):[],u=V(o);return await S(t,[u,...i]),await S(n,null),u});r&&(await O(e.runtimeDir,r.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:r.id,command:r.command,message:"Recovered stale current-run entry back into queue."}));}async function Ge(e,t,n){try{if((n.shouldEnforceSubscriptionOnJobExecution?.(e)??n.enforceSubscriptionOnJobExecution)&&n.assertActiveSubscriptionForIdentity){let u=await _e(e,t);await n.assertActiveSubscriptionForIdentity(u);}let o=ze(e),s=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await C(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,o,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await C(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await C(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="crawl")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await C(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,o,e.id),{exitCode:0};if(e.command==="deploy")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await C(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="invalidate")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await C(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await C(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,o,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await C(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,o,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(r){return r instanceof I?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:r.request,stoppedStep:r.step}:{exitCode:1,error:r instanceof Error?r.message:String(r)}}}function X(e){return async function(n=process.argv.slice(2)){let r=Re(n),o=p.join(r.runtimeDir,"export.lock"),s=p.join(r.runtimeDir,"last-run.json");await q(r.runtimeDir),await b(r,"starting",{message:"queue-runner starting"});try{await We(o);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await b(r,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await Ke(r);let i=await Ue(r,e),u=0;for(let a=0;a<r.maxJobs;a++){let c=await Ze(r,e);if(c==="none"||(u+=1,c==="stopped"))break}if(u===0)await q(r.runtimeDir),await b(r,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await q(r.runtimeDir);let a=await w(s,null),c=String(a?.status||"").trim();await b(r,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:a?.id,lastJobCommand:a?.command,lastJobStatus:a?.status,lastJobExitCode:a?.exitCode,lastJobError:a?.error,...c==="stopped"?{currentStep:a?.stoppedStep,stopRequestedAt:a?.stopRequestedAt,stopRequestedByLogin:a?.stopRequestedByLogin,stopRequestedMode:a?.stopMode,lastStoppedStep:a?.stoppedStep,message:a?.stopMode==="requeue"?`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and requeued.`:`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await b(r,"error",{message:u}),await A(r.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await Be(o);}}}async function Ye(e,t,n){let r=p.join(e.runtimeDir,"queue.json"),o=p.join(e.runtimeDir,"current-run.json");await D(e.runtimeDir,async()=>{let s=await w(r,[]),i=Array.isArray(s)?s.filter(c=>c?.id!==t.id):[],u=V(t),a=$e(t,n);a?u.resumeFromStep=a:delete u.resumeFromStep,await S(r,[u,...i]),await S(o,null);});}async function Ze(e,t){let n=p.join(e.runtimeDir,"queue.json"),r=p.join(e.runtimeDir,"current-run.json"),o=p.join(e.runtimeDir,"last-run.json"),s=new Date().toISOString(),i=await D(e.runtimeDir,async()=>{let d=await w(n,[]);if(!Array.isArray(d)||d.length===0)return null;let l={...d[0],status:"running",startedAt:s};return await S(n,d.slice(1)),await S(r,l),l});if(!i)return "none";await q(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:s,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await b(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await Ge(i,e,t),a=new Date().toISOString();if(u.stopped){let d=await He(e,u.stoppedStep||i.command),l=u.stopRequest?.mode==="requeue"?"requeue":"stop",m={...i,status:"stopped",endedAt:a,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:l,stoppedStep:d};l==="requeue"?await Ye(e,i,d):await S(r,null);try{let h=await H(m,e);m.logArchiveDir=h.archiveDir,m.logArchiveCreatedAt=h.archiveCreatedAt,m.logArchiveFileCount=h.archivedFiles.length;}catch(h){m.logArchiveError=h instanceof Error?h.message:String(h);}return await S(o,m),await O(e.runtimeDir,i.id),await q(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:l==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:a,deploymentProfile:i.deploymentProfile||"",stopMode:l,stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:d,logArchiveKey:m.logArchiveError||!m.logArchiveDir?"":p.basename(m.logArchiveDir),logArchiveDir:m.logArchiveDir||"",logArchiveCreatedAt:m.logArchiveCreatedAt||"",logArchiveFileCount:m.logArchiveFileCount??0,logArchiveError:m.logArchiveError||""}}),"stopped"}let c={...i,status:u.exitCode===0?"success":"failed",endedAt:a,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await S(r,null),await O(e.runtimeDir,i.id);try{let d=await H(c,e);c.logArchiveDir=d.archiveDir,c.logArchiveCreatedAt=d.archiveCreatedAt,c.logArchiveFileCount=d.archivedFiles.length;}catch(d){c.logArchiveError=d instanceof Error?d.message:String(d);}await S(o,c);let g=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:g,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":p.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}async function Xe(e){if(!e)throw new Error("Missing remote WPSuite publisher state. Refresh the runtime config before running subscription-gated jobs.");if(e.subscriptionType!=="PROFESSIONAL"&&e.subscriptionType!=="AGENCY")throw new Error("This job requires an active WPSuite publisher subscription in the remote site configuration.")}var et={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0,assertActiveSubscriptionForIdentity:Xe},ee=X(et),tt=ee,St=ee;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&tt().catch(e=>{console.error(e),process.exit(1);});export{St as default,tt as main,ee as runQueueRunner};
|
|
1
|
+
import {pathToFileURL}from'url';import f from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import g from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';import ne from'http';import ie from'https';function U(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function oe(e){let t=String(e??"").trim();if(!t)return "";let n=t;if(/^https?:\/\//i.test(n))try{n=new URL(n).pathname;}catch{return ""}if(n=n.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!n)return "";let r=n.endsWith("/"),o=n.replace(/^\/+/,"").split("/").map(i=>i.replace(/[^A-Za-z0-9._-]/g,"")).filter(i=>i.length>0&&i!=="."&&i!=="..");if(o.length===0)return "";let s=`/${o.join("/")}`;return r&&g.extname(s)===""&&(s+="/"),s==="/"?"":s}function x(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,n])=>[t.trim(),String(n??"")]).filter(([t])=>t.length>0))}function se(e){if(!e||typeof e!="object")return {};let t={};for(let[n,r]of Object.entries(e)){let o=n.trim();if(!o||!r||typeof r!="object")continue;let s=r,i={};if(typeof s.targetOrigin=="string"){let a=U(s.targetOrigin);a&&(i.targetOrigin=a);}let u=x(s.extraReplacements);if(Object.keys(u).length>0&&(i.extraReplacements=u),s.s3&&typeof s.s3=="object"){let a={},c=s.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(i.s3=a);}if(s.cloudFront&&typeof s.cloudFront=="object"){let a={},c=s.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(p=>String(p??"").trim()).filter(p=>p.length>0)),Object.keys(a).length>0&&(i.cloudFront=a);}t[o]=i;}return t}function W(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function ue(e,t){if(!e||typeof e!="object")return null;let n=e,r=n.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let o=Number.parseInt(String(n.intervalMinutes??"0"),10);if(!Number.isFinite(o)||o<1)return null;let s=String(n.id??`${r}-${t+1}`).trim();if(!s)return null;let i=String(n.deploymentProfile??"").trim(),u=String(n.url??"").trim();return {id:s,enabled:n.enabled!==false,command:r,intervalMinutes:o,...r==="publish"||r==="crawl"?{crawlMode:n.crawlMode==="incremental"?"incremental":"full"}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&i?{deploymentProfile:i}:{},...u?{url:u}:{}}}function B(e){let t=e&&typeof e=="object"?e:{},n=Array.isArray(t.rules)?t.rules.map((r,o)=>ue(r,o)).filter(r=>!!r):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:n}}function ae(e){if(!e||typeof e!="object")return;let t=e,n={};for(let o of ["accountId","siteId"]){let s=t[o];typeof s=="string"&&s.trim()!==""&&(n[o]=s.trim());}let r=Number(t.lastUpdate??0);return Number.isFinite(r)&&r>0&&(n.lastUpdate=Math.floor(r)),t.subscriber===true&&(n.subscriber=true),Object.keys(n).length>0?n:void 0}function T(e){let t=e&&typeof e=="object"?e:{},n=String(t.accountId??"").trim(),r=String(t.siteId??"").trim(),o=t.subscriber===true,s=ae(t.siteSettings),i={...s??{},...n?{accountId:s?.accountId??n}:{},...r?{siteId:s?.siteId??r}:{},...s?.subscriber===true||o?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let c=String(t.runtimeToken??t.nonce??"").trim();c&&(u.runtimeToken=c);let p=String(t.uploadUrl??"").trim();p&&(u.uploadUrl=p),Object.keys(i).length>0&&(u.siteSettings=i);let d=W(t.subscriptionType);return d&&(u.subscriptionType=d),u}function ce(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Q(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function le(e,t){let n=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[r,o]of new Headers(t.headers).entries())n.set(r,o);return n}async function z(e,t,n=5){let r=new URL(Q(e)),o=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),s=r.protocol==="http:"?ne:ie,i=le(e,t);return await new Promise((u,a)=>{let c=s.request(r,{method:o,headers:Object.fromEntries(i.entries()),...r.protocol==="https:"?{rejectUnauthorized:false}:{}},p=>{let d=[];p.on("data",l=>{d.push(Buffer.isBuffer(l)?l:Buffer.from(l));}),p.on("error",a),p.on("end",()=>{let l=p.statusCode??0,m=p.headers.location;if(n>0&&m&&[301,302,303,307,308].includes(l)){u(z(new URL(m,r),{method:l===303?"GET":o},n-1));return}let h=new Headers;for(let[P,y]of Object.entries(p.headers))Array.isArray(y)?h.set(P,y.join(", ")):typeof y=="string"&&h.set(P,y);u(new Response(Buffer.concat(d),{status:l,statusText:p.statusMessage??"",headers:h}));});});c.on("error",a),c.end();})}function de(e){let t=T(e.wpsuite),n={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},r=String(n.accountId??"").trim(),o=String(n.siteId??"").trim(),s=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...s?{uploadUrl:s}:{},...Object.keys(n).length>0?{siteSettings:n}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!r||!o||!s)return false;ce(e.sourceOrigin);let i=globalThis,a={...i.WpSuite??{},siteSettings:n,uploadUrl:s};return t.apiBase&&(a.apiBase=t.apiBase),i.WpSuite=a,true}async function ge(e){if(!de(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),n=globalThis.fetch;try{e.ignoreHttpsErrors&&typeof n=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(s,i)=>Q(s).startsWith(t)?z(s,i):n(s,i)));let{getConfig:r}=await import('@smart-cloud/wpsuite-core'),o=await r("publisher");return o&&typeof o=="object"?o:null}catch{return null}finally{typeof n=="function"&&(globalThis.fetch=n);}}function pe(e,t){let n=se(t?.deploymentProfiles),r=String(t?.defaultDeploymentProfile??"").trim(),o=n[r]?r:"",s=W(t?.subscriptionType),i=T(e.wpsuite);return {...e,scheduler:B(t?.scheduler),deploymentProfiles:n,defaultDeploymentProfile:o,...s?{subscriptionType:s}:{},wpsuite:{...i,...i.siteSettings||s?{siteSettings:{...i.siteSettings??{},...i.siteSettings?.subscriber===true||s==="PROFESSIONAL"||s==="AGENCY"?{subscriber:true}:{}}}:{},...s?{subscriptionType:s}:{}}}}function J(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let o=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):t}function me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?g.resolve(e):""}function L(e,t,n){let r=String(t||"").trim();return r&&g.isAbsolute(r)?g.resolve(r):g.resolve(e,J(r,n))}async function k(e){let t=String(e??"").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await f.readFile(t,"utf8"),r=JSON.parse(n);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=U(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=oe(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=x(r.extraReplacements),r.postCrawlCopyMap=x(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.wpsuite=T(r.wpsuite),r.scheduler=B(void 0),r.deploymentProfiles={},r.defaultDeploymentProfile="",r.deploymentTargetOverride=String(r.deploymentTargetOverride??"").trim();let o=pe(r,await ge(r)),s=me(),i=s?g.resolve(s,".."):"";return i?(o.outputDir=L(i,o.outputDir,"export"),o.logDir=L(i,o.logDir,"logs")):(g.isAbsolute(o.outputDir)||(o.outputDir=J(o.outputDir,"export")),g.isAbsolute(o.logDir)||(o.logDir=J(o.logDir,"logs"))),o}var I=class extends Error{request;step;constructor(t,n){super(`Stop requested during ${n}.`),this.name="StopRequestedError",this.request=t,this.step=n;}},Pe="queue-mutation.lock",ve=5e3,Ce=3e4;function Re(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",n=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),r=process.env.PUBLISHER_CONFIG||"",o=1;for(let u=0;u<e.length;u++){let a=e[u];a==="--runtime-dir"?t=e[++u]||"":a.startsWith("--runtime-dir=")?t=a.slice(14):a==="--exporter-dir"?n=e[++u]||n:a.startsWith("--exporter-dir=")?n=a.slice(15):a==="--config"?r=e[++u]||"":a.startsWith("--config=")?r=a.slice(9):a==="--max-jobs"?o=Number.parseInt(e[++u]||"1",10):a.startsWith("--max-jobs=")&&(o=Number.parseInt(a.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let s=g.resolve(t),i=r?g.resolve(r):g.join(s,"config.json");return {runtimeDir:s,exporterDir:g.resolve(n),configPath:i,maxJobs:Number.isFinite(o)&&o>0?o:1}}async function w(e,t){try{let n=await f.readFile(e,"utf8");return JSON.parse(n)}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="ENOENT"){let o=n instanceof Error?n.message:String(n);console.warn(`[queue-runner] failed to read JSON ${e}: ${o}`);}return t}}async function S(e,t){await f.mkdir(g.dirname(e),{recursive:true}),await f.writeFile(e,JSON.stringify(t,null,2),"utf8");}function G(e){return g.join(e,Pe)}async function Ae(e){await new Promise(t=>setTimeout(t,e));}async function qe(e){let t=G(e),n=Date.now()+ve;for(;;)try{await f.mkdir(g.dirname(t),{recursive:!0}),await f.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="EEXIST")throw r;let s=await f.stat(t).catch(()=>null);if(s&&Date.now()-s.mtimeMs>Ce){await f.unlink(t).catch(()=>{});continue}if(Date.now()>=n)throw new Error("Timed out acquiring queue mutation lock.",{cause:r});await Ae(50);}}async function Ie(e){await f.unlink(G(e)).catch(()=>{});}async function D(e,t){await qe(e);try{return await t()}finally{await Ie(e);}}function De(e){return g.join(e,"queue-runner-heartbeat.json")}function F(e){return g.join(e,"current-progress.json")}function N(e){return g.join(e,"scheduler-state.json")}function xe(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let o=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):t}async function Y(e){let t=await w(e.configPath||"",null),n=g.resolve(e.runtimeDir,".."),o=(typeof t?.logDir=="string"?t.logDir:"").trim();return o&&g.isAbsolute(o)?g.resolve(o):g.resolve(n,xe(o,"logs"))}function E(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function Je(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function Te(e){return [Je(e.endedAt||e.startedAt),E(e.command||"job","job"),E(e.id||"job","job"),E(e.status||"finished","finished")].join("-")}function ke(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")||e.endsWith(".rejected.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function Ee(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"||e.endsWith(".rejected.jsonl")?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function je(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function _(e,t,n){let r=`${n}.gz`,o=g.join(t,r);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(o));let[s,i]=await Promise.all([f.stat(e),f.stat(o)]);return {role:Ee(n),originalFileName:n,storedFileName:r,compressed:true,compression:"gzip",contentType:je(r),originalSize:s.size,storedSize:i.size}}async function H(e,t){let n=await Y(t),r=new Date().toISOString(),o=Te(e),s=g.join(n,"archive",o),i=[],u=[];await f.mkdir(s,{recursive:true});try{let p=await f.readdir(n,{withFileTypes:!0});for(let d of p){if(!d.isFile()||!ke(d.name))continue;let l=await _(g.join(n,d.name),s,d.name);u.push(l),i.push(l.storedFileName);}}catch(p){if((p&&typeof p=="object"&&"code"in p?String(p.code||""):"")!=="ENOENT")throw p}let a=F(t.runtimeDir);if(existsSync(a)){let p=await _(a,s,"current-progress.json");u.push(p),i.push(p.storedFileName);}let c=[...i,"job.json"];return await S(g.join(s,"job.json"),{manifestVersion:1,archivedAt:r,archiveKey:o,archiveDir:s,logDir:n,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:u,job:e}),{archiveKey:o,archiveDir:s,archiveCreatedAt:r,archivedFiles:c,artifacts:u}}function Oe(e){return g.join(e,"audit-events.jsonl")}function j(e){return g.join(e,"stop-request.json")}async function Z(e){try{let t=await f.readFile(j(e),"utf8"),n=JSON.parse(t);return n&&typeof n=="object"?n:null}catch{return null}}async function $(e,t){let n=await Z(e);if(!n)return null;let r=String(n.targetJobId||"").trim();return r&&r!==t?null:n}async function O(e,t){if(!t){await f.unlink(j(e)).catch(()=>{});return}let n=await Z(e);if(!n)return;let r=String(n.targetJobId||"").trim();(!r||r===t)&&await f.unlink(j(e)).catch(()=>{});}async function A(e,t){try{let n={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await f.mkdir(e,{recursive:!0}),await f.appendFile(Oe(e),`${JSON.stringify(n)}
|
|
2
|
+
`,"utf8");}catch{}}async function b(e,t,n){let r={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...n??{}};try{await S(De(e.runtimeDir),r);}catch{}}async function q(e){try{await f.unlink(F(e));}catch{}}function Fe(e,t){let n=e.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let r=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(r)||r<1)return null;let o=(e.id||`${n}-${t+1}`).trim();if(!o)return null;let s=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",a=(n==="publish"||n==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return n==="url"&&!i?null:{id:o,enabled:s,command:n,intervalMinutes:r,...n==="publish"||n==="crawl"?{crawlMode:a}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function Me(e){let t=await k(e.configPath).catch(()=>null),n=t?.scheduler,r=Array.isArray(n?.rules)?n.rules.map((o,s)=>Fe(o,s)).filter(o=>!!o):[];return {enabled:!!n?.enabled,timezone:typeof n?.timezone=="string"&&n.timezone.trim()?n.timezone.trim():"UTC",rules:r,wpsuite:t?.wpsuite}}function Le(e,t){let n=(t.url||"").trim(),r=t.crawlMode||"full",o=(t.deploymentProfile||"").trim();return e.some(s=>s.command===t.command&&(s.url||"").trim()===n&&(s.crawlMode||"full")===r&&(s.deploymentProfile||"").trim()===o&&(s.status===void 0||s.status==="queued"||s.status==="running"))}async function Ue(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let n=await Me(e);if(!n.enabled||n.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(n.wpsuite??null);}catch(a){let c=a instanceof Error?a.message:String(a);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let r=g.join(e.runtimeDir,"queue.json"),o=g.join(e.runtimeDir,"current-run.json"),s=Date.now(),i=0,u=[];await D(e.runtimeDir,async()=>{let a=await w(r,[]),c=await w(o,null),p=await w(N(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),d=[...a];c&&c.status==="running"&&d.unshift(c);for(let l of n.rules){if(!l.enabled)continue;let m=l.intervalMinutes*60*1e3,h=Math.floor(s/m),P=p.lastEnqueuedBucketByRuleId[l.id]??-1;if(h<=P||Le(d,{command:l.command,url:l.url,crawlMode:l.crawlMode,deploymentProfile:l.deploymentProfile}))continue;let y={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...n.wpsuite?{wpsuite:n.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};a.push(y),d.push(y),u.push({job:y,rule:l}),p.lastEnqueuedBucketByRuleId[l.id]=h,i+=1;}i>0&&(await S(r,a),await S(N(e.runtimeDir),p));});for(let a of u)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:a.job.id,command:a.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:a.rule.id,intervalMinutes:a.rule.intervalMinutes,timezone:n.timezone,deploymentProfile:a.rule.deploymentProfile||"",url:a.rule.url||""}});return i}async function We(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await f.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function Be(e){try{await f.unlink(e);}catch{}}function Qe(e,t){let n=[g.join(e,"dist",`${t}.js`),g.join(e,`${t}.js`)];for(let r of n)if(existsSync(r))return r;throw new Error(`Cannot find ${t}.js in ${g.join(e,"dist")} or ${e}`)}async function C(e,t,n,r,o,s,i){if(i){let c=await $(t,i);if(c)throw new I(c,n)}let u=Qe(e,n),a={...process.env,...s??{}};return a.STATIC_PUBLISHER_RUNTIME_DIR=t,o&&(a.PUBLISHER_CONFIG=o),await new Promise((c,p)=>{let d=null,l=null,m=null,h=false,P=spawn(process.execPath,[u,...r],{cwd:e,env:a,stdio:"inherit"}),y=()=>{l&&clearInterval(l),m&&clearTimeout(m);},te=async()=>{if(!(!i||d||h)){h=true;try{let R=await $(t,i);if(!R)return;d=R,P.kill("SIGTERM"),m=setTimeout(()=>{P.kill("SIGKILL");},1e4);}finally{h=false;}}};i&&(l=setInterval(()=>{te();},1e3)),P.on("error",R=>{y(),p(R);}),P.on("close",(R,M)=>{if(y(),d){p(new I(d,n));return}R===0?c():p(new Error(`${n} exited with code ${R??-1}${M?` (signal ${M})`:""}`));});}),0}function ze(e){let t=e.awsTempCreds;if(!t)return {};let n={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(n.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(n.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(n.AWS_SESSION_TOKEN=t.sessionToken.trim()),n}async function Ne(e,t){let n=i=>{if(!i||typeof i!="object")return null;let u=i,a={...u.siteSettings??{},...String(u.accountId??"").trim()&&!u.siteSettings?.accountId?{accountId:String(u.accountId).trim()}:{},...String(u.siteId??"").trim()&&!u.siteSettings?.siteId?{siteId:String(u.siteId).trim()}:{},...u.siteSettings?.subscriber===true||u.subscriber===true?{subscriber:true}:{}},c={...String(u.apiBase??"").trim()?{apiBase:String(u.apiBase).trim()}:{},...String(u.runtimeToken??u.nonce??"").trim()?{runtimeToken:String(u.runtimeToken??u.nonce).trim()}:{},...String(u.uploadUrl??"").trim()?{uploadUrl:String(u.uploadUrl).trim()}:{},...Object.keys(a).length>0?{siteSettings:a}:{},...u.subscriptionType?{subscriptionType:u.subscriptionType}:{}};return Object.keys(c).length>0?c:null},r=await k(t.configPath).catch(()=>null),o=n(r?.wpsuite);if(o)return o;let s=n(e.wpsuite);return s||null}async function _e(e,t){return await Ne(e,t)}function V(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function He(e,t){let n=await w(F(e.runtimeDir),null),r=n&&n.details&&typeof n.details.phase=="string"?n.details.phase.trim():"";if(r)return r;let o=n&&typeof n.currentStep=="string"?n.currentStep.trim():"";if(o)return o;let s=await Y(e).catch(()=>"");if(s){let i=await w(g.join(s,"current-crawl-event.json"),null),u=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(u)return u}return t}function $e(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function Ke(e){let t=g.join(e.runtimeDir,"queue.json"),n=g.join(e.runtimeDir,"current-run.json"),r=await D(e.runtimeDir,async()=>{let o=await w(n,null);if(!o||o.status!=="running"&&o.status!=="queued")return null;let s=await w(t,[]),i=Array.isArray(s)?s.filter(a=>a?.id!==o.id):[],u=V(o);return await S(t,[u,...i]),await S(n,null),u});r&&(await O(e.runtimeDir,r.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:r.id,command:r.command,message:"Recovered stale current-run entry back into queue."}));}async function Ge(e,t,n){try{if((n.shouldEnforceSubscriptionOnJobExecution?.(e)??n.enforceSubscriptionOnJobExecution)&&n.assertActiveSubscriptionForIdentity){let u=await _e(e,t);await n.assertActiveSubscriptionForIdentity(u);}let o=ze(e),s=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await C(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,o,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await C(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await C(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="crawl")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await C(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,o,e.id),{exitCode:0};if(e.command==="deploy")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await C(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="invalidate")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await C(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await C(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,o,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await C(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,o,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(r){return r instanceof I?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:r.request,stoppedStep:r.step}:{exitCode:1,error:r instanceof Error?r.message:String(r)}}}function X(e){return async function(n=process.argv.slice(2)){let r=Re(n),o=g.join(r.runtimeDir,"export.lock"),s=g.join(r.runtimeDir,"last-run.json");await q(r.runtimeDir),await b(r,"starting",{message:"queue-runner starting"});try{await We(o);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await b(r,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await Ke(r);let i=await Ue(r,e),u=0;for(let a=0;a<r.maxJobs;a++){let c=await Ze(r,e);if(c==="none"||(u+=1,c==="stopped"))break}if(u===0)await q(r.runtimeDir),await b(r,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await q(r.runtimeDir);let a=await w(s,null),c=String(a?.status||"").trim();await b(r,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:a?.id,lastJobCommand:a?.command,lastJobStatus:a?.status,lastJobExitCode:a?.exitCode,lastJobError:a?.error,...c==="stopped"?{currentStep:a?.stoppedStep,stopRequestedAt:a?.stopRequestedAt,stopRequestedByLogin:a?.stopRequestedByLogin,stopRequestedMode:a?.stopMode,lastStoppedStep:a?.stoppedStep,message:a?.stopMode==="requeue"?`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and requeued.`:`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await b(r,"error",{message:u}),await A(r.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await Be(o);}}}async function Ye(e,t,n){let r=g.join(e.runtimeDir,"queue.json"),o=g.join(e.runtimeDir,"current-run.json");await D(e.runtimeDir,async()=>{let s=await w(r,[]),i=Array.isArray(s)?s.filter(c=>c?.id!==t.id):[],u=V(t),a=$e(t,n);a?u.resumeFromStep=a:delete u.resumeFromStep,await S(r,[u,...i]),await S(o,null);});}async function Ze(e,t){let n=g.join(e.runtimeDir,"queue.json"),r=g.join(e.runtimeDir,"current-run.json"),o=g.join(e.runtimeDir,"last-run.json"),s=new Date().toISOString(),i=await D(e.runtimeDir,async()=>{let d=await w(n,[]);if(!Array.isArray(d)||d.length===0)return null;let l={...d[0],status:"running",startedAt:s};return await S(n,d.slice(1)),await S(r,l),l});if(!i)return "none";await q(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:s,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await b(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await Ge(i,e,t),a=new Date().toISOString();if(u.stopped){let d=await He(e,u.stoppedStep||i.command),l=u.stopRequest?.mode==="requeue"?"requeue":"stop",m={...i,status:"stopped",endedAt:a,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:l,stoppedStep:d};l==="requeue"?await Ye(e,i,d):await S(r,null);try{let h=await H(m,e);m.logArchiveDir=h.archiveDir,m.logArchiveCreatedAt=h.archiveCreatedAt,m.logArchiveFileCount=h.archivedFiles.length;}catch(h){m.logArchiveError=h instanceof Error?h.message:String(h);}return await S(o,m),await O(e.runtimeDir,i.id),await q(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:l==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:a,deploymentProfile:i.deploymentProfile||"",stopMode:l,stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:d,logArchiveKey:m.logArchiveError||!m.logArchiveDir?"":g.basename(m.logArchiveDir),logArchiveDir:m.logArchiveDir||"",logArchiveCreatedAt:m.logArchiveCreatedAt||"",logArchiveFileCount:m.logArchiveFileCount??0,logArchiveError:m.logArchiveError||""}}),"stopped"}let c={...i,status:u.exitCode===0?"success":"failed",endedAt:a,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await S(r,null),await O(e.runtimeDir,i.id);try{let d=await H(c,e);c.logArchiveDir=d.archiveDir,c.logArchiveCreatedAt=d.archiveCreatedAt,c.logArchiveFileCount=d.archivedFiles.length;}catch(d){c.logArchiveError=d instanceof Error?d.message:String(d);}await S(o,c);let p=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:p,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":g.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}var Xe={allowSchedulerAutoEnqueue:false,enforceSubscriptionOnJobExecution:false},ee=X(Xe),et=ee,bt=ee;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&et().catch(e=>{console.error(e),process.exit(1);});export{bt as default,et as main,ee as runQueueRunner};
|
package/package.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.33",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|
|
7
7
|
"bin": {
|
|
8
|
-
"publisher-exporter": "
|
|
8
|
+
"publisher-exporter": "dist/cli.js"
|
|
9
9
|
},
|
|
10
10
|
"files": [
|
|
11
11
|
"dist",
|
|
@@ -36,29 +36,37 @@
|
|
|
36
36
|
"queue:drain": "node dist/queue-runner.js --max-jobs=100",
|
|
37
37
|
"publish:site": "npm run build && npm run crawl && npm run deploy && npm run invalidate",
|
|
38
38
|
"install:browsers": "playwright install chromium",
|
|
39
|
-
"release:npm": "WPSUITE_PREMIUM=true npm run build && npm publish --access=public"
|
|
39
|
+
"release:npm": "WPSUITE_PREMIUM=true npm run lint && npm run check && npm run build && npm publish --access=public --registry=https://registry.npmjs.org/"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@aws-sdk/client-cloudfront": "^3.
|
|
43
|
-
"@aws-sdk/client-s3": "^3.
|
|
44
|
-
"@aws-sdk/lib-storage": "^3.
|
|
45
|
-
"@smart-cloud/publisher-core": "^1.1.
|
|
46
|
-
"@smart-cloud/wpsuite-core": "^2.3.
|
|
42
|
+
"@aws-sdk/client-cloudfront": "^3.1100.0",
|
|
43
|
+
"@aws-sdk/client-s3": "^3.1100.0",
|
|
44
|
+
"@aws-sdk/lib-storage": "^3.1100.0",
|
|
45
|
+
"@smart-cloud/publisher-core": "^1.1.5",
|
|
46
|
+
"@smart-cloud/wpsuite-core": "^2.3.2",
|
|
47
47
|
"fast-glob": "^3.3.3",
|
|
48
48
|
"mime-types": "^3.0.2",
|
|
49
|
-
"playwright": "^1.
|
|
49
|
+
"playwright": "^1.62.1"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@eslint/js": "^10.0.1",
|
|
53
53
|
"@types/mime-types": "^3.0.1",
|
|
54
|
-
"@types/node": "^
|
|
55
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
56
|
-
"@typescript-eslint/parser": "^8.
|
|
57
|
-
"esbuild": "^0.28.
|
|
58
|
-
"eslint": "^10.
|
|
59
|
-
"globals": "^17.
|
|
54
|
+
"@types/node": "^26.1.2",
|
|
55
|
+
"@typescript-eslint/eslint-plugin": "^8.65.0",
|
|
56
|
+
"@typescript-eslint/parser": "^8.65.0",
|
|
57
|
+
"esbuild": "^0.28.1",
|
|
58
|
+
"eslint": "^10.8.0",
|
|
59
|
+
"globals": "^17.8.0",
|
|
60
60
|
"tsup": "^8.5.1",
|
|
61
61
|
"typescript": "^6.0.3",
|
|
62
|
-
"typescript-eslint": "^8.
|
|
62
|
+
"typescript-eslint": "^8.65.0"
|
|
63
|
+
},
|
|
64
|
+
"overrides": {
|
|
65
|
+
"brace-expansion": "^5.0.8",
|
|
66
|
+
"esbuild": "^0.28.1"
|
|
67
|
+
},
|
|
68
|
+
"resolutions": {
|
|
69
|
+
"brace-expansion": "^5.0.8",
|
|
70
|
+
"esbuild": "^0.28.1"
|
|
63
71
|
}
|
|
64
72
|
}
|