@smart-cloud/publisher-exporter 1.1.10 → 1.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +4 -4
- package/dist/deploy.js +4 -4
- package/dist/invalidate.js +3 -3
- package/dist/queue-runner.js +2 -2
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import S from'fs/promises';import m from'path';import {gunzipSync}from'zlib';import {getConfig}from'@smart-cloud/wpsuite-core';import Dt from'crypto';import $t from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Pt=".deploy-plan.json",St="deploy-plan.json";function bt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?m.resolve(e):""}function Fe(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function pe(e){return [...new Set(e.map(Fe).filter(t=>t!==null))]}function yt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:m.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:pe(t.changedFiles||[]),deletedFiles:pe(t.deletedFiles||[]),rewriteTargets:pe(t.rewriteTargets||[])}}function vt(e){let t=m.join(m.resolve(e.outputDir),Pt),r=bt();return r?[m.join(r,St),t]:[t]}function te(e,t){let r=m.resolve(e),s=m.resolve(t),n=m.relative(r,s).replace(/\\/g,"/");return Fe(n)}async function De(e,t){let r=yt(e,t);for(let s of vt(e))await S.mkdir(m.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function $e(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function xt(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(a=>a.replace(/[^A-Za-z0-9._-]/g,"")).filter(a=>a.length>0&&a!=="."&&a!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&m.extname(i)===""&&(i+="/"),i==="/"?"":i}function ge(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function kt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,a={};if(typeof i.targetOrigin=="string"){let u=$e(i.targetOrigin);u&&(a.targetOrigin=u);}let l=ge(i.extraReplacements);if(Object.keys(l).length>0&&(a.extraReplacements=l),i.s3&&typeof i.s3=="object"){let u={},o=i.s3;typeof o.bucket=="string"&&(u.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(u.prefix=o.prefix.trim()),typeof o.region=="string"&&(u.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(u.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(u.assetCacheControl=o.assetCacheControl.trim()),Object.keys(u).length>0&&(a.s3=u);}if(i.cloudFront&&typeof i.cloudFront=="object"){let u={},o=i.cloudFront;typeof o.distributionId=="string"&&(u.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(u.invalidationPaths=o.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(u).length>0&&(a.cloudFront=u);}t[n]=a;}return t}function je(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Tt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let a=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&a?{deploymentProfile:a}:{},...l?{url:l}:{}}}function Ue(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Tt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function At(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function re(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=At(t.siteSettings),a={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},u=String(t.apiBase??"").trim();u&&(l.apiBase=u);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(a).length>0&&(l.siteSettings=a);let p=je(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Mt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function It(e){let t=re(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Mt(e.sourceOrigin);let a=globalThis,u={...a.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(u.apiBase=t.apiBase),a.WpSuite=u,true}async function Lt(e){let t=re(e.wpsuite),r=t.siteSettings??{},s=String(r.accountId??"").trim()||null,n=String(r.siteId??"").trim()||null,i=String(t.uploadUrl??"").trim();if(!It(e))return console.info(`[INFO] Skipping remote WPSuite publisher config load: ${JSON.stringify({accountId:s,siteId:n,uploadUrl:i||null,subscriber:r.subscriber===true,lastUpdate:r.lastUpdate??null})}`),null;try{let a=await getConfig("publisher"),l=a&&typeof a=="object"?a:null,u=l?.siteSettings&&typeof l.siteSettings=="object"?l.siteSettings:null;return console.info(`[INFO] Loaded remote WPSuite publisher config: ${JSON.stringify({accountId:s,siteId:n,uploadUrl:i||null,subscriber:r.subscriber===!0,lastUpdate:r.lastUpdate??null,remoteType:a===null?"null":typeof a,remoteSubscriptionType:typeof l?.subscriptionType=="string"?l.subscriptionType:null,remoteKeys:l?Object.keys(l).sort():[],remoteAccountId:typeof l?.accountId=="string"?l.accountId:typeof u?.accountId=="string"?u.accountId:null,remoteSiteId:typeof l?.siteId=="string"?l.siteId:typeof u?.siteId=="string"?u.siteId:null,remoteSiteSettingsKeys:u?Object.keys(u).sort():[]})}`),a&&typeof a=="object"?a:null}catch(a){return console.warn(`[WARN] Remote WPSuite publisher config load failed: ${JSON.stringify({accountId:s,siteId:n,uploadUrl:i||null,subscriber:r.subscriber===true,lastUpdate:r.lastUpdate??null,error:a instanceof Error?a.message:String(a)})}`),null}}function Ot(e,t){let r=kt(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=je(t?.subscriptionType),a=re(e.wpsuite);return {...e,scheduler:Ue(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...a,...a.siteSettings||i?{siteSettings:{...a.siteSettings??{},...a.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function me(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Ft(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?m.resolve(e):""}function Ee(e,t,r){let s=String(t||"").trim();return s&&m.isAbsolute(s)?m.resolve(s):m.resolve(e,me(s,r))}async function he(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await S.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=$e(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=xt(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=ge(s.extraReplacements),s.postCrawlCopyMap=ge(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=re(s.wpsuite),s.scheduler=Ue(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Ot(s,await Lt(s)),i=Ft(),a=i?m.resolve(i,".."):"";return a?(n.outputDir=Ee(a,n.outputDir,"export"),n.logDir=Ee(a,n.logDir,"logs")):(m.isAbsolute(n.outputDir)||(n.outputDir=me(n.outputDir,"export")),m.isAbsolute(n.logDir)||(n.logDir=me(n.logDir,"logs"))),n}function fe(e,t=10){return Dt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function we(e){await S.mkdir(m.dirname(e),{recursive:true});}function q(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(m.extname(e).toLowerCase())}function j(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function W(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function se(e){return e.replace(/\//g,"\\/")}function Qe(e){return e.replace(/\//g,"\\\\/")}function ne(e,t,r){if(!t)return;e[t]=r;let s=se(t),n=se(r);e[s]=n;let i=Qe(t),a=Qe(r);e[i]=a;let l=W(t),u=W(r);e[l]=u;let o=W(s),c=W(n);e[o]=c;let p=W(i),d=W(a);e[p]=d;}function Pe(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Ne(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function qe(e,t,r){if(!t)return r;let s=m.dirname(m.resolve(t)),n=m.resolve(e,r.replace(/^\/+/,"")),i=m.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Qt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Nt=new Set([".html",".htm"]),_e="WPSuite.io Static Publisher",qt=_e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Wt(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=m.extname(r).toLowerCase();return Nt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function _t(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${qt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${_e}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import b from'fs/promises';import h from'path';import {gunzipSync}from'zlib';import Tt from'http';import kt from'https';import jt from'crypto';import Qt from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var bt=".deploy-plan.json",yt="deploy-plan.json";function vt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Fe(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function de(e){return [...new Set(e.map(Fe).filter(t=>t!==null))]}function Rt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:h.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:de(t.changedFiles||[]),deletedFiles:de(t.deletedFiles||[]),rewriteTargets:de(t.rewriteTargets||[])}}function Ct(e){let t=h.join(h.resolve(e.outputDir),bt),r=vt();return r?[h.join(r,yt),t]:[t]}function te(e,t){let r=h.resolve(e),s=h.resolve(t),n=h.relative(r,s).replace(/\\/g,"/");return Fe(n)}async function Oe(e,t){let r=Rt(e,t);for(let s of Ct(e))await b.mkdir(h.dirname(s),{recursive:true}),await b.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function At(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&h.extname(i)===""&&(i+="/"),i==="/"?"":i}function pe(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Mt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let a=Ue(i.targetOrigin);a&&(u.targetOrigin=a);}let l=pe(i.extraReplacements);if(Object.keys(l).length>0&&(u.extraReplacements=l),i.s3&&typeof i.s3=="object"){let a={},o=i.s3;typeof o.bucket=="string"&&(a.bucket=o.bucket.trim()),typeof o.prefix=="string"&&(a.prefix=o.prefix.trim()),typeof o.region=="string"&&(a.region=o.region.trim()),typeof o.htmlCacheControl=="string"&&(a.htmlCacheControl=o.htmlCacheControl.trim()),typeof o.assetCacheControl=="string"&&(a.assetCacheControl=o.assetCacheControl.trim()),Object.keys(a).length>0&&(u.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},o=i.cloudFront;typeof o.distributionId=="string"&&(a.distributionId=o.distributionId.trim()),Array.isArray(o.invalidationPaths)&&(a.invalidationPaths=o.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(u.cloudFront=a);}t[n]=u;}return t}function $e(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Lt(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate")&&u?{deploymentProfile:u}:{},...l?{url:l}:{}}}function je(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Lt(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function It(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function he(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=It(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let o=String(t.runtimeToken??t.nonce??"").trim();o&&(l.runtimeToken=o);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(u).length>0&&(l.siteSettings=u);let p=$e(t.subscriptionType);return p&&(l.subscriptionType=p),l}function Dt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function qe(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Ft(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function Qe(e,t,r=5){let s=new URL(qe(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Tt:kt,u=Ft(e,t);return await new Promise((l,a)=>{let o=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},c=>{let p=[];c.on("data",d=>{p.push(Buffer.isBuffer(d)?d:Buffer.from(d));}),c.on("error",a),c.on("end",()=>{let d=c.statusCode??0,g=c.headers.location;if(r>0&&g&&[301,302,303,307,308].includes(d)){l(Qe(new URL(g,s),{method:d===303?"GET":n},r-1));return}let f=new Headers;for(let[m,P]of Object.entries(c.headers))Array.isArray(P)?f.set(m,P.join(", ")):typeof P=="string"&&f.set(m,P);l(new Response(Buffer.concat(p),{status:d,statusText:c.statusMessage??"",headers:f}));});});o.on("error",a),o.end();})}function Ot(e){let t=he(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{uploadUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Dt(e.sourceOrigin);let u=globalThis,a={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),u.WpSuite=a,true}async function Et(e){if(!Ot(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{let s=globalThis,n=()=>({sourceOrigin:e.sourceOrigin,locationHref:s.location?.href??null,locationHostname:s.location?.hostname??null,wpsuite:s.WpSuite?{apiBase:s.WpSuite.apiBase??null,uploadUrl:s.WpSuite.uploadUrl??null,siteSettings:s.WpSuite.siteSettings??null}:null});console.info(`[DEBUG] Remote WPSuite config bootstrap before import: ${JSON.stringify(n())}`),e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(l,a)=>qe(l).startsWith(t)?Qe(l,a):r(l,a)));let{getConfig:i}=await import('@smart-cloud/wpsuite-core');console.info(`[DEBUG] Remote WPSuite config bootstrap before getConfig: ${JSON.stringify(n())}`);let u=await i("publisher");return u&&typeof u=="object"?u:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Ut(e,t){let r=Mt(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=$e(t?.subscriptionType),u=he(e.wpsuite);return {...e,scheduler:je(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function ge(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function $t(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Ee(e,t,r){let s=String(t||"").trim();return s&&h.isAbsolute(s)?h.resolve(s):h.resolve(e,ge(s,r))}async function fe(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await b.readFile(t,"utf8"),s=JSON.parse(r);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Ue(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=At(s.generated404RequestPath),s.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.concurrency||=1,s.maxPages||=0,s.extraReplacements=pe(s.extraReplacements),s.postCrawlCopyMap=pe(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency,s.rewriteConcurrency=Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency,s.wpsuite=he(s.wpsuite),s.scheduler=je(void 0),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let n=Ut(s,await Et(s)),i=$t(),u=i?h.resolve(i,".."):"";return u?(n.outputDir=Ee(u,n.outputDir,"export"),n.logDir=Ee(u,n.logDir,"logs")):(h.isAbsolute(n.outputDir)||(n.outputDir=ge(n.outputDir,"export")),h.isAbsolute(n.logDir)||(n.logDir=ge(n.logDir,"logs"))),n}function me(e,t=10){return jt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function we(e){await b.mkdir(h.dirname(e),{recursive:true});}function N(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(h.extname(e).toLowerCase())}function $(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function W(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function re(e){return e.replace(/\//g,"\\/")}function Ne(e){return e.replace(/\//g,"\\\\/")}function se(e,t,r){if(!t)return;e[t]=r;let s=re(t),n=re(r);e[s]=n;let i=Ne(t),u=Ne(r);e[i]=u;let l=W(t),a=W(r);e[l]=a;let o=W(s),c=W(n);e[o]=c;let p=W(i),d=W(u);e[p]=d;}function Pe(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function We(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function _e(e,t,r){if(!t)return r;let s=h.dirname(h.resolve(t)),n=h.resolve(e,r.replace(/^\/+/,"")),i=h.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var _t=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Bt=new Set([".html",".htm"]),He="WPSuite.io Static Publisher",Ht=He.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function zt(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=h.extname(r).toLowerCase();return Bt.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Jt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Ht}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${He}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Zt(t,r));}updateCurrentEvent(t,r,s){let n=tr(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(U[this.level]>=U.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}debug(t,r){U[this.level]<U.debug||this.push("debug",t,r);}progress(t,r){Xt("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),U[this.level]>=U.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),U[this.level]>=U.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var ir=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],or=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),ar=new Set(["",".html",".htm"]),ur=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),lr=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function cr(e){let t=[],r="full",s="full",n=false,i=false;for(let a=0;a<e.length;a++){let l=e[a];if(l==="--retry-timeouts")r="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let u=e[++a];if(!u)throw new Error("--crawl-mode requires a value");s=u==="incremental"?"incremental":"full",n||=s==="incremental";}else if(l.startsWith("--crawl-mode="))s=l.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(l==="--incremental")s="incremental",n=true;else if(l==="--url"){let u=e[++a];if(!u)throw new Error("--url requires a value");r="single-url",n=true,t.push(u);}else if(l.startsWith("--url="))r="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let u=e[++a];if(!u)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${u}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Te(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?m.resolve(e):""}function dr(){let e=Te();return e?m.resolve(e,".."):""}function pr(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?m.resolve(e):""}function gr(e){let t=e.trim(),r=[{alias:"@storage-root",root:dr(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Te(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:pr(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?m.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:m.resolve(t)}}function it(e){let t=Te();return t?m.join(t,"crawl-manifest.json"):m.join(e.outputDir,".crawl-manifest.json")}function ot(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function mr(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function Q(e,t,r){let s=te(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Ve(e,t,r){let s=te(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ue(e,t){e.rewriteTargets.add(m.resolve(t));}async function hr(e){try{let t=await S.readFile(it(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return ot()}function Ge(e){return JSON.parse(JSON.stringify(e))}async function fr(e,t){let r=it(e);await S.mkdir(m.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}function le(e){if(!e)return null;let t=e.trim();return t||null}function wr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],a=t[n];!i||!a||i===a||r.add(n);}return r}async function Pr(e,t,r,s){let n=[...new Set(r.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!l.ok())return s.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:a,status:l.status()}),new Map;let u=await l.json().catch(()=>null);if(!u||!Array.isArray(u.items))return new Map;let o=new Map;for(let c of u.items)!c||typeof c.url!="string"||o.set(c.url,c);return o}catch(l){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:String(l)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await Pr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,a),a}async function Sr(e,t,r,s,n,i){if(dt(e,n))return {action:"render",changeToken:null};if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await ve(e,t,s,n,i):null};let a=s.manifest.pages[n];if(!a)return {action:"render",changeToken:await ve(e,t,s,n,i)};let l=await ve(e,t,s,n,i);if(l?.supported&&l.token)return a.changeToken===l.token?{action:"reuse",changeToken:l}:{action:"render",changeToken:l};let u=le(r.sitemapLastmodByPage[n]);return u&&a.sitemapLastmod&&a.sitemapLastmod===u?{action:"reuse",changeToken:l}:{action:"render",changeToken:l}}async function br(e,t,r,s,n){let i=Ke(r.previousManifest),a=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ve(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(h){(h.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(h)});}delete r.manifest.pages[d],a++;}let l=Ke(r.manifest,r.runId),u=new Set,o=new Set;for(let d of l){for(let f of G(e,d))u.add(f);let g=r.manifest.assets[d]?.outputPath;g&&o.add(g);let h=ce(e,t.assetMap,d);h&&o.add(h);}for(let d of Object.keys(r.manifest.assets))l.has(d)||delete r.manifest.assets[d];let c=0,p=0;for(let d of i){let g=new Set,h=r.previousManifest.assets[d]?.outputPath;h&&g.add(h);let f=ce(e,t.assetMap,d);if(!l.has(d)&&f)g.add(f);else if(!l.has(d)){let b=Me(e,d);g.add(b.originalFilePath),b.hashedFilePath&&g.add(b.hashedFilePath);}for(let b of g)if(!o.has(b)){Ve(n,e,b);try{await S.unlink(b),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:b,error:String(w)});}}for(let b of G(e,d))u.has(b)||Object.prototype.hasOwnProperty.call(t.assetMap,b)&&(delete t.assetMap[b],p++);}(a>0||c>0||p>0)&&s.info(`Incremental cleanup removed ${a} stale pages, ${c} stale assets, ${p} stale asset mappings`,{removedPages:a,removedAssets:c,removedAssetMappings:p,phase:"incremental-cleanup"});}function Ke(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function Ye(e,t,r){for(let s of t)for(let n of G(e,s))if(r.has(n))return true;return false}function yr(e,t,r,s,n){let i=new Set(r.rewriteTargets),a=wr(s,n);if(a.size===0)return [...i];for(let l of Object.values(t.manifest.pages))Ye(e,l.discoveredAssets,a)&&i.add(m.resolve(l.outputPath));for(let[l,u]of Object.entries(t.manifest.assets)){u.isText&&Ye(e,u.discoveredAssets,a)&&i.add(m.resolve(u.outputPath));for(let o of G(e,l))if(a.has(o)){Q(r,e,u.outputPath),u.isText&&i.add(m.resolve(u.outputPath));break}}return [...i]}function G(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function vr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function at(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return m.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function Rr(e,t,r){for(let s of vr(e,r)){let n=t[s];if(!n)continue;let i=at(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Cr(e,t,r){for(let s of G(e,r)){let n=t[s];if(!n)continue;let i=at(e,n);if(i)return i}return null}function ce(e,t,r){let s=Cr(e,t,r);return s||Me(e,r).originalFilePath}function xr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?m.join(m.resolve(e),"queue-runner-heartbeat.json"):""}async function Xe(e){let t=xr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function kr(e,t){let r=new Set;for(let s of t.donePages)r.add(m.resolve(H(e,s)));for(let s of t.doneAssets){let n=ce(e,t.assetMap,s);!n||!q(n)||r.add(m.resolve(n));}return [...r]}var de="re:",oe=new Map;function Tr(e){if(!e.startsWith(de))return null;let t=oe.get(e);if(t!==void 0)return t||null;let r=e.slice(de.length).trim();if(!r)return oe.set(e,false),null;try{let s=new RegExp(r);return oe.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),oe.set(e,false),null}}function Ae(e,t){let r=Tr(t);return r?r.test(e):t.startsWith(de)?false:e.startsWith(t)}function Ar(e,t){return t.find(r=>Ae(e,r))}function ut(e,t){let r=Ar(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(de)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Mr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Ir(e,t){return Mr(e).has(t.hostname)}function Lr(e){let t=j(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function lt(e){let t=j(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function K(e){return lr.some(t=>e.pathname.startsWith(t))}function k(e,t,r=e.sourceOrigin,s,n="url"){let i=Lr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(lt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let a=new URL(i,r);if(!["http:","https:"].includes(a.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Ir(e,a))return s?.ignore(n,a.toString(),"host not allowed",r),null;let l=ut(e,a);if(l)return s?.reject(n,a.toString(),l,r),null;let u=new URL(e.sourceOrigin);return a.protocol=u.protocol,a.host=u.host,a.hash="",a}catch{return s?.reject(n,t,"invalid URL",r),null}}function ct(e){return or.has(m.extname(e.pathname).toLowerCase())}function z(e){return K(e)?false:ar.has(m.extname(e.pathname).toLowerCase())}function Z(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function x(e,t){return Z(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:ct(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>Ae(t.pathname,r)):false}function H(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":m.extname(s)||(s+="/index.html"),m.join(e.outputDir,s)}function Or(e){if(!e.generated404RequestPath)return null;try{return new URL(e.generated404RequestPath,e.sourceOrigin).toString()}catch{return null}}function dt(e,t){let r=Or(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Me(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=m.join(e.outputDir,i),l=Pe(e.targetOrigin,i),u=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:l,originalPublicUrlWithSearch:u,preferQueryHashed:n};let o=m.extname(s),p=`${o?s.slice(0,-o.length):s}.${fe(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:l,originalPublicUrlWithSearch:u,preferQueryHashed:n,hashedPathname:p,hashedFilePath:m.join(e.outputDir,p),hashedPublicUrl:Pe(e.targetOrigin,p)}}function Fr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let u of l)try{let o=new URL(u);t[o.origin+i.pathname+i.search]=s;}catch{}try{let u=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=u;}catch{}try{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}try{let u=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}}catch{}}async function pt(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function gt(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Dr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function mt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Er(e,t){let r=mt(t),s=Dr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function $r(e,t){if(!t)return;let r=mt(t);if(r===null)return;let s=await gt(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function Re(e,t){let r=m.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return ur.has(r)||ir.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function jr(e){return fe(e.toString("base64"),40)}function Ur(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function Qr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function ke(e,t,r,s,n,i,a){let l=Me(e,r),u=jr(s),o=l.originalFilePath,c=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&Ur(n,l.originalFilePath,u)){let h=Rr(e,t,r);h?(o=h.outputPath,c=h.publicUrl):(o=l.hashedFilePath,c=l.hashedPublicUrl);}Fr(e,t,r,c,l.originalPublicUrl);let p=q(o)?Buffer.from(ye(s.toString("utf8"),e,t,o),"utf8"):s,d=await pt(o),g=false;return (d===null||!d.equals(p))&&(await we(o),await S.writeFile(o,p),g=true),Qr(n,l.originalFilePath,u),await $r(o,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function Y(e,t,r,s,n){let i=()=>{let o=String(n??"").trim();if(!o)return {discoveredFrom:"",discoveredFromType:"unknown"};if(o==="seed-path"||o==="cli"||o==="generated-404-request-path")return {discoveredFrom:o,discoveredFromType:o};if(/^https?:\/\//i.test(o))try{let c=new URL(o);return Z(c)?{discoveredFrom:o,discoveredFromType:"sitemap"}:x(e,c)?{discoveredFrom:o,discoveredFromType:"asset"}:z(c)&&!K(c)?{discoveredFrom:o,discoveredFromType:"page"}:{discoveredFrom:o,discoveredFromType:"url"}}catch{return {discoveredFrom:o,discoveredFromType:"url"}}return {discoveredFrom:o,discoveredFromType:"url"}},a=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=k(e,r,a,s,"page");if(!l)return;if(!z(l)||ct(l)){if(x(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),T(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),K(l)?"asset/internal path cannot be page":"not page-like",n);return}let u=l.toString();if(!t.donePages.has(u)&&!t.queuedPages.has(u)){let{discoveredFrom:o,discoveredFromType:c}=i();t.queuedPages.add(u),t.pageQueue.push(u),t.stats.pagesQueued++,s.debug(`Queued discovered page ${u}`,{phase:"discovery",url:u,discoveredFrom:o,discoveredFromType:c,outputPath:H(e,u)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"});}}function T(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=k(e,r,i,s,"asset");if(!a)return;if(!x(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let l=a.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Ie(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=k(e,r,i,s,"sitemap");if(!a)return;if(!Z(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let l=a.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Nr(e){let t=j(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:le(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function qr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Wr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function xe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function ht(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=m.join(e,n.name),a=m.join(t,n.name);if(n.isDirectory()){r.push(...await ht(i,a));continue}await S.copyFile(i,a),r.push(a);}return r}async function _r(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,a])=>({sourcePath:i.trim(),prefix:String(a||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let a=i.sourcePath,l=i.prefix;if(!a||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:l});continue}let u=gr(a);if(!u.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:u.alias||"",requiredEnv:u.requiredEnv||"",prefixPath:l});continue}let o=u.resolvedPath,c=xe(l);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:l});continue}let p;try{p=await S.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:o,prefixPath:l});continue}let d=m.resolve(e.outputDir,c);if(p.isDirectory()){n.push(...await ht(o,d)),s++,t.progress(`Copied static directory to export: ${o} -> /${c}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${c}`,copiedItems:s});continue}let g=m.basename(o),h=l.endsWith("/")?m.join(d,g):d;await S.mkdir(m.dirname(h),{recursive:true}),await S.copyFile(o,h),n.push(h),s++,t.progress(`Copied static file to export: ${o} -> /${xe(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${xe(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function X(e,t,r,s,n){let i=new Set,a=j(r).replace(/\\\//g,"/");for(let o of Wr(a)){let c=k(e,o,t,s,`${n}:css-url`);c&&x(e,c)&&i.add(c.toString());}for(let o of a.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=k(e,o[1],t,s,`${n}:css-import`);c&&x(e,c)&&i.add(c.toString());}for(let o of a.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=k(e,o[1],t,s,`${n}:xml-stylesheet`);c&&x(e,c)&&i.add(c.toString());}let l="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",u=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let o of a.matchAll(u)){let c=o[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${o[0]}`:o[0];if(lt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=k(e,c,t,s,`${n}:serialized-url`);p&&x(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Br(e,t,r,s){let n=new Set,i=j(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=a[1],u=k(e,l,t,s,"page-link");u&&(ut(e,u)||z(u)&&!K(u)&&!x(e,u)&&n.add(u.toString()));}return [...n]}async function Ze(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let a=await i.body(),l=i.url()||n;await ke(e,r.assetMap,l,a,r,s);let u=a.toString("utf8");for(let o of X(e,l,u,s,"sitemap"))T(e,r,o,s,l);for(let o of Nr(u)){let c=k(e,o.loc,l,s,"sitemap-loc");c&&(Z(c)?Ie(e,r,c.toString(),s,l):x(e,c)?T(e,r,c.toString(),s,l):z(c)?(o.lastmod&&(r.sitemapLastmodByPage[c.toString()]=o.lastmod),Y(e,r,c.toString(),s,l)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",l));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Hr(e,t,r,s,n,i,a){let l=k(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!x(e,l))return;let u=l.toString();if(r.doneAssets.has(u))return;let o=i.enabled?i.previousManifest.assets[u]:void 0,c=(p,d={})=>{if(!o)return false;i.manifest.assets[u]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(u);for(let g of o.discoveredAssets||[])T(e,r,g,n,u);return n.warn(`Keeping previous incremental asset ${u} after fetch issue.`,{url:u,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${u}`,{url:u});try{let p=ce(e,r.assetMap,u),d=p?await gt(p):null;if(p&&d)try{let v=await t.head(u,{timeout:6e4}),R=v.headers();if(v.ok()&&Er(d,R)){let N=await pt(p);if(N!==null){let L=v.url()||u,O=await ke(e,r.assetMap,L,N,r,n,R);O.wroteFile&&(Q(a,e,O.outputPath),q(O.outputPath)&&ue(a,O.outputPath)),r.doneAssets.add(u),n.info(`Reused existing asset ${u} based on response headers.`,{url:u,filePath:p,lastModified:R["last-modified"]||"",contentLength:R["content-length"]||"",phase:"download-assets"}),Ce(r,n);let F=Re(l,R),P=o?.discoveredAssets?[...o.discoveredAssets]:[];if(F||o?.isText){let C=N.toString("utf8"),J=X(e,L,C,n,"asset:cached");(J.length>0||!o)&&(P=J);for(let ee of P)T(e,r,ee,n,L);}i.trackManifest&&(i.manifest.assets[u]={url:u,outputPath:O.outputPath,isText:F||!!o?.isText,discoveredAssets:P,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let g=await t.get(u,{timeout:6e4});if(!g.ok()){if(c(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",u,g.status());return}let h=await g.body(),f=g.url()||u,b=g.headers(),w=await ke(e,r.assetMap,f,h,r,n,b);w.wroteFile&&(Q(a,e,w.outputPath),q(w.outputPath)&&ue(a,w.outputPath)),r.doneAssets.add(u),Ce(r,n);let y=[];if(Re(l,b)){let v=h.toString("utf8");y=X(e,f,v,n,`asset:${m.extname(l.pathname).toLowerCase()||b["content-type"]||"unknown"}`);for(let R of y)T(e,r,R,n,f);}i.trackManifest&&(i.manifest.assets[u]={url:u,outputPath:w.outputPath,isText:Re(l,b),discoveredAssets:y,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(p){if(c("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${u}`,{url:u,error:String(p)});}}async function zr(e,t,r,s,n,i){let a=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:a},async()=>{for(;r.assetQueue.length>0;){let u=r.assetQueue.shift();u&&(r.queuedAssets.delete(u),await Hr(e,t,r,u,s,n,i));}});await Promise.all(l);}function Jr(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function B(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function ae(e){return Math.max(15e3,Math.min(6e4,B(e)))}function Vr(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function Le(e){return e.stats.pagesCompleted}function ft(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function Gr(e,t,r){let s=ft(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function E(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${Jr(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function Kr(e,t){let r=Vr(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,a=700,l=Date.now(),u=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){u();return}window.scrollBy(0,a),i+=a;let c=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(c,p);i>=d+window.innerHeight&&u();},120);});},{maxDurationMs:r});}async function Yr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Kr(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Xr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Zr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Ae(t,r))}async function es(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let u=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let h of o){let f=d.getAttribute(h);f&&u.add(f);}for(let h of p){let f=d.getAttribute(h);f&&f.split(",").forEach(b=>{let w=b.trim().split(/\s+/)[0];w&&u.add(w);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...u],styles:[...c]}});for(let u of l.attrs)for(let o of qr(u)){let c=k(e,o,r,s,"dom-attr");c&&x(e,c)&&i.add(c.toString());}for(let u of l.styles)for(let o of X(e,r,u,s,"dom-style"))i.add(o);}let a=await t.content();for(let l of X(e,r,a,s,"page-html"))i.add(l);return [...i]}async function et(e,t,r,s,n){let i=H(e,t);await E(`save page ${t}`,B(e),async()=>{await we(i),await S.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let a=Math.min(s.stats.pagesQueued,Le(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${a}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:a,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ts(e){let t=Le(e);return t%5===0||t===1||t===e.stats.pagesQueued}function rs(e,t){let r=Le(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function ss(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function ns(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await ss(s),s}async function is(e,t,r,s,n,i,a){let l=await chromium.launch({headless:true}),u=ft(l);u.pid!==null&&r.debug(`Launched crawl worker browser PID ${u.pid}`,{browserPid:u.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let c=null,p=null;try{let d=new URL(o),g=dt(e,o),h=Zr(e,d.pathname);if(!z(d)||x(e,d)||K(d)){if(x(e,d)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),T(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let f=await Sr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let P=i.manifest.pages[o];if(f.action==="reuse"&&P){for(let C of P.discoveredAssets)T(e,t,C,r,o);for(let C of P.discoveredPages)Y(e,t,C,r,o);P.lastSeenRunId=i.runId,P.sitemapLastmod=le(t.sitemapLastmodByPage[o])??P.sitemapLastmod,f.changeToken?.supported&&(P.changeToken=f.changeToken.token,P.tokenSource=f.changeToken.tokenSource??P.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:f.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}c=await ns(l,e,h),p=await c.newPage();let b=!1,w=new Set;await p.route("**/*",async P=>{let C=P.request();if(C.isNavigationRequest()&&C.frame()===p.mainFrame())try{let J=new URL(C.url()),ee=new URL(o);if(J.hash="",ee.hash="",J.href===ee.href){if(b){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:C.url()}),await P.abort("aborted");return}b=!0;}}catch{}await P.continue();}),p.on("response",P=>{try{let C=k(e,P.url(),e.sourceOrigin,void 0,"network-response");C&&x(e,C)&&(w.add(C.toString()),T(e,t,C.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await E(`read response body ${o}`,B(e),()=>y.text());}catch{v=null;}}catch(P){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(P)});}if(g){let P=y?.status()??null;if(P!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:P,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:P,outputPath:H(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status()),await p.close();continue}h?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await Xr(e,p)):await Yr(e,p),t.stats.pagesRendered++;let R=await E(`extract rendered assets ${o}`,B(e),()=>es(e,p,o,r,!h));for(let P of R)T(e,t,P,r,o);let N=[...new Set([...R,...w])],L=v??await E(`capture rendered html ${o}`,B(e),()=>p.content()),O=s!=="single-url"&&!g?Br(e,o,L,r):[];if(s!=="single-url"&&!g)for(let P of O)Y(e,t,P,r,o);await et(e,o,L,t,r);let F=H(e,o);Q(a,e,F),ue(a,F),i.trackManifest&&(i.manifest.pages[o]={url:o,outputPath:F,changeToken:f.changeToken?.supported?f.changeToken.token:null,tokenSource:f.changeToken?.supported?f.changeToken.tokenSource??null:null,sitemapLastmod:le(t.sitemapLastmodByPage[o])??null,discoveredPages:O,discoveredAssets:N,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(d){try{let g=p?await E(`capture partial html ${o}`,B(e),()=>p.content()):"";if(g&&g.trim()){await et(e,o,g,t,r);let h=H(e,o);Q(a,e,h),ue(a,h),r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(d)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(d)});}finally{t.stats.pagesCompleted++,ts(t)&&rs(t,r),p&&await E(`close page ${o}`,ae(e),()=>p.close()).catch(d=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:u.pid,error:String(d)});}),c&&await E(`close page context ${o}`,ae(e),()=>c.close()).catch(d=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:u.pid,error:String(d)});});}}}finally{await E("close worker browser",ae(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:u.pid,error:String(o)}),Gr(l,r,"browser.close timeout")));}}async function os(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function tt(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function as(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function us(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function wt(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function rt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function ls(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>rt(s)-rt(n)).map(s=>m.join(e,String(s.storedFileName))):[];return [...new Set([...r,m.join(e,"errors.jsonl"),m.join(e,"errors.json"),m.join(e,"errors.jsonl.gz"),m.join(e,"errors.json.gz")])]}async function st(e){let t=await wt(m.join(e,"job.json"));for(let r of ls(e,t)){let s=await os(r);if(s.length>0)return s}return []}async function cs(e){let t=m.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=m.join(t,i);if(!(await S.stat(a).catch(()=>null))?.isDirectory())continue;let u=await wt(m.join(a,"job.json"));if(!as(u))continue;let o=us(u);o>=n&&(n=o,s=a);}return s}async function ds(e){let t=await cs(e),r=t!==""?await st(t):await st(e.logDir),s=new Set;for(let n of r)if(tt(n.error)||tt(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function ps(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function gs(e){try{let t=await S.readFile(m.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ms(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(m.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function nt(e,t,r,s,n){let i=k(e,r,e.sourceOrigin,s,"manual-url");i&&(Z(i)?Ie(e,t,i.toString(),s,n):x(e,i)?T(e,t,i.toString(),s,n):z(i)?Y(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function hs(){let e=await he(),t=cr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await ds(e)),t.mode==="single-url"&&(r=await ps(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,a=t.mode!=="full"||t.resumeRewrite||n,l=a?await gs(e):{},u=n?await hr(e):ot(),o={enabled:n,trackManifest:i,manifest:Ge(u),previousManifest:Ge(u),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await ms(e.logDir);let c={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=mr();a||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let d=new V(e.logDir,e.logLevel);if(d.info("Resolved WPSuite publisher subscription state for crawl startup.",{phase:"startup",mode:t.mode,requestedMode:t.crawlMode,requestedIncremental:s,incrementalEnabled:n,configSubscriptionType:e.subscriptionType??null,wpsuiteSubscriptionType:e.wpsuite?.subscriptionType??null,subscriber:e.wpsuite?.siteSettings?.subscriber===true,hasAccountId:!!String(e.wpsuite?.siteSettings?.accountId??"").trim(),hasSiteId:!!String(e.wpsuite?.siteSettings?.siteId??"").trim(),hasUploadUrl:!!String(e.wpsuite?.uploadUrl??"").trim(),hasRuntimeToken:!!String(e.wpsuite?.runtimeToken??"").trim(),lastUpdate:e.wpsuite?.siteSettings?.lastUpdate??null,apiBase:e.wpsuite?.apiBase??null,uploadUrl:e.wpsuite?.uploadUrl??null}),s&&!n&&d.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)d.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(d.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Ie(e,c,v,d)),await Ze(e,w,c,d),e.seedPaths.forEach(v=>nt(e,c,v,d,"seed-path")),e.generated404RequestPath&&Y(e,c,e.generated404RequestPath,d,"generated-404-request-path")):(r.forEach(v=>nt(e,c,v,d,"cli")),await Ze(e,w,c,d)),d.endMark("discovery",{pages:c.pageQueue.length,assets:c.assetQueue.length,sitemaps:c.sitemapQueue.length}),d.summary(`Queued ${c.pageQueue.length} pages, ${c.assetQueue.length} assets, ${c.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:c.pageQueue.length,queuedAssets:c.assetQueue.length,queuedSitemaps:c.sitemapQueue.length}),t.mode==="retry-timeouts"&&c.pageQueue.length===0&&c.assetQueue.length===0&&c.sitemapQueue.length===0){d.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await d.flush();return}d.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>is(e,c,d,t.mode,w,o,p));if(await Promise.all(y),d.endMark("render-pages",{pages:c.stats.pagesCompleted}),d.mark("download-assets"),d.progress("Asset download phase started.",{phase:"download-assets",doneAssets:c.doneAssets.size,assetsQueued:c.stats.assetsQueued,assetQueue:c.assetQueue.length}),await zr(e,w,c,d,o,p),d.endMark("download-assets",{assets:c.doneAssets.size}),o.enabled&&e.maxPages===0&&await br(e,c,o,d,p),t.mode==="full"){let v=await _r(e,d);for(let R of v)Q(p,e,R);}}finally{await E("dispose crawl request context",ae(e),()=>w.dispose()).catch(y=>{d.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&(o.manifest.updatedAt=new Date().toISOString(),await fr(e,o.manifest)),await S.writeFile(m.join(e.outputDir,"asset-map.json"),JSON.stringify(c.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(c.assetMap,null,2)&&Q(p,e,m.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?kr(e,c):o.enabled&&!t.resumeRewrite?yr(e,o,p,l,c.assetMap):void 0;d.mark("rewrite-text");let h=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";d.progress(h,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await Xe(h);let f=Date.now(),b=await Be(e,c.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:R,changed:N})=>{N&&Q(p,e,m.join(e.outputDir,R)),d.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R});let L=Date.now();if(!(w===1||w===y||L-f>=5e3))return;f=L;let F=`Rewriting text files: ${w}/${y}`;d.progress(F,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R}),await Xe(F);}});d.endMark("rewrite-text",{changedTextFiles:b}),await De(e,{generatedAt:new Date().toISOString(),outputDir:m.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>te(e.outputDir,m.isAbsolute(w)?w:m.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),d.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${b} text files.`:`Done. Rendered ${c.stats.pagesRendered} pages, processed ${c.doneSitemaps.size} sitemaps, downloaded ${c.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...c.stats,pagesRendered:c.stats.pagesRendered,donePages:c.stats.pagesCompleted,claimedPages:c.donePages.size,doneSitemaps:c.doneSitemaps.size,doneAssets:c.doneAssets.size,changedTextFiles:b}),await d.flush();}hs().catch(async e=>{console.error(e);try{let r=(await he().catch(()=>null))?.logDir??"logs",s=new V(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Gt(e){let t=$(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Vt(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,re(i)]).filter(([i,u])=>i!==u).sort((i,u)=>u[0].length-i[0].length);if(!r.length)return e;let s=i=>{let u=i;for(let[l,a]of r)u=u.split(l).join(a);return u},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,u,l,a)=>`${u}${s(l)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,u,l,a,o)=>Gt(a)?`${u}${s(a)}${o}`:i),n}function Kt(e,t){if(!t)return null;let r=h.dirname(h.resolve(t)),s=h.resolve(e.outputDir),n=h.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Yt(e,t,r){let s=Kt(t,r);if(!s)return e;let n=e;for(let i of _t){let u=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${u}`,"g"),a=new RegExp(`(?<!\\.)/${u}`,"g");n=n.replace(l,`${s}${i}`).replace(a,`${s}${i}`);let o=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),p=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),f=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${c}${o}`).replace(g,`${c}${o}`).replace(f,`${c}${o}`);}return n}function be(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:We(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${_e(e.outputDir,r,n)}${i}`}function Se(e,t,r,s,n){se(e,r,be(t,s,n));}function Xt(e,t,r,s,n){if(!r||r===s)return;se(e,r,s);let i=be(t,r,n),u=be(t,s,n);se(e,i,u);}function ye(e,t,r,s,n={}){let i=e,u={};Se(u,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,o]of Object.entries(t.extraReplacements))Se(u,t,a,o,s);for(let[a,o]of Object.entries(r))Se(u,t,a,o,s);for(let[a,o]of Object.entries(n)){let c=r[a];!c||c===o||Xt(u,t,o,c,s);}let l=Object.entries(u).sort((a,o)=>o[0].length-a[0].length);for(let[a,o]of l)i=i.split(a).join(o);return t.urlRewriteMode!=="absolute"&&(i=Yt(i,t,s)),i=Vt(i,l.map(([,a])=>a)),zt(t,i,s)&&(i=Jt(i)),i}async function Zt(e,t,r,s){let n=h.join(e.outputDir,s),i=h.extname(n).toLowerCase(),u;try{u=await b.readFile(n,"utf8");}catch(o){if(o.code!=="ENOENT")throw o;return {changed:false}}let a=ye(u,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==u?(await b.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function er(e,t){return await new Promise((r,s)=>{let n=a=>{if(l(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{l(),s(a);},u=a=>{l(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",n),e.off("error",i),e.off("exit",u);};e.on("message",n),e.on("error",i),e.on("exit",u),e.postMessage({file:t});})}async function ze(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>h.resolve(d)).filter(d=>d.startsWith(h.resolve(e.outputDir))).map(d=>h.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Qt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&N(d)),i=0;if(n.length===0)return 0;let u=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(u,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:l});let a=0,o=0,c=null;if(l===1){for(let d of n){let g=await Zt(e,t,r.previousAssetMap??{},d);g.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:d,changed:g.changed});}return i}let p=Array.from({length:l},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let g=a;if(g>=n.length)return;a+=1;let f=n[g],m=await er(d,f);m.changed&&(i+=1),o+=1,await r.onProgress?.({index:o,totalFiles:n.length,changedTextFiles:i,file:f,changed:m.changed});}}catch(g){throw c=g instanceof Error?g:new Error(String(g)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var j={error:0,warn:1,info:2,debug:3},Je=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",ne=Je?h.join(Je,"current-progress.json"):"",Ge=Promise.resolve();function Ve(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function tr(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let u=r[n];if(typeof u=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(u,i);continue}r[n]=i;}return r}function rr(e,t,r){ne&&(Ge=Ge.then(async()=>{let s=new Date,n=s.toISOString();await b.mkdir(h.dirname(ne),{recursive:true});let i=await b.readFile(ne,"utf8").then(m=>JSON.parse(m)).catch(()=>null),u=i&&typeof i.details=="object"&&i.details?i.details:{},l=Ve(e),a=i?.currentStep||Ve(i?.source||""),o=i?.startedAt||n,c={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==l&&i?.stepStartedAt){let m=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+m,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(o).getTime())/1e3)),f={checkedAt:n,source:e,message:t,details:tr(u,r??{}),startedAt:o,currentStep:l,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:c};await b.writeFile(ne,JSON.stringify(f,null,2),"utf8");}).catch(()=>{}));}async function sr(e,t){await b.mkdir(h.dirname(e),{recursive:true}),await b.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function nr(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function ir(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:nr(e,r),level:e,message:t,details:r??{}}}var G=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return h.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return h.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return h.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return h.join(this.logDir,"ignored.jsonl")}get skippedPath(){return h.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return h.join(this.logDir,"errors.jsonl")}get timingsPath(){return h.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await b.mkdir(this.logDir,{recursive:true}),await b.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await b.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>sr(t,r));}updateCurrentEvent(t,r,s){let n=ir(t,r,s);this.enqueueTask(async()=>{await b.mkdir(this.logDir,{recursive:true}),await b.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(j[this.level]>=j.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}debug(t,r){j[this.level]<j.debug||this.push("debug",t,r);}progress(t,r){rr("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),j[this.level]>=j.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),j[this.level]>=j.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var lr=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],cr=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),dr=new Set(["",".html",".htm"]),pr=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),gr=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function hr(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let l=e[u];if(l==="--retry-timeouts")r="retry-timeouts",n=true;else if(l==="--resume")n=true;else if(l==="--resume-rewrite")n=true,i=true;else if(l==="--crawl-mode"){let a=e[++u];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(l.startsWith("--crawl-mode="))s=l.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(l==="--incremental")s="incremental",n=true;else if(l==="--url"){let a=e[++u];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(l.startsWith("--url="))r="single-url",n=true,t.push(l.slice(6));else if(l==="--urls"){let a=e[++u];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function ke(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function fr(){let e=ke();return e?h.resolve(e,".."):""}function mr(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?h.resolve(e):""}function wr(e){let t=e.trim(),r=[{alias:"@storage-root",root:fr(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:ke(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:mr(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?h.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:h.resolve(t)}}function at(e){let t=ke();return t?h.join(t,"crawl-manifest.json"):h.join(e.outputDir,".crawl-manifest.json")}function ut(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Pr(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function q(e,t,r){let s=te(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Ke(e,t,r){let s=te(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ae(e,t){e.rewriteTargets.add(h.resolve(t));}async function Sr(e){try{let t=await b.readFile(at(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return ut()}function Ye(e){return JSON.parse(JSON.stringify(e))}async function br(e,t){let r=at(e);await b.mkdir(h.dirname(r),{recursive:true}),await b.writeFile(r,JSON.stringify(t,null,2),"utf8");}function ue(e){if(!e)return null;let t=e.trim();return t||null}function yr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function vr(e,t,r,s){let n=[...new Set(r.map(l=>l.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let l=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!l.ok())return s.warn(`Change token lookup failed with HTTP ${l.status()}`,{endpoint:u,status:l.status()}),new Map;let a=await l.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let o=new Map;for(let c of a.items)!c||typeof c.url!="string"||o.set(c.url,c);return o}catch(l){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(l)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await vr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function Rr(e,t,r,s,n,i){if(gt(e,n))return {action:"render",changeToken:null};if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await ve(e,t,s,n,i):null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await ve(e,t,s,n,i)};let l=await ve(e,t,s,n,i);if(l?.supported&&l.token)return u.changeToken===l.token?{action:"reuse",changeToken:l}:{action:"render",changeToken:l};let a=ue(r.sitemapLastmodByPage[n]);return a&&u.sitemapLastmod&&u.sitemapLastmod===a?{action:"reuse",changeToken:l}:{action:"render",changeToken:l}}async function Cr(e,t,r,s,n){let i=Xe(r.previousManifest),u=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ke(n,e,g.outputPath);try{await b.unlink(g.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(f)});}delete r.manifest.pages[d],u++;}let l=Xe(r.manifest,r.runId),a=new Set,o=new Set;for(let d of l){for(let m of V(e,d))a.add(m);let g=r.manifest.assets[d]?.outputPath;g&&o.add(g);let f=le(e,t.assetMap,d);f&&o.add(f);}for(let d of Object.keys(r.manifest.assets))l.has(d)||delete r.manifest.assets[d];let c=0,p=0;for(let d of i){let g=new Set,f=r.previousManifest.assets[d]?.outputPath;f&&g.add(f);let m=le(e,t.assetMap,d);if(!l.has(d)&&m)g.add(m);else if(!l.has(d)){let P=Me(e,d);g.add(P.originalFilePath),P.hashedFilePath&&g.add(P.hashedFilePath);}for(let P of g)if(!o.has(P)){Ke(n,e,P);try{await b.unlink(P),c++;}catch(w){(w.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:P,error:String(w)});}}for(let P of V(e,d))a.has(P)||Object.prototype.hasOwnProperty.call(t.assetMap,P)&&(delete t.assetMap[P],p++);}(u>0||c>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${c} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:c,removedAssetMappings:p,phase:"incremental-cleanup"});}function Xe(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function Ze(e,t,r){for(let s of t)for(let n of V(e,s))if(r.has(n))return true;return false}function xr(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=yr(s,n);if(u.size===0)return [...i];for(let l of Object.values(t.manifest.pages))Ze(e,l.discoveredAssets,u)&&i.add(h.resolve(l.outputPath));for(let[l,a]of Object.entries(t.manifest.assets)){a.isText&&Ze(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let o of V(e,l))if(u.has(o)){q(r,e,a.outputPath),a.isText&&i.add(h.resolve(a.outputPath));break}}return [...i]}function V(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Tr(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function lt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function kr(e,t,r){for(let s of Tr(e,r)){let n=t[s];if(!n)continue;let i=lt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Ar(e,t,r){for(let s of V(e,r)){let n=t[s];if(!n)continue;let i=lt(e,n);if(i)return i}return null}function le(e,t,r){let s=Ar(e,t,r);return s||Me(e,r).originalFilePath}function Mr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function et(e){let t=Mr();if(t)try{let r=await b.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await b.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Lr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(H(e,s)));for(let s of t.doneAssets){let n=le(e,t.assetMap,s);!n||!N(n)||r.add(h.resolve(n));}return [...r]}var ce="re:",ie=new Map;function Ir(e){if(!e.startsWith(ce))return null;let t=ie.get(e);if(t!==void 0)return t||null;let r=e.slice(ce.length).trim();if(!r)return ie.set(e,false),null;try{let s=new RegExp(r);return ie.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),ie.set(e,false),null}}function Ae(e,t){let r=Ir(t);return r?r.test(e):t.startsWith(ce)?false:e.startsWith(t)}function Dr(e,t){return t.find(r=>Ae(e,r))}function ct(e,t){let r=Dr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(ce)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Fr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Or(e,t){return Fr(e).has(t.hostname)}function Er(e){let t=$(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function dt(e){let t=$(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function K(e){return gr.some(t=>e.pathname.startsWith(t))}function T(e,t,r=e.sourceOrigin,s,n="url"){let i=Er(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(dt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Or(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let l=ct(e,u);if(l)return s?.reject(n,u.toString(),l,r),null;let a=new URL(e.sourceOrigin);return u.protocol=a.protocol,u.host=a.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function pt(e){return cr.has(h.extname(e.pathname).toLowerCase())}function z(e){return K(e)?false:dr.has(h.extname(e.pathname).toLowerCase())}function Z(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function x(e,t){return Z(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:pt(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>Ae(t.pathname,r)):false}function H(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function Ur(e){if(!e.generated404RequestPath)return null;try{return new URL(e.generated404RequestPath,e.sourceOrigin).toString()}catch{return null}}function gt(e,t){let r=Ur(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Me(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),l=Pe(e.targetOrigin,i),a=`${l}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n};let o=h.extname(s),p=`${o?s.slice(0,-o.length):s}.${me(r.search)}${o||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:l,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:h.join(e.outputDir,p),hashedPublicUrl:Pe(e.targetOrigin,p)}}function $r(e,t,r,s,n){t[r]=s;try{let i=new URL(r),l=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{l.push(new URL(e.targetOrigin).origin);}catch{}for(let a of l)try{let o=new URL(a);t[o.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${o.pathname}${o.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function ht(e){try{return await b.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ft(e){try{return await b.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function jr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function mt(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function qr(e,t){let r=mt(t),s=jr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function Qr(e,t){if(!t)return;let r=mt(t);if(r===null)return;let s=await ft(e);if(!s)return;let n=new Date(r);await b.utimes(e,s.atime,n).catch(()=>{});}function Re(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return pr.has(r)||lr.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Nr(e){return me(e.toString("base64"),40)}function Wr(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function _r(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Te(e,t,r,s,n,i,u){let l=Me(e,r),a=Nr(s),o=l.originalFilePath,c=l.originalPublicUrlWithSearch;if(l.preferQueryHashed&&l.hashedFilePath&&l.hashedPublicUrl&&Wr(n,l.originalFilePath,a)){let f=kr(e,t,r);f?(o=f.outputPath,c=f.publicUrl):(o=l.hashedFilePath,c=l.hashedPublicUrl);}$r(e,t,r,c,l.originalPublicUrl);let p=N(o)?Buffer.from(ye(s.toString("utf8"),e,t,o),"utf8"):s,d=await ht(o),g=false;return (d===null||!d.equals(p))&&(await we(o),await b.writeFile(o,p),g=true),_r(n,l.originalFilePath,a),await Qr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:g}}function Y(e,t,r,s,n){let i=()=>{let o=String(n??"").trim();if(!o)return {discoveredFrom:"",discoveredFromType:"unknown"};if(o==="seed-path"||o==="cli"||o==="generated-404-request-path")return {discoveredFrom:o,discoveredFromType:o};if(/^https?:\/\//i.test(o))try{let c=new URL(o);return Z(c)?{discoveredFrom:o,discoveredFromType:"sitemap"}:x(e,c)?{discoveredFrom:o,discoveredFromType:"asset"}:z(c)&&!K(c)?{discoveredFrom:o,discoveredFromType:"page"}:{discoveredFrom:o,discoveredFromType:"url"}}catch{return {discoveredFrom:o,discoveredFromType:"url"}}return {discoveredFrom:o,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,l=T(e,r,u,s,"page");if(!l)return;if(!z(l)||pt(l)){if(x(e,l)){s.info(`Seeded/non-page URL queued as asset: ${l.toString()}`,{url:l.toString(),source:n}),k(e,t,l.toString(),s,n);return}s.reject("page",l.toString(),K(l)?"asset/internal path cannot be page":"not page-like",n);return}let a=l.toString();if(!t.donePages.has(a)&&!t.queuedPages.has(a)){let{discoveredFrom:o,discoveredFromType:c}=i();t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:o,discoveredFromType:c,outputPath:H(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"});}}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"asset");if(!u)return;if(!x(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let l=u.toString();!t.doneAssets.has(l)&&!t.queuedAssets.has(l)&&(t.queuedAssets.add(l),t.assetQueue.push(l),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Le(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=T(e,r,i,s,"sitemap");if(!u)return;if(!Z(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let l=u.toString();!t.doneSitemaps.has(l)&&!t.queuedSitemaps.has(l)&&(t.queuedSitemaps.add(l),t.sitemapQueue.push(l),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Br(e){let t=$(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:ue(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Hr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function zr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function xe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function wt(e,t){let r=[];await b.mkdir(t,{recursive:true});let s=await b.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await wt(i,u));continue}await b.copyFile(i,u),r.push(u);}return r}async function Jr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,l=i.prefix;if(!u||!l){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:l});continue}let a=wr(u);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:l});continue}let o=a.resolvedPath,c=xe(l);if(!c){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:l});continue}let p;try{p=await b.stat(o);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:o,prefixPath:l});continue}let d=h.resolve(e.outputDir,c);if(p.isDirectory()){n.push(...await wt(o,d)),s++,t.progress(`Copied static directory to export: ${o} -> /${c}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${c}`,copiedItems:s});continue}let g=h.basename(o),f=l.endsWith("/")?h.join(d,g):d;await b.mkdir(h.dirname(f),{recursive:true}),await b.copyFile(o,f),n.push(f),s++,t.progress(`Copied static file to export: ${o} -> /${xe(l.endsWith("/")?`${l}${g}`:l)}`,{phase:"copy-extra-paths",sourcePath:o,targetPath:`/${xe(l.endsWith("/")?`${l}${g}`:l)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function X(e,t,r,s,n){let i=new Set,u=$(r).replace(/\\\//g,"/");for(let o of zr(u)){let c=T(e,o,t,s,`${n}:css-url`);c&&x(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=T(e,o[1],t,s,`${n}:css-import`);c&&x(e,c)&&i.add(c.toString());}for(let o of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=T(e,o[1],t,s,`${n}:xml-stylesheet`);c&&x(e,c)&&i.add(c.toString());}let l="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${l}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let o of u.matchAll(a)){let c=o[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${o[0]}`:o[0];if(dt(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=T(e,c,t,s,`${n}:serialized-url`);p&&x(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Gr(e,t,r,s){let n=new Set,i=$(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let l=u[1],a=T(e,l,t,s,"page-link");a&&(ct(e,a)||z(a)&&!K(a)&&!x(e,a)&&n.add(a.toString()));}return [...n]}async function tt(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),l=i.url()||n;await Te(e,r.assetMap,l,u,r,s);let a=u.toString("utf8");for(let o of X(e,l,a,s,"sitemap"))k(e,r,o,s,l);for(let o of Br(a)){let c=T(e,o.loc,l,s,"sitemap-loc");c&&(Z(c)?Le(e,r,c.toString(),s,l):x(e,c)?k(e,r,c.toString(),s,l):z(c)?(o.lastmod&&(r.sitemapLastmodByPage[c.toString()]=o.lastmod),Y(e,r,c.toString(),s,l)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",l));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Vr(e,t,r,s,n,i,u){let l=T(e,s,e.sourceOrigin,n,"asset-fetch");if(!l||!x(e,l))return;let a=l.toString();if(r.doneAssets.has(a))return;let o=i.enabled?i.previousManifest.assets[a]:void 0,c=(p,d={})=>{if(!o)return false;i.manifest.assets[a]={...o,discoveredAssets:[...o.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let g of o.discoveredAssets||[])k(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),Ce(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=le(e,r.assetMap,a),d=p?await ft(p):null;if(p&&d)try{let v=await t.head(a,{timeout:6e4}),R=v.headers();if(v.ok()&&qr(d,R)){let Q=await ht(p);if(Q!==null){let I=v.url()||a,D=await Te(e,r.assetMap,I,Q,r,n,R);D.wroteFile&&(q(u,e,D.outputPath),N(D.outputPath)&&ae(u,D.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:R["last-modified"]||"",contentLength:R["content-length"]||"",phase:"download-assets"}),Ce(r,n);let F=Re(l,R),S=o?.discoveredAssets?[...o.discoveredAssets]:[];if(F||o?.isText){let C=Q.toString("utf8"),J=X(e,I,C,n,"asset:cached");(J.length>0||!o)&&(S=J);for(let ee of S)k(e,r,ee,n,I);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:D.outputPath,isText:F||!!o?.isText,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(c(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let f=await g.body(),m=g.url()||a,P=g.headers(),w=await Te(e,r.assetMap,m,f,r,n,P);w.wroteFile&&(q(u,e,w.outputPath),N(w.outputPath)&&ae(u,w.outputPath)),r.doneAssets.add(a),Ce(r,n);let y=[];if(Re(l,P)){let v=f.toString("utf8");y=X(e,m,v,n,`asset:${h.extname(l.pathname).toLowerCase()||P["content-type"]||"unknown"}`);for(let R of y)k(e,r,R,n,m);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:w.outputPath,isText:Re(l,P),discoveredAssets:y,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(p){if(c("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function Kr(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),l=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Vr(e,t,r,a,s,n,i));}});await Promise.all(l);}function Yr(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function B(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function oe(e){return Math.max(15e3,Math.min(6e4,B(e)))}function Xr(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function Ie(e){return e.stats.pagesCompleted}function Pt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function Zr(e,t,r){let s=Pt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function E(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${Yr(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function es(e,t){let r=Xr(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,l=Date.now(),a=()=>{window.clearInterval(o),window.scrollTo(0,0),n();},o=window.setInterval(()=>{if(Date.now()-l>=s.maxDurationMs){a();return}window.scrollBy(0,u),i+=u;let c=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(c,p);i>=d+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function ts(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await es(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function rs(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function ss(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>Ae(t,r))}async function ns(e,t,r,s,n=true){let i=new Set;if(n){let l=await t.evaluate(()=>{let a=new Set,o=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of o){let m=d.getAttribute(f);m&&a.add(m);}for(let f of p){let m=d.getAttribute(f);m&&m.split(",").forEach(P=>{let w=P.trim().split(/\s+/)[0];w&&a.add(w);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...a],styles:[...c]}});for(let a of l.attrs)for(let o of Hr(a)){let c=T(e,o,r,s,"dom-attr");c&&x(e,c)&&i.add(c.toString());}for(let a of l.styles)for(let o of X(e,r,a,s,"dom-style"))i.add(o);}let u=await t.content();for(let l of X(e,r,u,s,"page-html"))i.add(l);return [...i]}async function rt(e,t,r,s,n){let i=H(e,t);await E(`save page ${t}`,B(e),async()=>{await we(i),await b.writeFile(i,r,"utf8");}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,Ie(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function is(e){let t=Ie(e);return t%5===0||t===1||t===e.stats.pagesQueued}function os(e,t){let r=Ie(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function as(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function us(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await as(s),s}async function ls(e,t,r,s,n,i,u){let l=await chromium.launch({headless:true}),a=Pt(l);a.pid!==null&&r.debug(`Launched crawl worker browser PID ${a.pid}`,{browserPid:a.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let o=t.pageQueue.shift();if(!o)break;if(t.queuedPages.delete(o),t.donePages.has(o))continue;t.donePages.add(o),i.enabled&&i.seenPages.add(o);let c=null,p=null;try{let d=new URL(o),g=gt(e,o),f=ss(e,d.pathname);if(!z(d)||x(e,d)||K(d)){if(x(e,d)){r.info(`Worker redirected non-page URL to asset queue: ${o}`,{url:o,source:"worker-guard"}),k(e,t,o,r,"worker-guard");continue}r.reject("page",o,"guard rejected non-page URL before rendering","worker");continue}let m=await Rr(e,n,t,i,o,r);if(i.enabled&&s!=="single-url"){let S=i.manifest.pages[o];if(m.action==="reuse"&&S){for(let C of S.discoveredAssets)k(e,t,C,r,o);for(let C of S.discoveredPages)Y(e,t,C,r,o);S.lastSeenRunId=i.runId,S.sitemapLastmod=ue(t.sitemapLastmodByPage[o])??S.sitemapLastmod,m.changeToken?.supported&&(S.changeToken=m.changeToken.token,S.tokenSource=m.changeToken.tokenSource??S.tokenSource),r.info(`Incremental reuse skipped unchanged page ${o}`,{url:o,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}c=await us(l,e,f),p=await c.newPage();let P=!1,w=new Set;await p.route("**/*",async S=>{let C=S.request();if(C.isNavigationRequest()&&C.frame()===p.mainFrame())try{let J=new URL(C.url()),ee=new URL(o);if(J.hash="",ee.hash="",J.href===ee.href){if(P){r.warn(`Blocked same-page navigation/reload for ${o}`,{url:o,requestUrl:C.url()}),await S.abort("aborted");return}P=!0;}}catch{}await S.continue();}),p.on("response",S=>{try{let C=T(e,S.url(),e.sourceOrigin,void 0,"network-response");C&&x(e,C)&&(w.add(C.toString()),k(e,t,C.toString(),r,o));}catch{}}),r.page(`Rendering ${o}`,{url:o});let y=null,v=null;try{if(y=await p.goto(o,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),y&&y.ok())try{v=await E(`read response body ${o}`,B(e),()=>y.text());}catch{v=null;}}catch(S){r.warn(`Navigation issue for ${o}; saving current DOM if available`,{url:o,error:String(S)});}if(g){let S=y?.status()??null;if(S!==404){r.error(`Generated 404 request path returned unexpected status for ${o}`,{url:o,expectedStatus:404,status:S,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${o}`,{url:o,status:S,outputPath:H(e,o)});}else if(y&&!y.ok()){r.skip("page",o,y.status()),await p.close();continue}f?(r.info(`Rendering without JS execution for ${o}`,{url:o,mode:"no-js"}),await rs(e,p)):await ts(e,p),t.stats.pagesRendered++;let R=await E(`extract rendered assets ${o}`,B(e),()=>ns(e,p,o,r,!f));for(let S of R)k(e,t,S,r,o);let Q=[...new Set([...R,...w])],I=v??await E(`capture rendered html ${o}`,B(e),()=>p.content()),D=s!=="single-url"&&!g?Gr(e,o,I,r):[];if(s!=="single-url"&&!g)for(let S of D)Y(e,t,S,r,o);await rt(e,o,I,t,r);let F=H(e,o);q(u,e,F),ae(u,F),i.trackManifest&&(i.manifest.pages[o]={url:o,outputPath:F,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:ue(t.sitemapLastmodByPage[o])??null,discoveredPages:D,discoveredAssets:Q,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(d){try{let g=p?await E(`capture partial html ${o}`,B(e),()=>p.content()):"";if(g&&g.trim()){await rt(e,o,g,t,r);let f=H(e,o);q(u,e,f),ae(u,f),r.warn(`Saved partial DOM for ${o}`,{url:o,error:String(d)});}}catch(g){r.error(`Could not save partial DOM for ${o}`,{url:o,error:String(g)});}r.error(`Failed page ${o}`,{url:o,error:String(d)});}finally{t.stats.pagesCompleted++,is(t)&&os(t,r),p&&await E(`close page ${o}`,oe(e),()=>p.close()).catch(d=>{r.warn(`Timed out while closing page ${o}`,{url:o,browserPid:a.pid,error:String(d)});}),c&&await E(`close page context ${o}`,oe(e),()=>c.close()).catch(d=>{r.warn(`Timed out while closing page context for ${o}`,{url:o,browserPid:a.pid,error:String(d)});});}}}finally{await E("close worker browser",oe(e),()=>l.close()).catch(o=>(r.warn("Timed out while closing crawl worker browser",{browserPid:a.pid,error:String(o)}),Zr(l,r,"browser.close timeout")));}}async function cs(e){try{let t=await b.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function st(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function ds(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function ps(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function St(e){try{let t=await b.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function nt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function gs(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>nt(s)-nt(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function it(e){let t=await St(h.join(e,"job.json"));for(let r of gs(e,t)){let s=await cs(r);if(s.length>0)return s}return []}async function hs(e){let t=h.join(e.logDir,"archive"),r;try{r=await b.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await b.stat(u).catch(()=>null))?.isDirectory())continue;let a=await St(h.join(u,"job.json"));if(!ds(a))continue;let o=ps(a);o>=n&&(n=o,s=u);}return s}async function fs(e){let t=await hs(e),r=t!==""?await it(t):await it(e.logDir),s=new Set;for(let n of r)if(st(n.error)||st(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function ms(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await b.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function ws(e){try{let t=await b.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Ps(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await b.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await b.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function ot(e,t,r,s,n){let i=T(e,r,e.sourceOrigin,s,"manual-url");i&&(Z(i)?Le(e,t,i.toString(),s,n):x(e,i)?k(e,t,i.toString(),s,n):z(i)?Y(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Ss(){let e=await fe(),t=hr(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await fs(e)),t.mode==="single-url"&&(r=await ms(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s&&typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!=="",i=t.mode==="full"&&!t.resumeRewrite,u=t.mode!=="full"||t.resumeRewrite||n,l=u?await ws(e):{},a=n?await Sr(e):ut(),o={enabled:n,trackManifest:i,manifest:Ye(a),previousManifest:Ye(a),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Ps(e.logDir);let c={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...l},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},p=Pr();u||await b.rm(e.outputDir,{recursive:true,force:true}),await b.mkdir(e.outputDir,{recursive:true});let d=new G(e.logDir,e.logLevel);if(d.info("Resolved WPSuite publisher subscription state for crawl startup.",{phase:"startup",mode:t.mode,requestedMode:t.crawlMode,requestedIncremental:s,incrementalEnabled:n,configSubscriptionType:e.subscriptionType??null,wpsuiteSubscriptionType:e.wpsuite?.subscriptionType??null,subscriber:e.wpsuite?.siteSettings?.subscriber===true,hasAccountId:!!String(e.wpsuite?.siteSettings?.accountId??"").trim(),hasSiteId:!!String(e.wpsuite?.siteSettings?.siteId??"").trim(),hasUploadUrl:!!String(e.wpsuite?.uploadUrl??"").trim(),hasRuntimeToken:!!String(e.wpsuite?.runtimeToken??"").trim(),lastUpdate:e.wpsuite?.siteSettings?.lastUpdate??null,apiBase:e.wpsuite?.apiBase??null,uploadUrl:e.wpsuite?.uploadUrl??null}),s&&!n&&d.warn("Incremental crawl requested without active remote publisher subscription. Falling back to full crawl.",{requestedMode:t.crawlMode,subscriptionType:e.subscriptionType??null}),t.resumeRewrite)d.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await request.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors});try{if(d.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Le(e,c,v,d)),await tt(e,w,c,d),e.seedPaths.forEach(v=>ot(e,c,v,d,"seed-path")),e.generated404RequestPath&&Y(e,c,e.generated404RequestPath,d,"generated-404-request-path")):(r.forEach(v=>ot(e,c,v,d,"cli")),await tt(e,w,c,d)),d.endMark("discovery",{pages:c.pageQueue.length,assets:c.assetQueue.length,sitemaps:c.sitemapQueue.length}),d.summary(`Queued ${c.pageQueue.length} pages, ${c.assetQueue.length} assets, ${c.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:c.pageQueue.length,queuedAssets:c.assetQueue.length,queuedSitemaps:c.sitemapQueue.length}),t.mode==="retry-timeouts"&&c.pageQueue.length===0&&c.assetQueue.length===0&&c.sitemapQueue.length===0){d.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await d.flush();return}d.mark("render-pages");let y=Array.from({length:Math.max(1,e.concurrency)},()=>ls(e,c,d,t.mode,w,o,p));if(await Promise.all(y),d.endMark("render-pages",{pages:c.stats.pagesCompleted}),d.mark("download-assets"),d.progress("Asset download phase started.",{phase:"download-assets",doneAssets:c.doneAssets.size,assetsQueued:c.stats.assetsQueued,assetQueue:c.assetQueue.length}),await Kr(e,w,c,d,o,p),d.endMark("download-assets",{assets:c.doneAssets.size}),o.enabled&&e.maxPages===0&&await Cr(e,c,o,d,p),t.mode==="full"){let v=await Jr(e,d);for(let R of v)q(p,e,R);}}finally{await E("dispose crawl request context",oe(e),()=>w.dispose()).catch(y=>{d.warn("Timed out while disposing crawl request context",{error:String(y)});});}}o.trackManifest&&(o.manifest.updatedAt=new Date().toISOString(),await br(e,o.manifest)),await b.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(c.assetMap,null,2),"utf8"),JSON.stringify(l,null,2)!==JSON.stringify(c.assetMap,null,2)&&q(p,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?Lr(e,c):o.enabled&&!t.resumeRewrite?xr(e,o,p,l,c.assetMap):void 0;d.mark("rewrite-text");let f=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";d.progress(f,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await et(f);let m=Date.now(),P=await ze(e,c.assetMap,{files:g,previousAssetMap:l,onProgress:async({index:w,totalFiles:y,changedTextFiles:v,file:R,changed:Q})=>{Q&&q(p,e,h.join(e.outputDir,R)),d.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R});let I=Date.now();if(!(w===1||w===y||I-m>=5e3))return;m=I;let F=`Rewriting text files: ${w}/${y}`;d.progress(F,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:v,file:R}),await et(F);}});d.endMark("rewrite-text",{changedTextFiles:P}),await Oe(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(o.enabled&&!t.resumeRewrite),changedFiles:o.enabled&&!t.resumeRewrite?[...p.changedFiles].sort():[],deletedFiles:o.enabled&&!t.resumeRewrite?[...p.deletedFiles].sort():[],rewriteTargets:o.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>te(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),d.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${P} text files.`:`Done. Rendered ${c.stats.pagesRendered} pages, processed ${c.doneSitemaps.size} sitemaps, downloaded ${c.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...c.stats,pagesRendered:c.stats.pagesRendered,donePages:c.stats.pagesCompleted,claimedPages:c.donePages.size,doneSitemaps:c.doneSitemaps.size,doneAssets:c.doneAssets.size,changedTextFiles:P}),await d.flush();}Ss().catch(async e=>{console.error(e);try{let r=(await fe().catch(()=>null))?.logDir??"logs",s=new G(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/dist/deploy.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {S3Client,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand,ListObjectsV2Command}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {createHash}from'crypto';import St from'fast-glob';import $ from'fs/promises';import v from'path';import Bt from'mime-types';import {getConfig}from'@smart-cloud/wpsuite-core';import {availableParallelism}from'os';import {Worker}from'worker_threads';function we(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Ze(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let n=r.endsWith("/"),i=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(i.length===0)return "";let s=`/${i.join("/")}`;return n&&v.extname(s)===""&&(s+="/"),s==="/"?"":s}function ee(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Xe(e){if(!e||typeof e!="object")return {};let t={};for(let[r,n]of Object.entries(e)){let i=r.trim();if(!i||!n||typeof n!="object")continue;let s=n,o={};if(typeof s.targetOrigin=="string"){let a=we(s.targetOrigin);a&&(o.targetOrigin=a);}let l=ee(s.extraReplacements);if(Object.keys(l).length>0&&(o.extraReplacements=l),s.s3&&typeof s.s3=="object"){let a={},u=s.s3;typeof u.bucket=="string"&&(a.bucket=u.bucket.trim()),typeof u.prefix=="string"&&(a.prefix=u.prefix.trim()),typeof u.region=="string"&&(a.region=u.region.trim()),typeof u.htmlCacheControl=="string"&&(a.htmlCacheControl=u.htmlCacheControl.trim()),typeof u.assetCacheControl=="string"&&(a.assetCacheControl=u.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(s.cloudFront&&typeof s.cloudFront=="object"){let a={},u=s.cloudFront;typeof u.distributionId=="string"&&(a.distributionId=u.distributionId.trim()),Array.isArray(u.invalidationPaths)&&(a.invalidationPaths=u.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[i]=o;}return t}function be(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function et(e,t){if(!e||typeof e!="object")return null;let r=e,n=r.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let i=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(i)||i<1)return null;let s=String(r.id??`${n}-${t+1}`).trim();if(!s)return null;let o=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:s,enabled:r.enabled!==false,command:n,intervalMinutes:i,...n==="publish"||n==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&o?{deploymentProfile:o}:{},...l?{url:l}:{}}}function ye(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((n,i)=>et(n,i)).filter(n=>!!n):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function tt(e){if(!e||typeof e!="object")return;let t=e,r={};for(let i of ["accountId","siteId"]){let s=t[i];typeof s=="string"&&s.trim()!==""&&(r[i]=s.trim());}let n=Number(t.lastUpdate??0);return Number.isFinite(n)&&n>0&&(r.lastUpdate=Math.floor(n)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function q(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),n=String(t.siteId??"").trim(),i=t.subscriber===true,s=tt(t.siteSettings),o={...s??{},...r?{accountId:s?.accountId??r}:{},...n?{siteId:s?.siteId??n}:{},...s?.subscriber===true||i?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let u=String(t.runtimeToken??t.nonce??"").trim();u&&(l.runtimeToken=u);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(o).length>0&&(l.siteSettings=o);let f=be(t.subscriptionType);return f&&(l.subscriptionType=f),l}function rt(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function nt(e){let t=q(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},n=String(r.accountId??"").trim(),i=String(r.siteId??"").trim(),s=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...s?{uploadUrl:s}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!n||!i||!s)return false;rt(e.sourceOrigin);let o=globalThis,a={...o.WpSuite??{},siteSettings:r,uploadUrl:s};return t.apiBase&&(a.apiBase=t.apiBase),o.WpSuite=a,true}async function it(e){let t=q(e.wpsuite),r=t.siteSettings??{},n=String(r.accountId??"").trim()||null,i=String(r.siteId??"").trim()||null,s=String(t.uploadUrl??"").trim();if(!nt(e))return console.info(`[INFO] Skipping remote WPSuite publisher config load: ${JSON.stringify({accountId:n,siteId:i,uploadUrl:s||null,subscriber:r.subscriber===true,lastUpdate:r.lastUpdate??null})}`),null;try{let o=await getConfig("publisher"),l=o&&typeof o=="object"?o:null,a=l?.siteSettings&&typeof l.siteSettings=="object"?l.siteSettings:null;return console.info(`[INFO] Loaded remote WPSuite publisher config: ${JSON.stringify({accountId:n,siteId:i,uploadUrl:s||null,subscriber:r.subscriber===!0,lastUpdate:r.lastUpdate??null,remoteType:o===null?"null":typeof o,remoteSubscriptionType:typeof l?.subscriptionType=="string"?l.subscriptionType:null,remoteKeys:l?Object.keys(l).sort():[],remoteAccountId:typeof l?.accountId=="string"?l.accountId:typeof a?.accountId=="string"?a.accountId:null,remoteSiteId:typeof l?.siteId=="string"?l.siteId:typeof a?.siteId=="string"?a.siteId:null,remoteSiteSettingsKeys:a?Object.keys(a).sort():[]})}`),o&&typeof o=="object"?o:null}catch(o){return console.warn(`[WARN] Remote WPSuite publisher config load failed: ${JSON.stringify({accountId:n,siteId:i,uploadUrl:s||null,subscriber:r.subscriber===true,lastUpdate:r.lastUpdate??null,error:o instanceof Error?o.message:String(o)})}`),null}}function st(e,t){let r=Xe(t?.deploymentProfiles),n=String(t?.defaultDeploymentProfile??"").trim(),i=r[n]?n:"",s=be(t?.subscriptionType),o=q(e.wpsuite);return {...e,scheduler:ye(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:i,...s?{subscriptionType:s}:{},wpsuite:{...o,...o.siteSettings||s?{siteSettings:{...o.siteSettings??{},...o.siteSettings?.subscriber===true||s==="PROFESSIONAL"||s==="AGENCY"?{subscriber:true}:{}}}:{},...s?{subscriptionType:s}:{}}}}function te(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let i=n.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return i.length>0?i.join("/"):t}function ot(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?v.resolve(e):""}function he(e,t,r){let n=String(t||"").trim();return n&&v.isAbsolute(n)?v.resolve(n):v.resolve(e,te(n,r))}async function re(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await $.readFile(t,"utf8"),n=JSON.parse(r);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=we(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=Ze(n.generated404RequestPath),n.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=ee(n.extraReplacements),n.postCrawlCopyMap=ee(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=q(n.wpsuite),n.scheduler=ye(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let i=st(n,await it(n)),s=ot(),o=s?v.resolve(s,".."):"";return o?(i.outputDir=he(o,i.outputDir,"export"),i.logDir=he(o,i.logDir,"logs")):(v.isAbsolute(i.outputDir)||(i.outputDir=te(i.outputDir,"export")),v.isAbsolute(i.logDir)||(i.logDir=te(i.logDir,"logs"))),i}function at(e){let t=e/6e4;if(Number.isInteger(t)&&t>=1)return `${t} minute${t===1?"":"s"}`;let r=Math.ceil(e/1e3);return `${r} second${r===1?"":"s"}`}async function C(e,t,r){let n=new AbortController,i=globalThis.setTimeout(()=>{n.abort(new Error(`${e} timed out after ${at(t)}.`));},t);try{return await r({abortController:n,httpOptions:{abortSignal:n.signal,requestTimeout:t}})}finally{globalThis.clearTimeout(i);}}var lt=".deploy-plan.json",ut="deploy-plan.json";function ct(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?v.resolve(e):""}function gt(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ne(e){return [...new Set(e.map(gt).filter(t=>t!==null))]}function pt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:v.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ne(t.changedFiles||[]),deletedFiles:ne(t.deletedFiles||[]),rewriteTargets:ne(t.rewriteTargets||[])}}function dt(e){if(!e||typeof e!="object")return false;let t=e;return t.schemaVersion===1&&typeof t.generatedAt=="string"&&typeof t.outputDir=="string"&&(t.runMode==="full"||t.runMode==="retry-timeouts"||t.runMode==="single-url")&&(t.crawlMode==="full"||t.crawlMode==="incremental")&&typeof t.fullSyncRequired=="boolean"&&Array.isArray(t.changedFiles)&&Array.isArray(t.deletedFiles)&&Array.isArray(t.rewriteTargets)}function Pe(e){let t=v.join(v.resolve(e.outputDir),lt),r=ct();return r?[v.join(r,ut),t]:[t]}async function ve(e){let t=v.resolve(e.outputDir);for(let r of Pe(e))try{let n=await $.readFile(r,"utf8"),i=JSON.parse(n);if(!dt(i)||v.resolve(i.outputDir)!==t)continue;return pt(e,i)}catch{}return null}async function $e(e){for(let t of Pe(e))try{await $.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function ft(e){return typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!==""}function xe(e){let t="";for(let n=0;n<e.length;n++){let i=e[n];if(i==="--profile"){let s=(e[n+1]||"").trim();if(!s)throw new Error("Missing value for --profile.");t=s,n+=1;}else if(i.startsWith("--profile=")){let s=i.slice(10).trim();if(!s)throw new Error("Missing value for --profile.");t=s;}}let r=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||r||null}function mt(e,t){return {...e,...t??{}}}function ht(e,t){return {...e,...t??{},invalidationPaths:[...(t?.invalidationPaths??e.invalidationPaths)||[]]}}function Re(e,t){let r=String(t??e.deploymentTargetOverride??"").trim();if(!ft(e)){if(r)throw new Error(`Deployment target "${r}" requires an active WPSuite subscription and remote publisher config.`);return {name:null,profile:null,config:e}}let n=String(r||e.defaultDeploymentProfile||"").trim();if(!n)return {name:null,profile:null,config:e};let i=e.deploymentProfiles?.[n];if(!i)throw new Error(`Unknown deployment profile "${n}". Check the linked WPSuite publisher configuration.`);return {name:n,profile:i,config:{...e,targetOrigin:i.targetOrigin??e.targetOrigin,s3:mt(e.s3,i.s3),cloudFront:ht(e.cloudFront,i.cloudFront)}}}var De={error:0,warn:1,info:2,debug:3},Ce=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",J=Ce?v.join(Ce,"current-progress.json"):"",Te=Promise.resolve();function Ee(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function wt(e,t){let r={...e},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[i,s]of Object.entries(t)){let o=r[i];if(typeof o=="number"&&typeof s=="number"&&n.has(i)){r[i]=Math.max(o,s);continue}r[i]=s;}return r}function bt(e,t,r){J&&(Te=Te.then(async()=>{let n=new Date,i=n.toISOString();await $.mkdir(v.dirname(J),{recursive:true});let s=await $.readFile(J,"utf8").then(y=>JSON.parse(y)).catch(()=>null),o=s&&typeof s.details=="object"&&s.details?s.details:{},l=Ee(e),a=s?.currentStep||Ee(s?.source||""),u=s?.startedAt||i,c={...s?.stepDurationsSec??{}},f=s?.stepStartedAt||i;if(a&&a!==l&&s?.stepStartedAt){let y=Math.max(0,Math.round((n.getTime()-new Date(s.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+y,f=i;}let g=Math.max(0,Math.round((n.getTime()-new Date(f).getTime())/1e3)),m=Math.max(0,Math.round((n.getTime()-new Date(u).getTime())/1e3)),b={checkedAt:i,source:e,message:t,details:wt(o,r??{}),startedAt:u,currentStep:l,stepStartedAt:f,stepElapsedSec:g,totalElapsedSec:m,stepDurationsSec:c};await $.writeFile(J,JSON.stringify(b,null,2),"utf8");}).catch(()=>{}));}async function yt(e,t){await $.mkdir(v.dirname(e),{recursive:true}),await $.appendFile(e,`${JSON.stringify(t)}
|
|
2
|
-
`,"utf8");}var
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>
|
|
1
|
+
import {S3Client,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand,ListObjectsV2Command}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {createHash}from'crypto';import xt from'fast-glob';import R from'fs/promises';import v from'path';import Kt from'mime-types';import Xe from'http';import et from'https';import {availableParallelism}from'os';import {Worker}from'worker_threads';function we(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function tt(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let n=r.endsWith("/"),i=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(i.length===0)return "";let s=`/${i.join("/")}`;return n&&v.extname(s)===""&&(s+="/"),s==="/"?"":s}function X(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function rt(e){if(!e||typeof e!="object")return {};let t={};for(let[r,n]of Object.entries(e)){let i=r.trim();if(!i||!n||typeof n!="object")continue;let s=n,o={};if(typeof s.targetOrigin=="string"){let a=we(s.targetOrigin);a&&(o.targetOrigin=a);}let l=X(s.extraReplacements);if(Object.keys(l).length>0&&(o.extraReplacements=l),s.s3&&typeof s.s3=="object"){let a={},u=s.s3;typeof u.bucket=="string"&&(a.bucket=u.bucket.trim()),typeof u.prefix=="string"&&(a.prefix=u.prefix.trim()),typeof u.region=="string"&&(a.region=u.region.trim()),typeof u.htmlCacheControl=="string"&&(a.htmlCacheControl=u.htmlCacheControl.trim()),typeof u.assetCacheControl=="string"&&(a.assetCacheControl=u.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(s.cloudFront&&typeof s.cloudFront=="object"){let a={},u=s.cloudFront;typeof u.distributionId=="string"&&(a.distributionId=u.distributionId.trim()),Array.isArray(u.invalidationPaths)&&(a.invalidationPaths=u.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[i]=o;}return t}function be(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function nt(e,t){if(!e||typeof e!="object")return null;let r=e,n=r.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let i=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(i)||i<1)return null;let s=String(r.id??`${n}-${t+1}`).trim();if(!s)return null;let o=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:s,enabled:r.enabled!==false,command:n,intervalMinutes:i,...n==="publish"||n==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&o?{deploymentProfile:o}:{},...l?{url:l}:{}}}function ye(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((n,i)=>nt(n,i)).filter(n=>!!n):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function it(e){if(!e||typeof e!="object")return;let t=e,r={};for(let i of ["accountId","siteId"]){let s=t[i];typeof s=="string"&&s.trim()!==""&&(r[i]=s.trim());}let n=Number(t.lastUpdate??0);return Number.isFinite(n)&&n>0&&(r.lastUpdate=Math.floor(n)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function te(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),n=String(t.siteId??"").trim(),i=t.subscriber===true,s=it(t.siteSettings),o={...s??{},...r?{accountId:s?.accountId??r}:{},...n?{siteId:s?.siteId??n}:{},...s?.subscriber===true||i?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let u=String(t.runtimeToken??t.nonce??"").trim();u&&(l.runtimeToken=u);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(o).length>0&&(l.siteSettings=o);let d=be(t.subscriptionType);return d&&(l.subscriptionType=d),l}function st(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Se(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function ot(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[n,i]of new Headers(t.headers).entries())r.set(n,i);return r}async function Pe(e,t,r=5){let n=new URL(Se(e)),i=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),s=n.protocol==="http:"?Xe:et,o=ot(e,t);return await new Promise((l,a)=>{let u=s.request(n,{method:i,headers:Object.fromEntries(o.entries()),...n.protocol==="https:"?{rejectUnauthorized:false}:{}},c=>{let d=[];c.on("data",g=>{d.push(Buffer.isBuffer(g)?g:Buffer.from(g));}),c.on("error",a),c.on("end",()=>{let g=c.statusCode??0,m=c.headers.location;if(r>0&&m&&[301,302,303,307,308].includes(g)){l(Pe(new URL(m,n),{method:g===303?"GET":i},r-1));return}let w=new Headers;for(let[b,p]of Object.entries(c.headers))Array.isArray(p)?w.set(b,p.join(", ")):typeof p=="string"&&w.set(b,p);l(new Response(Buffer.concat(d),{status:g,statusText:c.statusMessage??"",headers:w}));});});u.on("error",a),u.end();})}function at(e){let t=te(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},n=String(r.accountId??"").trim(),i=String(r.siteId??"").trim(),s=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...s?{uploadUrl:s}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!n||!i||!s)return false;st(e.sourceOrigin);let o=globalThis,a={...o.WpSuite??{},siteSettings:r,uploadUrl:s};return t.apiBase&&(a.apiBase=t.apiBase),o.WpSuite=a,true}async function lt(e){if(!at(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{let n=globalThis,i=()=>({sourceOrigin:e.sourceOrigin,locationHref:n.location?.href??null,locationHostname:n.location?.hostname??null,wpsuite:n.WpSuite?{apiBase:n.WpSuite.apiBase??null,uploadUrl:n.WpSuite.uploadUrl??null,siteSettings:n.WpSuite.siteSettings??null}:null});console.info(`[DEBUG] Remote WPSuite config bootstrap before import: ${JSON.stringify(i())}`),e.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(l,a)=>Se(l).startsWith(t)?Pe(l,a):r(l,a)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core');console.info(`[DEBUG] Remote WPSuite config bootstrap before getConfig: ${JSON.stringify(i())}`);let o=await s("publisher");return o&&typeof o=="object"?o:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function ut(e,t){let r=rt(t?.deploymentProfiles),n=String(t?.defaultDeploymentProfile??"").trim(),i=r[n]?n:"",s=be(t?.subscriptionType),o=te(e.wpsuite);return {...e,scheduler:ye(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:i,...s?{subscriptionType:s}:{},wpsuite:{...o,...o.siteSettings||s?{siteSettings:{...o.siteSettings??{},...o.siteSettings?.subscriber===true||s==="PROFESSIONAL"||s==="AGENCY"?{subscriber:true}:{}}}:{},...s?{subscriptionType:s}:{}}}}function ee(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let i=n.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return i.length>0?i.join("/"):t}function ct(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?v.resolve(e):""}function he(e,t,r){let n=String(t||"").trim();return n&&v.isAbsolute(n)?v.resolve(n):v.resolve(e,ee(n,r))}async function re(e){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await R.readFile(t,"utf8"),n=JSON.parse(r);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=we(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=tt(n.generated404RequestPath),n.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=X(n.extraReplacements),n.postCrawlCopyMap=X(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=te(n.wpsuite),n.scheduler=ye(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let i=ut(n,await lt(n)),s=ct(),o=s?v.resolve(s,".."):"";return o?(i.outputDir=he(o,i.outputDir,"export"),i.logDir=he(o,i.logDir,"logs")):(v.isAbsolute(i.outputDir)||(i.outputDir=ee(i.outputDir,"export")),v.isAbsolute(i.logDir)||(i.logDir=ee(i.logDir,"logs"))),i}function gt(e){let t=e/6e4;if(Number.isInteger(t)&&t>=1)return `${t} minute${t===1?"":"s"}`;let r=Math.ceil(e/1e3);return `${r} second${r===1?"":"s"}`}async function C(e,t,r){let n=new AbortController,i=globalThis.setTimeout(()=>{n.abort(new Error(`${e} timed out after ${gt(t)}.`));},t);try{return await r({abortController:n,httpOptions:{abortSignal:n.signal,requestTimeout:t}})}finally{globalThis.clearTimeout(i);}}var pt=".deploy-plan.json",dt="deploy-plan.json";function ft(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?v.resolve(e):""}function mt(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ne(e){return [...new Set(e.map(mt).filter(t=>t!==null))]}function ht(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:v.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ne(t.changedFiles||[]),deletedFiles:ne(t.deletedFiles||[]),rewriteTargets:ne(t.rewriteTargets||[])}}function wt(e){if(!e||typeof e!="object")return false;let t=e;return t.schemaVersion===1&&typeof t.generatedAt=="string"&&typeof t.outputDir=="string"&&(t.runMode==="full"||t.runMode==="retry-timeouts"||t.runMode==="single-url")&&(t.crawlMode==="full"||t.crawlMode==="incremental")&&typeof t.fullSyncRequired=="boolean"&&Array.isArray(t.changedFiles)&&Array.isArray(t.deletedFiles)&&Array.isArray(t.rewriteTargets)}function Re(e){let t=v.join(v.resolve(e.outputDir),pt),r=ft();return r?[v.join(r,dt),t]:[t]}async function xe(e){let t=v.resolve(e.outputDir);for(let r of Re(e))try{let n=await R.readFile(r,"utf8"),i=JSON.parse(n);if(!wt(i)||v.resolve(i.outputDir)!==t)continue;return ht(e,i)}catch{}return null}async function $e(e){for(let t of Re(e))try{await R.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function bt(e){return typeof e.subscriptionType=="string"&&e.subscriptionType.trim()!==""}function De(e){let t="";for(let n=0;n<e.length;n++){let i=e[n];if(i==="--profile"){let s=(e[n+1]||"").trim();if(!s)throw new Error("Missing value for --profile.");t=s,n+=1;}else if(i.startsWith("--profile=")){let s=i.slice(10).trim();if(!s)throw new Error("Missing value for --profile.");t=s;}}let r=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||r||null}function yt(e,t){return {...e,...t??{}}}function St(e,t){return {...e,...t??{},invalidationPaths:[...(t?.invalidationPaths??e.invalidationPaths)||[]]}}function Ce(e,t){let r=String(t??e.deploymentTargetOverride??"").trim();if(!bt(e)){if(r)throw new Error(`Deployment target "${r}" requires an active WPSuite subscription and remote publisher config.`);return {name:null,profile:null,config:e}}let n=String(r||e.defaultDeploymentProfile||"").trim();if(!n)return {name:null,profile:null,config:e};let i=e.deploymentProfiles?.[n];if(!i)throw new Error(`Unknown deployment profile "${n}". Check the linked WPSuite publisher configuration.`);return {name:n,profile:i,config:{...e,targetOrigin:i.targetOrigin??e.targetOrigin,s3:yt(e.s3,i.s3),cloudFront:St(e.cloudFront,i.cloudFront)}}}var Te={error:0,warn:1,info:2,debug:3},Ee=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",q=Ee?v.join(Ee,"current-progress.json"):"",Me=Promise.resolve();function Le(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function Pt(e,t){let r={...e},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[i,s]of Object.entries(t)){let o=r[i];if(typeof o=="number"&&typeof s=="number"&&n.has(i)){r[i]=Math.max(o,s);continue}r[i]=s;}return r}function vt(e,t,r){q&&(Me=Me.then(async()=>{let n=new Date,i=n.toISOString();await R.mkdir(v.dirname(q),{recursive:true});let s=await R.readFile(q,"utf8").then(b=>JSON.parse(b)).catch(()=>null),o=s&&typeof s.details=="object"&&s.details?s.details:{},l=Le(e),a=s?.currentStep||Le(s?.source||""),u=s?.startedAt||i,c={...s?.stepDurationsSec??{}},d=s?.stepStartedAt||i;if(a&&a!==l&&s?.stepStartedAt){let b=Math.max(0,Math.round((n.getTime()-new Date(s.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+b,d=i;}let g=Math.max(0,Math.round((n.getTime()-new Date(d).getTime())/1e3)),m=Math.max(0,Math.round((n.getTime()-new Date(u).getTime())/1e3)),w={checkedAt:i,source:e,message:t,details:Pt(o,r??{}),startedAt:u,currentStep:l,stepStartedAt:d,stepElapsedSec:g,totalElapsedSec:m,stepDurationsSec:c};await R.writeFile(q,JSON.stringify(w,null,2),"utf8");}).catch(()=>{}));}async function Rt(e,t){await R.mkdir(v.dirname(e),{recursive:true}),await R.appendFile(e,`${JSON.stringify(t)}
|
|
2
|
+
`,"utf8");}var N=class{constructor(t,r,n="info"){this.logDir=t;this.logFile=r;this.level=n;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return v.join(this.logDir,this.logFile)}get errorsPath(){return v.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await R.mkdir(this.logDir,{recursive:true}),await R.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await R.appendFile(this.logPath,`${t}
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Rt(t,r));}accepts(t){return Te[t]<=Te[this.level]}push(t,r,n){if(!this.accepts(t))return;let i=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...n||{}});this.enqueueLine(i),t==="error"?console.error(`[ERROR] ${r}`,n?JSON.stringify(n):""):t==="warn"?console.warn(`[WARN] ${r}`,n?JSON.stringify(n):""):console.log(`[${t.toUpperCase().padEnd(5)}] ${r}`);}info(t,r){this.push("info",t,r);}progress(t,r){vt(this.logFile,t,r);}debug(t,r){this.push("debug",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let n=this.marks.get(t);if(!n)return;let i=Date.now()-n,s=Number((i/1e3).toFixed(2));this.push("info",`Timing ${t}: ${s}s`,{name:t,ms:i,seconds:s,...r||{}});}async flush(){let t=Date.now()-this.startedAt,r=Number((t/1e3).toFixed(2));if(this.push("info",`Total time: ${r}s`,{name:"total",ms:t,seconds:r}),await this.writeQueue,this.writeError)throw this.writeError}};function Ie(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(v.extname(e).toLowerCase())}function Oe(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function k(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function H(e){return e.replace(/\//g,"\\/")}function ke(e){return e.replace(/\//g,"\\\\/")}function J(e,t,r){if(!t)return;e[t]=r;let n=H(t),i=H(r);e[n]=i;let s=ke(t),o=ke(r);e[s]=o;let l=k(t),a=k(r);e[l]=a;let u=k(n),c=k(i);e[u]=c;let d=k(s),g=k(o);e[d]=g;}function Fe(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function je(e,t,r){if(!t)return r;let n=v.dirname(v.resolve(t)),i=v.resolve(e,r.replace(/^\/+/,"")),s=v.relative(n,i).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s):"."}var Ct=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Tt=new Set([".html",".htm"]),Ae="WPSuite.io Static Publisher",Et=Ae.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Mt(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let i=v.extname(r).toLowerCase();return Tt.has(i)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Lt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Et}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ae}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let n=e.includes(`\r
|
|
4
4
|
`)?`\r
|
|
5
5
|
`:`
|
|
6
|
-
`;return e.replace(/<head\b[^>]*>/i,i=>`${i}${n} ${t}`)}function Tt(e){let t=ke(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Et(e,t){let r=[...new Set(t)].filter(s=>s.includes("/")&&!/\\+\//.test(s)).map(s=>[s,H(s)]).filter(([s,o])=>s!==o).sort((s,o)=>o[0].length-s[0].length);if(!r.length)return e;let n=s=>{let o=s;for(let[l,a]of r)o=o.split(l).join(a);return o},i=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(s,o,l,a)=>`${o}${n(l)}${a}`);return i=i.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(s,o,l,a,u)=>Tt(a)?`${o}${n(a)}${u}`:s),i}function It(e,t){if(!t)return null;let r=v.dirname(v.resolve(t)),n=v.resolve(e.outputDir),i=v.relative(r,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i.endsWith("/")?i:`${i}/`):"./"}function Mt(e,t,r){let n=It(t,r);if(!n)return e;let i=e;for(let s of $t){let o=s.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");i=i.replace(l,`${n}${s}`).replace(a,`${n}${s}`);let u=s.replace(/\//g,"\\/"),c=n.replace(/\//g,"\\/"),f=u.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),g=new RegExp(`(?:\\.{1,}\\\\/)+${f}`,"g"),m=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${f}`,"g"),b=new RegExp(`(?<![\\.\\\\])\\/${f}`,"g");i=i.replace(g,`${c}${u}`).replace(m,`${c}${u}`).replace(b,`${c}${u}`);}return i}function se(e,t,r){if(e.urlRewriteMode==="absolute")return t;let n=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),i=n?n.pathname:Le(t,e.targetOrigin),s=n?`${n.search}${n.hash}`:"";return e.urlRewriteMode==="root-relative"?`${i}${s}`:`${Oe(e.outputDir,r,i)}${s}`}function ie(e,t,r,n,i){K(e,r,se(t,n,i));}function kt(e,t,r,n,i){if(!r||r===n)return;K(e,r,n);let s=se(t,r,i),o=se(t,n,i);K(e,s,o);}function Lt(e,t,r,n,i={}){let s=e,o={};ie(o,t,t.sourceOrigin,t.targetOrigin,n);for(let[a,u]of Object.entries(t.extraReplacements))ie(o,t,a,u,n);for(let[a,u]of Object.entries(r))ie(o,t,a,u,n);for(let[a,u]of Object.entries(i)){let c=r[a];!c||c===u||kt(o,t,u,c,n);}let l=Object.entries(o).sort((a,u)=>u[0].length-a[0].length);for(let[a,u]of l)s=s.split(a).join(u);return t.urlRewriteMode!=="absolute"&&(s=Mt(s,t,n)),s=Et(s,l.map(([,a])=>a)),Dt(t,s,n)&&(s=Ct(s)),s}async function Ot(e,t,r,n){let i=v.join(e.outputDir,n),s=v.extname(i).toLowerCase(),o;try{o=await $.readFile(i,"utf8");}catch(u){if(u.code!=="ENOENT")throw u;return {changed:false}}let a=Lt(o,e,t,s===".js"||s===".mjs"?void 0:i,r);return a!==o?(await $.writeFile(i,a,"utf8"),{changed:true}):{changed:false}}async function Ft(e,t){return await new Promise((r,n)=>{let i=a=>{if(l(),a?.error){n(new Error(a.error));return}r({changed:!!a?.changed});},s=a=>{l(),n(a);},o=a=>{l(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",i),e.off("error",s),e.off("exit",o);};e.on("message",i),e.on("error",s),e.on("exit",o),e.postMessage({file:t});})}async function _e(e,t,r={}){let i=(r.files?[...new Set(r.files)].map(g=>v.resolve(g)).filter(g=>g.startsWith(v.resolve(e.outputDir))).map(g=>v.relative(e.outputDir,g).replace(/\\/g,"/")).filter(g=>g.length>0):await St(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(g=>g!=="asset-map.json"&&Me(g)),s=0;if(i.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(o,i.length,availableParallelism()));await r.onStart?.({totalFiles:i.length,workerCount:l});let a=0,u=0,c=null;if(l===1){for(let g of i){let m=await Ot(e,t,r.previousAssetMap??{},g);m.changed&&(s+=1),u+=1,await r.onProgress?.({index:u,totalFiles:i.length,changedTextFiles:s,file:g,changed:m.changed});}return s}let f=Array.from({length:l},async()=>{let g=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let m=a;if(m>=i.length)return;a+=1;let b=i[m],y=await Ft(g,b);y.changed&&(s+=1),u+=1,await r.onProgress?.({index:u,totalFiles:i.length,changedTextFiles:s,file:b,changed:y.changed});}}catch(m){throw c=m instanceof Error?m:new Error(String(m)),c}finally{await g.terminate().catch(()=>{});}});return await Promise.all(f),s}var Ue="wpsuite-sha256",We="wpsuite-normalized-sha256",qt=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,pe=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,Jt=["SIGINT","SIGTERM"],Q=new Set,G=new Map,Ae=null;function Ht(e){Q.add(e);let t=false;return ()=>{t||(t=true,Q.delete(e));}}async function Kt(){let e=[...Q];e.length!==0&&await Promise.allSettled(e.map(async t=>{Q.delete(t),await $.rm(t,{recursive:true,force:true});}));}function Gt(){if(!(G.size>0))for(let e of Jt){let t=()=>{Ae||(Ae=Kt().finally(()=>{for(let[r,n]of G)process.off(r,n);G.clear(),process.exit(1);}));};G.set(e,t),process.on(e,t);}}Gt();function Qt(e){return e&&(pe.test(e)?"__uuid__":/^[0-9a-f]{6,}$/i.test(e)?"__hex__":/^[0-9]{8,}$/.test(e)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(e)?"__alnum__":e)}function le(e){return e.split(/([_:.\-/=?&#]+)/g).map((t,r)=>r%2===1?t:Qt(t)).join("")}function Vt(e){return e.split(/([_:.\-/=?&#]+)/g).map((t,r)=>r%2===1?t:pe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{6,}$/.test(t)?"__num__":/^[a-z0-9_-]{6,}$/i.test(t)?"__id__":t).join("")}function V(e,t){return Array.isArray(e)?e.map(r=>V(r,t)):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([r,n])=>[r,V(n,r)])):typeof e!="string"?e:t&&qt.test(t)?Vt(e):pe.test(e)?"__uuid__":/^[0-9a-f]{6,}$/i.test(e)||/^[0-9]{8,}$/.test(e)?le(e):e}function Yt(e){return e.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function Zt(e,t){let r=t.trim();if((r.startsWith("{")||r.startsWith("[")||r.startsWith('"'))&&e.startsWith("data-"))try{let i=JSON.parse(r);return JSON.stringify(V(i))}catch{}if(e.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(r))try{let i=Buffer.from(r,"base64").toString("utf8"),s=JSON.parse(i),o=JSON.stringify(V(s));return Buffer.from(o,"utf8").toString("base64")}catch{}return t}function Xt(e,t){let r=e.toLowerCase();return r==="class"?t.split(/(\s+)/).map((n,i)=>i%2===1?n:le(n)).join(""):r==="id"||r==="for"||r==="aria-controls"||r==="aria-labelledby"||r==="aria-describedby"||r==="aria-owns"||r==="aria-activedescendant"||r==="name"?le(t):r==="href"||r==="src"?Yt(t):Zt(r,t)}function er(e){return e.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(t,r,n)=>{if(r.startsWith("/"))return t;let i=n.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(s,o,l,a)=>{let u=Xt(o,a);return ` ${o}=${l}${u}${l}`});return `<${r}${i}>`})}function oe(e){let t=v.extname(e).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(t)||["robots.txt","sitemap.xml"].includes(v.basename(e))}function ae(e,t){return [e.replace(/^\/+|\/+$/g,""),t.replace(/^\/+/,"")].filter(Boolean).join("/")}async function tr(e,t,r,n){let i=new Map,s,o=0;do{o++;let l=await C(`S3 list objects page ${o} for s3://${t}/${r||""}`,3e5,({httpOptions:a})=>e.send(new ListObjectsV2Command({Bucket:t,Prefix:r||void 0,ContinuationToken:s}),a));for(let a of l.Contents||[]){if(!a.Key)continue;let u=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),c=Number(a.Size??0);i.set(a.Key,{etag:u,size:c});}s=l.NextContinuationToken,n.debug(`Listed S3 keys page ${o}: ${i.size} keys so far`);}while(s);return i}async function rr(e,t,r,n){if(n.has(r))return n.get(r)??null;try{let i=await C(`S3 head object ${r}`,3e5,({httpOptions:o})=>e.send(new HeadObjectCommand({Bucket:t,Key:r}),o)),s={etag:(i.ETag||"").replace(/^"|"$/g,"").toLowerCase(),size:Number(i.ContentLength??0)};return n.set(r,s),s}catch{return n.set(r,null),null}}async function ze(e,t,r,n){let i=[...new Set(Array.from(r).filter(Boolean))];if(i.length===0)return n.info("No stale S3 objects to delete."),[];n.info(`Deleting ${i.length} stale S3 object(s)\u2026`);for(let s=0;s<i.length;s+=1e3){let o=i.slice(s,s+1e3);if(o.length!==0){await C(`S3 delete objects batch ${Math.floor(s/1e3)+1}`,3e5,({httpOptions:l})=>e.send(new DeleteObjectsCommand({Bucket:t,Delete:{Objects:o.map(a=>({Key:a}))}}),l)),n.info(`Deleted batch of ${o.length} object(s).`);for(let l of o)n.debug(` deleted: ${l}`);}}return i}async function nr(e,t,r,n,i){return ze(e,t,Array.from(r).filter(s=>!n.has(s)),i)}function ir(e){return createHash("md5").update(e).digest("hex")}function Ne(e){return createHash("sha256").update(e).digest("hex")}function ue(e){return createHash("sha256").update(er(e.toString("utf8"))).digest("hex")}async function sr(e,t,r,n){if(n.has(r))return n.get(r)??null;try{let o=((await C(`S3 head object metadata ${r}`,3e5,({httpOptions:l})=>e.send(new HeadObjectCommand({Bucket:t,Key:r}),l))).Metadata?.[Ue]||"").trim().toLowerCase()||null;return n.set(r,o),o}catch{return n.set(r,null),null}}async function or(e,t,r,n,i){if(n.has(r))return n.get(r)??null;try{let o=((await C(`S3 head object normalized metadata ${r}`,3e5,({httpOptions:l})=>e.send(new HeadObjectCommand({Bucket:t,Key:r}),l))).Metadata?.[We]||"").trim().toLowerCase();if(o)return n.set(r,o),o}catch{return n.set(r,null),null}if(i.has(r)){let s=i.get(r)??null;return n.set(r,s),s}try{let o=await(await C(`S3 get object ${r}`,3e5,({httpOptions:a})=>e.send(new GetObjectCommand({Bucket:t,Key:r}),a))).Body?.transformToByteArray(),l=o?ue(Buffer.from(o)):null;return i.set(r,l),n.set(r,l),l}catch{return i.set(r,null),n.set(r,null),null}}function ar(e){return e.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function lr(e,t){let r=e,n=Object.entries(t).sort((i,s)=>s[0].length-i[0].length);for(let[i,s]of n)i&&(r=r.split(i).join(s));return r}async function ur(e,t,r,n){let i=v.join(e,"asset-map.json"),s;try{s=await $.readFile(i,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(s);if(!o||typeof o!="object")return;let l=Object.fromEntries(Object.entries(o).map(([a,u])=>{let c=String(u??"");return t.urlRewriteMode==="absolute"&&t.targetOrigin&&r.targetOrigin&&t.targetOrigin!==r.targetOrigin&&(c=c.split(t.targetOrigin).join(r.targetOrigin)),c=lr(c,n),[a,c]}));await $.writeFile(i,JSON.stringify(l,null,2),"utf8");}async function cr(e,t,r,n,i){let s=e.targetOrigin!==t.targetOrigin,o=Object.keys(n).length>0;if(!r||!s&&!o)return {outputDir:e.outputDir,cleanup:async()=>{}};if(s&&e.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${r}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let l=v.join(v.dirname(e.outputDir),`.deploy-profile-${ar(r)}-${Date.now()}`),a=Ht(l);i.info(`Preparing deployment profile "${r}" from existing crawl output to ${l}...`);try{await $.rm(l,{recursive:!0,force:!0}),await $.cp(e.outputDir,l,{recursive:!0});let u={...e,sourceOrigin:e.targetOrigin,targetOrigin:t.targetOrigin,extraReplacements:n,outputDir:l},c=1;i.info(`Rewriting text files for deployment profile "${r}"...`);let f=await _e(u,{},{onStart:({totalFiles:g,workerCount:m})=>{c=m,i.info(`Rewriting ${g} text file(s) for deployment profile "${r}" with ${m} worker(s).`);},onProgress:({index:g,totalFiles:m,changedTextFiles:b,file:y,changed:p})=>{let S=`Rewriting [${g}/${m}] ${y} (${p?"rewrote":"checked"}; ${b} changed so far)`;m<=200||g<=5||g%10===0||g===m?(i.progress(S,{file:y,index:g,totalFiles:m,changed:p,changedTextFiles:b,workerCount:c}),i.info(S)):i.debug(S);}});return await ur(l,e,t,n),i.info(`Prepared deployment profile "${r}" in ${l} (${f} rewritten text file(s)).`),{outputDir:l,cleanup:async()=>{a(),await $.rm(l,{recursive:!0,force:!0});}}}catch(u){throw i.error(`Failed to prepare deployment profile "${r}" temporary output.`,{tempDir:l,outputDir:e.outputDir,error:u instanceof Error?u.message:String(u),...u instanceof Error&&u.stack?{stack:u.stack}:{}}),await $.rm(l,{recursive:true,force:true}).catch(()=>{}),a(),await i.flush().catch(()=>{}),u}}async function gr(){let e=await re(),t=xe(process.argv.slice(2)),r=Re(e,t),n=v.resolve(r.config.logDir||"logs"),i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",s=i?v.join(v.resolve(i),"deploy-diff.json"):"",o=new U(n,"deploy.log.jsonl",r.config.logLevel??"info"),l=r.profile?.extraReplacements??{},a=async()=>{};try{let u=await cr(e,r.config,r.name,l,o);a=u.cleanup;let c={...r.config,outputDir:u.outputDir},f=c.s3SyncMode??"sdk-upload-delete",g=await ve(e),m=u.outputDir!==e.outputDir&&(e.targetOrigin!==r.config.targetOrigin||Object.keys(l).length>0),b=!!g&&!g.fullSyncRequired&&!m;r.name&&o.info(`Using deployment profile: ${r.name}`),o.info(`Starting deploy \u2014 s3://${c.s3.bucket}/${c.s3.prefix||""} (region: ${c.s3.region})`),o.info(`S3 sync mode: ${f}`),g&&g.fullSyncRequired?o.info("Deploy plan requests a full sync; scanning the whole output tree."):g&&m?o.info("Ignoring incremental deploy plan because deployment-profile rewrites require a full sync."):b&&o.info(`Using incremental deploy plan: ${g.changedFiles.length} changed file(s), ${g.deletedFiles.length} deleted file(s).`),o.mark("deploy");let y=new S3Client({region:c.s3.region}),p=b?(await Promise.all(g.changedFiles.map(async h=>{try{return (await $.stat(v.join(c.outputDir,h))).isFile()?h:null}catch{return null}}))).filter(h=>h!==null):await St(["**/*"],{cwd:c.outputDir,onlyFiles:!0,dot:!0}),de=b?new Set:new Set(p.map(h=>ae(c.s3.prefix,h)));b?o.info(`Using ${p.length} planned changed file(s) from deploy plan for upload.`):o.info(`Found ${p.length} file(s) in output directory "${c.outputDir}".`),o.progress("Deploy preflight: comparing local output with remote S3 state.",{phase:"deploy",mode:f,totalFiles:p.length,deploymentProfile:r.name||"",s3Bucket:c.s3.bucket,s3Prefix:c.s3.prefix||"",targetedPlan:b});let S=0,P=0,E=0,Y=[],O=[],Z=[],z=[],F=new Map,Be=new Map,qe=new Map,Je=new Map,He=new Map;b||(o.info("Listing existing S3 objects to detect unchanged files..."),F=await tr(y,c.s3.bucket,c.s3.prefix,o),o.info(`Found ${F.size} existing key(s) in S3.`));let Ke=async h=>F.has(h)?F.get(h)??null:b?rr(y,c.s3.bucket,h,Be):null;for(let h of p){let fe=v.join(c.outputDir,h),d=ae(c.s3.prefix,h),j=Bt.lookup(h)||"application/octet-stream",Ge=oe(h)?c.s3.htmlCacheControl:c.s3.assetCacheControl,w=S+P+E+1;p.length<=200||w<=5||w%10===0||w===p.length?(o.progress(`Uploading [${w}/${p.length}] ${d} (${j})`,{file:h,key:d,index:w,totalFiles:p.length,contentType:j,mode:f}),o.info(`Uploading [${w}/${p.length}] ${d} (${j})`)):o.debug(`Uploading [${w}/${p.length}] ${d} (${j})`);try{let x=await $.readFile(fe),_=await Ke(d),A=null,M=null;if(_){let X=x.byteLength;if(_.size===X){let B=_.etag.includes("-");if(_.etag!==""&&!B){let Qe=ir(x);if(_.etag===Qe){P++,p.length<=200||P<=5||P%25===0?(o.progress(`Skipped unchanged [${w}/${p.length}] ${d}`,{file:h,key:d,index:w,totalFiles:p.length,uploaded:S,skipped:P,failed:E,mode:f,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${w}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${w}/${p.length}] ${d}`),O.push(d);continue}}A=Ne(x);let me=await sr(y,c.s3.bucket,d,qe);if(me&&me===A){P++,p.length<=200||P<=5||P%25===0?(o.progress(`Skipped unchanged [${w}/${p.length}] ${d}`,{file:h,key:d,index:w,totalFiles:p.length,uploaded:S,skipped:P,failed:E,mode:f,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${w}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${w}/${p.length}] ${d}`),O.push(d);continue}}if(oe(h)){M=ue(x);let B=await or(y,c.s3.bucket,d,Je,He);if(B&&B===M){P++,p.length<=200||P<=5||P%25===0?(o.progress(`Skipped semantically unchanged [${w}/${p.length}] ${d}`,{file:h,key:d,index:w,totalFiles:p.length,uploaded:S,skipped:P,failed:E,mode:f,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${w}/${p.length}] ${d}`)):o.debug(`Skipped semantically unchanged [${w}/${p.length}] ${d}`),O.push(d);continue}}}A=A??Ne(x),M=M??(oe(h)?ue(x):null),await C(`S3 upload ${d}`,9e5,({abortController:X})=>new Upload({client:y,abortController:X,params:{Bucket:c.s3.bucket,Key:d,Body:x,ContentType:j,CacheControl:Ge,Metadata:{[Ue]:A,...M?{[We]:M}:{}}}}).done()),S++,Y.push(d),(S%25===0||S===p.length)&&(o.progress(`Uploaded ${S}/${p.length} file(s)\u2026`,{uploaded:S,skipped:P,failed:E,totalFiles:p.length,mode:f}),o.info(`Uploaded ${S}/${p.length} file(s)\u2026`));}catch(x){E++,Z.push(d),o.error(`Failed to upload ${d}: ${x.message}`,{key:d,error:String(x)});}}o.info(`Upload complete: ${S} uploaded, ${P} skipped unchanged, ${E} failed.`),f==="sdk-upload-delete"?b?z=await ze(y,c.s3.bucket,g.deletedFiles.map(h=>ae(c.s3.prefix,h)),o):z=await nr(y,c.s3.bucket,F.keys(),de,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion."),s&&(await $.mkdir(v.dirname(s),{recursive:!0}),await $.writeFile(s,JSON.stringify({generatedAt:new Date().toISOString(),mode:f,summary:{totalFiles:p.length,uploaded:Y.length,skipped:O.length,failed:Z.length,deleted:z.length,...r.name?{profile:r.name}:{}},uploadedKeys:Y,skippedKeys:O,failedKeys:Z,deletedKeys:z},null,2),"utf8")),await $e(e),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}gr().catch(async e=>{console.error(e);try{let r=(await re().catch(()=>null))?.logDir??"logs",n=new U(r,"deploy.log.jsonl","debug");n.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
|
6
|
+
`;return e.replace(/<head\b[^>]*>/i,i=>`${i}${n} ${t}`)}function kt(e){let t=Oe(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function It(e,t){let r=[...new Set(t)].filter(s=>s.includes("/")&&!/\\+\//.test(s)).map(s=>[s,H(s)]).filter(([s,o])=>s!==o).sort((s,o)=>o[0].length-s[0].length);if(!r.length)return e;let n=s=>{let o=s;for(let[l,a]of r)o=o.split(l).join(a);return o},i=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(s,o,l,a)=>`${o}${n(l)}${a}`);return i=i.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(s,o,l,a,u)=>kt(a)?`${o}${n(a)}${u}`:s),i}function Ot(e,t){if(!t)return null;let r=v.dirname(v.resolve(t)),n=v.resolve(e.outputDir),i=v.relative(r,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i.endsWith("/")?i:`${i}/`):"./"}function Ft(e,t,r){let n=Ot(t,r);if(!n)return e;let i=e;for(let s of Ct){let o=s.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");i=i.replace(l,`${n}${s}`).replace(a,`${n}${s}`);let u=s.replace(/\//g,"\\/"),c=n.replace(/\//g,"\\/"),d=u.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),g=new RegExp(`(?:\\.{1,}\\\\/)+${d}`,"g"),m=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${d}`,"g"),w=new RegExp(`(?<![\\.\\\\])\\/${d}`,"g");i=i.replace(g,`${c}${u}`).replace(m,`${c}${u}`).replace(w,`${c}${u}`);}return i}function se(e,t,r){if(e.urlRewriteMode==="absolute")return t;let n=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),i=n?n.pathname:Fe(t,e.targetOrigin),s=n?`${n.search}${n.hash}`:"";return e.urlRewriteMode==="root-relative"?`${i}${s}`:`${je(e.outputDir,r,i)}${s}`}function ie(e,t,r,n,i){J(e,r,se(t,n,i));}function jt(e,t,r,n,i){if(!r||r===n)return;J(e,r,n);let s=se(t,r,i),o=se(t,n,i);J(e,s,o);}function _t(e,t,r,n,i={}){let s=e,o={};ie(o,t,t.sourceOrigin,t.targetOrigin,n);for(let[a,u]of Object.entries(t.extraReplacements))ie(o,t,a,u,n);for(let[a,u]of Object.entries(r))ie(o,t,a,u,n);for(let[a,u]of Object.entries(i)){let c=r[a];!c||c===u||jt(o,t,u,c,n);}let l=Object.entries(o).sort((a,u)=>u[0].length-a[0].length);for(let[a,u]of l)s=s.split(a).join(u);return t.urlRewriteMode!=="absolute"&&(s=Ft(s,t,n)),s=It(s,l.map(([,a])=>a)),Mt(t,s,n)&&(s=Lt(s)),s}async function At(e,t,r,n){let i=v.join(e.outputDir,n),s=v.extname(i).toLowerCase(),o;try{o=await R.readFile(i,"utf8");}catch(u){if(u.code!=="ENOENT")throw u;return {changed:false}}let a=_t(o,e,t,s===".js"||s===".mjs"?void 0:i,r);return a!==o?(await R.writeFile(i,a,"utf8"),{changed:true}):{changed:false}}async function Ut(e,t){return await new Promise((r,n)=>{let i=a=>{if(l(),a?.error){n(new Error(a.error));return}r({changed:!!a?.changed});},s=a=>{l(),n(a);},o=a=>{l(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{e.off("message",i),e.off("error",s),e.off("exit",o);};e.on("message",i),e.on("error",s),e.on("exit",o),e.postMessage({file:t});})}async function Ue(e,t,r={}){let i=(r.files?[...new Set(r.files)].map(g=>v.resolve(g)).filter(g=>g.startsWith(v.resolve(e.outputDir))).map(g=>v.relative(e.outputDir,g).replace(/\\/g,"/")).filter(g=>g.length>0):await xt(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(g=>g!=="asset-map.json"&&Ie(g)),s=0;if(i.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(o,i.length,availableParallelism()));await r.onStart?.({totalFiles:i.length,workerCount:l});let a=0,u=0,c=null;if(l===1){for(let g of i){let m=await At(e,t,r.previousAssetMap??{},g);m.changed&&(s+=1),u+=1,await r.onProgress?.({index:u,totalFiles:i.length,changedTextFiles:s,file:g,changed:m.changed});}return s}let d=Array.from({length:l},async()=>{let g=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!c;){let m=a;if(m>=i.length)return;a+=1;let w=i[m],b=await Ut(g,w);b.changed&&(s+=1),u+=1,await r.onProgress?.({index:u,totalFiles:i.length,changedTextFiles:s,file:w,changed:b.changed});}}catch(m){throw c=m instanceof Error?m:new Error(String(m)),c}finally{await g.terminate().catch(()=>{});}});return await Promise.all(d),s}var ze="wpsuite-sha256",Be="wpsuite-normalized-sha256",Gt=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,pe=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,Qt=["SIGINT","SIGTERM"],G=new Set,K=new Map,Ne=null;function Vt(e){G.add(e);let t=false;return ()=>{t||(t=true,G.delete(e));}}async function Yt(){let e=[...G];e.length!==0&&await Promise.allSettled(e.map(async t=>{G.delete(t),await R.rm(t,{recursive:true,force:true});}));}function Zt(){if(!(K.size>0))for(let e of Qt){let t=()=>{Ne||(Ne=Yt().finally(()=>{for(let[r,n]of K)process.off(r,n);K.clear(),process.exit(1);}));};K.set(e,t),process.on(e,t);}}Zt();function Xt(e){return e&&(pe.test(e)?"__uuid__":/^[0-9a-f]{6,}$/i.test(e)?"__hex__":/^[0-9]{8,}$/.test(e)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(e)?"__alnum__":e)}function le(e){return e.split(/([_:.\-/=?&#]+)/g).map((t,r)=>r%2===1?t:Xt(t)).join("")}function er(e){return e.split(/([_:.\-/=?&#]+)/g).map((t,r)=>r%2===1?t:pe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{6,}$/.test(t)?"__num__":/^[a-z0-9_-]{6,}$/i.test(t)?"__id__":t).join("")}function Q(e,t){return Array.isArray(e)?e.map(r=>Q(r,t)):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([r,n])=>[r,Q(n,r)])):typeof e!="string"?e:t&&Gt.test(t)?er(e):pe.test(e)?"__uuid__":/^[0-9a-f]{6,}$/i.test(e)||/^[0-9]{8,}$/.test(e)?le(e):e}function tr(e){return e.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function rr(e,t){let r=t.trim();if((r.startsWith("{")||r.startsWith("[")||r.startsWith('"'))&&e.startsWith("data-"))try{let i=JSON.parse(r);return JSON.stringify(Q(i))}catch{}if(e.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(r))try{let i=Buffer.from(r,"base64").toString("utf8"),s=JSON.parse(i),o=JSON.stringify(Q(s));return Buffer.from(o,"utf8").toString("base64")}catch{}return t}function nr(e,t){let r=e.toLowerCase();return r==="class"?t.split(/(\s+)/).map((n,i)=>i%2===1?n:le(n)).join(""):r==="id"||r==="for"||r==="aria-controls"||r==="aria-labelledby"||r==="aria-describedby"||r==="aria-owns"||r==="aria-activedescendant"||r==="name"?le(t):r==="href"||r==="src"?tr(t):rr(r,t)}function ir(e){return e.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(t,r,n)=>{if(r.startsWith("/"))return t;let i=n.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(s,o,l,a)=>{let u=nr(o,a);return ` ${o}=${l}${u}${l}`});return `<${r}${i}>`})}function oe(e){let t=v.extname(e).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(t)||["robots.txt","sitemap.xml"].includes(v.basename(e))}function ae(e,t){return [e.replace(/^\/+|\/+$/g,""),t.replace(/^\/+/,"")].filter(Boolean).join("/")}async function sr(e,t,r,n){let i=new Map,s,o=0;do{o++;let l=await C(`S3 list objects page ${o} for s3://${t}/${r||""}`,3e5,({httpOptions:a})=>e.send(new ListObjectsV2Command({Bucket:t,Prefix:r||void 0,ContinuationToken:s}),a));for(let a of l.Contents||[]){if(!a.Key)continue;let u=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),c=Number(a.Size??0);i.set(a.Key,{etag:u,size:c});}s=l.NextContinuationToken,n.debug(`Listed S3 keys page ${o}: ${i.size} keys so far`);}while(s);return i}async function or(e,t,r,n){if(n.has(r))return n.get(r)??null;try{let i=await C(`S3 head object ${r}`,3e5,({httpOptions:o})=>e.send(new HeadObjectCommand({Bucket:t,Key:r}),o)),s={etag:(i.ETag||"").replace(/^"|"$/g,"").toLowerCase(),size:Number(i.ContentLength??0)};return n.set(r,s),s}catch{return n.set(r,null),null}}async function qe(e,t,r,n){let i=[...new Set(Array.from(r).filter(Boolean))];if(i.length===0)return n.info("No stale S3 objects to delete."),[];n.info(`Deleting ${i.length} stale S3 object(s)\u2026`);for(let s=0;s<i.length;s+=1e3){let o=i.slice(s,s+1e3);if(o.length!==0){await C(`S3 delete objects batch ${Math.floor(s/1e3)+1}`,3e5,({httpOptions:l})=>e.send(new DeleteObjectsCommand({Bucket:t,Delete:{Objects:o.map(a=>({Key:a}))}}),l)),n.info(`Deleted batch of ${o.length} object(s).`);for(let l of o)n.debug(` deleted: ${l}`);}}return i}async function ar(e,t,r,n,i){return qe(e,t,Array.from(r).filter(s=>!n.has(s)),i)}function lr(e){return createHash("md5").update(e).digest("hex")}function We(e){return createHash("sha256").update(e).digest("hex")}function ue(e){return createHash("sha256").update(ir(e.toString("utf8"))).digest("hex")}async function ur(e,t,r,n){if(n.has(r))return n.get(r)??null;try{let o=((await C(`S3 head object metadata ${r}`,3e5,({httpOptions:l})=>e.send(new HeadObjectCommand({Bucket:t,Key:r}),l))).Metadata?.[ze]||"").trim().toLowerCase()||null;return n.set(r,o),o}catch{return n.set(r,null),null}}async function cr(e,t,r,n,i){if(n.has(r))return n.get(r)??null;try{let o=((await C(`S3 head object normalized metadata ${r}`,3e5,({httpOptions:l})=>e.send(new HeadObjectCommand({Bucket:t,Key:r}),l))).Metadata?.[Be]||"").trim().toLowerCase();if(o)return n.set(r,o),o}catch{return n.set(r,null),null}if(i.has(r)){let s=i.get(r)??null;return n.set(r,s),s}try{let o=await(await C(`S3 get object ${r}`,3e5,({httpOptions:a})=>e.send(new GetObjectCommand({Bucket:t,Key:r}),a))).Body?.transformToByteArray(),l=o?ue(Buffer.from(o)):null;return i.set(r,l),n.set(r,l),l}catch{return i.set(r,null),n.set(r,null),null}}function gr(e){return e.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function pr(e,t){let r=e,n=Object.entries(t).sort((i,s)=>s[0].length-i[0].length);for(let[i,s]of n)i&&(r=r.split(i).join(s));return r}async function dr(e,t,r,n){let i=v.join(e,"asset-map.json"),s;try{s=await R.readFile(i,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(s);if(!o||typeof o!="object")return;let l=Object.fromEntries(Object.entries(o).map(([a,u])=>{let c=String(u??"");return t.urlRewriteMode==="absolute"&&t.targetOrigin&&r.targetOrigin&&t.targetOrigin!==r.targetOrigin&&(c=c.split(t.targetOrigin).join(r.targetOrigin)),c=pr(c,n),[a,c]}));await R.writeFile(i,JSON.stringify(l,null,2),"utf8");}async function fr(e,t,r,n,i){let s=e.targetOrigin!==t.targetOrigin,o=Object.keys(n).length>0;if(!r||!s&&!o)return {outputDir:e.outputDir,cleanup:async()=>{}};if(s&&e.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${r}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let l=v.join(v.dirname(e.outputDir),`.deploy-profile-${gr(r)}-${Date.now()}`),a=Vt(l);i.info(`Preparing deployment profile "${r}" from existing crawl output to ${l}...`);try{await R.rm(l,{recursive:!0,force:!0}),await R.cp(e.outputDir,l,{recursive:!0});let u={...e,sourceOrigin:e.targetOrigin,targetOrigin:t.targetOrigin,extraReplacements:n,outputDir:l},c=1;i.info(`Rewriting text files for deployment profile "${r}"...`);let d=await Ue(u,{},{onStart:({totalFiles:g,workerCount:m})=>{c=m,i.info(`Rewriting ${g} text file(s) for deployment profile "${r}" with ${m} worker(s).`);},onProgress:({index:g,totalFiles:m,changedTextFiles:w,file:b,changed:p})=>{let S=`Rewriting [${g}/${m}] ${b} (${p?"rewrote":"checked"}; ${w} changed so far)`;m<=200||g<=5||g%10===0||g===m?(i.progress(S,{file:b,index:g,totalFiles:m,changed:p,changedTextFiles:w,workerCount:c}),i.info(S)):i.debug(S);}});return await dr(l,e,t,n),i.info(`Prepared deployment profile "${r}" in ${l} (${d} rewritten text file(s)).`),{outputDir:l,cleanup:async()=>{a(),await R.rm(l,{recursive:!0,force:!0});}}}catch(u){throw i.error(`Failed to prepare deployment profile "${r}" temporary output.`,{tempDir:l,outputDir:e.outputDir,error:u instanceof Error?u.message:String(u),...u instanceof Error&&u.stack?{stack:u.stack}:{}}),await R.rm(l,{recursive:true,force:true}).catch(()=>{}),a(),await i.flush().catch(()=>{}),u}}async function mr(){let e=await re(),t=De(process.argv.slice(2)),r=Ce(e,t),n=v.resolve(r.config.logDir||"logs"),i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",s=i?v.join(v.resolve(i),"deploy-diff.json"):"",o=new N(n,"deploy.log.jsonl",r.config.logLevel??"info"),l=r.profile?.extraReplacements??{},a=async()=>{};try{let u=await fr(e,r.config,r.name,l,o);a=u.cleanup;let c={...r.config,outputDir:u.outputDir},d=c.s3SyncMode??"sdk-upload-delete",g=await xe(e),m=u.outputDir!==e.outputDir&&(e.targetOrigin!==r.config.targetOrigin||Object.keys(l).length>0),w=!!g&&!g.fullSyncRequired&&!m;r.name&&o.info(`Using deployment profile: ${r.name}`),o.info(`Starting deploy \u2014 s3://${c.s3.bucket}/${c.s3.prefix||""} (region: ${c.s3.region})`),o.info(`S3 sync mode: ${d}`),g&&g.fullSyncRequired?o.info("Deploy plan requests a full sync; scanning the whole output tree."):g&&m?o.info("Ignoring incremental deploy plan because deployment-profile rewrites require a full sync."):w&&o.info(`Using incremental deploy plan: ${g.changedFiles.length} changed file(s), ${g.deletedFiles.length} deleted file(s).`),o.mark("deploy");let b=new S3Client({region:c.s3.region}),p=w?(await Promise.all(g.changedFiles.map(async h=>{try{return (await R.stat(v.join(c.outputDir,h))).isFile()?h:null}catch{return null}}))).filter(h=>h!==null):await xt(["**/*"],{cwd:c.outputDir,onlyFiles:!0,dot:!0}),de=w?new Set:new Set(p.map(h=>ae(c.s3.prefix,h)));w?o.info(`Using ${p.length} planned changed file(s) from deploy plan for upload.`):o.info(`Found ${p.length} file(s) in output directory "${c.outputDir}".`),o.progress("Deploy preflight: comparing local output with remote S3 state.",{phase:"deploy",mode:d,totalFiles:p.length,deploymentProfile:r.name||"",s3Bucket:c.s3.bucket,s3Prefix:c.s3.prefix||"",targetedPlan:w});let S=0,P=0,E=0,V=[],O=[],Y=[],z=[],F=new Map,He=new Map,Je=new Map,Ke=new Map,Ge=new Map;w||(o.info("Listing existing S3 objects to detect unchanged files..."),F=await sr(b,c.s3.bucket,c.s3.prefix,o),o.info(`Found ${F.size} existing key(s) in S3.`));let Qe=async h=>F.has(h)?F.get(h)??null:w?or(b,c.s3.bucket,h,He):null;for(let h of p){let fe=v.join(c.outputDir,h),f=ae(c.s3.prefix,h),j=Kt.lookup(h)||"application/octet-stream",Ve=oe(h)?c.s3.htmlCacheControl:c.s3.assetCacheControl,y=S+P+E+1;p.length<=200||y<=5||y%10===0||y===p.length?(o.progress(`Uploading [${y}/${p.length}] ${f} (${j})`,{file:h,key:f,index:y,totalFiles:p.length,contentType:j,mode:d}),o.info(`Uploading [${y}/${p.length}] ${f} (${j})`)):o.debug(`Uploading [${y}/${p.length}] ${f} (${j})`);try{let x=await R.readFile(fe),_=await Qe(f),A=null,L=null;if(_){let Z=x.byteLength;if(_.size===Z){let B=_.etag.includes("-");if(_.etag!==""&&!B){let Ye=lr(x);if(_.etag===Ye){P++,p.length<=200||P<=5||P%25===0?(o.progress(`Skipped unchanged [${y}/${p.length}] ${f}`,{file:h,key:f,index:y,totalFiles:p.length,uploaded:S,skipped:P,failed:E,mode:d,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${y}/${p.length}] ${f}`)):o.debug(`Skipped unchanged [${y}/${p.length}] ${f}`),O.push(f);continue}}A=We(x);let me=await ur(b,c.s3.bucket,f,Je);if(me&&me===A){P++,p.length<=200||P<=5||P%25===0?(o.progress(`Skipped unchanged [${y}/${p.length}] ${f}`,{file:h,key:f,index:y,totalFiles:p.length,uploaded:S,skipped:P,failed:E,mode:d,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${y}/${p.length}] ${f}`)):o.debug(`Skipped unchanged [${y}/${p.length}] ${f}`),O.push(f);continue}}if(oe(h)){L=ue(x);let B=await cr(b,c.s3.bucket,f,Ke,Ge);if(B&&B===L){P++,p.length<=200||P<=5||P%25===0?(o.progress(`Skipped semantically unchanged [${y}/${p.length}] ${f}`,{file:h,key:f,index:y,totalFiles:p.length,uploaded:S,skipped:P,failed:E,mode:d,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${y}/${p.length}] ${f}`)):o.debug(`Skipped semantically unchanged [${y}/${p.length}] ${f}`),O.push(f);continue}}}A=A??We(x),L=L??(oe(h)?ue(x):null),await C(`S3 upload ${f}`,9e5,({abortController:Z})=>new Upload({client:b,abortController:Z,params:{Bucket:c.s3.bucket,Key:f,Body:x,ContentType:j,CacheControl:Ve,Metadata:{[ze]:A,...L?{[Be]:L}:{}}}}).done()),S++,V.push(f),(S%25===0||S===p.length)&&(o.progress(`Uploaded ${S}/${p.length} file(s)\u2026`,{uploaded:S,skipped:P,failed:E,totalFiles:p.length,mode:d}),o.info(`Uploaded ${S}/${p.length} file(s)\u2026`));}catch(x){E++,Y.push(f),o.error(`Failed to upload ${f}: ${x.message}`,{key:f,error:String(x)});}}o.info(`Upload complete: ${S} uploaded, ${P} skipped unchanged, ${E} failed.`),d==="sdk-upload-delete"?w?z=await qe(b,c.s3.bucket,g.deletedFiles.map(h=>ae(c.s3.prefix,h)),o):z=await ar(b,c.s3.bucket,F.keys(),de,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion."),s&&(await R.mkdir(v.dirname(s),{recursive:!0}),await R.writeFile(s,JSON.stringify({generatedAt:new Date().toISOString(),mode:d,summary:{totalFiles:p.length,uploaded:V.length,skipped:O.length,failed:Y.length,deleted:z.length,...r.name?{profile:r.name}:{}},uploadedKeys:V,skippedKeys:O,failedKeys:Y,deletedKeys:z},null,2),"utf8")),await $e(e),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}mr().catch(async e=>{console.error(e);try{let r=(await re().catch(()=>null))?.logDir??"logs",n=new N(r,"deploy.log.jsonl","debug");n.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
package/dist/invalidate.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {CloudFrontClient,CreateInvalidationCommand}from'@aws-sdk/client-cloudfront';import d from'fs/promises';import p from'path';import {getConfig}from'@smart-cloud/wpsuite-core';function A(r){let t=r/6e4;if(Number.isInteger(t)&&t>=1)return `${t} minute${"s"}`;let i=Math.ceil(r/1e3);return `${i} second${i===1?"":"s"}`}async function I(r,t,i){let e=new AbortController,o=globalThis.setTimeout(()=>{e.abort(new Error(`${r} timed out after ${A(t)}.`));},t);try{return await i({abortController:e,httpOptions:{abortSignal:e.signal,requestTimeout:t}})}finally{globalThis.clearTimeout(o);}}function T(r){let t=r.trim();return t==="."?".":t.replace(/\/$/,"")}function $(r){let t=String(r??"").trim();if(!t)return "";let i=t;if(/^https?:\/\//i.test(i))try{i=new URL(i).pathname;}catch{return ""}if(i=i.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!i)return "";let e=i.endsWith("/"),o=i.replace(/^\/+/,"").split("/").map(s=>s.replace(/[^A-Za-z0-9._-]/g,"")).filter(s=>s.length>0&&s!=="."&&s!=="..");if(o.length===0)return "";let n=`/${o.join("/")}`;return e&&p.extname(n)===""&&(n+="/"),n==="/"?"":n}function y(r){return !r||typeof r!="object"?{}:Object.fromEntries(Object.entries(r).map(([t,i])=>[t.trim(),String(i??"")]).filter(([t])=>t.length>0))}function W(r){if(!r||typeof r!="object")return {};let t={};for(let[i,e]of Object.entries(r)){let o=i.trim();if(!o||!e||typeof e!="object")continue;let n=e,s={};if(typeof n.targetOrigin=="string"){let a=T(n.targetOrigin);a&&(s.targetOrigin=a);}let u=y(n.extraReplacements);if(Object.keys(u).length>0&&(s.extraReplacements=u),n.s3&&typeof n.s3=="object"){let a={},l=n.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(s.s3=a);}if(n.cloudFront&&typeof n.cloudFront=="object"){let a={},l=n.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(s.cloudFront=a);}t[o]=s;}return t}function L(r){return r==="PROFESSIONAL"||r==="AGENCY"?r:void 0}function _(r,t){if(!r||typeof r!="object")return null;let i=r,e=i.command;if(e!=="publish"&&e!=="crawl"&&e!=="deploy"&&e!=="invalidate"&&e!=="retry-timeouts"&&e!=="url")return null;let o=Number.parseInt(String(i.intervalMinutes??"0"),10);if(!Number.isFinite(o)||o<1)return null;let n=String(i.id??`${e}-${t+1}`).trim();if(!n)return null;let s=String(i.deploymentProfile??"").trim(),u=String(i.url??"").trim();return {id:n,enabled:i.enabled!==false,command:e,intervalMinutes:o,...e==="publish"||e==="crawl"?{crawlMode:i.crawlMode==="incremental"?"incremental":"full"}:{},...(e==="publish"||e==="deploy"||e==="invalidate")&&s?{deploymentProfile:s}:{},...u?{url:u}:{}}}function D(r){let t=r&&typeof r=="object"?r:{},i=Array.isArray(t.rules)?t.rules.map((e,o)=>_(e,o)).filter(e=>!!e):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:i}}function q(r){if(!r||typeof r!="object")return;let t=r,i={};for(let o of ["accountId","siteId"]){let n=t[o];typeof n=="string"&&n.trim()!==""&&(i[o]=n.trim());}let e=Number(t.lastUpdate??0);return Number.isFinite(e)&&e>0&&(i.lastUpdate=Math.floor(e)),t.subscriber===true&&(i.subscriber=true),Object.keys(i).length>0?i:void 0}function f(r){let t=r&&typeof r=="object"?r:{},i=String(t.accountId??"").trim(),e=String(t.siteId??"").trim(),o=t.subscriber===true,n=q(t.siteSettings),s={...n??{},...i?{accountId:n?.accountId??i}:{},...e?{siteId:n?.siteId??e}:{},...n?.subscriber===true||o?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let l=String(t.runtimeToken??t.nonce??"").trim();l&&(u.runtimeToken=l);let c=String(t.uploadUrl??"").trim();c&&(u.uploadUrl=c),Object.keys(s).length>0&&(u.siteSettings=s);let g=L(t.subscriptionType);return g&&(u.subscriptionType=g),u}function J(r){try{let t=new URL(r);globalThis.location=t;}catch{}}function B(r){let t=f(r.wpsuite),i={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},e=String(i.accountId??"").trim(),o=String(i.siteId??"").trim(),n=String(t.uploadUrl??"").trim();if(r.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...n?{uploadUrl:n}:{},...Object.keys(i).length>0?{siteSettings:i}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!e||!o||!n)return false;J(r.sourceOrigin);let s=globalThis,a={...s.WpSuite??{},siteSettings:i,uploadUrl:n};return t.apiBase&&(a.apiBase=t.apiBase),s.WpSuite=a,true}async function z(r){let t=f(r.wpsuite),i=t.siteSettings??{},e=String(i.accountId??"").trim()||null,o=String(i.siteId??"").trim()||null,n=String(t.uploadUrl??"").trim();if(!B(r))return console.info(`[INFO] Skipping remote WPSuite publisher config load: ${JSON.stringify({accountId:e,siteId:o,uploadUrl:n||null,subscriber:i.subscriber===true,lastUpdate:i.lastUpdate??null})}`),null;try{let s=await getConfig("publisher"),u=s&&typeof s=="object"?s:null,a=u?.siteSettings&&typeof u.siteSettings=="object"?u.siteSettings:null;return console.info(`[INFO] Loaded remote WPSuite publisher config: ${JSON.stringify({accountId:e,siteId:o,uploadUrl:n||null,subscriber:i.subscriber===!0,lastUpdate:i.lastUpdate??null,remoteType:s===null?"null":typeof s,remoteSubscriptionType:typeof u?.subscriptionType=="string"?u.subscriptionType:null,remoteKeys:u?Object.keys(u).sort():[],remoteAccountId:typeof u?.accountId=="string"?u.accountId:typeof a?.accountId=="string"?a.accountId:null,remoteSiteId:typeof u?.siteId=="string"?u.siteId:typeof a?.siteId=="string"?a.siteId:null,remoteSiteSettingsKeys:a?Object.keys(a).sort():[]})}`),s&&typeof s=="object"?s:null}catch(s){return console.warn(`[WARN] Remote WPSuite publisher config load failed: ${JSON.stringify({accountId:e,siteId:o,uploadUrl:n||null,subscriber:i.subscriber===true,lastUpdate:i.lastUpdate??null,error:s instanceof Error?s.message:String(s)})}`),null}}function Q(r,t){let i=W(t?.deploymentProfiles),e=String(t?.defaultDeploymentProfile??"").trim(),o=i[e]?e:"",n=L(t?.subscriptionType),s=f(r.wpsuite);return {...r,scheduler:D(t?.scheduler),deploymentProfiles:i,defaultDeploymentProfile:o,...n?{subscriptionType:n}:{},wpsuite:{...s,...s.siteSettings||n?{siteSettings:{...s.siteSettings??{},...s.siteSettings?.subscriber===true||n==="PROFESSIONAL"||n==="AGENCY"?{subscriber:true}:{}}}:{},...n?{subscriptionType:n}:{}}}}function w(r,t){let e=String(r||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!e)return t;let o=e.split("/").map(n=>n.trim()).filter(n=>n.length>0&&n!=="."&&n!=="..");return o.length>0?o.join("/"):t}function H(){let r=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return r.trim()?p.resolve(r):""}function C(r,t,i){let e=String(t||"").trim();return e&&p.isAbsolute(e)?p.resolve(e):p.resolve(r,w(e,i))}async function v(r){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",i=await d.readFile(t,"utf8"),e=JSON.parse(i);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=T(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=$(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=y(e.extraReplacements),e.postCrawlCopyMap=y(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.wpsuite=f(e.wpsuite),e.scheduler=D(void 0),e.deploymentProfiles={},e.defaultDeploymentProfile="",e.deploymentTargetOverride=String(e.deploymentTargetOverride??"").trim();let o=Q(e,await z(e)),n=H(),s=n?p.resolve(n,".."):"";return s?(o.outputDir=C(s,o.outputDir,"export"),o.logDir=C(s,o.logDir,"logs")):(p.isAbsolute(o.outputDir)||(o.outputDir=w(o.outputDir,"export")),p.isAbsolute(o.logDir)||(o.logDir=w(o.logDir,"logs"))),o}function G(r){return typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!==""}function R(r){let t="";for(let e=0;e<r.length;e++){let o=r[e];if(o==="--profile"){let n=(r[e+1]||"").trim();if(!n)throw new Error("Missing value for --profile.");t=n,e+=1;}else if(o.startsWith("--profile=")){let n=o.slice(10).trim();if(!n)throw new Error("Missing value for --profile.");t=n;}}let i=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||i||null}function Y(r,t){return {...r,...t??{}}}function K(r,t){return {...r,...t??{},invalidationPaths:[...(t?.invalidationPaths??r.invalidationPaths)||[]]}}function k(r,t){let i=String(t??r.deploymentTargetOverride??"").trim();if(!G(r)){if(i)throw new Error(`Deployment target "${i}" requires an active WPSuite subscription and remote publisher config.`);return {name:null,profile:null,config:r}}let e=String(i||r.defaultDeploymentProfile||"").trim();if(!e)return {name:null,profile:null,config:r};let o=r.deploymentProfiles?.[e];if(!o)throw new Error(`Unknown deployment profile "${e}". Check the linked WPSuite publisher configuration.`);return {name:e,profile:o,config:{...r,targetOrigin:o.targetOrigin??r.targetOrigin,s3:Y(r.s3,o.s3),cloudFront:K(r.cloudFront,o.cloudFront)}}}var F={error:0,warn:1,info:2,debug:3},E=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",b=E?p.join(E,"current-progress.json"):"",O=Promise.resolve();function x(r){return r.includes("crawl")?"crawl":r.includes("deploy")?"deploy":r.includes("invalidate")?"invalidate":r}function V(r,t){let i={...r},e=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[o,n]of Object.entries(t)){let s=i[o];if(typeof s=="number"&&typeof n=="number"&&e.has(o)){i[o]=Math.max(s,n);continue}i[o]=n;}return i}function Z(r,t,i){b&&(O=O.then(async()=>{let e=new Date,o=e.toISOString();await d.mkdir(p.dirname(b),{recursive:true});let n=await d.readFile(b,"utf8").then(P=>JSON.parse(P)).catch(()=>null),s=n&&typeof n.details=="object"&&n.details?n.details:{},u=x(r),a=n?.currentStep||x(n?.source||""),l=n?.startedAt||o,c={...n?.stepDurationsSec??{}},g=n?.stepStartedAt||o;if(a&&a!==u&&n?.stepStartedAt){let P=Math.max(0,Math.round((e.getTime()-new Date(n.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+P,g=o;}let S=Math.max(0,Math.round((e.getTime()-new Date(g).getTime())/1e3)),M=Math.max(0,Math.round((e.getTime()-new Date(l).getTime())/1e3)),j={checkedAt:o,source:r,message:t,details:V(s,i??{}),startedAt:l,currentStep:u,stepStartedAt:g,stepElapsedSec:S,totalElapsedSec:M,stepDurationsSec:c};await d.writeFile(b,JSON.stringify(j,null,2),"utf8");}).catch(()=>{}));}async function X(r,t){await d.mkdir(p.dirname(r),{recursive:true}),await d.appendFile(r,`${JSON.stringify(t)}
|
|
2
|
-
`,"utf8");}var
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(t,
|
|
1
|
+
import {CloudFrontClient,CreateInvalidationCommand}from'@aws-sdk/client-cloudfront';import m from'fs/promises';import q from'http';import _ from'https';import d from'path';function W(i){let t=i/6e4;if(Number.isInteger(t)&&t>=1)return `${t} minute${"s"}`;let r=Math.ceil(i/1e3);return `${r} second${r===1?"":"s"}`}async function T(i,t,r){let e=new AbortController,o=globalThis.setTimeout(()=>{e.abort(new Error(`${i} timed out after ${W(t)}.`));},t);try{return await r({abortController:e,httpOptions:{abortSignal:e.signal,requestTimeout:t}})}finally{globalThis.clearTimeout(o);}}function D(i){let t=i.trim();return t==="."?".":t.replace(/\/$/,"")}function B(i){let t=String(i??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let e=r.endsWith("/"),o=r.replace(/^\/+/,"").split("/").map(s=>s.replace(/[^A-Za-z0-9._-]/g,"")).filter(s=>s.length>0&&s!=="."&&s!=="..");if(o.length===0)return "";let n=`/${o.join("/")}`;return e&&d.extname(n)===""&&(n+="/"),n==="/"?"":n}function v(i){return !i||typeof i!="object"?{}:Object.fromEntries(Object.entries(i).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function J(i){if(!i||typeof i!="object")return {};let t={};for(let[r,e]of Object.entries(i)){let o=r.trim();if(!o||!e||typeof e!="object")continue;let n=e,s={};if(typeof n.targetOrigin=="string"){let a=D(n.targetOrigin);a&&(s.targetOrigin=a);}let l=v(n.extraReplacements);if(Object.keys(l).length>0&&(s.extraReplacements=l),n.s3&&typeof n.s3=="object"){let a={},u=n.s3;typeof u.bucket=="string"&&(a.bucket=u.bucket.trim()),typeof u.prefix=="string"&&(a.prefix=u.prefix.trim()),typeof u.region=="string"&&(a.region=u.region.trim()),typeof u.htmlCacheControl=="string"&&(a.htmlCacheControl=u.htmlCacheControl.trim()),typeof u.assetCacheControl=="string"&&(a.assetCacheControl=u.assetCacheControl.trim()),Object.keys(a).length>0&&(s.s3=a);}if(n.cloudFront&&typeof n.cloudFront=="object"){let a={},u=n.cloudFront;typeof u.distributionId=="string"&&(a.distributionId=u.distributionId.trim()),Array.isArray(u.invalidationPaths)&&(a.invalidationPaths=u.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(a).length>0&&(s.cloudFront=a);}t[o]=s;}return t}function k(i){return i==="PROFESSIONAL"||i==="AGENCY"?i:void 0}function H(i,t){if(!i||typeof i!="object")return null;let r=i,e=r.command;if(e!=="publish"&&e!=="crawl"&&e!=="deploy"&&e!=="invalidate"&&e!=="retry-timeouts"&&e!=="url")return null;let o=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(o)||o<1)return null;let n=String(r.id??`${e}-${t+1}`).trim();if(!n)return null;let s=String(r.deploymentProfile??"").trim(),l=String(r.url??"").trim();return {id:n,enabled:r.enabled!==false,command:e,intervalMinutes:o,...e==="publish"||e==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(e==="publish"||e==="deploy"||e==="invalidate")&&s?{deploymentProfile:s}:{},...l?{url:l}:{}}}function E(i){let t=i&&typeof i=="object"?i:{},r=Array.isArray(t.rules)?t.rules.map((e,o)=>H(e,o)).filter(e=>!!e):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function z(i){if(!i||typeof i!="object")return;let t=i,r={};for(let o of ["accountId","siteId"]){let n=t[o];typeof n=="string"&&n.trim()!==""&&(r[o]=n.trim());}let e=Number(t.lastUpdate??0);return Number.isFinite(e)&&e>0&&(r.lastUpdate=Math.floor(e)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function I(i){let t=i&&typeof i=="object"?i:{},r=String(t.accountId??"").trim(),e=String(t.siteId??"").trim(),o=t.subscriber===true,n=z(t.siteSettings),s={...n??{},...r?{accountId:n?.accountId??r}:{},...e?{siteId:n?.siteId??e}:{},...n?.subscriber===true||o?{subscriber:true}:{}},l={},a=String(t.apiBase??"").trim();a&&(l.apiBase=a);let u=String(t.runtimeToken??t.nonce??"").trim();u&&(l.runtimeToken=u);let c=String(t.uploadUrl??"").trim();c&&(l.uploadUrl=c),Object.keys(s).length>0&&(l.siteSettings=s);let g=k(t.subscriptionType);return g&&(l.subscriptionType=g),l}function Q(i){try{let t=new URL(i);globalThis.location=t;}catch{}}function F(i){return typeof i=="string"?i:i instanceof URL?i.toString():i.url}function G(i,t){let r=new Headers(i instanceof Request?i.headers:void 0);if(t?.headers)for(let[e,o]of new Headers(t.headers).entries())r.set(e,o);return r}async function O(i,t,r=5){let e=new URL(F(i)),o=String(t?.method??(i instanceof Request?i.method:"GET")).toUpperCase(),n=e.protocol==="http:"?q:_,s=G(i,t);return await new Promise((l,a)=>{let u=n.request(e,{method:o,headers:Object.fromEntries(s.entries()),...e.protocol==="https:"?{rejectUnauthorized:false}:{}},c=>{let g=[];c.on("data",p=>{g.push(Buffer.isBuffer(p)?p:Buffer.from(p));}),c.on("error",a),c.on("end",()=>{let p=c.statusCode??0,P=c.headers.location;if(r>0&&P&&[301,302,303,307,308].includes(p)){l(O(new URL(P,e),{method:p===303?"GET":o},r-1));return}let h=new Headers;for(let[f,w]of Object.entries(c.headers))Array.isArray(w)?h.set(f,w.join(", ")):typeof w=="string"&&h.set(f,w);l(new Response(Buffer.concat(g),{status:p,statusText:c.statusMessage??"",headers:h}));});});u.on("error",a),u.end();})}function Y(i){let t=I(i.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},e=String(r.accountId??"").trim(),o=String(r.siteId??"").trim(),n=String(t.uploadUrl??"").trim();if(i.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...n?{uploadUrl:n}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!e||!o||!n)return false;Q(i.sourceOrigin);let s=globalThis,a={...s.WpSuite??{},siteSettings:r,uploadUrl:n};return t.apiBase&&(a.apiBase=t.apiBase),s.WpSuite=a,true}async function K(i){if(!Y(i))return null;let t=String(i.wpsuite?.uploadUrl??"").trim(),r=globalThis.fetch;try{let e=globalThis,o=()=>({sourceOrigin:i.sourceOrigin,locationHref:e.location?.href??null,locationHostname:e.location?.hostname??null,wpsuite:e.WpSuite?{apiBase:e.WpSuite.apiBase??null,uploadUrl:e.WpSuite.uploadUrl??null,siteSettings:e.WpSuite.siteSettings??null}:null});console.info(`[DEBUG] Remote WPSuite config bootstrap before import: ${JSON.stringify(o())}`),i.ignoreHttpsErrors&&typeof r=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(l,a)=>F(l).startsWith(t)?O(l,a):r(l,a)));let{getConfig:n}=await import('@smart-cloud/wpsuite-core');console.info(`[DEBUG] Remote WPSuite config bootstrap before getConfig: ${JSON.stringify(o())}`);let s=await n("publisher");return s&&typeof s=="object"?s:null}catch{return null}finally{typeof r=="function"&&(globalThis.fetch=r);}}function V(i,t){let r=J(t?.deploymentProfiles),e=String(t?.defaultDeploymentProfile??"").trim(),o=r[e]?e:"",n=k(t?.subscriptionType),s=I(i.wpsuite);return {...i,scheduler:E(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:o,...n?{subscriptionType:n}:{},wpsuite:{...s,...s.siteSettings||n?{siteSettings:{...s.siteSettings??{},...s.siteSettings?.subscriber===true||n==="PROFESSIONAL"||n==="AGENCY"?{subscriber:true}:{}}}:{},...n?{subscriptionType:n}:{}}}}function C(i,t){let e=String(i||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!e)return t;let o=e.split("/").map(n=>n.trim()).filter(n=>n.length>0&&n!=="."&&n!=="..");return o.length>0?o.join("/"):t}function Z(){let i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return i.trim()?d.resolve(i):""}function L(i,t,r){let e=String(t||"").trim();return e&&d.isAbsolute(e)?d.resolve(e):d.resolve(i,C(e,r))}async function R(i){let t=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await m.readFile(t,"utf8"),e=JSON.parse(r);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=D(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=B(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=v(e.extraReplacements),e.postCrawlCopyMap=v(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.wpsuite=I(e.wpsuite),e.scheduler=E(void 0),e.deploymentProfiles={},e.defaultDeploymentProfile="",e.deploymentTargetOverride=String(e.deploymentTargetOverride??"").trim();let o=V(e,await K(e)),n=Z(),s=n?d.resolve(n,".."):"";return s?(o.outputDir=L(s,o.outputDir,"export"),o.logDir=L(s,o.logDir,"logs")):(d.isAbsolute(o.outputDir)||(o.outputDir=C(o.outputDir,"export")),d.isAbsolute(o.logDir)||(o.logDir=C(o.logDir,"logs"))),o}function X(i){return typeof i.subscriptionType=="string"&&i.subscriptionType.trim()!==""}function x(i){let t="";for(let e=0;e<i.length;e++){let o=i[e];if(o==="--profile"){let n=(i[e+1]||"").trim();if(!n)throw new Error("Missing value for --profile.");t=n,e+=1;}else if(o.startsWith("--profile=")){let n=o.slice(10).trim();if(!n)throw new Error("Missing value for --profile.");t=n;}}let r=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||r||null}function ee(i,t){return {...i,...t??{}}}function te(i,t){return {...i,...t??{},invalidationPaths:[...(t?.invalidationPaths??i.invalidationPaths)||[]]}}function M(i,t){let r=String(t??i.deploymentTargetOverride??"").trim();if(!X(i)){if(r)throw new Error(`Deployment target "${r}" requires an active WPSuite subscription and remote publisher config.`);return {name:null,profile:null,config:i}}let e=String(r||i.defaultDeploymentProfile||"").trim();if(!e)return {name:null,profile:null,config:i};let o=i.deploymentProfiles?.[e];if(!o)throw new Error(`Unknown deployment profile "${e}". Check the linked WPSuite publisher configuration.`);return {name:e,profile:o,config:{...i,targetOrigin:o.targetOrigin??i.targetOrigin,s3:ee(i.s3,o.s3),cloudFront:te(i.cloudFront,o.cloudFront)}}}var j={error:0,warn:1,info:2,debug:3},A=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",y=A?d.join(A,"current-progress.json"):"",U=Promise.resolve();function N(i){return i.includes("crawl")?"crawl":i.includes("deploy")?"deploy":i.includes("invalidate")?"invalidate":i}function ie(i,t){let r={...i},e=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[o,n]of Object.entries(t)){let s=r[o];if(typeof s=="number"&&typeof n=="number"&&e.has(o)){r[o]=Math.max(s,n);continue}r[o]=n;}return r}function re(i,t,r){y&&(U=U.then(async()=>{let e=new Date,o=e.toISOString();await m.mkdir(d.dirname(y),{recursive:true});let n=await m.readFile(y,"utf8").then(f=>JSON.parse(f)).catch(()=>null),s=n&&typeof n.details=="object"&&n.details?n.details:{},l=N(i),a=n?.currentStep||N(n?.source||""),u=n?.startedAt||o,c={...n?.stepDurationsSec??{}},g=n?.stepStartedAt||o;if(a&&a!==l&&n?.stepStartedAt){let f=Math.max(0,Math.round((e.getTime()-new Date(n.stepStartedAt).getTime())/1e3));c[a]=(c[a]??0)+f,g=o;}let p=Math.max(0,Math.round((e.getTime()-new Date(g).getTime())/1e3)),P=Math.max(0,Math.round((e.getTime()-new Date(u).getTime())/1e3)),h={checkedAt:o,source:i,message:t,details:ie(s,r??{}),startedAt:u,currentStep:l,stepStartedAt:g,stepElapsedSec:p,totalElapsedSec:P,stepDurationsSec:c};await m.writeFile(y,JSON.stringify(h,null,2),"utf8");}).catch(()=>{}));}async function ne(i,t){await m.mkdir(d.dirname(i),{recursive:true}),await m.appendFile(i,`${JSON.stringify(t)}
|
|
2
|
+
`,"utf8");}var S=class{constructor(t,r,e="info"){this.logDir=t;this.logFile=r;this.level=e;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return d.join(this.logDir,this.logFile)}get errorsPath(){return d.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await m.mkdir(this.logDir,{recursive:true}),await m.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await m.appendFile(this.logPath,`${t}
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>ne(t,r));}accepts(t){return j[t]<=j[this.level]}push(t,r,e){if(!this.accepts(t))return;let o=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...e||{}});this.enqueueLine(o),t==="error"?console.error(`[ERROR] ${r}`,e?JSON.stringify(e):""):t==="warn"?console.warn(`[WARN] ${r}`,e?JSON.stringify(e):""):console.log(`[${t.toUpperCase().padEnd(5)}] ${r}`);}info(t,r){this.push("info",t,r);}progress(t,r){re(this.logFile,t,r);}debug(t,r){this.push("debug",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let e=this.marks.get(t);if(!e)return;let o=Date.now()-e,n=Number((o/1e3).toFixed(2));this.push("info",`Timing ${t}: ${n}s`,{name:t,ms:o,seconds:n,...r||{}});}async flush(){let t=Date.now()-this.startedAt,r=Number((t/1e3).toFixed(2));if(this.push("info",`Total time: ${r}s`,{name:"total",ms:t,seconds:r}),await this.writeQueue,this.writeError)throw this.writeError}};async function ue(){let i=await R(),t=x(process.argv.slice(2)),r=M(i,t),e=r.config,o=e.logDir||"logs",n=new S(o,"invalidate.log.jsonl",e.logLevel??"info");if(r.name&&n.info(`Using deployment profile: ${r.name}`),!e.cloudFront.distributionId){n.info("No CloudFront distributionId configured; skipping invalidation."),await n.flush();return}let s=e.cloudFront.invalidationPaths.length>0?e.cloudFront.invalidationPaths:["/*"];n.info(`Starting CloudFront invalidation for distribution ${e.cloudFront.distributionId}`),n.info(`Invalidation paths (${s.length}): ${s.join(", ")}`),n.mark("invalidate"),n.progress("CloudFront invalidation request queued.",{phase:"invalidate",distributionId:e.cloudFront.distributionId,pathCount:s.length});let l=new CloudFrontClient({region:"us-east-1"}),a=`smartcloud-static-publisher-${Date.now()}`;n.debug(`CallerReference: ${a}`);let u=await T(`CloudFront invalidation for distribution ${e.cloudFront.distributionId}`,3e5,({httpOptions:p})=>l.send(new CreateInvalidationCommand({DistributionId:e.cloudFront.distributionId,InvalidationBatch:{CallerReference:a,Paths:{Quantity:s.length,Items:s}}}),p)),c=u.Invalidation?.Id??"(unknown)",g=u.Invalidation?.Status??"(unknown)";n.info(`CloudFront invalidation created. ID: ${c} Status: ${g}`),n.debug(`Full invalidation response: ${JSON.stringify(u.Invalidation)}`),n.endMark("invalidate"),await n.flush();}ue().catch(async i=>{console.error(i);try{let r=(await R().catch(()=>null))?.logDir??"logs",e=new S(r,"invalidate.log.jsonl","debug");e.error(`Unhandled error: ${i instanceof Error?i.message:String(i)}`,i instanceof Error?{stack:i.stack}:void 0),await e.flush();}catch{}process.exit(1);});
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import m from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import l from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';import {getConfig}from'@smart-cloud/wpsuite-core';function U(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function re(e){let t=String(e??"").trim();if(!t)return "";let n=t;if(/^https?:\/\//i.test(n))try{n=new URL(n).pathname;}catch{return ""}if(n=n.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!n)return "";let r=n.endsWith("/"),u=n.replace(/^\/+/,"").split("/").map(i=>i.replace(/[^A-Za-z0-9._-]/g,"")).filter(i=>i.length>0&&i!=="."&&i!=="..");if(u.length===0)return "";let o=`/${u.join("/")}`;return r&&l.extname(o)===""&&(o+="/"),o==="/"?"":o}function J(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,n])=>[t.trim(),String(n??"")]).filter(([t])=>t.length>0))}function ne(e){if(!e||typeof e!="object")return {};let t={};for(let[n,r]of Object.entries(e)){let u=n.trim();if(!u||!r||typeof r!="object")continue;let o=r,i={};if(typeof o.targetOrigin=="string"){let a=U(o.targetOrigin);a&&(i.targetOrigin=a);}let s=J(o.extraReplacements);if(Object.keys(s).length>0&&(i.extraReplacements=s),o.s3&&typeof o.s3=="object"){let a={},c=o.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(i.s3=a);}if(o.cloudFront&&typeof o.cloudFront=="object"){let a={},c=o.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(a).length>0&&(i.cloudFront=a);}t[u]=i;}return t}function L(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function ie(e,t){if(!e||typeof e!="object")return null;let n=e,r=n.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let u=Number.parseInt(String(n.intervalMinutes??"0"),10);if(!Number.isFinite(u)||u<1)return null;let o=String(n.id??`${r}-${t+1}`).trim();if(!o)return null;let i=String(n.deploymentProfile??"").trim(),s=String(n.url??"").trim();return {id:o,enabled:n.enabled!==false,command:r,intervalMinutes:u,...r==="publish"||r==="crawl"?{crawlMode:n.crawlMode==="incremental"?"incremental":"full"}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&i?{deploymentProfile:i}:{},...s?{url:s}:{}}}function Q(e){let t=e&&typeof e=="object"?e:{},n=Array.isArray(t.rules)?t.rules.map((r,u)=>ie(r,u)).filter(r=>!!r):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:n}}function oe(e){if(!e||typeof e!="object")return;let t=e,n={};for(let u of ["accountId","siteId"]){let o=t[u];typeof o=="string"&&o.trim()!==""&&(n[u]=o.trim());}let r=Number(t.lastUpdate??0);return Number.isFinite(r)&&r>0&&(n.lastUpdate=Math.floor(r)),t.subscriber===true&&(n.subscriber=true),Object.keys(n).length>0?n:void 0}function D(e){let t=e&&typeof e=="object"?e:{},n=String(t.accountId??"").trim(),r=String(t.siteId??"").trim(),u=t.subscriber===true,o=oe(t.siteSettings),i={...o??{},...n?{accountId:o?.accountId??n}:{},...r?{siteId:o?.siteId??r}:{},...o?.subscriber===true||u?{subscriber:true}:{}},s={},a=String(t.apiBase??"").trim();a&&(s.apiBase=a);let c=String(t.runtimeToken??t.nonce??"").trim();c&&(s.runtimeToken=c);let g=String(t.uploadUrl??"").trim();g&&(s.uploadUrl=g),Object.keys(i).length>0&&(s.siteSettings=i);let p=L(t.subscriptionType);return p&&(s.subscriptionType=p),s}function se(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function ue(e){let t=D(e.wpsuite),n={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},r=String(n.accountId??"").trim(),u=String(n.siteId??"").trim(),o=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...o?{uploadUrl:o}:{},...Object.keys(n).length>0?{siteSettings:n}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!r||!u||!o)return false;se(e.sourceOrigin);let i=globalThis,a={...i.WpSuite??{},siteSettings:n,uploadUrl:o};return t.apiBase&&(a.apiBase=t.apiBase),i.WpSuite=a,true}async function ae(e){let t=D(e.wpsuite),n=t.siteSettings??{},r=String(n.accountId??"").trim()||null,u=String(n.siteId??"").trim()||null,o=String(t.uploadUrl??"").trim();if(!ue(e))return console.info(`[INFO] Skipping remote WPSuite publisher config load: ${JSON.stringify({accountId:r,siteId:u,uploadUrl:o||null,subscriber:n.subscriber===true,lastUpdate:n.lastUpdate??null})}`),null;try{let i=await getConfig("publisher"),s=i&&typeof i=="object"?i:null,a=s?.siteSettings&&typeof s.siteSettings=="object"?s.siteSettings:null;return console.info(`[INFO] Loaded remote WPSuite publisher config: ${JSON.stringify({accountId:r,siteId:u,uploadUrl:o||null,subscriber:n.subscriber===!0,lastUpdate:n.lastUpdate??null,remoteType:i===null?"null":typeof i,remoteSubscriptionType:typeof s?.subscriptionType=="string"?s.subscriptionType:null,remoteKeys:s?Object.keys(s).sort():[],remoteAccountId:typeof s?.accountId=="string"?s.accountId:typeof a?.accountId=="string"?a.accountId:null,remoteSiteId:typeof s?.siteId=="string"?s.siteId:typeof a?.siteId=="string"?a.siteId:null,remoteSiteSettingsKeys:a?Object.keys(a).sort():[]})}`),i&&typeof i=="object"?i:null}catch(i){return console.warn(`[WARN] Remote WPSuite publisher config load failed: ${JSON.stringify({accountId:r,siteId:u,uploadUrl:o||null,subscriber:n.subscriber===true,lastUpdate:n.lastUpdate??null,error:i instanceof Error?i.message:String(i)})}`),null}}function ce(e,t){let n=ne(t?.deploymentProfiles),r=String(t?.defaultDeploymentProfile??"").trim(),u=n[r]?r:"",o=L(t?.subscriptionType),i=D(e.wpsuite);return {...e,scheduler:Q(t?.scheduler),deploymentProfiles:n,defaultDeploymentProfile:u,...o?{subscriptionType:o}:{},wpsuite:{...i,...i.siteSettings||o?{siteSettings:{...i.siteSettings??{},...i.siteSettings?.subscriber===true||o==="PROFESSIONAL"||o==="AGENCY"?{subscriber:true}:{}}}:{},...o?{subscriptionType:o}:{}}}}function k(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let u=r.split("/").map(o=>o.trim()).filter(o=>o.length>0&&o!=="."&&o!=="..");return u.length>0?u.join("/"):t}function le(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?l.resolve(e):""}function W(e,t,n){let r=String(t||"").trim();return r&&l.isAbsolute(r)?l.resolve(r):l.resolve(e,k(r,n))}async function T(e){let t=String(e??"").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await m.readFile(t,"utf8"),r=JSON.parse(n);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=U(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=re(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=J(r.extraReplacements),r.postCrawlCopyMap=J(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.wpsuite=D(r.wpsuite),r.scheduler=Q(void 0),r.deploymentProfiles={},r.defaultDeploymentProfile="",r.deploymentTargetOverride=String(r.deploymentTargetOverride??"").trim();let u=ce(r,await ae(r)),o=le(),i=o?l.resolve(o,".."):"";return i?(u.outputDir=W(i,u.outputDir,"export"),u.logDir=W(i,u.logDir,"logs")):(l.isAbsolute(u.outputDir)||(u.outputDir=k(u.outputDir,"export")),l.isAbsolute(u.logDir)||(u.logDir=k(u.logDir,"logs"))),u}var q=class extends Error{request;step;constructor(t,n){super(`Stop requested during ${n}.`),this.name="StopRequestedError",this.request=t,this.step=n;}},he="queue-mutation.lock",Se=5e3,ye=3e4;function we(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",n=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),r=process.env.PUBLISHER_CONFIG||"",u=1;for(let s=0;s<e.length;s++){let a=e[s];a==="--runtime-dir"?t=e[++s]||"":a.startsWith("--runtime-dir=")?t=a.slice(14):a==="--exporter-dir"?n=e[++s]||n:a.startsWith("--exporter-dir=")?n=a.slice(15):a==="--config"?r=e[++s]||"":a.startsWith("--config=")?r=a.slice(9):a==="--max-jobs"?u=Number.parseInt(e[++s]||"1",10):a.startsWith("--max-jobs=")&&(u=Number.parseInt(a.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let o=l.resolve(t),i=r?l.resolve(r):l.join(o,"config.json");return {runtimeDir:o,exporterDir:l.resolve(n),configPath:i,maxJobs:Number.isFinite(u)&&u>0?u:1}}async function y(e,t){try{let n=await m.readFile(e,"utf8");return JSON.parse(n)}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="ENOENT"){let u=n instanceof Error?n.message:String(n);console.warn(`[queue-runner] failed to read JSON ${e}: ${u}`);}return t}}async function S(e,t){await m.mkdir(l.dirname(e),{recursive:true}),await m.writeFile(e,JSON.stringify(t,null,2),"utf8");}function H(e){return l.join(e,he)}async function Pe(e){await new Promise(t=>setTimeout(t,e));}async function ve(e){let t=H(e),n=Date.now()+Se;for(;;)try{await m.mkdir(l.dirname(t),{recursive:!0}),await m.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="EEXIST")throw r;let o=await m.stat(t).catch(()=>null);if(o&&Date.now()-o.mtimeMs>ye){await m.unlink(t).catch(()=>{});continue}if(Date.now()>=n)throw new Error("Timed out acquiring queue mutation lock.",{cause:r});await Pe(50);}}async function Ce(e){await m.unlink(H(e)).catch(()=>{});}async function x(e,t){await ve(e);try{return await t()}finally{await Ce(e);}}function Re(e){return l.join(e,"queue-runner-heartbeat.json")}function F(e){return l.join(e,"current-progress.json")}function B(e){return l.join(e,"scheduler-state.json")}function Ie(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let u=r.split("/").map(o=>o.trim()).filter(o=>o.length>0&&o!=="."&&o!=="..");return u.length>0?u.join("/"):t}async function K(e){let t=await y(e.configPath||"",null),n=l.resolve(e.runtimeDir,".."),u=(typeof t?.logDir=="string"?t.logDir:"").trim();return u&&l.isAbsolute(u)?l.resolve(u):l.resolve(n,Ie(u,"logs"))}function E(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function Ae(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function qe(e){return [Ae(e.endedAt||e.startedAt),E(e.command||"job","job"),E(e.id||"job","job"),E(e.status||"finished","finished")].join("-")}function De(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function xe(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function Je(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function N(e,t,n){let r=`${n}.gz`,u=l.join(t,r);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(u));let[o,i]=await Promise.all([m.stat(e),m.stat(u)]);return {role:xe(n),originalFileName:n,storedFileName:r,compressed:true,compression:"gzip",contentType:Je(r),originalSize:o.size,storedSize:i.size}}async function z(e,t){let n=await K(t),r=new Date().toISOString(),u=qe(e),o=l.join(n,"archive",u),i=[],s=[];await m.mkdir(o,{recursive:true});try{let g=await m.readdir(n,{withFileTypes:!0});for(let p of g){if(!p.isFile()||!De(p.name))continue;let d=await N(l.join(n,p.name),o,p.name);s.push(d),i.push(d.storedFileName);}}catch(g){if((g&&typeof g=="object"&&"code"in g?String(g.code||""):"")!=="ENOENT")throw g}let a=F(t.runtimeDir);if(existsSync(a)){let g=await N(a,o,"current-progress.json");s.push(g),i.push(g.storedFileName);}let c=[...i,"job.json"];return await S(l.join(o,"job.json"),{manifestVersion:1,archivedAt:r,archiveKey:u,archiveDir:o,logDir:n,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:s,job:e}),{archiveKey:u,archiveDir:o,archiveCreatedAt:r,archivedFiles:c,artifacts:s}}function ke(e){return l.join(e,"audit-events.jsonl")}function j(e){return l.join(e,"stop-request.json")}async function G(e){try{let t=await m.readFile(j(e),"utf8"),n=JSON.parse(t);return n&&typeof n=="object"?n:null}catch{return null}}async function _(e,t){let n=await G(e);if(!n)return null;let r=String(n.targetJobId||"").trim();return r&&r!==t?null:n}async function O(e,t){if(!t){await m.unlink(j(e)).catch(()=>{});return}let n=await G(e);if(!n)return;let r=String(n.targetJobId||"").trim();(!r||r===t)&&await m.unlink(j(e)).catch(()=>{});}async function I(e,t){try{let n={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await m.mkdir(e,{recursive:!0}),await m.appendFile(ke(e),`${JSON.stringify(n)}
|
|
2
|
-
`,"utf8");}catch{}}async function h(e,t,n){let r={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...n??{}};try{await S(Re(e.runtimeDir),r);}catch{}}async function A(e){try{await m.unlink(F(e));}catch{}}function Te(e,t){let n=e.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let r=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(r)||r<1)return null;let u=(e.id||`${n}-${t+1}`).trim();if(!u)return null;let o=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",s=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",a=(n==="publish"||n==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return n==="url"&&!i?null:{id:u,enabled:o,command:n,intervalMinutes:r,...n==="publish"||n==="crawl"?{crawlMode:a}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&s?{deploymentProfile:s}:{},...i?{url:i}:{}}}async function Ee(e){let t=await T(e.configPath).catch(()=>null),n=t?.scheduler,r=Array.isArray(n?.rules)?n.rules.map((u,o)=>Te(u,o)).filter(u=>!!u):[];return {enabled:!!n?.enabled,timezone:typeof n?.timezone=="string"&&n.timezone.trim()?n.timezone.trim():"UTC",rules:r,wpsuite:t?.wpsuite}}function je(e,t){let n=(t.url||"").trim(),r=t.crawlMode||"full",u=(t.deploymentProfile||"").trim();return e.some(o=>o.command===t.command&&(o.url||"").trim()===n&&(o.crawlMode||"full")===r&&(o.deploymentProfile||"").trim()===u&&(o.status===void 0||o.status==="queued"||o.status==="running"))}async function Oe(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let n=await Ee(e);if(!n.enabled||n.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(n.wpsuite??null);}catch(a){let c=a instanceof Error?a.message:String(a);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let r=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json"),o=Date.now(),i=0,s=[];await x(e.runtimeDir,async()=>{let a=await y(r,[]),c=await y(u,null),g=await y(B(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),p=[...a];c&&c.status==="running"&&p.unshift(c);for(let d of n.rules){if(!d.enabled)continue;let f=d.intervalMinutes*60*1e3,b=Math.floor(o/f),C=g.lastEnqueuedBucketByRuleId[d.id]??-1;if(b<=C||je(p,{command:d.command,url:d.url,crawlMode:d.crawlMode,deploymentProfile:d.deploymentProfile}))continue;let R={id:randomUUID(),command:d.command,...(d.command==="publish"||d.command==="crawl")&&d.crawlMode?{crawlMode:d.crawlMode}:{},...(d.command==="publish"||d.command==="deploy"||d.command==="invalidate")&&d.deploymentProfile?{deploymentProfile:d.deploymentProfile}:{},...d.url?{url:d.url}:{},enqueueSource:"scheduler",...n.wpsuite?{wpsuite:n.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};a.push(R),p.push(R),s.push({job:R,rule:d}),g.lastEnqueuedBucketByRuleId[d.id]=b,i+=1;}i>0&&(await S(r,a),await S(B(e.runtimeDir),g));});for(let a of s)await I(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:a.job.id,command:a.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:a.rule.id,intervalMinutes:a.rule.intervalMinutes,timezone:n.timezone,deploymentProfile:a.rule.deploymentProfile||"",url:a.rule.url||""}});return i}async function Fe(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await m.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function Me(e){try{await m.unlink(e);}catch{}}function We(e,t){let n=[l.join(e,"dist",`${t}.js`),l.join(e,`${t}.js`)];for(let r of n)if(existsSync(r))return r;throw new Error(`Cannot find ${t}.js in ${l.join(e,"dist")} or ${e}`)}async function P(e,t,n,r,u,o,i){if(i){let c=await _(t,i);if(c)throw new q(c,n)}let s=We(e,n),a={...process.env,...o??{}};return a.STATIC_PUBLISHER_RUNTIME_DIR=t,u&&(a.PUBLISHER_CONFIG=u),await new Promise((c,g)=>{let p=null,d=null,f=null,b=false,C=spawn(process.execPath,[s,...r],{cwd:e,env:a,stdio:"inherit"}),R=()=>{d&&clearInterval(d),f&&clearTimeout(f);},X=async()=>{if(!(!i||p||b)){b=true;try{let v=await _(t,i);if(!v)return;p=v,C.kill("SIGTERM"),f=setTimeout(()=>{C.kill("SIGKILL");},1e4);}finally{b=false;}}};i&&(d=setInterval(()=>{X();},1e3)),C.on("error",v=>{R(),g(v);}),C.on("close",(v,M)=>{if(R(),p){g(new q(p,n));return}v===0?c():g(new Error(`${n} exited with code ${v??-1}${M?` (signal ${M})`:""}`));});}),0}function Ue(e){let t=e.awsTempCreds;if(!t)return {};let n={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(n.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(n.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(n.AWS_SESSION_TOKEN=t.sessionToken.trim()),n}async function Le(e,t){let n=i=>{if(!i||typeof i!="object")return null;let s=i,a={...s.siteSettings??{},...String(s.accountId??"").trim()&&!s.siteSettings?.accountId?{accountId:String(s.accountId).trim()}:{},...String(s.siteId??"").trim()&&!s.siteSettings?.siteId?{siteId:String(s.siteId).trim()}:{},...s.siteSettings?.subscriber===true||s.subscriber===true?{subscriber:true}:{}},c={...String(s.apiBase??"").trim()?{apiBase:String(s.apiBase).trim()}:{},...String(s.runtimeToken??s.nonce??"").trim()?{runtimeToken:String(s.runtimeToken??s.nonce).trim()}:{},...String(s.uploadUrl??"").trim()?{uploadUrl:String(s.uploadUrl).trim()}:{},...Object.keys(a).length>0?{siteSettings:a}:{},...s.subscriptionType?{subscriptionType:s.subscriptionType}:{}};return Object.keys(c).length>0?c:null},r=await T(t.configPath).catch(()=>null),u=n(r?.wpsuite);if(u)return u;let o=n(e.wpsuite);return o||null}async function Qe(e,t){return await Le(e,t)}function Y(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function Be(e,t){let n=await y(F(e.runtimeDir),null),r=n&&n.details&&typeof n.details.phase=="string"?n.details.phase.trim():"";if(r)return r;let u=n&&typeof n.currentStep=="string"?n.currentStep.trim():"";if(u)return u;let o=await K(e).catch(()=>"");if(o){let i=await y(l.join(o,"current-crawl-event.json"),null),s=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(s)return s}return t}function Ne(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function ze(e){let t=l.join(e.runtimeDir,"queue.json"),n=l.join(e.runtimeDir,"current-run.json"),r=await x(e.runtimeDir,async()=>{let u=await y(n,null);if(!u||u.status!=="running"&&u.status!=="queued")return null;let o=await y(t,[]),i=Array.isArray(o)?o.filter(a=>a?.id!==u.id):[],s=Y(u);return await S(t,[s,...i]),await S(n,null),s});r&&(await O(e.runtimeDir,r.id),await I(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:r.id,command:r.command,message:"Recovered stale current-run entry back into queue."}));}async function _e(e,t,n){try{if((n.shouldEnforceSubscriptionOnJobExecution?.(e)??n.enforceSubscriptionOnJobExecution)&&n.assertActiveSubscriptionForIdentity){let s=await Qe(e,t);await n.assertActiveSubscriptionForIdentity(s);}let u=Ue(e),o=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await P(t.exporterDir,t.runtimeDir,"crawl",o,t.configPath,u,e.id),await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await P(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await P(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="crawl")return await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await P(t.exporterDir,t.runtimeDir,"crawl",o,t.configPath,u,e.id),{exitCode:0};if(e.command==="deploy")return await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await P(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="invalidate")return await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await P(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await P(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,u,e.id),{exitCode:0};if(e.command==="url"){let s=(e.url||"").trim();return s?(await h(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await P(t.exporterDir,t.runtimeDir,"crawl",["--url",s],t.configPath,u,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(r){return r instanceof q?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:r.request,stoppedStep:r.step}:{exitCode:1,error:r instanceof Error?r.message:String(r)}}}function Z(e){return async function(n=process.argv.slice(2)){let r=we(n),u=l.join(r.runtimeDir,"export.lock"),o=l.join(r.runtimeDir,"last-run.json");await A(r.runtimeDir),await h(r,"starting",{message:"queue-runner starting"});try{await Fe(u);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await h(r,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await ze(r);let i=await Oe(r,e),s=0;for(let a=0;a<r.maxJobs;a++){let c=await He(r,e);if(c==="none"||(s+=1,c==="stopped"))break}if(s===0)await A(r.runtimeDir),await h(r,"idle",{processedJobs:s,schedulerEnqueued:i,message:"no queued jobs"});else {await A(r.runtimeDir);let a=await y(o,null),c=String(a?.status||"").trim();await h(r,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:s,schedulerEnqueued:i,lastJobId:a?.id,lastJobCommand:a?.command,lastJobStatus:a?.status,lastJobExitCode:a?.exitCode,lastJobError:a?.error,...c==="stopped"?{currentStep:a?.stoppedStep,stopRequestedAt:a?.stopRequestedAt,stopRequestedByLogin:a?.stopRequestedByLogin,stopRequestedMode:a?.stopMode,lastStoppedStep:a?.stoppedStep,message:a?.stopMode==="requeue"?`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and requeued.`:`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let s=i instanceof Error?i.message:String(i);throw await h(r,"error",{message:s}),await I(r.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:s}}),i}finally{await Me(u);}}}async function $e(e,t,n){let r=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json");await x(e.runtimeDir,async()=>{let o=await y(r,[]),i=Array.isArray(o)?o.filter(c=>c?.id!==t.id):[],s=Y(t),a=Ne(t,n);a?s.resumeFromStep=a:delete s.resumeFromStep,await S(r,[s,...i]),await S(u,null);});}async function He(e,t){let n=l.join(e.runtimeDir,"queue.json"),r=l.join(e.runtimeDir,"current-run.json"),u=l.join(e.runtimeDir,"last-run.json"),o=new Date().toISOString(),i=await x(e.runtimeDir,async()=>{let p=await y(n,[]);if(!Array.isArray(p)||p.length===0)return null;let d={...p[0],status:"running",startedAt:o};return await S(n,p.slice(1)),await S(r,d),d});if(!i)return "none";await A(e.runtimeDir),await I(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:o,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await h(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let s=await _e(i,e,t),a=new Date().toISOString();if(s.stopped){let p=await Be(e,s.stoppedStep||i.command),d=s.stopRequest?.mode==="requeue"?"requeue":"stop",f={...i,status:"stopped",endedAt:a,exitCode:s.exitCode,...s.error?{error:s.error}:{},stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopMode:d,stoppedStep:p};d==="requeue"?await $e(e,i,p):await S(r,null);try{let b=await z(f,e);f.logArchiveDir=b.archiveDir,f.logArchiveCreatedAt=b.archiveCreatedAt,f.logArchiveFileCount=b.archivedFiles.length;}catch(b){f.logArchiveError=b instanceof Error?b.message:String(b);}return await S(u,f),await O(e.runtimeDir,i.id),await A(e.runtimeDir),await I(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:d==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:a,deploymentProfile:i.deploymentProfile||"",stopMode:d,stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stoppedStep:p,logArchiveKey:f.logArchiveError||!f.logArchiveDir?"":l.basename(f.logArchiveDir),logArchiveDir:f.logArchiveDir||"",logArchiveCreatedAt:f.logArchiveCreatedAt||"",logArchiveFileCount:f.logArchiveFileCount??0,logArchiveError:f.logArchiveError||""}}),"stopped"}let c={...i,status:s.exitCode===0?"success":"failed",endedAt:a,exitCode:s.exitCode,...s.error?{error:s.error}:{}};await S(r,null),await O(e.runtimeDir,i.id);try{let p=await z(c,e);c.logArchiveDir=p.archiveDir,c.logArchiveCreatedAt=p.archiveCreatedAt,c.logArchiveFileCount=p.archivedFiles.length;}catch(p){c.logArchiveError=p instanceof Error?p.message:String(p);}await S(u,c);let g=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await I(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:g,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":l.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}async function Ge(e){if(!e)throw new Error("Missing remote WPSuite publisher state. Refresh the runtime config before running subscription-gated jobs.");if(e.subscriptionType!=="PROFESSIONAL"&&e.subscriptionType!=="AGENCY")throw new Error("This job requires an active WPSuite publisher subscription in the remote site configuration.")}var Ye={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0,assertActiveSubscriptionForIdentity:Ge},V=Z(Ye),Ze=V,gt=V;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Ze().catch(e=>{console.error(e),process.exit(1);});export{gt as default,Ze as main,V as runQueueRunner};
|
|
1
|
+
import {pathToFileURL}from'url';import f from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import p from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';import ne from'http';import ie from'https';function U(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function oe(e){let t=String(e??"").trim();if(!t)return "";let n=t;if(/^https?:\/\//i.test(n))try{n=new URL(n).pathname;}catch{return ""}if(n=n.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!n)return "";let r=n.endsWith("/"),o=n.replace(/^\/+/,"").split("/").map(i=>i.replace(/[^A-Za-z0-9._-]/g,"")).filter(i=>i.length>0&&i!=="."&&i!=="..");if(o.length===0)return "";let s=`/${o.join("/")}`;return r&&p.extname(s)===""&&(s+="/"),s==="/"?"":s}function x(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,n])=>[t.trim(),String(n??"")]).filter(([t])=>t.length>0))}function se(e){if(!e||typeof e!="object")return {};let t={};for(let[n,r]of Object.entries(e)){let o=n.trim();if(!o||!r||typeof r!="object")continue;let s=r,i={};if(typeof s.targetOrigin=="string"){let a=U(s.targetOrigin);a&&(i.targetOrigin=a);}let u=x(s.extraReplacements);if(Object.keys(u).length>0&&(i.extraReplacements=u),s.s3&&typeof s.s3=="object"){let a={},c=s.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(i.s3=a);}if(s.cloudFront&&typeof s.cloudFront=="object"){let a={},c=s.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(a).length>0&&(i.cloudFront=a);}t[o]=i;}return t}function L(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function ue(e,t){if(!e||typeof e!="object")return null;let n=e,r=n.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let o=Number.parseInt(String(n.intervalMinutes??"0"),10);if(!Number.isFinite(o)||o<1)return null;let s=String(n.id??`${r}-${t+1}`).trim();if(!s)return null;let i=String(n.deploymentProfile??"").trim(),u=String(n.url??"").trim();return {id:s,enabled:n.enabled!==false,command:r,intervalMinutes:o,...r==="publish"||r==="crawl"?{crawlMode:n.crawlMode==="incremental"?"incremental":"full"}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&i?{deploymentProfile:i}:{},...u?{url:u}:{}}}function B(e){let t=e&&typeof e=="object"?e:{},n=Array.isArray(t.rules)?t.rules.map((r,o)=>ue(r,o)).filter(r=>!!r):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:n}}function ae(e){if(!e||typeof e!="object")return;let t=e,n={};for(let o of ["accountId","siteId"]){let s=t[o];typeof s=="string"&&s.trim()!==""&&(n[o]=s.trim());}let r=Number(t.lastUpdate??0);return Number.isFinite(r)&&r>0&&(n.lastUpdate=Math.floor(r)),t.subscriber===true&&(n.subscriber=true),Object.keys(n).length>0?n:void 0}function T(e){let t=e&&typeof e=="object"?e:{},n=String(t.accountId??"").trim(),r=String(t.siteId??"").trim(),o=t.subscriber===true,s=ae(t.siteSettings),i={...s??{},...n?{accountId:s?.accountId??n}:{},...r?{siteId:s?.siteId??r}:{},...s?.subscriber===true||o?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let c=String(t.runtimeToken??t.nonce??"").trim();c&&(u.runtimeToken=c);let g=String(t.uploadUrl??"").trim();g&&(u.uploadUrl=g),Object.keys(i).length>0&&(u.siteSettings=i);let d=L(t.subscriptionType);return d&&(u.subscriptionType=d),u}function ce(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Q(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function le(e,t){let n=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[r,o]of new Headers(t.headers).entries())n.set(r,o);return n}async function N(e,t,n=5){let r=new URL(Q(e)),o=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),s=r.protocol==="http:"?ne:ie,i=le(e,t);return await new Promise((u,a)=>{let c=s.request(r,{method:o,headers:Object.fromEntries(i.entries()),...r.protocol==="https:"?{rejectUnauthorized:false}:{}},g=>{let d=[];g.on("data",l=>{d.push(Buffer.isBuffer(l)?l:Buffer.from(l));}),g.on("error",a),g.on("end",()=>{let l=g.statusCode??0,m=g.headers.location;if(n>0&&m&&[301,302,303,307,308].includes(l)){u(N(new URL(m,r),{method:l===303?"GET":o},n-1));return}let h=new Headers;for(let[P,y]of Object.entries(g.headers))Array.isArray(y)?h.set(P,y.join(", ")):typeof y=="string"&&h.set(P,y);u(new Response(Buffer.concat(d),{status:l,statusText:g.statusMessage??"",headers:h}));});});c.on("error",a),c.end();})}function de(e){let t=T(e.wpsuite),n={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},r=String(n.accountId??"").trim(),o=String(n.siteId??"").trim(),s=String(t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...s?{uploadUrl:s}:{},...Object.keys(n).length>0?{siteSettings:n}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!r||!o||!s)return false;ce(e.sourceOrigin);let i=globalThis,a={...i.WpSuite??{},siteSettings:n,uploadUrl:s};return t.apiBase&&(a.apiBase=t.apiBase),i.WpSuite=a,true}async function pe(e){if(!de(e))return null;let t=String(e.wpsuite?.uploadUrl??"").trim(),n=globalThis.fetch;try{let r=globalThis,o=()=>({sourceOrigin:e.sourceOrigin,locationHref:r.location?.href??null,locationHostname:r.location?.hostname??null,wpsuite:r.WpSuite?{apiBase:r.WpSuite.apiBase??null,uploadUrl:r.WpSuite.uploadUrl??null,siteSettings:r.WpSuite.siteSettings??null}:null});console.info(`[DEBUG] Remote WPSuite config bootstrap before import: ${JSON.stringify(o())}`),e.ignoreHttpsErrors&&typeof n=="function"&&t.startsWith("https://")&&(globalThis.fetch=(async(u,a)=>Q(u).startsWith(t)?N(u,a):n(u,a)));let{getConfig:s}=await import('@smart-cloud/wpsuite-core');console.info(`[DEBUG] Remote WPSuite config bootstrap before getConfig: ${JSON.stringify(o())}`);let i=await s("publisher");return i&&typeof i=="object"?i:null}catch{return null}finally{typeof n=="function"&&(globalThis.fetch=n);}}function ge(e,t){let n=se(t?.deploymentProfiles),r=String(t?.defaultDeploymentProfile??"").trim(),o=n[r]?r:"",s=L(t?.subscriptionType),i=T(e.wpsuite);return {...e,scheduler:B(t?.scheduler),deploymentProfiles:n,defaultDeploymentProfile:o,...s?{subscriptionType:s}:{},wpsuite:{...i,...i.siteSettings||s?{siteSettings:{...i.siteSettings??{},...i.siteSettings?.subscriber===true||s==="PROFESSIONAL"||s==="AGENCY"?{subscriber:true}:{}}}:{},...s?{subscriptionType:s}:{}}}}function J(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let o=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):t}function me(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?p.resolve(e):""}function W(e,t,n){let r=String(t||"").trim();return r&&p.isAbsolute(r)?p.resolve(r):p.resolve(e,J(r,n))}async function k(e){let t=String(e??"").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await f.readFile(t,"utf8"),r=JSON.parse(n);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=U(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=oe(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=x(r.extraReplacements),r.postCrawlCopyMap=x(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.wpsuite=T(r.wpsuite),r.scheduler=B(void 0),r.deploymentProfiles={},r.defaultDeploymentProfile="",r.deploymentTargetOverride=String(r.deploymentTargetOverride??"").trim();let o=ge(r,await pe(r)),s=me(),i=s?p.resolve(s,".."):"";return i?(o.outputDir=W(i,o.outputDir,"export"),o.logDir=W(i,o.logDir,"logs")):(p.isAbsolute(o.outputDir)||(o.outputDir=J(o.outputDir,"export")),p.isAbsolute(o.logDir)||(o.logDir=J(o.logDir,"logs"))),o}var I=class extends Error{request;step;constructor(t,n){super(`Stop requested during ${n}.`),this.name="StopRequestedError",this.request=t,this.step=n;}},Pe="queue-mutation.lock",ve=5e3,Re=3e4;function Ce(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",n=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),r=process.env.PUBLISHER_CONFIG||"",o=1;for(let u=0;u<e.length;u++){let a=e[u];a==="--runtime-dir"?t=e[++u]||"":a.startsWith("--runtime-dir=")?t=a.slice(14):a==="--exporter-dir"?n=e[++u]||n:a.startsWith("--exporter-dir=")?n=a.slice(15):a==="--config"?r=e[++u]||"":a.startsWith("--config=")?r=a.slice(9):a==="--max-jobs"?o=Number.parseInt(e[++u]||"1",10):a.startsWith("--max-jobs=")&&(o=Number.parseInt(a.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let s=p.resolve(t),i=r?p.resolve(r):p.join(s,"config.json");return {runtimeDir:s,exporterDir:p.resolve(n),configPath:i,maxJobs:Number.isFinite(o)&&o>0?o:1}}async function w(e,t){try{let n=await f.readFile(e,"utf8");return JSON.parse(n)}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="ENOENT"){let o=n instanceof Error?n.message:String(n);console.warn(`[queue-runner] failed to read JSON ${e}: ${o}`);}return t}}async function S(e,t){await f.mkdir(p.dirname(e),{recursive:true}),await f.writeFile(e,JSON.stringify(t,null,2),"utf8");}function K(e){return p.join(e,Pe)}async function Ae(e){await new Promise(t=>setTimeout(t,e));}async function qe(e){let t=K(e),n=Date.now()+ve;for(;;)try{await f.mkdir(p.dirname(t),{recursive:!0}),await f.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="EEXIST")throw r;let s=await f.stat(t).catch(()=>null);if(s&&Date.now()-s.mtimeMs>Re){await f.unlink(t).catch(()=>{});continue}if(Date.now()>=n)throw new Error("Timed out acquiring queue mutation lock.",{cause:r});await Ae(50);}}async function Ie(e){await f.unlink(K(e)).catch(()=>{});}async function D(e,t){await qe(e);try{return await t()}finally{await Ie(e);}}function De(e){return p.join(e,"queue-runner-heartbeat.json")}function F(e){return p.join(e,"current-progress.json")}function z(e){return p.join(e,"scheduler-state.json")}function xe(e,t){let r=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!r)return t;let o=r.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):t}async function Y(e){let t=await w(e.configPath||"",null),n=p.resolve(e.runtimeDir,".."),o=(typeof t?.logDir=="string"?t.logDir:"").trim();return o&&p.isAbsolute(o)?p.resolve(o):p.resolve(n,xe(o,"logs"))}function E(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function Je(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function Te(e){return [Je(e.endedAt||e.startedAt),E(e.command||"job","job"),E(e.id||"job","job"),E(e.status||"finished","finished")].join("-")}function ke(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function Ee(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function je(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function _(e,t,n){let r=`${n}.gz`,o=p.join(t,r);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(o));let[s,i]=await Promise.all([f.stat(e),f.stat(o)]);return {role:Ee(n),originalFileName:n,storedFileName:r,compressed:true,compression:"gzip",contentType:je(r),originalSize:s.size,storedSize:i.size}}async function H(e,t){let n=await Y(t),r=new Date().toISOString(),o=Te(e),s=p.join(n,"archive",o),i=[],u=[];await f.mkdir(s,{recursive:true});try{let g=await f.readdir(n,{withFileTypes:!0});for(let d of g){if(!d.isFile()||!ke(d.name))continue;let l=await _(p.join(n,d.name),s,d.name);u.push(l),i.push(l.storedFileName);}}catch(g){if((g&&typeof g=="object"&&"code"in g?String(g.code||""):"")!=="ENOENT")throw g}let a=F(t.runtimeDir);if(existsSync(a)){let g=await _(a,s,"current-progress.json");u.push(g),i.push(g.storedFileName);}let c=[...i,"job.json"];return await S(p.join(s,"job.json"),{manifestVersion:1,archivedAt:r,archiveKey:o,archiveDir:s,logDir:n,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:u,job:e}),{archiveKey:o,archiveDir:s,archiveCreatedAt:r,archivedFiles:c,artifacts:u}}function Oe(e){return p.join(e,"audit-events.jsonl")}function j(e){return p.join(e,"stop-request.json")}async function Z(e){try{let t=await f.readFile(j(e),"utf8"),n=JSON.parse(t);return n&&typeof n=="object"?n:null}catch{return null}}async function $(e,t){let n=await Z(e);if(!n)return null;let r=String(n.targetJobId||"").trim();return r&&r!==t?null:n}async function O(e,t){if(!t){await f.unlink(j(e)).catch(()=>{});return}let n=await Z(e);if(!n)return;let r=String(n.targetJobId||"").trim();(!r||r===t)&&await f.unlink(j(e)).catch(()=>{});}async function A(e,t){try{let n={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await f.mkdir(e,{recursive:!0}),await f.appendFile(Oe(e),`${JSON.stringify(n)}
|
|
2
|
+
`,"utf8");}catch{}}async function b(e,t,n){let r={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...n??{}};try{await S(De(e.runtimeDir),r);}catch{}}async function q(e){try{await f.unlink(F(e));}catch{}}function Fe(e,t){let n=e.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url")return null;let r=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(r)||r<1)return null;let o=(e.id||`${n}-${t+1}`).trim();if(!o)return null;let s=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",a=(n==="publish"||n==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return n==="url"&&!i?null:{id:o,enabled:s,command:n,intervalMinutes:r,...n==="publish"||n==="crawl"?{crawlMode:a}:{},...(n==="publish"||n==="deploy"||n==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function Me(e){let t=await k(e.configPath).catch(()=>null),n=t?.scheduler,r=Array.isArray(n?.rules)?n.rules.map((o,s)=>Fe(o,s)).filter(o=>!!o):[];return {enabled:!!n?.enabled,timezone:typeof n?.timezone=="string"&&n.timezone.trim()?n.timezone.trim():"UTC",rules:r,wpsuite:t?.wpsuite}}function We(e,t){let n=(t.url||"").trim(),r=t.crawlMode||"full",o=(t.deploymentProfile||"").trim();return e.some(s=>s.command===t.command&&(s.url||"").trim()===n&&(s.crawlMode||"full")===r&&(s.deploymentProfile||"").trim()===o&&(s.status===void 0||s.status==="queued"||s.status==="running"))}async function Ue(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let n=await Me(e);if(!n.enabled||n.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(n.wpsuite??null);}catch(a){let c=a instanceof Error?a.message:String(a);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let r=p.join(e.runtimeDir,"queue.json"),o=p.join(e.runtimeDir,"current-run.json"),s=Date.now(),i=0,u=[];await D(e.runtimeDir,async()=>{let a=await w(r,[]),c=await w(o,null),g=await w(z(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),d=[...a];c&&c.status==="running"&&d.unshift(c);for(let l of n.rules){if(!l.enabled)continue;let m=l.intervalMinutes*60*1e3,h=Math.floor(s/m),P=g.lastEnqueuedBucketByRuleId[l.id]??-1;if(h<=P||We(d,{command:l.command,url:l.url,crawlMode:l.crawlMode,deploymentProfile:l.deploymentProfile}))continue;let y={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...n.wpsuite?{wpsuite:n.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};a.push(y),d.push(y),u.push({job:y,rule:l}),g.lastEnqueuedBucketByRuleId[l.id]=h,i+=1;}i>0&&(await S(r,a),await S(z(e.runtimeDir),g));});for(let a of u)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:a.job.id,command:a.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:a.rule.id,intervalMinutes:a.rule.intervalMinutes,timezone:n.timezone,deploymentProfile:a.rule.deploymentProfile||"",url:a.rule.url||""}});return i}async function Le(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await f.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function Be(e){try{await f.unlink(e);}catch{}}function Qe(e,t){let n=[p.join(e,"dist",`${t}.js`),p.join(e,`${t}.js`)];for(let r of n)if(existsSync(r))return r;throw new Error(`Cannot find ${t}.js in ${p.join(e,"dist")} or ${e}`)}async function R(e,t,n,r,o,s,i){if(i){let c=await $(t,i);if(c)throw new I(c,n)}let u=Qe(e,n),a={...process.env,...s??{}};return a.STATIC_PUBLISHER_RUNTIME_DIR=t,o&&(a.PUBLISHER_CONFIG=o),await new Promise((c,g)=>{let d=null,l=null,m=null,h=false,P=spawn(process.execPath,[u,...r],{cwd:e,env:a,stdio:"inherit"}),y=()=>{l&&clearInterval(l),m&&clearTimeout(m);},te=async()=>{if(!(!i||d||h)){h=true;try{let C=await $(t,i);if(!C)return;d=C,P.kill("SIGTERM"),m=setTimeout(()=>{P.kill("SIGKILL");},1e4);}finally{h=false;}}};i&&(l=setInterval(()=>{te();},1e3)),P.on("error",C=>{y(),g(C);}),P.on("close",(C,M)=>{if(y(),d){g(new I(d,n));return}C===0?c():g(new Error(`${n} exited with code ${C??-1}${M?` (signal ${M})`:""}`));});}),0}function Ne(e){let t=e.awsTempCreds;if(!t)return {};let n={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(n.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(n.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(n.AWS_SESSION_TOKEN=t.sessionToken.trim()),n}async function ze(e,t){let n=i=>{if(!i||typeof i!="object")return null;let u=i,a={...u.siteSettings??{},...String(u.accountId??"").trim()&&!u.siteSettings?.accountId?{accountId:String(u.accountId).trim()}:{},...String(u.siteId??"").trim()&&!u.siteSettings?.siteId?{siteId:String(u.siteId).trim()}:{},...u.siteSettings?.subscriber===true||u.subscriber===true?{subscriber:true}:{}},c={...String(u.apiBase??"").trim()?{apiBase:String(u.apiBase).trim()}:{},...String(u.runtimeToken??u.nonce??"").trim()?{runtimeToken:String(u.runtimeToken??u.nonce).trim()}:{},...String(u.uploadUrl??"").trim()?{uploadUrl:String(u.uploadUrl).trim()}:{},...Object.keys(a).length>0?{siteSettings:a}:{},...u.subscriptionType?{subscriptionType:u.subscriptionType}:{}};return Object.keys(c).length>0?c:null},r=await k(t.configPath).catch(()=>null),o=n(r?.wpsuite);if(o)return o;let s=n(e.wpsuite);return s||null}async function _e(e,t){return await ze(e,t)}function V(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function He(e,t){let n=await w(F(e.runtimeDir),null),r=n&&n.details&&typeof n.details.phase=="string"?n.details.phase.trim():"";if(r)return r;let o=n&&typeof n.currentStep=="string"?n.currentStep.trim():"";if(o)return o;let s=await Y(e).catch(()=>"");if(s){let i=await w(p.join(s,"current-crawl-event.json"),null),u=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(u)return u}return t}function $e(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function Ge(e){let t=p.join(e.runtimeDir,"queue.json"),n=p.join(e.runtimeDir,"current-run.json"),r=await D(e.runtimeDir,async()=>{let o=await w(n,null);if(!o||o.status!=="running"&&o.status!=="queued")return null;let s=await w(t,[]),i=Array.isArray(s)?s.filter(a=>a?.id!==o.id):[],u=V(o);return await S(t,[u,...i]),await S(n,null),u});r&&(await O(e.runtimeDir,r.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:r.id,command:r.command,message:"Recovered stale current-run entry back into queue."}));}async function Ke(e,t,n){try{if((n.shouldEnforceSubscriptionOnJobExecution?.(e)??n.enforceSubscriptionOnJobExecution)&&n.assertActiveSubscriptionForIdentity){let u=await _e(e,t);await n.assertActiveSubscriptionForIdentity(u);}let o=Ne(e),s=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await R(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,o,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await R(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await R(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="crawl")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await R(t.exporterDir,t.runtimeDir,"crawl",s,t.configPath,o,e.id),{exitCode:0};if(e.command==="deploy")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await R(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="invalidate")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await R(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await R(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,o,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await b(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await R(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,o,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(r){return r instanceof I?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:r.request,stoppedStep:r.step}:{exitCode:1,error:r instanceof Error?r.message:String(r)}}}function X(e){return async function(n=process.argv.slice(2)){let r=Ce(n),o=p.join(r.runtimeDir,"export.lock"),s=p.join(r.runtimeDir,"last-run.json");await q(r.runtimeDir),await b(r,"starting",{message:"queue-runner starting"});try{await Le(o);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await b(r,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await Ge(r);let i=await Ue(r,e),u=0;for(let a=0;a<r.maxJobs;a++){let c=await Ze(r,e);if(c==="none"||(u+=1,c==="stopped"))break}if(u===0)await q(r.runtimeDir),await b(r,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await q(r.runtimeDir);let a=await w(s,null),c=String(a?.status||"").trim();await b(r,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:a?.id,lastJobCommand:a?.command,lastJobStatus:a?.status,lastJobExitCode:a?.exitCode,lastJobError:a?.error,...c==="stopped"?{currentStep:a?.stoppedStep,stopRequestedAt:a?.stopRequestedAt,stopRequestedByLogin:a?.stopRequestedByLogin,stopRequestedMode:a?.stopMode,lastStoppedStep:a?.stoppedStep,message:a?.stopMode==="requeue"?`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and requeued.`:`Job stopped during ${a?.stoppedStep||a?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await b(r,"error",{message:u}),await A(r.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await Be(o);}}}async function Ye(e,t,n){let r=p.join(e.runtimeDir,"queue.json"),o=p.join(e.runtimeDir,"current-run.json");await D(e.runtimeDir,async()=>{let s=await w(r,[]),i=Array.isArray(s)?s.filter(c=>c?.id!==t.id):[],u=V(t),a=$e(t,n);a?u.resumeFromStep=a:delete u.resumeFromStep,await S(r,[u,...i]),await S(o,null);});}async function Ze(e,t){let n=p.join(e.runtimeDir,"queue.json"),r=p.join(e.runtimeDir,"current-run.json"),o=p.join(e.runtimeDir,"last-run.json"),s=new Date().toISOString(),i=await D(e.runtimeDir,async()=>{let d=await w(n,[]);if(!Array.isArray(d)||d.length===0)return null;let l={...d[0],status:"running",startedAt:s};return await S(n,d.slice(1)),await S(r,l),l});if(!i)return "none";await q(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:s,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await b(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await Ke(i,e,t),a=new Date().toISOString();if(u.stopped){let d=await He(e,u.stoppedStep||i.command),l=u.stopRequest?.mode==="requeue"?"requeue":"stop",m={...i,status:"stopped",endedAt:a,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:l,stoppedStep:d};l==="requeue"?await Ye(e,i,d):await S(r,null);try{let h=await H(m,e);m.logArchiveDir=h.archiveDir,m.logArchiveCreatedAt=h.archiveCreatedAt,m.logArchiveFileCount=h.archivedFiles.length;}catch(h){m.logArchiveError=h instanceof Error?h.message:String(h);}return await S(o,m),await O(e.runtimeDir,i.id),await q(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:l==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:a,deploymentProfile:i.deploymentProfile||"",stopMode:l,stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:d,logArchiveKey:m.logArchiveError||!m.logArchiveDir?"":p.basename(m.logArchiveDir),logArchiveDir:m.logArchiveDir||"",logArchiveCreatedAt:m.logArchiveCreatedAt||"",logArchiveFileCount:m.logArchiveFileCount??0,logArchiveError:m.logArchiveError||""}}),"stopped"}let c={...i,status:u.exitCode===0?"success":"failed",endedAt:a,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await S(r,null),await O(e.runtimeDir,i.id);try{let d=await H(c,e);c.logArchiveDir=d.archiveDir,c.logArchiveCreatedAt=d.archiveCreatedAt,c.logArchiveFileCount=d.archivedFiles.length;}catch(d){c.logArchiveError=d instanceof Error?d.message:String(d);}await S(o,c);let g=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:g,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":p.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}async function Xe(e){if(!e)throw new Error("Missing remote WPSuite publisher state. Refresh the runtime config before running subscription-gated jobs.");if(e.subscriptionType!=="PROFESSIONAL"&&e.subscriptionType!=="AGENCY")throw new Error("This job requires an active WPSuite publisher subscription in the remote site configuration.")}var et={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0,assertActiveSubscriptionForIdentity:Xe},ee=X(et),tt=ee,St=ee;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&tt().catch(e=>{console.error(e),process.exit(1);});export{St as default,tt as main,ee as runQueueRunner};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.12",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|