@smart-cloud/publisher-exporter 1.1.57 → 1.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/content-sync.js +10 -10
- package/dist/crawl.js +8 -8
- package/dist/deploy.js +4 -4
- package/dist/invalidate.js +3 -3
- package/dist/queue-runner.js +3 -3
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import R from'fs/promises';import y from'path';import {gunzipSync}from'zlib';import Lt,{createHash}from'crypto';import Or from'http';import Dr from'https';import Vr from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Cr=".deploy-plan.json",Rr="deploy-plan.json",kr=".crawl-incomplete.json",Ar="crawl-incomplete.json";function ht(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function mt(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim(),r=y.posix.normalize(t);return !t||!r||r==="."||/^[a-zA-Z]:/.test(r)||r.startsWith("../")||r===".."?null:r}function xr(e){let t=String(e||"").trim();if(!t.startsWith("/")||t.startsWith("//")||t.includes("*")||t.includes("#"))return null;try{let r=new URL(t,"https://content-sync.invalid");return r.origin!=="https://content-sync.invalid"?null:`${y.posix.normalize(r.pathname)}${r.search}`}catch{return null}}function ee(e){return [...new Set(e.map(mt).filter(t=>t!==null))]}function Tr(e,t){let r={generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:y.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ee(t.changedFiles||[]),deletedFiles:ee(t.deletedFiles||[]),rewriteTargets:ee(t.rewriteTargets||[])};return t.schemaVersion===2?{...r,schemaVersion:2,runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,consumerId:String(t.consumerId||"").trim(),fromSequence:Math.max(0,Number(t.fromSequence)||0),toSequence:Math.max(0,Number(t.toSequence)||0),impactHash:String(t.impactHash||"").trim(),resolvedSitemapUrls:[...new Set((t.resolvedSitemapUrls||[]).map(s=>String(s||"").trim()).filter(s=>{try{let n=new URL(s);return n.protocol==="http:"||n.protocol==="https:"}catch{return false}}))],uploadGroups:{assets:ee(t.uploadGroups?.assets||[]),content:ee(t.uploadGroups?.content||[]),listings:ee(t.uploadGroups?.listings||[]),sitemapLeaves:ee(t.uploadGroups?.sitemapLeaves||[]),sitemapIndexes:ee(t.uploadGroups?.sitemapIndexes||[])},invalidationPaths:[...new Set((t.invalidationPaths||[]).map(xr).filter(s=>s!==null))].sort()}:{schemaVersion:1,...r,runMode:t.runMode}}function wt(e){let t=y.join(y.resolve(e.outputDir),Cr),r=ht();return r?[y.join(r,Rr),t]:[t]}function yt(e){let t=y.join(y.resolve(e.outputDir),kr),r=ht();return r?[y.join(r,Ar),t]:[t]}function re(e,t){let r=y.resolve(e),s=y.resolve(t),n=y.relative(r,s).replace(/\\/g,"/");return mt(n)}async function We(e,t){let r=Tr(e,t);for(let s of wt(e)){await R.mkdir(y.dirname(s),{recursive:true});let n=`${s}.${process.pid}.${Date.now()}.tmp`;await R.writeFile(n,JSON.stringify(r,null,2),"utf8"),await R.rename(n,s);}}async function St(e){for(let t of wt(e))try{await R.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}async function Pt(e,t){let r={schemaVersion:1,startedAt:String(t.startedAt||new Date().toISOString()),outputDir:y.resolve(e.outputDir)};for(let s of yt(e)){await R.mkdir(y.dirname(s),{recursive:true});let n=`${s}.${process.pid}.tmp`;await R.writeFile(n,JSON.stringify(r,null,2),"utf8"),await R.rename(n,s);}}async function bt(e){for(let t of yt(e))try{await R.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function Mr(e){return new URL(e).pathname.split("/").filter(Boolean).some(r=>{try{return decodeURIComponent(r).endsWith("__trashed")}catch{return r.endsWith("__trashed")}})}function _e(e){return Array.isArray(e)?e.map(_e):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[r])=>t.localeCompare(r)).map(([t,r])=>[t,_e(r)])):e}function $r(e){return JSON.stringify(_e(e))}function Lr(e){return createHash("sha256").update($r(e)).digest("hex")}function vt(e){let{impactHash:t,...r}=e;if(e.schemaVersion!==1||!t)throw new Error("Invalid content-sync impact plan schema.");if(Lr(r)!==t)throw new Error("Content-sync impact plan hash mismatch.");for(let s of [...e.directRenderUrls,...e.deleteUrls])if(Mr(s))throw new Error(`Content-sync impact contains a WordPress trash alias instead of the last public URL: ${s}`)}var Qe=["/sitemap_index.xml","/wp-sitemap.xml","/sitemap.xml"];function Ct(e){let t=new Set;for(let r of [...e,...Qe]){let s=String(r||"").trim();s&&t.add(s);}return [...t]}function He(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function Rt(e){if(!e||!/^https?:\/\//i.test(e))return false;try{return He(new URL(e))}catch{return false}}function At(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Er(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&y.extname(i)===""&&(i+="/"),i==="/"?"":i}function ze(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Fr(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,o={};if(typeof i.targetOrigin=="string"){let a=At(i.targetOrigin);a&&(o.targetOrigin=a);}let u=ze(i.extraReplacements);if(Object.keys(u).length>0&&(o.extraReplacements=u),i.s3&&typeof i.s3=="object"){let a={},l=i.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},l=i.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[n]=o;}return t}function Ve(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function jr(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url"&&s!=="content-sync")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let o=String(r.deploymentProfile??"").trim(),u=String(r.url??"").trim(),a=Array.isArray(r.postTypes)?[...new Set(r.postTypes.map(d=>String(d??"").trim().toLowerCase().replace(/[^a-z0-9_-]/g,"")).filter(Boolean))]:[],l=Array.isArray(r.listingPaths)?[...new Set(r.listingPaths.map(d=>String(d??"").trim()).filter(Boolean))]:[];return s==="content-sync"&&a.length===0?null:{id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate"||s==="content-sync")&&o?{deploymentProfile:o}:{},...u?{url:u}:{},...a.length>0?{postTypes:a}:{},...l.length>0?{listingPaths:l}:{},...typeof r.includeSubsites=="boolean"?{includeSubsites:r.includeSubsites}:{},...typeof r.includePostTypeArchives=="boolean"?{includePostTypeArchives:r.includePostTypeArchives}:{},...typeof r.includeTaxonomyArchives=="boolean"?{includeTaxonomyArchives:r.includeTaxonomyArchives}:{},...typeof r.includeAuthorArchives=="boolean"?{includeAuthorArchives:r.includeAuthorArchives}:{},...typeof r.includeDateArchives=="boolean"?{includeDateArchives:r.includeDateArchives}:{},...typeof r.includePostsPage=="boolean"?{includePostsPage:r.includePostsPage}:{},...typeof r.includeSitemapChain=="boolean"?{includeSitemapChain:r.includeSitemapChain}:{}}}function xt(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>jr(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function qr(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function Ge(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=qr(t.siteSettings),o={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let l=String(t.runtimeToken??t.nonce??"").trim();l&&(u.runtimeToken=l);let d=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();d&&(u.virtualAssetBaseUrl=d),Object.keys(o).length>0&&(u.siteSettings=o);let c=Ve(t.subscriptionType);return c&&(u.subscriptionType=c),u}function Nr(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Tt(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Br(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function It(e,t,r=5){let s=new URL(Tt(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Or:Dr,o=Br(e,t);return await new Promise((u,a)=>{let l=i.request(s,{method:n,headers:Object.fromEntries(o.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let c=[];d.on("data",p=>{c.push(Buffer.isBuffer(p)?p:Buffer.from(p));}),d.on("error",a),d.on("end",()=>{let p=d.statusCode??0,w=d.headers.location;if(r>0&&w&&[301,302,303,307,308].includes(p)){u(It(new URL(w,s),{method:p===303?"GET":n},r-1));return}let m=new Headers;for(let[b,h]of Object.entries(d.headers))Array.isArray(h)?m.set(b,h.join(", ")):typeof h=="string"&&m.set(b,h);u(new Response(Buffer.concat(c),{status:p,statusText:d.statusMessage??"",headers:m}));});});l.on("error",a),l.end();})}function Wr(e){let t=Ge(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{virtualAssetBaseUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Nr(e.sourceOrigin);let o=globalThis,a={...o.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),o.WpSuite=a,true}async function _r(e){let t=String(e.wpsuite?.virtualAssetBaseUrl??e.wpsuite?.uploadUrl??"").trim();if(!Wr(e)){let n=e.wpsuite?.siteSettings,i=[String(n?.accountId??"").trim()?"":"accountId",String(n?.siteId??"").trim()?"":"siteId",t?"":"virtualAssetBaseUrl"].filter(Boolean);return {remote:null,diagnostic:{status:"invalid-runtime",virtualAssetBaseUrl:t,subscriptionType:null,error:`Missing WP Suite runtime field(s): ${i.join(", ")}.`,requests:[]}}}let r=globalThis.fetch,s=[];try{typeof r=="function"&&/^https?:\/\//i.test(t)&&(globalThis.fetch=(async(a,l)=>{let d=Tt(a);if(d.startsWith(t)){let c=(()=>{try{return new URL(d).pathname}catch{return d}})();try{let p=e.ignoreHttpsErrors&&d.startsWith("https://")?await It(a,l):await r(a,l);return s.push({path:c,status:p.status,ok:p.ok,error:null}),p}catch(p){throw s.push({path:c,status:null,ok:!1,error:p instanceof Error?p.message:String(p)}),p}}return r(a,l)}));let{getConfig:n}=await import('@smart-cloud/wpsuite-core'),i=await n("publisher"),o=i&&typeof i=="object"?i:null,u=Ve(o?.subscriptionType);return {remote:o,diagnostic:{status:o?"loaded":"unavailable",virtualAssetBaseUrl:t,subscriptionType:u??null,error:o?null:"WP Suite publisher config loader returned no configuration.",requests:s}}}catch(n){return {remote:null,diagnostic:{status:"unavailable",virtualAssetBaseUrl:t,subscriptionType:null,error:n instanceof Error?n.message:String(n),requests:s}}}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Qr(e,t){let r=Fr(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=Ve(t?.subscriptionType),o=Ge(e.wpsuite);return {...e,scheduler:xt(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...o,...o.siteSettings||i?{siteSettings:{...o.siteSettings??{},...o.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function Je(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Hr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function kt(e,t,r){let s=String(t||"").trim();return s&&y.isAbsolute(s)?y.resolve(s):y.resolve(e,Je(s,r))}async function Te(e,t={}){let r=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",s=await R.readFile(r,"utf8"),n=JSON.parse(s);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=At(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=Er(n.generated404RequestPath),n.sitemapPaths||=[...Qe],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=ze(n.extraReplacements),n.postCrawlCopyMap=ze(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=Ge(n.wpsuite),n.scheduler=xt(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let i=await _r(n);t.onRemotePublisherConfigDiagnostic?.(i.diagnostic);let o=Qr(n,i.remote),u=Hr(),a=u?y.resolve(u,".."):"";return a?(o.outputDir=kt(a,o.outputDir,"export"),o.logDir=kt(a,o.logDir,"logs")):(y.isAbsolute(o.outputDir)||(o.outputDir=Je(o.outputDir,"export")),y.isAbsolute(o.logDir)||(o.logDir=Je(o.logDir,"logs"))),o}function ye(e,t=10){return Lt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function zr(e){await R.mkdir(y.dirname(e),{recursive:true});}async function Se(e,t,r){await zr(e);let s=y.join(y.dirname(e),`.${y.basename(e)}.${process.pid}.${Date.now()}.${Lt.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await R.writeFile(s,t,r??"utf8"):await R.writeFile(s,t),await R.rename(s,e);}catch(n){throw await R.unlink(s).catch(()=>{}),n}}function ne(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(y.extname(e).toLowerCase())}function K(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function le(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function Ie(e){return e.replace(/\//g,"\\/")}function Mt(e){return e.replace(/\//g,"\\\\/")}function Jr(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function $t(e,t,r){if(!t)return;e[t]=r;let s=Ie(t),n=Ie(r);e[s]=n;let i=Mt(t),o=Mt(r);e[i]=o;let u=le(t),a=le(r);e[u]=a;let l=le(s),d=le(n);e[l]=d;let c=le(i),p=le(o);e[c]=p;}function Me(e,t,r){$t(e,t,r);let s=Jr(t);s&&s!==t&&$t(e,s,r);}function Ke(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Ut(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ot(e,t,r){if(!t)return r;let s=y.dirname(y.resolve(t)),n=y.resolve(e,r.replace(/^\/+/,"")),i=y.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Yr=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Xr=new Set([".html",".htm"]),Dt="WPSuite.io Static Publisher",es=Dt.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function ts(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function rs(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function ss(e,t,r){return rs(t)?e.replace(new RegExp(`(?<!:)${ts(t)}`,"g"),r):e.split(t).join(r)}function ns(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=y.extname(r).toLowerCase();return Xr.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function is(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${es}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Dt}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import R from'fs/promises';import w from'path';import {gunzipSync}from'zlib';import Xr from'fast-glob';import Ot,{createHash}from'crypto';import qr from'http';import Nr from'https';import {availableParallelism}from'os';import {Worker}from'worker_threads';var Tr=".deploy-plan.json",Ir="deploy-plan.json",Mr=".crawl-incomplete.json",$r="crawl-incomplete.json";function wt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function yt(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim(),r=w.posix.normalize(t);return !t||!r||r==="."||/^[a-zA-Z]:/.test(r)||r.startsWith("../")||r===".."?null:r}function Ur(e){let t=String(e||"").trim();if(!t.startsWith("/")||t.startsWith("//")||t.includes("*")||t.includes("#"))return null;try{let r=new URL(t,"https://content-sync.invalid");return r.origin!=="https://content-sync.invalid"?null:`${w.posix.normalize(r.pathname)}${r.search}`}catch{return null}}function G(e){return [...new Set(e.map(yt).filter(t=>t!==null))]}function Lr(e,t){let r={generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:w.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:G(t.changedFiles||[]),deletedFiles:G(t.deletedFiles||[]),rewriteTargets:G(t.rewriteTargets||[])};return t.schemaVersion===2?{...r,schemaVersion:2,runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,consumerId:String(t.consumerId||"").trim(),fromSequence:Math.max(0,Number(t.fromSequence)||0),toSequence:Math.max(0,Number(t.toSequence)||0),impactHash:String(t.impactHash||"").trim(),resolvedSitemapUrls:[...new Set((t.resolvedSitemapUrls||[]).map(s=>String(s||"").trim()).filter(s=>{try{let n=new URL(s);return n.protocol==="http:"||n.protocol==="https:"}catch{return false}}))],uploadGroups:{assets:G(t.uploadGroups?.assets||[]),content:G(t.uploadGroups?.content||[]),listings:G(t.uploadGroups?.listings||[]),resources:G(t.uploadGroups?.resources||[]),sitemapLeaves:G(t.uploadGroups?.sitemapLeaves||[]),sitemapIndexes:G(t.uploadGroups?.sitemapIndexes||[])},invalidationPaths:[...new Set((t.invalidationPaths||[]).map(Ur).filter(s=>s!==null))].sort()}:{schemaVersion:1,...r,runMode:t.runMode}}function St(e){let t=w.join(w.resolve(e.outputDir),Tr),r=wt();return r?[w.join(r,Ir),t]:[t]}function Pt(e){let t=w.join(w.resolve(e.outputDir),Mr),r=wt();return r?[w.join(r,$r),t]:[t]}function K(e,t){let r=w.resolve(e),s=w.resolve(t),n=w.relative(r,s).replace(/\\/g,"/");return yt(n)}async function Qe(e,t){let r=Lr(e,t);for(let s of St(e)){await R.mkdir(w.dirname(s),{recursive:true});let n=`${s}.${process.pid}.${Date.now()}.tmp`;await R.writeFile(n,JSON.stringify(r,null,2),"utf8"),await R.rename(n,s);}}async function bt(e){for(let t of St(e))try{await R.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}async function vt(e,t){let r={schemaVersion:1,startedAt:String(t.startedAt||new Date().toISOString()),outputDir:w.resolve(e.outputDir)};for(let s of Pt(e)){await R.mkdir(w.dirname(s),{recursive:true});let n=`${s}.${process.pid}.tmp`;await R.writeFile(n,JSON.stringify(r,null,2),"utf8"),await R.rename(n,s);}}async function Ct(e){for(let t of Pt(e))try{await R.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function Dr(e){return new URL(e).pathname.split("/").filter(Boolean).some(r=>{try{return decodeURIComponent(r).endsWith("__trashed")}catch{return r.endsWith("__trashed")}})}function He(e){return Array.isArray(e)?e.map(He):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[r])=>t.localeCompare(r)).map(([t,r])=>[t,He(r)])):e}function Er(e){return JSON.stringify(He(e))}function Fr(e){return createHash("sha256").update(Er(e)).digest("hex")}function Rt(e){let{impactHash:t,...r}=e;if(e.schemaVersion!==1||!t)throw new Error("Invalid content-sync impact plan schema.");if(!Array.isArray(e.resourceRenderUrls)||!Array.isArray(e.resourceDeleteUrls))throw new Error("Invalid content-sync resource impact contract.");if(Fr(r)!==t)throw new Error("Content-sync impact plan hash mismatch.");for(let s of [...e.directRenderUrls,...e.deleteUrls,...e.resourceRenderUrls,...e.resourceDeleteUrls])if(Dr(s))throw new Error(`Content-sync impact contains a WordPress trash alias instead of the last public URL: ${s}`)}var ze=["/sitemap_index.xml","/wp-sitemap.xml","/sitemap.xml"];function kt(e){let t=new Set;for(let r of [...e,...ze]){let s=String(r||"").trim();s&&t.add(s);}return [...t]}function Je(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function At(e){if(!e||!/^https?:\/\//i.test(e))return false;try{return Je(new URL(e))}catch{return false}}function Tt(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function Br(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(a=>a.replace(/[^A-Za-z0-9._-]/g,"")).filter(a=>a.length>0&&a!=="."&&a!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&w.extname(i)===""&&(i+="/"),i==="/"?"":i}function Ve(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Wr(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,a={};if(typeof i.targetOrigin=="string"){let o=Tt(i.targetOrigin);o&&(a.targetOrigin=o);}let u=Ve(i.extraReplacements);if(Object.keys(u).length>0&&(a.extraReplacements=u),i.s3&&typeof i.s3=="object"){let o={},c=i.s3;typeof c.bucket=="string"&&(o.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(o.prefix=c.prefix.trim()),typeof c.region=="string"&&(o.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(o.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(o.assetCacheControl=c.assetCacheControl.trim()),Object.keys(o).length>0&&(a.s3=o);}if(i.cloudFront&&typeof i.cloudFront=="object"){let o={},c=i.cloudFront;typeof c.distributionId=="string"&&(o.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(o.invalidationPaths=c.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(o).length>0&&(a.cloudFront=o);}t[n]=a;}return t}function Ke(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function _r(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url"&&s!=="content-sync")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let a=String(r.deploymentProfile??"").trim(),u=String(r.url??"").trim(),o=Array.isArray(r.postTypes)?[...new Set(r.postTypes.map(d=>String(d??"").trim().toLowerCase().replace(/[^a-z0-9_-]/g,"")).filter(Boolean))]:[],c=Array.isArray(r.listingPaths)?[...new Set(r.listingPaths.map(d=>String(d??"").trim()).filter(Boolean))]:[];return s==="content-sync"&&o.length===0?null:{id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate"||s==="content-sync")&&a?{deploymentProfile:a}:{},...u?{url:u}:{},...o.length>0?{postTypes:o}:{},...c.length>0?{listingPaths:c}:{},...typeof r.includeSubsites=="boolean"?{includeSubsites:r.includeSubsites}:{},...typeof r.includePostTypeArchives=="boolean"?{includePostTypeArchives:r.includePostTypeArchives}:{},...typeof r.includeTaxonomyArchives=="boolean"?{includeTaxonomyArchives:r.includeTaxonomyArchives}:{},...typeof r.includeAuthorArchives=="boolean"?{includeAuthorArchives:r.includeAuthorArchives}:{},...typeof r.includeDateArchives=="boolean"?{includeDateArchives:r.includeDateArchives}:{},...typeof r.includePostsPage=="boolean"?{includePostsPage:r.includePostsPage}:{},...typeof r.includeSitemapChain=="boolean"?{includeSitemapChain:r.includeSitemapChain}:{}}}function It(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>_r(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Qr(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function Ze(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Qr(t.siteSettings),a={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},u={},o=String(t.apiBase??"").trim();o&&(u.apiBase=o);let c=String(t.runtimeToken??t.nonce??"").trim();c&&(u.runtimeToken=c);let d=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();d&&(u.virtualAssetBaseUrl=d),Object.keys(a).length>0&&(u.siteSettings=a);let l=Ke(t.subscriptionType);return l&&(u.subscriptionType=l),u}function Hr(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Mt(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function zr(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function $t(e,t,r=5){let s=new URL(Mt(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?qr:Nr,a=zr(e,t);return await new Promise((u,o)=>{let c=i.request(s,{method:n,headers:Object.fromEntries(a.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let l=[];d.on("data",p=>{l.push(Buffer.isBuffer(p)?p:Buffer.from(p));}),d.on("error",o),d.on("end",()=>{let p=d.statusCode??0,y=d.headers.location;if(r>0&&y&&[301,302,303,307,308].includes(p)){u($t(new URL(y,s),{method:p===303?"GET":n},r-1));return}let m=new Headers;for(let[b,f]of Object.entries(d.headers))Array.isArray(f)?m.set(b,f.join(", ")):typeof f=="string"&&m.set(b,f);u(new Response(Buffer.concat(l),{status:p,statusText:d.statusMessage??"",headers:m}));});});c.on("error",o),c.end();})}function Jr(e){let t=Ze(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{virtualAssetBaseUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Hr(e.sourceOrigin);let a=globalThis,o={...a.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(o.apiBase=t.apiBase),a.WpSuite=o,true}async function Vr(e){let t=String(e.wpsuite?.virtualAssetBaseUrl??e.wpsuite?.uploadUrl??"").trim();if(!Jr(e)){let n=e.wpsuite?.siteSettings,i=[String(n?.accountId??"").trim()?"":"accountId",String(n?.siteId??"").trim()?"":"siteId",t?"":"virtualAssetBaseUrl"].filter(Boolean);return {remote:null,diagnostic:{status:"invalid-runtime",virtualAssetBaseUrl:t,subscriptionType:null,error:`Missing WP Suite runtime field(s): ${i.join(", ")}.`,requests:[]}}}let r=globalThis.fetch,s=[];try{typeof r=="function"&&/^https?:\/\//i.test(t)&&(globalThis.fetch=(async(o,c)=>{let d=Mt(o);if(d.startsWith(t)){let l=(()=>{try{return new URL(d).pathname}catch{return d}})();try{let p=e.ignoreHttpsErrors&&d.startsWith("https://")?await $t(o,c):await r(o,c);return s.push({path:l,status:p.status,ok:p.ok,error:null}),p}catch(p){throw s.push({path:l,status:null,ok:!1,error:p instanceof Error?p.message:String(p)}),p}}return r(o,c)}));let{getConfig:n}=await import('@smart-cloud/wpsuite-core'),i=await n("publisher"),a=i&&typeof i=="object"?i:null,u=Ke(a?.subscriptionType);return {remote:a,diagnostic:{status:a?"loaded":"unavailable",virtualAssetBaseUrl:t,subscriptionType:u??null,error:a?null:"WP Suite publisher config loader returned no configuration.",requests:s}}}catch(n){return {remote:null,diagnostic:{status:"unavailable",virtualAssetBaseUrl:t,subscriptionType:null,error:n instanceof Error?n.message:String(n),requests:s}}}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Gr(e,t){let r=Wr(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=Ke(t?.subscriptionType),a=Ze(e.wpsuite);return {...e,scheduler:It(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...a,...a.siteSettings||i?{siteSettings:{...a.siteSettings??{},...a.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function Ge(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Kr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function xt(e,t,r){let s=String(t||"").trim();return s&&w.isAbsolute(s)?w.resolve(s):w.resolve(e,Ge(s,r))}async function Me(e,t={}){let r=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",s=await R.readFile(r,"utf8"),n=JSON.parse(s);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=Tt(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=Br(n.generated404RequestPath),n.sitemapPaths||=[...ze],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=Ve(n.extraReplacements),n.postCrawlCopyMap=Ve(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=Ze(n.wpsuite),n.scheduler=It(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let i=await Vr(n);t.onRemotePublisherConfigDiagnostic?.(i.diagnostic);let a=Gr(n,i.remote),u=Kr(),o=u?w.resolve(u,".."):"";return o?(a.outputDir=xt(o,a.outputDir,"export"),a.logDir=xt(o,a.logDir,"logs")):(w.isAbsolute(a.outputDir)||(a.outputDir=Ge(a.outputDir,"export")),w.isAbsolute(a.logDir)||(a.logDir=Ge(a.logDir,"logs"))),a}function Se(e,t=10){return Ot.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Zr(e){await R.mkdir(w.dirname(e),{recursive:true});}async function Pe(e,t,r){await Zr(e);let s=w.join(w.dirname(e),`.${w.basename(e)}.${process.pid}.${Date.now()}.${Ot.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await R.writeFile(s,t,r??"utf8"):await R.writeFile(s,t),await R.rename(s,e);}catch(n){throw await R.unlink(s).catch(()=>{}),n}}function ne(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(w.extname(e).toLowerCase())}function Y(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function le(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function $e(e){return e.replace(/\//g,"\\/")}function Ut(e){return e.replace(/\//g,"\\\\/")}function Yr(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Lt(e,t,r){if(!t)return;e[t]=r;let s=$e(t),n=$e(r);e[s]=n;let i=Ut(t),a=Ut(r);e[i]=a;let u=le(t),o=le(r);e[u]=o;let c=le(s),d=le(n);e[c]=d;let l=le(i),p=le(a);e[l]=p;}function Ue(e,t,r){Lt(e,t,r);let s=Yr(t);s&&s!==t&&Lt(e,s,r);}function Ye(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Dt(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Et(e,t,r){if(!t)return r;let s=w.dirname(w.resolve(t)),n=w.resolve(e,r.replace(/^\/+/,"")),i=w.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var ss=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],ns=new Set([".html",".htm"]),Ft="WPSuite.io Static Publisher",is=Ft.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function os(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function as(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function us(e,t,r){return as(t)?e.replace(new RegExp(`(?<!:)${os(t)}`,"g"),r):e.split(t).join(r)}function cs(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=w.extname(r).toLowerCase();return ns.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function ls(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${is}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ft}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function ds(e){let t=Y(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function ps(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,$e(i)]).filter(([i,a])=>i!==a).sort((i,a)=>a[0].length-i[0].length);if(!r.length)return e;let s=i=>{let a=i;for(let[u,o]of r)a=a.split(u).join(o);return a},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,a,u,o)=>`${a}${s(u)}${o}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,a,u,o,c)=>ds(o)?`${a}${s(o)}${c}`:i),n}function gs(e,t){if(!t)return null;let r=w.dirname(w.resolve(t)),s=w.resolve(e.outputDir),n=w.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function fs(e,t,r){let s=gs(t,r);if(!s)return e;let n=e;for(let i of ss){let a=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${a}`,"g"),o=new RegExp(`(?<!\\.)/${a}`,"g");n=n.replace(u,`${s}${i}`).replace(o,`${s}${i}`);let c=i.replace(/\//g,"\\/"),d=s.replace(/\//g,"\\/"),l=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),p=new RegExp(`(?:\\.{1,}\\\\/)+${l}`,"g"),y=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${l}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${l}`,"g");n=n.replace(p,`${d}${c}`).replace(y,`${d}${c}`).replace(m,`${d}${c}`);}return n}function et(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:Dt(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Et(e.outputDir,r,n)}${i}`}function Xe(e,t,r,s,n){Ue(e,r,et(t,s,n));}function hs(e,t,r,s,n){if(!r||r===s)return;Ue(e,r,s);let i=et(t,r,n),a=et(t,s,n);Ue(e,i,a);}function tt(e,t,r,s,n={}){let i=e,a={};Xe(a,t,t.sourceOrigin,t.targetOrigin,s);for(let[o,c]of Object.entries(t.extraReplacements))Xe(a,t,o,c,s);for(let[o,c]of Object.entries(r))Xe(a,t,o,c,s);for(let[o,c]of Object.entries(n)){let d=r[o];!d||d===c||hs(a,t,c,d,s);}let u=Object.entries(a).sort((o,c)=>c[0].length-o[0].length);for(let[o,c]of u)i=us(i,o,c);return t.urlRewriteMode!=="absolute"&&(i=fs(i,t,s)),i=ps(i,u.map(([,o])=>o)),cs(t,i,s)&&(i=ls(i)),i}async function ms(e,t,r,s){let n=w.join(e.outputDir,s),i=w.extname(n).toLowerCase(),a;try{a=await R.readFile(n,"utf8");}catch(c){if(c.code!=="ENOENT")throw c;return {changed:false}}let o=tt(a,e,t,i===".js"||i===".mjs"?void 0:n,r);return o!==a?(await Pe(n,o,"utf8"),{changed:true}):{changed:false}}async function ws(e,t){return await new Promise((r,s)=>{let n=o=>{if(u(),o?.error){s(new Error(o.error));return}r({changed:!!o?.changed});},i=o=>{u(),s(o);},a=o=>{u(),o!==0&&s(new Error(`Rewrite worker exited with code ${o}.`));},u=()=>{e.off("message",n),e.off("error",i),e.off("exit",a);};e.on("message",n),e.on("error",i),e.on("exit",a),e.postMessage({file:t});})}async function jt(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(p=>w.resolve(p)).filter(p=>p.startsWith(w.resolve(e.outputDir))).map(p=>w.relative(e.outputDir,p).replace(/\\/g,"/")).filter(p=>p.length>0):await Xr(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(p=>p!=="asset-map.json"&&ne(p)),i=0;if(n.length===0)return 0;let a=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),u=Math.max(1,Math.min(a,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:u});let o=0,c=0,d=null;if(u===1){for(let p of n){let y=await ms(e,t,r.previousAssetMap??{},p);y.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:p,changed:y.changed});}return i}let l=Array.from({length:u},async()=>{let p=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!d;){let y=o;if(y>=n.length)return;o+=1;let m=n[y],b=await ws(p,m);b.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:m,changed:b.changed});}}catch(y){throw d=y instanceof Error?y:new Error(String(y)),d}finally{await p.terminate().catch(()=>{});}});return await Promise.all(l),i}var ie={error:0,warn:1,info:2,debug:3},ys=/^(?:authorization|proxyAuthorization|runtimeToken|siteKey|nonce|accessToken|refreshToken|accessKeyId|secretAccessKey|sessionToken|xStaticPublisherToken|xApiKey)$/i;function X(e){return e.replace(/((?:x-static-publisher-token|authorization|proxy-authorization|x-api-key|aws-access-key-id|aws-secret-access-key|aws-session-token)\s*:\s*)[^\r\n]+/gi,"$1[REDACTED]").replace(/\b(Basic|Bearer)\s+[^\s]+/gi,"$1 [REDACTED]").replace(/([?&](?:token|access_token|refresh_token|api_key|key|nonce)=)[^&\s]+/gi,"$1[REDACTED]").replace(/((?:"|')?(?:runtimeToken|siteKey|nonce|accessToken|refreshToken|accessKeyId|secretAccessKey|sessionToken)(?:"|')?\s*[:=]\s*(?:"|'))[^"']+/gi,"$1[REDACTED]")}function Oe(e){return typeof e=="string"?X(e):Array.isArray(e)?e.map(t=>Oe(t)):e instanceof Error?{name:e.name,message:X(e.message),...e.stack?{stack:X(e.stack)}:{}}:e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([t,r])=>[t,ys.test(t)?"[REDACTED]":Oe(r)])):e}function rt(e){return e?Oe(e):void 0}var qt=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",Le=qt?w.join(qt,"current-progress.json"):"",Nt=Promise.resolve();function Bt(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function Ss(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let a=r[n];if(typeof a=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(a,i);continue}r[n]=i;}return r}function Ps(e,t,r){Le&&(Nt=Nt.then(async()=>{let s=new Date,n=s.toISOString();await R.mkdir(w.dirname(Le),{recursive:true});let i=await R.readFile(Le,"utf8").then(g=>JSON.parse(g)).catch(()=>null),a=i&&typeof i.details=="object"&&i.details?i.details:{},u=Bt(e),o=i?.currentStep||Bt(i?.source||""),c=i?.startedAt||n,d={...i?.stepDurationsSec??{}},l=i?.stepStartedAt||n;if(o&&o!==u&&i?.stepStartedAt){let g=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));d[o]=(d[o]??0)+g,l=n;}let p=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),y=Math.max(0,Math.round((s.getTime()-new Date(c).getTime())/1e3)),m=X(t),b=rt(r)??{},f={checkedAt:n,source:e,message:m,details:Ss(a,b),startedAt:c,currentStep:u,stepStartedAt:l,stepElapsedSec:p,totalElapsedSec:y,stepDurationsSec:d};await R.writeFile(Le,JSON.stringify(f,null,2),"utf8");}).catch(()=>{}));}async function bs(e,t){await R.mkdir(w.dirname(e),{recursive:true}),await R.appendFile(e,`${JSON.stringify(Oe(t))}
|
|
5
|
+
`,"utf8");}function vs(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function Cs(e,t,r){let s=rt(r);return {checkedAt:new Date().toISOString(),currentStep:vs(e,s),level:e,message:X(t),details:s??{}}}var de=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return w.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return w.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return w.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return w.join(this.logDir,"ignored.jsonl")}get skippedPath(){return w.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return w.join(this.logDir,"errors.jsonl")}get timingsPath(){return w.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await R.mkdir(this.logDir,{recursive:true}),await R.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await R.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>bs(t,r));}updateCurrentEvent(t,r,s){let n=Cs(t,r,s);this.enqueueTask(async()=>{await R.mkdir(this.logDir,{recursive:true}),await R.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=X(r),i=rt(s),a=JSON.stringify({time:new Date().toISOString(),level:t,message:n,...i||{}});this.enqueueLine(a),this.updateCurrentEvent(t,n,i),t==="error"?console.error(n,i||""):t==="warn"?console.warn(n,i||""):(ie[this.level]>=ie.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(n);}info(t,r){this.push("info",t,r);}debug(t,r){ie[this.level]<ie.debug||this.push("debug",t,r);}progress(t,r){Ps("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),ie[this.level]>=ie.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),ie[this.level]>=ie.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Wt="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)";function Qt(e){let t=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${Wt}(?:[?#][^\\s'"<>\\);]*)?(?=[\\s'"<>\\);]|$)|(?<![A-Za-z0-9._-])/[^\\s'"<>\\);]+\\.${Wt}(?:[?#][^\\s'"<>\\);]*)?(?=[\\s'"<>\\);]|$)`,"gi");return [...e.matchAll(t)].map(r=>r[0])}function _t(e){return !!e&&/^(?:\.{1,2}\/)?[A-Za-z0-9@_][A-Za-z0-9@._/-]*\.(?:js|mjs)(?:[?#][^\s<>\\]*)?$/i.test(e)&&!e.startsWith("/")&&!e.startsWith("//")&&!/^[a-z][a-z0-9+.-]*:/i.test(e)&&!/(?:^|\/)node_modules(?:\/|$)|(?:^|\/)\.pnpm(?:\/|$)/i.test(e)}function Ht(e){let t=new Set,r=[],s=/\bimport\s*\(\s*["'`]([^"'`\s<>\\]+)["'`]\s*\)/g,n=/\b(?:[A-Za-z_$][\w$]*\.)?u\s*=\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>\s*["'`]([^"'`\s<>\\]+)["'`]/g;for(let i of e.matchAll(s))i[1]&&_t(i[1])&&r.push({index:i.index??0,value:i[1]});for(let i of e.matchAll(n))i[1]&&_t(i[1])&&r.push({index:i.index??0,value:i[1]});for(let i of r.sort((a,u)=>a.index-u.index))t.add(i.value);return [...t]}function zt(e,t){let r=w.relative(t,e);return r===""||!r.startsWith("..")&&!w.isAbsolute(r)}function Jt(e,t,r,s){let n=w.resolve(e),i=w.resolve(t),a=w.extname(n).toLowerCase();return !zt(n,i)||a!==".html"&&a!==".htm"||r.has(n)?false:!s.some(u=>zt(n,w.resolve(u)))}function Vt(e,t){return typeof e=="string"&&e.length>0&&t?.supported===true&&typeof t.token=="string"&&t.token.length>0&&e===t.token}function Gt(e,t){try{let r=new URL(e),s=new URL(t,r);if(s.origin!==r.origin)return !1;let n=r.pathname.replace(/\/+$/,"");if(n&&s.pathname!==n&&!s.pathname.startsWith(`${n}/`))return !1;let i=n?s.pathname.slice(n.length)||"/":s.pathname;return ["/wp-includes/","/wp-content/plugins/","/wp-content/themes/","/wp-content/mu-plugins/"].some(a=>i.startsWith(a))}catch{return false}}function Kt(e){return e<1}function Zt(e){let t=String(e??"").toLowerCase();return t==="strict"?"Strict":t==="lax"?"Lax":t==="none"?"None":"Unspecified"}function ks(e){let t=e.pathname||"/";if(!t.startsWith("/")||t==="/")return "/";let r=t.lastIndexOf("/");return r<=0?"/":t.slice(0,r)}function be(e){return `${e.name.toLowerCase()}
|
|
7
7
|
${e.domain.toLowerCase()}
|
|
8
|
-
${e.path}`}function
|
|
9
|
-
`,{encoding:"utf8",mode:384}),await R.rename(r,e);}var
|
|
10
|
-
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function
|
|
8
|
+
${e.path}`}function Yt(e,t){let r;try{r=new URL(t);}catch{return null}let s=e.split(";"),n=s.shift()?.trim()??"",i=n.indexOf("=");if(i<=0)return null;let a=n.slice(0,i).trim();if(!a||[...a].some(b=>b<=" "||"=;,".includes(b)))return null;let u=r.hostname.toLowerCase(),o=ks(r),c=null,d=true,l=false,p=false,y="Unspecified",m=false;for(let b of s){let f=b.indexOf("="),g=(f<0?b:b.slice(0,f)).trim().toLowerCase(),h=f<0?"":b.slice(f+1).trim();if(g==="domain"&&h)u=h.toLowerCase();else if(g==="path"&&h.startsWith("/"))o=h;else if(g==="expires"){let v=Date.parse(h);Number.isFinite(v)&&(c=Math.floor(v/1e3),d=false);}else if(g==="max-age"&&/^-?\d+$/u.test(h)){let v=Number.parseInt(h,10);c=Math.floor(Date.now()/1e3)+v,d=false;}else g==="secure"?l=true:g==="httponly"?p=true:g==="samesite"?y=Zt(h):g==="partitioned"&&(m=true);}return {name:a,domain:u,path:o,expires:c,session:d,secure:l,httpOnly:p,sameSite:y,partitioned:m}}function As(e){let t=!Number.isFinite(e.expires)||e.expires<=0;return {name:e.name,domain:e.domain.toLowerCase(),path:e.path||"/",expires:t?null:e.expires,session:t,secure:e.secure,httpOnly:e.httpOnly,sameSite:Zt(e.sameSite),partitioned:e.partitioned===true}}function Xt(e,t){let r=new Map(e.map(s=>[be(s),s.value]));return t.filter(s=>r.get(be(s))!==s.value).map(As)}function xs(e,t){let r=e.session||t.session;return {...e,expires:r?null:Math.max(e.expires??0,t.expires??0)||null,session:r,secure:e.secure&&t.secure,httpOnly:e.httpOnly&&t.httpOnly,sameSite:e.sameSite===t.sameSite?e.sameSite:"Unspecified",partitioned:e.partitioned&&t.partitioned}}var De=class{#e=new Map;add(t){if(!t)return;let r={...t,domain:t.domain.toLowerCase(),path:t.path||"/"},s=be(r),n=this.#e.get(s);this.#e.set(s,n?xs(n,r):r);}addAll(t){for(let r of t)this.add(r);}artifact(){return {schemaVersion:1,observations:[...this.#e.values()].sort((t,r)=>be(t).localeCompare(be(r)))}}};async function Ee(e,t){await R.mkdir(w.dirname(e),{recursive:true});let r=`${e}.${process.pid}.tmp`;await R.writeFile(r,`${JSON.stringify(t,null,2)}
|
|
9
|
+
`,{encoding:"utf8",mode:384}),await R.rename(r,e);}var $s=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml"],pr=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),gr=new Set(["",".html",".htm"]),Us=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Ls=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Os(e){let t=[],r="full",s="full",n=false,i=false,a="";for(let u=0;u<e.length;u++){let o=e[u];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let c=e[++u];if(!c)throw new Error("--crawl-mode requires a value");s=c==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let c=e[++u];if(!c)throw new Error("--url requires a value");r="single-url",n=true,t.push(c);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let c=e[++u];if(!c)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${c}`);}else if(o==="--content-sync-plan"){let c=e[++u];if(!c)throw new Error("--content-sync-plan requires a file path");r="content-sync",s="incremental",n=true,a=w.resolve(c);}else o.startsWith("--content-sync-plan=")&&(r="content-sync",s="incremental",n=true,a=w.resolve(o.slice(20)));}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i,contentSyncPlanPath:a}}function We(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function Ds(){let e=We();return e?w.join(e,"privacy","cookie-observations.json"):null}function Es(){let e=We();return e?w.resolve(e,".."):""}function Fs(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?w.resolve(e):""}function js(e){let t=e.trim(),r=[{alias:"@storage-root",root:Es(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:We(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Fs(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?w.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:w.resolve(t)}}function fr(e){let t=String(process.env.STATIC_PUBLISHER_CRAWL_MANIFEST_PATH||"").trim();if(t)return w.resolve(t);let r=We();return r?w.join(r,"crawl-manifest.json"):w.join(e.outputDir,".crawl-manifest.json")}function hr(){return {schemaVersion:5,updatedAt:"",pages:{},assets:{}}}function tr(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function qs(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),unreferencedRuns:Math.max(0,Number(e.unreferencedRuns)||0),rewriteComplete:e.rewriteComplete===true||!t}}function it(e){let t=e?.["content-type"]?.trim();return t||null}function Ns(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function Q(e,t,r){let s=K(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function ot(e,t,r){let s=K(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ce(e,t){e.rewriteTargets.add(w.resolve(t));}async function Bs(e){try{let t=await R.readFile(fr(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3||r.schemaVersion===4||r.schemaVersion===5)&&r.assets&&typeof r.assets=="object")return {schemaVersion:5,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,tr(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,qs(n)]))};if(r.schemaVersion===1)return {schemaVersion:5,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,tr(n)])),assets:{}}}}catch{}return hr()}function at(e){return JSON.parse(JSON.stringify(e))}async function Ws(e,t){let r=fr(e);await R.mkdir(w.dirname(r),{recursive:true}),await R.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function oe(e,t){if(!t.trackManifest)return;let r=async()=>{let s=at(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await Ws(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function Re(e){if(!e)return null;let t=e.trim();return t||null}function _s(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],a=t[n];!i||!a||i===a||r.add(n);}return r}async function mr(e,t,r,s){let n=[...new Set(r.map(u=>u.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let u=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!u.ok())return s.warn(`Change token lookup failed with HTTP ${u.status()}`,{endpoint:a,status:u.status()}),new Map;let o=await u.json().catch(()=>null);if(!o||!Array.isArray(o.items))return new Map;let c=new Map;for(let d of o.items)!d||typeof d.url!="string"||c.set(d.url,d);return c}catch(u){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:u instanceof Error?`${u.name}: ${u.message.split(/\r?\n/u,1)[0]}`:"Unknown change-token request failure"}),new Map}}async function je(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await mr(e,t,[s],n)).get(s)??null;return ft(e,s)&&(!a||a.supported!==true||!a.token)&&(a=await zs(e,t,s,n)),a=wr(e,a),r.changeTokenCache.set(s,a),a}async function Qs(e,t,r,s,n){if(!r.trackManifest)return;let i=[...new Set(s.map(a=>a.trim()).filter(Boolean))].filter(a=>!r.changeTokenCache.has(a));for(let a=0;a<i.length;a+=250){let u=i.slice(a,a+250),o=await mr(e,t,u,n);for(let c of u)r.changeTokenCache.set(c,wr(e,o.get(c)??null));}}function Hs(e){return `subscriber=${e.wpsuite?.siteSettings?.subscriber===true?"1":"0"}`}function wr(e,t){if(!t?.supported||!t.token||t.tokenSource==="wp-media-file")return t;let r=Hs(e);return {...t,token:Se(`${t.token}
|
|
10
|
+
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function zs(e,t,r,s){try{let n=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(n.status()!==404)return s.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:n.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let i=await n.text().catch(()=>""),a=Se(`${r}
|
|
11
11
|
404
|
|
12
|
-
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:o,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Bs(e,t,r,s,n,i){let o=gt(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Fe(e,t,s,n,i):null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await Fe(e,t,s,n,i)};let a=await Fe(e,t,s,n,i);if(a?.supported&&a.token)return u.changeToken===a.token?o&&!await Ws(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let l=ve(r.sitemapLastmodByPage[n]);return l&&u.sitemapLastmod&&u.sitemapLastmod===l?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Ws(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function _s(e,t,r,s,n){let i=Zt(r.previousManifest,void 0,s,"previous-manifest"),o=0;for(let[p,w]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){Kt(n,e,w.outputPath);try{await R.unlink(w.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:w.outputPath,error:String(m)});}delete r.manifest.pages[p],o++;}let u=Zt(r.manifest,r.runId,s,"active-manifest"),a=new Set,l=new Set;for(let p of u){for(let b of Ce(e,p))a.add(b);let w=r.manifest.assets[p]?.outputPath;w&&l.add(w);let m=fe(e,t.assetMap,p);m&&l.add(m);}for(let p of Object.keys(r.manifest.assets)){if(u.has(p)){r.manifest.assets[p].unreferencedRuns=0;continue}let w=r.manifest.assets[p];if(Qt(w.unreferencedRuns)){w.unreferencedRuns+=1,s.warn(`Deferred deletion of newly unreferenced incremental asset ${p}.`,{url:p,unreferencedRuns:w.unreferencedRuns,phase:"incremental-cleanup",source:"asset-deletion-grace"});continue}delete r.manifest.assets[p];}let d=0,c=0;for(let p of i){if(r.manifest.assets[p])continue;let w=new Set,m=r.previousManifest.assets[p]?.outputPath;m&&w.add(m);let b=fe(e,t.assetMap,p);if(!u.has(p)&&b)w.add(b);else if(!u.has(p)){let h=an(e,p,s,"incremental-cleanup");h&&(w.add(h.originalFilePath),h.hashedFilePath&&w.add(h.hashedFilePath));}for(let h of w)if(!l.has(h)){Kt(n,e,h);try{await R.unlink(h),d++;}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:h,error:String(g)});}}for(let h of Ce(e,p))a.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],c++);}(o>0||d>0||c>0)&&s.info(`Incremental cleanup removed ${o} stale pages, ${d} stale assets, ${c} stale asset mappings`,{removedPages:o,removedAssets:d,removedAssetMappings:c,phase:"incremental-cleanup"});}function Zt(e,t,r,s){let n=new Set,i=o=>{let u=o.trim();if(u){if(r){let a=ae(u,r,"asset",s??"manifest");if(!a)return;n.add(a.toString());return}n.add(u);}};for(let o of Object.values(e.pages))for(let u of o.discoveredAssets||[])i(u);for(let[o,u]of Object.entries(e.assets||{}))t&&u.lastSeenRunId!==t||i(o);return n}function Yt(e,t,r){for(let s of t)for(let n of Ce(e,s))if(r.has(n))return true;return false}function Qs(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let u of r.changedFiles)i.add(y.resolve(e.outputDir,u));for(let u of Object.values(t.manifest.pages))u.rewriteComplete||i.add(y.resolve(u.outputPath));for(let u of Object.values(t.manifest.assets))u.isText&&!u.rewriteComplete&&i.add(y.resolve(u.outputPath));let o=Fs(s,n);if(o.size===0)return [...i];for(let u of Object.values(t.manifest.pages))Yt(e,u.discoveredAssets,o)&&i.add(y.resolve(u.outputPath));for(let[u,a]of Object.entries(t.manifest.assets)){a.isText&&Yt(e,a.discoveredAssets,o)&&i.add(y.resolve(a.outputPath));for(let l of Ce(e,u))if(o.has(l)){Q(r,e,a.outputPath),a.isText&&i.add(y.resolve(a.outputPath));break}}return [...i]}function Hs(e,t){let r=t?new Set(Array.from(t,n=>y.resolve(n))):null,s=n=>r===null||r.has(y.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function zs(e,t,r){let s=new Set,n=i=>{if(!i)return;let o=re(e.outputDir,i);o&&s.add(o);};for(let i of t.donePages)n(X(e,i));for(let i of Object.values(r.manifest.pages))i.lastSeenRunId===r.runId&&n(i.outputPath);for(let i of t.doneAssets)n(fe(e,t.assetMap,i));for(let i of Object.values(r.manifest.assets))i.lastSeenRunId===r.runId&&n(i.outputPath);return s}async function Js(e,t,r,s,n){let i=[...s.deletedFiles].sort(),o=zs(e,t,r),u=0;for(let d of [...o].sort()){try{if((await R.stat(y.join(e.outputDir,d))).isFile())continue}catch{}u++,n.warn(`Active output is missing after crawl: ${d}`,{phase:"deploy-plan",file:d,outputPath:y.join(e.outputDir,d)});}if(u>0&&n.warn(`Detected ${u} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:u}),i.length===0)return [];let a=[],l=0;for(let d of i){if(!o.has(d)){a.push(d);continue}l++;let c=y.join(e.outputDir,d),p;try{p=(await R.stat(c)).isFile();}catch{p=false;}n.warn(`Deploy plan protected active output from deletion: ${d}`,{phase:"deploy-plan",file:d,outputPath:c,exists:p});}return l>0&&n.info(`Deploy plan skipped ${l} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:l,deletedFiles:a.length}),a}function Ce(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Vs(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function gr(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return y.join(e.outputDir,pt(s.pathname))}catch{return null}}function Gs(e,t,r){for(let s of Vs(e,r)){let n=t[s];if(!n)continue;let i=gr(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function Ks(e,t,r){for(let s of Ce(e,r)){let n=t[s];if(!n)continue;let i=gr(e,n);if(i)return i}return null}function fe(e,t,r){let s=Ks(e,t,r);if(s)return s;try{return Be(e,r).originalFilePath}catch{return null}}function Zs(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.join(y.resolve(e),"queue-runner-heartbeat.json"):""}async function Xt(e){let t=Zs();if(t)try{let r=await R.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await R.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Ys(e,t){let r=new Set;for(let s of t.donePages)r.add(y.resolve(X(e,s)));for(let s of t.doneAssets){let n=fe(e,t.assetMap,s);!n||!ne(n)||r.add(y.resolve(n));}for(let s of t.doneSitemaps){let n=fe(e,t.assetMap,s);!n||!ne(n)||r.add(y.resolve(n));}return [...r]}var je="re:",De=new Map;function Xs(e){if(!e.startsWith(je))return null;let t=De.get(e);if(t!==void 0)return t||null;let r=e.slice(je.length).trim();if(!r)return De.set(e,false),null;try{let s=new RegExp(r);return De.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),De.set(e,false),null}}function dt(e,t){let r=Xs(t);return r?r.test(e):t.startsWith(je)?false:e.startsWith(t)}function en(e,t){return t.find(r=>dt(e,r))}function fr(e,t){let r=en(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(je)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function tn(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function rn(e,t){return tn(e).has(t.hostname)}function sn(e){let t=K(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function at(e){let t=K(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function nn(e){let t=K(e).replace(/\\\//g,"/").trim(),r=y.extname(t).toLowerCase();return ar.has(r)||ur.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function he(e){return xs.some(t=>e.pathname.startsWith(t))}function N(e,t,r=e.sourceOrigin,s,n="url",i=false){let o=sn(t);if(!o||o.startsWith("data:")||o.startsWith("blob:")||o.startsWith("mailto:")||o.startsWith("tel:")||o.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(at(o))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(nn(o))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(o,r);if(hr(u))return s?.reject(n,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!rn(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let a=fr(e,u);if(a&&!(i&&He(u)))return s?.reject(n,u.toString(),a,r),null;let l=new URL(e.sourceOrigin);return u.protocol=l.protocol,u.host=l.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function hr(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function ae(e,t,r,s){try{let n=new URL(e);return hr(n)?(t.reject(r,e,"malformed URL encoding",s),null):n.protocol!=="http:"&&n.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${n.protocol}`,s),null):n}catch{return t.reject(r,e,"invalid URL",s),null}}function mr(e){return ar.has(y.extname(e.pathname).toLowerCase())}function ue(e){return he(e)?false:ur.has(y.extname(e.pathname).toLowerCase())}function me(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function O(e,t){return !!(me(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>dt(t.pathname,r))||mr(t)||!ue(t)&&!he(t))}function X(e,t){let r=new URL(t),s=pt(r.pathname);return s.endsWith("/")?s+="index.html":y.extname(s)||(s+="/index.html"),y.join(e.outputDir,s)}function pt(e){try{return decodeURIComponent(e)}catch{return e}}function on(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&y.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function gt(e,t){let r=on(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Be(e,t){let r=new URL(t),s=pt(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,o=y.join(e.outputDir,i),u=Ke(e.targetOrigin,i),a=`${u}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:o,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=y.extname(s),c=`${l?s.slice(0,-l.length):s}.${ye(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:o,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:c,hashedFilePath:y.join(e.outputDir,c),hashedPublicUrl:Ke(e.targetOrigin,c)}}function an(e,t,r,s){let n=ae(t,r,"asset",s);return n?Be(e,n.toString()):null}function un(e,t,r,s,n){t[r]=s;try{let i=new URL(r),u=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{u.push(new URL(e.targetOrigin).origin);}catch{}for(let a of u)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function wr(e){try{return await R.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ge(e){try{return await R.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function cn(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function yr(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function ln(e,t){let r=yr(t),s=cn(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function dn(e,t){if(!t)return;let r=yr(t);if(r===null)return;let s=await ge(e);if(!s)return;let n=new Date(r);await R.utimes(e,s.atime,n).catch(()=>{});}function er(e,t){let r=y.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return As.has(r)||ks.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function pn(e){try{let t=y.extname(new URL(e).pathname).toLowerCase();return t===".js"||t===".mjs"}catch{return false}}function pe(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset processing progress: processed ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function gn(e){return ye(e.toString("base64"),40)}function fn(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function hn(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function ut(e,t,r,s,n,i,o){let u=Be(e,r),a=gn(s),l=u.originalFilePath,d=u.originalPublicUrlWithSearch;if(u.preferQueryHashed&&u.hashedFilePath&&u.hashedPublicUrl&&fn(n,u.originalFilePath,a)){let m=Gs(e,t,r);m?(l=m.outputPath,d=m.publicUrl):(l=u.hashedFilePath,d=u.hashedPublicUrl);}un(e,t,r,d,u.originalPublicUrl);let c=ne(l)?Buffer.from(Xe(s.toString("utf8"),e,t,l),"utf8"):s,p=await wr(l),w=false;return (p===null||!p.equals(c))&&(await Se(l,c),w=true),i?.debug(`${w?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:l,bytes:c.length,wroteFile:w}),hn(n,u.originalFilePath,a),await dn(l,o),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:l,publicUrl:d,wroteFile:w}}function Re(e,t,r,s,n){let i=()=>{let c=String(n??"").trim();if(!c)return {discoveredFrom:"",discoveredFromType:"unknown"};if(c==="seed-path"||c==="cli"||c==="generated-404-request-path")return {discoveredFrom:c,discoveredFromType:c};if(/^https?:\/\//i.test(c))try{let p=new URL(c);return me(p)?{discoveredFrom:c,discoveredFromType:"sitemap"}:O(e,p)?{discoveredFrom:c,discoveredFromType:"asset"}:ue(p)&&!he(p)?{discoveredFrom:c,discoveredFromType:"page"}:{discoveredFrom:c,discoveredFromType:"url"}}catch{return {discoveredFrom:c,discoveredFromType:"url"}}return {discoveredFrom:c,discoveredFromType:"url"}},o=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=N(e,r,o,s,"page");if(!u)return;if(!ue(u)||mr(u)){if(O(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),q(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),he(u)?"asset/internal path cannot be page":"not page-like",n);return}let a=u.toString(),{discoveredFrom:l,discoveredFromType:d}=i();l&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,l),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:l,discoveredFromType:d,outputPath:X(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function q(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=N(e,r,i,s,"asset");if(!o)return;if(!O(e,o)){s.reject("asset",o.toString(),"not a safe asset path/prefix",n);return}let u=o.toString();!t.doneAssets.has(u)&&!t.queuedAssets.has(u)&&(t.queuedAssets.add(u),t.assetQueue.push(u),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function qe(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=N(e,r,i,s,"sitemap",Rt(n));if(!o)return;if(!me(o)){s.reject("sitemap",o.toString(),"not sitemap-like",n);return}let u=o.toString();!t.doneSitemaps.has(u)&&!t.queuedSitemaps.has(u)&&(t.queuedSitemaps.add(u),t.sitemapQueue.push(u),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function mn(e){let t=K(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:ve(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function wn(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function yn(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function Sn(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function Pn(e,t){if(t==="dom-style")return true;let r=(()=>{try{return y.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function st(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Sr(e,t,r,s){let n=[];await R.mkdir(t,{recursive:true});let i=await R.readdir(e,{withFileTypes:true});for(let o of i){let u=y.join(e,o.name),a=y.join(t,o.name);if(o.isDirectory()){n.push(...await Sr(u,a,r,s));continue}await Pr(u,a,r,s)&&n.push(a);}return n}async function bn(e,t){try{let[r,s]=await Promise.all([R.stat(e),R.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([R.readFile(e),R.readFile(t)]);return n.equals(i)}catch{return false}}async function Pr(e,t,r,s){let n=y.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await bn(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await R.mkdir(y.dirname(t),{recursive:true}),await R.copyFile(e,t),true)}async function vn(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([u,a])=>({sourcePath:u.trim(),prefix:String(a||"").trim()}));if(s.length===0)return [];let n=0,i=[],o=new Set([...t.donePages].map(u=>y.resolve(X(e,u))));r.mark("copy-extra-paths");for(let u of s){let a=u.sourcePath,l=u.prefix;if(!a||!l){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:l});continue}let d=Ls(a);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:l});continue}let c=d.resolvedPath,p=st(l);if(!p){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:l});continue}let w;try{w=await R.stat(c);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:c,prefixPath:l});continue}let m=y.resolve(e.outputDir,p);if(w.isDirectory()){let f=await Sr(c,m,o,r);i.push(...f),n+=f.length,f.length>0&&r.progress(`Copied static directory to export: ${c} -> /${p}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${p}`,copiedItems:n,changedFiles:f.length});continue}let b=y.basename(c),h=l.endsWith("/")?y.join(m,b):m;await Pr(c,h,o,r)&&(i.push(h),n++,r.progress(`Copied static file to export: ${c} -> /${st(l.endsWith("/")?`${l}${b}`:l)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${st(l.endsWith("/")?`${l}${b}`:l)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function ke(e,t,r,s,n){let i=new Set,o=K(r).replace(/\\\//g,"/"),u=n==="page-html"?Sn(o):o,a=Pn(t,n);if(a){for(let d of yn(u)){let c=N(e,d,t,s,`${n}:css-url`);c&&O(e,c)&&i.add(c.toString());}for(let d of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=N(e,d[1],t,s,`${n}:css-import`);c&&O(e,c)&&i.add(c.toString());}}if(a||n==="page-html")for(let d of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=N(e,d[1],t,s,`${n}:xml-stylesheet`);c&&O(e,c)&&i.add(c.toString());}let l=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${tt}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${tt}(?:\\?[^\\s'"<>\\);]*)?`,"gi");if(n==="page-html")for(let d of o.matchAll(l)){let c=d[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${d[0]}`:d[0];if(at(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=N(e,c,t,s,`${n}:serialized-url`);p&&O(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}if(n!=="page-html"&&pn(t))for(let d of Bt(o)){if(at(d)||!/\.(?:js|mjs)(?:[?#]|$)/i.test(d))continue;let c=N(e,d,t,s,`${n}:relative-asset`);c&&O(e,c)&&i.add(c.toString());}return [...i]}function Cn(e,t,r,s){let n=new Set,i=K(r);for(let o of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let u=o[1],a=N(e,u,t,s,"page-link");a&&(fr(e,a)||ue(a)&&!he(a)&&!O(e,a)&&n.add(a.toString()));}return [...n]}async function tr(e,t,r,s,n,i,o=false,u=[]){let a=new Set(u),l=0,d=new Map,c=(p,w)=>{let m=[w],b=new Set;for(;m.length>0;){let h=m.pop();if(h===p)return true;b.has(h)||(b.add(h),m.push(...d.get(h)??[]));}return false};for(;r.sitemapQueue.length>0;){let p=r.sitemapQueue.shift();r.queuedSitemaps.delete(p);let w=ae(p,s,"sitemap","sitemap-queue");if(!w)continue;let m=w.toString();if(r.doneSitemaps.has(m))continue;r.doneSitemaps.add(m),s.sitemap(`Fetching sitemap ${m}`,{url:m});let b=true;try{let h=await t.get(m,{timeout:6e4});if(b=!1,!h.ok()){if(s.skip("sitemap",m,h.status()),o&&!a.has(m))throw new Error(`Required child content-sync sitemap returned HTTP ${h.status()}: ${m}`);continue}let g=await h.body(),f=h.url()||m,C=ae(f,s,"sitemap-response",m);if(!C)continue;let v=C.toString();r.successfulSitemaps.add(m),r.successfulSitemaps.add(v),a.has(m)&&(r.resolvedSitemapRoots.add(v),l+=1);let U=await ut(e,r.assetMap,v,g,r,s);Q(n,e,U.outputPath),ce(n,U.outputPath);let M=g.toString("utf8");/<sitemapindex(?:\s|>)/i.test(M)?r.sitemapIndexUrls.add(v):r.sitemapLeafUrls.add(v);let B=ke(e,v,M,s,"sitemap");for(let A of B)q(e,r,A,s,v);let j=[];for(let A of mn(M)){let k=N(e,A.loc,v,s,"sitemap-loc",!0);if(!k){if(o)throw new Error(`Content-sync sitemap contains an inadmissible location: ${A.loc}`);continue}if(me(k)){if(c(v,k.toString()))throw new Error(`Sitemap index cycle detected: ${v} -> ${k.toString()}`);if(d.has(v)||d.set(v,new Set),d.get(v).add(k.toString()),j.push(k.toString()),o&&r.doneSitemaps.has(k.toString())&&!r.successfulSitemaps.has(k.toString()))throw new Error(`Required child content-sync sitemap was not resolved: ${k.toString()}`);qe(e,r,k.toString(),s,v);}else if(o){if(ue(k)&&!O(e,k)){let W=i.requiredUrls.has(k.toString()),S=i.manifest.pages[k.toString()],x=!!S&&!!await ge(S.outputPath);if(!W&&!x)throw new Error(`Content-sync sitemap coverage is incomplete for ${k.toString()}.`)}s.ignore("sitemap-loc",k.toString(),"content-sync sitemap-only traversal does not enqueue urlset locations",v);}else O(e,k)?q(e,r,k.toString(),s,v):ue(k)?(A.lastmod&&(r.sitemapLastmodByPage[k.toString()]=A.lastmod),Re(e,r,k.toString(),s,v)):s.reject("sitemap-loc",k.toString(),"not page/sitemap/asset-like",v);}i.trackManifest&&(i.manifest.assets[v]={url:v,outputPath:U.outputPath,isText:!0,contentType:h.headers()["content-type"]??"application/xml",changeToken:null,tokenSource:null,discoveredAssets:[...new Set([...B,...j])],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!1},await oe(e,i));}catch(h){s.error(`Failed sitemap ${m}`,{url:m,error:String(h)});let g=o&&a.has(m)&&b;if(o&&!g)throw h}}if(o&&a.size>0&&l===0)throw new Error(`None of the configured content-sync sitemap roots could be resolved: ${[...a].join(", ")}`)}async function Rn(e,t,r,s,n,i,o){let u=N(e,s,e.sourceOrigin,n,"asset-fetch");if(!u||!O(e,u))return;let a=u.toString();if(r.doneAssets.has(a))return;let l=i.enabled||i.strictTargeted?i.previousManifest.assets[a]:void 0,d=i.trackManifest?await Fe(e,t,i,a,n):null,c=d?.token??null,p=Wt(l?.changeToken,d),w=i.strictTargeted&&(!d?.supported||!d.token)&&_t(e.sourceOrigin,a),m=i.strictTargeted?i.manifest.assets[a]:void 0;if(m?.lastSeenRunId===i.runId&&await ge(m.outputPath)){r.doneAssets.add(a);for(let h of m.discoveredAssets||[])q(e,r,h,n,a);Q(o,e,m.outputPath),m.isText&&ce(o,m.outputPath),n.info(`Resumed completed content-sync asset ${a}`,{url:a,source:"content-sync-checkpoint"}),pe(r,n);return}if(l&&(p||w)&&await ge(l.outputPath)){i.manifest.assets[a]={...l,changeToken:c,tokenSource:d?.tokenSource??l.tokenSource,discoveredAssets:[...l.discoveredAssets||[]],lastSeenRunId:i.runId,unreferencedRuns:0},delete i.manifest.pages[a],r.doneAssets.add(a);for(let h of l.discoveredAssets||[])q(e,r,h,n,a);n.info(`Reused unchanged trusted asset ${a}.`,{url:a,tokenSource:d?.tokenSource??"unknown",phase:"download-assets",source:p?"asset-change-token-match":"content-sync-release-fingerprint"}),pe(r,n);return}let b=(h,g={})=>{if(!l||i.strictTargeted)return false;i.manifest.assets[a]={...l,discoveredAssets:[...l.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0},delete i.manifest.pages[a],r.doneAssets.add(a);for(let f of l.discoveredAssets||[])q(e,r,f,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:h,preservedFromPreviousManifest:true,...g,phase:"download-assets"}),pe(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let h=fe(e,r.assetMap,a),g=h?await ge(h):null;if(h&&g)try{let W=await F(`fetch asset headers ${a}`,6e4,()=>t.head(a,{timeout:6e4})),S=W.headers();if(W.ok()&&ln(g,S)){let x=await wr(h);if(x!==null){let L=W.url()||a,D=ae(L,n,"asset-response",a);if(!D)return;let P=D.toString(),T=await ut(e,r.assetMap,P,x,r,n,S);T.wroteFile&&(Q(o,e,T.outputPath),ne(T.outputPath)&&ce(o,T.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:h,lastModified:S["last-modified"]||"",contentLength:S["content-length"]||"",phase:"download-assets"}),pe(r,n);let $=er(u,S),_=l?.discoveredAssets?[...l.discoveredAssets]:[];if($||l?.isText){let V=x.toString("utf8"),I=ke(e,P,V,n,"asset:cached");(I.length>0||!l)&&(_=I);for(let xe of _)q(e,r,xe,n,P);}if(i.trackManifest){let V=$||!!l?.isText;i.manifest.assets[a]={url:a,outputPath:T.outputPath,isText:V,contentType:it(S)||l?.contentType||null,changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:_,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!V},delete i.manifest.pages[a],(i.enabled||i.strictTargeted)&&V&&await oe(e,i);}return}}}catch{}let f=await F(`fetch asset ${a}`,6e4,()=>t.get(a,{timeout:6e4}));if(!f.ok()){if(b(`http-${f.status()}`,{status:f.status()}))return;if(i.strictTargeted)throw new Error(`Required content-sync asset returned HTTP ${f.status()}: ${a}`);n.skip("asset",a,f.status());return}let C=await F(`read asset response ${a}`,6e4,()=>f.body()),v=f.url()||a,U=ae(v,n,"asset-response",a);if(!U)return;let M=U.toString(),B=f.headers(),j=await ut(e,r.assetMap,M,C,r,n,B);j.wroteFile&&(Q(o,e,j.outputPath),ne(j.outputPath)&&ce(o,j.outputPath)),r.doneAssets.add(a),pe(r,n);let A=er(u,B),k=[];if(A){let W=C.toString("utf8");k=ke(e,M,W,n,`asset:${y.extname(u.pathname).toLowerCase()||B["content-type"]||"unknown"}`);for(let S of k)q(e,r,S,n,M);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:j.outputPath,isText:A,contentType:it(B),changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:k,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!A},delete i.manifest.pages[a],(i.enabled||i.strictTargeted)&&A&&await oe(e,i));}catch(h){if(b("fetch-error",{error:String(h)}))return;if(n.error(`Failed asset ${a}`,{url:a,error:String(h)}),i.strictTargeted)throw h}finally{r.doneAssets.has(a)||(r.doneAssets.add(a),pe(r,n));}}async function kn(e,t,r,s,n,i){let o=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),u=Array.from({length:o},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Rn(e,t,r,a,s,n,i));}});await Promise.all(u);}function An(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function Y(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function be(e){return Math.max(15e3,Math.min(6e4,Y(e)))}function xn(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function ft(e){return e.stats.pagesCompleted}function ct(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function nt(e,t,r){let s=ct(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function F(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Ae(e,t));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}var Ae=class extends Error{constructor(r,s){super(`${r} timed out after ${An(s)}.`);this.operation=r;this.timeoutMs=s;this.name="OperationTimeoutError";}operation;timeoutMs};async function Tn(e,t){let r=xn(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,o=700,u=Date.now(),a=()=>{window.clearInterval(l),window.scrollTo(0,0),n();},l=window.setInterval(()=>{if(Date.now()-u>=s.maxDurationMs){a();return}window.scrollBy(0,o),i+=o;let d=document.body?.scrollHeight||0,c=document.documentElement?.scrollHeight||0,p=Math.max(d,c);i>=p+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function In(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Tn(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Mn(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function $n(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>dt(t,r))}async function Ln(e,t,r,s,n=true){let i=new Set;if(n){let u=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,c=["srcset","data-srcset","data-lazy-srcset"],p=w=>{w.querySelectorAll("*").forEach(m=>{for(let h of l){let g=m.getAttribute(h);g&&a.add(g);}for(let h of c){let g=m.getAttribute(h);g&&g.split(",").forEach(f=>{let C=f.trim().split(/\s+/)[0];C&&a.add(C);});}let b=m.getAttribute("style");b&&d.add(b),m.shadowRoot&&p(m.shadowRoot);});};return p(document),{attrs:[...a],styles:[...d]}});for(let a of u.attrs)for(let l of wn(a)){let d=N(e,l,r,s,"dom-attr");d&&O(e,d)&&i.add(d.toString());}for(let a of u.styles)for(let l of ke(e,r,a,s,"dom-style"))i.add(l);}let o=await t.content();for(let u of ke(e,r,o,s,"page-html"))i.add(u);return [...i]}async function rr(e,t,r,s,n){let i=X(e,t);await Se(i,r,"utf8");let o=await R.stat(i);n?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:i,bytes:o.size}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,ft(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Un(e){let t=ft(e);return t%5===0||t===1||t===e.stats.pagesQueued}function On(e,t){let r=ft(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Dn(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function En(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Dn(s),s}async function Fn(e,t,r,s,n,i,o,u){let a=await chromium.launch({headless:true}),l=ct(a);l.pid!==null&&r.debug(`Launched crawl worker browser PID ${l.pid}`,{browserPid:l.pid});try{for(;t.pageQueue.length>0;){a||(a=await chromium.launch({headless:!0}),l=ct(a),l.pid!==null&&r.debug(`Relaunched crawl worker browser PID ${l.pid}`,{browserPid:l.pid}));let d=a;if(e.maxPages>0&&t.donePages.size>=e.maxPages)break;let c=t.pageQueue.shift();if(!c)break;t.queuedPages.delete(c);let p=null,w=null,m=!1,b=[],h=new Set;try{let g=ae(c,r,"page","page-queue");if(!g)continue;let f=g.toString();if(c=f,t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),i.enabled&&i.seenPages.add(c);let C=gt(e,c),v=$n(e,g.pathname);if(!ue(g)||O(e,g)||he(g)){if(O(e,g)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),q(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let U=i.strictTargeted?i.manifest.pages[c]:void 0;if(U?.lastSeenRunId===i.runId&&await ge(U.outputPath)){i.seenPages.add(c);for(let P of U.discoveredAssets||[])q(e,t,P,r,c);Q(o,e,U.outputPath),ce(o,U.outputPath),r.info(`Resumed completed content-sync page ${c}`,{url:c,source:"content-sync-checkpoint"});continue}let M=await Bs(e,n,t,i,c,r);if(i.enabled&&s!=="single-url"&&s!=="content-sync"){let P=i.manifest.pages[c];if(M.action==="reuse"&&P){for(let T of P.discoveredAssets)q(e,t,T,r,c);for(let T of P.discoveredPages)Re(e,t,T,r,c);P.lastSeenRunId=i.runId,P.sitemapLastmod=ve(t.sitemapLastmodByPage[c])??P.sitemapLastmod,M.changeToken?.supported&&(P.changeToken=M.changeToken.token,P.tokenSource=M.changeToken.tokenSource??P.tokenSource),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:M.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}p=await F(`create page context ${c}`,Y(e),()=>En(d,e,v)),w=await F(`create page ${c}`,Y(e),()=>p.newPage()),b=await p.cookies();let B=!1,j=new Set;await w.route("**/*",async P=>{let T=P.request();try{let $=N(e,T.url(),e.sourceOrigin,r,"network-request");$&&O(e,$)&&(j.add($.toString()),q(e,t,$.toString(),r,c));}catch{}if(T.isNavigationRequest()&&T.frame()===w.mainFrame())try{let $=new URL(T.url()),_=new URL(f);if($.hash="",_.hash="",$.href===_.href){if(B){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:T.url()});try{await P.abort("aborted");}catch(V){r.debug(`Ignored canceled route while blocking reload for ${c}`,{url:c,error:String(V)});}return}B=!0;}}catch{}try{await P.continue();}catch($){r.debug(`Ignored canceled network route for ${c}`,{url:c,requestUrl:T.url(),error:String($)});}}),w.on("response",P=>{try{let $=N(e,P.url(),e.sourceOrigin,r,"network-response");$&&O(e,$)&&(j.add($.toString()),q(e,t,$.toString(),r,c));}catch{}let T=P.headerValues("set-cookie").then($=>{for(let _ of $)u.add(zt(_,P.url()));}).catch(()=>{}).then(()=>{});h.add(T),T.finally(()=>h.delete(T));}),r.page(`Rendering ${c}`,{url:c});let A=null,k=null;try{if(A=await w.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),A&&A.ok())try{k=await F(`read response body ${c}`,Y(e),()=>A.text());}catch{k=null;}}catch(P){r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(P)});}if(s==="content-sync"&&A){let P=ae(A.url(),r,"content-sync-final-response",c)?.toString();if(P&&P!==f){if(i.deleteOnMissingUrls.has(c)){await lt(e,i,o,c,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${P}`,{url:c,finalResponseUrl:P,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${c} -> ${P}`)}}if(C){let P=A?.status()??null;if(P!==404){r.error(`Generated 404 request path returned unexpected status for ${c}`,{url:c,expectedStatus:404,status:P,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${c}`,{url:c,status:P,outputPath:X(e,c)});}else if(A&&!A.ok()){if(s==="content-sync"&&(A.status()===404||A.status()===410)&&i.deleteOnMissingUrls.has(c)){await lt(e,i,o,c,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${c}`,{url:c,status:A.status(),source:"content-sync-tombstone"}),await w.close();continue}if(s==="content-sync")throw new Error(`Required content-sync route returned HTTP ${A.status()}: ${c}`);r.skip("page",c,A.status(),t.pageDiscoverySources.get(c)),await w.close();continue}v?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await F(`wait for rendered page ${c}`,Y(e),()=>Mn(e,w))):await F(`wait for rendered page ${c}`,Y(e),()=>In(e,w)),t.stats.pagesRendered++;let W=await F(`extract rendered assets ${c}`,Y(e),()=>Ln(e,w,f,r,!v));for(let P of W)q(e,t,P,r,c);let S=[...new Set([...W,...j])],x=k??await F(`capture rendered html ${c}`,Y(e),()=>w.content()),L=s!=="single-url"&&s!=="content-sync"&&!C?Cn(e,c,x,r):[];if(s!=="single-url"&&s!=="content-sync"&&!C)for(let P of L)Re(e,t,P,r,c);await rr(e,c,x,t,r);let D=X(e,c);if(Q(o,e,D),ce(o,D),i.trackManifest){let P=A?.headers();i.manifest.pages[c]={url:c,outputPath:D,contentType:it(P),changeToken:M.changeToken?.supported?M.changeToken.token:null,tokenSource:M.changeToken?.supported?M.changeToken.tokenSource??null:null,sitemapLastmod:ve(t.sitemapLastmodByPage[c])??null,discoveredPages:L,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await oe(e,i);}}catch(g){if(m=g instanceof Ae,i.strictTargeted)throw r.error(`Failed required content-sync page ${c}`,{url:c,error:String(g)}),g;if(m)r.warn(`Skipped partial DOM for timed-out page ${c}`,{url:c,error:String(g),recovery:"recycle-worker-browser"});else try{let f=w?await F(`capture partial html ${c}`,Y(e),()=>w.content()):"";if(f&&f.trim()){await rr(e,c,f,t,r);let C=X(e,c);if(Q(o,e,C),ce(o,C),i.trackManifest){let v=i.manifest.pages[c]??i.previousManifest.pages[c];i.manifest.pages[c]={url:c,outputPath:C,contentType:v?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:ve(t.sitemapLastmodByPage[c])??v?.sitemapLastmod??null,discoveredPages:v?.discoveredPages?[...v.discoveredPages]:[],discoveredAssets:v?.discoveredAssets?[...v.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await oe(e,i);}r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(g)});}}catch(f){f instanceof Ae&&(m=!0),r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(f)});}r.error(`Failed page ${c}`,{url:c,error:String(g)});}finally{if(!m&&w&&await F(`drain network routes ${c}`,be(e),()=>w.unrouteAll({behavior:"wait"})).catch(g=>{m=!0,r.warn(`Could not drain network routes for ${c}`,{url:c,browserPid:l.pid,error:String(g),recovery:"recycle-worker-browser"});}),m&&(a=null,await nt(d,r,`page operation timeout for ${c}`),await d.close().catch(()=>{})),h.size>0&&await Promise.allSettled([...h]),!m&&p)try{let g=await p.cookies();u.addAll(Jt(b,g));}catch{}t.stats.pagesCompleted++,Un(t)&&On(t,r),!m&&w&&await F(`close page ${c}`,be(e),()=>w.close()).catch(g=>{m=!0,r.warn(`Timed out while closing page ${c}`,{url:c,browserPid:l.pid,error:String(g),recovery:"recycle-worker-browser"});}),!m&&p&&await F(`close page context ${c}`,be(e),()=>p.close()).catch(g=>{m=!0,r.warn(`Timed out while closing page context for ${c}`,{url:c,browserPid:l.pid,error:String(g),recovery:"recycle-worker-browser"});}),m&&a===d&&(a=null,await nt(d,r,`page cleanup timeout for ${c}`),await d.close().catch(()=>{}));}}}finally{if(a){let d=a;await F("close worker browser",be(e),()=>d.close()).catch(c=>(r.warn("Timed out while closing crawl worker browser",{browserPid:l.pid,error:String(c)}),nt(d,r,"browser.close timeout")));}}}async function jn(e){try{let t=await R.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function sr(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function qn(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Nn(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function br(e){try{let t=await R.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function nr(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Bn(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>nr(s)-nr(n)).map(s=>y.join(e,String(s.storedFileName))):[];return [...new Set([...r,y.join(e,"errors.jsonl"),y.join(e,"errors.json"),y.join(e,"errors.jsonl.gz"),y.join(e,"errors.json.gz")])]}async function ir(e){let t=await br(y.join(e,"job.json"));for(let r of Bn(e,t)){let s=await jn(r);if(s.length>0)return s}return []}async function Wn(e){let t=y.join(e.logDir,"archive"),r;try{r=await R.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let o=y.join(t,i);if(!(await R.stat(o).catch(()=>null))?.isDirectory())continue;let a=await br(y.join(o,"job.json"));if(!qn(a))continue;let l=Nn(a);l>=n&&(n=l,s=o);}return s}async function _n(e){let t=await Wn(e),r=t!==""?await ir(t):await ir(e.logDir),s=new Set;for(let n of r)if(sr(n.error)||sr(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Qn(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await R.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Hn(e){if(!e)throw new Error("Content-sync crawl requires an immutable impact plan.");let t=JSON.parse(await R.readFile(e,"utf8"));return vt(t),t}function zn(e,t){let r;try{r=new URL(e);}catch{return false}return t.archiveFamilies.some(s=>{let n=new URL(s.url);if(r.origin!==n.origin)return false;if(r.toString()===n.toString())return true;if(r.searchParams.has("paged")&&r.pathname===n.pathname)return /^\d+$/.test(r.searchParams.get("paged")||"");let i=n.pathname.replace(/\/+$/,""),o=r.pathname.slice(i.length).replace(/^\/+/,""),u=String(s.paginationBase||"page").replace(/^\/+|\/+$/g,"").replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return r.pathname.startsWith(`${i}/`)&&new RegExp(`^${u}/\\d+/?$`).test(o)})}function Ee(e,t,r){let s=r==="page"?X(e,t):Be(e,t).originalFilePath;return re(e.outputDir,s)}async function lt(e,t,r,s,n){let i=t.manifest.pages[s];if(!i)return;let o=re(e.outputDir,i.outputPath);if(!o)throw new Error(`Manifest-owned page path escaped output root: ${s}`);let u=y.resolve(i.outputPath);Object.entries(t.manifest.pages).some(([l,d])=>l!==s&&!n.has(l)&&y.resolve(d.outputPath)===u)||(await R.rm(i.outputPath,{force:true}),r.deletedFiles.add(o)),delete t.manifest.pages[s];}function Jn(e,t){let r=new Set,s=[...t];for(;s.length>0;){let n=s.shift();if(r.has(n))continue;r.add(n);let i=e.assets[n];for(let o of i?.discoveredAssets??[])try{me(new URL(o))&&s.push(o);}catch{}}return r}async function Vn(e,t,r,s,n){let i=Jn(r.previousManifest,n);for(let o of i){if(t.successfulSitemaps.has(o))continue;let u=r.previousManifest.assets[o];if(!u)continue;let a=re(e.outputDir,u.outputPath);if(!a)throw new Error(`Manifest-owned sitemap path escaped output root: ${o}`);await R.rm(u.outputPath,{force:true}),s.deletedFiles.add(a),delete r.manifest.assets[o];}await oe(e,r);}async function Gn(e){try{let t=await R.readFile(y.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Kn(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await R.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await R.unlink(y.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function or(e,t,r,s,n){let i=N(e,r,e.sourceOrigin,s,"manual-url");i&&(me(i)?qe(e,t,i.toString(),s,n):O(e,i)?q(e,t,i.toString(),s,n):ue(i)?Re(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function vr(e=process.argv.slice(2)){let t=[],r=await Te(void 0,{onRemotePublisherConfigDiagnostic:S=>{t.push(S);}}),s=Ct(r.sitemapPaths),n=t.at(-1)??null,i=Ts(e),o=i.mode==="content-sync"?await Hn(i.contentSyncPlanPath):null;if(i.resumeRewrite&&i.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let u=[];i.mode==="retry-timeouts"&&(u=await _n(r)),i.mode==="single-url"&&(u=await Qn(i.urls)),o&&(u=[...o.directRenderUrls,...o.archiveFamilyUrls]);let a=i.crawlMode==="incremental"&&i.mode==="full",l=a&&typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!=="";if(o&&r.subscriptionType!=="PROFESSIONAL"&&r.subscriptionType!=="AGENCY")throw new Error("Content-sync requires an active Professional or Agency subscription.");let d=(i.mode==="full"||i.mode==="single-url"||i.mode==="content-sync")&&!i.resumeRewrite,c=i.mode!=="full"||i.resumeRewrite||l,p=c?await Gn(r):{},w=l||i.mode==="single-url"||i.mode==="content-sync"?await Ds(r):lr(),m=new Set(u),b=new Set;if(o){for(let S of o.archiveFamilies)if(S.foundPosts===0)for(let x of S.pageUrls)b.add(x);for(let S of Object.keys(w.pages))zn(S,o)&&!m.has(S)&&(u.push(S),b.add(S));}let h={enabled:l,trackManifest:d,manifest:ot(w),previousManifest:ot(w),runId:o?.jobId??`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve(),strictTargeted:!!o,requiredUrls:m,deleteOnMissingUrls:b};await Kn(r.logDir);let g=new de(r.logDir,r.logLevel);if(n){let S={status:n.status,virtualAssetBaseUrl:n.virtualAssetBaseUrl,subscriptionType:n.subscriptionType??null,error:n.error,requests:n.requests,source:"remote-publisher-config"};n.status==="loaded"&&n.subscriptionType?g.info("Loaded remote WP Suite publisher configuration.",S):g.warn("Remote WP Suite publisher configuration is unavailable or has no active subscription type.",S);}if(a&&!l){let S="Incremental crawl requires an active remote publisher subscription; refusing to fall back to a destructive full crawl.";throw g.error(S,{requestedMode:i.crawlMode,subscriptionType:r.subscriptionType??null,source:"incremental-subscription-guard"}),await g.flush(),new Error(S)}let f={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,successfulSitemaps:new Set,resolvedSitemapRoots:new Set,sitemapIndexUrls:new Set,sitemapLeafUrls:new Set,sitemapLastmodByPage:{},assetMap:{...p},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},C=Os(),v=new Ue,U=Is();if(U&&!i.resumeRewrite&&await Oe(U,v.artifact()),await St(r),await Pt(r,{startedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir)}),c||await R.rm(r.outputDir,{recursive:true,force:true}),await R.mkdir(r.outputDir,{recursive:true}),o){let S=new Set([...o.deleteUrls,...h.deleteOnMissingUrls]);for(let x of o.deleteUrls)await lt(r,h,C,x,S);}if(i.resumeRewrite)g.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:i.mode,crawlMode:i.crawlMode});else {let S=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{if(g.mark("discovery"),i.mode==="full"?(s.forEach(L=>qe(r,f,L,g)),await tr(r,S,f,g,C,h),r.seedPaths.forEach(L=>or(r,f,L,g,"seed-path")),r.generated404RequestPath&&Re(r,f,r.generated404RequestPath,g,"generated-404-request-path")):(u.forEach(L=>or(r,f,L,g,"cli")),o?.sitemapUrls.forEach(L=>qe(r,f,L,g,"content-sync-impact")),await tr(r,S,f,g,C,h,i.mode==="content-sync",o?.sitemapUrls??[]),o&&await Vn(r,f,h,C,o.sitemapUrls)),g.endMark("discovery",{pages:f.pageQueue.length,assets:f.assetQueue.length,sitemaps:f.sitemapQueue.length}),g.summary(`Queued ${f.pageQueue.length} pages, ${f.assetQueue.length} assets, ${f.sitemapQueue.length} sitemaps.`,{mode:i.mode,crawlMode:i.crawlMode,queuedPages:f.pageQueue.length,queuedAssets:f.assetQueue.length,queuedSitemaps:f.sitemapQueue.length}),i.mode==="retry-timeouts"&&f.pageQueue.length===0&&f.assetQueue.length===0&&f.sitemapQueue.length===0){g.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:i.mode,crawlMode:i.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await g.flush(),U&&await Oe(U,v.artifact());return}g.mark("render-pages");let x=Array.from({length:Math.max(1,r.concurrency)},()=>Fn(r,f,g,i.mode,S,h,C,v));if(await Promise.all(x),U&&await Oe(U,v.artifact()),g.endMark("render-pages",{pages:f.stats.pagesCompleted}),g.mark("download-assets"),g.progress("Asset download phase started.",{phase:"download-assets",doneAssets:f.doneAssets.size,assetsQueued:f.stats.assetsQueued,assetQueue:f.assetQueue.length}),await js(r,S,h,[...f.assetQueue],g),await kn(r,S,f,g,h,C),g.endMark("download-assets",{assets:f.doneAssets.size}),h.enabled&&r.maxPages===0&&await _s(r,f,h,g,C),i.mode==="full"){let L=await vn(r,f,g);for(let D of L)Q(C,r,D);}}finally{await F("dispose crawl request context",be(r),()=>S.dispose()).catch(x=>{g.warn("Timed out while disposing crawl request context",{error:String(x)});});}}h.trackManifest&&await oe(r,h),await R.writeFile(y.join(r.outputDir,"asset-map.json"),JSON.stringify(f.assetMap,null,2),"utf8"),JSON.stringify(p,null,2)!==JSON.stringify(f.assetMap,null,2)&&Q(C,r,y.join(r.outputDir,"asset-map.json"));let M=i.mode==="single-url"||i.mode==="content-sync"?Ys(r,f):h.enabled&&!i.resumeRewrite?Qs(r,h,C,p,f.assetMap):void 0;g.mark("rewrite-text");let B=i.mode==="single-url"||i.mode==="content-sync"?`Rewriting text files touched by URL crawl: 0/${M?.length??0}`:"Rewriting text files...";g.progress(B,{phase:"rewrite-text",index:0,totalFiles:M?.length,changedTextFiles:0}),await Xt(B);let j=Date.now(),A=await Et(r,f.assetMap,{files:M,previousAssetMap:p,onStart:({totalFiles:S,workerCount:x})=>{g.info(`Rewrite phase scanning ${S} text files.`,{phase:"rewrite-text",totalFiles:S,workerCount:x});},onProgress:async({index:S,totalFiles:x,changedTextFiles:L,file:D,changed:P})=>{P&&Q(C,r,y.join(r.outputDir,D)),g.checkpoint(`Rewriting text file ${S}/${x}`,{phase:"rewrite-text",index:S,totalFiles:x,changedTextFiles:L,file:D}),g.debug(`${P?"Rewrote":"Checked unchanged"} text file ${S}/${x}: ${D}`,{phase:"rewrite-text",index:S,totalFiles:x,changedTextFiles:L,file:D,changed:P});let T=Date.now();if(!(S===1||S===x||S%50===0||T-j>=5e3))return;j=T;let _=`Rewriting text files: ${S}/${x}`;g.progress(_,{phase:"rewrite-text",index:S,totalFiles:x,changedTextFiles:L,file:D}),await Xt(_);}});h.trackManifest&&(Hs(h,M),await oe(r,h)),g.endMark("rewrite-text",{changedTextFiles:A});let k=o?[...C.deletedFiles].sort():h.enabled&&!i.resumeRewrite?await Js(r,f,h,C,g):[],W=[...new Set((M||[]).map(S=>re(r.outputDir,y.isAbsolute(S)?S:y.join(r.outputDir,S))).filter(S=>S!==null))].sort();if(o){let S=[...C.changedFiles].sort(),x=new Set(S),L=new Set(o.directRenderUrls.map(I=>Ee(r,I,"page")).filter(I=>!!I&&x.has(I))),D=new Set(o.archiveFamilyUrls.map(I=>Ee(r,I,"page")).filter(I=>!!I&&x.has(I))),P=new Set([...f.sitemapLeafUrls].map(I=>Ee(r,I,"asset")).filter(I=>!!I&&x.has(I))),T=new Set([...f.sitemapIndexUrls].map(I=>Ee(r,I,"asset")).filter(I=>!!I&&x.has(I))),$=new Set([...L,...D,...P,...T]),_=S.filter(I=>!$.has(I)),V=[...new Set([...o.directRenderUrls,...o.archiveFamilyUrls,...f.resolvedSitemapRoots,...f.sitemapLeafUrls,...f.sitemapIndexUrls,...o.deleteUrls].map(I=>{let xe=new URL(I);return `${xe.pathname}${xe.search}`}))].sort();await We(r,{schemaVersion:2,generatedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir),runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,changedFiles:S,deletedFiles:k,rewriteTargets:W,consumerId:o.consumerId,fromSequence:o.fromSequence,toSequence:o.toSequence,impactHash:o.impactHash,resolvedSitemapUrls:[...f.resolvedSitemapRoots],uploadGroups:{assets:_,content:[...L].sort(),listings:[...D].sort(),sitemapLeaves:[...P].sort(),sitemapIndexes:[...T].reverse()},invalidationPaths:V});}else await We(r,{schemaVersion:1,generatedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir),runMode:i.mode,crawlMode:i.crawlMode,fullSyncRequired:!(h.enabled&&!i.resumeRewrite),changedFiles:h.enabled&&!i.resumeRewrite?[...C.changedFiles].sort():[],deletedFiles:h.enabled&&!i.resumeRewrite?k:[],rewriteTargets:h.enabled&&!i.resumeRewrite?W:[]});await bt(r),g.summary(i.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${A} text files.`:`Done. Rendered ${f.stats.pagesRendered} pages, processed ${f.doneSitemaps.size} sitemaps and ${f.doneAssets.size} assets.`,{mode:i.mode,crawlMode:i.crawlMode,resumeRewrite:i.resumeRewrite,...f.stats,pagesRendered:f.stats.pagesRendered,donePages:f.stats.pagesCompleted,claimedPages:f.donePages.size,doneSitemaps:f.doneSitemaps.size,doneAssets:f.doneAssets.size,changedTextFiles:A}),await g.flush();}vr().catch(async e=>{let t=Z(e instanceof Error?e.message:String(e));console.error(t);try{let s=(await Te().catch(()=>null))?.logDir??"logs",n=new de(s,"debug");n.error(`Unhandled error: ${t}`,e instanceof Error&&e.stack?{stack:Z(e.stack)}:void 0),await n.flush();}catch{}process.exit(1);});
|
|
12
|
+
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:a,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Js(e,t,r,s,n,i){let a=ft(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await je(e,t,s,n,i):null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await je(e,t,s,n,i)};let o=await je(e,t,s,n,i);if(o?.supported&&o.token)return u.changeToken===o.token?a&&!await Vs(e,t,n,i)?{action:"render",changeToken:o}:{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let c=Re(r.sitemapLastmodByPage[n]);return c&&u.sitemapLastmod&&u.sitemapLastmod===c?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function Vs(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Gs(e,t,r,s,n){let i=rr(r.previousManifest,void 0,s,"previous-manifest"),a=0;for(let[p,y]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){ot(n,e,y.outputPath);try{await R.unlink(y.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:y.outputPath,error:String(m)});}delete r.manifest.pages[p],a++;}let u=rr(r.manifest,r.runId,s,"active-manifest"),o=new Set,c=new Set;for(let p of u){for(let b of ke(e,p))o.add(b);let y=r.manifest.assets[p]?.outputPath;y&&c.add(y);let m=he(e,t.assetMap,p);m&&c.add(m);}for(let p of Object.keys(r.manifest.assets)){if(u.has(p)){r.manifest.assets[p].unreferencedRuns=0;continue}let y=r.manifest.assets[p];if(Kt(y.unreferencedRuns)){y.unreferencedRuns+=1,s.warn(`Deferred deletion of newly unreferenced incremental asset ${p}.`,{url:p,unreferencedRuns:y.unreferencedRuns,phase:"incremental-cleanup",source:"asset-deletion-grace"});continue}delete r.manifest.assets[p];}let d=0,l=0;for(let p of i){if(r.manifest.assets[p])continue;let y=new Set,m=r.previousManifest.assets[p]?.outputPath;m&&y.add(m);let b=he(e,t.assetMap,p);if(!u.has(p)&&b)y.add(b);else if(!u.has(p)){let f=fn(e,p,s,"incremental-cleanup");f&&(y.add(f.originalFilePath),f.hashedFilePath&&y.add(f.hashedFilePath));}for(let f of y)if(!c.has(f)){ot(n,e,f);try{await R.unlink(f),d++;}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:f,error:String(g)});}}for(let f of ke(e,p))o.has(f)||Object.prototype.hasOwnProperty.call(t.assetMap,f)&&(delete t.assetMap[f],l++);}(a>0||d>0||l>0)&&s.info(`Incremental cleanup removed ${a} stale pages, ${d} stale assets, ${l} stale asset mappings`,{removedPages:a,removedAssets:d,removedAssetMappings:l,phase:"incremental-cleanup"});}async function Ks(e,t,r,s){let n=new Set([...Object.values(t.manifest.pages),...Object.values(t.manifest.assets)].map(o=>String(o.outputPath||"").trim()).filter(Boolean).map(o=>w.resolve(o))),i=Object.values(e.postCrawlCopyMap||{}).map(o=>qe(String(o||""))).filter(o=>!!o).map(o=>w.resolve(e.outputDir,o)),u=(await Xr(["**/*.html","**/*.htm"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(o=>Jt(w.join(e.outputDir,o),e.outputDir,n,i));for(let o of u){let c=w.join(e.outputDir,o);ot(s,e,c),await R.unlink(c);}u.length>0&&r.info(`Removed ${u.length} orphaned HTML output file(s) that are absent from the current crawl manifest.`,{phase:"incremental-cleanup",source:"manifest-output-reconciliation",removedPages:u.length,samples:u.slice(0,20)});}function rr(e,t,r,s){let n=new Set,i=a=>{let u=a.trim();if(u){if(r){let o=ae(u,r,"asset",s??"manifest");if(!o)return;n.add(o.toString());return}n.add(u);}};for(let a of Object.values(e.pages))for(let u of a.discoveredAssets||[])i(u);for(let[a,u]of Object.entries(e.assets||{}))t&&u.lastSeenRunId!==t||i(a);return n}function sr(e,t,r){for(let s of t)for(let n of ke(e,s))if(r.has(n))return true;return false}function Zs(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let u of r.changedFiles)i.add(w.resolve(e.outputDir,u));for(let u of Object.values(t.manifest.pages))u.rewriteComplete||i.add(w.resolve(u.outputPath));for(let u of Object.values(t.manifest.assets))u.isText&&!u.rewriteComplete&&i.add(w.resolve(u.outputPath));let a=_s(s,n);if(a.size===0)return [...i];for(let u of Object.values(t.manifest.pages))sr(e,u.discoveredAssets,a)&&i.add(w.resolve(u.outputPath));for(let[u,o]of Object.entries(t.manifest.assets)){o.isText&&sr(e,o.discoveredAssets,a)&&i.add(w.resolve(o.outputPath));for(let c of ke(e,u))if(a.has(c)){Q(r,e,o.outputPath),o.isText&&i.add(w.resolve(o.outputPath));break}}return [...i]}function Ys(e,t){let r=t?new Set(Array.from(t,n=>w.resolve(n))):null,s=n=>r===null||r.has(w.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function Xs(e,t,r){let s=new Set,n=i=>{if(!i)return;let a=K(e.outputDir,i);a&&s.add(a);};for(let i of t.donePages)n(te(e,i));for(let i of Object.values(r.manifest.pages))i.lastSeenRunId===r.runId&&n(i.outputPath);for(let i of t.doneAssets)n(he(e,t.assetMap,i));for(let i of Object.values(r.manifest.assets))i.lastSeenRunId===r.runId&&n(i.outputPath);return s}async function en(e,t,r,s,n){let i=[...s.deletedFiles].sort(),a=Xs(e,t,r),u=0;for(let d of [...a].sort()){try{if((await R.stat(w.join(e.outputDir,d))).isFile())continue}catch{}u++,n.warn(`Active output is missing after crawl: ${d}`,{phase:"deploy-plan",file:d,outputPath:w.join(e.outputDir,d)});}if(u>0&&n.warn(`Detected ${u} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:u}),i.length===0)return [];let o=[],c=0;for(let d of i){if(!a.has(d)){o.push(d);continue}c++;let l=w.join(e.outputDir,d),p;try{p=(await R.stat(l)).isFile();}catch{p=false;}n.warn(`Deploy plan protected active output from deletion: ${d}`,{phase:"deploy-plan",file:d,outputPath:l,exists:p});}return c>0&&n.info(`Deploy plan skipped ${c} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:c,deletedFiles:o.length}),o}function ke(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function tn(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function yr(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return w.join(e.outputDir,gt(s.pathname))}catch{return null}}function rn(e,t,r){for(let s of tn(e,r)){let n=t[s];if(!n)continue;let i=yr(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function sn(e,t,r){for(let s of ke(e,r)){let n=t[s];if(!n)continue;let i=yr(e,n);if(i)return i}return null}function he(e,t,r){let s=sn(e,t,r);if(s)return s;try{return _e(e,r).originalFilePath}catch{return null}}function nn(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.join(w.resolve(e),"queue-runner-heartbeat.json"):""}async function nr(e){let t=nn();if(t)try{let r=await R.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await R.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function on(e,t){let r=new Set;for(let s of t.donePages)r.add(w.resolve(te(e,s)));for(let s of t.doneAssets){let n=he(e,t.assetMap,s);!n||!ne(n)||r.add(w.resolve(n));}for(let s of t.doneSitemaps){let n=he(e,t.assetMap,s);!n||!ne(n)||r.add(w.resolve(n));}return [...r]}var Ne="re:",Fe=new Map;function an(e){if(!e.startsWith(Ne))return null;let t=Fe.get(e);if(t!==void 0)return t||null;let r=e.slice(Ne.length).trim();if(!r)return Fe.set(e,false),null;try{let s=new RegExp(r);return Fe.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Fe.set(e,false),null}}function pt(e,t){let r=an(t);return r?r.test(e):t.startsWith(Ne)?false:e.startsWith(t)}function un(e,t){return t.find(r=>pt(e,r))}function Sr(e,t){let r=un(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Ne)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function cn(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function ln(e,t){return cn(e).has(t.hostname)}function dn(e){let t=Y(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function ut(e){let t=Y(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function pn(e){let t=Y(e).replace(/\\\//g,"/").trim(),r=w.extname(t).toLowerCase();return pr.has(r)||gr.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function me(e){return Ls.some(t=>e.pathname.startsWith(t))}function N(e,t,r=e.sourceOrigin,s,n="url",i=false){let a=dn(t);if(!a||a.startsWith("data:")||a.startsWith("blob:")||a.startsWith("mailto:")||a.startsWith("tel:")||a.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(ut(a))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(pn(a))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(a,r);if(Pr(u))return s?.reject(n,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!ln(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let o=Sr(e,u);if(o&&!(i&&Je(u)))return s?.reject(n,u.toString(),o,r),null;let c=new URL(e.sourceOrigin);return u.protocol=c.protocol,u.host=c.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function Pr(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function ae(e,t,r,s){try{let n=new URL(e);return Pr(n)?(t.reject(r,e,"malformed URL encoding",s),null):n.protocol!=="http:"&&n.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${n.protocol}`,s),null):n}catch{return t.reject(r,e,"invalid URL",s),null}}function br(e){return pr.has(w.extname(e.pathname).toLowerCase())}function ue(e){return me(e)?false:gr.has(w.extname(e.pathname).toLowerCase())}function we(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function O(e,t){return !!(we(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>pt(t.pathname,r))||br(t)||!ue(t)&&!me(t))}function te(e,t){let r=new URL(t),s=gt(r.pathname);return s.endsWith("/")?s+="index.html":w.extname(s)||(s+="/index.html"),w.join(e.outputDir,s)}function gt(e){try{return decodeURIComponent(e)}catch{return e}}function gn(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&w.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function ft(e,t){let r=gn(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function _e(e,t){let r=new URL(t),s=gt(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=w.join(e.outputDir,i),u=Ye(e.targetOrigin,i),o=`${u}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:u,originalPublicUrlWithSearch:o,preferQueryHashed:n};let c=w.extname(s),l=`${c?s.slice(0,-c.length):s}.${Se(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:u,originalPublicUrlWithSearch:o,preferQueryHashed:n,hashedPathname:l,hashedFilePath:w.join(e.outputDir,l),hashedPublicUrl:Ye(e.targetOrigin,l)}}function fn(e,t,r,s){let n=ae(t,r,"asset",s);return n?_e(e,n.toString()):null}function hn(e,t,r,s,n){t[r]=s;try{let i=new URL(r),u=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{u.push(new URL(e.targetOrigin).origin);}catch{}for(let o of u)try{let c=new URL(o);t[c.origin+i.pathname+i.search]=s;}catch{}try{let o=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=o;}catch{}try{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}try{let o=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}}catch{}}async function vr(e){try{return await R.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function fe(e){try{return await R.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function mn(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Cr(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function wn(e,t){let r=Cr(t),s=mn(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function yn(e,t){if(!t)return;let r=Cr(t);if(r===null)return;let s=await fe(e);if(!s)return;let n=new Date(r);await R.utimes(e,s.atime,n).catch(()=>{});}function ir(e,t){let r=w.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Us.has(r)||$s.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Sn(e){try{let t=w.extname(new URL(e).pathname).toLowerCase();return t===".js"||t===".mjs"}catch{return false}}function ge(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset processing progress: processed ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Pn(e){return Se(e.toString("base64"),40)}function bn(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function vn(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function ct(e,t,r,s,n,i,a){let u=_e(e,r),o=Pn(s),c=u.originalFilePath,d=u.originalPublicUrlWithSearch;if(u.preferQueryHashed&&u.hashedFilePath&&u.hashedPublicUrl&&bn(n,u.originalFilePath,o)){let m=rn(e,t,r);m?(c=m.outputPath,d=m.publicUrl):(c=u.hashedFilePath,d=u.hashedPublicUrl);}hn(e,t,r,d,u.originalPublicUrl);let l=ne(c)?Buffer.from(tt(s.toString("utf8"),e,t,c),"utf8"):s,p=await vr(c),y=false;return (p===null||!p.equals(l))&&(await Pe(c,l),y=true),i?.debug(`${y?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:c,bytes:l.length,wroteFile:y}),vn(n,u.originalFilePath,o),await yn(c,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:c,publicUrl:d,wroteFile:y}}function Ae(e,t,r,s,n){let i=()=>{let l=String(n??"").trim();if(!l)return {discoveredFrom:"",discoveredFromType:"unknown"};if(l==="seed-path"||l==="cli"||l==="generated-404-request-path")return {discoveredFrom:l,discoveredFromType:l};if(/^https?:\/\//i.test(l))try{let p=new URL(l);return we(p)?{discoveredFrom:l,discoveredFromType:"sitemap"}:O(e,p)?{discoveredFrom:l,discoveredFromType:"asset"}:ue(p)&&!me(p)?{discoveredFrom:l,discoveredFromType:"page"}:{discoveredFrom:l,discoveredFromType:"url"}}catch{return {discoveredFrom:l,discoveredFromType:"url"}}return {discoveredFrom:l,discoveredFromType:"url"}},a=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=N(e,r,a,s,"page");if(!u)return;if(!ue(u)||br(u)){if(O(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),q(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),me(u)?"asset/internal path cannot be page":"not page-like",n);return}let o=u.toString(),{discoveredFrom:c,discoveredFromType:d}=i();c&&!t.pageDiscoverySources.has(o)&&t.pageDiscoverySources.set(o,c),!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,s.debug(`Queued discovered page ${o}`,{phase:"discovery",url:o,discoveredFrom:c,discoveredFromType:d,outputPath:te(e,o)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function q(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=N(e,r,i,s,"asset");if(!a)return;if(!O(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let u=a.toString();!t.doneAssets.has(u)&&!t.queuedAssets.has(u)&&(t.queuedAssets.add(u),t.assetQueue.push(u),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Be(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=N(e,r,i,s,"sitemap",At(n));if(!a)return;if(!we(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let u=a.toString();!t.doneSitemaps.has(u)&&!t.queuedSitemaps.has(u)&&(t.queuedSitemaps.add(u),t.sitemapQueue.push(u),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Cn(e){let t=Y(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:Re(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Rn(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function kn(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function An(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function xn(e,t){if(t==="dom-style")return true;let r=(()=>{try{return w.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function qe(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Rr(e,t,r,s){let n=[];await R.mkdir(t,{recursive:true});let i=await R.readdir(e,{withFileTypes:true});for(let a of i){let u=w.join(e,a.name),o=w.join(t,a.name);if(a.isDirectory()){n.push(...await Rr(u,o,r,s));continue}await kr(u,o,r,s)&&n.push(o);}return n}async function Tn(e,t){try{let[r,s]=await Promise.all([R.stat(e),R.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([R.readFile(e),R.readFile(t)]);return n.equals(i)}catch{return false}}async function kr(e,t,r,s){let n=w.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await Tn(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await R.mkdir(w.dirname(t),{recursive:true}),await R.copyFile(e,t),true)}async function In(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([u,o])=>({sourcePath:u.trim(),prefix:String(o||"").trim()}));if(s.length===0)return [];let n=0,i=[],a=new Set([...t.donePages].map(u=>w.resolve(te(e,u))));r.mark("copy-extra-paths");for(let u of s){let o=u.sourcePath,c=u.prefix;if(!o||!c){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:o,prefixPath:c});continue}let d=js(o);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:o,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:c});continue}let l=d.resolvedPath,p=qe(c);if(!p){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:o,prefixPath:c});continue}let y;try{y=await R.stat(l);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:o,sourceAbs:l,prefixPath:c});continue}let m=w.resolve(e.outputDir,p);if(y.isDirectory()){let h=await Rr(l,m,a,r);i.push(...h),n+=h.length,h.length>0&&r.progress(`Copied static directory to export: ${l} -> /${p}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${p}`,copiedItems:n,changedFiles:h.length});continue}let b=w.basename(l),f=c.endsWith("/")?w.join(m,b):m;await kr(l,f,a,r)&&(i.push(f),n++,r.progress(`Copied static file to export: ${l} -> /${qe(c.endsWith("/")?`${c}${b}`:c)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${qe(c.endsWith("/")?`${c}${b}`:c)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function xe(e,t,r,s,n){let i=new Set,a=Y(r).replace(/\\\//g,"/"),u=n==="page-html"?An(a):a,o=xn(t,n);if(o){for(let c of kn(u)){let d=N(e,c,t,s,`${n}:css-url`);d&&O(e,d)&&i.add(d.toString());}for(let c of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let d=N(e,c[1],t,s,`${n}:css-import`);d&&O(e,d)&&i.add(d.toString());}}if(o||n==="page-html")for(let c of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let d=N(e,c[1],t,s,`${n}:xml-stylesheet`);d&&O(e,d)&&i.add(d.toString());}if(n==="page-html")for(let c of Qt(a)){let d=c.startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c}`:c;if(ut(d)){s.ignore(`${n}:serialized-url`,d,"looks like JavaScript/code fragment",t);continue}let l=N(e,d,t,s,`${n}:serialized-url`);l&&O(e,l)?i.add(l.toString()):l&&s.ignore("asset",l.toString(),"serialized URL did not pass safe asset path filter",t);}if(n!=="page-html"&&Sn(t))for(let c of Ht(a)){if(ut(c)||!/\.(?:js|mjs)(?:[?#]|$)/i.test(c))continue;let d=N(e,c,t,s,`${n}:relative-asset`);d&&O(e,d)&&i.add(d.toString());}return [...i]}function Mn(e,t,r,s){let n=new Set,i=Y(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let u=a[1],o=N(e,u,t,s,"page-link");o&&(Sr(e,o)||ue(o)&&!me(o)&&!O(e,o)&&n.add(o.toString()));}return [...n]}async function or(e,t,r,s,n,i,a=false,u=[]){let o=new Set(u),c=0,d=new Map,l=(p,y)=>{let m=[y],b=new Set;for(;m.length>0;){let f=m.pop();if(f===p)return true;b.has(f)||(b.add(f),m.push(...d.get(f)??[]));}return false};for(;r.sitemapQueue.length>0;){let p=r.sitemapQueue.shift();r.queuedSitemaps.delete(p);let y=ae(p,s,"sitemap","sitemap-queue");if(!y)continue;let m=y.toString();if(r.doneSitemaps.has(m))continue;r.doneSitemaps.add(m),s.sitemap(`Fetching sitemap ${m}`,{url:m});let b=true;try{let f=await t.get(m,{timeout:6e4});if(b=!1,!f.ok()){if(s.skip("sitemap",m,f.status()),a&&!o.has(m))throw new Error(`Required child content-sync sitemap returned HTTP ${f.status()}: ${m}`);continue}let g=await f.body(),h=f.url()||m,v=ae(h,s,"sitemap-response",m);if(!v)continue;let C=v.toString();r.successfulSitemaps.add(m),r.successfulSitemaps.add(C),o.has(m)&&(r.resolvedSitemapRoots.add(C),c+=1);let L=await ct(e,r.assetMap,C,g,r,s);Q(n,e,L.outputPath),ce(n,L.outputPath);let $=g.toString("utf8");/<sitemapindex(?:\s|>)/i.test($)?r.sitemapIndexUrls.add(C):r.sitemapLeafUrls.add(C);let B=xe(e,C,$,s,"sitemap");for(let x of B)q(e,r,x,s,C);let j=[];for(let x of Cn($)){let k=N(e,x.loc,C,s,"sitemap-loc",!0);if(!k){if(a)throw new Error(`Content-sync sitemap contains an inadmissible location: ${x.loc}`);continue}if(we(k)){if(l(C,k.toString()))throw new Error(`Sitemap index cycle detected: ${C} -> ${k.toString()}`);if(d.has(C)||d.set(C,new Set),d.get(C).add(k.toString()),j.push(k.toString()),a&&r.doneSitemaps.has(k.toString())&&!r.successfulSitemaps.has(k.toString()))throw new Error(`Required child content-sync sitemap was not resolved: ${k.toString()}`);Be(e,r,k.toString(),s,C);}else if(a){if(ue(k)&&!O(e,k)){let W=i.requiredUrls.has(k.toString()),S=i.manifest.pages[k.toString()],A=!!S&&!!await fe(S.outputPath);if(!W&&!A)throw new Error(`Content-sync sitemap coverage is incomplete for ${k.toString()}.`)}s.ignore("sitemap-loc",k.toString(),"content-sync sitemap-only traversal does not enqueue urlset locations",C);}else O(e,k)?q(e,r,k.toString(),s,C):ue(k)?(x.lastmod&&(r.sitemapLastmodByPage[k.toString()]=x.lastmod),Ae(e,r,k.toString(),s,C)):s.reject("sitemap-loc",k.toString(),"not page/sitemap/asset-like",C);}i.trackManifest&&(i.manifest.assets[C]={url:C,outputPath:L.outputPath,isText:!0,contentType:f.headers()["content-type"]??"application/xml",changeToken:null,tokenSource:null,discoveredAssets:[...new Set([...B,...j])],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!1},await oe(e,i));}catch(f){s.error(`Failed sitemap ${m}`,{url:m,error:String(f)});let g=a&&o.has(m)&&b;if(a&&!g)throw f}}if(a&&o.size>0&&c===0)throw new Error(`None of the configured content-sync sitemap roots could be resolved: ${[...o].join(", ")}`)}async function $n(e,t,r,s,n,i,a){let u=N(e,s,e.sourceOrigin,n,"asset-fetch");if(!u||!O(e,u))return;let o=u.toString();if(r.doneAssets.has(o))return;let c=i.enabled||i.strictTargeted?i.previousManifest.assets[o]:void 0,d=i.trackManifest?await je(e,t,i,o,n):null,l=d?.token??null,p=Vt(c?.changeToken,d),y=i.strictTargeted&&(!d?.supported||!d.token)&&Gt(e.sourceOrigin,o),m=i.strictTargeted?i.manifest.assets[o]:void 0;if(m?.lastSeenRunId===i.runId&&await fe(m.outputPath)){r.doneAssets.add(o);for(let f of m.discoveredAssets||[])q(e,r,f,n,o);Q(a,e,m.outputPath),m.isText&&ce(a,m.outputPath),n.info(`Resumed completed content-sync asset ${o}`,{url:o,source:"content-sync-checkpoint"}),ge(r,n);return}if(c&&(p||y)&&await fe(c.outputPath)){i.manifest.assets[o]={...c,changeToken:l,tokenSource:d?.tokenSource??c.tokenSource,discoveredAssets:[...c.discoveredAssets||[]],lastSeenRunId:i.runId,unreferencedRuns:0},delete i.manifest.pages[o],r.doneAssets.add(o);for(let f of c.discoveredAssets||[])q(e,r,f,n,o);n.info(`Reused unchanged trusted asset ${o}.`,{url:o,tokenSource:d?.tokenSource??"unknown",phase:"download-assets",source:p?"asset-change-token-match":"content-sync-release-fingerprint"}),ge(r,n);return}let b=(f,g={})=>{if(!c||i.strictTargeted)return false;i.manifest.assets[o]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0},delete i.manifest.pages[o],r.doneAssets.add(o);for(let h of c.discoveredAssets||[])q(e,r,h,n,o);return n.warn(`Keeping previous incremental asset ${o} after fetch issue.`,{url:o,reason:f,preservedFromPreviousManifest:true,...g,phase:"download-assets"}),ge(r,n),true};n.asset(`Fetching asset ${o}`,{url:o});try{let f=he(e,r.assetMap,o),g=f?await fe(f):null;if(f&&g)try{let W=await F(`fetch asset headers ${o}`,6e4,()=>t.head(o,{timeout:6e4})),S=W.headers();if(W.ok()&&wn(g,S)){let A=await vr(f);if(A!==null){let M=W.url()||o,D=ae(M,n,"asset-response",o);if(!D)return;let P=D.toString(),I=await ct(e,r.assetMap,P,A,r,n,S);I.wroteFile&&(Q(a,e,I.outputPath),ne(I.outputPath)&&ce(a,I.outputPath)),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:f,lastModified:S["last-modified"]||"",contentLength:S["content-length"]||"",phase:"download-assets"}),ge(r,n);let U=ir(u,S),_=c?.discoveredAssets?[...c.discoveredAssets]:[];if(U||c?.isText){let V=A.toString("utf8"),Ie=xe(e,P,V,n,"asset:cached");(Ie.length>0||!c)&&(_=Ie);for(let T of _)q(e,r,T,n,P);}if(i.trackManifest){let V=U||!!c?.isText;i.manifest.assets[o]={url:o,outputPath:I.outputPath,isText:V,contentType:it(S)||c?.contentType||null,changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:_,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!V},delete i.manifest.pages[o],(i.enabled||i.strictTargeted)&&V&&await oe(e,i);}return}}}catch{}let h=await F(`fetch asset ${o}`,6e4,()=>t.get(o,{timeout:6e4}));if(!h.ok()){if(b(`http-${h.status()}`,{status:h.status()}))return;if(i.strictTargeted)throw new Error(`Required content-sync asset returned HTTP ${h.status()}: ${o}`);n.skip("asset",o,h.status());return}let v=await F(`read asset response ${o}`,6e4,()=>h.body()),C=h.url()||o,L=ae(C,n,"asset-response",o);if(!L)return;let $=L.toString(),B=h.headers(),j=await ct(e,r.assetMap,$,v,r,n,B);j.wroteFile&&(Q(a,e,j.outputPath),ne(j.outputPath)&&ce(a,j.outputPath)),r.doneAssets.add(o),ge(r,n);let x=ir(u,B),k=[];if(x){let W=v.toString("utf8");k=xe(e,$,W,n,`asset:${w.extname(u.pathname).toLowerCase()||B["content-type"]||"unknown"}`);for(let S of k)q(e,r,S,n,$);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:j.outputPath,isText:x,contentType:it(B),changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:k,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!x},delete i.manifest.pages[o],(i.enabled||i.strictTargeted)&&x&&await oe(e,i));}catch(f){if(b("fetch-error",{error:String(f)}))return;if(n.error(`Failed asset ${o}`,{url:o,error:String(f)}),i.strictTargeted)throw f}finally{r.doneAssets.has(o)||(r.doneAssets.add(o),ge(r,n));}}async function Un(e,t,r,s,n,i){let a=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),u=Array.from({length:a},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await $n(e,t,r,o,s,n,i));}});await Promise.all(u);}function Ln(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function ee(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function Ce(e){return Math.max(15e3,Math.min(6e4,ee(e)))}function On(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function ht(e){return e.stats.pagesCompleted}function lt(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function nt(e,t,r){let s=lt(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function F(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Te(e,t));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}var Te=class extends Error{constructor(r,s){super(`${r} timed out after ${Ln(s)}.`);this.operation=r;this.timeoutMs=s;this.name="OperationTimeoutError";}operation;timeoutMs};async function Dn(e,t){let r=On(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,a=700,u=Date.now(),o=()=>{window.clearInterval(c),window.scrollTo(0,0),n();},c=window.setInterval(()=>{if(Date.now()-u>=s.maxDurationMs){o();return}window.scrollBy(0,a),i+=a;let d=document.body?.scrollHeight||0,l=document.documentElement?.scrollHeight||0,p=Math.max(d,l);i>=p+window.innerHeight&&o();},120);});},{maxDurationMs:r});}async function En(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Dn(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Fn(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function jn(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>pt(t,r))}async function qn(e,t,r,s,n=true){let i=new Set;if(n){let u=await t.evaluate(()=>{let o=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],d=new Set,l=["srcset","data-srcset","data-lazy-srcset"],p=y=>{y.querySelectorAll("*").forEach(m=>{for(let f of c){let g=m.getAttribute(f);g&&o.add(g);}for(let f of l){let g=m.getAttribute(f);g&&g.split(",").forEach(h=>{let v=h.trim().split(/\s+/)[0];v&&o.add(v);});}let b=m.getAttribute("style");b&&d.add(b),m.shadowRoot&&p(m.shadowRoot);});};return p(document),{attrs:[...o],styles:[...d]}});for(let o of u.attrs)for(let c of Rn(o)){let d=N(e,c,r,s,"dom-attr");d&&O(e,d)&&i.add(d.toString());}for(let o of u.styles)for(let c of xe(e,r,o,s,"dom-style"))i.add(c);}let a=await t.content();for(let u of xe(e,r,a,s,"page-html"))i.add(u);return [...i]}async function ar(e,t,r,s,n){let i=te(e,t);await Pe(i,r,"utf8");let a=await R.stat(i);n?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:i,bytes:a.size}),s.stats.pagesSaved++;let u=Math.min(s.stats.pagesQueued,ht(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${u}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:u,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Nn(e){let t=ht(e);return t%5===0||t===1||t===e.stats.pagesQueued}function Bn(e,t){let r=ht(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Wn(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function _n(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Wn(s),s}async function Qn(e,t,r,s,n,i,a,u){let o=await chromium.launch({headless:true}),c=lt(o);c.pid!==null&&r.debug(`Launched crawl worker browser PID ${c.pid}`,{browserPid:c.pid});try{for(;t.pageQueue.length>0;){o||(o=await chromium.launch({headless:!0}),c=lt(o),c.pid!==null&&r.debug(`Relaunched crawl worker browser PID ${c.pid}`,{browserPid:c.pid}));let d=o;if(e.maxPages>0&&t.donePages.size>=e.maxPages)break;let l=t.pageQueue.shift();if(!l)break;t.queuedPages.delete(l);let p=null,y=null,m=!1,b=[],f=new Set;try{let g=ae(l,r,"page","page-queue");if(!g)continue;let h=g.toString();if(l=h,t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let v=ft(e,l),C=jn(e,g.pathname);if(!ue(g)||O(e,g)||me(g)){if(O(e,g)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),q(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let L=i.strictTargeted?i.manifest.pages[l]:void 0;if(L?.lastSeenRunId===i.runId&&await fe(L.outputPath)){i.seenPages.add(l);for(let P of L.discoveredAssets||[])q(e,t,P,r,l);Q(a,e,L.outputPath),ce(a,L.outputPath),r.info(`Resumed completed content-sync page ${l}`,{url:l,source:"content-sync-checkpoint"});continue}let $=await Js(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"&&s!=="content-sync"){let P=i.manifest.pages[l];if($.action==="reuse"&&P){for(let I of P.discoveredAssets)q(e,t,I,r,l);for(let I of P.discoveredPages)Ae(e,t,I,r,l);P.lastSeenRunId=i.runId,P.sitemapLastmod=Re(t.sitemapLastmodByPage[l])??P.sitemapLastmod,$.changeToken?.supported&&(P.changeToken=$.changeToken.token,P.tokenSource=$.changeToken.tokenSource??P.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:$.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}p=await F(`create page context ${l}`,ee(e),()=>_n(d,e,C)),y=await F(`create page ${l}`,ee(e),()=>p.newPage()),b=await p.cookies();let B=!1,j=new Set;await y.route("**/*",async P=>{let I=P.request();try{let U=N(e,I.url(),e.sourceOrigin,r,"network-request");U&&O(e,U)&&(j.add(U.toString()),q(e,t,U.toString(),r,l));}catch{}if(I.isNavigationRequest()&&I.frame()===y.mainFrame())try{let U=new URL(I.url()),_=new URL(h);if(U.hash="",_.hash="",U.href===_.href){if(B){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:I.url()});try{await P.abort("aborted");}catch(V){r.debug(`Ignored canceled route while blocking reload for ${l}`,{url:l,error:String(V)});}return}B=!0;}}catch{}try{await P.continue();}catch(U){r.debug(`Ignored canceled network route for ${l}`,{url:l,requestUrl:I.url(),error:String(U)});}}),y.on("response",P=>{try{let U=N(e,P.url(),e.sourceOrigin,r,"network-response");U&&O(e,U)&&(j.add(U.toString()),q(e,t,U.toString(),r,l));}catch{}let I=P.headerValues("set-cookie").then(U=>{for(let _ of U)u.add(Yt(_,P.url()));}).catch(()=>{}).then(()=>{});f.add(I),I.finally(()=>f.delete(I));}),r.page(`Rendering ${l}`,{url:l});let x=null,k=null;try{if(x=await y.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),x&&x.ok())try{k=await F(`read response body ${l}`,ee(e),()=>x.text());}catch{k=null;}}catch(P){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(P)});}if(s==="content-sync"&&x){let P=ae(x.url(),r,"content-sync-final-response",l)?.toString();if(P&&P!==h){if(i.deleteOnMissingUrls.has(l)){await dt(e,i,a,l,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${P}`,{url:l,finalResponseUrl:P,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${l} -> ${P}`)}}if(v){let P=x?.status()??null;if(P!==404){r.error(`Generated 404 request path returned unexpected status for ${l}`,{url:l,expectedStatus:404,status:P,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${l}`,{url:l,status:P,outputPath:te(e,l)});}else if(x&&!x.ok()){if(s==="content-sync"&&(x.status()===404||x.status()===410)&&i.deleteOnMissingUrls.has(l)){await dt(e,i,a,l,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${l}`,{url:l,status:x.status(),source:"content-sync-tombstone"}),await y.close();continue}if(s==="content-sync")throw new Error(`Required content-sync route returned HTTP ${x.status()}: ${l}`);r.skip("page",l,x.status(),t.pageDiscoverySources.get(l)),await y.close();continue}C?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await F(`wait for rendered page ${l}`,ee(e),()=>Fn(e,y))):await F(`wait for rendered page ${l}`,ee(e),()=>En(e,y)),t.stats.pagesRendered++;let W=await F(`extract rendered assets ${l}`,ee(e),()=>qn(e,y,h,r,!C));for(let P of W)q(e,t,P,r,l);let S=[...new Set([...W,...j])],A=k??await F(`capture rendered html ${l}`,ee(e),()=>y.content()),M=s!=="single-url"&&s!=="content-sync"&&!v?Mn(e,l,A,r):[];if(s!=="single-url"&&s!=="content-sync"&&!v)for(let P of M)Ae(e,t,P,r,l);await ar(e,l,A,t,r);let D=te(e,l);if(Q(a,e,D),ce(a,D),i.trackManifest){let P=x?.headers();i.manifest.pages[l]={url:l,outputPath:D,contentType:it(P),changeToken:$.changeToken?.supported?$.changeToken.token:null,tokenSource:$.changeToken?.supported?$.changeToken.tokenSource??null:null,sitemapLastmod:Re(t.sitemapLastmodByPage[l])??null,discoveredPages:M,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await oe(e,i);}}catch(g){if(m=g instanceof Te,i.strictTargeted)throw r.error(`Failed required content-sync page ${l}`,{url:l,error:String(g)}),g;if(m)r.warn(`Skipped partial DOM for timed-out page ${l}`,{url:l,error:String(g),recovery:"recycle-worker-browser"});else try{let h=y?await F(`capture partial html ${l}`,ee(e),()=>y.content()):"";if(h&&h.trim()){await ar(e,l,h,t,r);let v=te(e,l);if(Q(a,e,v),ce(a,v),i.trackManifest){let C=i.manifest.pages[l]??i.previousManifest.pages[l];i.manifest.pages[l]={url:l,outputPath:v,contentType:C?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:Re(t.sitemapLastmodByPage[l])??C?.sitemapLastmod??null,discoveredPages:C?.discoveredPages?[...C.discoveredPages]:[],discoveredAssets:C?.discoveredAssets?[...C.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await oe(e,i);}r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(g)});}}catch(h){h instanceof Te&&(m=!0),r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(h)});}r.error(`Failed page ${l}`,{url:l,error:String(g)});}finally{if(!m&&y&&await F(`drain network routes ${l}`,Ce(e),()=>y.unrouteAll({behavior:"wait"})).catch(g=>{m=!0,r.warn(`Could not drain network routes for ${l}`,{url:l,browserPid:c.pid,error:String(g),recovery:"recycle-worker-browser"});}),m&&(o=null,await nt(d,r,`page operation timeout for ${l}`),await d.close().catch(()=>{})),f.size>0&&await Promise.allSettled([...f]),!m&&p)try{let g=await p.cookies();u.addAll(Xt(b,g));}catch{}t.stats.pagesCompleted++,Nn(t)&&Bn(t,r),!m&&y&&await F(`close page ${l}`,Ce(e),()=>y.close()).catch(g=>{m=!0,r.warn(`Timed out while closing page ${l}`,{url:l,browserPid:c.pid,error:String(g),recovery:"recycle-worker-browser"});}),!m&&p&&await F(`close page context ${l}`,Ce(e),()=>p.close()).catch(g=>{m=!0,r.warn(`Timed out while closing page context for ${l}`,{url:l,browserPid:c.pid,error:String(g),recovery:"recycle-worker-browser"});}),m&&o===d&&(o=null,await nt(d,r,`page cleanup timeout for ${l}`),await d.close().catch(()=>{}));}}}finally{if(o){let d=o;await F("close worker browser",Ce(e),()=>d.close()).catch(l=>(r.warn("Timed out while closing crawl worker browser",{browserPid:c.pid,error:String(l)}),nt(d,r,"browser.close timeout")));}}}async function Hn(e){try{let t=await R.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function ur(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function zn(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Jn(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function Ar(e){try{let t=await R.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function cr(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Vn(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>cr(s)-cr(n)).map(s=>w.join(e,String(s.storedFileName))):[];return [...new Set([...r,w.join(e,"errors.jsonl"),w.join(e,"errors.json"),w.join(e,"errors.jsonl.gz"),w.join(e,"errors.json.gz")])]}async function lr(e){let t=await Ar(w.join(e,"job.json"));for(let r of Vn(e,t)){let s=await Hn(r);if(s.length>0)return s}return []}async function Gn(e){let t=w.join(e.logDir,"archive"),r;try{r=await R.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=w.join(t,i);if(!(await R.stat(a).catch(()=>null))?.isDirectory())continue;let o=await Ar(w.join(a,"job.json"));if(!zn(o))continue;let c=Jn(o);c>=n&&(n=c,s=a);}return s}async function Kn(e){let t=await Gn(e),r=t!==""?await lr(t):await lr(e.logDir),s=new Set;for(let n of r)if(ur(n.error)||ur(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Zn(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await R.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Yn(e){if(!e)throw new Error("Content-sync crawl requires an immutable impact plan.");let t=JSON.parse(await R.readFile(e,"utf8"));return Rt(t),t}function Xn(e,t){let r;try{r=new URL(e);}catch{return false}return t.archiveFamilies.some(s=>{let n=new URL(s.url);if(r.origin!==n.origin)return false;if(r.toString()===n.toString())return true;if(r.searchParams.has("paged")&&r.pathname===n.pathname)return /^\d+$/.test(r.searchParams.get("paged")||"");let i=n.pathname.replace(/\/+$/,""),a=r.pathname.slice(i.length).replace(/^\/+/,""),u=String(s.paginationBase||"page").replace(/^\/+|\/+$/g,"").replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return r.pathname.startsWith(`${i}/`)&&new RegExp(`^${u}/\\d+/?$`).test(a)})}function ve(e,t,r){let s=r==="page"?te(e,t):_e(e,t).originalFilePath;return K(e.outputDir,s)}async function dt(e,t,r,s,n){let i=t.manifest.pages[s];if(!i)return;let a=K(e.outputDir,i.outputPath);if(!a)throw new Error(`Manifest-owned page path escaped output root: ${s}`);let u=w.resolve(i.outputPath);Object.entries(t.manifest.pages).some(([c,d])=>c!==s&&!n.has(c)&&w.resolve(d.outputPath)===u)||(await R.rm(i.outputPath,{force:true}),r.deletedFiles.add(a)),delete t.manifest.pages[s];}async function ei(e,t,r,s,n){let i=t.manifest.assets[s];if(!i)return;let a=K(e.outputDir,i.outputPath);if(!a)throw new Error(`Manifest-owned resource path escaped output root: ${s}`);let u=w.resolve(i.outputPath);Object.entries(t.manifest.assets).some(([c,d])=>c!==s&&!n.has(c)&&w.resolve(d.outputPath)===u)||(await R.rm(i.outputPath,{force:true}),r.deletedFiles.add(a)),delete t.manifest.assets[s];}function ti(e,t){let r=new Set,s=[...t];for(;s.length>0;){let n=s.shift();if(r.has(n))continue;r.add(n);let i=e.assets[n];for(let a of i?.discoveredAssets??[])try{we(new URL(a))&&s.push(a);}catch{}}return r}async function ri(e,t,r,s,n){let i=ti(r.previousManifest,n);for(let a of i){if(t.successfulSitemaps.has(a))continue;let u=r.previousManifest.assets[a];if(!u)continue;let o=K(e.outputDir,u.outputPath);if(!o)throw new Error(`Manifest-owned sitemap path escaped output root: ${a}`);await R.rm(u.outputPath,{force:true}),s.deletedFiles.add(o),delete r.manifest.assets[a];}await oe(e,r);}async function si(e){try{let t=await R.readFile(w.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ni(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await R.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await R.unlink(w.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function dr(e,t,r,s,n){let i=N(e,r,e.sourceOrigin,s,"manual-url");i&&(we(i)?Be(e,t,i.toString(),s,n):O(e,i)?q(e,t,i.toString(),s,n):ue(i)?Ae(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function xr(e=process.argv.slice(2)){let t=[],r=await Me(void 0,{onRemotePublisherConfigDiagnostic:S=>{t.push(S);}}),s=kt(r.sitemapPaths),n=t.at(-1)??null,i=Os(e),a=i.mode==="content-sync"?await Yn(i.contentSyncPlanPath):null;if(i.resumeRewrite&&i.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let u=[];i.mode==="retry-timeouts"&&(u=await Kn(r)),i.mode==="single-url"&&(u=await Zn(i.urls)),a&&(u=[...a.directRenderUrls,...a.archiveFamilyUrls,...a.resourceRenderUrls]);let o=i.crawlMode==="incremental"&&i.mode==="full",c=o&&typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!=="";if(a&&r.subscriptionType!=="PROFESSIONAL"&&r.subscriptionType!=="AGENCY")throw new Error("Content-sync requires an active Professional or Agency subscription.");let d=(i.mode==="full"||i.mode==="single-url"||i.mode==="content-sync")&&!i.resumeRewrite,l=i.mode!=="full"||i.resumeRewrite||c,p=l?await si(r):{},y=c||i.mode==="single-url"||i.mode==="content-sync"?await Bs(r):hr(),m=new Set(u),b=new Set;if(a){for(let S of a.archiveFamilies)if(S.foundPosts===0)for(let A of S.pageUrls)b.add(A);for(let S of Object.keys(y.pages))Xn(S,a)&&!m.has(S)&&(u.push(S),b.add(S));}let f={enabled:c,trackManifest:d,manifest:at(y),previousManifest:at(y),runId:a?.jobId??`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve(),strictTargeted:!!a,requiredUrls:m,deleteOnMissingUrls:b};await ni(r.logDir);let g=new de(r.logDir,r.logLevel);if(n){let S={status:n.status,virtualAssetBaseUrl:n.virtualAssetBaseUrl,subscriptionType:n.subscriptionType??null,error:n.error,requests:n.requests,source:"remote-publisher-config"};n.status==="loaded"&&n.subscriptionType?g.info("Loaded remote WP Suite publisher configuration.",S):g.warn("Remote WP Suite publisher configuration is unavailable or has no active subscription type.",S);}if(o&&!c){let S="Incremental crawl requires an active remote publisher subscription; refusing to fall back to a destructive full crawl.";throw g.error(S,{requestedMode:i.crawlMode,subscriptionType:r.subscriptionType??null,source:"incremental-subscription-guard"}),await g.flush(),new Error(S)}let h={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,successfulSitemaps:new Set,resolvedSitemapRoots:new Set,sitemapIndexUrls:new Set,sitemapLeafUrls:new Set,sitemapLastmodByPage:{},assetMap:{...p},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},v=Ns(),C=new De,L=Ds();if(L&&!i.resumeRewrite&&await Ee(L,C.artifact()),await bt(r),await vt(r,{startedAt:new Date().toISOString(),outputDir:w.resolve(r.outputDir)}),l||await R.rm(r.outputDir,{recursive:true,force:true}),await R.mkdir(r.outputDir,{recursive:true}),a){let S=new Set([...a.deleteUrls,...f.deleteOnMissingUrls]);for(let M of a.deleteUrls)await dt(r,f,v,M,S);let A=new Set(a.resourceDeleteUrls);for(let M of a.resourceDeleteUrls)await ei(r,f,v,M,A);}if(i.resumeRewrite)g.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:i.mode,crawlMode:i.crawlMode});else {let S=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{if(g.mark("discovery"),i.mode==="full"?(s.forEach(M=>Be(r,h,M,g)),await or(r,S,h,g,v,f),r.seedPaths.forEach(M=>dr(r,h,M,g,"seed-path")),r.generated404RequestPath&&Ae(r,h,r.generated404RequestPath,g,"generated-404-request-path")):(u.forEach(M=>dr(r,h,M,g,"cli")),a?.sitemapUrls.forEach(M=>Be(r,h,M,g,"content-sync-impact")),await or(r,S,h,g,v,f,i.mode==="content-sync",a?.sitemapUrls??[]),a&&await ri(r,h,f,v,a.sitemapUrls)),g.endMark("discovery",{pages:h.pageQueue.length,assets:h.assetQueue.length,sitemaps:h.sitemapQueue.length}),g.summary(`Queued ${h.pageQueue.length} pages, ${h.assetQueue.length} assets, ${h.sitemapQueue.length} sitemaps.`,{mode:i.mode,crawlMode:i.crawlMode,queuedPages:h.pageQueue.length,queuedAssets:h.assetQueue.length,queuedSitemaps:h.sitemapQueue.length}),i.mode==="retry-timeouts"&&h.pageQueue.length===0&&h.assetQueue.length===0&&h.sitemapQueue.length===0){g.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:i.mode,crawlMode:i.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await g.flush(),L&&await Ee(L,C.artifact());return}g.mark("render-pages");let A=Array.from({length:Math.max(1,r.concurrency)},()=>Qn(r,h,g,i.mode,S,f,v,C));if(await Promise.all(A),L&&await Ee(L,C.artifact()),g.endMark("render-pages",{pages:h.stats.pagesCompleted}),g.mark("download-assets"),g.progress("Asset download phase started.",{phase:"download-assets",doneAssets:h.doneAssets.size,assetsQueued:h.stats.assetsQueued,assetQueue:h.assetQueue.length}),await Qs(r,S,f,[...h.assetQueue],g),await Un(r,S,h,g,f,v),g.endMark("download-assets",{assets:h.doneAssets.size}),f.enabled&&r.maxPages===0&&(await Gs(r,h,f,g,v),i.mode==="full"&&await Ks(r,f,g,v)),i.mode==="full"){let M=await In(r,h,g);for(let D of M)Q(v,r,D);}}finally{await F("dispose crawl request context",Ce(r),()=>S.dispose()).catch(A=>{g.warn("Timed out while disposing crawl request context",{error:String(A)});});}}f.trackManifest&&await oe(r,f),await R.writeFile(w.join(r.outputDir,"asset-map.json"),JSON.stringify(h.assetMap,null,2),"utf8"),JSON.stringify(p,null,2)!==JSON.stringify(h.assetMap,null,2)&&Q(v,r,w.join(r.outputDir,"asset-map.json"));let $=i.mode==="single-url"||i.mode==="content-sync"?on(r,h):f.enabled&&!i.resumeRewrite?Zs(r,f,v,p,h.assetMap):void 0;g.mark("rewrite-text");let B=i.mode==="single-url"||i.mode==="content-sync"?`Rewriting text files touched by URL crawl: 0/${$?.length??0}`:"Rewriting text files...";g.progress(B,{phase:"rewrite-text",index:0,totalFiles:$?.length,changedTextFiles:0}),await nr(B);let j=Date.now(),x=await jt(r,h.assetMap,{files:$,previousAssetMap:p,onStart:({totalFiles:S,workerCount:A})=>{g.info(`Rewrite phase scanning ${S} text files.`,{phase:"rewrite-text",totalFiles:S,workerCount:A});},onProgress:async({index:S,totalFiles:A,changedTextFiles:M,file:D,changed:P})=>{P&&Q(v,r,w.join(r.outputDir,D)),g.checkpoint(`Rewriting text file ${S}/${A}`,{phase:"rewrite-text",index:S,totalFiles:A,changedTextFiles:M,file:D}),g.debug(`${P?"Rewrote":"Checked unchanged"} text file ${S}/${A}: ${D}`,{phase:"rewrite-text",index:S,totalFiles:A,changedTextFiles:M,file:D,changed:P});let I=Date.now();if(!(S===1||S===A||S%50===0||I-j>=5e3))return;j=I;let _=`Rewriting text files: ${S}/${A}`;g.progress(_,{phase:"rewrite-text",index:S,totalFiles:A,changedTextFiles:M,file:D}),await nr(_);}});f.trackManifest&&(Ys(f,$),await oe(r,f)),g.endMark("rewrite-text",{changedTextFiles:x});let k=a?[...v.deletedFiles].sort():f.enabled&&!i.resumeRewrite?await en(r,h,f,v,g):[],W=[...new Set(($||[]).map(S=>K(r.outputDir,w.isAbsolute(S)?S:w.join(r.outputDir,S))).filter(S=>S!==null))].sort();if(a){let S=[...v.changedFiles].sort(),A=new Set(S),M=new Set(a.directRenderUrls.map(T=>ve(r,T,"page")).filter(T=>!!T&&A.has(T))),D=new Set(a.archiveFamilyUrls.map(T=>ve(r,T,"page")).filter(T=>!!T&&A.has(T))),P=new Set(a.resourceRenderUrls.map(T=>ve(r,T,"asset")).filter(T=>!!T&&A.has(T))),I=new Set([...h.sitemapLeafUrls].map(T=>ve(r,T,"asset")).filter(T=>!!T&&A.has(T))),U=new Set([...h.sitemapIndexUrls].map(T=>ve(r,T,"asset")).filter(T=>!!T&&A.has(T))),_=new Set([...M,...D,...P,...I,...U]),V=S.filter(T=>!_.has(T)),Ie=[...new Set([...a.directRenderUrls,...a.archiveFamilyUrls,...h.resolvedSitemapRoots,...h.sitemapLeafUrls,...h.sitemapIndexUrls,...a.deleteUrls,...a.resourceRenderUrls,...a.resourceDeleteUrls].map(T=>{let mt=new URL(T);return `${mt.pathname}${mt.search}`}))].sort();await Qe(r,{schemaVersion:2,generatedAt:new Date().toISOString(),outputDir:w.resolve(r.outputDir),runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,changedFiles:S,deletedFiles:k,rewriteTargets:W,consumerId:a.consumerId,fromSequence:a.fromSequence,toSequence:a.toSequence,impactHash:a.impactHash,resolvedSitemapUrls:[...h.resolvedSitemapRoots],uploadGroups:{assets:V,content:[...M].sort(),listings:[...D].sort(),resources:[...P].sort(),sitemapLeaves:[...I].sort(),sitemapIndexes:[...U].reverse()},invalidationPaths:Ie});}else await Qe(r,{schemaVersion:1,generatedAt:new Date().toISOString(),outputDir:w.resolve(r.outputDir),runMode:i.mode,crawlMode:i.crawlMode,fullSyncRequired:!(f.enabled&&!i.resumeRewrite),changedFiles:f.enabled&&!i.resumeRewrite?[...v.changedFiles].sort():[],deletedFiles:f.enabled&&!i.resumeRewrite?k:[],rewriteTargets:f.enabled&&!i.resumeRewrite?W:[]});await Ct(r),g.summary(i.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${x} text files.`:`Done. Rendered ${h.stats.pagesRendered} pages, processed ${h.doneSitemaps.size} sitemaps and ${h.doneAssets.size} assets.`,{mode:i.mode,crawlMode:i.crawlMode,resumeRewrite:i.resumeRewrite,...h.stats,pagesRendered:h.stats.pagesRendered,donePages:h.stats.pagesCompleted,claimedPages:h.donePages.size,doneSitemaps:h.doneSitemaps.size,doneAssets:h.doneAssets.size,changedTextFiles:x}),await g.flush();}xr().catch(async e=>{let t=X(e instanceof Error?e.message:String(e));console.error(t);try{let s=(await Me().catch(()=>null))?.logDir??"logs",n=new de(s,"debug");n.error(`Unhandled error: ${t}`,e instanceof Error&&e.stack?{stack:X(e.stack)}:void 0),await n.flush();}catch{}process.exit(1);});
|