@smart-cloud/publisher-exporter 1.1.49 → 1.1.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-sync.js +11 -8
- package/dist/cookie-observations.js +4 -0
- package/dist/crawl.js +9 -6
- package/dist/deploy.js +4 -4
- package/dist/invalidate.js +3 -3
- package/dist/queue-runner.js +3 -3
- package/package.json +3 -3
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import d from'fs/promises';import x from'path';function y(e){let t=String(e??"").toLowerCase();return t==="strict"?"Strict":t==="lax"?"Lax":t==="none"?"None":"Unspecified"}function g(e){let t=e.pathname||"/";if(!t.startsWith("/")||t==="/")return "/";let i=t.lastIndexOf("/");return i<=0?"/":t.slice(0,i)}function l(e){return `${e.name.toLowerCase()}
|
|
2
|
+
${e.domain.toLowerCase()}
|
|
3
|
+
${e.path}`}function N(e,t){let i;try{i=new URL(t);}catch{return null}let o=e.split(";"),a=o.shift()?.trim()??"",h=a.indexOf("=");if(h<=0)return null;let m=a.slice(0,h).trim();if(!m||[...m].some(s=>s<=" "||"=;,".includes(s)))return null;let C=i.hostname.toLowerCase(),b=g(i),f=null,c=true,S=false,O=false,w="Unspecified",k=false;for(let s of o){let p=s.indexOf("="),n=(p<0?s:s.slice(0,p)).trim().toLowerCase(),r=p<0?"":s.slice(p+1).trim();if(n==="domain"&&r)C=r.toLowerCase();else if(n==="path"&&r.startsWith("/"))b=r;else if(n==="expires"){let u=Date.parse(r);Number.isFinite(u)&&(f=Math.floor(u/1e3),c=false);}else if(n==="max-age"&&/^-?\d+$/u.test(r)){let u=Number.parseInt(r,10);f=Math.floor(Date.now()/1e3)+u,c=false;}else n==="secure"?S=true:n==="httponly"?O=true:n==="samesite"?w=y(r):n==="partitioned"&&(k=true);}return {name:m,domain:C,path:b,expires:f,session:c,secure:S,httpOnly:O,sameSite:w,partitioned:k}}function L(e){let t=!Number.isFinite(e.expires)||e.expires<=0;return {name:e.name,domain:e.domain.toLowerCase(),path:e.path||"/",expires:t?null:e.expires,session:t,secure:e.secure,httpOnly:e.httpOnly,sameSite:y(e.sameSite),partitioned:e.partitioned===true}}function A(e,t){let i=new Map(e.map(o=>[l(o),o.value]));return t.filter(o=>i.get(l(o))!==o.value).map(L)}function M(e,t){let i=e.session||t.session;return {...e,expires:i?null:Math.max(e.expires??0,t.expires??0)||null,session:i,secure:e.secure&&t.secure,httpOnly:e.httpOnly&&t.httpOnly,sameSite:e.sameSite===t.sameSite?e.sameSite:"Unspecified",partitioned:e.partitioned&&t.partitioned}}var v=class{#e=new Map;add(t){if(!t)return;let i={...t,domain:t.domain.toLowerCase(),path:t.path||"/"},o=l(i),a=this.#e.get(o);this.#e.set(o,a?M(a,i):i);}addAll(t){for(let i of t)this.add(i);}artifact(){return {schemaVersion:1,observations:[...this.#e.values()].sort((t,i)=>l(t).localeCompare(l(i)))}}};async function B(e,t){await d.mkdir(x.dirname(e),{recursive:true});let i=`${e}.${process.pid}.tmp`;await d.writeFile(i,`${JSON.stringify(t,null,2)}
|
|
4
|
+
`,{encoding:"utf8",mode:384}),await d.rename(i,e);}export{v as CookieObservationCollector,A as diffBrowserCookieMetadata,N as parseSetCookieMetadata,B as writeCookieObservationArtifact};
|
package/dist/crawl.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import v from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import St,{createHash}from'crypto';import Pr from'http';import Sr from'https';import Dr from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var ur=".deploy-plan.json",cr="deploy-plan.json",lr=".crawl-incomplete.json",dr="crawl-incomplete.json";function it(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function ot(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim(),r=f.posix.normalize(t);return !t||!r||r==="."||/^[a-zA-Z]:/.test(r)||r.startsWith("../")||r===".."?null:r}function pr(e){let t=String(e||"").trim();if(!t.startsWith("/")||t.startsWith("//")||t.includes("*")||t.includes("#"))return null;try{let r=new URL(t,"https://content-sync.invalid");return r.origin!=="https://content-sync.invalid"?null:`${f.posix.normalize(r.pathname)}${r.search}`}catch{return null}}function G(e){return [...new Set(e.map(ot).filter(t=>t!==null))]}function gr(e,t){let r={generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:G(t.changedFiles||[]),deletedFiles:G(t.deletedFiles||[]),rewriteTargets:G(t.rewriteTargets||[])};return t.schemaVersion===2?{...r,schemaVersion:2,runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,consumerId:String(t.consumerId||"").trim(),fromSequence:Math.max(0,Number(t.fromSequence)||0),toSequence:Math.max(0,Number(t.toSequence)||0),impactHash:String(t.impactHash||"").trim(),uploadGroups:{assets:G(t.uploadGroups?.assets||[]),content:G(t.uploadGroups?.content||[]),listings:G(t.uploadGroups?.listings||[]),sitemapLeaves:G(t.uploadGroups?.sitemapLeaves||[]),sitemapIndexes:G(t.uploadGroups?.sitemapIndexes||[])},invalidationPaths:[...new Set((t.invalidationPaths||[]).map(pr).filter(s=>s!==null))].sort()}:{schemaVersion:1,...r,runMode:t.runMode}}function at(e){let t=f.join(f.resolve(e.outputDir),ur),r=it();return r?[f.join(r,cr),t]:[t]}function ut(e){let t=f.join(f.resolve(e.outputDir),lr),r=it();return r?[f.join(r,dr),t]:[t]}function Z(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return ot(n)}async function Fe(e,t){let r=gr(e,t);for(let s of at(e)){await v.mkdir(f.dirname(s),{recursive:true});let n=`${s}.${process.pid}.${Date.now()}.tmp`;await v.writeFile(n,JSON.stringify(r,null,2),"utf8"),await v.rename(n,s);}}async function ct(e){for(let t of at(e))try{await v.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}async function lt(e,t){let r={schemaVersion:1,startedAt:String(t.startedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir)};for(let s of ut(e)){await v.mkdir(f.dirname(s),{recursive:true});let n=`${s}.${process.pid}.tmp`;await v.writeFile(n,JSON.stringify(r,null,2),"utf8"),await v.rename(n,s);}}async function dt(e){for(let t of ut(e))try{await v.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function fr(e){return new URL(e).pathname.split("/").filter(Boolean).some(r=>{try{return decodeURIComponent(r).endsWith("__trashed")}catch{return r.endsWith("__trashed")}})}function Ee(e){return Array.isArray(e)?e.map(Ee):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[r])=>t.localeCompare(r)).map(([t,r])=>[t,Ee(r)])):e}function mr(e){return JSON.stringify(Ee(e))}function wr(e){return createHash("sha256").update(mr(e)).digest("hex")}function pt(e){let{impactHash:t,...r}=e;if(e.schemaVersion!==1||!t)throw new Error("Invalid content-sync impact plan schema.");if(wr(r)!==t)throw new Error("Content-sync impact plan hash mismatch.");for(let s of [...e.directRenderUrls,...e.deleteUrls])if(fr(s))throw new Error(`Content-sync impact contains a WordPress trash alias instead of the last public URL: ${s}`)}function ht(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function br(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(u=>u.replace(/[^A-Za-z0-9._-]/g,"")).filter(u=>u.length>0&&u!=="."&&u!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&f.extname(i)===""&&(i+="/"),i==="/"?"":i}function Oe(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function vr(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,u={};if(typeof i.targetOrigin=="string"){let o=ht(i.targetOrigin);o&&(u.targetOrigin=o);}let c=Oe(i.extraReplacements);if(Object.keys(c).length>0&&(u.extraReplacements=c),i.s3&&typeof i.s3=="object"){let o={},a=i.s3;typeof a.bucket=="string"&&(o.bucket=a.bucket.trim()),typeof a.prefix=="string"&&(o.prefix=a.prefix.trim()),typeof a.region=="string"&&(o.region=a.region.trim()),typeof a.htmlCacheControl=="string"&&(o.htmlCacheControl=a.htmlCacheControl.trim()),typeof a.assetCacheControl=="string"&&(o.assetCacheControl=a.assetCacheControl.trim()),Object.keys(o).length>0&&(u.s3=o);}if(i.cloudFront&&typeof i.cloudFront=="object"){let o={},a=i.cloudFront;typeof a.distributionId=="string"&&(o.distributionId=a.distributionId.trim()),Array.isArray(a.invalidationPaths)&&(o.invalidationPaths=a.invalidationPaths.map(l=>String(l??"").trim()).filter(l=>l.length>0)),Object.keys(o).length>0&&(u.cloudFront=o);}t[n]=u;}return t}function qe(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Cr(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url"&&s!=="content-sync")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let u=String(r.deploymentProfile??"").trim(),c=String(r.url??"").trim(),o=Array.isArray(r.postTypes)?[...new Set(r.postTypes.map(l=>String(l??"").trim().toLowerCase().replace(/[^a-z0-9_-]/g,"")).filter(Boolean))]:[],a=Array.isArray(r.listingPaths)?[...new Set(r.listingPaths.map(l=>String(l??"").trim()).filter(Boolean))]:[];return s==="content-sync"&&o.length===0?null:{id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate"||s==="content-sync")&&u?{deploymentProfile:u}:{},...c?{url:c}:{},...o.length>0?{postTypes:o}:{},...a.length>0?{listingPaths:a}:{},...typeof r.includeSubsites=="boolean"?{includeSubsites:r.includeSubsites}:{},...typeof r.includePostTypeArchives=="boolean"?{includePostTypeArchives:r.includePostTypeArchives}:{},...typeof r.includeTaxonomyArchives=="boolean"?{includeTaxonomyArchives:r.includeTaxonomyArchives}:{},...typeof r.includeAuthorArchives=="boolean"?{includeAuthorArchives:r.includeAuthorArchives}:{},...typeof r.includeDateArchives=="boolean"?{includeDateArchives:r.includeDateArchives}:{},...typeof r.includePostsPage=="boolean"?{includePostsPage:r.includePostsPage}:{},...typeof r.includeSitemapChain=="boolean"?{includeSitemapChain:r.includeSitemapChain}:{}}}function ft(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Cr(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Rr(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function Ne(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Rr(t.siteSettings),u={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},c={},o=String(t.apiBase??"").trim();o&&(c.apiBase=o);let a=String(t.runtimeToken??t.nonce??"").trim();a&&(c.runtimeToken=a);let l=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();l&&(c.virtualAssetBaseUrl=l),Object.keys(u).length>0&&(c.siteSettings=u);let p=qe(t.subscriptionType);return p&&(c.subscriptionType=p),c}function Ar(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function mt(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function kr(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function wt(e,t,r=5){let s=new URL(mt(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Pr:Sr,u=kr(e,t);return await new Promise((c,o)=>{let a=i.request(s,{method:n,headers:Object.fromEntries(u.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},l=>{let p=[];l.on("data",d=>{p.push(Buffer.isBuffer(d)?d:Buffer.from(d));}),l.on("error",o),l.on("end",()=>{let d=l.statusCode??0,m=l.headers.location;if(r>0&&m&&[301,302,303,307,308].includes(d)){c(wt(new URL(m,s),{method:d===303?"GET":n},r-1));return}let S=new Headers;for(let[w,g]of Object.entries(l.headers))Array.isArray(g)?S.set(w,g.join(", ")):typeof g=="string"&&S.set(w,g);c(new Response(Buffer.concat(p),{status:d,statusText:l.statusMessage??"",headers:S}));});});a.on("error",o),a.end();})}function xr(e){let t=Ne(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{virtualAssetBaseUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Ar(e.sourceOrigin);let u=globalThis,o={...u.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(o.apiBase=t.apiBase),u.WpSuite=o,true}async function Tr(e){let t=String(e.wpsuite?.virtualAssetBaseUrl??e.wpsuite?.uploadUrl??"").trim();if(!xr(e)){let n=e.wpsuite?.siteSettings,i=[String(n?.accountId??"").trim()?"":"accountId",String(n?.siteId??"").trim()?"":"siteId",t?"":"virtualAssetBaseUrl"].filter(Boolean);return {remote:null,diagnostic:{status:"invalid-runtime",virtualAssetBaseUrl:t,subscriptionType:null,error:`Missing WP Suite runtime field(s): ${i.join(", ")}.`,requests:[]}}}let r=globalThis.fetch,s=[];try{typeof r=="function"&&/^https?:\/\//i.test(t)&&(globalThis.fetch=(async(o,a)=>{let l=mt(o);if(l.startsWith(t)){let p=(()=>{try{return new URL(l).pathname}catch{return l}})();try{let d=e.ignoreHttpsErrors&&l.startsWith("https://")?await wt(o,a):await r(o,a);return s.push({path:p,status:d.status,ok:d.ok,error:null}),d}catch(d){throw s.push({path:p,status:null,ok:!1,error:d instanceof Error?d.message:String(d)}),d}}return r(o,a)}));let{getConfig:n}=await import('@smart-cloud/wpsuite-core'),i=await n("publisher"),u=i&&typeof i=="object"?i:null,c=qe(u?.subscriptionType);return {remote:u,diagnostic:{status:u?"loaded":"unavailable",virtualAssetBaseUrl:t,subscriptionType:c??null,error:u?null:"WP Suite publisher config loader returned no configuration.",requests:s}}}catch(n){return {remote:null,diagnostic:{status:"unavailable",virtualAssetBaseUrl:t,subscriptionType:null,error:n instanceof Error?n.message:String(n),requests:s}}}finally{typeof r=="function"&&(globalThis.fetch=r);}}function Ir(e,t){let r=vr(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=qe(t?.subscriptionType),u=Ne(e.wpsuite);return {...e,scheduler:ft(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...u,...u.siteSettings||i?{siteSettings:{...u.siteSettings??{},...u.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function je(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Mr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function gt(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,je(s,r))}async function Ce(e,t={}){let r=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",s=await v.readFile(r,"utf8"),n=JSON.parse(s);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=ht(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=br(n.generated404RequestPath),n.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=Oe(n.extraReplacements),n.postCrawlCopyMap=Oe(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=Ne(n.wpsuite),n.scheduler=ft(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let i=await Tr(n);t.onRemotePublisherConfigDiagnostic?.(i.diagnostic);let u=Ir(n,i.remote),c=Mr(),o=c?f.resolve(c,".."):"";return o?(u.outputDir=gt(o,u.outputDir,"export"),u.logDir=gt(o,u.logDir,"logs")):(f.isAbsolute(u.outputDir)||(u.outputDir=je(u.outputDir,"export")),f.isAbsolute(u.logDir)||(u.logDir=je(u.logDir,"logs"))),u}function we(e,t=10){return St.createHash("sha1").update(e).digest("hex").slice(0,t)}async function $r(e){await v.mkdir(f.dirname(e),{recursive:true});}async function ye(e,t,r){await $r(e);let s=f.join(f.dirname(e),`.${f.basename(e)}.${process.pid}.${Date.now()}.${St.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await v.writeFile(s,t,r??"utf8"):await v.writeFile(s,t),await v.rename(s,e);}catch(n){throw await v.unlink(s).catch(()=>{}),n}}function X(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function Q(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function ae(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function Re(e){return e.replace(/\//g,"\\/")}function yt(e){return e.replace(/\//g,"\\\\/")}function Lr(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Pt(e,t,r){if(!t)return;e[t]=r;let s=Re(t),n=Re(r);e[s]=n;let i=yt(t),u=yt(r);e[i]=u;let c=ae(t),o=ae(r);e[c]=o;let a=ae(s),l=ae(n);e[a]=l;let p=ae(i),d=ae(u);e[p]=d;}function Ae(e,t,r){Pt(e,t,r);let s=Lr(t);s&&s!==t&&Pt(e,s,r);}function We(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function bt(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function vt(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Or=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],jr=new Set([".html",".htm"]),Ct="WPSuite.io Static Publisher",qr=Ct.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Nr(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function Wr(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function _r(e,t,r){return Wr(t)?e.replace(new RegExp(`(?<!:)${Nr(t)}`,"g"),r):e.split(t).join(r)}function Br(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=f.extname(r).toLowerCase();return jr.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Qr(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${qr}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ct}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import C from'fs/promises';import w from'path';import {gunzipSync}from'zlib';import xt,{createHash}from'crypto';import Ir from'http';import Mr from'https';import Br from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var yr=".deploy-plan.json",Sr="deploy-plan.json",Pr=".crawl-incomplete.json",br="crawl-incomplete.json";function dt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function pt(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim(),r=w.posix.normalize(t);return !t||!r||r==="."||/^[a-zA-Z]:/.test(r)||r.startsWith("../")||r===".."?null:r}function vr(e){let t=String(e||"").trim();if(!t.startsWith("/")||t.startsWith("//")||t.includes("*")||t.includes("#"))return null;try{let r=new URL(t,"https://content-sync.invalid");return r.origin!=="https://content-sync.invalid"?null:`${w.posix.normalize(r.pathname)}${r.search}`}catch{return null}}function Y(e){return [...new Set(e.map(pt).filter(t=>t!==null))]}function Cr(e,t){let r={generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:w.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:Y(t.changedFiles||[]),deletedFiles:Y(t.deletedFiles||[]),rewriteTargets:Y(t.rewriteTargets||[])};return t.schemaVersion===2?{...r,schemaVersion:2,runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,consumerId:String(t.consumerId||"").trim(),fromSequence:Math.max(0,Number(t.fromSequence)||0),toSequence:Math.max(0,Number(t.toSequence)||0),impactHash:String(t.impactHash||"").trim(),resolvedSitemapUrls:[...new Set((t.resolvedSitemapUrls||[]).map(s=>String(s||"").trim()).filter(s=>{try{let n=new URL(s);return n.protocol==="http:"||n.protocol==="https:"}catch{return false}}))],uploadGroups:{assets:Y(t.uploadGroups?.assets||[]),content:Y(t.uploadGroups?.content||[]),listings:Y(t.uploadGroups?.listings||[]),sitemapLeaves:Y(t.uploadGroups?.sitemapLeaves||[]),sitemapIndexes:Y(t.uploadGroups?.sitemapIndexes||[])},invalidationPaths:[...new Set((t.invalidationPaths||[]).map(vr).filter(s=>s!==null))].sort()}:{schemaVersion:1,...r,runMode:t.runMode}}function gt(e){let t=w.join(w.resolve(e.outputDir),yr),r=dt();return r?[w.join(r,Sr),t]:[t]}function ft(e){let t=w.join(w.resolve(e.outputDir),Pr),r=dt();return r?[w.join(r,br),t]:[t]}function ee(e,t){let r=w.resolve(e),s=w.resolve(t),n=w.relative(r,s).replace(/\\/g,"/");return pt(n)}async function _e(e,t){let r=Cr(e,t);for(let s of gt(e)){await C.mkdir(w.dirname(s),{recursive:true});let n=`${s}.${process.pid}.${Date.now()}.tmp`;await C.writeFile(n,JSON.stringify(r,null,2),"utf8"),await C.rename(n,s);}}async function ht(e){for(let t of gt(e))try{await C.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}async function mt(e,t){let r={schemaVersion:1,startedAt:String(t.startedAt||new Date().toISOString()),outputDir:w.resolve(e.outputDir)};for(let s of ft(e)){await C.mkdir(w.dirname(s),{recursive:true});let n=`${s}.${process.pid}.tmp`;await C.writeFile(n,JSON.stringify(r,null,2),"utf8"),await C.rename(n,s);}}async function wt(e){for(let t of ft(e))try{await C.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function Ar(e){return new URL(e).pathname.split("/").filter(Boolean).some(r=>{try{return decodeURIComponent(r).endsWith("__trashed")}catch{return r.endsWith("__trashed")}})}function We(e){return Array.isArray(e)?e.map(We):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[r])=>t.localeCompare(r)).map(([t,r])=>[t,We(r)])):e}function kr(e){return JSON.stringify(We(e))}function xr(e){return createHash("sha256").update(kr(e)).digest("hex")}function yt(e){let{impactHash:t,...r}=e;if(e.schemaVersion!==1||!t)throw new Error("Invalid content-sync impact plan schema.");if(xr(r)!==t)throw new Error("Content-sync impact plan hash mismatch.");for(let s of [...e.directRenderUrls,...e.deleteUrls])if(Ar(s))throw new Error(`Content-sync impact contains a WordPress trash alias instead of the last public URL: ${s}`)}var Be=["/sitemap_index.xml","/wp-sitemap.xml","/sitemap.xml"];function St(e){let t=new Set;for(let r of [...e,...Be]){let s=String(r||"").trim();s&&t.add(s);}return [...t]}function bt(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function $r(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&w.extname(i)===""&&(i+="/"),i==="/"?"":i}function Qe(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Lr(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,o={};if(typeof i.targetOrigin=="string"){let a=bt(i.targetOrigin);a&&(o.targetOrigin=a);}let c=Qe(i.extraReplacements);if(Object.keys(c).length>0&&(o.extraReplacements=c),i.s3&&typeof i.s3=="object"){let a={},l=i.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},l=i.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(u=>String(u??"").trim()).filter(u=>u.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[n]=o;}return t}function ze(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function Ur(e,t){if(!e||typeof e!="object")return null;let r=e,s=r.command;if(s!=="publish"&&s!=="crawl"&&s!=="deploy"&&s!=="invalidate"&&s!=="retry-timeouts"&&s!=="url"&&s!=="content-sync")return null;let n=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let i=String(r.id??`${s}-${t+1}`).trim();if(!i)return null;let o=String(r.deploymentProfile??"").trim(),c=String(r.url??"").trim(),a=Array.isArray(r.postTypes)?[...new Set(r.postTypes.map(u=>String(u??"").trim().toLowerCase().replace(/[^a-z0-9_-]/g,"")).filter(Boolean))]:[],l=Array.isArray(r.listingPaths)?[...new Set(r.listingPaths.map(u=>String(u??"").trim()).filter(Boolean))]:[];return s==="content-sync"&&a.length===0?null:{id:i,enabled:r.enabled!==false,command:s,intervalMinutes:n,...s==="publish"||s==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(s==="publish"||s==="deploy"||s==="invalidate"||s==="content-sync")&&o?{deploymentProfile:o}:{},...c?{url:c}:{},...a.length>0?{postTypes:a}:{},...l.length>0?{listingPaths:l}:{},...typeof r.includeSubsites=="boolean"?{includeSubsites:r.includeSubsites}:{},...typeof r.includePostTypeArchives=="boolean"?{includePostTypeArchives:r.includePostTypeArchives}:{},...typeof r.includeTaxonomyArchives=="boolean"?{includeTaxonomyArchives:r.includeTaxonomyArchives}:{},...typeof r.includeAuthorArchives=="boolean"?{includeAuthorArchives:r.includeAuthorArchives}:{},...typeof r.includeDateArchives=="boolean"?{includeDateArchives:r.includeDateArchives}:{},...typeof r.includePostsPage=="boolean"?{includePostsPage:r.includePostsPage}:{},...typeof r.includeSitemapChain=="boolean"?{includeSitemapChain:r.includeSitemapChain}:{}}}function vt(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((s,n)=>Ur(s,n)).filter(s=>!!s):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function Or(e){if(!e||typeof e!="object")return;let t=e,r={};for(let n of ["accountId","siteId"]){let i=t[n];typeof i=="string"&&i.trim()!==""&&(r[n]=i.trim());}let s=Number(t.lastUpdate??0);return Number.isFinite(s)&&s>0&&(r.lastUpdate=Math.floor(s)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function Je(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),s=String(t.siteId??"").trim(),n=t.subscriber===true,i=Or(t.siteSettings),o={...i??{},...r?{accountId:i?.accountId??r}:{},...s?{siteId:i?.siteId??s}:{},...i?.subscriber===true||n?{subscriber:true}:{}},c={},a=String(t.apiBase??"").trim();a&&(c.apiBase=a);let l=String(t.runtimeToken??t.nonce??"").trim();l&&(c.runtimeToken=l);let u=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();u&&(c.virtualAssetBaseUrl=u),Object.keys(o).length>0&&(c.siteSettings=o);let p=ze(t.subscriptionType);return p&&(c.subscriptionType=p),c}function Er(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Ct(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function Dr(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[s,n]of new Headers(t.headers).entries())r.set(s,n);return r}async function Rt(e,t,r=5){let s=new URL(Ct(e)),n=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),i=s.protocol==="http:"?Ir:Mr,o=Dr(e,t);return await new Promise((c,a)=>{let l=i.request(s,{method:n,headers:Object.fromEntries(o.entries()),...s.protocol==="https:"?{rejectUnauthorized:false}:{}},u=>{let p=[];u.on("data",d=>{p.push(Buffer.isBuffer(d)?d:Buffer.from(d));}),u.on("error",a),u.on("end",()=>{let d=u.statusCode??0,S=u.headers.location;if(r>0&&S&&[301,302,303,307,308].includes(d)){c(Rt(new URL(S,s),{method:d===303?"GET":n},r-1));return}let m=new Headers;for(let[P,g]of Object.entries(u.headers))Array.isArray(g)?m.set(P,g.join(", ")):typeof g=="string"&&m.set(P,g);c(new Response(Buffer.concat(p),{status:d,statusText:u.statusMessage??"",headers:m}));});});l.on("error",a),l.end();})}function Fr(e){let t=Je(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},s=String(r.accountId??"").trim(),n=String(r.siteId??"").trim(),i=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...i?{virtualAssetBaseUrl:i}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!s||!n||!i)return false;Er(e.sourceOrigin);let o=globalThis,a={...o.WpSuite??{},siteSettings:r,uploadUrl:i};return t.apiBase&&(a.apiBase=t.apiBase),o.WpSuite=a,true}async function jr(e){let t=String(e.wpsuite?.virtualAssetBaseUrl??e.wpsuite?.uploadUrl??"").trim();if(!Fr(e)){let n=e.wpsuite?.siteSettings,i=[String(n?.accountId??"").trim()?"":"accountId",String(n?.siteId??"").trim()?"":"siteId",t?"":"virtualAssetBaseUrl"].filter(Boolean);return {remote:null,diagnostic:{status:"invalid-runtime",virtualAssetBaseUrl:t,subscriptionType:null,error:`Missing WP Suite runtime field(s): ${i.join(", ")}.`,requests:[]}}}let r=globalThis.fetch,s=[];try{typeof r=="function"&&/^https?:\/\//i.test(t)&&(globalThis.fetch=(async(a,l)=>{let u=Ct(a);if(u.startsWith(t)){let p=(()=>{try{return new URL(u).pathname}catch{return u}})();try{let d=e.ignoreHttpsErrors&&u.startsWith("https://")?await Rt(a,l):await r(a,l);return s.push({path:p,status:d.status,ok:d.ok,error:null}),d}catch(d){throw s.push({path:p,status:null,ok:!1,error:d instanceof Error?d.message:String(d)}),d}}return r(a,l)}));let{getConfig:n}=await import('@smart-cloud/wpsuite-core'),i=await n("publisher"),o=i&&typeof i=="object"?i:null,c=ze(o?.subscriptionType);return {remote:o,diagnostic:{status:o?"loaded":"unavailable",virtualAssetBaseUrl:t,subscriptionType:c??null,error:o?null:"WP Suite publisher config loader returned no configuration.",requests:s}}}catch(n){return {remote:null,diagnostic:{status:"unavailable",virtualAssetBaseUrl:t,subscriptionType:null,error:n instanceof Error?n.message:String(n),requests:s}}}finally{typeof r=="function"&&(globalThis.fetch=r);}}function qr(e,t){let r=Lr(t?.deploymentProfiles),s=String(t?.defaultDeploymentProfile??"").trim(),n=r[s]?s:"",i=ze(t?.subscriptionType),o=Je(e.wpsuite);return {...e,scheduler:vt(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:n,...i?{subscriptionType:i}:{},wpsuite:{...o,...o.siteSettings||i?{siteSettings:{...o.siteSettings??{},...o.siteSettings?.subscriber===true||i==="PROFESSIONAL"||i==="AGENCY"?{subscriber:true}:{}}}:{},...i?{subscriptionType:i}:{}}}}function He(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Nr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function Pt(e,t,r){let s=String(t||"").trim();return s&&w.isAbsolute(s)?w.resolve(s):w.resolve(e,He(s,r))}async function ke(e,t={}){let r=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",s=await C.readFile(r,"utf8"),n=JSON.parse(s);n.sourceOrigin=n.sourceOrigin.replace(/\/$/,""),n.targetOrigin=bt(n.targetOrigin),n.ignoreHttpsErrors??=false,n.outputDir=String(n.outputDir||"export").trim()||"export",n.urlRewriteMode||=n.targetOrigin==="."?"relative":"absolute",n.noJavaScriptRenderPathPrefixes||=[],n.seedPaths||=[],n.generated404RequestPath=$r(n.generated404RequestPath),n.sitemapPaths||=[...Be],n.allowedAssetHosts||=[],n.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],n.blockedPathPrefixes||=[],n.blockedSearchFragments||=[],n.concurrency||=1,n.maxPages||=0,n.extraReplacements=Qe(n.extraReplacements),n.postCrawlCopyMap=Qe(n.postCrawlCopyMap),n.logDir=String(n.logDir||"logs").trim()||"logs",n.verbose??=false,n.logLevel||=n.verbose?"debug":"info",n.s3SyncMode||="sdk-upload-delete",n.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},n.readiness.timeoutMs??=1500,n.readiness.fallbackWaitMs??=1500,n.viewport||={width:1440,height:1200},n.navigationTimeoutMs||=3e4,n.assetDownloadConcurrency=Number(n.assetDownloadConcurrency)>0?Number(n.assetDownloadConcurrency):n.concurrency,n.rewriteConcurrency=Number(n.rewriteConcurrency)>0?Number(n.rewriteConcurrency):n.assetDownloadConcurrency,n.wpsuite=Je(n.wpsuite),n.scheduler=vt(void 0),n.deploymentProfiles={},n.defaultDeploymentProfile="",n.deploymentTargetOverride=String(n.deploymentTargetOverride??"").trim();let i=await jr(n);t.onRemotePublisherConfigDiagnostic?.(i.diagnostic);let o=qr(n,i.remote),c=Nr(),a=c?w.resolve(c,".."):"";return a?(o.outputDir=Pt(a,o.outputDir,"export"),o.logDir=Pt(a,o.logDir,"logs")):(w.isAbsolute(o.outputDir)||(o.outputDir=He(o.outputDir,"export")),w.isAbsolute(o.logDir)||(o.logDir=He(o.logDir,"logs"))),o}function ye(e,t=10){return xt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function _r(e){await C.mkdir(w.dirname(e),{recursive:true});}async function Se(e,t,r){await _r(e);let s=w.join(w.dirname(e),`.${w.basename(e)}.${process.pid}.${Date.now()}.${xt.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await C.writeFile(s,t,r??"utf8"):await C.writeFile(s,t),await C.rename(s,e);}catch(n){throw await C.unlink(s).catch(()=>{}),n}}function re(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(w.extname(e).toLowerCase())}function J(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function ce(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function xe(e){return e.replace(/\//g,"\\/")}function At(e){return e.replace(/\//g,"\\\\/")}function Wr(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function kt(e,t,r){if(!t)return;e[t]=r;let s=xe(t),n=xe(r);e[s]=n;let i=At(t),o=At(r);e[i]=o;let c=ce(t),a=ce(r);e[c]=a;let l=ce(s),u=ce(n);e[l]=u;let p=ce(i),d=ce(o);e[p]=d;}function Te(e,t,r){kt(e,t,r);let s=Wr(t);s&&s!==t&&kt(e,s,r);}function Ve(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function Tt(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function It(e,t,r){if(!t)return r;let s=w.dirname(w.resolve(t)),n=w.resolve(e,r.replace(/^\/+/,"")),i=w.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Jr=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Vr=new Set([".html",".htm"]),Mt="WPSuite.io Static Publisher",Gr=Mt.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Kr(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function Zr(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function Yr(e,t,r){return Zr(t)?e.replace(new RegExp(`(?<!:)${Kr(t)}`,"g"),r):e.split(t).join(r)}function Xr(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let n=w.extname(r).toLowerCase();return Vr.has(n)?/<head\b|<html\b|<!doctype html/i.test(t):false}function es(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Gr}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Mt}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>ts(t,r));}updateCurrentEvent(t,r,s){let n=ss(t,r,s);this.enqueueTask(async()=>{await v.mkdir(this.logDir,{recursive:true}),await v.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=H(r),i=He(s),u=JSON.stringify({time:new Date().toISOString(),level:t,message:n,...i||{}});this.enqueueLine(u),this.updateCurrentEvent(t,n,i),t==="error"?console.error(n,i||""):t==="warn"?console.warn(n,i||""):(ee[this.level]>=ee.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(n);}info(t,r){this.push("info",t,r);}debug(t,r){ee[this.level]<ee.debug||this.push("debug",t,r);}progress(t,r){es("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),ee[this.level]>=ee.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),ee[this.level]>=ee.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var ze="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)";function Tt(e){return !!e&&/^(?:\.{1,2}\/)?[A-Za-z0-9@_][A-Za-z0-9@._/-]*\.(?:js|mjs)(?:[?#][^\s<>\\]*)?$/i.test(e)&&!e.startsWith("/")&&!e.startsWith("//")&&!/^[a-z][a-z0-9+.-]*:/i.test(e)&&!/(?:^|\/)node_modules(?:\/|$)|(?:^|\/)\.pnpm(?:\/|$)/i.test(e)}function It(e){let t=new Set,r=[],s=/\bimport\s*\(\s*["'`]([^"'`\s<>\\]+)["'`]\s*\)/g,n=/\b(?:[A-Za-z_$][\w$]*\.)?u\s*=\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>\s*["'`]([^"'`\s<>\\]+)["'`]/g;for(let i of e.matchAll(s))i[1]&&Tt(i[1])&&r.push({index:i.index??0,value:i[1]});for(let i of e.matchAll(n))i[1]&&Tt(i[1])&&r.push({index:i.index??0,value:i[1]});for(let i of r.sort((u,c)=>u.index-c.index))t.add(i.value);return [...t]}function Mt(e,t){return typeof e=="string"&&e.length>0&&t?.supported===true&&typeof t.token=="string"&&t.token.length>0&&e===t.token}function $t(e,t){try{let r=new URL(e),s=new URL(t,r);if(s.origin!==r.origin)return !1;let n=r.pathname.replace(/\/+$/,"");if(n&&s.pathname!==n&&!s.pathname.startsWith(`${n}/`))return !1;let i=n?s.pathname.slice(n.length)||"/":s.pathname;return ["/wp-includes/","/wp-content/plugins/","/wp-content/themes/","/wp-content/mu-plugins/"].some(u=>i.startsWith(u))}catch{return false}}var as=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml"],Qt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Ht=new Set(["",".html",".htm"]),us=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),cs=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function ls(e){let t=[],r="full",s="full",n=false,i=false,u="";for(let c=0;c<e.length;c++){let o=e[c];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let a=e[++c];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let a=e[++c];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let a=e[++c];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}else if(o==="--content-sync-plan"){let a=e[++c];if(!a)throw new Error("--content-sync-plan requires a file path");r="content-sync",s="incremental",n=true,u=f.resolve(a);}else o.startsWith("--content-sync-plan=")&&(r="content-sync",s="incremental",n=true,u=f.resolve(o.slice(20)));}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i,contentSyncPlanPath:u}}function Xe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function ds(){let e=Xe();return e?f.resolve(e,".."):""}function ps(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function gs(e){let t=e.trim(),r=[{alias:"@storage-root",root:ds(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Xe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:ps(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function zt(e){let t=String(process.env.STATIC_PUBLISHER_CRAWL_MANIFEST_PATH||"").trim();if(t)return f.resolve(t);let r=Xe();return r?f.join(r,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function Jt(){return {schemaVersion:4,updatedAt:"",pages:{},assets:{}}}function Lt(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function hs(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true||!t}}function Ve(e){let t=e?.["content-type"]?.trim();return t||null}function fs(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function j(e,t,r){let s=Z(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Dt(e,t,r){let s=Z(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ie(e,t){e.rewriteTargets.add(f.resolve(t));}async function ms(e){try{let t=await v.readFile(zt(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3||r.schemaVersion===4)&&r.assets&&typeof r.assets=="object")return {schemaVersion:4,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Lt(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,hs(n)]))};if(r.schemaVersion===1)return {schemaVersion:4,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Lt(n)])),assets:{}}}}catch{}return Jt()}function Ge(e){return JSON.parse(JSON.stringify(e))}async function ws(e,t){let r=zt(e);await v.mkdir(f.dirname(r),{recursive:true}),await v.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function te(e,t){if(!t.trackManifest)return;let r=async()=>{let s=Ge(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await ws(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function Pe(e){if(!e)return null;let t=e.trim();return t||null}function ys(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function Vt(e,t,r,s){let n=[...new Set(r.map(c=>c.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let c=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!c.ok())return s.warn(`Change token lookup failed with HTTP ${c.status()}`,{endpoint:u,status:c.status()}),new Map;let o=await c.json().catch(()=>null);if(!o||!Array.isArray(o.items))return new Map;let a=new Map;for(let l of o.items)!l||typeof l.url!="string"||a.set(l.url,l);return a}catch(c){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:c instanceof Error?`${c.name}: ${c.message.split(/\r?\n/u,1)[0]}`:"Unknown change-token request failure"}),new Map}}async function Me(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await Vt(e,t,[s],n)).get(s)??null;return rt(e,s)&&(!u||u.supported!==true||!u.token)&&(u=await bs(e,t,s,n)),u=Gt(e,u),r.changeTokenCache.set(s,u),u}async function Ps(e,t,r,s,n){if(!r.trackManifest)return;let i=[...new Set(s.map(u=>u.trim()).filter(Boolean))].filter(u=>!r.changeTokenCache.has(u));for(let u=0;u<i.length;u+=250){let c=i.slice(u,u+250),o=await Vt(e,t,c,n);for(let a of c)r.changeTokenCache.set(a,Gt(e,o.get(a)??null));}}function Ss(e){return `subscriber=${e.wpsuite?.siteSettings?.subscriber===true?"1":"0"}`}function Gt(e,t){if(!t?.supported||!t.token||t.tokenSource==="wp-media-file")return t;let r=Ss(e);return {...t,token:we(`${t.token}
|
|
7
|
-
${
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function ts(e){let t=J(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function rs(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,xe(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!r.length)return e;let s=i=>{let o=i;for(let[c,a]of r)o=o.split(c).join(a);return o},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,c,a)=>`${o}${s(c)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,c,a,l)=>ts(a)?`${o}${s(a)}${l}`:i),n}function ss(e,t){if(!t)return null;let r=w.dirname(w.resolve(t)),s=w.resolve(e.outputDir),n=w.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function ns(e,t,r){let s=ss(t,r);if(!s)return e;let n=e;for(let i of Jr){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");n=n.replace(c,`${s}${i}`).replace(a,`${s}${i}`);let l=i.replace(/\//g,"\\/"),u=s.replace(/\//g,"\\/"),p=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),S=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${u}${l}`).replace(S,`${u}${l}`).replace(m,`${u}${l}`);}return n}function Ke(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:Tt(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${It(e.outputDir,r,n)}${i}`}function Ge(e,t,r,s,n){Te(e,r,Ke(t,s,n));}function is(e,t,r,s,n){if(!r||r===s)return;Te(e,r,s);let i=Ke(t,r,n),o=Ke(t,s,n);Te(e,i,o);}function Ze(e,t,r,s,n={}){let i=e,o={};Ge(o,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,l]of Object.entries(t.extraReplacements))Ge(o,t,a,l,s);for(let[a,l]of Object.entries(r))Ge(o,t,a,l,s);for(let[a,l]of Object.entries(n)){let u=r[a];!u||u===l||is(o,t,l,u,s);}let c=Object.entries(o).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of c)i=Yr(i,a,l);return t.urlRewriteMode!=="absolute"&&(i=ns(i,t,s)),i=rs(i,c.map(([,a])=>a)),Xr(t,i,s)&&(i=es(i)),i}async function os(e,t,r,s){let n=w.join(e.outputDir,s),i=w.extname(n).toLowerCase(),o;try{o=await C.readFile(n,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=Ze(o,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==o?(await Se(n,a,"utf8"),{changed:true}):{changed:false}}async function as(e,t){return await new Promise((r,s)=>{let n=a=>{if(c(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{c(),s(a);},o=a=>{c(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},c=()=>{e.off("message",n),e.off("error",i),e.off("exit",o);};e.on("message",n),e.on("error",i),e.on("exit",o),e.postMessage({file:t});})}async function $t(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>w.resolve(d)).filter(d=>d.startsWith(w.resolve(e.outputDir))).map(d=>w.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Br(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&re(d)),i=0;if(n.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),c=Math.max(1,Math.min(o,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:c});let a=0,l=0,u=null;if(c===1){for(let d of n){let S=await os(e,t,r.previousAssetMap??{},d);S.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:d,changed:S.changed});}return i}let p=Array.from({length:c},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!u;){let S=a;if(S>=n.length)return;a+=1;let m=n[S],P=await as(d,m);P.changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:m,changed:P.changed});}}catch(S){throw u=S instanceof Error?S:new Error(String(S)),u}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var se={error:0,warn:1,info:2,debug:3},us=/^(?:authorization|proxyAuthorization|runtimeToken|siteKey|nonce|accessToken|refreshToken|accessKeyId|secretAccessKey|sessionToken|xStaticPublisherToken|xApiKey)$/i;function V(e){return e.replace(/((?:x-static-publisher-token|authorization|proxy-authorization|x-api-key|aws-access-key-id|aws-secret-access-key|aws-session-token)\s*:\s*)[^\r\n]+/gi,"$1[REDACTED]").replace(/\b(Basic|Bearer)\s+[^\s]+/gi,"$1 [REDACTED]").replace(/([?&](?:token|access_token|refresh_token|api_key|key|nonce)=)[^&\s]+/gi,"$1[REDACTED]").replace(/((?:"|')?(?:runtimeToken|siteKey|nonce|accessToken|refreshToken|accessKeyId|secretAccessKey|sessionToken)(?:"|')?\s*[:=]\s*(?:"|'))[^"']+/gi,"$1[REDACTED]")}function Me(e){return typeof e=="string"?V(e):Array.isArray(e)?e.map(t=>Me(t)):e instanceof Error?{name:e.name,message:V(e.message),...e.stack?{stack:V(e.stack)}:{}}:e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([t,r])=>[t,us.test(t)?"[REDACTED]":Me(r)])):e}function Ye(e){return e?Me(e):void 0}var Lt=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",Ie=Lt?w.join(Lt,"current-progress.json"):"",Ut=Promise.resolve();function Ot(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function cs(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let o=r[n];if(typeof o=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(o,i);continue}r[n]=i;}return r}function ls(e,t,r){Ie&&(Ut=Ut.then(async()=>{let s=new Date,n=s.toISOString();await C.mkdir(w.dirname(Ie),{recursive:true});let i=await C.readFile(Ie,"utf8").then(h=>JSON.parse(h)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},c=Ot(e),a=i?.currentStep||Ot(i?.source||""),l=i?.startedAt||n,u={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==c&&i?.stepStartedAt){let h=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));u[a]=(u[a]??0)+h,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),S=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),m=V(t),P=Ye(r)??{},g={checkedAt:n,source:e,message:m,details:cs(o,P),startedAt:l,currentStep:c,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:S,stepDurationsSec:u};await C.writeFile(Ie,JSON.stringify(g,null,2),"utf8");}).catch(()=>{}));}async function ds(e,t){await C.mkdir(w.dirname(e),{recursive:true}),await C.appendFile(e,`${JSON.stringify(Me(t))}
|
|
5
|
+
`,"utf8");}function ps(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function gs(e,t,r){let s=Ye(r);return {checkedAt:new Date().toISOString(),currentStep:ps(e,s),level:e,message:V(t),details:s??{}}}var le=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return w.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return w.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return w.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return w.join(this.logDir,"ignored.jsonl")}get skippedPath(){return w.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return w.join(this.logDir,"errors.jsonl")}get timingsPath(){return w.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await C.mkdir(this.logDir,{recursive:true}),await C.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await C.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>ds(t,r));}updateCurrentEvent(t,r,s){let n=gs(t,r,s);this.enqueueTask(async()=>{await C.mkdir(this.logDir,{recursive:true}),await C.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=V(r),i=Ye(s),o=JSON.stringify({time:new Date().toISOString(),level:t,message:n,...i||{}});this.enqueueLine(o),this.updateCurrentEvent(t,n,i),t==="error"?console.error(n,i||""):t==="warn"?console.warn(n,i||""):(se[this.level]>=se.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(n);}info(t,r){this.push("info",t,r);}debug(t,r){se[this.level]<se.debug||this.push("debug",t,r);}progress(t,r){ls("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),se[this.level]>=se.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),se[this.level]>=se.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Xe="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)";function Et(e){return !!e&&/^(?:\.{1,2}\/)?[A-Za-z0-9@_][A-Za-z0-9@._/-]*\.(?:js|mjs)(?:[?#][^\s<>\\]*)?$/i.test(e)&&!e.startsWith("/")&&!e.startsWith("//")&&!/^[a-z][a-z0-9+.-]*:/i.test(e)&&!/(?:^|\/)node_modules(?:\/|$)|(?:^|\/)\.pnpm(?:\/|$)/i.test(e)}function Dt(e){let t=new Set,r=[],s=/\bimport\s*\(\s*["'`]([^"'`\s<>\\]+)["'`]\s*\)/g,n=/\b(?:[A-Za-z_$][\w$]*\.)?u\s*=\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>\s*["'`]([^"'`\s<>\\]+)["'`]/g;for(let i of e.matchAll(s))i[1]&&Et(i[1])&&r.push({index:i.index??0,value:i[1]});for(let i of e.matchAll(n))i[1]&&Et(i[1])&&r.push({index:i.index??0,value:i[1]});for(let i of r.sort((o,c)=>o.index-c.index))t.add(i.value);return [...t]}function Ft(e,t){return typeof e=="string"&&e.length>0&&t?.supported===true&&typeof t.token=="string"&&t.token.length>0&&e===t.token}function jt(e,t){try{let r=new URL(e),s=new URL(t,r);if(s.origin!==r.origin)return !1;let n=r.pathname.replace(/\/+$/,"");if(n&&s.pathname!==n&&!s.pathname.startsWith(`${n}/`))return !1;let i=n?s.pathname.slice(n.length)||"/":s.pathname;return ["/wp-includes/","/wp-content/plugins/","/wp-content/themes/","/wp-content/mu-plugins/"].some(o=>i.startsWith(o))}catch{return false}}function qt(e){return e<1}function Nt(e){let t=String(e??"").toLowerCase();return t==="strict"?"Strict":t==="lax"?"Lax":t==="none"?"None":"Unspecified"}function hs(e){let t=e.pathname||"/";if(!t.startsWith("/")||t==="/")return "/";let r=t.lastIndexOf("/");return r<=0?"/":t.slice(0,r)}function Pe(e){return `${e.name.toLowerCase()}
|
|
7
|
+
${e.domain.toLowerCase()}
|
|
8
|
+
${e.path}`}function _t(e,t){let r;try{r=new URL(t);}catch{return null}let s=e.split(";"),n=s.shift()?.trim()??"",i=n.indexOf("=");if(i<=0)return null;let o=n.slice(0,i).trim();if(!o||[...o].some(P=>P<=" "||"=;,".includes(P)))return null;let c=r.hostname.toLowerCase(),a=hs(r),l=null,u=true,p=false,d=false,S="Unspecified",m=false;for(let P of s){let g=P.indexOf("="),h=(g<0?P:P.slice(0,g)).trim().toLowerCase(),f=g<0?"":P.slice(g+1).trim();if(h==="domain"&&f)c=f.toLowerCase();else if(h==="path"&&f.startsWith("/"))a=f;else if(h==="expires"){let R=Date.parse(f);Number.isFinite(R)&&(l=Math.floor(R/1e3),u=false);}else if(h==="max-age"&&/^-?\d+$/u.test(f)){let R=Number.parseInt(f,10);l=Math.floor(Date.now()/1e3)+R,u=false;}else h==="secure"?p=true:h==="httponly"?d=true:h==="samesite"?S=Nt(f):h==="partitioned"&&(m=true);}return {name:o,domain:c,path:a,expires:l,session:u,secure:p,httpOnly:d,sameSite:S,partitioned:m}}function ms(e){let t=!Number.isFinite(e.expires)||e.expires<=0;return {name:e.name,domain:e.domain.toLowerCase(),path:e.path||"/",expires:t?null:e.expires,session:t,secure:e.secure,httpOnly:e.httpOnly,sameSite:Nt(e.sameSite),partitioned:e.partitioned===true}}function Wt(e,t){let r=new Map(e.map(s=>[Pe(s),s.value]));return t.filter(s=>r.get(Pe(s))!==s.value).map(ms)}function ws(e,t){let r=e.session||t.session;return {...e,expires:r?null:Math.max(e.expires??0,t.expires??0)||null,session:r,secure:e.secure&&t.secure,httpOnly:e.httpOnly&&t.httpOnly,sameSite:e.sameSite===t.sameSite?e.sameSite:"Unspecified",partitioned:e.partitioned&&t.partitioned}}var $e=class{#e=new Map;add(t){if(!t)return;let r={...t,domain:t.domain.toLowerCase(),path:t.path||"/"},s=Pe(r),n=this.#e.get(s);this.#e.set(s,n?ws(n,r):r);}addAll(t){for(let r of t)this.add(r);}artifact(){return {schemaVersion:1,observations:[...this.#e.values()].sort((t,r)=>Pe(t).localeCompare(Pe(r)))}}};async function Le(e,t){await C.mkdir(w.dirname(e),{recursive:true});let r=`${e}.${process.pid}.tmp`;await C.writeFile(r,`${JSON.stringify(t,null,2)}
|
|
9
|
+
`,{encoding:"utf8",mode:384}),await C.rename(r,e);}var bs=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml"],tr=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),rr=new Set(["",".html",".htm"]),vs=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Cs=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Rs(e){let t=[],r="full",s="full",n=false,i=false,o="";for(let c=0;c<e.length;c++){let a=e[c];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--resume-rewrite")n=true,i=true;else if(a==="--crawl-mode"){let l=e[++c];if(!l)throw new Error("--crawl-mode requires a value");s=l==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let l=e[++c];if(!l)throw new Error("--url requires a value");r="single-url",n=true,t.push(l);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let l=e[++c];if(!l)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${l}`);}else if(a==="--content-sync-plan"){let l=e[++c];if(!l)throw new Error("--content-sync-plan requires a file path");r="content-sync",s="incremental",n=true,o=w.resolve(l);}else a.startsWith("--content-sync-plan=")&&(r="content-sync",s="incremental",n=true,o=w.resolve(a.slice(20)));}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i,contentSyncPlanPath:o}}function qe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function As(){let e=qe();return e?w.join(e,"privacy","cookie-observations.json"):null}function ks(){let e=qe();return e?w.resolve(e,".."):""}function xs(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?w.resolve(e):""}function Ts(e){let t=e.trim(),r=[{alias:"@storage-root",root:ks(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:qe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:xs(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?w.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:w.resolve(t)}}function sr(e){let t=String(process.env.STATIC_PUBLISHER_CRAWL_MANIFEST_PATH||"").trim();if(t)return w.resolve(t);let r=qe();return r?w.join(r,"crawl-manifest.json"):w.join(e.outputDir,".crawl-manifest.json")}function nr(){return {schemaVersion:5,updatedAt:"",pages:{},assets:{}}}function Bt(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function Is(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),unreferencedRuns:Math.max(0,Number(e.unreferencedRuns)||0),rewriteComplete:e.rewriteComplete===true||!t}}function rt(e){let t=e?.["content-type"]?.trim();return t||null}function Ms(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function W(e,t,r){let s=ee(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Qt(e,t,r){let s=ee(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ue(e,t){e.rewriteTargets.add(w.resolve(t));}async function $s(e){try{let t=await C.readFile(sr(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3||r.schemaVersion===4||r.schemaVersion===5)&&r.assets&&typeof r.assets=="object")return {schemaVersion:5,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Bt(n)])),assets:Object.fromEntries(Object.entries(r.assets).map(([s,n])=>[s,Is(n)]))};if(r.schemaVersion===1)return {schemaVersion:5,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([s,n])=>[s,Bt(n)])),assets:{}}}}catch{}return nr()}function st(e){return JSON.parse(JSON.stringify(e))}async function Ls(e,t){let r=sr(e);await C.mkdir(w.dirname(r),{recursive:true}),await C.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function ne(e,t){if(!t.trackManifest)return;let r=async()=>{let s=st(t.manifest);s.updatedAt=new Date().toISOString(),t.manifest.updatedAt=s.updatedAt,await Ls(e,s);};t.manifestSaveChain=t.manifestSaveChain.then(r,r),await t.manifestSaveChain;}function be(e){if(!e)return null;let t=e.trim();return t||null}function Us(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],o=t[n];!i||!o||i===o||r.add(n);}return r}async function ir(e,t,r,s){let n=[...new Set(r.map(c=>c.trim()).filter(Boolean))],i=String(e.wpsuite?.runtimeToken||"").trim();if(n.length===0||!i)return new Map;let o=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let c=await t.post(o,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":i},data:JSON.stringify({urls:n})});if(!c.ok())return s.warn(`Change token lookup failed with HTTP ${c.status()}`,{endpoint:o,status:c.status()}),new Map;let a=await c.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let u of a.items)!u||typeof u.url!="string"||l.set(u.url,u);return l}catch(c){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:o,error:c instanceof Error?`${c.name}: ${c.message.split(/\r?\n/u,1)[0]}`:"Unknown change-token request failure"}),new Map}}async function Ee(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let o=(await ir(e,t,[s],n)).get(s)??null;return ct(e,s)&&(!o||o.supported!==true||!o.token)&&(o=await Ds(e,t,s,n)),o=or(e,o),r.changeTokenCache.set(s,o),o}async function Os(e,t,r,s,n){if(!r.trackManifest)return;let i=[...new Set(s.map(o=>o.trim()).filter(Boolean))].filter(o=>!r.changeTokenCache.has(o));for(let o=0;o<i.length;o+=250){let c=i.slice(o,o+250),a=await ir(e,t,c,n);for(let l of c)r.changeTokenCache.set(l,or(e,a.get(l)??null));}}function Es(e){return `subscriber=${e.wpsuite?.siteSettings?.subscriber===true?"1":"0"}`}function or(e,t){if(!t?.supported||!t.token||t.tokenSource==="wp-media-file")return t;let r=Es(e);return {...t,token:ye(`${t.token}
|
|
10
|
+
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function Ds(e,t,r,s){try{let n=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(n.status()!==404)return s.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:n.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let i=await n.text().catch(()=>""),o=ye(`${r}
|
|
8
11
|
404
|
|
9
|
-
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:u,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function vs(e,t,r,s,n,i){let u=rt(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Me(e,t,s,n,i):null};let c=s.manifest.pages[n];if(!c)return {action:"render",changeToken:await Me(e,t,s,n,i)};let o=await Me(e,t,s,n,i);if(o?.supported&&o.token)return c.changeToken===o.token?u&&!await Cs(e,t,n,i)?{action:"render",changeToken:o}:{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let a=Pe(r.sitemapLastmodByPage[n]);return a&&c.sitemapLastmod&&c.sitemapLastmod===a?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function Cs(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Rs(e,t,r,s,n){let i=Ut(r.previousManifest,void 0,s,"previous-manifest"),u=0;for(let[d,m]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Dt(n,e,m.outputPath);try{await v.unlink(m.outputPath);}catch(S){(S.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:m.outputPath,error:String(S)});}delete r.manifest.pages[d],u++;}let c=Ut(r.manifest,r.runId,s,"active-manifest"),o=new Set,a=new Set;for(let d of c){for(let w of Se(e,d))o.add(w);let m=r.manifest.assets[d]?.outputPath;m&&a.add(m);let S=de(e,t.assetMap,d);S&&a.add(S);}for(let d of Object.keys(r.manifest.assets))c.has(d)||delete r.manifest.assets[d];let l=0,p=0;for(let d of i){let m=new Set,S=r.previousManifest.assets[d]?.outputPath;S&&m.add(S);let w=de(e,t.assetMap,d);if(!c.has(d)&&w)m.add(w);else if(!c.has(d)){let g=Ws(e,d,s,"incremental-cleanup");g&&(m.add(g.originalFilePath),g.hashedFilePath&&m.add(g.hashedFilePath));}for(let g of m)if(!a.has(g)){Dt(n,e,g);try{await v.unlink(g),l++;}catch(h){(h.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:g,error:String(h)});}}for(let g of Se(e,d))o.has(g)||Object.prototype.hasOwnProperty.call(t.assetMap,g)&&(delete t.assetMap[g],p++);}(u>0||l>0||p>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${l} stale assets, ${p} stale asset mappings`,{removedPages:u,removedAssets:l,removedAssetMappings:p,phase:"incremental-cleanup"});}function Ut(e,t,r,s){let n=new Set,i=u=>{let c=u.trim();if(c){if(r){let o=re(c,r,"asset",s??"manifest");if(!o)return;n.add(o.toString());return}n.add(c);}};for(let u of Object.values(e.pages))for(let c of u.discoveredAssets||[])i(c);for(let[u,c]of Object.entries(e.assets||{}))t&&c.lastSeenRunId!==t||i(u);return n}function Ft(e,t,r){for(let s of t)for(let n of Se(e,s))if(r.has(n))return true;return false}function As(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let c of Object.values(t.manifest.pages))c.rewriteComplete||i.add(f.resolve(c.outputPath));for(let c of Object.values(t.manifest.assets))c.isText&&!c.rewriteComplete&&i.add(f.resolve(c.outputPath));let u=ys(s,n);if(u.size===0)return [...i];for(let c of Object.values(t.manifest.pages))Ft(e,c.discoveredAssets,u)&&i.add(f.resolve(c.outputPath));for(let[c,o]of Object.entries(t.manifest.assets)){o.isText&&Ft(e,o.discoveredAssets,u)&&i.add(f.resolve(o.outputPath));for(let a of Se(e,c))if(u.has(a)){j(r,e,o.outputPath),o.isText&&i.add(f.resolve(o.outputPath));break}}return [...i]}function ks(e,t){let r=t?new Set(Array.from(t,n=>f.resolve(n))):null,s=n=>r===null||r.has(f.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function xs(e,t,r){let s=new Set,n=i=>{if(!i)return;let u=Z(e.outputDir,i);u&&s.add(u);};for(let i of t.donePages)n(J(e,i));for(let i of Object.values(r.manifest.pages))i.lastSeenRunId===r.runId&&n(i.outputPath);for(let i of t.doneAssets)n(de(e,t.assetMap,i));for(let i of Object.values(r.manifest.assets))i.lastSeenRunId===r.runId&&n(i.outputPath);return s}async function Ts(e,t,r,s,n){let i=[...s.deletedFiles].sort(),u=xs(e,t,r),c=0;for(let l of [...u].sort()){try{if((await v.stat(f.join(e.outputDir,l))).isFile())continue}catch{}c++,n.warn(`Active output is missing after crawl: ${l}`,{phase:"deploy-plan",file:l,outputPath:f.join(e.outputDir,l)});}if(c>0&&n.warn(`Detected ${c} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:c}),i.length===0)return [];let o=[],a=0;for(let l of i){if(!u.has(l)){o.push(l);continue}a++;let p=f.join(e.outputDir,l),d;try{d=(await v.stat(p)).isFile();}catch{d=false;}n.warn(`Deploy plan protected active output from deletion: ${l}`,{phase:"deploy-plan",file:l,outputPath:p,exists:d});}return a>0&&n.info(`Deploy plan skipped ${a} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:a,deletedFiles:o.length}),o}function Se(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Is(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function Kt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,tt(s.pathname))}catch{return null}}function Ms(e,t,r){for(let s of Is(e,r)){let n=t[s];if(!n)continue;let i=Kt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function $s(e,t,r){for(let s of Se(e,r)){let n=t[s];if(!n)continue;let i=Kt(e,n);if(i)return i}return null}function de(e,t,r){let s=$s(e,t,r);if(s)return s;try{return Ue(e,r).originalFilePath}catch{return null}}function Ls(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function Et(e){let t=Ls();if(t)try{let r=await v.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await v.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Ds(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(J(e,s)));for(let s of t.doneAssets){let n=de(e,t.assetMap,s);!n||!X(n)||r.add(f.resolve(n));}for(let s of t.doneSitemaps){let n=de(e,t.assetMap,s);!n||!X(n)||r.add(f.resolve(n));}return [...r]}var Le="re:",Te=new Map;function Us(e){if(!e.startsWith(Le))return null;let t=Te.get(e);if(t!==void 0)return t||null;let r=e.slice(Le.length).trim();if(!r)return Te.set(e,false),null;try{let s=new RegExp(r);return Te.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Te.set(e,false),null}}function et(e,t){let r=Us(t);return r?r.test(e):t.startsWith(Le)?false:e.startsWith(t)}function Fs(e,t){return t.find(r=>et(e,r))}function Zt(e,t){let r=Fs(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Le)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Es(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Os(e,t){return Es(e).has(t.hostname)}function js(e){let t=Q(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Ke(e){let t=Q(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function qs(e){let t=Q(e).replace(/\\\//g,"/").trim(),r=f.extname(t).toLowerCase();return Qt.has(r)||Ht.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function pe(e){return cs.some(t=>e.pathname.startsWith(t))}function U(e,t,r=e.sourceOrigin,s,n="url"){let i=js(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Ke(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(qs(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(Yt(u))return s?.reject(n,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Os(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let c=Zt(e,u);if(c)return s?.reject(n,u.toString(),c,r),null;let o=new URL(e.sourceOrigin);return u.protocol=o.protocol,u.host=o.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function Yt(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function re(e,t,r,s){try{let n=new URL(e);return Yt(n)?(t.reject(r,e,"malformed URL encoding",s),null):n.protocol!=="http:"&&n.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${n.protocol}`,s),null):n}catch{return t.reject(r,e,"invalid URL",s),null}}function Xt(e){return Qt.has(f.extname(e.pathname).toLowerCase())}function se(e){return pe(e)?false:Ht.has(f.extname(e.pathname).toLowerCase())}function ge(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function T(e,t){return !!(ge(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>et(t.pathname,r))||Xt(t)||!se(t)&&!pe(t))}function J(e,t){let r=new URL(t),s=tt(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function tt(e){try{return decodeURIComponent(e)}catch{return e}}function Ns(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&f.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function rt(e,t){let r=Ns(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Ue(e,t){let r=new URL(t),s=tt(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=f.join(e.outputDir,i),c=We(e.targetOrigin,i),o=`${c}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:c,originalPublicUrlWithSearch:o,preferQueryHashed:n};let a=f.extname(s),p=`${a?s.slice(0,-a.length):s}.${we(r.search)}${a||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:c,originalPublicUrlWithSearch:o,preferQueryHashed:n,hashedPathname:p,hashedFilePath:f.join(e.outputDir,p),hashedPublicUrl:We(e.targetOrigin,p)}}function Ws(e,t,r,s){let n=re(t,r,"asset",s);return n?Ue(e,n.toString()):null}function _s(e,t,r,s,n){t[r]=s;try{let i=new URL(r),c=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{c.push(new URL(e.targetOrigin).origin);}catch{}for(let o of c)try{let a=new URL(o);t[a.origin+i.pathname+i.search]=s;}catch{}try{let o=(()=>{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${a.pathname}${a.search}`})();t[i.pathname+i.search]=o;}catch{}try{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}try{let o=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}}catch{}}async function er(e){try{return await v.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function le(e){try{return await v.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Bs(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function tr(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Qs(e,t){let r=tr(t),s=Bs(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function Hs(e,t){if(!t)return;let r=tr(t);if(r===null)return;let s=await le(e);if(!s)return;let n=new Date(r);await v.utimes(e,s.atime,n).catch(()=>{});}function Ot(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return us.has(r)||as.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function zs(e){try{let t=f.extname(new URL(e).pathname).toLowerCase();return t===".js"||t===".mjs"}catch{return false}}function ce(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset processing progress: processed ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function Js(e){return we(e.toString("base64"),40)}function Vs(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function Gs(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function Ze(e,t,r,s,n,i,u){let c=Ue(e,r),o=Js(s),a=c.originalFilePath,l=c.originalPublicUrlWithSearch;if(c.preferQueryHashed&&c.hashedFilePath&&c.hashedPublicUrl&&Vs(n,c.originalFilePath,o)){let S=Ms(e,t,r);S?(a=S.outputPath,l=S.publicUrl):(a=c.hashedFilePath,l=c.hashedPublicUrl);}_s(e,t,r,l,c.originalPublicUrl);let p=X(a)?Buffer.from(Qe(s.toString("utf8"),e,t,a),"utf8"):s,d=await er(a),m=false;return (d===null||!d.equals(p))&&(await ye(a,p),m=true),i?.debug(`${m?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:a,bytes:p.length,wroteFile:m}),Gs(n,c.originalFilePath,o),await Hs(a,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:l,wroteFile:m}}function be(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let d=new URL(p);return ge(d)?{discoveredFrom:p,discoveredFromType:"sitemap"}:T(e,d)?{discoveredFrom:p,discoveredFromType:"asset"}:se(d)&&!pe(d)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},u=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,c=U(e,r,u,s,"page");if(!c)return;if(!se(c)||Xt(c)){if(T(e,c)){s.info(`Seeded/non-page URL queued as asset: ${c.toString()}`,{url:c.toString(),source:n}),F(e,t,c.toString(),s,n);return}s.reject("page",c.toString(),pe(c)?"asset/internal path cannot be page":"not page-like",n);return}let o=c.toString(),{discoveredFrom:a,discoveredFromType:l}=i();a&&!t.pageDiscoverySources.has(o)&&t.pageDiscoverySources.set(o,a),!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,s.debug(`Queued discovered page ${o}`,{phase:"discovery",url:o,discoveredFrom:a,discoveredFromType:l,outputPath:J(e,o)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function F(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=U(e,r,i,s,"asset");if(!u)return;if(!T(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let c=u.toString();!t.doneAssets.has(c)&&!t.queuedAssets.has(c)&&(t.queuedAssets.add(c),t.assetQueue.push(c),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function De(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=U(e,r,i,s,"sitemap");if(!u)return;if(!ge(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let c=u.toString();!t.doneSitemaps.has(c)&&!t.queuedSitemaps.has(c)&&(t.queuedSitemaps.add(c),t.sitemapQueue.push(c),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Ks(e){let t=Q(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:Pe(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Zs(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Ys(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function Xs(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function en(e,t){if(t==="dom-style")return true;let r=(()=>{try{return f.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function Je(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function rr(e,t,r,s){let n=[];await v.mkdir(t,{recursive:true});let i=await v.readdir(e,{withFileTypes:true});for(let u of i){let c=f.join(e,u.name),o=f.join(t,u.name);if(u.isDirectory()){n.push(...await rr(c,o,r,s));continue}await sr(c,o,r,s)&&n.push(o);}return n}async function tn(e,t){try{let[r,s]=await Promise.all([v.stat(e),v.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([v.readFile(e),v.readFile(t)]);return n.equals(i)}catch{return false}}async function sr(e,t,r,s){let n=f.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await tn(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await v.mkdir(f.dirname(t),{recursive:true}),await v.copyFile(e,t),true)}async function rn(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([c,o])=>({sourcePath:c.trim(),prefix:String(o||"").trim()}));if(s.length===0)return [];let n=0,i=[],u=new Set([...t.donePages].map(c=>f.resolve(J(e,c))));r.mark("copy-extra-paths");for(let c of s){let o=c.sourcePath,a=c.prefix;if(!o||!a){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:o,prefixPath:a});continue}let l=gs(o);if(!l.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:o,alias:l.alias||"",requiredEnv:l.requiredEnv||"",prefixPath:a});continue}let p=l.resolvedPath,d=Je(a);if(!d){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:o,prefixPath:a});continue}let m;try{m=await v.stat(p);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:o,sourceAbs:p,prefixPath:a});continue}let S=f.resolve(e.outputDir,d);if(m.isDirectory()){let b=await rr(p,S,u,r);i.push(...b),n+=b.length,b.length>0&&r.progress(`Copied static directory to export: ${p} -> /${d}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${d}`,copiedItems:n,changedFiles:b.length});continue}let w=f.basename(p),g=a.endsWith("/")?f.join(S,w):S;await sr(p,g,u,r)&&(i.push(g),n++,r.progress(`Copied static file to export: ${p} -> /${Je(a.endsWith("/")?`${a}${w}`:a)}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${Je(a.endsWith("/")?`${a}${w}`:a)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function ve(e,t,r,s,n){let i=new Set,u=Q(r).replace(/\\\//g,"/"),c=n==="page-html"?Xs(u):u,o=en(t,n);if(o){for(let l of Ys(c)){let p=U(e,l,t,s,`${n}:css-url`);p&&T(e,p)&&i.add(p.toString());}for(let l of c.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let p=U(e,l[1],t,s,`${n}:css-import`);p&&T(e,p)&&i.add(p.toString());}}if(o||n==="page-html")for(let l of c.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let p=U(e,l[1],t,s,`${n}:xml-stylesheet`);p&&T(e,p)&&i.add(p.toString());}let a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${ze}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${ze}(?:\\?[^\\s'"<>\\);]*)?`,"gi");if(n==="page-html")for(let l of u.matchAll(a)){let p=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(Ke(p)){s.ignore(`${n}:serialized-url`,p,"looks like JavaScript/code fragment",t);continue}let d=U(e,p,t,s,`${n}:serialized-url`);d&&T(e,d)?i.add(d.toString()):d&&s.ignore("asset",d.toString(),"serialized URL did not pass safe asset path filter",t);}if(n!=="page-html"&&zs(t))for(let l of It(u)){if(Ke(l)||!/\.(?:js|mjs)(?:[?#]|$)/i.test(l))continue;let p=U(e,l,t,s,`${n}:relative-asset`);p&&T(e,p)&&i.add(p.toString());}return [...i]}function sn(e,t,r,s){let n=new Set,i=Q(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let c=u[1],o=U(e,c,t,s,"page-link");o&&(Zt(e,o)||se(o)&&!pe(o)&&!T(e,o)&&n.add(o.toString()));}return [...n]}async function jt(e,t,r,s,n,i,u=false){let c=new Map,o=(a,l)=>{let p=[l],d=new Set;for(;p.length>0;){let m=p.pop();if(m===a)return true;d.has(m)||(d.add(m),p.push(...c.get(m)??[]));}return false};for(;r.sitemapQueue.length>0;){let a=r.sitemapQueue.shift();r.queuedSitemaps.delete(a);let l=re(a,s,"sitemap","sitemap-queue");if(!l)continue;let p=l.toString();if(!r.doneSitemaps.has(p)){r.doneSitemaps.add(p),s.sitemap(`Fetching sitemap ${p}`,{url:p});try{let d=await t.get(p,{timeout:6e4});if(!d.ok()){if(s.skip("sitemap",p,d.status()),u)throw new Error(`Required content-sync sitemap returned HTTP ${d.status()}: ${p}`);continue}let m=await d.body(),S=d.url()||p,w=re(S,s,"sitemap-response",p);if(!w)continue;let g=w.toString(),h=await Ze(e,r.assetMap,g,m,r,s);j(n,e,h.outputPath),ie(n,h.outputPath);let b=m.toString("utf8");/<sitemapindex(?:\s|>)/i.test(b)?r.sitemapIndexUrls.add(g):r.sitemapLeafUrls.add(g);let L=ve(e,g,b,s,"sitemap");for(let I of L)F(e,r,I,s,g);let x=[];for(let I of Ks(b)){let R=U(e,I.loc,g,s,"sitemap-loc");if(!R){if(u)throw new Error(`Content-sync sitemap contains an inadmissible location: ${I.loc}`);continue}if(ge(R)){if(o(g,R.toString()))throw new Error(`Sitemap index cycle detected: ${g} -> ${R.toString()}`);c.has(g)||c.set(g,new Set),c.get(g).add(R.toString()),x.push(R.toString()),De(e,r,R.toString(),s,g);}else if(u){if(se(R)&&!T(e,R)){let O=i.requiredUrls.has(R.toString()),D=i.manifest.pages[R.toString()],P=!!D&&!!await le(D.outputPath);if(!O&&!P)throw new Error(`Content-sync sitemap coverage is incomplete for ${R.toString()}.`)}s.ignore("sitemap-loc",R.toString(),"content-sync sitemap-only traversal does not enqueue urlset locations",g);}else T(e,R)?F(e,r,R.toString(),s,g):se(R)?(I.lastmod&&(r.sitemapLastmodByPage[R.toString()]=I.lastmod),be(e,r,R.toString(),s,g)):s.reject("sitemap-loc",R.toString(),"not page/sitemap/asset-like",g);}i.trackManifest&&(i.manifest.assets[g]={url:g,outputPath:h.outputPath,isText:!0,contentType:d.headers()["content-type"]??"application/xml",changeToken:null,tokenSource:null,discoveredAssets:[...new Set([...L,...x])],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},await te(e,i));}catch(d){if(s.error(`Failed sitemap ${p}`,{url:p,error:String(d)}),u)throw d}}}}async function nn(e,t,r,s,n,i,u){let c=U(e,s,e.sourceOrigin,n,"asset-fetch");if(!c||!T(e,c))return;let o=c.toString();if(r.doneAssets.has(o))return;let a=i.enabled||i.strictTargeted?i.previousManifest.assets[o]:void 0,l=i.trackManifest?await Me(e,t,i,o,n):null,p=l?.token??null,d=Mt(a?.changeToken,l),m=i.strictTargeted&&(!l?.supported||!l.token)&&$t(e.sourceOrigin,o),S=i.strictTargeted?i.manifest.assets[o]:void 0;if(S?.lastSeenRunId===i.runId&&await le(S.outputPath)){r.doneAssets.add(o);for(let g of S.discoveredAssets||[])F(e,r,g,n,o);j(u,e,S.outputPath),S.isText&&ie(u,S.outputPath),n.info(`Resumed completed content-sync asset ${o}`,{url:o,source:"content-sync-checkpoint"}),ce(r,n);return}if(a&&(d||m)&&await le(a.outputPath)){i.manifest.assets[o]={...a,changeToken:p,tokenSource:l?.tokenSource??a.tokenSource,discoveredAssets:[...a.discoveredAssets||[]],lastSeenRunId:i.runId},delete i.manifest.pages[o],r.doneAssets.add(o);for(let g of a.discoveredAssets||[])F(e,r,g,n,o);n.info(`Reused unchanged trusted asset ${o}.`,{url:o,tokenSource:l?.tokenSource??"unknown",phase:"download-assets",source:d?"asset-change-token-match":"content-sync-release-fingerprint"}),ce(r,n);return}let w=(g,h={})=>{if(!a||i.strictTargeted)return false;i.manifest.assets[o]={...a,discoveredAssets:[...a.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},delete i.manifest.pages[o],r.doneAssets.add(o);for(let b of a.discoveredAssets||[])F(e,r,b,n,o);return n.warn(`Keeping previous incremental asset ${o} after fetch issue.`,{url:o,reason:g,preservedFromPreviousManifest:true,...h,phase:"download-assets"}),ce(r,n),true};n.asset(`Fetching asset ${o}`,{url:o});try{let g=de(e,r.assetMap,o),h=g?await le(g):null;if(g&&h)try{let y=await $(`fetch asset headers ${o}`,6e4,()=>t.head(o,{timeout:6e4})),C=y.headers();if(y.ok()&&Qs(h,C)){let E=await er(g);if(E!==null){let q=y.url()||o,he=re(q,n,"asset-response",o);if(!he)return;let V=he.toString(),ne=await Ze(e,r.assetMap,V,E,r,n,C);ne.wroteFile&&(j(u,e,ne.outputPath),X(ne.outputPath)&&ie(u,ne.outputPath)),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:g,lastModified:C["last-modified"]||"",contentLength:C["content-length"]||"",phase:"download-assets"}),ce(r,n);let k=Ot(c,C),oe=a?.discoveredAssets?[...a.discoveredAssets]:[];if(k||a?.isText){let fe=E.toString("utf8"),nt=ve(e,V,fe,n,"asset:cached");(nt.length>0||!a)&&(oe=nt);for(let ar of oe)F(e,r,ar,n,V);}if(i.trackManifest){let fe=k||!!a?.isText;i.manifest.assets[o]={url:o,outputPath:ne.outputPath,isText:fe,contentType:Ve(C)||a?.contentType||null,changeToken:l?.supported?l.token:null,tokenSource:l?.supported?l.tokenSource??null:null,discoveredAssets:oe,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!fe},delete i.manifest.pages[o],(i.enabled||i.strictTargeted)&&fe&&await te(e,i);}return}}}catch{}let b=await $(`fetch asset ${o}`,6e4,()=>t.get(o,{timeout:6e4}));if(!b.ok()){if(w(`http-${b.status()}`,{status:b.status()}))return;if(i.strictTargeted)throw new Error(`Required content-sync asset returned HTTP ${b.status()}: ${o}`);n.skip("asset",o,b.status());return}let L=await $(`read asset response ${o}`,6e4,()=>b.body()),x=b.url()||o,I=re(x,n,"asset-response",o);if(!I)return;let R=I.toString(),O=b.headers(),D=await Ze(e,r.assetMap,R,L,r,n,O);D.wroteFile&&(j(u,e,D.outputPath),X(D.outputPath)&&ie(u,D.outputPath)),r.doneAssets.add(o),ce(r,n);let P=Ot(c,O),A=[];if(P){let y=L.toString("utf8");A=ve(e,R,y,n,`asset:${f.extname(c.pathname).toLowerCase()||O["content-type"]||"unknown"}`);for(let C of A)F(e,r,C,n,R);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:D.outputPath,isText:P,contentType:Ve(O),changeToken:l?.supported?l.token:null,tokenSource:l?.supported?l.tokenSource??null:null,discoveredAssets:A,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!P},delete i.manifest.pages[o],(i.enabled||i.strictTargeted)&&P&&await te(e,i));}catch(g){if(w("fetch-error",{error:String(g)}))return;if(n.error(`Failed asset ${o}`,{url:o,error:String(g)}),i.strictTargeted)throw g}finally{r.doneAssets.has(o)||(r.doneAssets.add(o),ce(r,n));}}async function on(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),c=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await nn(e,t,r,o,s,n,i));}});await Promise.all(c);}function an(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function z(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function $e(e){return Math.max(15e3,Math.min(6e4,z(e)))}function un(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function st(e){return e.stats.pagesCompleted}function nr(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function cn(e,t,r){let s=nr(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function $(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${an(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function ln(e,t){let r=un(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,u=700,c=Date.now(),o=()=>{window.clearInterval(a),window.scrollTo(0,0),n();},a=window.setInterval(()=>{if(Date.now()-c>=s.maxDurationMs){o();return}window.scrollBy(0,u),i+=u;let l=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(l,p);i>=d+window.innerHeight&&o();},120);});},{maxDurationMs:r});}async function dn(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await ln(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function pn(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function gn(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>et(t,r))}async function hn(e,t,r,s,n=true){let i=new Set;if(n){let c=await t.evaluate(()=>{let o=new Set,a=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let S of a){let w=d.getAttribute(S);w&&o.add(w);}for(let S of p){let w=d.getAttribute(S);w&&w.split(",").forEach(g=>{let h=g.trim().split(/\s+/)[0];h&&o.add(h);});}let m=d.getAttribute("style");m&&l.add(m);}),{attrs:[...o],styles:[...l]}});for(let o of c.attrs)for(let a of Zs(o)){let l=U(e,a,r,s,"dom-attr");l&&T(e,l)&&i.add(l.toString());}for(let o of c.styles)for(let a of ve(e,r,o,s,"dom-style"))i.add(a);}let u=await t.content();for(let c of ve(e,r,u,s,"page-html"))i.add(c);return [...i]}async function qt(e,t,r,s,n){let i=J(e,t);await ye(i,r,"utf8");let u=await v.stat(i);n?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:i,bytes:u.size}),s.stats.pagesSaved++;let c=Math.min(s.stats.pagesQueued,st(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${c}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:c,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function fn(e){let t=st(e);return t%5===0||t===1||t===e.stats.pagesQueued}function mn(e,t){let r=st(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function wn(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function yn(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await wn(s),s}async function Pn(e,t,r,s,n,i,u){let c=await chromium.launch({headless:true}),o=nr(c);o.pid!==null&&r.debug(`Launched crawl worker browser PID ${o.pid}`,{browserPid:o.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let a=t.pageQueue.shift();if(!a)break;t.queuedPages.delete(a);let l=null,p=null;try{let d=re(a,r,"page","page-queue");if(!d)continue;let m=d.toString();if(a=m,t.queuedPages.delete(a),t.donePages.has(a))continue;t.donePages.add(a),i.enabled&&i.seenPages.add(a);let S=rt(e,a),w=gn(e,d.pathname);if(!se(d)||T(e,d)||pe(d)){if(T(e,d)){r.info(`Worker redirected non-page URL to asset queue: ${a}`,{url:a,source:"worker-guard"}),F(e,t,a,r,"worker-guard");continue}r.reject("page",a,"guard rejected non-page URL before rendering","worker");continue}let g=i.strictTargeted?i.manifest.pages[a]:void 0;if(g?.lastSeenRunId===i.runId&&await le(g.outputPath)){i.seenPages.add(a);for(let y of g.discoveredAssets||[])F(e,t,y,r,a);j(u,e,g.outputPath),ie(u,g.outputPath),r.info(`Resumed completed content-sync page ${a}`,{url:a,source:"content-sync-checkpoint"});continue}let h=await vs(e,n,t,i,a,r);if(i.enabled&&s!=="single-url"&&s!=="content-sync"){let y=i.manifest.pages[a];if(h.action==="reuse"&&y){for(let C of y.discoveredAssets)F(e,t,C,r,a);for(let C of y.discoveredPages)be(e,t,C,r,a);y.lastSeenRunId=i.runId,y.sitemapLastmod=Pe(t.sitemapLastmodByPage[a])??y.sitemapLastmod,h.changeToken?.supported&&(y.changeToken=h.changeToken.token,y.tokenSource=h.changeToken.tokenSource??y.tokenSource),r.info(`Incremental reuse skipped unchanged page ${a}`,{url:a,mode:"incremental",reason:h.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}l=await $(`create page context ${a}`,z(e),()=>yn(c,e,w)),p=await $(`create page ${a}`,z(e),()=>l.newPage());let b=!1,L=new Set;await p.route("**/*",async y=>{let C=y.request();if(C.isNavigationRequest()&&C.frame()===p.mainFrame())try{let E=new URL(C.url()),q=new URL(m);if(E.hash="",q.hash="",E.href===q.href){if(b){r.warn(`Blocked same-page navigation/reload for ${a}`,{url:a,requestUrl:C.url()}),await y.abort("aborted");return}b=!0;}}catch{}await y.continue();}),p.on("response",y=>{try{let C=U(e,y.url(),e.sourceOrigin,r,"network-response");C&&T(e,C)&&(L.add(C.toString()),F(e,t,C.toString(),r,a));}catch{}}),r.page(`Rendering ${a}`,{url:a});let x=null,I=null;try{if(x=await p.goto(a,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),x&&x.ok())try{I=await $(`read response body ${a}`,z(e),()=>x.text());}catch{I=null;}}catch(y){r.warn(`Navigation issue for ${a}; saving current DOM if available`,{url:a,error:String(y)});}if(s==="content-sync"&&x){let y=re(x.url(),r,"content-sync-final-response",a)?.toString();if(y&&y!==m){if(i.deleteOnMissingUrls.has(a)){await Ye(e,i,u,a,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${y}`,{url:a,finalResponseUrl:y,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${a} -> ${y}`)}}if(S){let y=x?.status()??null;if(y!==404){r.error(`Generated 404 request path returned unexpected status for ${a}`,{url:a,expectedStatus:404,status:y,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${a}`,{url:a,status:y,outputPath:J(e,a)});}else if(x&&!x.ok()){if(s==="content-sync"&&(x.status()===404||x.status()===410)&&i.deleteOnMissingUrls.has(a)){await Ye(e,i,u,a,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${a}`,{url:a,status:x.status(),source:"content-sync-tombstone"}),await p.close();continue}if(s==="content-sync")throw new Error(`Required content-sync route returned HTTP ${x.status()}: ${a}`);r.skip("page",a,x.status(),t.pageDiscoverySources.get(a)),await p.close();continue}w?(r.info(`Rendering without JS execution for ${a}`,{url:a,mode:"no-js"}),await $(`wait for rendered page ${a}`,z(e),()=>pn(e,p))):await $(`wait for rendered page ${a}`,z(e),()=>dn(e,p)),t.stats.pagesRendered++;let R=await $(`extract rendered assets ${a}`,z(e),()=>hn(e,p,m,r,!w));for(let y of R)F(e,t,y,r,a);let O=[...new Set([...R,...L])],D=I??await $(`capture rendered html ${a}`,z(e),()=>p.content()),P=s!=="single-url"&&s!=="content-sync"&&!S?sn(e,a,D,r):[];if(s!=="single-url"&&s!=="content-sync"&&!S)for(let y of P)be(e,t,y,r,a);await qt(e,a,D,t,r);let A=J(e,a);if(j(u,e,A),ie(u,A),i.trackManifest){let y=x?.headers();i.manifest.pages[a]={url:a,outputPath:A,contentType:Ve(y),changeToken:h.changeToken?.supported?h.changeToken.token:null,tokenSource:h.changeToken?.supported?h.changeToken.tokenSource??null:null,sitemapLastmod:Pe(t.sitemapLastmodByPage[a])??null,discoveredPages:P,discoveredAssets:O,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await te(e,i);}}catch(d){if(i.strictTargeted)throw r.error(`Failed required content-sync page ${a}`,{url:a,error:String(d)}),d;try{let m=p?await $(`capture partial html ${a}`,z(e),()=>p.content()):"";if(m&&m.trim()){await qt(e,a,m,t,r);let S=J(e,a);if(j(u,e,S),ie(u,S),i.trackManifest){let w=i.manifest.pages[a]??i.previousManifest.pages[a];i.manifest.pages[a]={url:a,outputPath:S,contentType:w?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:Pe(t.sitemapLastmodByPage[a])??w?.sitemapLastmod??null,discoveredPages:w?.discoveredPages?[...w.discoveredPages]:[],discoveredAssets:w?.discoveredAssets?[...w.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await te(e,i);}r.warn(`Saved partial DOM for ${a}`,{url:a,error:String(d)});}}catch(m){r.error(`Could not save partial DOM for ${a}`,{url:a,error:String(m)});}r.error(`Failed page ${a}`,{url:a,error:String(d)});}finally{t.stats.pagesCompleted++,fn(t)&&mn(t,r),p&&await $(`close page ${a}`,$e(e),()=>p.close()).catch(d=>{r.warn(`Timed out while closing page ${a}`,{url:a,browserPid:o.pid,error:String(d)});}),l&&await $(`close page context ${a}`,$e(e),()=>l.close()).catch(d=>{r.warn(`Timed out while closing page context for ${a}`,{url:a,browserPid:o.pid,error:String(d)});});}}}finally{await $("close worker browser",$e(e),()=>c.close()).catch(a=>(r.warn("Timed out while closing crawl worker browser",{browserPid:o.pid,error:String(a)}),cn(c,r,"browser.close timeout")));}}async function Sn(e){try{let t=await v.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Nt(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function bn(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function vn(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function ir(e){try{let t=await v.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Wt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Cn(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Wt(s)-Wt(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function _t(e){let t=await ir(f.join(e,"job.json"));for(let r of Cn(e,t)){let s=await Sn(r);if(s.length>0)return s}return []}async function Rn(e){let t=f.join(e.logDir,"archive"),r;try{r=await v.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=f.join(t,i);if(!(await v.stat(u).catch(()=>null))?.isDirectory())continue;let o=await ir(f.join(u,"job.json"));if(!bn(o))continue;let a=vn(o);a>=n&&(n=a,s=u);}return s}async function An(e){let t=await Rn(e),r=t!==""?await _t(t):await _t(e.logDir),s=new Set;for(let n of r)if(Nt(n.error)||Nt(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function kn(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await v.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function xn(e){if(!e)throw new Error("Content-sync crawl requires an immutable impact plan.");let t=JSON.parse(await v.readFile(e,"utf8"));return pt(t),t}function Tn(e,t){let r;try{r=new URL(e);}catch{return false}return t.archiveFamilies.some(s=>{let n=new URL(s.url);if(r.origin!==n.origin)return false;if(r.toString()===n.toString())return true;if(r.searchParams.has("paged")&&r.pathname===n.pathname)return /^\d+$/.test(r.searchParams.get("paged")||"");let i=n.pathname.replace(/\/+$/,""),u=r.pathname.slice(i.length).replace(/^\/+/,""),c=String(s.paginationBase||"page").replace(/^\/+|\/+$/g,"").replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return r.pathname.startsWith(`${i}/`)&&new RegExp(`^${c}/\\d+/?$`).test(u)})}function Ie(e,t,r){let s=r==="page"?J(e,t):Ue(e,t).originalFilePath;return Z(e.outputDir,s)}async function Ye(e,t,r,s,n){let i=t.manifest.pages[s];if(!i)return;let u=Z(e.outputDir,i.outputPath);if(!u)throw new Error(`Manifest-owned page path escaped output root: ${s}`);let c=f.resolve(i.outputPath);Object.entries(t.manifest.pages).some(([a,l])=>a!==s&&!n.has(a)&&f.resolve(l.outputPath)===c)||(await v.rm(i.outputPath,{force:true}),r.deletedFiles.add(u)),delete t.manifest.pages[s];}function In(e,t){let r=new Set,s=[...t];for(;s.length>0;){let n=s.shift();if(r.has(n))continue;r.add(n);let i=e.assets[n];for(let u of i?.discoveredAssets??[])try{ge(new URL(u))&&s.push(u);}catch{}}return r}async function Mn(e,t,r,s,n){let i=In(r.previousManifest,n);for(let u of i){if(t.doneSitemaps.has(u))continue;let c=r.previousManifest.assets[u];if(!c)continue;let o=Z(e.outputDir,c.outputPath);if(!o)throw new Error(`Manifest-owned sitemap path escaped output root: ${u}`);await v.rm(c.outputPath,{force:true}),s.deletedFiles.add(o),delete r.manifest.assets[u];}await te(e,r);}async function $n(e){try{let t=await v.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Ln(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await v.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await v.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Bt(e,t,r,s,n){let i=U(e,r,e.sourceOrigin,s,"manual-url");i&&(ge(i)?De(e,t,i.toString(),s,n):T(e,i)?F(e,t,i.toString(),s,n):se(i)?be(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function or(e=process.argv.slice(2)){let t=[],r=await Ce(void 0,{onRemotePublisherConfigDiagnostic:P=>{t.push(P);}}),s=t.at(-1)??null,n=ls(e),i=n.mode==="content-sync"?await xn(n.contentSyncPlanPath):null;if(n.resumeRewrite&&n.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let u=[];n.mode==="retry-timeouts"&&(u=await An(r)),n.mode==="single-url"&&(u=await kn(n.urls)),i&&(u=[...i.directRenderUrls,...i.archiveFamilyUrls]);let c=n.crawlMode==="incremental"&&n.mode==="full",o=c&&typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!=="";if(i&&r.subscriptionType!=="PROFESSIONAL"&&r.subscriptionType!=="AGENCY")throw new Error("Content-sync requires an active Professional or Agency subscription.");let a=(n.mode==="full"||n.mode==="single-url"||n.mode==="content-sync")&&!n.resumeRewrite,l=n.mode!=="full"||n.resumeRewrite||o,p=l?await $n(r):{},d=o||n.mode==="single-url"||n.mode==="content-sync"?await ms(r):Jt(),m=new Set(u),S=new Set;if(i){for(let P of i.archiveFamilies)if(P.foundPosts===0)for(let A of P.pageUrls)S.add(A);for(let P of Object.keys(d.pages))Tn(P,i)&&!m.has(P)&&(u.push(P),S.add(P));}let w={enabled:o,trackManifest:a,manifest:Ge(d),previousManifest:Ge(d),runId:i?.jobId??`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve(),strictTargeted:!!i,requiredUrls:m,deleteOnMissingUrls:S};await Ln(r.logDir);let g=new ue(r.logDir,r.logLevel);if(s){let P={status:s.status,virtualAssetBaseUrl:s.virtualAssetBaseUrl,subscriptionType:s.subscriptionType??null,error:s.error,requests:s.requests,source:"remote-publisher-config"};s.status==="loaded"&&s.subscriptionType?g.info("Loaded remote WP Suite publisher configuration.",P):g.warn("Remote WP Suite publisher configuration is unavailable or has no active subscription type.",P);}if(c&&!o){let P="Incremental crawl requires an active remote publisher subscription; refusing to fall back to a destructive full crawl.";throw g.error(P,{requestedMode:n.crawlMode,subscriptionType:r.subscriptionType??null,source:"incremental-subscription-guard"}),await g.flush(),new Error(P)}let h={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapIndexUrls:new Set,sitemapLeafUrls:new Set,sitemapLastmodByPage:{},assetMap:{...p},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},b=fs();if(await ct(r),await lt(r,{startedAt:new Date().toISOString(),outputDir:f.resolve(r.outputDir)}),l||await v.rm(r.outputDir,{recursive:true,force:true}),await v.mkdir(r.outputDir,{recursive:true}),i){let P=new Set([...i.deleteUrls,...w.deleteOnMissingUrls]);for(let A of i.deleteUrls)await Ye(r,w,b,A,P);}if(n.resumeRewrite)g.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:n.mode,crawlMode:n.crawlMode});else {let P=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{if(g.mark("discovery"),n.mode==="full"?(r.sitemapPaths.forEach(y=>De(r,h,y,g)),await jt(r,P,h,g,b,w),r.seedPaths.forEach(y=>Bt(r,h,y,g,"seed-path")),r.generated404RequestPath&&be(r,h,r.generated404RequestPath,g,"generated-404-request-path")):(u.forEach(y=>Bt(r,h,y,g,"cli")),i?.sitemapUrls.forEach(y=>De(r,h,y,g,"content-sync-impact")),await jt(r,P,h,g,b,w,n.mode==="content-sync"),i&&await Mn(r,h,w,b,i.sitemapUrls)),g.endMark("discovery",{pages:h.pageQueue.length,assets:h.assetQueue.length,sitemaps:h.sitemapQueue.length}),g.summary(`Queued ${h.pageQueue.length} pages, ${h.assetQueue.length} assets, ${h.sitemapQueue.length} sitemaps.`,{mode:n.mode,crawlMode:n.crawlMode,queuedPages:h.pageQueue.length,queuedAssets:h.assetQueue.length,queuedSitemaps:h.sitemapQueue.length}),n.mode==="retry-timeouts"&&h.pageQueue.length===0&&h.assetQueue.length===0&&h.sitemapQueue.length===0){g.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:n.mode,crawlMode:n.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await g.flush();return}g.mark("render-pages");let A=Array.from({length:Math.max(1,r.concurrency)},()=>Pn(r,h,g,n.mode,P,w,b));if(await Promise.all(A),g.endMark("render-pages",{pages:h.stats.pagesCompleted}),g.mark("download-assets"),g.progress("Asset download phase started.",{phase:"download-assets",doneAssets:h.doneAssets.size,assetsQueued:h.stats.assetsQueued,assetQueue:h.assetQueue.length}),await Ps(r,P,w,[...h.assetQueue],g),await on(r,P,h,g,w,b),g.endMark("download-assets",{assets:h.doneAssets.size}),w.enabled&&r.maxPages===0&&await Rs(r,h,w,g,b),n.mode==="full"){let y=await rn(r,h,g);for(let C of y)j(b,r,C);}}finally{await $("dispose crawl request context",$e(r),()=>P.dispose()).catch(A=>{g.warn("Timed out while disposing crawl request context",{error:String(A)});});}}w.trackManifest&&await te(r,w),await v.writeFile(f.join(r.outputDir,"asset-map.json"),JSON.stringify(h.assetMap,null,2),"utf8"),JSON.stringify(p,null,2)!==JSON.stringify(h.assetMap,null,2)&&j(b,r,f.join(r.outputDir,"asset-map.json"));let L=n.mode==="single-url"||n.mode==="content-sync"?Ds(r,h):w.enabled&&!n.resumeRewrite?As(r,w,b,p,h.assetMap):void 0;g.mark("rewrite-text");let x=n.mode==="single-url"||n.mode==="content-sync"?`Rewriting text files touched by URL crawl: 0/${L?.length??0}`:"Rewriting text files...";g.progress(x,{phase:"rewrite-text",index:0,totalFiles:L?.length,changedTextFiles:0}),await Et(x);let I=Date.now(),R=await Rt(r,h.assetMap,{files:L,previousAssetMap:p,onStart:({totalFiles:P,workerCount:A})=>{g.info(`Rewrite phase scanning ${P} text files.`,{phase:"rewrite-text",totalFiles:P,workerCount:A});},onProgress:async({index:P,totalFiles:A,changedTextFiles:y,file:C,changed:E})=>{E&&j(b,r,f.join(r.outputDir,C)),g.checkpoint(`Rewriting text file ${P}/${A}`,{phase:"rewrite-text",index:P,totalFiles:A,changedTextFiles:y,file:C}),g.debug(`${E?"Rewrote":"Checked unchanged"} text file ${P}/${A}: ${C}`,{phase:"rewrite-text",index:P,totalFiles:A,changedTextFiles:y,file:C,changed:E});let q=Date.now();if(!(P===1||P===A||P%50===0||q-I>=5e3))return;I=q;let V=`Rewriting text files: ${P}/${A}`;g.progress(V,{phase:"rewrite-text",index:P,totalFiles:A,changedTextFiles:y,file:C}),await Et(V);}});w.trackManifest&&(ks(w,L),await te(r,w)),g.endMark("rewrite-text",{changedTextFiles:R});let O=i?[...b.deletedFiles].sort():w.enabled&&!n.resumeRewrite?await Ts(r,h,w,b,g):[],D=[...new Set((L||[]).map(P=>Z(r.outputDir,f.isAbsolute(P)?P:f.join(r.outputDir,P))).filter(P=>P!==null))].sort();if(i){let P=[...b.changedFiles].sort(),A=new Set(P),y=new Set(i.directRenderUrls.map(k=>Ie(r,k,"page")).filter(k=>!!k&&A.has(k))),C=new Set(i.archiveFamilyUrls.map(k=>Ie(r,k,"page")).filter(k=>!!k&&A.has(k))),E=new Set([...h.sitemapLeafUrls].map(k=>Ie(r,k,"asset")).filter(k=>!!k&&A.has(k))),q=new Set([...h.sitemapIndexUrls].map(k=>Ie(r,k,"asset")).filter(k=>!!k&&A.has(k))),he=new Set([...y,...C,...E,...q]),V=P.filter(k=>!he.has(k)),ne=[...new Set([...i.directRenderUrls,...i.archiveFamilyUrls,...i.sitemapUrls,...h.sitemapLeafUrls,...h.sitemapIndexUrls,...i.deleteUrls].map(k=>{let oe=new URL(k);return `${oe.pathname}${oe.search}`}))].sort();await Fe(r,{schemaVersion:2,generatedAt:new Date().toISOString(),outputDir:f.resolve(r.outputDir),runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,changedFiles:P,deletedFiles:O,rewriteTargets:D,consumerId:i.consumerId,fromSequence:i.fromSequence,toSequence:i.toSequence,impactHash:i.impactHash,uploadGroups:{assets:V,content:[...y].sort(),listings:[...C].sort(),sitemapLeaves:[...E].sort(),sitemapIndexes:[...q].reverse()},invalidationPaths:ne});}else await Fe(r,{schemaVersion:1,generatedAt:new Date().toISOString(),outputDir:f.resolve(r.outputDir),runMode:n.mode,crawlMode:n.crawlMode,fullSyncRequired:!(w.enabled&&!n.resumeRewrite),changedFiles:w.enabled&&!n.resumeRewrite?[...b.changedFiles].sort():[],deletedFiles:w.enabled&&!n.resumeRewrite?O:[],rewriteTargets:w.enabled&&!n.resumeRewrite?D:[]});await dt(r),g.summary(n.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${R} text files.`:`Done. Rendered ${h.stats.pagesRendered} pages, processed ${h.doneSitemaps.size} sitemaps and ${h.doneAssets.size} assets.`,{mode:n.mode,crawlMode:n.crawlMode,resumeRewrite:n.resumeRewrite,...h.stats,pagesRendered:h.stats.pagesRendered,donePages:h.stats.pagesCompleted,claimedPages:h.donePages.size,doneSitemaps:h.doneSitemaps.size,doneAssets:h.doneAssets.size,changedTextFiles:R}),await g.flush();}or().catch(async e=>{let t=H(e instanceof Error?e.message:String(e));console.error(t);try{let s=(await Ce().catch(()=>null))?.logDir??"logs",n=new ue(s,"debug");n.error(`Unhandled error: ${t}`,e instanceof Error&&e.stack?{stack:H(e.stack)}:void 0),await n.flush();}catch{}process.exit(1);});
|
|
12
|
+
${i}`,40);return s.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:o,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(n){return s.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function Fs(e,t,r,s,n,i){let o=ct(e,n);if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await Ee(e,t,s,n,i):null};let c=s.manifest.pages[n];if(!c)return {action:"render",changeToken:await Ee(e,t,s,n,i)};let a=await Ee(e,t,s,n,i);if(a?.supported&&a.token)return c.changeToken===a.token?o&&!await js(e,t,n,i)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let l=be(r.sitemapLastmodByPage[n]);return l&&c.sitemapLastmod&&c.sitemapLastmod===l?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function js(e,t,r,s){try{let i=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return i===404?!0:(s.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:i,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(n){return s.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(n),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function qs(e,t,r,s,n){let i=Ht(r.previousManifest,void 0,s,"previous-manifest"),o=0;for(let[d,S]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Qt(n,e,S.outputPath);try{await C.unlink(S.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:S.outputPath,error:String(m)});}delete r.manifest.pages[d],o++;}let c=Ht(r.manifest,r.runId,s,"active-manifest"),a=new Set,l=new Set;for(let d of c){for(let P of ve(e,d))a.add(P);let S=r.manifest.assets[d]?.outputPath;S&&l.add(S);let m=ge(e,t.assetMap,d);m&&l.add(m);}for(let d of Object.keys(r.manifest.assets)){if(c.has(d)){r.manifest.assets[d].unreferencedRuns=0;continue}let S=r.manifest.assets[d];if(qt(S.unreferencedRuns)){S.unreferencedRuns+=1,s.warn(`Deferred deletion of newly unreferenced incremental asset ${d}.`,{url:d,unreferencedRuns:S.unreferencedRuns,phase:"incremental-cleanup",source:"asset-deletion-grace"});continue}delete r.manifest.assets[d];}let u=0,p=0;for(let d of i){if(r.manifest.assets[d])continue;let S=new Set,m=r.previousManifest.assets[d]?.outputPath;m&&S.add(m);let P=ge(e,t.assetMap,d);if(!c.has(d)&&P)S.add(P);else if(!c.has(d)){let g=rn(e,d,s,"incremental-cleanup");g&&(S.add(g.originalFilePath),g.hashedFilePath&&S.add(g.hashedFilePath));}for(let g of S)if(!l.has(g)){Qt(n,e,g);try{await C.unlink(g),u++;}catch(h){(h.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:g,error:String(h)});}}for(let g of ve(e,d))a.has(g)||Object.prototype.hasOwnProperty.call(t.assetMap,g)&&(delete t.assetMap[g],p++);}(o>0||u>0||p>0)&&s.info(`Incremental cleanup removed ${o} stale pages, ${u} stale assets, ${p} stale asset mappings`,{removedPages:o,removedAssets:u,removedAssetMappings:p,phase:"incremental-cleanup"});}function Ht(e,t,r,s){let n=new Set,i=o=>{let c=o.trim();if(c){if(r){let a=ie(c,r,"asset",s??"manifest");if(!a)return;n.add(a.toString());return}n.add(c);}};for(let o of Object.values(e.pages))for(let c of o.discoveredAssets||[])i(c);for(let[o,c]of Object.entries(e.assets||{}))t&&c.lastSeenRunId!==t||i(o);return n}function zt(e,t,r){for(let s of t)for(let n of ve(e,s))if(r.has(n))return true;return false}function Ns(e,t,r,s,n){let i=new Set(r.rewriteTargets);for(let c of Object.values(t.manifest.pages))c.rewriteComplete||i.add(w.resolve(c.outputPath));for(let c of Object.values(t.manifest.assets))c.isText&&!c.rewriteComplete&&i.add(w.resolve(c.outputPath));let o=Us(s,n);if(o.size===0)return [...i];for(let c of Object.values(t.manifest.pages))zt(e,c.discoveredAssets,o)&&i.add(w.resolve(c.outputPath));for(let[c,a]of Object.entries(t.manifest.assets)){a.isText&&zt(e,a.discoveredAssets,o)&&i.add(w.resolve(a.outputPath));for(let l of ve(e,c))if(o.has(l)){W(r,e,a.outputPath),a.isText&&i.add(w.resolve(a.outputPath));break}}return [...i]}function _s(e,t){let r=t?new Set(Array.from(t,n=>w.resolve(n))):null,s=n=>r===null||r.has(w.resolve(n));for(let n of Object.values(e.manifest.pages))s(n.outputPath)&&(n.rewriteComplete=true);for(let n of Object.values(e.manifest.assets)){if(!n.isText){n.rewriteComplete=true;continue}s(n.outputPath)&&(n.rewriteComplete=true);}}function Ws(e,t,r){let s=new Set,n=i=>{if(!i)return;let o=ee(e.outputDir,i);o&&s.add(o);};for(let i of t.donePages)n(K(e,i));for(let i of Object.values(r.manifest.pages))i.lastSeenRunId===r.runId&&n(i.outputPath);for(let i of t.doneAssets)n(ge(e,t.assetMap,i));for(let i of Object.values(r.manifest.assets))i.lastSeenRunId===r.runId&&n(i.outputPath);return s}async function Bs(e,t,r,s,n){let i=[...s.deletedFiles].sort(),o=Ws(e,t,r),c=0;for(let u of [...o].sort()){try{if((await C.stat(w.join(e.outputDir,u))).isFile())continue}catch{}c++,n.warn(`Active output is missing after crawl: ${u}`,{phase:"deploy-plan",file:u,outputPath:w.join(e.outputDir,u)});}if(c>0&&n.warn(`Detected ${c} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:c}),i.length===0)return [];let a=[],l=0;for(let u of i){if(!o.has(u)){a.push(u);continue}l++;let p=w.join(e.outputDir,u),d;try{d=(await C.stat(p)).isFile();}catch{d=false;}n.warn(`Deploy plan protected active output from deletion: ${u}`,{phase:"deploy-plan",file:u,outputPath:p,exists:d});}return l>0&&n.info(`Deploy plan skipped ${l} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:l,deletedFiles:a.length}),a}function ve(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Qs(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function ar(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return w.join(e.outputDir,ut(s.pathname))}catch{return null}}function Hs(e,t,r){for(let s of Qs(e,r)){let n=t[s];if(!n)continue;let i=ar(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function zs(e,t,r){for(let s of ve(e,r)){let n=t[s];if(!n)continue;let i=ar(e,n);if(i)return i}return null}function ge(e,t,r){let s=zs(e,t,r);if(s)return s;try{return Ne(e,r).originalFilePath}catch{return null}}function Js(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.join(w.resolve(e),"queue-runner-heartbeat.json"):""}async function Jt(e){let t=Js();if(t)try{let r=await C.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await C.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Vs(e,t){let r=new Set;for(let s of t.donePages)r.add(w.resolve(K(e,s)));for(let s of t.doneAssets){let n=ge(e,t.assetMap,s);!n||!re(n)||r.add(w.resolve(n));}for(let s of t.doneSitemaps){let n=ge(e,t.assetMap,s);!n||!re(n)||r.add(w.resolve(n));}return [...r]}var Fe="re:",Ue=new Map;function Gs(e){if(!e.startsWith(Fe))return null;let t=Ue.get(e);if(t!==void 0)return t||null;let r=e.slice(Fe.length).trim();if(!r)return Ue.set(e,false),null;try{let s=new RegExp(r);return Ue.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Ue.set(e,false),null}}function at(e,t){let r=Gs(t);return r?r.test(e):t.startsWith(Fe)?false:e.startsWith(t)}function Ks(e,t){return t.find(r=>at(e,r))}function ur(e,t){let r=Ks(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(Fe)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Zs(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Ys(e,t){return Zs(e).has(t.hostname)}function Xs(e){let t=J(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=J(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function en(e){let t=J(e).replace(/\\\//g,"/").trim(),r=w.extname(t).toLowerCase();return tr.has(r)||rr.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function fe(e){return Cs.some(t=>e.pathname.startsWith(t))}function N(e,t,r=e.sourceOrigin,s,n="url"){let i=Xs(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;if(en(i))return s?.ignore(n,t,"looks like JavaScript property access",r),null;try{let o=new URL(i,r);if(cr(o))return s?.reject(n,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(o.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Ys(e,o))return s?.ignore(n,o.toString(),"host not allowed",r),null;let c=ur(e,o);if(c)return s?.reject(n,o.toString(),c,r),null;let a=new URL(e.sourceOrigin);return o.protocol=a.protocol,o.host=a.host,o.hash="",o}catch{return s?.reject(n,t,"invalid URL",r),null}}function cr(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function ie(e,t,r,s){try{let n=new URL(e);return cr(n)?(t.reject(r,e,"malformed URL encoding",s),null):n.protocol!=="http:"&&n.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${n.protocol}`,s),null):n}catch{return t.reject(r,e,"invalid URL",s),null}}function lr(e){return tr.has(w.extname(e.pathname).toLowerCase())}function oe(e){return fe(e)?false:rr.has(w.extname(e.pathname).toLowerCase())}function he(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function L(e,t){return !!(he(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>at(t.pathname,r))||lr(t)||!oe(t)&&!fe(t))}function K(e,t){let r=new URL(t),s=ut(r.pathname);return s.endsWith("/")?s+="index.html":w.extname(s)||(s+="/index.html"),w.join(e.outputDir,s)}function ut(e){try{return decodeURIComponent(e)}catch{return e}}function tn(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&w.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function ct(e,t){let r=tn(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Ne(e,t){let r=new URL(t),s=ut(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,o=w.join(e.outputDir,i),c=Ve(e.targetOrigin,i),a=`${c}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:o,originalPublicUrl:c,originalPublicUrlWithSearch:a,preferQueryHashed:n};let l=w.extname(s),p=`${l?s.slice(0,-l.length):s}.${ye(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:o,originalPublicUrl:c,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:w.join(e.outputDir,p),hashedPublicUrl:Ve(e.targetOrigin,p)}}function rn(e,t,r,s){let n=ie(t,r,"asset",s);return n?Ne(e,n.toString()):null}function sn(e,t,r,s,n){t[r]=s;try{let i=new URL(r),c=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{c.push(new URL(e.targetOrigin).origin);}catch{}for(let a of c)try{let l=new URL(a);t[l.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function dr(e){try{return await C.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function pe(e){try{return await C.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function nn(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function pr(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function on(e,t){let r=pr(t),s=nn(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function an(e,t){if(!t)return;let r=pr(t);if(r===null)return;let s=await pe(e);if(!s)return;let n=new Date(r);await C.utimes(e,s.atime,n).catch(()=>{});}function Vt(e,t){let r=w.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return vs.has(r)||bs.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function un(e){try{let t=w.extname(new URL(e).pathname).toLowerCase();return t===".js"||t===".mjs"}catch{return false}}function de(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset processing progress: processed ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function cn(e){return ye(e.toString("base64"),40)}function ln(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);if(!s||s.size===0)return false;for(let n of s)if(n!==r)return true;return false}function dn(e,t,r){let s=e.assetVariantDigestsByOriginalPath.get(t);s||(s=new Set,e.assetVariantDigestsByOriginalPath.set(t,s)),s.add(r);}async function it(e,t,r,s,n,i,o){let c=Ne(e,r),a=cn(s),l=c.originalFilePath,u=c.originalPublicUrlWithSearch;if(c.preferQueryHashed&&c.hashedFilePath&&c.hashedPublicUrl&&ln(n,c.originalFilePath,a)){let m=Hs(e,t,r);m?(l=m.outputPath,u=m.publicUrl):(l=c.hashedFilePath,u=c.hashedPublicUrl);}sn(e,t,r,u,c.originalPublicUrl);let p=re(l)?Buffer.from(Ze(s.toString("utf8"),e,t,l),"utf8"):s,d=await dr(l),S=false;return (d===null||!d.equals(p))&&(await Se(l,p),S=true),i?.debug(`${S?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:l,bytes:p.length,wroteFile:S}),dn(n,c.originalFilePath,a),await an(l,o),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:l,publicUrl:u,wroteFile:S}}function Ce(e,t,r,s,n){let i=()=>{let p=String(n??"").trim();if(!p)return {discoveredFrom:"",discoveredFromType:"unknown"};if(p==="seed-path"||p==="cli"||p==="generated-404-request-path")return {discoveredFrom:p,discoveredFromType:p};if(/^https?:\/\//i.test(p))try{let d=new URL(p);return he(d)?{discoveredFrom:p,discoveredFromType:"sitemap"}:L(e,d)?{discoveredFrom:p,discoveredFromType:"asset"}:oe(d)&&!fe(d)?{discoveredFrom:p,discoveredFromType:"page"}:{discoveredFrom:p,discoveredFromType:"url"}}catch{return {discoveredFrom:p,discoveredFromType:"url"}}return {discoveredFrom:p,discoveredFromType:"url"}},o=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,c=N(e,r,o,s,"page");if(!c)return;if(!oe(c)||lr(c)){if(L(e,c)){s.info(`Seeded/non-page URL queued as asset: ${c.toString()}`,{url:c.toString(),source:n}),q(e,t,c.toString(),s,n);return}s.reject("page",c.toString(),fe(c)?"asset/internal path cannot be page":"not page-like",n);return}let a=c.toString(),{discoveredFrom:l,discoveredFromType:u}=i();l&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,l),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,s.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:l,discoveredFromType:u,outputPath:K(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function q(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=N(e,r,i,s,"asset");if(!o)return;if(!L(e,o)){s.reject("asset",o.toString(),"not a safe asset path/prefix",n);return}let c=o.toString();!t.doneAssets.has(c)&&!t.queuedAssets.has(c)&&(t.queuedAssets.add(c),t.assetQueue.push(c),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function je(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=N(e,r,i,s,"sitemap");if(!o)return;if(!he(o)){s.reject("sitemap",o.toString(),"not sitemap-like",n);return}let c=o.toString();!t.doneSitemaps.has(c)&&!t.queuedSitemaps.has(c)&&(t.queuedSitemaps.add(c),t.sitemapQueue.push(c),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function pn(e){let t=J(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:be(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function gn(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function fn(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=e.slice(Math.max(0,s.index-8),s.index);if(/\bnew\s+$/i.test(n))continue;let i=(s[1]||s[2]||s[3]||"").trim();i&&t.push(i);}return t}function hn(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function mn(e,t){if(t==="dom-style")return true;let r=(()=>{try{return w.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function tt(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function gr(e,t,r,s){let n=[];await C.mkdir(t,{recursive:true});let i=await C.readdir(e,{withFileTypes:true});for(let o of i){let c=w.join(e,o.name),a=w.join(t,o.name);if(o.isDirectory()){n.push(...await gr(c,a,r,s));continue}await fr(c,a,r,s)&&n.push(a);}return n}async function wn(e,t){try{let[r,s]=await Promise.all([C.stat(e),C.stat(t)]);if(r.size!==s.size)return !1;let[n,i]=await Promise.all([C.readFile(e),C.readFile(t)]);return n.equals(i)}catch{return false}}async function fr(e,t,r,s){let n=w.resolve(t);return r.has(n)?(s.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await wn(e,t)?(s.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await C.mkdir(w.dirname(t),{recursive:true}),await C.copyFile(e,t),true)}async function yn(e,t,r){let s=Object.entries(e.postCrawlCopyMap||{}).map(([c,a])=>({sourcePath:c.trim(),prefix:String(a||"").trim()}));if(s.length===0)return [];let n=0,i=[],o=new Set([...t.donePages].map(c=>w.resolve(K(e,c))));r.mark("copy-extra-paths");for(let c of s){let a=c.sourcePath,l=c.prefix;if(!a||!l){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:l});continue}let u=Ts(a);if(!u.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:u.alias||"",requiredEnv:u.requiredEnv||"",prefixPath:l});continue}let p=u.resolvedPath,d=tt(l);if(!d){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:l});continue}let S;try{S=await C.stat(p);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:p,prefixPath:l});continue}let m=w.resolve(e.outputDir,d);if(S.isDirectory()){let f=await gr(p,m,o,r);i.push(...f),n+=f.length,f.length>0&&r.progress(`Copied static directory to export: ${p} -> /${d}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${d}`,copiedItems:n,changedFiles:f.length});continue}let P=w.basename(p),g=l.endsWith("/")?w.join(m,P):m;await fr(p,g,o,r)&&(i.push(g),n++,r.progress(`Copied static file to export: ${p} -> /${tt(l.endsWith("/")?`${l}${P}`:l)}`,{phase:"copy-extra-paths",sourcePath:p,targetPath:`/${tt(l.endsWith("/")?`${l}${P}`:l)}`,copiedItems:n}));}return r.endMark("copy-extra-paths",{mappedSources:s.length,copiedItems:n}),i}function Re(e,t,r,s,n){let i=new Set,o=J(r).replace(/\\\//g,"/"),c=n==="page-html"?hn(o):o,a=mn(t,n);if(a){for(let u of fn(c)){let p=N(e,u,t,s,`${n}:css-url`);p&&L(e,p)&&i.add(p.toString());}for(let u of c.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let p=N(e,u[1],t,s,`${n}:css-import`);p&&L(e,p)&&i.add(p.toString());}}if(a||n==="page-html")for(let u of c.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let p=N(e,u[1],t,s,`${n}:xml-stylesheet`);p&&L(e,p)&&i.add(p.toString());}let l=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${Xe}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${Xe}(?:\\?[^\\s'"<>\\);]*)?`,"gi");if(n==="page-html")for(let u of o.matchAll(l)){let p=u[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${u[0]}`:u[0];if(nt(p)){s.ignore(`${n}:serialized-url`,p,"looks like JavaScript/code fragment",t);continue}let d=N(e,p,t,s,`${n}:serialized-url`);d&&L(e,d)?i.add(d.toString()):d&&s.ignore("asset",d.toString(),"serialized URL did not pass safe asset path filter",t);}if(n!=="page-html"&&un(t))for(let u of Dt(o)){if(nt(u)||!/\.(?:js|mjs)(?:[?#]|$)/i.test(u))continue;let p=N(e,u,t,s,`${n}:relative-asset`);p&&L(e,p)&&i.add(p.toString());}return [...i]}function Sn(e,t,r,s){let n=new Set,i=J(r);for(let o of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let c=o[1],a=N(e,c,t,s,"page-link");a&&(ur(e,a)||oe(a)&&!fe(a)&&!L(e,a)&&n.add(a.toString()));}return [...n]}async function Gt(e,t,r,s,n,i,o=false,c=[]){let a=new Set(c),l=0,u=new Map,p=(d,S)=>{let m=[S],P=new Set;for(;m.length>0;){let g=m.pop();if(g===d)return true;P.has(g)||(P.add(g),m.push(...u.get(g)??[]));}return false};for(;r.sitemapQueue.length>0;){let d=r.sitemapQueue.shift();r.queuedSitemaps.delete(d);let S=ie(d,s,"sitemap","sitemap-queue");if(!S)continue;let m=S.toString();if(r.doneSitemaps.has(m))continue;r.doneSitemaps.add(m),s.sitemap(`Fetching sitemap ${m}`,{url:m});let P=true;try{let g=await t.get(m,{timeout:6e4});if(P=!1,!g.ok()){if(s.skip("sitemap",m,g.status()),o&&!a.has(m))throw new Error(`Required child content-sync sitemap returned HTTP ${g.status()}: ${m}`);continue}let h=await g.body(),f=g.url()||m,R=ie(f,s,"sitemap-response",m);if(!R)continue;let v=R.toString();r.successfulSitemaps.add(m),r.successfulSitemaps.add(v),a.has(m)&&(r.resolvedSitemapRoots.add(v),l+=1);let O=await it(e,r.assetMap,v,h,r,s);W(n,e,O.outputPath),ue(n,O.outputPath);let U=h.toString("utf8");/<sitemapindex(?:\s|>)/i.test(U)?r.sitemapIndexUrls.add(v):r.sitemapLeafUrls.add(v);let T=Re(e,v,U,s,"sitemap");for(let $ of T)q(e,r,$,s,v);let F=[];for(let $ of pn(U)){let k=N(e,$.loc,v,s,"sitemap-loc");if(!k){if(o)throw new Error(`Content-sync sitemap contains an inadmissible location: ${$.loc}`);continue}if(he(k)){if(p(v,k.toString()))throw new Error(`Sitemap index cycle detected: ${v} -> ${k.toString()}`);if(u.has(v)||u.set(v,new Set),u.get(v).add(k.toString()),F.push(k.toString()),o&&r.doneSitemaps.has(k.toString())&&!r.successfulSitemaps.has(k.toString()))throw new Error(`Required child content-sync sitemap was not resolved: ${k.toString()}`);je(e,r,k.toString(),s,v);}else if(o){if(oe(k)&&!L(e,k)){let _=i.requiredUrls.has(k.toString()),y=i.manifest.pages[k.toString()],A=!!y&&!!await pe(y.outputPath);if(!_&&!A)throw new Error(`Content-sync sitemap coverage is incomplete for ${k.toString()}.`)}s.ignore("sitemap-loc",k.toString(),"content-sync sitemap-only traversal does not enqueue urlset locations",v);}else L(e,k)?q(e,r,k.toString(),s,v):oe(k)?($.lastmod&&(r.sitemapLastmodByPage[k.toString()]=$.lastmod),Ce(e,r,k.toString(),s,v)):s.reject("sitemap-loc",k.toString(),"not page/sitemap/asset-like",v);}i.trackManifest&&(i.manifest.assets[v]={url:v,outputPath:O.outputPath,isText:!0,contentType:g.headers()["content-type"]??"application/xml",changeToken:null,tokenSource:null,discoveredAssets:[...new Set([...T,...F])],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!1},await ne(e,i));}catch(g){s.error(`Failed sitemap ${m}`,{url:m,error:String(g)});let h=o&&a.has(m)&&P;if(o&&!h)throw g}}if(o&&a.size>0&&l===0)throw new Error(`None of the configured content-sync sitemap roots could be resolved: ${[...a].join(", ")}`)}async function Pn(e,t,r,s,n,i,o){let c=N(e,s,e.sourceOrigin,n,"asset-fetch");if(!c||!L(e,c))return;let a=c.toString();if(r.doneAssets.has(a))return;let l=i.enabled||i.strictTargeted?i.previousManifest.assets[a]:void 0,u=i.trackManifest?await Ee(e,t,i,a,n):null,p=u?.token??null,d=Ft(l?.changeToken,u),S=i.strictTargeted&&(!u?.supported||!u.token)&&jt(e.sourceOrigin,a),m=i.strictTargeted?i.manifest.assets[a]:void 0;if(m?.lastSeenRunId===i.runId&&await pe(m.outputPath)){r.doneAssets.add(a);for(let g of m.discoveredAssets||[])q(e,r,g,n,a);W(o,e,m.outputPath),m.isText&&ue(o,m.outputPath),n.info(`Resumed completed content-sync asset ${a}`,{url:a,source:"content-sync-checkpoint"}),de(r,n);return}if(l&&(d||S)&&await pe(l.outputPath)){i.manifest.assets[a]={...l,changeToken:p,tokenSource:u?.tokenSource??l.tokenSource,discoveredAssets:[...l.discoveredAssets||[]],lastSeenRunId:i.runId,unreferencedRuns:0},delete i.manifest.pages[a],r.doneAssets.add(a);for(let g of l.discoveredAssets||[])q(e,r,g,n,a);n.info(`Reused unchanged trusted asset ${a}.`,{url:a,tokenSource:u?.tokenSource??"unknown",phase:"download-assets",source:d?"asset-change-token-match":"content-sync-release-fingerprint"}),de(r,n);return}let P=(g,h={})=>{if(!l||i.strictTargeted)return false;i.manifest.assets[a]={...l,discoveredAssets:[...l.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0},delete i.manifest.pages[a],r.doneAssets.add(a);for(let f of l.discoveredAssets||[])q(e,r,f,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:g,preservedFromPreviousManifest:true,...h,phase:"download-assets"}),de(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let g=ge(e,r.assetMap,a),h=g?await pe(g):null;if(g&&h)try{let _=await j(`fetch asset headers ${a}`,6e4,()=>t.head(a,{timeout:6e4})),y=_.headers();if(_.ok()&&on(h,y)){let A=await dr(g);if(A!==null){let b=_.url()||a,x=ie(b,n,"asset-response",a);if(!x)return;let M=x.toString(),E=await it(e,r.assetMap,M,A,r,n,y);E.wroteFile&&(W(o,e,E.outputPath),re(E.outputPath)&&ue(o,E.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:g,lastModified:y["last-modified"]||"",contentLength:y["content-length"]||"",phase:"download-assets"}),de(r,n);let me=Vt(c,y),Z=l?.discoveredAssets?[...l.discoveredAssets]:[];if(me||l?.isText){let ae=A.toString("utf8"),I=Re(e,M,ae,n,"asset:cached");(I.length>0||!l)&&(Z=I);for(let Ae of Z)q(e,r,Ae,n,M);}if(i.trackManifest){let ae=me||!!l?.isText;i.manifest.assets[a]={url:a,outputPath:E.outputPath,isText:ae,contentType:rt(y)||l?.contentType||null,changeToken:u?.supported?u.token:null,tokenSource:u?.supported?u.tokenSource??null:null,discoveredAssets:Z,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!ae},delete i.manifest.pages[a],(i.enabled||i.strictTargeted)&&ae&&await ne(e,i);}return}}}catch{}let f=await j(`fetch asset ${a}`,6e4,()=>t.get(a,{timeout:6e4}));if(!f.ok()){if(P(`http-${f.status()}`,{status:f.status()}))return;if(i.strictTargeted)throw new Error(`Required content-sync asset returned HTTP ${f.status()}: ${a}`);n.skip("asset",a,f.status());return}let R=await j(`read asset response ${a}`,6e4,()=>f.body()),v=f.url()||a,O=ie(v,n,"asset-response",a);if(!O)return;let U=O.toString(),T=f.headers(),F=await it(e,r.assetMap,U,R,r,n,T);F.wroteFile&&(W(o,e,F.outputPath),re(F.outputPath)&&ue(o,F.outputPath)),r.doneAssets.add(a),de(r,n);let $=Vt(c,T),k=[];if($){let _=R.toString("utf8");k=Re(e,U,_,n,`asset:${w.extname(c.pathname).toLowerCase()||T["content-type"]||"unknown"}`);for(let y of k)q(e,r,y,n,U);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:F.outputPath,isText:$,contentType:rt(T),changeToken:u?.supported?u.token:null,tokenSource:u?.supported?u.tokenSource??null:null,discoveredAssets:k,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,unreferencedRuns:0,rewriteComplete:!$},delete i.manifest.pages[a],(i.enabled||i.strictTargeted)&&$&&await ne(e,i));}catch(g){if(P("fetch-error",{error:String(g)}))return;if(n.error(`Failed asset ${a}`,{url:a,error:String(g)}),i.strictTargeted)throw g}finally{r.doneAssets.has(a)||(r.doneAssets.add(a),de(r,n));}}async function bn(e,t,r,s,n,i){let o=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),c=Array.from({length:o},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Pn(e,t,r,a,s,n,i));}});await Promise.all(c);}function vn(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function G(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function De(e){return Math.max(15e3,Math.min(6e4,G(e)))}function Cn(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function lt(e){return e.stats.pagesCompleted}function hr(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function Rn(e,t,r){let s=hr(e);try{if(s.kill){await s.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}if(s.pid!==null){process.kill(s.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:s.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(n){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:s.pid,reason:r,error:String(n)});}}async function j(e,t,r){let s;try{return await Promise.race([r(),new Promise((n,i)=>{s=globalThis.setTimeout(()=>{i(new Error(`${e} timed out after ${vn(t)}.`));},t);})])}finally{s!==void 0&&globalThis.clearTimeout(s);}}async function An(e,t){let r=Cn(e);await t.evaluate(async s=>{await new Promise(n=>{let i=0,o=700,c=Date.now(),a=()=>{window.clearInterval(l),window.scrollTo(0,0),n();},l=window.setInterval(()=>{if(Date.now()-c>=s.maxDurationMs){a();return}window.scrollBy(0,o),i+=o;let u=document.body?.scrollHeight||0,p=document.documentElement?.scrollHeight||0,d=Math.max(u,p);i>=d+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function kn(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await An(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function xn(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Tn(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>at(t,r))}async function In(e,t,r,s,n=true){let i=new Set;if(n){let c=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],u=new Set,p=["srcset","data-srcset","data-lazy-srcset"],d=S=>{S.querySelectorAll("*").forEach(m=>{for(let g of l){let h=m.getAttribute(g);h&&a.add(h);}for(let g of p){let h=m.getAttribute(g);h&&h.split(",").forEach(f=>{let R=f.trim().split(/\s+/)[0];R&&a.add(R);});}let P=m.getAttribute("style");P&&u.add(P),m.shadowRoot&&d(m.shadowRoot);});};return d(document),{attrs:[...a],styles:[...u]}});for(let a of c.attrs)for(let l of gn(a)){let u=N(e,l,r,s,"dom-attr");u&&L(e,u)&&i.add(u.toString());}for(let a of c.styles)for(let l of Re(e,r,a,s,"dom-style"))i.add(l);}let o=await t.content();for(let c of Re(e,r,o,s,"page-html"))i.add(c);return [...i]}async function Kt(e,t,r,s,n){let i=K(e,t);await Se(i,r,"utf8");let o=await C.stat(i);n?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:i,bytes:o.size}),s.stats.pagesSaved++;let c=Math.min(s.stats.pagesQueued,lt(s)+1);n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${c}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:c,claimedPages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Mn(e){let t=lt(e);return t%5===0||t===1||t===e.stats.pagesQueued}function $n(e,t){let r=lt(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Ln(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Un(e,t,r){let s=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Ln(s),s}async function On(e,t,r,s,n,i,o,c){let a=await chromium.launch({headless:true}),l=hr(a);l.pid!==null&&r.debug(`Launched crawl worker browser PID ${l.pid}`,{browserPid:l.pid});try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let u=t.pageQueue.shift();if(!u)break;t.queuedPages.delete(u);let p=null,d=null,S=[],m=new Set;try{let P=ie(u,r,"page","page-queue");if(!P)continue;let g=P.toString();if(u=g,t.queuedPages.delete(u),t.donePages.has(u))continue;t.donePages.add(u),i.enabled&&i.seenPages.add(u);let h=ct(e,u),f=Tn(e,P.pathname);if(!oe(P)||L(e,P)||fe(P)){if(L(e,P)){r.info(`Worker redirected non-page URL to asset queue: ${u}`,{url:u,source:"worker-guard"}),q(e,t,u,r,"worker-guard");continue}r.reject("page",u,"guard rejected non-page URL before rendering","worker");continue}let R=i.strictTargeted?i.manifest.pages[u]:void 0;if(R?.lastSeenRunId===i.runId&&await pe(R.outputPath)){i.seenPages.add(u);for(let b of R.discoveredAssets||[])q(e,t,b,r,u);W(o,e,R.outputPath),ue(o,R.outputPath),r.info(`Resumed completed content-sync page ${u}`,{url:u,source:"content-sync-checkpoint"});continue}let v=await Fs(e,n,t,i,u,r);if(i.enabled&&s!=="single-url"&&s!=="content-sync"){let b=i.manifest.pages[u];if(v.action==="reuse"&&b){for(let x of b.discoveredAssets)q(e,t,x,r,u);for(let x of b.discoveredPages)Ce(e,t,x,r,u);b.lastSeenRunId=i.runId,b.sitemapLastmod=be(t.sitemapLastmodByPage[u])??b.sitemapLastmod,v.changeToken?.supported&&(b.changeToken=v.changeToken.token,b.tokenSource=v.changeToken.tokenSource??b.tokenSource),r.info(`Incremental reuse skipped unchanged page ${u}`,{url:u,mode:"incremental",reason:v.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}p=await j(`create page context ${u}`,G(e),()=>Un(a,e,f)),d=await j(`create page ${u}`,G(e),()=>p.newPage()),S=await p.cookies();let O=!1,U=new Set;await d.route("**/*",async b=>{let x=b.request();try{let M=N(e,x.url(),e.sourceOrigin,r,"network-request");M&&L(e,M)&&(U.add(M.toString()),q(e,t,M.toString(),r,u));}catch{}if(x.isNavigationRequest()&&x.frame()===d.mainFrame())try{let M=new URL(x.url()),E=new URL(g);if(M.hash="",E.hash="",M.href===E.href){if(O){r.warn(`Blocked same-page navigation/reload for ${u}`,{url:u,requestUrl:x.url()}),await b.abort("aborted");return}O=!0;}}catch{}await b.continue();}),d.on("response",b=>{try{let M=N(e,b.url(),e.sourceOrigin,r,"network-response");M&&L(e,M)&&(U.add(M.toString()),q(e,t,M.toString(),r,u));}catch{}let x=b.headerValues("set-cookie").then(M=>{for(let E of M)c.add(_t(E,b.url()));}).catch(()=>{}).then(()=>{});m.add(x),x.finally(()=>m.delete(x));}),r.page(`Rendering ${u}`,{url:u});let T=null,F=null;try{if(T=await d.goto(u,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),T&&T.ok())try{F=await j(`read response body ${u}`,G(e),()=>T.text());}catch{F=null;}}catch(b){r.warn(`Navigation issue for ${u}; saving current DOM if available`,{url:u,error:String(b)});}if(s==="content-sync"&&T){let b=ie(T.url(),r,"content-sync-final-response",u)?.toString();if(b&&b!==g){if(i.deleteOnMissingUrls.has(u)){await ot(e,i,o,u,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${b}`,{url:u,finalResponseUrl:b,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${u} -> ${b}`)}}if(h){let b=T?.status()??null;if(b!==404){r.error(`Generated 404 request path returned unexpected status for ${u}`,{url:u,expectedStatus:404,status:b,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${u}`,{url:u,status:b,outputPath:K(e,u)});}else if(T&&!T.ok()){if(s==="content-sync"&&(T.status()===404||T.status()===410)&&i.deleteOnMissingUrls.has(u)){await ot(e,i,o,u,i.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${u}`,{url:u,status:T.status(),source:"content-sync-tombstone"}),await d.close();continue}if(s==="content-sync")throw new Error(`Required content-sync route returned HTTP ${T.status()}: ${u}`);r.skip("page",u,T.status(),t.pageDiscoverySources.get(u)),await d.close();continue}f?(r.info(`Rendering without JS execution for ${u}`,{url:u,mode:"no-js"}),await j(`wait for rendered page ${u}`,G(e),()=>xn(e,d))):await j(`wait for rendered page ${u}`,G(e),()=>kn(e,d)),t.stats.pagesRendered++;let $=await j(`extract rendered assets ${u}`,G(e),()=>In(e,d,g,r,!f));for(let b of $)q(e,t,b,r,u);let k=[...new Set([...$,...U])],_=F??await j(`capture rendered html ${u}`,G(e),()=>d.content()),y=s!=="single-url"&&s!=="content-sync"&&!h?Sn(e,u,_,r):[];if(s!=="single-url"&&s!=="content-sync"&&!h)for(let b of y)Ce(e,t,b,r,u);await Kt(e,u,_,t,r);let A=K(e,u);if(W(o,e,A),ue(o,A),i.trackManifest){let b=T?.headers();i.manifest.pages[u]={url:u,outputPath:A,contentType:rt(b),changeToken:v.changeToken?.supported?v.changeToken.token:null,tokenSource:v.changeToken?.supported?v.changeToken.tokenSource??null:null,sitemapLastmod:be(t.sitemapLastmodByPage[u])??null,discoveredPages:y,discoveredAssets:k,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await ne(e,i);}}catch(P){if(i.strictTargeted)throw r.error(`Failed required content-sync page ${u}`,{url:u,error:String(P)}),P;try{let g=d?await j(`capture partial html ${u}`,G(e),()=>d.content()):"";if(g&&g.trim()){await Kt(e,u,g,t,r);let h=K(e,u);if(W(o,e,h),ue(o,h),i.trackManifest){let f=i.manifest.pages[u]??i.previousManifest.pages[u];i.manifest.pages[u]={url:u,outputPath:h,contentType:f?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:be(t.sitemapLastmodByPage[u])??f?.sitemapLastmod??null,discoveredPages:f?.discoveredPages?[...f.discoveredPages]:[],discoveredAssets:f?.discoveredAssets?[...f.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId,rewriteComplete:!1},(i.enabled||i.strictTargeted)&&await ne(e,i);}r.warn(`Saved partial DOM for ${u}`,{url:u,error:String(P)});}}catch(g){r.error(`Could not save partial DOM for ${u}`,{url:u,error:String(g)});}r.error(`Failed page ${u}`,{url:u,error:String(P)});}finally{if(m.size>0&&await Promise.allSettled([...m]),p)try{let P=await p.cookies();c.addAll(Wt(S,P));}catch{}t.stats.pagesCompleted++,Mn(t)&&$n(t,r),d&&await j(`close page ${u}`,De(e),()=>d.close()).catch(P=>{r.warn(`Timed out while closing page ${u}`,{url:u,browserPid:l.pid,error:String(P)});}),p&&await j(`close page context ${u}`,De(e),()=>p.close()).catch(P=>{r.warn(`Timed out while closing page context for ${u}`,{url:u,browserPid:l.pid,error:String(P)});});}}}finally{await j("close worker browser",De(e),()=>a.close()).catch(u=>(r.warn("Timed out while closing crawl worker browser",{browserPid:l.pid,error:String(u)}),Rn(a,r,"browser.close timeout")));}}async function En(e){try{let t=await C.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Zt(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Dn(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Fn(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function mr(e){try{let t=await C.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Yt(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function jn(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Yt(s)-Yt(n)).map(s=>w.join(e,String(s.storedFileName))):[];return [...new Set([...r,w.join(e,"errors.jsonl"),w.join(e,"errors.json"),w.join(e,"errors.jsonl.gz"),w.join(e,"errors.json.gz")])]}async function Xt(e){let t=await mr(w.join(e,"job.json"));for(let r of jn(e,t)){let s=await En(r);if(s.length>0)return s}return []}async function qn(e){let t=w.join(e.logDir,"archive"),r;try{r=await C.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let o=w.join(t,i);if(!(await C.stat(o).catch(()=>null))?.isDirectory())continue;let a=await mr(w.join(o,"job.json"));if(!Dn(a))continue;let l=Fn(a);l>=n&&(n=l,s=o);}return s}async function Nn(e){let t=await qn(e),r=t!==""?await Xt(t):await Xt(e.logDir),s=new Set;for(let n of r)if(Zt(n.error)||Zt(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function _n(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await C.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Wn(e){if(!e)throw new Error("Content-sync crawl requires an immutable impact plan.");let t=JSON.parse(await C.readFile(e,"utf8"));return yt(t),t}function Bn(e,t){let r;try{r=new URL(e);}catch{return false}return t.archiveFamilies.some(s=>{let n=new URL(s.url);if(r.origin!==n.origin)return false;if(r.toString()===n.toString())return true;if(r.searchParams.has("paged")&&r.pathname===n.pathname)return /^\d+$/.test(r.searchParams.get("paged")||"");let i=n.pathname.replace(/\/+$/,""),o=r.pathname.slice(i.length).replace(/^\/+/,""),c=String(s.paginationBase||"page").replace(/^\/+|\/+$/g,"").replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return r.pathname.startsWith(`${i}/`)&&new RegExp(`^${c}/\\d+/?$`).test(o)})}function Oe(e,t,r){let s=r==="page"?K(e,t):Ne(e,t).originalFilePath;return ee(e.outputDir,s)}async function ot(e,t,r,s,n){let i=t.manifest.pages[s];if(!i)return;let o=ee(e.outputDir,i.outputPath);if(!o)throw new Error(`Manifest-owned page path escaped output root: ${s}`);let c=w.resolve(i.outputPath);Object.entries(t.manifest.pages).some(([l,u])=>l!==s&&!n.has(l)&&w.resolve(u.outputPath)===c)||(await C.rm(i.outputPath,{force:true}),r.deletedFiles.add(o)),delete t.manifest.pages[s];}function Qn(e,t){let r=new Set,s=[...t];for(;s.length>0;){let n=s.shift();if(r.has(n))continue;r.add(n);let i=e.assets[n];for(let o of i?.discoveredAssets??[])try{he(new URL(o))&&s.push(o);}catch{}}return r}async function Hn(e,t,r,s,n){let i=Qn(r.previousManifest,n);for(let o of i){if(t.successfulSitemaps.has(o))continue;let c=r.previousManifest.assets[o];if(!c)continue;let a=ee(e.outputDir,c.outputPath);if(!a)throw new Error(`Manifest-owned sitemap path escaped output root: ${o}`);await C.rm(c.outputPath,{force:true}),s.deletedFiles.add(a),delete r.manifest.assets[o];}await ne(e,r);}async function zn(e){try{let t=await C.readFile(w.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Jn(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await C.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await C.unlink(w.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function er(e,t,r,s,n){let i=N(e,r,e.sourceOrigin,s,"manual-url");i&&(he(i)?je(e,t,i.toString(),s,n):L(e,i)?q(e,t,i.toString(),s,n):oe(i)?Ce(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function wr(e=process.argv.slice(2)){let t=[],r=await ke(void 0,{onRemotePublisherConfigDiagnostic:y=>{t.push(y);}}),s=St(r.sitemapPaths),n=t.at(-1)??null,i=Rs(e),o=i.mode==="content-sync"?await Wn(i.contentSyncPlanPath):null;if(i.resumeRewrite&&i.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let c=[];i.mode==="retry-timeouts"&&(c=await Nn(r)),i.mode==="single-url"&&(c=await _n(i.urls)),o&&(c=[...o.directRenderUrls,...o.archiveFamilyUrls]);let a=i.crawlMode==="incremental"&&i.mode==="full",l=a&&typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!=="";if(o&&r.subscriptionType!=="PROFESSIONAL"&&r.subscriptionType!=="AGENCY")throw new Error("Content-sync requires an active Professional or Agency subscription.");let u=(i.mode==="full"||i.mode==="single-url"||i.mode==="content-sync")&&!i.resumeRewrite,p=i.mode!=="full"||i.resumeRewrite||l,d=p?await zn(r):{},S=l||i.mode==="single-url"||i.mode==="content-sync"?await $s(r):nr(),m=new Set(c),P=new Set;if(o){for(let y of o.archiveFamilies)if(y.foundPosts===0)for(let A of y.pageUrls)P.add(A);for(let y of Object.keys(S.pages))Bn(y,o)&&!m.has(y)&&(c.push(y),P.add(y));}let g={enabled:l,trackManifest:u,manifest:st(S),previousManifest:st(S),runId:o?.jobId??`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,manifestSaveChain:Promise.resolve(),strictTargeted:!!o,requiredUrls:m,deleteOnMissingUrls:P};await Jn(r.logDir);let h=new le(r.logDir,r.logLevel);if(n){let y={status:n.status,virtualAssetBaseUrl:n.virtualAssetBaseUrl,subscriptionType:n.subscriptionType??null,error:n.error,requests:n.requests,source:"remote-publisher-config"};n.status==="loaded"&&n.subscriptionType?h.info("Loaded remote WP Suite publisher configuration.",y):h.warn("Remote WP Suite publisher configuration is unavailable or has no active subscription type.",y);}if(a&&!l){let y="Incremental crawl requires an active remote publisher subscription; refusing to fall back to a destructive full crawl.";throw h.error(y,{requestedMode:i.crawlMode,subscriptionType:r.subscriptionType??null,source:"incremental-subscription-guard"}),await h.flush(),new Error(y)}let f={pageQueue:[],queuedPages:new Set,donePages:new Set,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,successfulSitemaps:new Set,resolvedSitemapRoots:new Set,sitemapIndexUrls:new Set,sitemapLeafUrls:new Set,sitemapLastmodByPage:{},assetMap:{...d},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},R=Ms(),v=new $e,O=As();if(O&&!i.resumeRewrite&&await Le(O,v.artifact()),await ht(r),await mt(r,{startedAt:new Date().toISOString(),outputDir:w.resolve(r.outputDir)}),p||await C.rm(r.outputDir,{recursive:true,force:true}),await C.mkdir(r.outputDir,{recursive:true}),o){let y=new Set([...o.deleteUrls,...g.deleteOnMissingUrls]);for(let A of o.deleteUrls)await ot(r,g,R,A,y);}if(i.resumeRewrite)h.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:i.mode,crawlMode:i.crawlMode});else {let y=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{if(h.mark("discovery"),i.mode==="full"?(s.forEach(b=>je(r,f,b,h)),await Gt(r,y,f,h,R,g),r.seedPaths.forEach(b=>er(r,f,b,h,"seed-path")),r.generated404RequestPath&&Ce(r,f,r.generated404RequestPath,h,"generated-404-request-path")):(c.forEach(b=>er(r,f,b,h,"cli")),o?.sitemapUrls.forEach(b=>je(r,f,b,h,"content-sync-impact")),await Gt(r,y,f,h,R,g,i.mode==="content-sync",o?.sitemapUrls??[]),o&&await Hn(r,f,g,R,o.sitemapUrls)),h.endMark("discovery",{pages:f.pageQueue.length,assets:f.assetQueue.length,sitemaps:f.sitemapQueue.length}),h.summary(`Queued ${f.pageQueue.length} pages, ${f.assetQueue.length} assets, ${f.sitemapQueue.length} sitemaps.`,{mode:i.mode,crawlMode:i.crawlMode,queuedPages:f.pageQueue.length,queuedAssets:f.assetQueue.length,queuedSitemaps:f.sitemapQueue.length}),i.mode==="retry-timeouts"&&f.pageQueue.length===0&&f.assetQueue.length===0&&f.sitemapQueue.length===0){h.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:i.mode,crawlMode:i.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await h.flush(),O&&await Le(O,v.artifact());return}h.mark("render-pages");let A=Array.from({length:Math.max(1,r.concurrency)},()=>On(r,f,h,i.mode,y,g,R,v));if(await Promise.all(A),O&&await Le(O,v.artifact()),h.endMark("render-pages",{pages:f.stats.pagesCompleted}),h.mark("download-assets"),h.progress("Asset download phase started.",{phase:"download-assets",doneAssets:f.doneAssets.size,assetsQueued:f.stats.assetsQueued,assetQueue:f.assetQueue.length}),await Os(r,y,g,[...f.assetQueue],h),await bn(r,y,f,h,g,R),h.endMark("download-assets",{assets:f.doneAssets.size}),g.enabled&&r.maxPages===0&&await qs(r,f,g,h,R),i.mode==="full"){let b=await yn(r,f,h);for(let x of b)W(R,r,x);}}finally{await j("dispose crawl request context",De(r),()=>y.dispose()).catch(A=>{h.warn("Timed out while disposing crawl request context",{error:String(A)});});}}g.trackManifest&&await ne(r,g),await C.writeFile(w.join(r.outputDir,"asset-map.json"),JSON.stringify(f.assetMap,null,2),"utf8"),JSON.stringify(d,null,2)!==JSON.stringify(f.assetMap,null,2)&&W(R,r,w.join(r.outputDir,"asset-map.json"));let U=i.mode==="single-url"||i.mode==="content-sync"?Vs(r,f):g.enabled&&!i.resumeRewrite?Ns(r,g,R,d,f.assetMap):void 0;h.mark("rewrite-text");let T=i.mode==="single-url"||i.mode==="content-sync"?`Rewriting text files touched by URL crawl: 0/${U?.length??0}`:"Rewriting text files...";h.progress(T,{phase:"rewrite-text",index:0,totalFiles:U?.length,changedTextFiles:0}),await Jt(T);let F=Date.now(),$=await $t(r,f.assetMap,{files:U,previousAssetMap:d,onStart:({totalFiles:y,workerCount:A})=>{h.info(`Rewrite phase scanning ${y} text files.`,{phase:"rewrite-text",totalFiles:y,workerCount:A});},onProgress:async({index:y,totalFiles:A,changedTextFiles:b,file:x,changed:M})=>{M&&W(R,r,w.join(r.outputDir,x)),h.checkpoint(`Rewriting text file ${y}/${A}`,{phase:"rewrite-text",index:y,totalFiles:A,changedTextFiles:b,file:x}),h.debug(`${M?"Rewrote":"Checked unchanged"} text file ${y}/${A}: ${x}`,{phase:"rewrite-text",index:y,totalFiles:A,changedTextFiles:b,file:x,changed:M});let E=Date.now();if(!(y===1||y===A||y%50===0||E-F>=5e3))return;F=E;let Z=`Rewriting text files: ${y}/${A}`;h.progress(Z,{phase:"rewrite-text",index:y,totalFiles:A,changedTextFiles:b,file:x}),await Jt(Z);}});g.trackManifest&&(_s(g,U),await ne(r,g)),h.endMark("rewrite-text",{changedTextFiles:$});let k=o?[...R.deletedFiles].sort():g.enabled&&!i.resumeRewrite?await Bs(r,f,g,R,h):[],_=[...new Set((U||[]).map(y=>ee(r.outputDir,w.isAbsolute(y)?y:w.join(r.outputDir,y))).filter(y=>y!==null))].sort();if(o){let y=[...R.changedFiles].sort(),A=new Set(y),b=new Set(o.directRenderUrls.map(I=>Oe(r,I,"page")).filter(I=>!!I&&A.has(I))),x=new Set(o.archiveFamilyUrls.map(I=>Oe(r,I,"page")).filter(I=>!!I&&A.has(I))),M=new Set([...f.sitemapLeafUrls].map(I=>Oe(r,I,"asset")).filter(I=>!!I&&A.has(I))),E=new Set([...f.sitemapIndexUrls].map(I=>Oe(r,I,"asset")).filter(I=>!!I&&A.has(I))),me=new Set([...b,...x,...M,...E]),Z=y.filter(I=>!me.has(I)),ae=[...new Set([...o.directRenderUrls,...o.archiveFamilyUrls,...f.resolvedSitemapRoots,...f.sitemapLeafUrls,...f.sitemapIndexUrls,...o.deleteUrls].map(I=>{let Ae=new URL(I);return `${Ae.pathname}${Ae.search}`}))].sort();await _e(r,{schemaVersion:2,generatedAt:new Date().toISOString(),outputDir:w.resolve(r.outputDir),runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,changedFiles:y,deletedFiles:k,rewriteTargets:_,consumerId:o.consumerId,fromSequence:o.fromSequence,toSequence:o.toSequence,impactHash:o.impactHash,resolvedSitemapUrls:[...f.resolvedSitemapRoots],uploadGroups:{assets:Z,content:[...b].sort(),listings:[...x].sort(),sitemapLeaves:[...M].sort(),sitemapIndexes:[...E].reverse()},invalidationPaths:ae});}else await _e(r,{schemaVersion:1,generatedAt:new Date().toISOString(),outputDir:w.resolve(r.outputDir),runMode:i.mode,crawlMode:i.crawlMode,fullSyncRequired:!(g.enabled&&!i.resumeRewrite),changedFiles:g.enabled&&!i.resumeRewrite?[...R.changedFiles].sort():[],deletedFiles:g.enabled&&!i.resumeRewrite?k:[],rewriteTargets:g.enabled&&!i.resumeRewrite?_:[]});await wt(r),h.summary(i.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${$} text files.`:`Done. Rendered ${f.stats.pagesRendered} pages, processed ${f.doneSitemaps.size} sitemaps and ${f.doneAssets.size} assets.`,{mode:i.mode,crawlMode:i.crawlMode,resumeRewrite:i.resumeRewrite,...f.stats,pagesRendered:f.stats.pagesRendered,donePages:f.stats.pagesCompleted,claimedPages:f.donePages.size,doneSitemaps:f.doneSitemaps.size,doneAssets:f.doneAssets.size,changedTextFiles:$}),await h.flush();}wr().catch(async e=>{let t=V(e instanceof Error?e.message:String(e));console.error(t);try{let s=(await ke().catch(()=>null))?.logDir??"logs",n=new le(s,"debug");n.error(`Unhandled error: ${t}`,e instanceof Error&&e.stack?{stack:V(e.stack)}:void 0),await n.flush();}catch{}process.exit(1);});
|