@smart-cloud/publisher-exporter 1.1.73 → 1.1.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/content-sync.js +15 -15
- package/dist/crawl.js +13 -13
- package/dist/deploy.js +6 -6
- package/dist/invalidate.js +1 -1
- package/dist/queue-runner.js +1 -1
- package/package.json +1 -1
- package/publisher.config.example.json +3 -0
package/dist/crawl.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import {request,chromium}from'playwright';import M,{readFile}from'fs/promises';import y from'path';import {gunzipSync}from'zlib';import {setTimeout as setTimeout$1}from'timers/promises';import Ss from'fast-glob';import Gr from'mime-types';import xr,{createHash}from'crypto';import os from'http';import is from'https';import {availableParallelism}from'os';import {Worker}from'worker_threads';import {LambdaClient,InvokeCommand}from'@aws-sdk/client-lambda';import {DynamoDBClient,GetItemCommand}from'@aws-sdk/client-dynamodb';import {S3Client,GetObjectCommand,PutObjectCommand,CopyObjectCommand}from'@aws-sdk/client-s3';import {fromTemporaryCredentials}from'@aws-sdk/credential-providers';var Kn=".deploy-plan.json",Jn="deploy-plan.json",Gn=".crawl-incomplete.json",Zn="crawl-incomplete.json";function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function dr(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim(),r=y.posix.normalize(t);return !t||!r||r==="."||/^[a-zA-Z]:/.test(r)||r.startsWith("../")||r===".."?null:r}function Yn(e){let t=String(e||"").trim();if(!t.startsWith("/")||t.startsWith("//")||t.includes("*")||t.includes("#"))return null;try{let r=new URL(t,"https://content-sync.invalid");return r.origin!=="https://content-sync.invalid"?null:`${y.posix.normalize(r.pathname)}${r.search}`}catch{return null}}function ae(e){return [...new Set(e.map(dr).filter(t=>t!==null))]}function Xn(e,t){let r={generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:y.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ae(t.changedFiles||[]),deletedFiles:ae(t.deletedFiles||[]),rewriteTargets:ae(t.rewriteTargets||[]),revalidateFiles:ae(t.revalidateFiles||[])};return t.schemaVersion===2?{...r,schemaVersion:2,runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,consumerId:String(t.consumerId||"").trim(),fromSequence:Math.max(0,Number(t.fromSequence)||0),toSequence:Math.max(0,Number(t.toSequence)||0),impactHash:String(t.impactHash||"").trim(),resolvedSitemapUrls:[...new Set((t.resolvedSitemapUrls||[]).map(n=>String(n||"").trim()).filter(n=>{try{let s=new URL(n);return s.protocol==="http:"||s.protocol==="https:"}catch{return false}}))],uploadGroups:{assets:ae(t.uploadGroups?.assets||[]),content:ae(t.uploadGroups?.content||[]),listings:ae(t.uploadGroups?.listings||[]),resources:ae(t.uploadGroups?.resources||[]),sitemapLeaves:ae(t.uploadGroups?.sitemapLeaves||[]),sitemapIndexes:ae(t.uploadGroups?.sitemapIndexes||[])},invalidationPaths:[...new Set((t.invalidationPaths||[]).map(Yn).filter(n=>n!==null))].sort()}:{schemaVersion:1,...r,runMode:t.runMode}}function pr(e){let t=y.join(y.resolve(e.outputDir),Kn),r=lr();return r?[y.join(r,Jn),t]:[t]}function mr(e){let t=y.join(y.resolve(e.outputDir),Gn),r=lr();return r?[y.join(r,Zn),t]:[t]}function Y(e,t){let r=y.resolve(e),n=y.resolve(t),s=y.relative(r,n).replace(/\\/g,"/");return dr(s)}async function kt(e,t){let r=Xn(e,t);for(let n of pr(e)){await M.mkdir(y.dirname(n),{recursive:true});let s=`${n}.${process.pid}.${Date.now()}.tmp`;await M.writeFile(s,JSON.stringify(r,null,2),"utf8"),await M.rename(s,n);}}async function gr(e){for(let t of pr(e))try{await M.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}async function Ct(e,t){let r={schemaVersion:1,startedAt:String(t.startedAt||new Date().toISOString()),outputDir:y.resolve(e.outputDir)};for(let n of mr(e)){await M.mkdir(y.dirname(n),{recursive:true});let s=`${n}.${process.pid}.tmp`;await M.writeFile(s,JSON.stringify(r,null,2),"utf8"),await M.rename(s,n);}}async function hr(e){for(let t of mr(e))try{await M.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function ts(e){return new URL(e).pathname.split("/").filter(Boolean).some(r=>{try{return decodeURIComponent(r).endsWith("__trashed")}catch{return r.endsWith("__trashed")}})}function xt(e){return Array.isArray(e)?e.map(xt):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[r])=>t.localeCompare(r)).map(([t,r])=>[t,xt(r)])):e}function rs(e){return JSON.stringify(xt(e))}function ns(e){return createHash("sha256").update(rs(e)).digest("hex")}function fr(e){let{impactHash:t,...r}=e;if(e.schemaVersion!==1||!t)throw new Error("Invalid content-sync impact plan schema.");if(!Array.isArray(e.resourceRenderUrls)||!Array.isArray(e.resourceDeleteUrls))throw new Error("Invalid content-sync resource impact contract.");if(ns(r)!==t)throw new Error("Content-sync impact plan hash mismatch.");for(let n of [...e.directRenderUrls,...e.deleteUrls,...e.resourceRenderUrls,...e.resourceDeleteUrls])if(ts(n))throw new Error(`Content-sync impact contains a WordPress trash alias instead of the last public URL: ${n}`)}var Tt=["/sitemap_index.xml","/wp-sitemap.xml","/sitemap.xml"];function wr(e){let t=new Set;for(let r of [...e,...Tt]){let n=String(r||"").trim();n&&t.add(n);}return [...t]}function At(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function yr(e){if(!e||!/^https?:\/\//i.test(e))return false;try{return At(new URL(e))}catch{return false}}function Sr(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function as(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let n=r.endsWith("/"),s=r.replace(/^\/+/,"").split("/").map(i=>i.replace(/[^A-Za-z0-9._-]/g,"")).filter(i=>i.length>0&&i!=="."&&i!=="..");if(s.length===0)return "";let o=`/${s.join("/")}`;return n&&y.extname(o)===""&&(o+="/"),o==="/"?"":o}function It(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function us(e){if(!e||typeof e!="object")return {};let t={};for(let[r,n]of Object.entries(e)){let s=r.trim();if(!s||!n||typeof n!="object")continue;let o=n,i={};if(typeof o.targetOrigin=="string"){let a=Sr(o.targetOrigin);a&&(i.targetOrigin=a);}let u=It(o.extraReplacements);if(Object.keys(u).length>0&&(i.extraReplacements=u),o.s3&&typeof o.s3=="object"){let a={},l=o.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(i.s3=a);}if(o.cloudFront&&typeof o.cloudFront=="object"){let a={},l=o.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(i.cloudFront=a);}t[s]=i;}return t}function Ot(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function cs(e,t){if(!e||typeof e!="object")return null;let r=e,n=r.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url"&&n!=="content-sync")return null;let s=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(s)||s<1)return null;let o=String(r.id??`${n}-${t+1}`).trim();if(!o)return null;let i=String(r.deploymentProfile??"").trim(),u=String(r.url??"").trim(),a=Array.isArray(r.postTypes)?[...new Set(r.postTypes.map(d=>String(d??"").trim().toLowerCase().replace(/[^a-z0-9_-]/g,"")).filter(Boolean))]:[],l=Array.isArray(r.listingPaths)?[...new Set(r.listingPaths.map(d=>String(d??"").trim()).filter(Boolean))]:[];return n==="content-sync"&&a.length===0?null:{id:o,enabled:r.enabled!==false,command:n,intervalMinutes:s,...n==="publish"||n==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(n==="publish"||n==="deploy"||n==="invalidate"||n==="content-sync")&&i?{deploymentProfile:i}:{},...u?{url:u}:{},...a.length>0?{postTypes:a}:{},...l.length>0?{listingPaths:l}:{},...typeof r.includeSubsites=="boolean"?{includeSubsites:r.includeSubsites}:{},...typeof r.includePostTypeArchives=="boolean"?{includePostTypeArchives:r.includePostTypeArchives}:{},...typeof r.includeTaxonomyArchives=="boolean"?{includeTaxonomyArchives:r.includeTaxonomyArchives}:{},...typeof r.includeAuthorArchives=="boolean"?{includeAuthorArchives:r.includeAuthorArchives}:{},...typeof r.includeDateArchives=="boolean"?{includeDateArchives:r.includeDateArchives}:{},...typeof r.includePostsPage=="boolean"?{includePostsPage:r.includePostsPage}:{},...typeof r.includeSitemapChain=="boolean"?{includeSitemapChain:r.includeSitemapChain}:{}}}function Pr(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((n,s)=>cs(n,s)).filter(n=>!!n):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function ls(e){let t=e&&typeof e=="object"?e:{},r=t.render&&typeof t.render=="object"?t.render:{},n=Number(r.concurrency??0),s=Number(r.batchSize??0),o=Number(r.maxAttempts??0),i=t.rewrite&&typeof t.rewrite=="object"?t.rewrite:{},u=t.deploy&&typeof t.deploy=="object"?t.deploy:{},a=(l,d)=>({enabled:typeof l.enabled=="boolean"?l.enabled:r.enabled===true,concurrency:Math.min(100,Math.max(1,Number(l.concurrency||d.concurrency))),batchSize:Math.min(500,Math.max(1,Number(l.batchSize||d.batchSize))),maxAttempts:Math.min(5,Math.max(1,Number(l.maxAttempts||2)))});return {render:{enabled:r.enabled===true,...Number.isInteger(n)&&n>0?{concurrency:n}:{},batchSize:Number.isInteger(s)&&s>0?Math.min(20,s):5,...Number.isInteger(o)&&o>0?{maxAttempts:Math.min(5,o)}:{}},rewrite:a(i,{concurrency:8,batchSize:25}),deploy:a(u,{concurrency:8,batchSize:50})}}function ds(e){if(!e||typeof e!="object")return;let t=e,r={};for(let s of ["accountId","siteId"]){let o=t[s];typeof o=="string"&&o.trim()!==""&&(r[s]=o.trim());}let n=Number(t.lastUpdate??0);return Number.isFinite(n)&&n>0&&(r.lastUpdate=Math.floor(n)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function Mt(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),n=String(t.siteId??"").trim(),s=t.subscriber===true,o=ds(t.siteSettings),i={...o??{},...r?{accountId:o?.accountId??r}:{},...n?{siteId:o?.siteId??n}:{},...o?.subscriber===true||s?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let l=String(t.runtimeToken??t.nonce??"").trim();l&&(u.runtimeToken=l);let d=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();d&&(u.virtualAssetBaseUrl=d),Object.keys(i).length>0&&(u.siteSettings=i);let c=Ot(t.subscriptionType);return c&&(u.subscriptionType=c),u}function ps(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Rr(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function ms(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[n,s]of new Headers(t.headers).entries())r.set(n,s);return r}async function vr(e,t,r=5){let n=new URL(Rr(e)),s=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),o=n.protocol==="http:"?os:is,i=ms(e,t);return await new Promise((u,a)=>{let l=o.request(n,{method:s,headers:Object.fromEntries(i.entries()),...n.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let c=[];d.on("data",m=>{c.push(Buffer.isBuffer(m)?m:Buffer.from(m));}),d.on("error",a),d.on("end",()=>{let m=d.statusCode??0,h=d.headers.location;if(r>0&&h&&[301,302,303,307,308].includes(m)){u(vr(new URL(h,n),{method:m===303?"GET":s},r-1));return}let f=new Headers;for(let[b,p]of Object.entries(d.headers))Array.isArray(p)?f.set(b,p.join(", ")):typeof p=="string"&&f.set(b,p);u(new Response(Buffer.concat(c),{status:m,statusText:d.statusMessage??"",headers:f}));});});l.on("error",a),l.end();})}function gs(e){let t=Mt(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},n=String(r.accountId??"").trim(),s=String(r.siteId??"").trim(),o=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...o?{virtualAssetBaseUrl:o}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!n||!s||!o)return false;ps(e.sourceOrigin);let i=globalThis,a={...i.WpSuite??{},siteSettings:r,uploadUrl:o};return t.apiBase&&(a.apiBase=t.apiBase),i.WpSuite=a,true}async function hs(e){let t=String(e.wpsuite?.virtualAssetBaseUrl??e.wpsuite?.uploadUrl??"").trim();if(!gs(e)){let s=e.wpsuite?.siteSettings,o=[String(s?.accountId??"").trim()?"":"accountId",String(s?.siteId??"").trim()?"":"siteId",t?"":"virtualAssetBaseUrl"].filter(Boolean);return {remote:null,diagnostic:{status:"invalid-runtime",virtualAssetBaseUrl:t,subscriptionType:null,error:`Missing WP Suite runtime field(s): ${o.join(", ")}.`,requests:[]}}}let r=globalThis.fetch,n=[];try{typeof r=="function"&&/^https?:\/\//i.test(t)&&(globalThis.fetch=(async(a,l)=>{let d=Rr(a);if(d.startsWith(t)){let c=(()=>{try{return new URL(d).pathname}catch{return d}})();try{let m=e.ignoreHttpsErrors&&d.startsWith("https://")?await vr(a,l):await r(a,l);return n.push({path:c,status:m.status,ok:m.ok,error:null}),m}catch(m){throw n.push({path:c,status:null,ok:!1,error:m instanceof Error?m.message:String(m)}),m}}return r(a,l)}));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),o=await s("publisher"),i=o&&typeof o=="object"?o:null,u=Ot(i?.subscriptionType);return {remote:i,diagnostic:{status:i?"loaded":"unavailable",virtualAssetBaseUrl:t,subscriptionType:u??null,error:i?null:"WP Suite publisher config loader returned no configuration.",requests:n}}}catch(s){return {remote:null,diagnostic:{status:"unavailable",virtualAssetBaseUrl:t,subscriptionType:null,error:s instanceof Error?s.message:String(s),requests:n}}}finally{typeof r=="function"&&(globalThis.fetch=r);}}function fs(e,t){let r=us(t?.deploymentProfiles),n=String(t?.defaultDeploymentProfile??"").trim(),s=r[n]?n:"",o=Ot(t?.subscriptionType),i=Mt(e.wpsuite);return {...e,scheduler:Pr(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:s,...o?{subscriptionType:o}:{},wpsuite:{...i,...i.siteSettings||o?{siteSettings:{...i.siteSettings??{},...i.siteSettings?.subscriber===true||o==="PROFESSIONAL"||o==="AGENCY"?{subscriber:true}:{}}}:{},...o?{subscriptionType:o}:{}}}}function Et(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let s=n.split("/").map(o=>o.trim()).filter(o=>o.length>0&&o!=="."&&o!=="..");return s.length>0?s.join("/"):t}function ws(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function br(e,t,r){let n=String(t||"").trim();return n&&y.isAbsolute(n)?y.resolve(n):y.resolve(e,Et(n,r))}async function rt(e,t={}){let r=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await M.readFile(r,"utf8"),s=JSON.parse(n);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Sr(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=as(s.generated404RequestPath),s.sitemapPaths||=[...Tt],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[];let o=Number(s.processingConcurrency)>0?Math.min(100,Math.max(1,Number(s.processingConcurrency))):null;s.concurrency=o??s.concurrency??1,s.maxPages||=0,s.extraReplacements=It(s.extraReplacements),s.postCrawlCopyMap=It(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=o??(Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency),s.rewriteConcurrency=o??(Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency),s.wpsuite=Mt(s.wpsuite),s.scheduler=Pr(void 0),s.remoteWorkers=ls(s.remoteWorkers),typeof s.lambdaDelegationEnabled=="boolean"&&(s.remoteWorkers.render??={},s.remoteWorkers.rewrite??={},s.remoteWorkers.deploy??={},s.remoteWorkers.render.enabled=s.lambdaDelegationEnabled,s.remoteWorkers.rewrite.enabled=s.lambdaDelegationEnabled,s.remoteWorkers.deploy.enabled=s.lambdaDelegationEnabled),o!==null&&(s.processingConcurrency=o,s.remoteWorkers.render??={},s.remoteWorkers.rewrite??={},s.remoteWorkers.deploy??={},s.remoteWorkers.render.concurrency=o,s.remoteWorkers.rewrite.concurrency=o,s.remoteWorkers.deploy.concurrency=o),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let i=await hs(s);t.onRemotePublisherConfigDiagnostic?.(i.diagnostic);let u=fs(s,i.remote),a=ws(),l=a?y.resolve(a,".."):"";return l?(u.outputDir=br(l,u.outputDir,"export"),u.logDir=br(l,u.logDir,"logs")):(y.isAbsolute(u.outputDir)||(u.outputDir=Et(u.outputDir,"export")),y.isAbsolute(u.logDir)||(u.logDir=Et(u.logDir,"logs"))),u}function ue(e,t=10){return xr.createHash("sha1").update(e).digest("hex").slice(0,t)}async function ys(e){await M.mkdir(y.dirname(e),{recursive:true});}async function he(e,t,r){await ys(e);let n=y.join(y.dirname(e),`.${y.basename(e)}.${process.pid}.${Date.now()}.${xr.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await M.writeFile(n,t,r??"utf8"):await M.writeFile(n,t),await M.rename(n,e);}catch(s){throw await M.unlink(n).catch(()=>{}),s}}function ce(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(y.extname(e).toLowerCase())}function fe(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function je(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function nt(e){return e.replace(/\//g,"\\/")}function kr(e){return e.replace(/\//g,"\\\\/")}function bs(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Cr(e,t,r){if(!t)return;e[t]=r;let n=nt(t),s=nt(r);e[n]=s;let o=kr(t),i=kr(r);e[o]=i;let u=je(t),a=je(r);e[u]=a;let l=je(n),d=je(s);e[l]=d;let c=je(o),m=je(i);e[c]=m;}function st(e,t,r){Cr(e,t,r);let n=bs(t);n&&n!==t&&Cr(e,n,r);}function $t(e,t){let r=t.startsWith("/")?t:`/${t}`,n=(e||"").replace(/\/$/,"");return !n||n==="."||n==="/"?r:`${n}${r}`}function Tr(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ar(e,t,r){if(!t)return r;let n=y.dirname(y.resolve(t)),s=y.resolve(e,r.replace(/^\/+/,"")),o=y.relative(n,s).replace(/\\/g,"/");return o?(o.startsWith(".")||(o=`./${o}`),o):"."}async function Lt(e,t){return (t?[...new Set(t)].map(n=>y.resolve(n)).filter(n=>n.startsWith(y.resolve(e.outputDir))).map(n=>y.relative(e.outputDir,n).replace(/\\/g,"/")).filter(n=>n.length>0):await Ss(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(n=>n!=="asset-map.json"&&ce(n))}var ks=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Cs=new Set([".html",".htm"]),Ir="WPSuite.io Static Publisher",xs=Ir.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Ts(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function As(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function Is(e,t,r){return As(t)?e.replace(new RegExp(`(?<!:)${Ts(t)}`,"g"),r):e.split(t).join(r)}function Es(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let s=y.extname(r).toLowerCase();return Cs.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Os(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${xs}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ir}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let n=e.includes(`\r
|
|
1
|
+
import {request,chromium}from'playwright';import $,{readFile}from'fs/promises';import y from'path';import {gunzipSync}from'zlib';import {setTimeout as setTimeout$1}from'timers/promises';import vs from'fast-glob';import Gr from'mime-types';import xr,{createHash}from'crypto';import us from'http';import cs from'https';import {availableParallelism}from'os';import {Worker}from'worker_threads';import {LambdaClient,InvokeCommand}from'@aws-sdk/client-lambda';import {DynamoDBClient,GetItemCommand}from'@aws-sdk/client-dynamodb';import {S3Client,GetObjectCommand,PutObjectCommand,CopyObjectCommand}from'@aws-sdk/client-s3';import {fromTemporaryCredentials}from'@aws-sdk/credential-providers';var Zn=".deploy-plan.json",Yn="deploy-plan.json",Xn=".crawl-incomplete.json",es="crawl-incomplete.json";function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function dr(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim(),r=y.posix.normalize(t);return !t||!r||r==="."||/^[a-zA-Z]:/.test(r)||r.startsWith("../")||r===".."?null:r}function ts(e){let t=String(e||"").trim();if(!t.startsWith("/")||t.startsWith("//")||t.includes("*")||t.includes("#"))return null;try{let r=new URL(t,"https://content-sync.invalid");return r.origin!=="https://content-sync.invalid"?null:`${y.posix.normalize(r.pathname)}${r.search}`}catch{return null}}function ae(e){return [...new Set(e.map(dr).filter(t=>t!==null))]}function rs(e,t){let r={generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:y.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ae(t.changedFiles||[]),deletedFiles:ae(t.deletedFiles||[]),rewriteTargets:ae(t.rewriteTargets||[]),revalidateFiles:ae(t.revalidateFiles||[])};return t.schemaVersion===2?{...r,schemaVersion:2,runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,consumerId:String(t.consumerId||"").trim(),fromSequence:Math.max(0,Number(t.fromSequence)||0),toSequence:Math.max(0,Number(t.toSequence)||0),impactHash:String(t.impactHash||"").trim(),resolvedSitemapUrls:[...new Set((t.resolvedSitemapUrls||[]).map(n=>String(n||"").trim()).filter(n=>{try{let s=new URL(n);return s.protocol==="http:"||s.protocol==="https:"}catch{return false}}))],uploadGroups:{assets:ae(t.uploadGroups?.assets||[]),content:ae(t.uploadGroups?.content||[]),listings:ae(t.uploadGroups?.listings||[]),resources:ae(t.uploadGroups?.resources||[]),sitemapLeaves:ae(t.uploadGroups?.sitemapLeaves||[]),sitemapIndexes:ae(t.uploadGroups?.sitemapIndexes||[])},invalidationPaths:[...new Set((t.invalidationPaths||[]).map(ts).filter(n=>n!==null))].sort()}:{schemaVersion:1,...r,runMode:t.runMode}}function pr(e){let t=y.join(y.resolve(e.outputDir),Zn),r=lr();return r?[y.join(r,Yn),t]:[t]}function mr(e){let t=y.join(y.resolve(e.outputDir),Xn),r=lr();return r?[y.join(r,es),t]:[t]}function Y(e,t){let r=y.resolve(e),n=y.resolve(t),s=y.relative(r,n).replace(/\\/g,"/");return dr(s)}async function xt(e,t){let r=rs(e,t);for(let n of pr(e)){await $.mkdir(y.dirname(n),{recursive:true});let s=`${n}.${process.pid}.${Date.now()}.tmp`;await $.writeFile(s,JSON.stringify(r,null,2),"utf8"),await $.rename(s,n);}}async function gr(e){for(let t of pr(e))try{await $.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}async function Tt(e,t){let r={schemaVersion:1,startedAt:String(t.startedAt||new Date().toISOString()),outputDir:y.resolve(e.outputDir)};for(let n of mr(e)){await $.mkdir(y.dirname(n),{recursive:true});let s=`${n}.${process.pid}.tmp`;await $.writeFile(s,JSON.stringify(r,null,2),"utf8"),await $.rename(s,n);}}async function hr(e){for(let t of mr(e))try{await $.unlink(t);}catch(r){if(r.code!=="ENOENT")throw r}}function ss(e){return new URL(e).pathname.split("/").filter(Boolean).some(r=>{try{return decodeURIComponent(r).endsWith("__trashed")}catch{return r.endsWith("__trashed")}})}function At(e){return Array.isArray(e)?e.map(At):e&&typeof e=="object"?Object.fromEntries(Object.entries(e).sort(([t],[r])=>t.localeCompare(r)).map(([t,r])=>[t,At(r)])):e}function os(e){return JSON.stringify(At(e))}function is(e){return createHash("sha256").update(os(e)).digest("hex")}function fr(e){let{impactHash:t,...r}=e;if(e.schemaVersion!==1||!t)throw new Error("Invalid content-sync impact plan schema.");if(!Array.isArray(e.resourceRenderUrls)||!Array.isArray(e.resourceDeleteUrls))throw new Error("Invalid content-sync resource impact contract.");if(is(r)!==t)throw new Error("Content-sync impact plan hash mismatch.");for(let n of [...e.directRenderUrls,...e.deleteUrls,...e.resourceRenderUrls,...e.resourceDeleteUrls])if(ss(n))throw new Error(`Content-sync impact contains a WordPress trash alias instead of the last public URL: ${n}`)}var It=["/sitemap_index.xml","/wp-sitemap.xml","/sitemap.xml"];function wr(e){let t=new Set;for(let r of [...e,...It]){let n=String(r||"").trim();n&&t.add(n);}return [...t]}function Et(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function yr(e){if(!e||!/^https?:\/\//i.test(e))return false;try{return Et(new URL(e))}catch{return false}}function Sr(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function ls(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let n=r.endsWith("/"),s=r.replace(/^\/+/,"").split("/").map(i=>i.replace(/[^A-Za-z0-9._-]/g,"")).filter(i=>i.length>0&&i!=="."&&i!=="..");if(s.length===0)return "";let o=`/${s.join("/")}`;return n&&y.extname(o)===""&&(o+="/"),o==="/"?"":o}function Ot(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function ds(e){if(!e||typeof e!="object")return {};let t={};for(let[r,n]of Object.entries(e)){let s=r.trim();if(!s||!n||typeof n!="object")continue;let o=n,i={};if(typeof o.targetOrigin=="string"){let a=Sr(o.targetOrigin);a&&(i.targetOrigin=a);}let u=Ot(o.extraReplacements);if(Object.keys(u).length>0&&(i.extraReplacements=u),o.s3&&typeof o.s3=="object"){let a={},l=o.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(i.s3=a);}if(o.cloudFront&&typeof o.cloudFront=="object"){let a={},l=o.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(d=>String(d??"").trim()).filter(d=>d.length>0)),Object.keys(a).length>0&&(i.cloudFront=a);}t[s]=i;}return t}function $t(e){return e==="PROFESSIONAL"||e==="AGENCY"?e:void 0}function ps(e,t){if(!e||typeof e!="object")return null;let r=e,n=r.command;if(n!=="publish"&&n!=="crawl"&&n!=="deploy"&&n!=="invalidate"&&n!=="retry-timeouts"&&n!=="url"&&n!=="content-sync")return null;let s=Number.parseInt(String(r.intervalMinutes??"0"),10);if(!Number.isFinite(s)||s<1)return null;let o=String(r.id??`${n}-${t+1}`).trim();if(!o)return null;let i=String(r.deploymentProfile??"").trim(),u=String(r.url??"").trim(),a=Array.isArray(r.postTypes)?[...new Set(r.postTypes.map(d=>String(d??"").trim().toLowerCase().replace(/[^a-z0-9_-]/g,"")).filter(Boolean))]:[],l=Array.isArray(r.listingPaths)?[...new Set(r.listingPaths.map(d=>String(d??"").trim()).filter(Boolean))]:[];return n==="content-sync"&&a.length===0?null:{id:o,enabled:r.enabled!==false,command:n,intervalMinutes:s,...n==="publish"||n==="crawl"?{crawlMode:r.crawlMode==="incremental"?"incremental":"full"}:{},...(n==="publish"||n==="deploy"||n==="invalidate"||n==="content-sync")&&i?{deploymentProfile:i}:{},...u?{url:u}:{},...a.length>0?{postTypes:a}:{},...l.length>0?{listingPaths:l}:{},...typeof r.includeSubsites=="boolean"?{includeSubsites:r.includeSubsites}:{},...typeof r.includePostTypeArchives=="boolean"?{includePostTypeArchives:r.includePostTypeArchives}:{},...typeof r.includeTaxonomyArchives=="boolean"?{includeTaxonomyArchives:r.includeTaxonomyArchives}:{},...typeof r.includeAuthorArchives=="boolean"?{includeAuthorArchives:r.includeAuthorArchives}:{},...typeof r.includeDateArchives=="boolean"?{includeDateArchives:r.includeDateArchives}:{},...typeof r.includePostsPage=="boolean"?{includePostsPage:r.includePostsPage}:{},...typeof r.includeSitemapChain=="boolean"?{includeSitemapChain:r.includeSitemapChain}:{}}}function Pr(e){let t=e&&typeof e=="object"?e:{},r=Array.isArray(t.rules)?t.rules.map((n,s)=>ps(n,s)).filter(n=>!!n):[];return {enabled:t.enabled===true,timezone:typeof t.timezone=="string"&&t.timezone.trim()!==""?t.timezone.trim():"UTC",rules:r}}function ms(e){let t=e&&typeof e=="object"?e:{},r=t.render&&typeof t.render=="object"?t.render:{},n=Number(r.concurrency??0),s=Number(r.batchSize??0),o=Number(r.maxAttempts??0),i=t.rewrite&&typeof t.rewrite=="object"?t.rewrite:{},u=t.deploy&&typeof t.deploy=="object"?t.deploy:{},a=(l,d)=>({enabled:typeof l.enabled=="boolean"?l.enabled:r.enabled===true,concurrency:Math.min(100,Math.max(1,Number(l.concurrency||d.concurrency))),batchSize:Math.min(500,Math.max(1,Number(l.batchSize||d.batchSize))),maxAttempts:Math.min(5,Math.max(1,Number(l.maxAttempts||2)))});return {render:{enabled:r.enabled===true,...Number.isInteger(n)&&n>0?{concurrency:n}:{},batchSize:Number.isInteger(s)&&s>0?Math.min(20,s):5,...Number.isInteger(o)&&o>0?{maxAttempts:Math.min(5,o)}:{}},rewrite:a(i,{concurrency:8,batchSize:25}),deploy:a(u,{concurrency:8,batchSize:50})}}function gs(e){if(!e||typeof e!="object")return;let t=e,r={};for(let s of ["accountId","siteId"]){let o=t[s];typeof o=="string"&&o.trim()!==""&&(r[s]=o.trim());}let n=Number(t.lastUpdate??0);return Number.isFinite(n)&&n>0&&(r.lastUpdate=Math.floor(n)),t.subscriber===true&&(r.subscriber=true),Object.keys(r).length>0?r:void 0}function jt(e){let t=e&&typeof e=="object"?e:{},r=String(t.accountId??"").trim(),n=String(t.siteId??"").trim(),s=t.subscriber===true,o=gs(t.siteSettings),i={...o??{},...r?{accountId:o?.accountId??r}:{},...n?{siteId:o?.siteId??n}:{},...o?.subscriber===true||s?{subscriber:true}:{}},u={},a=String(t.apiBase??"").trim();a&&(u.apiBase=a);let l=String(t.runtimeToken??t.nonce??"").trim();l&&(u.runtimeToken=l);let d=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();d&&(u.virtualAssetBaseUrl=d),Object.keys(i).length>0&&(u.siteSettings=i);let c=$t(t.subscriptionType);return c&&(u.subscriptionType=c),u}function hs(e){try{let t=new URL(e);globalThis.location=t;}catch{}}function Rr(e){return typeof e=="string"?e:e instanceof URL?e.toString():e.url}function fs(e,t){let r=new Headers(e instanceof Request?e.headers:void 0);if(t?.headers)for(let[n,s]of new Headers(t.headers).entries())r.set(n,s);return r}async function vr(e,t,r=5){let n=new URL(Rr(e)),s=String(t?.method??(e instanceof Request?e.method:"GET")).toUpperCase(),o=n.protocol==="http:"?us:cs,i=fs(e,t);return await new Promise((u,a)=>{let l=o.request(n,{method:s,headers:Object.fromEntries(i.entries()),...n.protocol==="https:"?{rejectUnauthorized:false}:{}},d=>{let c=[];d.on("data",m=>{c.push(Buffer.isBuffer(m)?m:Buffer.from(m));}),d.on("error",a),d.on("end",()=>{let m=d.statusCode??0,h=d.headers.location;if(r>0&&h&&[301,302,303,307,308].includes(m)){u(vr(new URL(h,n),{method:m===303?"GET":s},r-1));return}let f=new Headers;for(let[b,p]of Object.entries(d.headers))Array.isArray(p)?f.set(b,p.join(", ")):typeof p=="string"&&f.set(b,p);u(new Response(Buffer.concat(c),{status:m,statusText:d.statusMessage??"",headers:f}));});});l.on("error",a),l.end();})}function ws(e){let t=jt(e.wpsuite),r={...t.siteSettings??{},...t.siteSettings?.subscriber===true||t.subscriptionType==="PROFESSIONAL"||t.subscriptionType==="AGENCY"?{subscriber:true}:{}},n=String(r.accountId??"").trim(),s=String(r.siteId??"").trim(),o=String(t.virtualAssetBaseUrl??t.uploadUrl??"").trim();if(e.wpsuite={...t.apiBase?{apiBase:t.apiBase}:{},...t.runtimeToken?{runtimeToken:t.runtimeToken}:{},...o?{virtualAssetBaseUrl:o}:{},...Object.keys(r).length>0?{siteSettings:r}:{},...t.subscriptionType?{subscriptionType:t.subscriptionType}:{}},!n||!s||!o)return false;hs(e.sourceOrigin);let i=globalThis,a={...i.WpSuite??{},siteSettings:r,uploadUrl:o};return t.apiBase&&(a.apiBase=t.apiBase),i.WpSuite=a,true}async function ys(e){let t=String(e.wpsuite?.virtualAssetBaseUrl??e.wpsuite?.uploadUrl??"").trim();if(!ws(e)){let s=e.wpsuite?.siteSettings,o=[String(s?.accountId??"").trim()?"":"accountId",String(s?.siteId??"").trim()?"":"siteId",t?"":"virtualAssetBaseUrl"].filter(Boolean);return {remote:null,diagnostic:{status:"invalid-runtime",virtualAssetBaseUrl:t,subscriptionType:null,error:`Missing WP Suite runtime field(s): ${o.join(", ")}.`,requests:[]}}}let r=globalThis.fetch,n=[];try{typeof r=="function"&&/^https?:\/\//i.test(t)&&(globalThis.fetch=(async(a,l)=>{let d=Rr(a);if(d.startsWith(t)){let c=(()=>{try{return new URL(d).pathname}catch{return d}})();try{let m=e.ignoreHttpsErrors&&d.startsWith("https://")?await vr(a,l):await r(a,l);return n.push({path:c,status:m.status,ok:m.ok,error:null}),m}catch(m){throw n.push({path:c,status:null,ok:!1,error:m instanceof Error?m.message:String(m)}),m}}return r(a,l)}));let{getConfig:s}=await import('@smart-cloud/wpsuite-core'),o=await s("publisher"),i=o&&typeof o=="object"?o:null,u=$t(i?.subscriptionType);return {remote:i,diagnostic:{status:i?"loaded":"unavailable",virtualAssetBaseUrl:t,subscriptionType:u??null,error:i?null:"WP Suite publisher config loader returned no configuration.",requests:n}}}catch(s){return {remote:null,diagnostic:{status:"unavailable",virtualAssetBaseUrl:t,subscriptionType:null,error:s instanceof Error?s.message:String(s),requests:n}}}finally{typeof r=="function"&&(globalThis.fetch=r);}}function bs(e,t){let r=ds(t?.deploymentProfiles),n=String(t?.defaultDeploymentProfile??"").trim(),s=r[n]?n:"",o=$t(t?.subscriptionType),i=jt(e.wpsuite);return {...e,scheduler:Pr(t?.scheduler),deploymentProfiles:r,defaultDeploymentProfile:s,...o?{subscriptionType:o}:{},wpsuite:{...i,...i.siteSettings||o?{siteSettings:{...i.siteSettings??{},...i.siteSettings?.subscriber===true||o==="PROFESSIONAL"||o==="AGENCY"?{subscriber:true}:{}}}:{},...o?{subscriptionType:o}:{}}}}function Mt(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let s=n.split("/").map(o=>o.trim()).filter(o=>o.length>0&&o!=="."&&o!=="..");return s.length>0?s.join("/"):t}function Ss(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function br(e,t,r){let n=String(t||"").trim();return n&&y.isAbsolute(n)?y.resolve(n):y.resolve(e,Mt(n,r))}async function nt(e,t={}){let r=String("").trim()||process.env.PUBLISHER_CONFIG||"publisher.config.json",n=await $.readFile(r,"utf8"),s=JSON.parse(n);s.sourceOrigin=s.sourceOrigin.replace(/\/$/,""),s.targetOrigin=Sr(s.targetOrigin),s.ignoreHttpsErrors??=false,s.outputDir=String(s.outputDir||"export").trim()||"export",s.urlRewriteMode||=s.targetOrigin==="."?"relative":"absolute",s.noJavaScriptRenderPathPrefixes||=[],s.seedPaths||=[],s.generated404RequestPath=ls(s.generated404RequestPath),s.sitemapPaths||=[...It],s.allowedAssetHosts||=[],s.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],s.blockedPathPrefixes||=[],s.blockedSearchFragments||=[],s.pageCache={purgeBeforeCrawl:s.pageCache?.purgeBeforeCrawl===true};let o=Number(s.processingConcurrency)>0?Math.min(100,Math.max(1,Number(s.processingConcurrency))):null;s.concurrency=o??s.concurrency??1,s.maxPages||=0,s.extraReplacements=Ot(s.extraReplacements),s.postCrawlCopyMap=Ot(s.postCrawlCopyMap),s.logDir=String(s.logDir||"logs").trim()||"logs",s.verbose??=false,s.logLevel||=s.verbose?"debug":"info",s.s3SyncMode||="sdk-upload-delete",s.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},s.readiness.timeoutMs??=1500,s.readiness.fallbackWaitMs??=1500,s.viewport||={width:1440,height:1200},s.navigationTimeoutMs||=3e4,s.assetDownloadConcurrency=o??(Number(s.assetDownloadConcurrency)>0?Number(s.assetDownloadConcurrency):s.concurrency),s.rewriteConcurrency=o??(Number(s.rewriteConcurrency)>0?Number(s.rewriteConcurrency):s.assetDownloadConcurrency),s.wpsuite=jt(s.wpsuite),s.scheduler=Pr(void 0),s.remoteWorkers=ms(s.remoteWorkers),typeof s.lambdaDelegationEnabled=="boolean"&&(s.remoteWorkers.render??={},s.remoteWorkers.rewrite??={},s.remoteWorkers.deploy??={},s.remoteWorkers.render.enabled=s.lambdaDelegationEnabled,s.remoteWorkers.rewrite.enabled=s.lambdaDelegationEnabled,s.remoteWorkers.deploy.enabled=s.lambdaDelegationEnabled),o!==null&&(s.processingConcurrency=o,s.remoteWorkers.render??={},s.remoteWorkers.rewrite??={},s.remoteWorkers.deploy??={},s.remoteWorkers.render.concurrency=o,s.remoteWorkers.rewrite.concurrency=o,s.remoteWorkers.deploy.concurrency=o),s.deploymentProfiles={},s.defaultDeploymentProfile="",s.deploymentTargetOverride=String(s.deploymentTargetOverride??"").trim();let i=await ys(s);t.onRemotePublisherConfigDiagnostic?.(i.diagnostic);let u=bs(s,i.remote),a=Ss(),l=a?y.resolve(a,".."):"";return l?(u.outputDir=br(l,u.outputDir,"export"),u.logDir=br(l,u.logDir,"logs")):(y.isAbsolute(u.outputDir)||(u.outputDir=Mt(u.outputDir,"export")),y.isAbsolute(u.logDir)||(u.logDir=Mt(u.logDir,"logs"))),u}function ue(e,t=10){return xr.createHash("sha1").update(e).digest("hex").slice(0,t)}async function Ps(e){await $.mkdir(y.dirname(e),{recursive:true});}async function fe(e,t,r){await Ps(e);let n=y.join(y.dirname(e),`.${y.basename(e)}.${process.pid}.${Date.now()}.${xr.randomBytes(6).toString("hex")}.tmp`);try{typeof t=="string"?await $.writeFile(n,t,r??"utf8"):await $.writeFile(n,t),await $.rename(n,e);}catch(s){throw await $.unlink(n).catch(()=>{}),s}}function ce(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(y.extname(e).toLowerCase())}function we(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function je(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function st(e){return e.replace(/\//g,"\\/")}function kr(e){return e.replace(/\//g,"\\\\/")}function Rs(e){try{let t=new URL(e);if(t.protocol!=="http:"&&t.protocol!=="https:")return null;let r=t.pathname==="/"&&!t.search&&!t.hash?"":`${t.pathname}${t.search}${t.hash}`;return `//${t.host}${r}`}catch{return null}}function Cr(e,t,r){if(!t)return;e[t]=r;let n=st(t),s=st(r);e[n]=s;let o=kr(t),i=kr(r);e[o]=i;let u=je(t),a=je(r);e[u]=a;let l=je(n),d=je(s);e[l]=d;let c=je(o),m=je(i);e[c]=m;}function ot(e,t,r){Cr(e,t,r);let n=Rs(t);n&&n!==t&&Cr(e,n,r);}function Ut(e,t){let r=t.startsWith("/")?t:`/${t}`,n=(e||"").replace(/\/$/,"");return !n||n==="."||n==="/"?r:`${n}${r}`}function Tr(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ar(e,t,r){if(!t)return r;let n=y.dirname(y.resolve(t)),s=y.resolve(e,r.replace(/^\/+/,"")),o=y.relative(n,s).replace(/\\/g,"/");return o?(o.startsWith(".")||(o=`./${o}`),o):"."}async function Ft(e,t){return (t?[...new Set(t)].map(n=>y.resolve(n)).filter(n=>n.startsWith(y.resolve(e.outputDir))).map(n=>y.relative(e.outputDir,n).replace(/\\/g,"/")).filter(n=>n.length>0):await vs(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(n=>n!=="asset-map.json"&&ce(n))}var Ts=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],As=new Set([".html",".htm"]),Ir="WPSuite.io Static Publisher",Is=Ir.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Es(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function Os(e){return e.startsWith("//")||e.startsWith("\\/\\/")||e.startsWith("\\\\/\\\\/")}function Ms(e,t,r){return Os(t)?e.replace(new RegExp(`(?<!:)${Es(t)}`,"g"),r):e.split(t).join(r)}function $s(e,t,r){if(e.wpsuite?.siteSettings?.subscriber===true||e.wpsuite?.subscriptionType==="PROFESSIONAL"||e.wpsuite?.subscriptionType==="AGENCY"||!r)return false;let s=y.extname(r).toLowerCase();return As.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function js(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Is}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Ir}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let n=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,s=>`${s}${n} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,s=>`${s}${n} ${t}`)}function Us(e){let t=we(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Ls(e,t){let r=[...new Set(t)].filter(o=>o.includes("/")&&!/\\+\//.test(o)).map(o=>[o,st(o)]).filter(([o,i])=>o!==i).sort((o,i)=>i[0].length-o[0].length);if(!r.length)return e;let n=o=>{let i=o;for(let[u,a]of r)i=i.split(u).join(a);return i},s=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(o,i,u,a)=>`${i}${n(u)}${a}`);return s=s.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(o,i,u,a,l)=>Us(a)?`${i}${n(a)}${l}`:o),s}function Ds(e,t){if(!t)return null;let r=y.dirname(y.resolve(t)),n=y.resolve(e.outputDir),s=y.relative(r,n).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s.endsWith("/")?s:`${s}/`):"./"}function Fs(e,t,r){let n=Ds(t,r);if(!n)return e;let s=e;for(let o of Ts){let i=o.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${i}`,"g"),a=new RegExp(`(?<!\\.)/${i}`,"g");s=s.replace(u,`${n}${o}`).replace(a,`${n}${o}`);let l=o.replace(/\//g,"\\/"),d=n.replace(/\//g,"\\/"),c=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),m=new RegExp(`(?:\\.{1,}\\\\/)+${c}`,"g"),h=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${c}`,"g"),f=new RegExp(`(?<![\\.\\\\])\\/${c}`,"g");s=s.replace(m,`${d}${l}`).replace(h,`${d}${l}`).replace(f,`${d}${l}`);}return s}function Dt(e,t,r){if(e.urlRewriteMode==="absolute")return t;let n=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),s=n?n.pathname:Tr(t,e.targetOrigin),o=n?`${n.search}${n.hash}`:"";return e.urlRewriteMode==="root-relative"?`${s}${o}`:`${Ar(e.outputDir,r,s)}${o}`}function Lt(e,t,r,n,s){ot(e,r,Dt(t,n,s));}function Ws(e,t,r,n,s){if(!r||r===n)return;ot(e,r,n);let o=Dt(t,r,s),i=Dt(t,n,s);ot(e,o,i);}function Wt(e,t,r,n,s={}){let o=e,i={};Lt(i,t,t.sourceOrigin,t.targetOrigin,n);for(let[a,l]of Object.entries(t.extraReplacements))Lt(i,t,a,l,n);for(let[a,l]of Object.entries(r))Lt(i,t,a,l,n);for(let[a,l]of Object.entries(s)){let d=r[a];!d||d===l||Ws(i,t,l,d,n);}let u=Object.entries(i).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of u)o=Ms(o,a,l);return t.urlRewriteMode!=="absolute"&&(o=Fs(o,t,n)),o=Ls(o,u.map(([,a])=>a)),$s(t,o,n)&&(o=js(o)),o}async function qs(e,t,r,n){let s=y.join(e.outputDir,n),o=y.extname(s).toLowerCase(),i;try{i=await $.readFile(s,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=Wt(i,e,t,o===".js"||o===".mjs"?void 0:s,r);return a!==i?(await fe(s,a,"utf8"),{changed:true}):{changed:false}}async function Ns(e,t){return await new Promise((r,n)=>{let s=a=>{if(u(),a?.error){n(new Error(a.error));return}r({changed:!!a?.changed});},o=a=>{u(),n(a);},i=a=>{u(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},u=()=>{e.off("message",s),e.off("error",o),e.off("exit",i);};e.on("message",s),e.on("error",o),e.on("exit",i),e.postMessage({file:t});})}async function Er(e,t,r={}){let n=await Ft(e,r.files),s=0;if(n.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),i=Math.max(1,Math.min(o,n.length,availableParallelism()));await r.onStart?.({totalFiles:n.length,workerCount:i});let u=0,a=0,l=null;if(i===1){for(let c of n){let m=await qs(e,t,r.previousAssetMap??{},c);m.changed&&(s+=1),a+=1,await r.onProgress?.({index:a,totalFiles:n.length,changedTextFiles:s,file:c,changed:m.changed});}return s}let d=Array.from({length:i},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!l;){let m=u;if(m>=n.length)return;u+=1;let h=n[m],f=await Ns(c,h);f.changed&&(s+=1),a+=1,await r.onProgress?.({index:a,totalFiles:n.length,changedTextFiles:s,file:h,changed:f.changed});}}catch(m){throw l=m instanceof Error?m:new Error(String(m)),l}finally{await c.terminate().catch(()=>{});}});return await Promise.all(d),s}var Ce={error:0,warn:1,info:2,debug:3},_s=/^(?:authorization|proxyAuthorization|runtimeToken|siteKey|nonce|accessToken|refreshToken|accessKeyId|secretAccessKey|sessionToken|xStaticPublisherToken|xApiKey)$/i;function ye(e){return e.replace(/((?:x-static-publisher-token|authorization|proxy-authorization|x-api-key|aws-access-key-id|aws-secret-access-key|aws-session-token)\s*:\s*)[^\r\n]+/gi,"$1[REDACTED]").replace(/\b(Basic|Bearer)\s+[^\s]+/gi,"$1 [REDACTED]").replace(/([?&](?:token|access_token|refresh_token|api_key|key|nonce)=)[^&\s]+/gi,"$1[REDACTED]").replace(/((?:"|')?(?:runtimeToken|siteKey|nonce|accessToken|refreshToken|accessKeyId|secretAccessKey|sessionToken)(?:"|')?\s*[:=]\s*(?:"|'))[^"']+/gi,"$1[REDACTED]")}function at(e){return typeof e=="string"?ye(e):Array.isArray(e)?e.map(t=>at(t)):e instanceof Error?{name:e.name,message:ye(e.message),...e.stack?{stack:ye(e.stack)}:{}}:e&&typeof e=="object"?Object.fromEntries(Object.entries(e).map(([t,r])=>[t,_s.test(t)?"[REDACTED]":at(r)])):e}function qt(e){return e?at(e):void 0}var Or=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",it=Or?y.join(Or,"current-progress.json"):"",Mr=Promise.resolve();function $r(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function Hs(e,t){let r={...e},n=typeof e.phase=="string"?e.phase.trim():"",s=typeof t.phase=="string"?t.phase.trim():"";if(s!==""&&(s==="deploy"||s.startsWith("deploy-"))&&n!==s)for(let u of ["index","totalFiles","staged","uploaded","skipped","failed","deleted","workerCount","completedBatches","totalBatches"])delete r[u];let i=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","completedBatches","totalBatches","pageQueue","assetQueue","sitemapQueue"]);for(let[u,a]of Object.entries(t)){let l=r[u];if(typeof l=="number"&&typeof a=="number"&&i.has(u)){r[u]=Math.max(l,a);continue}r[u]=a;}return r}function Bs(e,t,r){it&&(Mr=Mr.then(async()=>{let n=new Date,s=n.toISOString();await $.mkdir(y.dirname(it),{recursive:true});let o=await $.readFile(it,"utf8").then(g=>JSON.parse(g)).catch(()=>null),i=o&&typeof o.details=="object"&&o.details?o.details:{},u=$r(e),a=o?.currentStep||$r(o?.source||""),l=o?.startedAt||s,d={...o?.stepDurationsSec??{}},c=o?.stepStartedAt||s;if(a&&a!==u&&o?.stepStartedAt){let g=Math.max(0,Math.round((n.getTime()-new Date(o.stepStartedAt).getTime())/1e3));d[a]=(d[a]??0)+g,c=s;}let m=Math.max(0,Math.round((n.getTime()-new Date(c).getTime())/1e3)),h=Math.max(0,Math.round((n.getTime()-new Date(l).getTime())/1e3)),f=ye(t),b=qt(r)??{},p={checkedAt:s,source:e,message:f,details:Hs(i,b),startedAt:l,currentStep:u,stepStartedAt:c,stepElapsedSec:m,totalElapsedSec:h,stepDurationsSec:d};await $.writeFile(it,JSON.stringify(p,null,2),"utf8");}).catch(()=>{}));}async function Vs(e,t){await $.mkdir(y.dirname(e),{recursive:true}),await $.appendFile(e,`${JSON.stringify(at(t))}
|
|
5
|
+
`,"utf8");}function zs(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function Qs(e,t,r){let n=qt(r);return {checkedAt:new Date().toISOString(),currentStep:zs(e,n),level:e,message:ye(t),details:n??{}}}var Ue=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return y.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return y.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return y.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return y.join(this.logDir,"ignored.jsonl")}get skippedPath(){return y.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return y.join(this.logDir,"errors.jsonl")}get timingsPath(){return y.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await $.mkdir(this.logDir,{recursive:true}),await $.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await $.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Vs(t,r));}updateCurrentEvent(t,r,n){let s=Qs(t,r,n);this.enqueueTask(async()=>{await $.mkdir(this.logDir,{recursive:true}),await $.writeFile(this.currentEventPath,JSON.stringify(s,null,2),"utf8");});}push(t,r,n){let s=ye(r),o=qt(n),i=JSON.stringify({time:new Date().toISOString(),level:t,message:s,...o||{}});this.enqueueLine(i),this.updateCurrentEvent(t,s,o),t==="error"?console.error(s,o||""):t==="warn"?console.warn(s,o||""):(Ce[this.level]>=Ce.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(s);}info(t,r){this.push("info",t,r);}debug(t,r){Ce[this.level]<Ce.debug||this.push("debug",t,r);}progress(t,r){Bs("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let n=this.marks.get(t);if(!n)return;let s=Date.now()-n,o={name:t,ms:s,seconds:Number((s/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,o),this.push("timing",`Timing ${t}: ${o.seconds}s`,o);}reject(t,r,n,s){let o={kind:t,url:r,reason:n,source:s};this.enqueueJsonLine(this.rejectedPath,o),Ce[this.level]>=Ce.debug&&this.push("reject",`Rejected ${t} ${r}: ${n}`,o);}ignore(t,r,n,s){let o={kind:t,url:r,reason:n,source:s};this.enqueueJsonLine(this.ignoredPath,o),Ce[this.level]>=Ce.debug&&this.push("ignore",`Ignored ${t} ${r}: ${n}`,o);}skip(t,r,n,s){let o={kind:t,url:r,status:n,source:s};this.enqueueJsonLine(this.skippedPath,o),this.push("warn",`Skipped ${t} ${r}: HTTP ${n}`,o);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var jr="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)";function Lr(e){let t=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${jr}(?:[?#][^\\s'"<>\\);]*)?(?=[\\s'"<>\\);]|$)|(?<![A-Za-z0-9._-])/[^\\s'"<>\\);]+\\.${jr}(?:[?#][^\\s'"<>\\);]*)?(?=[\\s'"<>\\);]|$)`,"gi");return [...e.matchAll(t)].map(r=>r[0])}function Ur(e){return !!e&&/^(?:\.{1,2}\/)?[A-Za-z0-9@_][A-Za-z0-9@._/-]*\.(?:js|mjs)(?:[?#][^\s<>\\]*)?$/i.test(e)&&!e.startsWith("/")&&!e.startsWith("//")&&!/^[a-z][a-z0-9+.-]*:/i.test(e)&&!/(?:^|\/)node_modules(?:\/|$)|(?:^|\/)\.pnpm(?:\/|$)/i.test(e)}function Dr(e){let t=new Set,r=[],n=/\bimport\s*\(\s*["'`]([^"'`\s<>\\]+)["'`]\s*\)/g,s=/\b(?:[A-Za-z_$][\w$]*\.)?u\s*=\s*(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>\s*["'`]([^"'`\s<>\\]+)["'`]/g;for(let o of e.matchAll(n))o[1]&&Ur(o[1])&&r.push({index:o.index??0,value:o[1]});for(let o of e.matchAll(s))o[1]&&Ur(o[1])&&r.push({index:o.index??0,value:o[1]});for(let o of r.sort((i,u)=>i.index-u.index))t.add(o.value);return [...t]}function Fr(e,t){let r=y.relative(t,e);return r===""||!r.startsWith("..")&&!y.isAbsolute(r)}function Wr(e,t,r,n){let s=y.resolve(e),o=y.resolve(t),i=y.extname(s).toLowerCase();return !Fr(s,o)||i!==".html"&&i!==".htm"||r.has(s)?false:!n.some(u=>Fr(s,y.resolve(u)))}function Nt(e,t){return typeof e=="string"&&e.length>0&&t?.supported===true&&typeof t.token=="string"&&t.token.length>0&&e===t.token}function _t(e,t){try{let r=new URL(e),n=new URL(t,r);if(n.origin!==r.origin)return !1;let s=r.pathname.replace(/\/+$/,"");if(s&&n.pathname!==s&&!n.pathname.startsWith(`${s}/`))return !1;let o=s?n.pathname.slice(s.length)||"/":n.pathname;return ["/wp-includes/","/wp-content/plugins/","/wp-content/themes/","/wp-content/mu-plugins/"].some(i=>o.startsWith(i))}catch{return false}}function qr(e){return e<1}function Nr(e){let t=String(e??"").toLowerCase();return t==="strict"?"Strict":t==="lax"?"Lax":t==="none"?"None":"Unspecified"}function Js(e){let t=e.pathname||"/";if(!t.startsWith("/")||t==="/")return "/";let r=t.lastIndexOf("/");return r<=0?"/":t.slice(0,r)}function Be(e){return `${e.name.toLowerCase()}
|
|
7
7
|
${e.domain.toLowerCase()}
|
|
8
|
-
${e.path}`}function _r(e,t){let r;try{r=new URL(t);}catch{return null}let n=e.split(";"),s=n.shift()?.trim()??"",o=s.indexOf("=");if(o<=0)return null;let i=s.slice(0,o).trim();if(!i||[...i].some(b=>b<=" "||"=;,".includes(b)))return null;let u=r.hostname.toLowerCase(),a=
|
|
9
|
-
`,{encoding:"utf8",mode:384}),await M.rename(r,e);}function Ve(e){let t=e;for(let n=0;n<3;n+=1){if(/%(?:2f|5c)/i.test(t))return null;try{let s=decodeURIComponent(t);if(s===t)break;t=s;}catch{return null}}return t.includes("\0")||t.includes("\\")||t.split("/").some(n=>n==="."||n==="..")?null:t}function ze(e,t){let r=Ve(t);if(r===null)throw new Error(`Unsafe output URL pathname: ${t}`);let n=y.resolve(e),s=y.resolve(n,r.replace(/^\/+/,""));if(s!==n&&!s.startsWith(`${n}${y.sep}`))throw new Error(`Output path escapes configured root: ${t}`);return s}var H=1,Br=/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;function te(e,t){let r=String(e??"").trim();if(!Br.test(r))throw new Error(`${t} must match ${Br.source} and be at most 128 characters.`);return r}function Vr(e){let t=String(e??"").trim().replace(/^\/+|\/+$/g,"");if(!t)return "";let r=t.split("/").filter(Boolean);if(r.some(n=>n==="."||n===".."))throw new Error("S3 prefix contains an unsafe dot path segment.");return `${r.join("/")}/`}function Ht(e,...t){let r=Vr(e),n=t.map((s,o)=>{let i=String(s??"").replace(/^\/+|\/+$/g,"");if(!i||i.split("/").some(u=>u==="."||u===".."))throw new Error(`Unsafe S3 key segment at index ${o}.`);return i});return `${r}${n.join("/")}`}function re(e){return createHash("sha256").update(e).digest("hex")}function ye(e,t){let r=String(e??"").replace(/^\/+/,""),n=Vr(t);if(!r||r.includes("\0"))throw new Error("S3 object key is empty or invalid.");if(n&&!r.startsWith(n))throw new Error(`S3 object key is outside the configured workspace prefix: ${r}`);if(r.split("/").some(s=>s===".."))throw new Error("S3 object key contains an unsafe parent segment.");return r}function Q(e,t,{optional:r=false}={}){if(!(r&&(e==null||e===""))){if(typeof e!="string"||!e.trim())throw new Error(`${t} must be a non-empty string.`);return e.trim()}}function oo(e){if(!e||typeof e!="object")throw new Error("Remote worker config must be a JSON object.");let t=e;if(t.schemaVersion!==1)throw new Error(`Unsupported remote worker config version: ${String(t.schemaVersion)}.`);if(t.protocolVersion!==H)throw new Error(`Remote worker protocol mismatch: expected ${H}, received ${String(t.protocolVersion)}.`);let r=t.workspace,n=t.functions;if(!r||!n)throw new Error("Remote worker config requires workspace and functions objects.");return {schemaVersion:1,region:Q(t.region,"region"),...Q(t.roleArn,"roleArn",{optional:true})?{roleArn:Q(t.roleArn,"roleArn")}:{},...Q(t.roleSessionName,"roleSessionName",{optional:true})?{roleSessionName:Q(t.roleSessionName,"roleSessionName")}:{},workspace:{bucket:Q(r.bucket,"workspace.bucket"),prefix:String(r.prefix??"")},...t.status&&typeof t.status=="object"?{status:{tableName:Q(t.status.tableName,"status.tableName")}}:{},functions:{render:Q(n.render,"functions.render"),...Q(n.asset,"functions.asset",{optional:true})?{asset:Q(n.asset,"functions.asset")}:{},rewrite:Q(n.rewrite,"functions.rewrite"),"deploy-copy":Q(n["deploy-copy"],"functions.deploy-copy")},targets:Array.isArray(t.targets)?t.targets.map((s,o)=>{if(!s||typeof s!="object")throw new Error(`targets[${o}] must be an object.`);let i=s;return {id:Q(i.id,`targets[${o}].id`),bucketName:Q(i.bucketName,`targets[${o}].bucketName`),prefix:String(i.prefix??""),region:Q(i.region,`targets[${o}].region`)}}):[],protocolVersion:H}}async function Kr(e){let t=y.resolve(e);try{return oo(JSON.parse(await readFile(t,"utf8")))}catch(r){throw new Error(`Unable to load remote worker config ${t}: ${r instanceof Error?r.message:String(r)}`,{cause:r})}}var ct=class{config;lambda;s3;dynamodb;#e;constructor(t,r={},n={}){this.config=t;let s=t.roleArn?fromTemporaryCredentials({params:{RoleArn:t.roleArn,RoleSessionName:t.roleSessionName??"publisher-exporter-remote-worker"},clientConfig:{region:t.region}}):void 0;this.lambda=new LambdaClient({region:t.region,...s?{credentials:s}:{},...r}),this.s3=new S3Client({region:t.region,...s?{credentials:s}:{}}),this.dynamodb=t.status?new DynamoDBClient({region:t.region,...s?{credentials:s}:{}}):null,this.#e=n;}async#n(t){if(!this.dynamodb||!this.config.status)return null;let n=(await this.dynamodb.send(new GetItemCommand({TableName:this.config.status.tableName,ConsistentRead:true,Key:{jobId:{S:t.jobId},taskId:{S:t.taskId}}}))).Item;if(!n)return null;let s=n.operation?.S,o=n.status?.S;return !s||!["render","asset","rewrite","deploy-copy"].includes(s)||!o||!["running","succeeded","partial","failed"].includes(o)?null:{jobId:n.jobId?.S??t.jobId,taskId:n.taskId?.S??t.taskId,operation:s,status:o,completedItems:Number(n.completedItems?.N??0),failedItems:Number(n.failedItems?.N??0),totalItems:Number(n.totalItems?.N??0),sequence:Number(n.sequence?.N??0),updatedAt:n.updatedAt?.S??new Date(0).toISOString(),requestId:n.requestId?.S??null}}async#t(t,r){let n=this.lambda.send(new InvokeCommand({FunctionName:r,InvocationType:"RequestResponse",Payload:Buffer.from(JSON.stringify(t),"utf8")}));if(t.operation==="health"||!this.dynamodb||!this.#e.onProgress)return await n;let s=false,o=n.finally(()=>{s=true;}),i=Math.max(500,Math.min(3e4,this.#e.progressPollIntervalMs??2e3)),u=-1;for(;!s;){await Promise.race([o.then(()=>{},()=>{}),new Promise(a=>setTimeout(a,i))]);try{let a=await this.#n(t);a&&a.sequence>u&&(u=a.sequence,await this.#e.onProgress(a));}catch(a){await this.#e.onProgressError?.(a instanceof Error?a:new Error(String(a)));}}return await o}async invoke(t){let r=t.operation==="health"?this.config.functions.render:this.config.functions[t.operation];if(!r)throw new Error(`Remote worker configuration does not provide a ${t.operation} function. Redeploy the Lambda workers and reinstall remote-workers.json.`);let n=await this.#t(t,r),s=n.Payload?JSON.parse(new TextDecoder().decode(n.Payload)):null;if(n.FunctionError)throw new Error(`Remote ${t.operation} worker failed (${n.FunctionError}): ${JSON.stringify(s)}`);if(!s||typeof s!="object")throw new Error(`Remote ${t.operation} worker returned no JSON response.`);let o=s;if(o.schemaVersion!==H||o.operation!==t.operation||o.jobId!==t.jobId||o.taskId!==t.taskId)throw new Error(`Remote ${t.operation} worker returned a mismatched response.`);return o}async readResult(t){if(!t.resultKey)throw new Error(`Remote ${t.operation} response does not contain a resultKey.`);let r=ye(t.resultKey,this.config.workspace.prefix),n=await this.s3.send(new GetObjectCommand({Bucket:this.config.workspace.bucket,Key:r}));if(!n.Body)throw new Error(`Remote worker result has no body: ${r}`);return JSON.parse(await n.Body.transformToString("utf8"))}async readObject(t){if(t.bucket!==this.config.workspace.bucket)throw new Error(`Remote worker object bucket mismatch: ${t.bucket}.`);let r=ye(t.key,this.config.workspace.prefix),n=await this.s3.send(new GetObjectCommand({Bucket:this.config.workspace.bucket,Key:r,...t.versionId?{VersionId:t.versionId}:{}}));if(!n.Body)throw new Error(`Remote worker object has no body: ${r}`);let s=await n.Body.transformToByteArray();if(s.byteLength!==t.bytes)throw new Error(`Remote worker object length mismatch for ${r}: expected ${t.bytes}, received ${s.byteLength}.`);if(re(s)!==t.sha256)throw new Error(`Remote worker object checksum mismatch for ${r}.`);return s}async readObjectText(t){return new TextDecoder("utf8",{fatal:true}).decode(await this.readObject(t))}async health(t){let r=this.config.functions[t];if(!r)throw new Error(`Remote worker configuration does not provide a ${t} function.`);let n=`${t}-${Date.now()}`,s=await this.lambda.send(new InvokeCommand({FunctionName:r,InvocationType:"RequestResponse",Payload:Buffer.from(JSON.stringify({schemaVersion:H,operation:"health",jobId:"health",taskId:n}),"utf8")})),o=s.Payload?JSON.parse(new TextDecoder().decode(s.Payload)):null;if(s.FunctionError||!o||typeof o!="object")throw new Error(`Remote ${t} health check failed: ${JSON.stringify(o)}`);return o}async writeWorkspaceObject(t){let r=ye(t.key,this.config.workspace.prefix),n=typeof t.body=="string"?Buffer.from(t.body,"utf8"):t.body,s=re(n),o=await this.s3.send(new PutObjectCommand({Bucket:this.config.workspace.bucket,Key:r,Body:n,ContentType:t.contentType,...t.cacheControl?{CacheControl:t.cacheControl}:{},Metadata:{sha256:s,...t.metadata??{}}}));return {bucket:this.config.workspace.bucket,key:r,sha256:s,bytes:n.byteLength,contentType:t.contentType,...o.ETag?{etag:o.ETag}:{},...o.VersionId?{versionId:o.VersionId}:{}}}async copyWorkspaceObject(t,r){if(t.bucket!==this.config.workspace.bucket)throw new Error(`Remote worker object bucket mismatch: ${t.bucket}.`);let n=ye(t.key,this.config.workspace.prefix),s=ye(r,this.config.workspace.prefix),o=`${t.bucket}/${n.split("/").map(u=>encodeURIComponent(u)).join("/")}`,i=await this.s3.send(new CopyObjectCommand({Bucket:t.bucket,Key:s,CopySource:o,MetadataDirective:"COPY"}));return {...t,key:s,...i.CopyObjectResult?.ETag?{etag:i.CopyObjectResult.ETag}:{},...i.VersionId?{versionId:i.VersionId}:{}}}};function lt(e){return Number.isInteger(e)&&e>=500&&e<=599}var Bt=3;function dt(e){return Math.min(3e4,5e3*2**Math.max(0,e-1))}var ee=class extends Error{constructor(r,n,s,o){let i=`Page failed after ${s} attempt(s), last server response HTTP ${n}: ${r}`;super(i+(o&&o!==i?`; ${o}`:""));this.url=r;this.status=n;this.attempts=s;this.name="PageHttpError";}url;status;attempts};function Jr(e){if(e.size===0)return;let t=[...e.values()].slice(0,5).map(r=>`HTTP ${r.status}: ${r.url}`).join("; ");throw new Error(`Crawl incomplete: ${e.size} page(s) failed after HTTP 5xx retries. Deployment is blocked; no complete deploy plan was produced. ${t}`)}function ao(e,t,r){if(!e||typeof e!="object")throw new Error("Remote render result must be a JSON object.");let n=e;if(n.schemaVersion!==H||!n.response||n.response.operation!=="render"||n.response.jobId!==t.jobId||n.response.taskId!==t.taskId||!n.detail||!Array.isArray(n.detail.pages)||n.detail.pages.length!==r.length)throw new Error("Remote render result has an invalid envelope.");for(let[s,o]of r.entries())if(n.detail.pages[s]?.requestedUrl!==o)throw new Error("Remote render result does not match the requested URLs.");return n}function Vt(e){return e instanceof Error?e.message:String(e)}var pt=class{#e;#n;#t;#s;#u;#c;#l;#r=[];#d=0;#o=0;#i=false;constructor(t){this.#e=t.client,this.#n=te(t.jobId,"jobId"),this.#t=Math.max(1,Math.min(5,t.maxAttempts??2)),this.#s=t.onRetry,this.#l=t.sleep??setTimeout$1,this.#u=Math.max(1,Math.min(20,t.batchSize??1)),this.#c=Math.max(1,Math.min(100,t.maxConcurrentBatches??1));}async renderBatch(t){let r=[...new Set(t.urls)];if(r.length<1||r.length>20)throw new Error("Remote render batches require between 1 and 20 unique URLs.");let n=new Map,s=new Map,o=new Map,i=new Map,u=r;for(let a=1;a<=this.#t&&u.length>0;a+=1){this.#d+=1;let l=te(`render-${this.#d}-${re(u.join(`
|
|
10
|
-
`)).slice(0,24)}-a${a}`,"taskId"),d=[];try{let c=await this.#e.invoke({schemaVersion:H,operation:"render",jobId:this.#n,taskId:l,sourceOrigin:new URL(t.sourceOrigin).origin,urls:u,options:t.options}),m=
|
|
11
|
-
`)).slice(0,24)}-a${s}`,"taskId");try{let i=await this.#e.invoke({schemaVersion:H,operation:"asset",jobId:this.#n,taskId:o,urls:r,options:{ignoreHttpsErrors:t.ignoreHttpsErrors,timeoutMs:Math.max(1e3,Math.min(12e4,t.timeoutMs??6e4))}});return {assets:
|
|
12
|
-
`)).slice(0,16)}`,"taskId"),z={schemaVersion:H,operation:"rewrite",jobId:o,taskId:x,configKey:u,assetMapKey:a,previousAssetMapKey:l,items:A.map(L=>({inputKey:L.key,outputKey:L.key,relativePath:L.relativePath,contentType:L.contentType}))};try{let L=await r.client.invoke(z);v=Xr(await r.client.readResult(L),L);break}catch(L){R=L;}}if(!v)throw new Error(`Remote rewrite batch ${I+1}/${p.length} failed: ${R instanceof Error?R.message:String(R)}`,{cause:R});for(let E of A){let x=v.detail.objects.find(z=>z.outputKey===E.key);if(!x)throw new Error(`Remote rewrite result omitted ${E.relativePath}.`);await
|
|
13
|
-
`)).slice(0,16)}`,"taskId");try{let A=await n.client.invoke({schemaVersion:H,operation:"rewrite",jobId:i,taskId:I,configKey:a,assetMapKey:l,items:g.map(R=>({inputKey:R.object.key,outputKey:R.object.key,relativePath:R.relativePath,contentType:R.contentType||R.object.contentType}))});w=Xr(await n.client.readResult(A),A);break}catch(A){k=A;}}if(!w)throw new Error(`Remote object rewrite batch ${p+1}/${c.length} failed: ${k instanceof Error?k.message:String(k)}`,{cause:k});for(let C of g){let I=w.detail.objects.find(A=>A.outputKey===C.object.key);if(!I)throw new Error(`Remote rewrite result omitted ${C.relativePath}.`);C.object=I.output,I.changed&&(b+=1),f+=1,await n.onProgress?.({index:f,totalFiles:o.length,changedTextFiles:b,file:C.relativePath,changed:I.changed});}}})),t.assetMap={...r},t.generatedAt=new Date().toISOString(),b}var
|
|
14
|
-
`,"utf8");}async function
|
|
15
|
-
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function
|
|
8
|
+
${e.path}`}function _r(e,t){let r;try{r=new URL(t);}catch{return null}let n=e.split(";"),s=n.shift()?.trim()??"",o=s.indexOf("=");if(o<=0)return null;let i=s.slice(0,o).trim();if(!i||[...i].some(b=>b<=" "||"=;,".includes(b)))return null;let u=r.hostname.toLowerCase(),a=Js(r),l=null,d=true,c=false,m=false,h="Unspecified",f=false;for(let b of n){let p=b.indexOf("="),g=(p<0?b:b.slice(0,p)).trim().toLowerCase(),w=p<0?"":b.slice(p+1).trim();if(g==="domain"&&w)u=w.toLowerCase();else if(g==="path"&&w.startsWith("/"))a=w;else if(g==="expires"){let k=Date.parse(w);Number.isFinite(k)&&(l=Math.floor(k/1e3),d=false);}else if(g==="max-age"&&/^-?\d+$/u.test(w)){let k=Number.parseInt(w,10);l=Math.floor(Date.now()/1e3)+k,d=false;}else g==="secure"?c=true:g==="httponly"?m=true:g==="samesite"?h=Nr(w):g==="partitioned"&&(f=true);}return {name:i,domain:u,path:a,expires:l,session:d,secure:c,httpOnly:m,sameSite:h,partitioned:f}}function Gs(e){let t=!Number.isFinite(e.expires)||e.expires<=0;return {name:e.name,domain:e.domain.toLowerCase(),path:e.path||"/",expires:t?null:e.expires,session:t,secure:e.secure,httpOnly:e.httpOnly,sameSite:Nr(e.sameSite),partitioned:e.partitioned===true}}function Hr(e,t){let r=new Map(e.map(n=>[Be(n),n.value]));return t.filter(n=>r.get(Be(n))!==n.value).map(Gs)}function Zs(e,t){let r=e.session||t.session;return {...e,expires:r?null:Math.max(e.expires??0,t.expires??0)||null,session:r,secure:e.secure&&t.secure,httpOnly:e.httpOnly&&t.httpOnly,sameSite:e.sameSite===t.sameSite?e.sameSite:"Unspecified",partitioned:e.partitioned&&t.partitioned}}var ut=class{#e=new Map;add(t){if(!t)return;let r={...t,domain:t.domain.toLowerCase(),path:t.path||"/"},n=Be(r),s=this.#e.get(n);this.#e.set(n,s?Zs(s,r):r);}addAll(t){for(let r of t)this.add(r);}artifact(){return {schemaVersion:1,observations:[...this.#e.values()].sort((t,r)=>Be(t).localeCompare(Be(r)))}}};async function ct(e,t){await $.mkdir(y.dirname(e),{recursive:true});let r=`${e}.${process.pid}.tmp`;await $.writeFile(r,`${JSON.stringify(t,null,2)}
|
|
9
|
+
`,{encoding:"utf8",mode:384}),await $.rename(r,e);}function Ve(e){let t=e;for(let n=0;n<3;n+=1){if(/%(?:2f|5c)/i.test(t))return null;try{let s=decodeURIComponent(t);if(s===t)break;t=s;}catch{return null}}return t.includes("\0")||t.includes("\\")||t.split("/").some(n=>n==="."||n==="..")?null:t}function ze(e,t){let r=Ve(t);if(r===null)throw new Error(`Unsafe output URL pathname: ${t}`);let n=y.resolve(e),s=y.resolve(n,r.replace(/^\/+/,""));if(s!==n&&!s.startsWith(`${n}${y.sep}`))throw new Error(`Output path escapes configured root: ${t}`);return s}var H=1,Br=/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;function X(e,t){let r=String(e??"").trim();if(!Br.test(r))throw new Error(`${t} must match ${Br.source} and be at most 128 characters.`);return r}function Vr(e){let t=String(e??"").trim().replace(/^\/+|\/+$/g,"");if(!t)return "";let r=t.split("/").filter(Boolean);if(r.some(n=>n==="."||n===".."))throw new Error("S3 prefix contains an unsafe dot path segment.");return `${r.join("/")}/`}function Qe(e,...t){let r=Vr(e),n=t.map((s,o)=>{let i=String(s??"").replace(/^\/+|\/+$/g,"");if(!i||i.split("/").some(u=>u==="."||u===".."))throw new Error(`Unsafe S3 key segment at index ${o}.`);return i});return `${r}${n.join("/")}`}function re(e){return createHash("sha256").update(e).digest("hex")}function me(e,t){let r=String(e??"").replace(/^\/+/,""),n=Vr(t);if(!r||r.includes("\0"))throw new Error("S3 object key is empty or invalid.");if(n&&!r.startsWith(n))throw new Error(`S3 object key is outside the configured workspace prefix: ${r}`);if(r.split("/").some(s=>s===".."))throw new Error("S3 object key contains an unsafe parent segment.");return r}function Q(e,t,{optional:r=false}={}){if(!(r&&(e==null||e===""))){if(typeof e!="string"||!e.trim())throw new Error(`${t} must be a non-empty string.`);return e.trim()}}function uo(e){if(!e||typeof e!="object")throw new Error("Remote worker config must be a JSON object.");let t=e;if(t.schemaVersion!==1)throw new Error(`Unsupported remote worker config version: ${String(t.schemaVersion)}.`);if(t.protocolVersion!==H)throw new Error(`Remote worker protocol mismatch: expected ${H}, received ${String(t.protocolVersion)}.`);let r=t.workspace,n=t.functions;if(!r||!n)throw new Error("Remote worker config requires workspace and functions objects.");return {schemaVersion:1,region:Q(t.region,"region"),...Q(t.roleArn,"roleArn",{optional:true})?{roleArn:Q(t.roleArn,"roleArn")}:{},...Q(t.roleSessionName,"roleSessionName",{optional:true})?{roleSessionName:Q(t.roleSessionName,"roleSessionName")}:{},workspace:{bucket:Q(r.bucket,"workspace.bucket"),prefix:String(r.prefix??"")},...t.status&&typeof t.status=="object"?{status:{tableName:Q(t.status.tableName,"status.tableName")}}:{},functions:{render:Q(n.render,"functions.render"),...Q(n.asset,"functions.asset",{optional:true})?{asset:Q(n.asset,"functions.asset")}:{},rewrite:Q(n.rewrite,"functions.rewrite"),"deploy-copy":Q(n["deploy-copy"],"functions.deploy-copy")},targets:Array.isArray(t.targets)?t.targets.map((s,o)=>{if(!s||typeof s!="object")throw new Error(`targets[${o}] must be an object.`);let i=s;return {id:Q(i.id,`targets[${o}].id`),bucketName:Q(i.bucketName,`targets[${o}].bucketName`),prefix:String(i.prefix??""),region:Q(i.region,`targets[${o}].region`)}}):[],protocolVersion:H}}async function Kr(e){let t=y.resolve(e);try{return uo(JSON.parse(await readFile(t,"utf8")))}catch(r){throw new Error(`Unable to load remote worker config ${t}: ${r instanceof Error?r.message:String(r)}`,{cause:r})}}var lt=class{config;lambda;s3;dynamodb;#e;constructor(t,r={},n={}){this.config=t;let s=t.roleArn?fromTemporaryCredentials({params:{RoleArn:t.roleArn,RoleSessionName:t.roleSessionName??"publisher-exporter-remote-worker"},clientConfig:{region:t.region}}):void 0;this.lambda=new LambdaClient({region:t.region,...s?{credentials:s}:{},...r}),this.s3=new S3Client({region:t.region,...s?{credentials:s}:{}}),this.dynamodb=t.status?new DynamoDBClient({region:t.region,...s?{credentials:s}:{}}):null,this.#e=n;}async#n(t){if(!this.dynamodb||!this.config.status)return null;let n=(await this.dynamodb.send(new GetItemCommand({TableName:this.config.status.tableName,ConsistentRead:true,Key:{jobId:{S:t.jobId},taskId:{S:t.taskId}}}))).Item;if(!n)return null;let s=n.operation?.S,o=n.status?.S;return !s||!["render","asset","rewrite","deploy-copy"].includes(s)||!o||!["running","succeeded","partial","failed"].includes(o)?null:{jobId:n.jobId?.S??t.jobId,taskId:n.taskId?.S??t.taskId,operation:s,status:o,completedItems:Number(n.completedItems?.N??0),failedItems:Number(n.failedItems?.N??0),totalItems:Number(n.totalItems?.N??0),sequence:Number(n.sequence?.N??0),updatedAt:n.updatedAt?.S??new Date(0).toISOString(),requestId:n.requestId?.S??null}}async#t(t,r){let n=this.lambda.send(new InvokeCommand({FunctionName:r,InvocationType:"RequestResponse",Payload:Buffer.from(JSON.stringify(t),"utf8")}));if(t.operation==="health"||!this.dynamodb||!this.#e.onProgress)return await n;let s=false,o=n.finally(()=>{s=true;}),i=Math.max(500,Math.min(3e4,this.#e.progressPollIntervalMs??2e3)),u=-1;for(;!s;){await Promise.race([o.then(()=>{},()=>{}),new Promise(a=>setTimeout(a,i))]);try{let a=await this.#n(t);a&&a.sequence>u&&(u=a.sequence,await this.#e.onProgress(a));}catch(a){await this.#e.onProgressError?.(a instanceof Error?a:new Error(String(a)));}}return await o}async invoke(t){let r=t.operation==="health"?this.config.functions.render:this.config.functions[t.operation];if(!r)throw new Error(`Remote worker configuration does not provide a ${t.operation} function. Redeploy the Lambda workers and reinstall remote-workers.json.`);let n=await this.#t(t,r),s=n.Payload?JSON.parse(new TextDecoder().decode(n.Payload)):null;if(n.FunctionError)throw new Error(`Remote ${t.operation} worker failed (${n.FunctionError}): ${JSON.stringify(s)}`);if(!s||typeof s!="object")throw new Error(`Remote ${t.operation} worker returned no JSON response.`);let o=s;if(o.schemaVersion!==H||o.operation!==t.operation||o.jobId!==t.jobId||o.taskId!==t.taskId)throw new Error(`Remote ${t.operation} worker returned a mismatched response.`);return o}async readResult(t){if(!t.resultKey)throw new Error(`Remote ${t.operation} response does not contain a resultKey.`);let r=me(t.resultKey,this.config.workspace.prefix),n=await this.s3.send(new GetObjectCommand({Bucket:this.config.workspace.bucket,Key:r}));if(!n.Body)throw new Error(`Remote worker result has no body: ${r}`);return JSON.parse(await n.Body.transformToString("utf8"))}async readObject(t){if(t.bucket!==this.config.workspace.bucket)throw new Error(`Remote worker object bucket mismatch: ${t.bucket}.`);let r=me(t.key,this.config.workspace.prefix),n=await this.s3.send(new GetObjectCommand({Bucket:this.config.workspace.bucket,Key:r,...t.versionId?{VersionId:t.versionId}:{}}));if(!n.Body)throw new Error(`Remote worker object has no body: ${r}`);let s=await n.Body.transformToByteArray();if(s.byteLength!==t.bytes)throw new Error(`Remote worker object length mismatch for ${r}: expected ${t.bytes}, received ${s.byteLength}.`);if(re(s)!==t.sha256)throw new Error(`Remote worker object checksum mismatch for ${r}.`);return s}async readObjectText(t){return new TextDecoder("utf8",{fatal:true}).decode(await this.readObject(t))}async health(t){let r=this.config.functions[t];if(!r)throw new Error(`Remote worker configuration does not provide a ${t} function.`);let n=`${t}-${Date.now()}`,s=await this.lambda.send(new InvokeCommand({FunctionName:r,InvocationType:"RequestResponse",Payload:Buffer.from(JSON.stringify({schemaVersion:H,operation:"health",jobId:"health",taskId:n}),"utf8")})),o=s.Payload?JSON.parse(new TextDecoder().decode(s.Payload)):null;if(s.FunctionError||!o||typeof o!="object")throw new Error(`Remote ${t} health check failed: ${JSON.stringify(o)}`);return o}async writeWorkspaceObject(t){let r=me(t.key,this.config.workspace.prefix),n=typeof t.body=="string"?Buffer.from(t.body,"utf8"):t.body,s=re(n),o=await this.s3.send(new PutObjectCommand({Bucket:this.config.workspace.bucket,Key:r,Body:n,ContentType:t.contentType,...t.cacheControl?{CacheControl:t.cacheControl}:{},Metadata:{sha256:s,...t.metadata??{}}}));return {bucket:this.config.workspace.bucket,key:r,sha256:s,bytes:n.byteLength,contentType:t.contentType,...o.ETag?{etag:o.ETag}:{},...o.VersionId?{versionId:o.VersionId}:{}}}async copyWorkspaceObject(t,r){if(t.bucket!==this.config.workspace.bucket)throw new Error(`Remote worker object bucket mismatch: ${t.bucket}.`);let n=me(t.key,this.config.workspace.prefix),s=me(r,this.config.workspace.prefix),o=`${t.bucket}/${n.split("/").map(u=>encodeURIComponent(u)).join("/")}`,i=await this.s3.send(new CopyObjectCommand({Bucket:t.bucket,Key:s,CopySource:o,MetadataDirective:"COPY"}));return {...t,key:s,...i.CopyObjectResult?.ETag?{etag:i.CopyObjectResult.ETag}:{},...i.VersionId?{versionId:i.VersionId}:{}}}};function dt(e){return Number.isInteger(e)&&e>=500&&e<=599}var Vt=3;function pt(e){return Math.min(3e4,5e3*2**Math.max(0,e-1))}var te=class extends Error{constructor(r,n,s,o){let i=`Page failed after ${s} attempt(s), last server response HTTP ${n}: ${r}`;super(i+(o&&o!==i?`; ${o}`:""));this.url=r;this.status=n;this.attempts=s;this.name="PageHttpError";}url;status;attempts};function Jr(e){if(e.size===0)return;let t=[...e.values()].slice(0,5).map(r=>`HTTP ${r.status}: ${r.url}`).join("; ");throw new Error(`Crawl incomplete: ${e.size} page(s) failed after HTTP 5xx retries. Deployment is blocked; no complete deploy plan was produced. ${t}`)}function lo(e,t,r){if(!e||typeof e!="object")throw new Error("Remote render result must be a JSON object.");let n=e;if(n.schemaVersion!==H||!n.response||n.response.operation!=="render"||n.response.jobId!==t.jobId||n.response.taskId!==t.taskId||!n.detail||!Array.isArray(n.detail.pages)||n.detail.pages.length!==r.length)throw new Error("Remote render result has an invalid envelope.");for(let[s,o]of r.entries())if(n.detail.pages[s]?.requestedUrl!==o)throw new Error("Remote render result does not match the requested URLs.");return n}function zt(e){return e instanceof Error?e.message:String(e)}var mt=class{#e;#n;#t;#s;#u;#c;#l;#r=[];#d=0;#o=0;#i=false;constructor(t){this.#e=t.client,this.#n=X(t.jobId,"jobId"),this.#t=Math.max(1,Math.min(5,t.maxAttempts??2)),this.#s=t.onRetry,this.#l=t.sleep??setTimeout$1,this.#u=Math.max(1,Math.min(20,t.batchSize??1)),this.#c=Math.max(1,Math.min(100,t.maxConcurrentBatches??1));}async renderBatch(t){let r=[...new Set(t.urls)];if(r.length<1||r.length>20)throw new Error("Remote render batches require between 1 and 20 unique URLs.");let n=new Map,s=new Map,o=new Map,i=new Map,u=r;for(let a=1;a<=this.#t&&u.length>0;a+=1){this.#d+=1;let l=X(`render-${this.#d}-${re(u.join(`
|
|
10
|
+
`)).slice(0,24)}-a${a}`,"taskId"),d=[];try{let c=await this.#e.invoke({schemaVersion:H,operation:"render",jobId:this.#n,taskId:l,sourceOrigin:new URL(t.sourceOrigin).origin,urls:u,options:t.options}),m=lo(await this.#e.readResult(c),c,u);for(let h of m.detail.pages){if(o.delete(h.requestedUrl),dt(h.httpStatus)){let f=h.httpStatus;o.set(h.requestedUrl,f),i.set(h.requestedUrl,f),s.set(h.requestedUrl,new te(h.requestedUrl,f,a).message),d.push(h.requestedUrl);continue}if(h.httpStatus!==null&&h.httpStatus!==void 0)i.delete(h.requestedUrl);else if(i.has(h.requestedUrl)){s.set(h.requestedUrl,h.error||"Remote render returned no HTTP status after a server error."),d.push(h.requestedUrl);continue}if(h.outcome==="failed"||h.outcome==="deferred"||!h.html){let f=h.error||`Remote render ended with ${h.outcome}.`;s.set(h.requestedUrl,f),d.push(h.requestedUrl);continue}try{n.set(h.requestedUrl,{page:h,html:t.downloadHtml===!1?null:await this.#e.readObjectText(h.html),taskId:l,attempts:a}),s.delete(h.requestedUrl);}catch(f){s.set(h.requestedUrl,zt(f)),d.push(h.requestedUrl);}}}catch(c){let m=zt(c);for(let h of u)o.delete(h),s.set(h,m),d.push(h);}if(a<this.#t){let c=d.some(m=>o.has(m))?pt(a):0;for(let m of d){let h=o.get(m);this.#s?.({url:m,attempt:a,maxAttempts:this.#t,reason:s.get(m)??"Remote render failed.",...h===void 0?{}:{status:h,delayMs:c}});}c>0&&await this.#l(c);}u=d;}return {pages:r.flatMap(a=>{let l=n.get(a);return l?[l]:[]}),failures:u.map(a=>({url:a,attempts:this.#t,error:s.get(a)??"Remote render failed.",...i.has(a)?{httpStatus:i.get(a)}:{}}))}}async render(t){let r=JSON.stringify({sourceOrigin:new URL(t.sourceOrigin).origin,options:t.options,downloadHtml:t.downloadHtml!==false});return await new Promise((n,s)=>{this.#r.push({input:t,key:r,resolve:n,reject:s}),this.#a();})}#a(){this.#i||this.#r.length===0||(this.#i=true,setTimeout(()=>{this.#i=false,this.#p();},0));}#p(){for(;this.#o<this.#c&&this.#r.length>0;){let t=this.#r.shift();if(!t)break;let r=[t];for(let n=0;n<this.#r.length&&r.length<this.#u;){let s=this.#r[n];s?.key===t.key?(r.push(s),this.#r.splice(n,1)):n+=1;}this.#o+=1,this.renderBatch({sourceOrigin:t.input.sourceOrigin,urls:r.map(n=>n.input.url),options:t.input.options,...t.input.downloadHtml===void 0?{}:{downloadHtml:t.input.downloadHtml}}).then(n=>{let s=new Map(n.pages.map(i=>[i.page.requestedUrl,i])),o=new Map(n.failures.map(i=>[i.url,i]));for(let i of r){let u=s.get(i.input.url);if(u){i.resolve(u);continue}let a=o.get(i.input.url);i.reject(a?.httpStatus===void 0?new Error(`Remote render failed after ${a?.attempts??this.#t} attempt(s) for ${i.input.url}: ${a?.error??"Unknown remote render failure."}`):new te(i.input.url,a.httpStatus,a.attempts,a.error));}}).catch(n=>{let s=zt(n);for(let o of r)o.reject(new Error(s,{cause:n}));}).finally(()=>{this.#o-=1,this.#a();});}this.#r.length>0&&this.#a();}};function po(e,t,r){if(!e||typeof e!="object")throw new Error("Remote asset result must be a JSON object.");let n=e;if(n.schemaVersion!==H||n.response?.operation!=="asset"||n.response.jobId!==t.jobId||n.response.taskId!==t.taskId||!n.detail||!Array.isArray(n.detail.assets)||n.detail.assets.length!==r.length)throw new Error("Remote asset result has an invalid envelope.");for(let s=0;s<r.length;s+=1)if(n.detail.assets[s]?.requestedUrl!==r[s])throw new Error("Remote asset result does not match the requested URLs.");return n}var gt=class{#e;#n;#t;#s=0;constructor(t){this.#e=t.client,this.#n=X(t.jobId,"jobId"),this.#t=Math.max(1,Math.min(5,t.maxAttempts??2));}async fetch(t){let r=[...t.urls];if(r.length<1||r.length>20)throw new Error("Remote asset batches require between 1 and 20 URLs.");let n=null;for(let s=1;s<=this.#t;s+=1){this.#s+=1;let o=X(`asset-${this.#s}-${re(r.join(`
|
|
11
|
+
`)).slice(0,24)}-a${s}`,"taskId");try{let i=await this.#e.invoke({schemaVersion:H,operation:"asset",jobId:this.#n,taskId:o,urls:r,options:{ignoreHttpsErrors:t.ignoreHttpsErrors,timeoutMs:Math.max(1e3,Math.min(12e4,t.timeoutMs??6e4))}});return {assets:po(await this.#e.readResult(i),i,r).detail.assets,taskId:o,attempts:s}}catch(i){n=i;}}throw new Error(`Remote asset batch failed after ${this.#t} attempt(s): ${n instanceof Error?n.message:String(n)}`,{cause:n})}};function Zr(e,t){let r=[];for(let n=0;n<e.length;n+=t)r.push(e.slice(n,n+t));return r}function Yr(e){return {sourceOrigin:e.sourceOrigin,targetOrigin:e.targetOrigin,urlRewriteMode:e.urlRewriteMode,outputDir:"/workspace",extraReplacements:e.extraReplacements,wpsuite:{...e.wpsuite?.subscriptionType?{subscriptionType:e.wpsuite.subscriptionType}:{},...e.wpsuite?.siteSettings?{siteSettings:{subscriber:e.wpsuite.siteSettings.subscriber===true}}:{}}}}function Xr(e,t){if(!e||typeof e!="object")throw new Error("Remote rewrite result must be a JSON object.");let r=e;if(r.schemaVersion!==H||r.response?.operation!=="rewrite"||r.response.jobId!==t.jobId||r.response.taskId!==t.taskId||!r.detail||!Array.isArray(r.detail.objects))throw new Error("Remote rewrite result has an invalid envelope.");return r}async function en(e,t,r){let n=r.options??{},s=await Ft(e,n.files);if(s.length===0)return 0;let o=X(r.jobId,"jobId"),i=Qe(r.client.config.workspace.prefix,"jobs",o,"rewrite-inputs"),u=`${i}/config.json`.replace(/\/+/g,"/"),a=`${i}/asset-map.json`.replace(/\/+/g,"/"),l=`${i}/previous-asset-map.json`.replace(/\/+/g,"/");await Promise.all([r.client.writeWorkspaceObject({key:u,body:JSON.stringify(Yr(e)),contentType:"application/json; charset=utf-8"}),r.client.writeWorkspaceObject({key:a,body:JSON.stringify(t),contentType:"application/json; charset=utf-8"}),r.client.writeWorkspaceObject({key:l,body:JSON.stringify(n.previousAssetMap??{}),contentType:"application/json; charset=utf-8"})]);let d=Math.max(1,Math.min(500,r.batchSize)),c=Math.ceil(s.length/d),m=Math.max(1,Math.min(Math.max(1,r.concurrency),c));await n.onStart?.({totalFiles:s.length,workerCount:m});let h=[],f=0,b=0;await Promise.all(Array.from({length:Math.min(16,s.length)},async()=>{for(;;){let I=f;if(I>=s.length)return;f+=1;let A=s[I],R=y.join(e.outputDir,A),v=Gr.lookup(A),E=v?Gr.contentType(v)||v:"text/plain; charset=utf-8",x=`${i}/files/${re(A)}.txt`.replace(/\/+/g,"/");await r.client.writeWorkspaceObject({key:x,body:await $.readFile(R),contentType:E}),h[I]={relativePath:A,key:x,fullPath:R,contentType:E},b+=1,await r.onStageProgress?.({index:b,totalFiles:s.length,file:A});}}));let p=Zr(h,d),g=0,w=0,k=0,C=Array.from({length:m},async()=>{for(;;){let I=g;if(I>=p.length)return;g+=1;let A=p[I],R=null,v=null;for(let E=1;E<=Math.max(1,Math.min(5,r.maxAttempts));E+=1){let x=X(`rewrite-${I+1}-a${E}-${re(A.map(L=>L.relativePath).join(`
|
|
12
|
+
`)).slice(0,16)}`,"taskId"),z={schemaVersion:H,operation:"rewrite",jobId:o,taskId:x,configKey:u,assetMapKey:a,previousAssetMapKey:l,items:A.map(L=>({inputKey:L.key,outputKey:L.key,relativePath:L.relativePath,contentType:L.contentType}))};try{let L=await r.client.invoke(z);v=Xr(await r.client.readResult(L),L);break}catch(L){R=L;}}if(!v)throw new Error(`Remote rewrite batch ${I+1}/${p.length} failed: ${R instanceof Error?R.message:String(R)}`,{cause:R});for(let E of A){let x=v.detail.objects.find(z=>z.outputKey===E.key);if(!x)throw new Error(`Remote rewrite result omitted ${E.relativePath}.`);await fe(E.fullPath,Buffer.from(await r.client.readObject(x.output))),x.changed&&(k+=1),w+=1,await n.onProgress?.({index:w,totalFiles:s.length,changedTextFiles:k,file:E.relativePath,changed:x.changed});}}});return await Promise.all(C),k}async function tn(e,t,r,n){let s=n.relativePaths?new Set([...n.relativePaths].map(p=>p.replace(/\\/g,"/"))):null,o=Object.values(t.objects).filter(p=>p.isText&&(!s||s.has(p.relativePath))).sort((p,g)=>p.relativePath.localeCompare(g.relativePath));if(o.length===0)return 0;let i=X(n.jobId,"jobId"),u=Qe(n.client.config.workspace.prefix,"jobs",i,"rewrite-config"),a=`${u}/config.json`.replace(/\/+/g,"/"),l=`${u}/asset-map.json`.replace(/\/+/g,"/");await Promise.all([n.client.writeWorkspaceObject({key:a,body:JSON.stringify(Yr(e)),contentType:"application/json; charset=utf-8"}),n.client.writeWorkspaceObject({key:l,body:JSON.stringify(r),contentType:"application/json; charset=utf-8"})]);let d=Math.max(1,Math.min(500,n.batchSize)),c=Zr(o,d),m=Math.max(1,Math.min(Math.max(1,n.concurrency),c.length)),h=0,f=0,b=0;return await Promise.all(Array.from({length:m},async()=>{for(;;){let p=h;if(p>=c.length)return;h+=1;let g=c[p],w=null,k=null;for(let C=1;C<=Math.max(1,Math.min(5,n.maxAttempts));C+=1){let I=X(`rewrite-object-${p+1}-a${C}-${re(g.map(A=>A.relativePath).join(`
|
|
13
|
+
`)).slice(0,16)}`,"taskId");try{let A=await n.client.invoke({schemaVersion:H,operation:"rewrite",jobId:i,taskId:I,configKey:a,assetMapKey:l,items:g.map(R=>({inputKey:R.object.key,outputKey:R.object.key,relativePath:R.relativePath,contentType:R.contentType||R.object.contentType}))});w=Xr(await n.client.readResult(A),A);break}catch(A){k=A;}}if(!w)throw new Error(`Remote object rewrite batch ${p+1}/${c.length} failed: ${k instanceof Error?k.message:String(k)}`,{cause:k});for(let C of g){let I=w.detail.objects.find(A=>A.outputKey===C.object.key);if(!I)throw new Error(`Remote rewrite result omitted ${C.relativePath}.`);C.object=I.output,I.changed&&(b+=1),f+=1,await n.onProgress?.({index:f,totalFiles:o.length,changedTextFiles:b,file:C.relativePath,changed:I.changed});}}})),t.assetMap={...r},t.generatedAt=new Date().toISOString(),b}var ho="remote-workers.json";function rn(e={}){let t=String(e.explicitPath??"").trim();if(t)return y.resolve(t);let r=String(e.runtimeDir??process.env.STATIC_PUBLISHER_RUNTIME_DIR??process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR??"").trim();return r?y.join(y.resolve(r),ho):""}function nn(e,t,r,n=250){let s=Math.max(1,Math.floor(n)),o=[],i=new Set;for(let u of [e,...t]){let a=u.trim();if(!(!a||i.has(a)||r(a))&&(i.add(a),o.push(a),o.length>=s))break}return o}async function sn(e,t){let r=String(e.wpsuite?.runtimeToken||"").trim();if(!r)throw new Error("Page-cache purge is enabled, but the WordPress runtime token is missing.");let n=new URL("/wp-json/smartcloud-static-publisher/v1/page-cache/purge",e.sourceOrigin).toString(),s=await t.post(n,{timeout:3e4,failOnStatusCode:false,maxRedirects:0,headers:{"content-type":"application/json","x-static-publisher-token":r},data:JSON.stringify({scope:"all"})});try{let o=await s.text();if(o.length>65536)throw new Error("WordPress page-cache purge response was too large.");let i=null;try{i=JSON.parse(o);}catch{throw new Error(`WordPress page-cache purge returned invalid JSON (HTTP ${s.status()}).`)}if(!s.ok()){let u=i&&typeof i=="object"&&"message"in i?String(i.message||""):"";throw new Error(`WordPress page-cache purge failed with HTTP ${s.status()}${u?`: ${u}`:""}`)}if(!i||typeof i!="object"||i.purged!==!0)throw new Error("WordPress page-cache provider did not confirm a complete purge.");return {purged:!0,provider:String(i.provider||"custom"),scope:"all",...typeof i.purgedAt=="string"?{purgedAt:i.purgedAt}:{}}}finally{await s.dispose().catch(()=>{});}}function fo(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function ht(e){let t=String(process.env.STATIC_PUBLISHER_REMOTE_EXPORT_MANIFEST_PATH||"").trim();if(t)return y.resolve(t);let r=fo();if(r)return y.join(r,"remote-export-manifest.json");let n=y.resolve(e.outputDir);return y.join(y.dirname(n),`.${y.basename(n)}-remote-export-manifest.json`)}function ft(e){let t=y.posix.normalize(String(e||"").replace(/\\/g,"/").replace(/^\/+/,""));if(!t||t==="."||t===".."||t.startsWith("../")||/^[A-Za-z]:/.test(t))throw new Error(`Invalid remote export relative path: ${e}`);return t}function an(e,t,r){let n=r?.workspace.bucket===t.workspace.bucket&&r.workspace.prefix===t.workspace.prefix;return {schemaVersion:1,generatedAt:new Date().toISOString(),jobId:e,workspace:{...t.workspace},objects:n?{...r.objects}:{},assetMap:n?{...r.assetMap}:{}}}function un(e,t){let r=e.objects[ft(t)];if(!r)return true;let n=`${Qe(e.workspace.prefix,"jobs",X(e.jobId,"jobId"))}/`;return !me(r.object.key,e.workspace.prefix).startsWith(n)}function De(e,t){let r=ft(t.relativePath);if(t.object.bucket!==e.workspace.bucket)throw new Error(`Remote export object bucket mismatch for ${r}.`);me(t.object.key,e.workspace.prefix),e.objects[r]={...t,relativePath:r},e.generatedAt=new Date().toISOString();}function cn(e,t){delete e.objects[ft(t)],e.generatedAt=new Date().toISOString();}async function ln(e,t){t.generatedAt=new Date().toISOString();let r=ht(e);await $.mkdir(y.dirname(r),{recursive:true}),await fe(r,`${JSON.stringify(t,null,2)}
|
|
14
|
+
`,"utf8");}async function dn(e){try{let t=JSON.parse(await $.readFile(ht(e),"utf8"));if(t.schemaVersion!==1||typeof t.jobId!="string"||!t.workspace||typeof t.workspace.bucket!="string"||typeof t.workspace.prefix!="string"||!t.objects||typeof t.objects!="object"||!t.assetMap||typeof t.assetMap!="object")return null;let r=t;for(let[n,s]of Object.entries(r.objects)){if(ft(n),!s||s.relativePath!==n||s.object?.bucket!==r.workspace.bucket)return null;me(s.object.key,r.workspace.prefix);}return r}catch{return null}}var So=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml"],Cn=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),xn=new Set(["",".html",".htm"]),Po=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Ro=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function vo(e){let t=[],r="full",n="full",s=false,o=false,i="",u=null,a="";for(let l=0;l<e.length;l++){let d=e[l];if(d==="--retry-timeouts")r="retry-timeouts",s=true;else if(d==="--resume")s=true;else if(d==="--resume-rewrite")s=true,o=true;else if(d==="--crawl-mode"){let c=e[++l];if(!c)throw new Error("--crawl-mode requires a value");n=c==="incremental"?"incremental":"full",s||=n==="incremental";}else if(d.startsWith("--crawl-mode="))n=d.slice(13)==="incremental"?"incremental":"full",s||=n==="incremental";else if(d==="--incremental")n="incremental",s=true;else if(d==="--url"){let c=e[++l];if(!c)throw new Error("--url requires a value");r="single-url",s=true,t.push(c);}else if(d.startsWith("--url="))r="single-url",s=true,t.push(d.slice(6));else if(d==="--urls"){let c=e[++l];if(!c)throw new Error("--urls requires a file path");r="single-url",s=true,t.push(`@${c}`);}else if(d==="--content-sync-plan"){let c=e[++l];if(!c)throw new Error("--content-sync-plan requires a file path");r="content-sync",n="incremental",s=true,i=y.resolve(c);}else if(d.startsWith("--content-sync-plan="))r="content-sync",n="incremental",s=true,i=y.resolve(d.slice(20));else if(d==="--remote-render")u=true;else if(d==="--local-render")u=false;else if(d==="--remote-worker-config"){let c=e[++l];if(!c)throw new Error("--remote-worker-config requires a file path");a=y.resolve(c);}else d.startsWith("--remote-worker-config=")&&(a=y.resolve(d.slice(23)));}return {mode:r,crawlMode:n,urls:t,preserveOutput:s,resumeRewrite:o,contentSyncPlanPath:i,remoteRender:u,remoteWorkerConfigPath:a}}function Xe(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.resolve(e):""}function ko(){let e=Xe();return e?y.join(e,"privacy","cookie-observations.json"):null}function Co(){let e=Xe();return e?y.resolve(e,".."):""}function xo(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?y.resolve(e):""}function To(e){let t=e.trim(),r=[{alias:"@storage-root",root:Co(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Xe(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:xo(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let n of r){if(t!==n.alias&&!t.startsWith(`${n.alias}/`))continue;if(!n.root)return {resolvedPath:null,alias:n.alias,requiredEnv:n.requiredEnv};let s=t.slice(n.alias.length).replace(/^\/+/,"");return {resolvedPath:s?y.resolve(n.root,s):n.root,alias:n.alias}}return {resolvedPath:y.resolve(t)}}function Tn(e){let t=String(process.env.STATIC_PUBLISHER_CRAWL_MANIFEST_PATH||"").trim();if(t)return y.resolve(t);let r=Xe();return r?y.join(r,"crawl-manifest.json"):y.join(e.outputDir,".crawl-manifest.json")}function An(){return {schemaVersion:6,updatedAt:"",pages:{},assets:{}}}function gn(e){return {url:String(e.url||""),outputPath:String(e.outputPath||""),contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,sitemapLastmod:typeof e.sitemapLastmod=="string"?e.sitemapLastmod:null,discoveredPages:Array.isArray(e.discoveredPages)?e.discoveredPages.map(t=>String(t)):[],discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(t=>String(t)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),rewriteComplete:e.rewriteComplete===true}}function Ao(e){let t=e.isText===true;return {url:String(e.url||""),outputPath:String(e.outputPath||""),isText:t,contentType:typeof e.contentType=="string"&&e.contentType?e.contentType:null,cacheControl:typeof e.cacheControl=="string"&&e.cacheControl?e.cacheControl:null,changeToken:typeof e.changeToken=="string"?e.changeToken:null,tokenSource:typeof e.tokenSource=="string"?e.tokenSource:null,discoveredAssets:Array.isArray(e.discoveredAssets)?e.discoveredAssets.map(r=>String(r)):[],lastCrawledAt:String(e.lastCrawledAt||""),lastSeenRunId:String(e.lastSeenRunId||""),unreferencedRuns:Math.max(0,Number(e.unreferencedRuns)||0),rewriteComplete:e.rewriteComplete===true||!t}}function Gt(e){let t=e?.["content-type"]?.trim();return t||null}function Zt(e){let t=e?.["cache-control"]?.trim();return t||null}function Io(e){return e?e.split(",").map(t=>t.trim().toLowerCase()).some(t=>t==="no-cache"||t==="no-store"||/^max-age\s*=\s*0$/.test(t)):false}function Eo(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set,revalidateFiles:new Set}}function Fe(e,t,r,n){let s=Y(t.outputDir,r);s&&(Io(n)?e.revalidateFiles.add(s):e.revalidateFiles.delete(s));}function G(e,t,r){let n=Y(t.outputDir,r);n&&(e.changedFiles.add(n),e.deletedFiles.delete(n));}function Yt(e,t,r){let n=Y(t.outputDir,r);n&&(e.deletedFiles.add(n),e.changedFiles.delete(n),e.revalidateFiles.delete(n));}function ne(e,t){e.rewriteTargets.add(y.resolve(t));}async function Oo(e){try{let t=await $.readFile(Tn(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if((r.schemaVersion===2||r.schemaVersion===3||r.schemaVersion===4||r.schemaVersion===5||r.schemaVersion===6)&&r.assets&&typeof r.assets=="object")return {schemaVersion:6,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([n,s])=>[n,gn(s)])),assets:Object.fromEntries(Object.entries(r.assets).map(([n,s])=>[n,Ao(s)]))};if(r.schemaVersion===1)return {schemaVersion:6,updatedAt:String(r.updatedAt||""),pages:Object.fromEntries(Object.entries(r.pages).map(([n,s])=>[n,gn(s)])),assets:{}}}}catch{}return An()}function Xt(e){return JSON.parse(JSON.stringify(e))}async function Mo(e,t){let r=Tn(e);await $.mkdir(y.dirname(r),{recursive:true}),await $.writeFile(r,JSON.stringify(t,null,2),"utf8");}async function Pe(e,t,r=false){if(!t.trackManifest)return;t.manifestPendingChanges+=1;let n=Date.now();if(!r&&t.manifestPendingChanges<25&&n-t.lastManifestSavedAt<5e3)return;t.manifestPendingChanges=0,t.lastManifestSavedAt=n;let s=async()=>{let o=Xt(t.manifest);o.updatedAt=new Date().toISOString(),t.manifest.updatedAt=o.updatedAt,await Mo(e,o);};t.manifestSaveChain=t.manifestSaveChain.then(s,s),await t.manifestSaveChain;}function Ee(e){if(!e)return null;let t=e.trim();return t||null}function $o(e,t){let r=new Set,n=new Set([...Object.keys(e),...Object.keys(t)]);for(let s of n){let o=e[s],i=t[s];!o||!i||o===i||r.add(s);}return r}async function In(e,t,r,n){let s=[...new Set(r.map(u=>u.trim()).filter(Boolean))],o=String(e.wpsuite?.runtimeToken||"").trim();if(s.length===0||!o)return new Map;let i=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let u=await t.post(i,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-static-publisher-token":o},data:JSON.stringify({urls:s})});if(!u.ok())return n.warn(`Change token lookup failed with HTTP ${u.status()}`,{endpoint:i,status:u.status()}),new Map;let a=await u.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let l=new Map;for(let d of a.items)!d||typeof d.url!="string"||l.set(d.url,d);return l}catch(u){return n.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:i,error:u instanceof Error?`${u.name}: ${u.message.split(/\r?\n/u,1)[0]}`:"Unknown change-token request failure"}),new Map}}async function yt(e,t,r,n,s){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(n))return r.changeTokenCache.get(n)??null;let i=(await In(e,t,[n],s)).get(n)??null;return Ct(e,n)&&(!i||i.supported!==true||!i.token)&&(i=await Lo(e,t,n,s)),i=En(e,i),r.changeTokenCache.set(n,i),i}async function ir(e,t,r,n,s){if(!r.trackManifest)return;let o=[...new Set(n.map(i=>i.trim()).filter(Boolean))].filter(i=>!r.changeTokenCache.has(i));for(let i=0;i<o.length;i+=250){let u=o.slice(i,i+250),a=await In(e,t,u,s);for(let l of u)r.changeTokenCache.set(l,En(e,a.get(l)??null));}}async function jo(e,t,r,n,s,o){if(!(!n.trackManifest||n.changeTokenCache.has(s)))for(;!n.changeTokenCache.has(s);){if(!n.pageChangeTokenPrimePromise){let i=nn(s,r.pageQueue,a=>n.changeTokenCache.has(a));if(i.length===0)return;o.info(`Loading ${i.length} page change tokens in bulk.`,{phase:"prime-page-change-tokens",batchSize:i.length,remainingPages:r.pageQueue.length});let u=ir(e,t,n,i,o).finally(()=>{n.pageChangeTokenPrimePromise===u&&(n.pageChangeTokenPrimePromise=null);});n.pageChangeTokenPrimePromise=u;}await n.pageChangeTokenPrimePromise;}}function Uo(e){return `subscriber=${e.wpsuite?.siteSettings?.subscriber===true?"1":"0"}`}function En(e,t){if(!t?.supported||!t.token||t.tokenSource==="wp-media-file")return t;let r=Uo(e);return {...t,token:ue(`${t.token}
|
|
15
|
+
${r}`,40),tokenSource:t.tokenSource?`${t.tokenSource}+site-subscriber`:"site-subscriber"}}async function Lo(e,t,r,n){try{let s=await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1});if(s.status()!==404)return n.warn(`Generated 404 fallback token skipped for ${r}`,{url:r,status:s.status(),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null;let o=await s.text().catch(()=>""),i=ue(`${r}
|
|
16
16
|
404
|
|
17
|
-
${o}`,40);return n.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:i,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(s){return n.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(s),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function An(e,t,r,n,s,o){let i=vt(e,s);if(await Mo(e,t,r,n,s,o),!n.enabled)return {action:"render",changeToken:n.trackManifest?await ft(e,t,n,s,o):null};let u=n.manifest.pages[s];if(!u)return {action:"render",changeToken:await ft(e,t,n,s,o)};let a=await ft(e,t,n,s,o);if(a?.supported&&a.token)return u.changeToken===a.token?i&&!await Uo(e,t,s,o)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let l=Ee(r.sitemapLastmodByPage[s]);return l&&u.sitemapLastmod&&u.sitemapLastmod===l?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Uo(e,t,r,n){try{let o=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return o===404?!0:(n.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:o,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(s){return n.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(s),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Lo(e,t,r,n,s){let o=pn(r.previousManifest,void 0,n,"previous-manifest"),i=0;for(let[m,h]of Object.entries(r.manifest.pages))if(!r.seenPages.has(m)){Yt(s,e,h.outputPath);try{await M.unlink(h.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&n.warn(`Failed to remove stale incremental page output for ${m}`,{url:m,outputPath:h.outputPath,error:String(f)});}delete r.manifest.pages[m],i++;}let u=pn(r.manifest,r.runId,n,"active-manifest"),a=new Set,l=new Set;for(let m of u){for(let b of Je(e,m))a.add(b);let h=r.manifest.assets[m]?.outputPath;h&&l.add(h);let f=We(e,t.assetMap,m);f&&l.add(f);}for(let m of Object.keys(r.manifest.assets)){if(u.has(m)){r.manifest.assets[m].unreferencedRuns=0;continue}let h=r.manifest.assets[m];if(qr(h.unreferencedRuns)){h.unreferencedRuns+=1,n.warn(`Deferred deletion of newly unreferenced incremental asset ${m}.`,{url:m,unreferencedRuns:h.unreferencedRuns,phase:"incremental-cleanup",source:"asset-deletion-grace"});continue}delete r.manifest.assets[m];}let d=0,c=0;for(let m of o){if(r.manifest.assets[m])continue;let h=new Set,f=r.previousManifest.assets[m]?.outputPath;f&&h.add(f);let b=We(e,t.assetMap,m);if(!u.has(m)&&b)h.add(b);else if(!u.has(m)){let p=Xo(e,m,n,"incremental-cleanup");p&&(h.add(p.originalFilePath),p.hashedFilePath&&h.add(p.hashedFilePath));}for(let p of h)if(!l.has(p)){Yt(s,e,p);try{await M.unlink(p),d++;}catch(g){(g.code||"")!=="ENOENT"&&n.warn(`Failed to remove stale incremental asset output for ${m}`,{url:m,outputPath:p,error:String(g)});}}for(let p of Je(e,m))a.has(p)||Object.prototype.hasOwnProperty.call(t.assetMap,p)&&(delete t.assetMap[p],c++);}(i>0||d>0||c>0)&&n.info(`Incremental cleanup removed ${i} stale pages, ${d} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:d,removedAssetMappings:c,phase:"incremental-cleanup"});}async function Do(e,t,r,n){let s=new Set([...Object.values(t.manifest.pages),...Object.values(t.manifest.assets)].map(a=>String(a.outputPath||"").trim()).filter(Boolean).map(a=>y.resolve(a))),o=Object.values(e.postCrawlCopyMap||{}).map(a=>wt(String(a||""))).filter(a=>!!a).map(a=>y.resolve(e.outputDir,a)),u=(await Ss(["**/*.html","**/*.htm"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(a=>Wr(y.join(e.outputDir,a),e.outputDir,s,o));for(let a of u){let l=y.join(e.outputDir,a);Yt(n,e,l),await M.unlink(l);}u.length>0&&r.info(`Removed ${u.length} orphaned HTML output file(s) that are absent from the current crawl manifest.`,{phase:"incremental-cleanup",source:"manifest-output-reconciliation",removedPages:u.length,samples:u.slice(0,20)});}function pn(e,t,r,n){let s=new Set,o=i=>{let u=i.trim();if(u){if(r){let a=se(u,r,"asset",n??"manifest");if(!a)return;s.add(a.toString());return}s.add(u);}};for(let i of Object.values(e.pages))for(let u of i.discoveredAssets||[])o(u);for(let[i,u]of Object.entries(e.assets||{}))t&&u.lastSeenRunId!==t||o(i);return s}function mn(e,t,r){for(let n of t)for(let s of Je(e,n))if(r.has(s))return true;return false}function Fo(e,t,r,n,s){let o=new Set(r.rewriteTargets);for(let u of r.changedFiles)o.add(y.resolve(e.outputDir,u));for(let u of Object.values(t.manifest.pages))u.rewriteComplete||o.add(y.resolve(u.outputPath));for(let u of Object.values(t.manifest.assets))u.isText&&!u.rewriteComplete&&o.add(y.resolve(u.outputPath));let i=Oo(n,s);if(i.size===0)return [...o];for(let u of Object.values(t.manifest.pages))mn(e,u.discoveredAssets,i)&&o.add(y.resolve(u.outputPath));for(let[u,a]of Object.entries(t.manifest.assets)){a.isText&&mn(e,a.discoveredAssets,i)&&o.add(y.resolve(a.outputPath));for(let l of Je(e,u))if(i.has(l)){G(r,e,a.outputPath),a.isText&&o.add(y.resolve(a.outputPath));break}}return [...o]}function Wo(e,t){let r=t?new Set(Array.from(t,s=>y.resolve(s))):null,n=s=>r===null||r.has(y.resolve(s));for(let s of Object.values(e.manifest.pages))n(s.outputPath)&&(s.rewriteComplete=true);for(let s of Object.values(e.manifest.assets)){if(!s.isText){s.rewriteComplete=true;continue}n(s.outputPath)&&(s.rewriteComplete=true);}}function qo(e,t,r){let n=new Set,s=o=>{if(!o)return;let i=Y(e.outputDir,o);i&&n.add(i);};for(let o of t.donePages)s(me(e,o));for(let o of Object.values(r.manifest.pages))o.lastSeenRunId===r.runId&&s(o.outputPath);for(let o of t.doneAssets)s(We(e,t.assetMap,o));for(let o of Object.values(r.manifest.assets))o.lastSeenRunId===r.runId&&s(o.outputPath);return n}async function No(e,t,r,n,s,o){let i=[...n.deletedFiles].sort(),u=qo(e,t,r),a=0;for(let c of [...u].sort())if(!o?.objects[c]){try{if((await M.stat(y.join(e.outputDir,c))).isFile())continue}catch{}a++,s.warn(`Active output is missing after crawl: ${c}`,{phase:"deploy-plan",file:c,outputPath:y.join(e.outputDir,c)});}if(a>0&&s.warn(`Detected ${a} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:a}),i.length===0)return [];let l=[],d=0;for(let c of i){if(!u.has(c)){l.push(c);continue}d++;let m=y.join(e.outputDir,c),h;try{h=(await M.stat(m)).isFile();}catch{h=false;}s.warn(`Deploy plan protected active output from deletion: ${c}`,{phase:"deploy-plan",file:c,outputPath:m,exists:h});}return d>0&&s.info(`Deploy plan skipped ${d} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:d,deletedFiles:l.length}),l}function Je(e,t){let r=new Set([t]);try{let n=new URL(t),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let i of o)try{r.add(new URL(i).origin+n.pathname+n.search);}catch{}r.add(n.pathname+n.search),r.add(n.pathname);}catch{}return [...r]}function _o(e,t){let r=new Set([t]);try{let n=new URL(t),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let i of o)try{r.add(new URL(i).origin+n.pathname+n.search);}catch{}r.add(n.pathname+n.search);}catch{}return [...r]}function In(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let n=new URL(t,r);return ze(e.outputDir,n.pathname)}catch{return null}}function En(e,t,r){for(let n of _o(e,r)){let s=t[n];if(!s)continue;let o=In(e,s);if(o)return {outputPath:o,publicUrl:s}}return null}function Ho(e,t,r){for(let n of Je(e,r)){let s=t[n];if(!s)continue;let o=In(e,s);if(o)return o}return null}function We(e,t,r){let n=Ho(e,t,r);if(n)return n;try{return Xe(e,r).originalFilePath}catch{return null}}function Bo(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.join(y.resolve(e),"queue-runner-heartbeat.json"):""}async function Kt(e){let t=Bo();if(t)try{let r=await M.readFile(t,"utf8"),n=JSON.parse(r);if(!n||typeof n!="object")return;await M.writeFile(t,JSON.stringify({...n,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Vo(e,t){let r=new Set;for(let n of t.donePages)r.add(y.resolve(me(e,n)));for(let n of t.doneAssets){let s=We(e,t.assetMap,n);!s||!ce(s)||r.add(y.resolve(s));}for(let n of t.doneSitemaps){let s=We(e,t.assetMap,n);!s||!ce(s)||r.add(y.resolve(s));}return [...r]}var yt="re:",ht=new Map;function zo(e){if(!e.startsWith(yt))return null;let t=ht.get(e);if(t!==void 0)return t||null;let r=e.slice(yt.length).trim();if(!r)return ht.set(e,false),null;try{let n=new RegExp(r);return ht.set(e,n),n}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),ht.set(e,false),null}}function ar(e,t){let r=zo(t);return r?r.test(e):t.startsWith(yt)?false:e.startsWith(t)}function Qo(e,t){return t.find(r=>ar(e,r))}function On(e,t){let r=Qo(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(yt)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let n=e.blockedSearchFragments.find(s=>t.search.includes(s));return n?`blocked search fragment: ${n}`:null}function Ko(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let n of Object.keys(e.extraReplacements||{}))try{r.add(new URL(n).hostname);}catch{}return r}function Jo(e,t){return Ko(e).has(t.hostname)}function Go(e){let t=fe(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function er(e){let t=fe(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function Zo(e){let t=fe(e).replace(/\\\//g,"/").trim(),r=y.extname(t).toLowerCase();return Rn.has(r)||vn.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function Oe(e){return So.some(t=>e.pathname.startsWith(t))}function V(e,t,r=e.sourceOrigin,n,s="url",o=false){let i=Go(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return n?.ignore(s,t,"empty or unsupported scheme",r),null;if(er(i))return n?.ignore(s,t,"looks like JavaScript/code fragment",r),null;if(Zo(i))return n?.ignore(s,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(Mn(u))return n?.reject(s,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(u.protocol))return n?.ignore(s,t,"unsupported protocol: ",r),null;if(!Jo(e,u))return n?.ignore(s,u.toString(),"host not allowed",r),null;let a=On(e,u);if(a&&!(o&&At(u)))return n?.reject(s,u.toString(),a,r),null;let l=new URL(e.sourceOrigin);return u.protocol=l.protocol,u.host=l.host,u.hash="",u}catch{return n?.reject(s,t,"invalid URL",r),null}}function Mn(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function se(e,t,r,n){try{let s=new URL(e);return Mn(s)?(t.reject(r,e,"malformed URL encoding",n),null):Ve(s.pathname)===null?(t.reject(r,e,"unsafe URL pathname",n),null):s.protocol!=="http:"&&s.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${s.protocol}`,n),null):s}catch{return t.reject(r,e,"invalid URL",n),null}}function $n(e){return Rn.has(y.extname(e.pathname).toLowerCase())}function Re(e){return Oe(e)?false:vn.has(y.extname(e.pathname).toLowerCase())}function Ne(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function q(e,t){return !!(Ne(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>ar(t.pathname,r))||$n(t)||!Re(t)&&!Oe(t))}function me(e,t){let r=new URL(t),n=Ve(r.pathname);if(n===null)throw new Error(`Unsafe page URL pathname: ${r.pathname}`);return n.endsWith("/")?n+="index.html":y.extname(n)||(n+="/index.html"),ze(e.outputDir,n)}function Yo(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&y.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function vt(e,t){let r=Yo(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function Xe(e,t){let r=new URL(t),n=Ve(r.pathname);if(n===null)throw new Error(`Unsafe asset URL pathname: ${r.pathname}`);n.endsWith("/")&&(n+="index.html");let s=!!r.search,o=n,i=ze(e.outputDir,o),u=$t(e.targetOrigin,o),a=`${u}${r.search}`;if(!s)return {originalPathname:o,originalFilePath:i,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:s};let l=y.extname(n),c=`${l?n.slice(0,-l.length):n}.${ue(r.search)}${l||".bin"}`;return {originalPathname:o,originalFilePath:i,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:s,hashedPathname:c,hashedFilePath:ze(e.outputDir,c),hashedPublicUrl:$t(e.targetOrigin,c)}}function Xo(e,t,r,n){let s=se(t,r,"asset",n);return s?Xe(e,s.toString()):null}function tr(e,t,r,n,s){t[r]=n;try{let o=new URL(r),u=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{u.push(new URL(e.targetOrigin).origin);}catch{}for(let a of u)try{let l=new URL(a);t[l.origin+o.pathname+o.search]=n;}catch{}try{let a=(()=>{let l=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[o.pathname+o.search]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[o.pathname]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[o.pathname]=a;}catch{}}catch{}}async function jn(e){try{return await M.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Te(e){try{return await M.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function ei(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Un(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function ti(e,t){let r=Un(t),n=ei(t),s=e?.mtime.getTime();return r!==null&&Math.abs((s??0)-r)>=1e3||n!==null&&e?.size!==n?false:r!==null||n!==null}async function ri(e,t){if(!t)return;let r=Un(t);if(r===null)return;let n=await Te(e);if(!n)return;let s=new Date(r);await M.utimes(e,n.atime,s).catch(()=>{});}function rr(e,t){let r=y.extname(e.pathname).toLowerCase(),n=t["content-type"]||"";return bo.has(r)||yo.some(s=>n.includes(s))&&!n.startsWith("image/")&&!n.includes("font")}function ni(e){try{let t=y.extname(new URL(e).pathname).toLowerCase();return t===".js"||t===".mjs"}catch{return false}}function xe(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset processing progress: processed ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function si(e){return ue(e.toString("base64"),40)}function Ln(e,t,r){let n=e.assetVariantDigestsByOriginalPath.get(t);if(!n||n.size===0)return false;for(let s of n)if(s!==r)return true;return false}function Dn(e,t,r){let n=e.assetVariantDigestsByOriginalPath.get(t);n||(n=new Set,e.assetVariantDigestsByOriginalPath.set(t,n)),n.add(r);}async function nr(e,t,r,n,s,o,i){let u=Xe(e,r),a=si(n),l=u.originalFilePath,d=u.originalPublicUrlWithSearch;if(u.preferQueryHashed&&u.hashedFilePath&&u.hashedPublicUrl&&Ln(s,u.originalFilePath,a)){let f=En(e,t,r);f?(l=f.outputPath,d=f.publicUrl):(l=u.hashedFilePath,d=u.hashedPublicUrl);}tr(e,t,r,d,u.originalPublicUrl);let c=ce(l)?Buffer.from(Dt(n.toString("utf8"),e,t,l),"utf8"):n,m=await jn(l),h=false;return (m===null||!m.equals(c))&&(await he(l,c),h=true),o?.debug(`${h?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:l,bytes:c.length,wroteFile:h}),Dn(s,u.originalFilePath,a),await ri(l,i),s.stats.assetsSaved++,o&&(s.stats.assetsSaved%50===0||s.stats.assetsSaved===1)&&o.progress(`Asset progress: saved ${s.stats.assetsSaved}, discovered ${s.stats.assetsQueued}, pending ${s.assetQueue.length}.`,{assetsSaved:s.stats.assetsSaved,assetsQueued:s.stats.assetsQueued,assetQueue:s.assetQueue.length,phase:"download-assets"}),{outputPath:l,publicUrl:d,wroteFile:h}}function Me(e,t,r,n,s){let o=()=>{let c=String(s??"").trim();if(!c)return {discoveredFrom:"",discoveredFromType:"unknown"};if(c==="seed-path"||c==="cli"||c==="generated-404-request-path")return {discoveredFrom:c,discoveredFromType:c};if(/^https?:\/\//i.test(c))try{let m=new URL(c);return Ne(m)?{discoveredFrom:c,discoveredFromType:"sitemap"}:q(e,m)?{discoveredFrom:c,discoveredFromType:"asset"}:Re(m)&&!Oe(m)?{discoveredFrom:c,discoveredFromType:"page"}:{discoveredFrom:c,discoveredFromType:"url"}}catch{return {discoveredFrom:c,discoveredFromType:"url"}}return {discoveredFrom:c,discoveredFromType:"url"}},i=s&&/^https?:\/\//i.test(s)?s:e.sourceOrigin,u=V(e,r,i,n,"page");if(!u)return;if(!Re(u)||$n(u)){if(q(e,u)){n.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:s}),W(e,t,u.toString(),n,s);return}n.reject("page",u.toString(),Oe(u)?"asset/internal path cannot be page":"not page-like",s);return}let a=u.toString(),{discoveredFrom:l,discoveredFromType:d}=o();l&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,l),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,n.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:l,discoveredFromType:d,outputPath:me(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&n.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function W(e,t,r,n,s){let o=s&&/^https?:\/\//i.test(s)?s:e.sourceOrigin,i=V(e,r,o,n,"asset");if(!i)return;if(!q(e,i)){n.reject("asset",i.toString(),"not a safe asset path/prefix",s);return}let u=i.toString();!t.doneAssets.has(u)&&!t.queuedAssets.has(u)&&(t.queuedAssets.add(u),t.assetQueue.push(u),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&n.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function bt(e,t,r,n,s){let o=s&&/^https?:\/\//i.test(s)?s:e.sourceOrigin,i=V(e,r,o,n,"sitemap",yr(s));if(!i)return;if(!Ne(i)){n.reject("sitemap",i.toString(),"not sitemap-like",s);return}let u=i.toString();!t.doneSitemaps.has(u)&&!t.queuedSitemaps.has(u)&&(t.queuedSitemaps.add(u),t.sitemapQueue.push(u),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&n.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function oi(e){let t=fe(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(n=>n[1]);return r.length>0?r.map(n=>{let s=n.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!s?.[1])return null;let o=n.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:s[1].trim(),lastmod:Ee(o?.[1]??null)}}).filter(n=>!!n):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(n=>({loc:n[1].trim(),lastmod:null})).filter(n=>n.loc)}function ii(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function ai(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let n of e.matchAll(r)){let s=e.slice(Math.max(0,n.index-8),n.index);if(/\bnew\s+$/i.test(s))continue;let o=(n[1]||n[2]||n[3]||"").trim();o&&t.push(o);}return t}function ui(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function ci(e,t){if(t==="dom-style")return true;let r=(()=>{try{return y.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function wt(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Fn(e,t,r,n){let s=[];await M.mkdir(t,{recursive:true});let o=await M.readdir(e,{withFileTypes:true});for(let i of o){let u=y.join(e,i.name),a=y.join(t,i.name);if(i.isDirectory()){s.push(...await Fn(u,a,r,n));continue}await Wn(u,a,r,n)&&s.push(a);}return s}async function li(e,t){try{let[r,n]=await Promise.all([M.stat(e),M.stat(t)]);if(r.size!==n.size)return !1;let[s,o]=await Promise.all([M.readFile(e),M.readFile(t)]);return s.equals(o)}catch{return false}}async function Wn(e,t,r,n){let s=y.resolve(t);return r.has(s)?(n.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await li(e,t)?(n.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await M.mkdir(y.dirname(t),{recursive:true}),await M.copyFile(e,t),true)}async function di(e,t,r){let n=Object.entries(e.postCrawlCopyMap||{}).map(([u,a])=>({sourcePath:u.trim(),prefix:String(a||"").trim()}));if(n.length===0)return [];let s=0,o=[],i=new Set([...t.donePages].map(u=>y.resolve(me(e,u))));r.mark("copy-extra-paths");for(let u of n){let a=u.sourcePath,l=u.prefix;if(!a||!l){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:l});continue}let d=Co(a);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:l});continue}let c=d.resolvedPath,m=wt(l);if(!m){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:l});continue}let h;try{h=await M.stat(c);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:c,prefixPath:l});continue}let f=y.resolve(e.outputDir,m);if(h.isDirectory()){let w=await Fn(c,f,i,r);o.push(...w),s+=w.length,w.length>0&&r.progress(`Copied static directory to export: ${c} -> /${m}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${m}`,copiedItems:s,changedFiles:w.length});continue}let b=y.basename(c),p=l.endsWith("/")?y.join(f,b):f;await Wn(c,p,i,r)&&(o.push(p),s++,r.progress(`Copied static file to export: ${c} -> /${wt(l.endsWith("/")?`${l}${b}`:l)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${wt(l.endsWith("/")?`${l}${b}`:l)}`,copiedItems:s}));}return r.endMark("copy-extra-paths",{mappedSources:n.length,copiedItems:s}),o}function qe(e,t,r,n,s){let o=new Set,i=fe(r).replace(/\\\//g,"/"),u=s==="page-html"?ui(i):i,a=ci(t,s);if(a){for(let l of ai(u)){let d=V(e,l,t,n,`${s}:css-url`);d&&q(e,d)&&o.add(d.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let d=V(e,l[1],t,n,`${s}:css-import`);d&&q(e,d)&&o.add(d.toString());}}if(a||s==="page-html")for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let d=V(e,l[1],t,n,`${s}:xml-stylesheet`);d&&q(e,d)&&o.add(d.toString());}if(s==="page-html")for(let l of Lr(i)){let d=l.startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l}`:l;if(er(d)){n.ignore(`${s}:serialized-url`,d,"looks like JavaScript/code fragment",t);continue}let c=V(e,d,t,n,`${s}:serialized-url`);c&&q(e,c)?o.add(c.toString()):c&&n.ignore("asset",c.toString(),"serialized URL did not pass safe asset path filter",t);}if(s!=="page-html"&&ni(t))for(let l of Dr(i)){if(er(l)||!/\.(?:js|mjs)(?:[?#]|$)/i.test(l))continue;let d=V(e,l,t,n,`${s}:relative-asset`);d&&q(e,d)&&o.add(d.toString());}return [...o]}function qn(e,t,r,n){let s=new Set,o=fe(r);for(let i of o.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let u=i[1],a=V(e,u,t,n,"page-link");a&&(On(e,a)||Re(a)&&!Oe(a)&&!q(e,a)&&s.add(a.toString()));}return [...s]}async function gn(e,t,r,n,s,o,i=false,u=[]){let a=new Set(u),l=0,d=new Map,c=(m,h)=>{let f=[h],b=new Set;for(;f.length>0;){let p=f.pop();if(p===m)return true;b.has(p)||(b.add(p),f.push(...d.get(p)??[]));}return false};for(;r.sitemapQueue.length>0;){let m=r.sitemapQueue.shift();r.queuedSitemaps.delete(m);let h=se(m,n,"sitemap","sitemap-queue");if(!h)continue;let f=h.toString();if(r.doneSitemaps.has(f))continue;r.doneSitemaps.add(f),n.sitemap(`Fetching sitemap ${f}`,{url:f});let b=true;try{let p=await t.get(f,{timeout:6e4});if(b=!1,!p.ok()){if(n.skip("sitemap",f,p.status()),i&&!a.has(f))throw new Error(`Required child content-sync sitemap returned HTTP ${p.status()}: ${f}`);continue}let g=await p.body(),w=p.url()||f,k=se(w,n,"sitemap-response",f);if(!k)continue;let C=k.toString();r.successfulSitemaps.add(f),r.successfulSitemaps.add(C),a.has(f)&&(r.resolvedSitemapRoots.add(C),l+=1);let I=await nr(e,r.assetMap,C,g,r,n);G(s,e,I.outputPath),ne(s,I.outputPath);let A=g.toString("utf8");/<sitemapindex(?:\s|>)/i.test(A)?r.sitemapIndexUrls.add(C):r.sitemapLeafUrls.add(C);let R=qe(e,C,A,n,"sitemap");for(let E of R)W(e,r,E,n,C);let v=[];for(let E of oi(A)){let x=V(e,E.loc,C,n,"sitemap-loc",!0);if(!x){if(i)throw new Error(`Content-sync sitemap contains an inadmissible location: ${E.loc}`);continue}if(Ne(x)){if(c(C,x.toString()))throw new Error(`Sitemap index cycle detected: ${C} -> ${x.toString()}`);if(d.has(C)||d.set(C,new Set),d.get(C).add(x.toString()),v.push(x.toString()),i&&r.doneSitemaps.has(x.toString())&&!r.successfulSitemaps.has(x.toString()))throw new Error(`Required child content-sync sitemap was not resolved: ${x.toString()}`);bt(e,r,x.toString(),n,C);}else if(i){if(Re(x)&&!q(e,x)){let z=o.requiredUrls.has(x.toString()),L=o.manifest.pages[x.toString()],P=!!L&&!!await Te(L.outputPath);if(!z&&!P)throw new Error(`Content-sync sitemap coverage is incomplete for ${x.toString()}.`)}n.ignore("sitemap-loc",x.toString(),"content-sync sitemap-only traversal does not enqueue urlset locations",C);}else q(e,x)?W(e,r,x.toString(),n,C):Re(x)?(E.lastmod&&(r.sitemapLastmodByPage[x.toString()]=E.lastmod),Me(e,r,x.toString(),n,C)):n.reject("sitemap-loc",x.toString(),"not page/sitemap/asset-like",C);}o.trackManifest&&(o.manifest.assets[C]={url:C,outputPath:I.outputPath,isText:!0,contentType:p.headers()["content-type"]??"application/xml",cacheControl:Zt(p.headers()),changeToken:null,tokenSource:null,discoveredAssets:[...new Set([...R,...v])],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0,rewriteComplete:!1},await Pe(e,o));}catch(p){n.error(`Failed sitemap ${f}`,{url:f,error:String(p)});let g=i&&a.has(f)&&b;if(i&&!g)throw p}}if(i&&a.size>0&&l===0)throw new Error(`None of the configured content-sync sitemap roots could be resolved: ${[...a].join(", ")}`)}async function pi(e,t,r,n,s,o,i){let u=V(e,n,e.sourceOrigin,s,"asset-fetch");if(!u||!q(e,u))return;let a=u.toString();if(r.doneAssets.has(a))return;let l=o.enabled||o.strictTargeted?o.previousManifest.assets[a]:void 0,d=o.trackManifest?await ft(e,t,o,a,s):null,c=d?.token??null,m=Wt(l?.changeToken,d),h=o.strictTargeted&&(!d?.supported||!d.token)&&qt(e.sourceOrigin,a),f=o.strictTargeted?o.manifest.assets[a]:void 0;if(f?.lastSeenRunId===o.runId&&await Te(f.outputPath)){r.doneAssets.add(a);for(let p of f.discoveredAssets||[])W(e,r,p,s,a);G(i,e,f.outputPath),Fe(i,e,f.outputPath,f.cacheControl),f.isText&&ne(i,f.outputPath),s.info(`Resumed completed content-sync asset ${a}`,{url:a,source:"content-sync-checkpoint"}),xe(r,s);return}if(l&&(m||h)&&await Te(l.outputPath)){o.manifest.assets[a]={...l,changeToken:c,tokenSource:d?.tokenSource??l.tokenSource,discoveredAssets:[...l.discoveredAssets||[]],lastSeenRunId:o.runId,unreferencedRuns:0},delete o.manifest.pages[a],r.doneAssets.add(a);for(let p of l.discoveredAssets||[])W(e,r,p,s,a);Fe(i,e,l.outputPath,l.cacheControl),s.info(`Reused unchanged trusted asset ${a}.`,{url:a,tokenSource:d?.tokenSource??"unknown",phase:"download-assets",source:m?"asset-change-token-match":"content-sync-release-fingerprint"}),xe(r,s);return}let b=(p,g={})=>{if(!l||o.strictTargeted)return false;o.manifest.assets[a]={...l,discoveredAssets:[...l.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0},delete o.manifest.pages[a],r.doneAssets.add(a);for(let w of l.discoveredAssets||[])W(e,r,w,s,a);return s.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...g,phase:"download-assets"}),xe(r,s),true};s.asset(`Fetching asset ${a}`,{url:a});try{let p=We(e,r.assetMap,a),g=p?await Te(p):null;if(p&&g&&!o.strictTargeted)try{let L=await B(`fetch asset headers ${a}`,6e4,()=>t.head(a,{timeout:6e4})),P=L.headers();if(L.ok()&&ti(g,P)){let j=await jn(p);if(j!==null){let oe=L.url()||a,X=se(oe,s,"asset-response",a);if(!X)return;let ie=X.toString(),T=await nr(e,r.assetMap,ie,j,r,s,P);T.wroteFile&&(G(i,e,T.outputPath),ce(T.outputPath)&&ne(i,T.outputPath));let F=Zt(P)||l?.cacheControl||null;Fe(i,e,T.outputPath,F),r.doneAssets.add(a),s.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:P["last-modified"]||"",contentLength:P["content-length"]||"",phase:"download-assets"}),xe(r,s);let D=rr(u,P),Z=l?.discoveredAssets?[...l.discoveredAssets]:[];if(D||l?.isText){let K=j.toString("utf8"),_e=qe(e,ie,K,s,"asset:cached");(_e.length>0||!l)&&(Z=_e);for(let et of Z)W(e,r,et,s,ie);}if(o.trackManifest){let K=D||!!l?.isText;o.manifest.assets[a]={url:a,outputPath:T.outputPath,isText:K,contentType:Gt(P)||l?.contentType||null,cacheControl:F,changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:Z,lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0,rewriteComplete:!K},delete o.manifest.pages[a],(o.enabled||o.strictTargeted)&&K&&await Pe(e,o);}return}}}catch{}let w=await B(`fetch asset ${a}`,6e4,()=>t.get(a,{timeout:6e4}));if(!w.ok()){if(b(`http-${w.status()}`,{status:w.status()}))return;if(o.strictTargeted)throw new Error(`Required content-sync asset returned HTTP ${w.status()}: ${a}`);s.skip("asset",a,w.status());return}let k=await B(`read asset response ${a}`,6e4,()=>w.body()),C=w.url()||a,I=se(C,s,"asset-response",a);if(!I)return;let A=I.toString(),R=w.headers(),v=await nr(e,r.assetMap,A,k,r,s,R);v.wroteFile&&(G(i,e,v.outputPath),ce(v.outputPath)&&ne(i,v.outputPath));let E=Zt(R);Fe(i,e,v.outputPath,E),r.doneAssets.add(a),xe(r,s);let x=rr(u,R),z=[];if(x){let L=k.toString("utf8");z=qe(e,A,L,s,`asset:${y.extname(u.pathname).toLowerCase()||R["content-type"]||"unknown"}`);for(let P of z)W(e,r,P,s,A);}o.trackManifest&&(o.manifest.assets[a]={url:a,outputPath:v.outputPath,isText:x,contentType:Gt(R),cacheControl:E,changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:z,lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0,rewriteComplete:!x},delete o.manifest.pages[a],(o.enabled||o.strictTargeted)&&x&&await Pe(e,o));}catch(p){if(b("fetch-error",{error:String(p)}))return;if(s.error(`Failed asset ${a}`,{url:a,error:String(p)}),o.strictTargeted)throw p}finally{r.doneAssets.has(a)||(r.doneAssets.add(a),xe(r,s));}}async function mi(e,t,r,n,s,o){let i=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),u=Array.from({length:i},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await pi(e,t,r,a,n,s,o));}});await Promise.all(u);}function St(e,t){let r=Y(e.outputDir,t);if(!r)throw new Error(`Remote export output is outside outputDir: ${t}`);return r}function gi(e,t){return rr(e,{"content-type":t})}function hn(e){return /(?:^|\/)index\.html?$|\.html?$/i.test(e)}async function hi(e,t,r,n,s,o,i,u,a){let l=Math.max(1,Math.min(100,a)),d=20;for(;r.assetQueue.length>0;){let c=r.assetQueue.splice(0,l*d);for(let b of c)r.queuedAssets.delete(b);await ir(e,t,s,c,n);let m=[];for(let b of c){let p=s.manifest.assets[b],g=p?Y(e.outputDir,p.outputPath):null,w=g?i.objects[g]:void 0,k=s.changeTokenCache.get(b)??null;if(!(s.enabled&&p&&w?.kind==="asset"&&w.sourceUrl===b&&(Wt(p.changeToken,k)||(!k?.supported||!k.token)&&qt(e.sourceOrigin,b)))||!p||!g||!w){m.push(b);continue}s.manifest.assets[b]={...p,changeToken:k?.token??null,tokenSource:k?.tokenSource??p.tokenSource,lastSeenRunId:s.runId,unreferencedRuns:0};for(let I of p.discoveredAssets||[])W(e,r,I,n,b);r.doneAssets.add(b),Fe(o,e,p.outputPath,p.cacheControl),n.info(`Reused unchanged S3-native asset ${b}.`,{url:b,relativePath:g,tokenSource:k?.tokenSource??"release-fingerprint",phase:"download-assets",execution:"lambda"}),xe(r,n);}if(m.length===0)continue;let h=[];for(let b=0;b<m.length;b+=d)h.push(m.slice(b,b+d));let f=await Promise.all(h.map(b=>u.fetch({urls:b,ignoreHttpsErrors:e.ignoreHttpsErrors,timeoutMs:6e4})));for(let b of f)for(let p of b.assets){let g=p.requestedUrl;try{if(p.outcome!=="fetched"||!p.object||!p.finalUrl){let E=p.error||`Remote asset fetch ended with ${p.outcome}.`;if(n.error(`Failed remote asset ${g}`,{url:g,status:p.httpStatus,outcome:p.outcome,error:E,execution:"lambda"}),s.strictTargeted)throw new Error(E);continue}let w=se(p.finalUrl,n,"remote-asset-response",g);if(!w)continue;let k=Xe(e,w.toString()),C=k.originalFilePath,I=k.originalPublicUrlWithSearch;if(k.preferQueryHashed&&k.hashedFilePath&&k.hashedPublicUrl&&Ln(r,k.originalFilePath,p.object.sha256)){let E=En(e,r.assetMap,w.toString());C=E?.outputPath??k.hashedFilePath,I=E?.publicUrl??k.hashedPublicUrl;}tr(e,r.assetMap,w.toString(),I,k.originalPublicUrl),w.toString()!==g&&tr(e,r.assetMap,g,I,k.originalPublicUrl),Dn(r,k.originalFilePath,p.object.sha256);let A=St(e,C),R=p.contentType?.trim()||"application/octet-stream",v=gi(w,R);De(i,{relativePath:A,sourceUrl:g,kind:"asset",contentType:R,cacheControl:p.cacheControl,isText:v,object:p.object}),G(o,e,C),v&&ne(o,C),Fe(o,e,C,p.cacheControl);for(let E of p.discoveredAssets)W(e,r,E,n,w.toString());r.stats.assetsSaved++,s.trackManifest&&(s.manifest.assets[g]={url:g,outputPath:C,isText:v,contentType:R,cacheControl:p.cacheControl,changeToken:null,tokenSource:null,discoveredAssets:[...p.discoveredAssets],lastCrawledAt:new Date().toISOString(),lastSeenRunId:s.runId,unreferencedRuns:0,rewriteComplete:!v},delete s.manifest.pages[g]),n.debug(`Remote asset fetch completed ${g}`,{url:g,finalUrl:w.toString(),relativePath:A,bytes:p.object.bytes,taskId:b.taskId,attempts:b.attempts,execution:"lambda"});}finally{r.doneAssets.add(g),xe(r,n);}}}}async function fi(e,t,r,n){for(let s of [...t.changedFiles].sort()){if(r.objects[s])continue;let o=y.join(e.outputDir,s),i=await M.readFile(o).catch(d=>{throw new Error(`Coordinator-owned output is missing before S3 staging: ${s}`,{cause:d})}),u=ce(o),a=String(Gr.lookup(s)||"")||(u?"text/plain; charset=utf-8":"application/octet-stream"),l=await n.writeWorkspaceObject({key:`${r.workspace.prefix}jobs/${r.jobId}/coordinator/${ue(s,40)}-${y.basename(s)}`,body:i,contentType:a,cacheControl:hn(s)?e.s3.htmlCacheControl:e.s3.assetCacheControl});De(r,{relativePath:s,sourceUrl:null,kind:"generated",contentType:a,cacheControl:hn(s)?e.s3.htmlCacheControl:e.s3.assetCacheControl,isText:u,object:l});}}function wi(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function Se(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function Ke(e){return Math.max(15e3,Math.min(6e4,Se(e)))}function Nn(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function ur(e){return e.stats.pagesCompleted}function sr(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function Jt(e,t,r){let n=sr(e);try{if(n.kill){await n.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:n.pid,reason:r});return}if(n.pid!==null){process.kill(n.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:n.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(s){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:n.pid,reason:r,error:String(s)});}}async function B(e,t,r){let n;try{return await Promise.race([r(),new Promise((s,o)=>{n=globalThis.setTimeout(()=>{o(new Ge(e,t));},t);})])}finally{n!==void 0&&globalThis.clearTimeout(n);}}var Ge=class extends Error{constructor(r,n){super(`${r} timed out after ${wi(n)}.`);this.operation=r;this.timeoutMs=n;this.name="OperationTimeoutError";}operation;timeoutMs};async function yi(e,t){let r=Nn(e);await t.evaluate(async n=>{await new Promise(s=>{let o=0,i=700,u=Date.now(),a=()=>{window.clearInterval(l),window.scrollTo(0,0),s();},l=window.setInterval(()=>{if(Date.now()-u>=n.maxDurationMs){a();return}window.scrollBy(0,i),o+=i;let d=document.body?.scrollHeight||0,c=document.documentElement?.scrollHeight||0,m=Math.max(d,c);o>=m+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function bi(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await yi(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Si(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function _n(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ar(t,r))}async function Pi(e,t,r,n,s=true){let o=new Set;if(s){let u=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href","data-resource-url"],d=new Set,c=["srcset","data-srcset","data-lazy-srcset"],m=h=>{h.querySelectorAll("*").forEach(f=>{for(let p of l){let g=f.getAttribute(p);g&&a.add(g);}for(let p of c){let g=f.getAttribute(p);g&&g.split(",").forEach(w=>{let k=w.trim().split(/\s+/)[0];k&&a.add(k);});}let b=f.getAttribute("style");b&&d.add(b),f.shadowRoot&&m(f.shadowRoot);});};return m(document),{attrs:[...a],styles:[...d]}});for(let a of u.attrs)for(let l of ii(a)){let d=V(e,l,r,n,"dom-attr");d&&q(e,d)&&o.add(d.toString());}for(let a of u.styles)for(let l of qe(e,r,a,n,"dom-style"))o.add(l);}let i=await t.content();for(let u of qe(e,r,i,n,"page-html"))o.add(u);return [...o]}async function fn(e){return await e.evaluate(()=>{document.querySelectorAll("#wpsuite-consent-root").forEach(t=>t.remove()),document.documentElement.classList.remove("wpsuite-consent-scroll-locked"),document.body.classList.remove("wpsuite-consent-scroll-locked");}),e.content()}async function or(e,t,r,n,s){let o=me(e,t);await he(o,r,"utf8");let i=await M.stat(o);s?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:o,bytes:i.size});let u=Pt(n,o),a=Math.min(n.stats.pagesQueued,ur(n)+1);s&&u&&(n.stats.pagesSaved%10===0||n.stats.pagesSaved===1)&&s.progress(`Page progress: processed ${a}, rendered ${n.stats.pagesRendered}, saved ${n.stats.pagesSaved}, discovered ${n.stats.pagesQueued}.`,{donePages:a,claimedPages:n.donePages.size,pagesRendered:n.stats.pagesRendered,pagesSaved:n.stats.pagesSaved,pagesQueued:n.stats.pagesQueued,pageQueue:n.pageQueue.length,phase:"save-pages"});}function Pt(e,t){let r=y.resolve(t);return e.savedPageOutputs.has(r)?false:(e.savedPageOutputs.add(r),e.stats.pagesSaved=e.savedPageOutputs.size,true)}async function Rt(e,t,r,n,s){let o=Y(e.outputDir,n);if(!o)return;if(s){let u=s.objects[o];u?.kind==="page"&&u.sourceUrl===r&&u.object?.bucket===s.workspace.bucket&&typeof u.object.key=="string"&&u.object.key.startsWith(s.workspace.prefix)&&Pt(t,n);return}(await Te(n).catch(()=>null))?.isFile()&&Pt(t,n);}function Hn(e){let t=ur(e);return t%5===0||t===1||t===e.stats.pagesQueued}function Bn(e,t){let r=ur(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Ri(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let n=new URL(String(r),window.location.href),s=new URL(window.location.href);return n.hash="",s.hash="",n.href===s.href}catch{return false}};try{let r=window.location.assign.bind(window.location),n=window.location.replace.bind(window.location),s=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:o=>{if(t(o)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",o??"");return}return r(o)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:o=>{if(t(o)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",o??"");return}return n(o)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function vi(e,t,r){let n=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await Ri(n),n}async function ki(e,t,r,n,s,o,i,u,a,l){for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let d=t.pageQueue.shift();if(!d)break;t.queuedPages.delete(d);try{let c=se(d,r,"page","page-queue");if(!c)continue;let m=c.toString();if(d=m,t.queuedPages.delete(d),t.donePages.has(d))continue;t.donePages.add(d),o.enabled&&o.seenPages.add(d);let h=vt(e,d),f=_n(e,c.pathname);if(!Re(c)||q(e,c)||Oe(c)){if(q(e,c)){r.info(`Remote worker redirected non-page URL to asset queue: ${d}`,{url:d,source:"remote-worker-guard"}),W(e,t,d,r,"remote-worker-guard");continue}r.reject("page",d,"guard rejected non-page URL before remote rendering","remote-worker");continue}let b=o.strictTargeted?o.manifest.pages[d]:void 0;if(b?.lastSeenRunId===o.runId&&await Te(b.outputPath)){o.seenPages.add(d);for(let v of b.discoveredAssets||[])W(e,t,v,r,d);G(i,e,b.outputPath),ne(i,b.outputPath),await Rt(e,t,d,b.outputPath,l),r.info(`Resumed completed content-sync page ${d}`,{url:d,source:"content-sync-checkpoint"});continue}let p=await An(e,s,t,o,d,r);if(o.enabled&&n!=="single-url"&&n!=="content-sync"){let v=o.manifest.pages[d];if(p.action==="reuse"&&v){for(let E of v.discoveredAssets)W(e,t,E,r,d);for(let E of v.discoveredPages)Me(e,t,E,r,d);v.lastSeenRunId=o.runId,v.sitemapLastmod=Ee(t.sitemapLastmodByPage[d])??v.sitemapLastmod,p.changeToken?.supported&&(v.changeToken=p.changeToken.token,v.tokenSource=p.changeToken.tokenSource??v.tokenSource),await Rt(e,t,d,v.outputPath,l),r.info(`Incremental reuse skipped unchanged page ${d}`,{url:d,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}r.page(`Rendering remotely ${d}`,{url:d,execution:"lambda"});let g=await a.render({sourceOrigin:e.sourceOrigin,url:d,options:{ignoreHttpsErrors:e.ignoreHttpsErrors,javaScriptEnabled:!f,viewport:e.viewport,navigationTimeoutMs:e.navigationTimeoutMs,autoScrollTimeoutMs:Nn(e),readiness:e.readiness},downloadHtml:!l}),w=g.page,k=w.finalUrl?se(w.finalUrl,r,"remote-final-response",d)?.toString():void 0;if(n==="content-sync"&&k&&k!==m){if(o.deleteOnMissingUrls.has(d)){await Ze(e,o,i,d,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${k}`,{url:d,finalResponseUrl:k,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${d} -> ${k}`)}let C=w.httpStatus;if(h){if(C!==404){r.error(`Generated 404 request path returned unexpected status for ${d}`,{url:d,expectedStatus:404,status:C,requestPath:e.generated404RequestPath,execution:"lambda"});continue}}else if(C!==null&&C>=400){if(n==="content-sync"&&(C===404||C===410)&&o.deleteOnMissingUrls.has(d)){await Ze(e,o,i,d,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${d}`,{url:d,status:C,source:"content-sync-tombstone"});continue}if(n==="content-sync")throw new Error(`Required content-sync route returned HTTP ${C}: ${d}`);r.skip("page",d,C,t.pageDiscoverySources.get(d));continue}t.stats.pagesRendered++;let I=new Set;if(!l){if(g.html===null)throw new Error("Remote render did not return HTML for local output mode.");for(let v of qe(e,m,g.html,r,"remote-page-html"))I.add(v);}for(let v of [...w.discoveredAssets,...w.networkUrls]){let E=V(e,v,m,r,"remote-render");E&&q(e,E)&&I.add(E.toString());}for(let v of I)W(e,t,v,r,d);let A=n!=="single-url"&&n!=="content-sync"&&!h?[...new Set([...!l&&g.html!==null?qn(e,d,g.html,r):[],...w.discoveredPages])]:[];for(let v of A)Me(e,t,v,r,d);for(let v of w.cookies){let E=v.sameSite==="Strict"||v.sameSite==="Lax"||v.sameSite==="None"?v.sameSite:"Unspecified",x=!Number.isFinite(v.expires)||v.expires<=0;u.add({name:v.name,domain:v.domain,path:v.path||"/",expires:x?null:v.expires,session:x,secure:v.secure,httpOnly:v.httpOnly,sameSite:E,partitioned:v.partitioned===!0});}let R=me(e,d);if(l){if(!w.html)throw new Error("Remote render completed without a workspace HTML object.");De(l,{relativePath:St(e,R),sourceUrl:d,kind:"page",contentType:w.contentType?.trim()||"text/html; charset=utf-8",cacheControl:e.s3.htmlCacheControl,isText:!0,object:w.html}),Pt(t,R);}else {if(g.html===null)throw new Error("Remote render did not return HTML for local output mode.");await or(e,d,g.html,t,r);}G(i,e,R),ne(i,R),o.trackManifest&&(o.manifest.pages[d]={url:d,outputPath:R,contentType:w.contentType?.trim()||null,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:Ee(t.sitemapLastmodByPage[d])??null,discoveredPages:A,discoveredAssets:[...I],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,rewriteComplete:!1},(o.enabled||o.strictTargeted)&&await Pe(e,o)),r.debug(`Remote render completed ${d}`,{url:d,taskId:g.taskId,attempts:g.attempts,execution:"lambda"});}catch(c){if(r.error(`Failed remote page ${d}`,{url:d,error:String(c),execution:"lambda",...c instanceof ee?{status:c.status,attempts:c.attempts}:{}}),c instanceof ee){t.pageHttpFailures.set(d,c);continue}if(o.strictTargeted)throw c}finally{t.stats.pagesCompleted++,Hn(t)&&Bn(t,r);}}}async function Ci(e,t,r,n,s,o,i,u){let a=await chromium.launch({headless:true}),l=sr(a);l.pid!==null&&r.debug(`Launched crawl worker browser PID ${l.pid}`,{browserPid:l.pid});try{for(;t.pageQueue.length>0;){a||(a=await chromium.launch({headless:!0}),l=sr(a),l.pid!==null&&r.debug(`Relaunched crawl worker browser PID ${l.pid}`,{browserPid:l.pid}));let d=a;if(e.maxPages>0&&t.donePages.size>=e.maxPages)break;let c=t.pageQueue.shift();if(!c)break;t.queuedPages.delete(c);let m=null,h=null,f=!1,b=null,p=[],g=new Set;try{let w=se(c,r,"page","page-queue");if(!w)continue;let k=w.toString();if(c=k,t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),o.enabled&&o.seenPages.add(c);let C=vt(e,c),I=_n(e,w.pathname);if(!Re(w)||q(e,w)||Oe(w)){if(q(e,w)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),W(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let A=o.strictTargeted?o.manifest.pages[c]:void 0;if(A?.lastSeenRunId===o.runId&&await Te(A.outputPath)){o.seenPages.add(c);for(let T of A.discoveredAssets||[])W(e,t,T,r,c);G(i,e,A.outputPath),ne(i,A.outputPath),await Rt(e,t,c,A.outputPath),r.info(`Resumed completed content-sync page ${c}`,{url:c,source:"content-sync-checkpoint"});continue}let R=await An(e,s,t,o,c,r);if(o.enabled&&n!=="single-url"&&n!=="content-sync"){let T=o.manifest.pages[c];if(R.action==="reuse"&&T){for(let F of T.discoveredAssets)W(e,t,F,r,c);for(let F of T.discoveredPages)Me(e,t,F,r,c);T.lastSeenRunId=o.runId,T.sitemapLastmod=Ee(t.sitemapLastmodByPage[c])??T.sitemapLastmod,R.changeToken?.supported&&(T.changeToken=R.changeToken.token,T.tokenSource=R.changeToken.tokenSource??T.tokenSource),await Rt(e,t,c,T.outputPath),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:R.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}m=await B(`create page context ${c}`,Se(e),()=>vi(d,e,I)),h=await B(`create page ${c}`,Se(e),()=>m.newPage()),p=await m.cookies();let v=!1,E=new Set;await h.route("**/*",async T=>{let F=T.request();try{let D=V(e,F.url(),e.sourceOrigin,r,"network-request");D&&q(e,D)&&(E.add(D.toString()),W(e,t,D.toString(),r,c));}catch{}if(F.isNavigationRequest()&&F.frame()===h.mainFrame())try{let D=new URL(F.url()),Z=new URL(k);if(D.hash="",Z.hash="",D.href===Z.href){if(v){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:F.url()});try{await T.abort("aborted");}catch(K){r.debug(`Ignored canceled route while blocking reload for ${c}`,{url:c,error:String(K)});}return}v=!0;}}catch{}try{await T.continue();}catch(D){r.debug(`Ignored canceled network route for ${c}`,{url:c,requestUrl:F.url(),error:String(D)});}}),h.on("response",T=>{try{let D=V(e,T.url(),e.sourceOrigin,r,"network-response");D&&q(e,D)&&(E.add(D.toString()),W(e,t,D.toString(),r,c));}catch{}let F=T.headerValues("set-cookie").then(D=>{for(let Z of D)u.add(_r(Z,T.url()));}).catch(()=>{}).then(()=>{});g.add(F),F.finally(()=>g.delete(F));}),r.page(`Rendering ${c}`,{url:c});let x=null,z=null;try{if(x=await h.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),x&&x.ok())try{z=await B(`read response body ${c}`,Se(e),()=>x.text());}catch{z=null;}}catch(T){let F=t.pageHttpFailures.get(c);if(F){let D=(t.pageHttpAttempts.get(c)??0)+1;throw t.pageHttpAttempts.set(c,D),new ee(c,F.status,D,`Retry navigation failed: ${String(T)}`)}r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(T)});}let L=x?.status();if(lt(L)){let T=(t.pageHttpAttempts.get(c)??0)+1;throw t.pageHttpAttempts.set(c,T),new ee(c,L,T)}if(x)t.pageHttpFailures.delete(c);else if(t.pageHttpFailures.has(c)){let T=t.pageHttpFailures.get(c),F=(t.pageHttpAttempts.get(c)??0)+1;throw t.pageHttpAttempts.set(c,F),new ee(c,T.status,F,"Retry navigation returned no HTTP response.")}if(n==="content-sync"&&x){let T=se(x.url(),r,"content-sync-final-response",c)?.toString();if(T&&T!==k){if(o.deleteOnMissingUrls.has(c)){await Ze(e,o,i,c,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${T}`,{url:c,finalResponseUrl:T,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${c} -> ${T}`)}}if(C){let T=x?.status()??null;if(T!==404){r.error(`Generated 404 request path returned unexpected status for ${c}`,{url:c,expectedStatus:404,status:T,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${c}`,{url:c,status:T,outputPath:me(e,c)});}else if(x&&!x.ok()){if(n==="content-sync"&&(x.status()===404||x.status()===410)&&o.deleteOnMissingUrls.has(c)){await Ze(e,o,i,c,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${c}`,{url:c,status:x.status(),source:"content-sync-tombstone"}),await h.close();continue}if(n==="content-sync")throw new Error(`Required content-sync route returned HTTP ${x.status()}: ${c}`);r.skip("page",c,x.status(),t.pageDiscoverySources.get(c)),await h.close();continue}I?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await B(`wait for rendered page ${c}`,Se(e),()=>Si(e,h))):await B(`wait for rendered page ${c}`,Se(e),()=>bi(e,h)),t.stats.pagesRendered++;let P=await B(`extract rendered assets ${c}`,Se(e),()=>Pi(e,h,k,r,!I));for(let T of P)W(e,t,T,r,c);let j=[...new Set([...P,...E])],oe=z??await B(`capture rendered html ${c}`,Se(e),()=>fn(h)),X=n!=="single-url"&&n!=="content-sync"&&!C?qn(e,c,oe,r):[];if(n!=="single-url"&&n!=="content-sync"&&!C)for(let T of X)Me(e,t,T,r,c);await or(e,c,oe,t,r);let ie=me(e,c);if(G(i,e,ie),ne(i,ie),o.trackManifest){let T=x?.headers();o.manifest.pages[c]={url:c,outputPath:ie,contentType:Gt(T),changeToken:R.changeToken?.supported?R.changeToken.token:null,tokenSource:R.changeToken?.supported?R.changeToken.tokenSource??null:null,sitemapLastmod:Ee(t.sitemapLastmodByPage[c])??null,discoveredPages:X,discoveredAssets:j,lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,rewriteComplete:!1},(o.enabled||o.strictTargeted)&&await Pe(e,o);}}catch(w){if(w instanceof ee){t.pageHttpFailures.set(c,w),w.attempts<Bt?(b=dt(w.attempts),r.warn(`Retrying page after HTTP ${w.status}: ${c}`,{url:c,status:w.status,attempt:w.attempts,maxAttempts:Bt,delayMs:b,reason:w.message,execution:"local"})):(t.pageHttpFailures.set(c,w),r.error(`Failed page after HTTP retries ${c}`,{url:c,status:w.status,attempts:w.attempts,error:String(w),execution:"local"}));continue}if(f=w instanceof Ge,o.strictTargeted)throw r.error(`Failed required content-sync page ${c}`,{url:c,error:String(w)}),w;if(f)r.warn(`Skipped partial DOM for timed-out page ${c}`,{url:c,error:String(w),recovery:"recycle-worker-browser"});else try{let k=h?await B(`capture partial html ${c}`,Se(e),()=>fn(h)):"";if(k&&k.trim()){await or(e,c,k,t,r);let C=me(e,c);if(G(i,e,C),ne(i,C),o.trackManifest){let I=o.manifest.pages[c]??o.previousManifest.pages[c];o.manifest.pages[c]={url:c,outputPath:C,contentType:I?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:Ee(t.sitemapLastmodByPage[c])??I?.sitemapLastmod??null,discoveredPages:I?.discoveredPages?[...I.discoveredPages]:[],discoveredAssets:I?.discoveredAssets?[...I.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,rewriteComplete:!1},(o.enabled||o.strictTargeted)&&await Pe(e,o);}r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(w)});}}catch(k){k instanceof Ge&&(f=!0),r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(k)});}r.error(`Failed page ${c}`,{url:c,error:String(w)});}finally{if(!f&&h&&await B(`drain network routes ${c}`,Ke(e),()=>h.unrouteAll({behavior:"wait"})).catch(w=>{f=!0,r.warn(`Could not drain network routes for ${c}`,{url:c,browserPid:l.pid,error:String(w),recovery:"recycle-worker-browser"});}),f&&(a=null,await Jt(d,r,`page operation timeout for ${c}`),await d.close().catch(()=>{})),g.size>0&&await Promise.allSettled([...g]),!f&&m)try{let w=await m.cookies();u.addAll(Hr(p,w));}catch{}b===null&&(t.stats.pagesCompleted++,Hn(t)&&Bn(t,r)),!f&&h&&await B(`close page ${c}`,Ke(e),()=>h.close()).catch(w=>{f=!0,r.warn(`Timed out while closing page ${c}`,{url:c,browserPid:l.pid,error:String(w),recovery:"recycle-worker-browser"});}),!f&&m&&await B(`close page context ${c}`,Ke(e),()=>m.close()).catch(w=>{f=!0,r.warn(`Timed out while closing page context for ${c}`,{url:c,browserPid:l.pid,error:String(w),recovery:"recycle-worker-browser"});}),f&&a===d&&(a=null,await Jt(d,r,`page cleanup timeout for ${c}`),await d.close().catch(()=>{})),b!==null&&(await setTimeout$1(b),t.donePages.delete(c),t.queuedPages.add(c),t.pageQueue.push(c));}}}finally{if(a){let d=a;await B("close worker browser",Ke(e),()=>d.close()).catch(c=>(r.warn("Timed out while closing crawl worker browser",{browserPid:l.pid,error:String(c)}),Jt(d,r,"browser.close timeout")));}}}async function xi(e){try{let t=await M.readFile(e),n=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!n)return [];try{let s=JSON.parse(n);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return n.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let o=JSON.parse(s);return o&&typeof o=="object"?o:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function wn(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Ti(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ai(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let n=Date.parse(String(r||""));if(Number.isFinite(n))return n}return 0}async function Vn(e){try{let t=await M.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function yn(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Ii(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(n=>String(n?.role||"").trim()==="errors"&&String(n?.storedFileName||"").trim()!=="").sort((n,s)=>yn(n)-yn(s)).map(n=>y.join(e,String(n.storedFileName))):[];return [...new Set([...r,y.join(e,"errors.jsonl"),y.join(e,"errors.json"),y.join(e,"errors.jsonl.gz"),y.join(e,"errors.json.gz")])]}async function bn(e){let t=await Vn(y.join(e,"job.json"));for(let r of Ii(e,t)){let n=await xi(r);if(n.length>0)return n}return []}async function Ei(e){let t=y.join(e.logDir,"archive"),r;try{r=await M.readdir(t);}catch{return ""}let n="",s=0;for(let o of r){let i=y.join(t,o);if(!(await M.stat(i).catch(()=>null))?.isDirectory())continue;let a=await Vn(y.join(i,"job.json"));if(!Ti(a))continue;let l=Ai(a);l>=s&&(s=l,n=i);}return n}async function Oi(e){let t=await Ei(e),r=t!==""?await bn(t):await bn(e.logDir),n=new Set;for(let s of r)if(wn(s.error)||wn(s.message)){let o=s.url;typeof o=="string"&&o&&n.add(o);}return [...n]}async function Mi(e){let t=[];for(let r of e)if(r.startsWith("@")){let n=r.slice(1),s=await M.readFile(n,"utf8");t.push(...s.split(/\r?\n/).map(o=>o.trim()).filter(o=>o&&!o.startsWith("#")));}else t.push(r);return t}async function $i(e){if(!e)throw new Error("Content-sync crawl requires an immutable impact plan.");let t=JSON.parse(await M.readFile(e,"utf8"));return fr(t),t}function ji(e,t){let r;try{r=new URL(e);}catch{return false}return t.archiveFamilies.some(n=>{let s=new URL(n.url);if(r.origin!==s.origin)return false;if(r.toString()===s.toString())return true;if(r.searchParams.has("paged")&&r.pathname===s.pathname)return /^\d+$/.test(r.searchParams.get("paged")||"");let o=s.pathname.replace(/\/+$/,""),i=r.pathname.slice(o.length).replace(/^\/+/,""),u=String(n.paginationBase||"page").replace(/^\/+|\/+$/g,"").replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return r.pathname.startsWith(`${o}/`)&&new RegExp(`^${u}/\\d+/?$`).test(i)})}function Qe(e,t,r){let n=r==="page"?me(e,t):Xe(e,t).originalFilePath;return Y(e.outputDir,n)}async function Ze(e,t,r,n,s){let o=t.manifest.pages[n];if(!o)return;let i=Y(e.outputDir,o.outputPath);if(!i)throw new Error(`Manifest-owned page path escaped output root: ${n}`);let u=y.resolve(o.outputPath);Object.entries(t.manifest.pages).some(([l,d])=>l!==n&&!s.has(l)&&y.resolve(d.outputPath)===u)||(await M.rm(o.outputPath,{force:true}),r.deletedFiles.add(i),r.revalidateFiles.delete(i)),delete t.manifest.pages[n];}async function Ui(e,t,r,n,s){let o=t.manifest.assets[n];if(!o)return;let i=Y(e.outputDir,o.outputPath);if(!i)throw new Error(`Manifest-owned resource path escaped output root: ${n}`);let u=y.resolve(o.outputPath);Object.entries(t.manifest.assets).some(([l,d])=>l!==n&&!s.has(l)&&y.resolve(d.outputPath)===u)||(await M.rm(o.outputPath,{force:true}),r.deletedFiles.add(i)),delete t.manifest.assets[n];}function Li(e,t){let r=new Set,n=[...t];for(;n.length>0;){let s=n.shift();if(r.has(s))continue;r.add(s);let o=e.assets[s];for(let i of o?.discoveredAssets??[])try{Ne(new URL(i))&&n.push(i);}catch{}}return r}async function Di(e,t,r,n,s){let o=Li(r.previousManifest,s);for(let i of o){if(t.successfulSitemaps.has(i))continue;let u=r.previousManifest.assets[i];if(!u)continue;let a=Y(e.outputDir,u.outputPath);if(!a)throw new Error(`Manifest-owned sitemap path escaped output root: ${i}`);await M.rm(u.outputPath,{force:true}),n.deletedFiles.add(a),delete r.manifest.assets[i];}await Pe(e,r);}async function Fi(e){try{let t=await M.readFile(y.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Wi(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await M.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await M.unlink(y.join(e,r));}catch(n){if(n.code!=="ENOENT")throw n}}));}function Sn(e,t,r,n,s){let o=V(e,r,e.sourceOrigin,n,"manual-url");o&&(Ne(o)?bt(e,t,o.toString(),n,s):q(e,o)?W(e,t,o.toString(),n,s):Re(o)?Me(e,t,o.toString(),n,s):n.reject("manual-url",o.toString(),"not page/sitemap/asset-like",s));}async function zn(e=process.argv.slice(2)){let t=[],r=await rt(void 0,{onRemotePublisherConfigDiagnostic:S=>{t.push(S);}}),n=wr(r.sitemapPaths),s=t.at(-1)??null,o=Po(e),i=o.mode==="content-sync"?await $i(o.contentSyncPlanPath):null;if(o.resumeRewrite&&o.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let u=[];o.mode==="retry-timeouts"&&(u=await Oi(r)),o.mode==="single-url"&&(u=await Mi(o.urls)),i&&(u=[...i.directRenderUrls,...i.archiveFamilyUrls,...i.resourceRenderUrls]);let a=o.crawlMode==="incremental"&&o.mode==="full",l=a&&typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!=="";if(i&&r.subscriptionType!=="PROFESSIONAL"&&r.subscriptionType!=="AGENCY")throw new Error("Content-sync requires an active Professional or Agency subscription.");let d=(o.mode==="full"||o.mode==="single-url"||o.mode==="content-sync")&&!o.resumeRewrite,c=o.mode!=="full"||o.resumeRewrite||l,m=c?await Fi(r):{},h=l||o.mode==="single-url"||o.mode==="content-sync"?await Io(r):Cn(),f=new Set(u),b=new Set;if(i){for(let S of i.archiveFamilies)if(S.foundPosts===0)for(let $ of S.pageUrls)b.add($);for(let S of Object.keys(h.pages))ji(S,i)&&!f.has(S)&&(u.push(S),b.add(S));}let p={enabled:l,trackManifest:d,manifest:Xt(h),previousManifest:Xt(h),runId:i?.jobId??`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,pageChangeTokenPrimePromise:null,manifestSaveChain:Promise.resolve(),manifestPendingChanges:0,lastManifestSavedAt:Date.now(),strictTargeted:!!i,requiredUrls:f,deleteOnMissingUrls:b};await Wi(r.logDir);let g=new Ue(r.logDir,r.logLevel),w=o.remoteRender??r.remoteWorkers?.render?.enabled===true,k=rn({explicitPath:o.remoteWorkerConfigPath,runtimeDir:Ye()}),C=null,I=null,A=null,R=null,v=Math.max(1,r.concurrency),E=1,x=r.remoteWorkers?.rewrite?.enabled===true,z=r.remoteWorkers?.deploy?.enabled===true,L=w&&x&&z&&!o.resumeRewrite;if(!L&&o.mode==="full"&&!o.resumeRewrite&&await M.rm(gt(r),{force:true}),w&&!o.resumeRewrite||x){if(!k)throw new Error("Remote Lambda workers are enabled, but the runtime directory is unknown. Run through queue-runner with --runtime-dir, set STATIC_PUBLISHER_RUNTIME_DIR, or use --remote-worker-config for a diagnostic run.");let S=await Kr(k),$=false;if(A=new ct(S,{},{onProgress:O=>{g.progress(`Lambda ${O.operation} batch: ${O.completedItems+O.failedItems}/${O.totalItems}`,{remoteWorkerOperation:O.operation,remoteWorkerTaskId:O.taskId,remoteWorkerStatus:O.status,remoteWorkerCompletedItems:O.completedItems,remoteWorkerFailedItems:O.failedItems,remoteWorkerTotalItems:O.totalItems,remoteWorkerUpdatedAt:O.updatedAt});},onProgressError:O=>{$||($=true,g.warn("Lambda progress status could not be read; task execution continues.",{error:O.message}));}}),L){if(!S.functions.asset)throw new Error("Full Lambda delegation requires an asset worker. Redeploy the Lambda worker stack and reinstall remote-workers.json.");let O=c?await cn(r):null;R=on(`crawl-${ue(p.runId,40)}`,S,O),I=new mt({client:A,jobId:R.jobId,maxAttempts:2}),g.info("S3-native Lambda data plane is enabled.",{execution:"lambda",assetWorker:S.functions.asset,workspaceBucket:S.workspace.bucket,workspacePrefix:S.workspace.prefix});}}if(w&&!o.resumeRewrite){if(!A)throw new Error("Remote render worker client was not initialized.");v=Math.max(1,Math.min(100,L?r.concurrency:r.remoteWorkers?.render?.concurrency??r.concurrency)),E=Math.max(1,Math.min(20,r.remoteWorkers?.render?.batchSize??5)),C=new pt({client:A,jobId:`crawl-${ue(p.runId,40)}`,maxAttempts:r.remoteWorkers?.render?.maxAttempts??2,batchSize:E,maxConcurrentBatches:v,onRetry:S=>{g.warn(`Retrying remote render for ${S.url}`,{url:S.url,attempt:S.attempt,maxAttempts:S.maxAttempts,reason:S.reason,status:S.status,delayMs:S.delayMs,execution:"lambda"});}}),g.info("Lambda page rendering is enabled.",{remoteWorkerConfigPath:y.resolve(k),concurrency:v,maxAttempts:r.remoteWorkers?.render?.maxAttempts??2,batchSize:E,execution:"lambda"});}if(s){let S={status:s.status,virtualAssetBaseUrl:s.virtualAssetBaseUrl,subscriptionType:s.subscriptionType??null,error:s.error,requests:s.requests,source:"remote-publisher-config"};s.status==="loaded"&&s.subscriptionType?g.info("Loaded remote WP Suite publisher configuration.",S):g.warn("Remote WP Suite publisher configuration is unavailable or has no active subscription type.",S);}if(a&&!l){let S="Incremental crawl requires an active remote publisher subscription; refusing to fall back to a destructive full crawl.";throw g.error(S,{requestedMode:o.crawlMode,subscriptionType:r.subscriptionType??null,source:"incremental-subscription-guard"}),await g.flush(),new Error(S)}let P={pageQueue:[],queuedPages:new Set,donePages:new Set,savedPageOutputs:new Set,pageHttpAttempts:new Map,pageHttpFailures:new Map,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,successfulSitemaps:new Set,resolvedSitemapRoots:new Set,sitemapIndexUrls:new Set,sitemapLeafUrls:new Set,sitemapLastmodByPage:{},assetMap:{...m},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},j=Ao(),oe=new at,X=Ro();if(X&&!o.resumeRewrite&&await ut(X,oe.artifact()),await gr(r),await Ct(r,{startedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir)}),c||await M.rm(r.outputDir,{recursive:true,force:true}),await M.mkdir(r.outputDir,{recursive:true}),c||await Ct(r,{startedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir)}),i){let S=new Set([...i.deleteUrls,...p.deleteOnMissingUrls]);for(let O of i.deleteUrls)await Ze(r,p,j,O,S);let $=new Set(i.resourceDeleteUrls);for(let O of i.resourceDeleteUrls)await Ui(r,p,j,O,$);}if(o.resumeRewrite)g.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:o.mode,crawlMode:o.crawlMode});else {let S=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{if(g.mark("discovery"),o.mode==="full"?(n.forEach(O=>bt(r,P,O,g)),await gn(r,S,P,g,j,p),r.seedPaths.forEach(O=>Sn(r,P,O,g,"seed-path")),r.generated404RequestPath&&Me(r,P,r.generated404RequestPath,g,"generated-404-request-path")):(u.forEach(O=>Sn(r,P,O,g,"cli")),i?.sitemapUrls.forEach(O=>bt(r,P,O,g,"content-sync-impact")),await gn(r,S,P,g,j,p,o.mode==="content-sync",i?.sitemapUrls??[]),i&&await Di(r,P,p,j,i.sitemapUrls)),g.endMark("discovery",{pages:P.pageQueue.length,assets:P.assetQueue.length,sitemaps:P.sitemapQueue.length}),g.summary(`Queued ${P.pageQueue.length} pages, ${P.assetQueue.length} assets, ${P.sitemapQueue.length} sitemaps.`,{mode:o.mode,crawlMode:o.crawlMode,queuedPages:P.pageQueue.length,queuedAssets:P.assetQueue.length,queuedSitemaps:P.sitemapQueue.length}),o.mode==="retry-timeouts"&&P.pageQueue.length===0&&P.assetQueue.length===0&&P.sitemapQueue.length===0){g.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:o.mode,crawlMode:o.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await g.flush(),X&&await ut(X,oe.artifact());return}g.mark("render-pages");let $=Array.from({length:C?v*E:v},()=>C?ki(r,P,g,o.mode,S,p,j,oe,C,R??void 0):Ci(r,P,g,o.mode,S,p,j,oe));if(await Promise.all($),P.pageHttpFailures.size>0&&await g.flush(),Jr(P.pageHttpFailures),X&&await ut(X,oe.artifact()),g.endMark("render-pages",{pages:P.stats.pagesCompleted}),g.mark("download-assets"),g.progress("Asset download phase started.",{phase:"download-assets",doneAssets:P.doneAssets.size,assetsQueued:P.stats.assetsQueued,assetQueue:P.assetQueue.length}),R&&I?await hi(r,S,P,g,p,j,R,I,r.concurrency):(await ir(r,S,p,[...P.assetQueue],g),await mi(r,S,P,g,p,j)),g.endMark("download-assets",{assets:P.doneAssets.size}),p.enabled&&r.maxPages===0&&(await Lo(r,P,p,g,j),o.mode==="full"&&await Do(r,p,g,j)),o.mode==="full"){let O=await di(r,P,g);for(let N of O)if(G(j,r,N),R&&A){let J=St(r,N),Ae=await M.readFile(N),Ie=ce(N),$e=String(Gr.lookup(J)||"")||(Ie?"text/plain; charset=utf-8":"application/octet-stream"),tt=await A.writeWorkspaceObject({key:`${R.workspace.prefix}jobs/${R.jobId}/copied/${ue(J,40)}-${y.basename(J)}`,body:Ae,contentType:$e});De(R,{relativePath:J,sourceUrl:null,kind:"generated",contentType:tt.contentType,cacheControl:null,isText:Ie,object:tt}),Ie&&ne(j,N);}}}finally{await B("dispose crawl request context",Ke(r),()=>S.dispose()).catch($=>{g.warn("Timed out while disposing crawl request context",{error:String($)});});}}if(p.trackManifest&&await Pe(r,p,true),R)for(let S of j.deletedFiles)an(R,S);let ie=JSON.stringify(P.assetMap,null,2);if(await M.writeFile(y.join(r.outputDir,"asset-map.json"),ie,"utf8"),R&&A){let S=await A.writeWorkspaceObject({key:`${R.workspace.prefix}jobs/${R.jobId}/generated/asset-map.json`,body:ie,contentType:"application/json; charset=utf-8"});De(R,{relativePath:"asset-map.json",sourceUrl:null,kind:"generated",contentType:"application/json; charset=utf-8",cacheControl:r.s3.assetCacheControl,isText:true,object:S});}JSON.stringify(m,null,2)!==JSON.stringify(P.assetMap,null,2)&&G(j,r,y.join(r.outputDir,"asset-map.json")),R&&A&&await fi(r,j,R,A);let T=o.mode==="single-url"||o.mode==="content-sync"?Vo(r,P):p.enabled&&!o.resumeRewrite?Fo(r,p,j,m,P.assetMap):void 0;g.mark("rewrite-text");let F=o.mode==="single-url"||o.mode==="content-sync"?`Rewriting text files touched by URL crawl: 0/${T?.length??0}`:"Rewriting text files...";g.progress(F,{phase:"rewrite-text",index:0,totalFiles:T?.length,changedTextFiles:0}),await Kt(F);let D=Date.now(),Z={files:T,previousAssetMap:m,onStart:({totalFiles:S,workerCount:$})=>{g.info(`Rewrite phase scanning ${S} text files.`,{phase:"rewrite-text",totalFiles:S,workerCount:$});},onProgress:async({index:S,totalFiles:$,changedTextFiles:O,file:N,changed:J})=>{J&&G(j,r,y.join(r.outputDir,N)),g.checkpoint(`Rewriting text file ${S}/${$}`,{phase:"rewrite-text",index:S,totalFiles:$,changedTextFiles:O,file:N}),g.debug(`${J?"Rewrote":"Checked unchanged"} text file ${S}/${$}: ${N}`,{phase:"rewrite-text",index:S,totalFiles:$,changedTextFiles:O,file:N,changed:J});let Ae=Date.now();if(!(S===1||S===$||S%50===0||Ae-D>=5e3))return;D=Ae;let $e=`Rewriting text files: ${S}/${$}`;g.progress($e,{phase:"rewrite-text",index:S,totalFiles:$,changedTextFiles:O,file:N}),await Kt($e);}},K;if(R&&A){let S=T?.map(O=>St(r,y.isAbsolute(O)?O:y.join(r.outputDir,O))),$=Object.values(R.objects).filter(O=>O.isText&&(!S||S.includes(O.relativePath))).length;await Z.onStart?.({totalFiles:$,workerCount:Math.min(r.concurrency,$)}),K=await tn(r,R,P.assetMap,{client:A,jobId:R.jobId,concurrency:r.concurrency,batchSize:25,maxAttempts:2,...S?{relativePaths:S}:{},onProgress:Z.onProgress}),await un(r,R);}else x&&A?K=await en(r,P.assetMap,{client:A,jobId:`crawl-${ue(p.runId,40)}`,concurrency:r.remoteWorkers?.rewrite?.concurrency??r.rewriteConcurrency,batchSize:r.remoteWorkers?.rewrite?.batchSize??25,maxAttempts:r.remoteWorkers?.rewrite?.maxAttempts??2,onStageProgress:async({index:S,totalFiles:$,file:O})=>{let N=Date.now();if(S!==1&&S!==$&&S%25!==0&&N-D<5e3)return;D=N;let J=`Staging text files for Lambda rewrite: ${S}/${$}`;g.progress(J,{phase:"rewrite-stage",index:S,totalFiles:$,file:O,execution:"lambda"}),await Kt(J);},options:Z}):K=await Er(r,P.assetMap,Z);p.trackManifest&&(Wo(p,T),await Pe(r,p,true)),g.endMark("rewrite-text",{changedTextFiles:K});let _e=i?[...j.deletedFiles].sort():p.enabled&&!o.resumeRewrite?await No(r,P,p,j,g,R):[],et=[...new Set((T||[]).map(S=>Y(r.outputDir,y.isAbsolute(S)?S:y.join(r.outputDir,S))).filter(S=>S!==null))].sort();if(i){let S=[...j.changedFiles].sort(),$=new Set(S),O=new Set(i.directRenderUrls.map(U=>Qe(r,U,"page")).filter(U=>!!U&&$.has(U))),N=new Set(i.archiveFamilyUrls.map(U=>Qe(r,U,"page")).filter(U=>!!U&&$.has(U))),J=new Set(i.resourceRenderUrls.map(U=>Qe(r,U,"asset")).filter(U=>!!U&&$.has(U))),Ae=new Set([...P.sitemapLeafUrls].map(U=>Qe(r,U,"asset")).filter(U=>!!U&&$.has(U))),Ie=new Set([...P.sitemapIndexUrls].map(U=>Qe(r,U,"asset")).filter(U=>!!U&&$.has(U))),$e=new Set([...O,...N,...J,...Ae,...Ie]),tt=S.filter(U=>!$e.has(U)),Qn=[...new Set([...i.directRenderUrls,...i.archiveFamilyUrls,...P.resolvedSitemapRoots,...P.sitemapLeafUrls,...P.sitemapIndexUrls,...i.deleteUrls,...i.resourceRenderUrls,...i.resourceDeleteUrls].map(U=>{let cr=new URL(U);return `${cr.pathname}${cr.search}`}))].sort();await kt(r,{schemaVersion:2,generatedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir),runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,changedFiles:S,deletedFiles:_e,rewriteTargets:et,revalidateFiles:[...j.revalidateFiles].sort(),consumerId:i.consumerId,fromSequence:i.fromSequence,toSequence:i.toSequence,impactHash:i.impactHash,resolvedSitemapUrls:[...P.resolvedSitemapRoots],uploadGroups:{assets:tt,content:[...O].sort(),listings:[...N].sort(),resources:[...J].sort(),sitemapLeaves:[...Ae].sort(),sitemapIndexes:[...Ie].reverse()},invalidationPaths:Qn});}else await kt(r,{schemaVersion:1,generatedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir),runMode:o.mode,crawlMode:o.crawlMode,fullSyncRequired:!(p.enabled&&!o.resumeRewrite),changedFiles:p.enabled&&!o.resumeRewrite?[...j.changedFiles].sort():[],deletedFiles:p.enabled&&!o.resumeRewrite?_e:[],rewriteTargets:p.enabled&&!o.resumeRewrite?et:[],revalidateFiles:[...j.revalidateFiles].sort()});await hr(r),g.summary(o.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${K} text files.`:`Done. Rendered ${P.stats.pagesRendered} pages, processed ${P.doneSitemaps.size} sitemaps and ${P.doneAssets.size} assets.`,{mode:o.mode,crawlMode:o.crawlMode,resumeRewrite:o.resumeRewrite,...P.stats,pagesRendered:P.stats.pagesRendered,donePages:P.stats.pagesCompleted,claimedPages:P.donePages.size,doneSitemaps:P.doneSitemaps.size,doneAssets:P.doneAssets.size,changedTextFiles:K}),await g.flush();}zn().catch(async e=>{let t=we(e instanceof Error?e.message:String(e));console.error(t);try{let n=(await rt().catch(()=>null))?.logDir??"logs",s=new Ue(n,"debug");s.error(`Unhandled error: ${t}`,e instanceof Error&&e.stack?{stack:we(e.stack)}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
17
|
+
${o}`,40);return n.info(`Using generated 404 fallback change token for ${r}`,{url:r,requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),{url:r,supported:!0,token:i,tokenSource:"generated-404-response-hash",postId:null,dependencyPostIds:[],reason:null}}catch(s){return n.warn(`Generated 404 fallback token lookup failed for ${r}`,{url:r,error:String(s),requestPath:e.generated404RequestPath,source:"generated-404-change-token-fallback"}),null}}async function On(e,t,r,n,s,o){let i=Ct(e,s);if(await jo(e,t,r,n,s,o),!n.enabled)return {action:"render",changeToken:n.trackManifest?await yt(e,t,n,s,o):null};let u=n.manifest.pages[s];if(!u)return {action:"render",changeToken:await yt(e,t,n,s,o)};let a=await yt(e,t,n,s,o);if(a?.supported&&a.token)return u.changeToken===a.token?i&&!await Do(e,t,s,o)?{action:"render",changeToken:a}:{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let l=Ee(r.sitemapLastmodByPage[s]);return l&&u.sitemapLastmod&&u.sitemapLastmod===l?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Do(e,t,r,n){try{let o=(await t.get(r,{timeout:Math.min(e.navigationTimeoutMs,15e3),failOnStatusCode:!1})).status();return o===404?!0:(n.warn(`Generated 404 request path no longer returns 404 for ${r}`,{url:r,status:o,requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),!1)}catch(s){return n.warn(`Generated 404 status probe failed for ${r}`,{url:r,error:String(s),requestPath:e.generated404RequestPath,source:"generated-404-status-probe"}),false}}async function Fo(e,t,r,n,s){let o=hn(r.previousManifest,void 0,n,"previous-manifest"),i=0;for(let[m,h]of Object.entries(r.manifest.pages))if(!r.seenPages.has(m)){Yt(s,e,h.outputPath);try{await $.unlink(h.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&n.warn(`Failed to remove stale incremental page output for ${m}`,{url:m,outputPath:h.outputPath,error:String(f)});}delete r.manifest.pages[m],i++;}let u=hn(r.manifest,r.runId,n,"active-manifest"),a=new Set,l=new Set;for(let m of u){for(let b of Ge(e,m))a.add(b);let h=r.manifest.assets[m]?.outputPath;h&&l.add(h);let f=We(e,t.assetMap,m);f&&l.add(f);}for(let m of Object.keys(r.manifest.assets)){if(u.has(m)){r.manifest.assets[m].unreferencedRuns=0;continue}let h=r.manifest.assets[m];if(qr(h.unreferencedRuns)){h.unreferencedRuns+=1,n.warn(`Deferred deletion of newly unreferenced incremental asset ${m}.`,{url:m,unreferencedRuns:h.unreferencedRuns,phase:"incremental-cleanup",source:"asset-deletion-grace"});continue}delete r.manifest.assets[m];}let d=0,c=0;for(let m of o){if(r.manifest.assets[m])continue;let h=new Set,f=r.previousManifest.assets[m]?.outputPath;f&&h.add(f);let b=We(e,t.assetMap,m);if(!u.has(m)&&b)h.add(b);else if(!u.has(m)){let p=ti(e,m,n,"incremental-cleanup");p&&(h.add(p.originalFilePath),p.hashedFilePath&&h.add(p.hashedFilePath));}for(let p of h)if(!l.has(p)){Yt(s,e,p);try{await $.unlink(p),d++;}catch(g){(g.code||"")!=="ENOENT"&&n.warn(`Failed to remove stale incremental asset output for ${m}`,{url:m,outputPath:p,error:String(g)});}}for(let p of Ge(e,m))a.has(p)||Object.prototype.hasOwnProperty.call(t.assetMap,p)&&(delete t.assetMap[p],c++);}(i>0||d>0||c>0)&&n.info(`Incremental cleanup removed ${i} stale pages, ${d} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:d,removedAssetMappings:c,phase:"incremental-cleanup"});}async function Wo(e,t,r,n){let s=new Set([...Object.values(t.manifest.pages),...Object.values(t.manifest.assets)].map(a=>String(a.outputPath||"").trim()).filter(Boolean).map(a=>y.resolve(a))),o=Object.values(e.postCrawlCopyMap||{}).map(a=>bt(String(a||""))).filter(a=>!!a).map(a=>y.resolve(e.outputDir,a)),u=(await vs(["**/*.html","**/*.htm"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(a=>Wr(y.join(e.outputDir,a),e.outputDir,s,o));for(let a of u){let l=y.join(e.outputDir,a);Yt(n,e,l),await $.unlink(l);}u.length>0&&r.info(`Removed ${u.length} orphaned HTML output file(s) that are absent from the current crawl manifest.`,{phase:"incremental-cleanup",source:"manifest-output-reconciliation",removedPages:u.length,samples:u.slice(0,20)});}function hn(e,t,r,n){let s=new Set,o=i=>{let u=i.trim();if(u){if(r){let a=se(u,r,"asset",n??"manifest");if(!a)return;s.add(a.toString());return}s.add(u);}};for(let i of Object.values(e.pages))for(let u of i.discoveredAssets||[])o(u);for(let[i,u]of Object.entries(e.assets||{}))t&&u.lastSeenRunId!==t||o(i);return s}function fn(e,t,r){for(let n of t)for(let s of Ge(e,n))if(r.has(s))return true;return false}function qo(e,t,r,n,s){let o=new Set(r.rewriteTargets);for(let u of r.changedFiles)o.add(y.resolve(e.outputDir,u));for(let u of Object.values(t.manifest.pages))u.rewriteComplete||o.add(y.resolve(u.outputPath));for(let u of Object.values(t.manifest.assets))u.isText&&!u.rewriteComplete&&o.add(y.resolve(u.outputPath));let i=$o(n,s);if(i.size===0)return [...o];for(let u of Object.values(t.manifest.pages))fn(e,u.discoveredAssets,i)&&o.add(y.resolve(u.outputPath));for(let[u,a]of Object.entries(t.manifest.assets)){a.isText&&fn(e,a.discoveredAssets,i)&&o.add(y.resolve(a.outputPath));for(let l of Ge(e,u))if(i.has(l)){G(r,e,a.outputPath),a.isText&&o.add(y.resolve(a.outputPath));break}}return [...o]}function No(e,t){let r=t?new Set(Array.from(t,s=>y.resolve(s))):null,n=s=>r===null||r.has(y.resolve(s));for(let s of Object.values(e.manifest.pages))n(s.outputPath)&&(s.rewriteComplete=true);for(let s of Object.values(e.manifest.assets)){if(!s.isText){s.rewriteComplete=true;continue}n(s.outputPath)&&(s.rewriteComplete=true);}}function _o(e,t,r){let n=new Set,s=o=>{if(!o)return;let i=Y(e.outputDir,o);i&&n.add(i);};for(let o of t.donePages)s(ge(e,o));for(let o of Object.values(r.manifest.pages))o.lastSeenRunId===r.runId&&s(o.outputPath);for(let o of t.doneAssets)s(We(e,t.assetMap,o));for(let o of Object.values(r.manifest.assets))o.lastSeenRunId===r.runId&&s(o.outputPath);return n}async function Ho(e,t,r,n,s,o){let i=[...n.deletedFiles].sort(),u=_o(e,t,r),a=0;for(let c of [...u].sort())if(!o?.objects[c]){try{if((await $.stat(y.join(e.outputDir,c))).isFile())continue}catch{}a++,s.warn(`Active output is missing after crawl: ${c}`,{phase:"deploy-plan",file:c,outputPath:y.join(e.outputDir,c)});}if(a>0&&s.warn(`Detected ${a} active output file(s) missing before deploy plan save.`,{phase:"deploy-plan",missingActiveOutputs:a}),i.length===0)return [];let l=[],d=0;for(let c of i){if(!u.has(c)){l.push(c);continue}d++;let m=y.join(e.outputDir,c),h;try{h=(await $.stat(m)).isFile();}catch{h=false;}s.warn(`Deploy plan protected active output from deletion: ${c}`,{phase:"deploy-plan",file:c,outputPath:m,exists:h});}return d>0&&s.info(`Deploy plan skipped ${d} active output deletion(s).`,{phase:"deploy-plan",protectedDeletes:d,deletedFiles:l.length}),l}function Ge(e,t){let r=new Set([t]);try{let n=new URL(t),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let i of o)try{r.add(new URL(i).origin+n.pathname+n.search);}catch{}r.add(n.pathname+n.search),r.add(n.pathname);}catch{}return [...r]}function Bo(e,t){let r=new Set([t]);try{let n=new URL(t),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let i of o)try{r.add(new URL(i).origin+n.pathname+n.search);}catch{}r.add(n.pathname+n.search);}catch{}return [...r]}function Mn(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let n=new URL(t,r);return ze(e.outputDir,n.pathname)}catch{return null}}function $n(e,t,r){for(let n of Bo(e,r)){let s=t[n];if(!s)continue;let o=Mn(e,s);if(o)return {outputPath:o,publicUrl:s}}return null}function Vo(e,t,r){for(let n of Ge(e,r)){let s=t[n];if(!s)continue;let o=Mn(e,s);if(o)return o}return null}function We(e,t,r){let n=Vo(e,t,r);if(n)return n;try{return et(e,r).originalFilePath}catch{return null}}function zo(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?y.join(y.resolve(e),"queue-runner-heartbeat.json"):""}async function Kt(e){let t=zo();if(t)try{let r=await $.readFile(t,"utf8"),n=JSON.parse(r);if(!n||typeof n!="object")return;await $.writeFile(t,JSON.stringify({...n,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Qo(e,t){let r=new Set;for(let n of t.donePages)r.add(y.resolve(ge(e,n)));for(let n of t.doneAssets){let s=We(e,t.assetMap,n);!s||!ce(s)||r.add(y.resolve(s));}for(let n of t.doneSitemaps){let s=We(e,t.assetMap,n);!s||!ce(s)||r.add(y.resolve(s));}return [...r]}var St="re:",wt=new Map;function Ko(e){if(!e.startsWith(St))return null;let t=wt.get(e);if(t!==void 0)return t||null;let r=e.slice(St.length).trim();if(!r)return wt.set(e,false),null;try{let n=new RegExp(r);return wt.set(e,n),n}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),wt.set(e,false),null}}function ar(e,t){let r=Ko(t);return r?r.test(e):t.startsWith(St)?false:e.startsWith(t)}function Jo(e,t){return t.find(r=>ar(e,r))}function jn(e,t){let r=Jo(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(St)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let n=e.blockedSearchFragments.find(s=>t.search.includes(s));return n?`blocked search fragment: ${n}`:null}function Go(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let n of Object.keys(e.extraReplacements||{}))try{r.add(new URL(n).hostname);}catch{}return r}function Zo(e,t){return Go(e).has(t.hostname)}function Yo(e){let t=we(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function er(e){let t=we(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)||/[()[\]'"]/.test(t)}function Xo(e){let t=we(e).replace(/\\\//g,"/").trim(),r=y.extname(t).toLowerCase();return Cn.has(r)||xn.has(r)?false:/^(?:this|[A-Za-z_$][\w$]*)\.[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*$/.test(t)}function Oe(e){return Ro.some(t=>e.pathname.startsWith(t))}function V(e,t,r=e.sourceOrigin,n,s="url",o=false){let i=Yo(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return n?.ignore(s,t,"empty or unsupported scheme",r),null;if(er(i))return n?.ignore(s,t,"looks like JavaScript/code fragment",r),null;if(Xo(i))return n?.ignore(s,t,"looks like JavaScript property access",r),null;try{let u=new URL(i,r);if(Un(u))return n?.reject(s,t,"malformed URL encoding",r),null;if(!["http:","https:"].includes(u.protocol))return n?.ignore(s,t,"unsupported protocol: ",r),null;if(!Zo(e,u))return n?.ignore(s,u.toString(),"host not allowed",r),null;let a=jn(e,u);if(a&&!(o&&Et(u)))return n?.reject(s,u.toString(),a,r),null;let l=new URL(e.sourceOrigin);return u.protocol=l.protocol,u.host=l.host,u.hash="",u}catch{return n?.reject(s,t,"invalid URL",r),null}}function Un(e){try{return decodeURIComponent(e.pathname),decodeURIComponent(e.search),decodeURIComponent(e.hash),!1}catch{return true}}function se(e,t,r,n){try{let s=new URL(e);return Un(s)?(t.reject(r,e,"malformed URL encoding",n),null):Ve(s.pathname)===null?(t.reject(r,e,"unsafe URL pathname",n),null):s.protocol!=="http:"&&s.protocol!=="https:"?(t.reject(r,e,`unsupported protocol: ${s.protocol}`,n),null):s}catch{return t.reject(r,e,"invalid URL",n),null}}function Ln(e){return Cn.has(y.extname(e.pathname).toLowerCase())}function Re(e){return Oe(e)?false:xn.has(y.extname(e.pathname).toLowerCase())}function Ne(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function q(e,t){return !!(Ne(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")||e.assetPathPrefixes.length>0&&e.assetPathPrefixes.some(r=>ar(t.pathname,r))||Ln(t)||!Re(t)&&!Oe(t))}function ge(e,t){let r=new URL(t),n=Ve(r.pathname);if(n===null)throw new Error(`Unsafe page URL pathname: ${r.pathname}`);return n.endsWith("/")?n+="index.html":y.extname(n)||(n+="/index.html"),ze(e.outputDir,n)}function ei(e){if(!e.generated404RequestPath)return null;try{let t=new URL(e.generated404RequestPath,e.sourceOrigin);return !t.pathname.endsWith("/")&&y.extname(t.pathname)===""&&(t.pathname+="/"),t.hash="",t.toString()}catch{return null}}function Ct(e,t){let r=ei(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function et(e,t){let r=new URL(t),n=Ve(r.pathname);if(n===null)throw new Error(`Unsafe asset URL pathname: ${r.pathname}`);n.endsWith("/")&&(n+="index.html");let s=!!r.search,o=n,i=ze(e.outputDir,o),u=Ut(e.targetOrigin,o),a=`${u}${r.search}`;if(!s)return {originalPathname:o,originalFilePath:i,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:s};let l=y.extname(n),c=`${l?n.slice(0,-l.length):n}.${ue(r.search)}${l||".bin"}`;return {originalPathname:o,originalFilePath:i,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:s,hashedPathname:c,hashedFilePath:ze(e.outputDir,c),hashedPublicUrl:Ut(e.targetOrigin,c)}}function ti(e,t,r,n){let s=se(t,r,"asset",n);return s?et(e,s.toString()):null}function tr(e,t,r,n,s){t[r]=n;try{let o=new URL(r),u=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{u.push(new URL(e.targetOrigin).origin);}catch{}for(let a of u)try{let l=new URL(a);t[l.origin+o.pathname+o.search]=n;}catch{}try{let a=(()=>{let l=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[o.pathname+o.search]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[o.pathname]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[o.pathname]=a;}catch{}}catch{}}async function Dn(e){try{return await $.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Te(e){try{return await $.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function ri(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Fn(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function ni(e,t){let r=Fn(t),n=ri(t),s=e?.mtime.getTime();return r!==null&&Math.abs((s??0)-r)>=1e3||n!==null&&e?.size!==n?false:r!==null||n!==null}async function si(e,t){if(!t)return;let r=Fn(t);if(r===null)return;let n=await Te(e);if(!n)return;let s=new Date(r);await $.utimes(e,n.atime,s).catch(()=>{});}function rr(e,t){let r=y.extname(e.pathname).toLowerCase(),n=t["content-type"]||"";return Po.has(r)||So.some(s=>n.includes(s))&&!n.startsWith("image/")&&!n.includes("font")}function oi(e){try{let t=y.extname(new URL(e).pathname).toLowerCase();return t===".js"||t===".mjs"}catch{return false}}function xe(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset processing progress: processed ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}function ii(e){return ue(e.toString("base64"),40)}function Wn(e,t,r){let n=e.assetVariantDigestsByOriginalPath.get(t);if(!n||n.size===0)return false;for(let s of n)if(s!==r)return true;return false}function qn(e,t,r){let n=e.assetVariantDigestsByOriginalPath.get(t);n||(n=new Set,e.assetVariantDigestsByOriginalPath.set(t,n)),n.add(r);}async function nr(e,t,r,n,s,o,i){let u=et(e,r),a=ii(n),l=u.originalFilePath,d=u.originalPublicUrlWithSearch;if(u.preferQueryHashed&&u.hashedFilePath&&u.hashedPublicUrl&&Wn(s,u.originalFilePath,a)){let f=$n(e,t,r);f?(l=f.outputPath,d=f.publicUrl):(l=u.hashedFilePath,d=u.hashedPublicUrl);}tr(e,t,r,d,u.originalPublicUrl);let c=ce(l)?Buffer.from(Wt(n.toString("utf8"),e,t,l),"utf8"):n,m=await Dn(l),h=false;return (m===null||!m.equals(c))&&(await fe(l,c),h=true),o?.debug(`${h?"Saved":"Skipped unchanged"} asset output ${r}`,{phase:"download-assets",url:r,outputPath:l,bytes:c.length,wroteFile:h}),qn(s,u.originalFilePath,a),await si(l,i),s.stats.assetsSaved++,o&&(s.stats.assetsSaved%50===0||s.stats.assetsSaved===1)&&o.progress(`Asset progress: saved ${s.stats.assetsSaved}, discovered ${s.stats.assetsQueued}, pending ${s.assetQueue.length}.`,{assetsSaved:s.stats.assetsSaved,assetsQueued:s.stats.assetsQueued,assetQueue:s.assetQueue.length,phase:"download-assets"}),{outputPath:l,publicUrl:d,wroteFile:h}}function Me(e,t,r,n,s){let o=()=>{let c=String(s??"").trim();if(!c)return {discoveredFrom:"",discoveredFromType:"unknown"};if(c==="seed-path"||c==="cli"||c==="generated-404-request-path")return {discoveredFrom:c,discoveredFromType:c};if(/^https?:\/\//i.test(c))try{let m=new URL(c);return Ne(m)?{discoveredFrom:c,discoveredFromType:"sitemap"}:q(e,m)?{discoveredFrom:c,discoveredFromType:"asset"}:Re(m)&&!Oe(m)?{discoveredFrom:c,discoveredFromType:"page"}:{discoveredFrom:c,discoveredFromType:"url"}}catch{return {discoveredFrom:c,discoveredFromType:"url"}}return {discoveredFrom:c,discoveredFromType:"url"}},i=s&&/^https?:\/\//i.test(s)?s:e.sourceOrigin,u=V(e,r,i,n,"page");if(!u)return;if(!Re(u)||Ln(u)){if(q(e,u)){n.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:s}),W(e,t,u.toString(),n,s);return}n.reject("page",u.toString(),Oe(u)?"asset/internal path cannot be page":"not page-like",s);return}let a=u.toString(),{discoveredFrom:l,discoveredFromType:d}=o();l&&!t.pageDiscoverySources.has(a)&&t.pageDiscoverySources.set(a,l),!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,n.debug(`Queued discovered page ${a}`,{phase:"discovery",url:a,discoveredFrom:l,discoveredFromType:d,outputPath:ge(e,a)}),(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&n.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function W(e,t,r,n,s){let o=s&&/^https?:\/\//i.test(s)?s:e.sourceOrigin,i=V(e,r,o,n,"asset");if(!i)return;if(!q(e,i)){n.reject("asset",i.toString(),"not a safe asset path/prefix",s);return}let u=i.toString();!t.doneAssets.has(u)&&!t.queuedAssets.has(u)&&(t.queuedAssets.add(u),t.assetQueue.push(u),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&n.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Pt(e,t,r,n,s){let o=s&&/^https?:\/\//i.test(s)?s:e.sourceOrigin,i=V(e,r,o,n,"sitemap",yr(s));if(!i)return;if(!Ne(i)){n.reject("sitemap",i.toString(),"not sitemap-like",s);return}let u=i.toString();!t.doneSitemaps.has(u)&&!t.queuedSitemaps.has(u)&&(t.queuedSitemaps.add(u),t.sitemapQueue.push(u),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&n.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function ai(e){let t=we(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(n=>n[1]);return r.length>0?r.map(n=>{let s=n.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!s?.[1])return null;let o=n.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:s[1].trim(),lastmod:Ee(o?.[1]??null)}}).filter(n=>!!n):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(n=>({loc:n[1].trim(),lastmod:null})).filter(n=>n.loc)}function ui(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function ci(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let n of e.matchAll(r)){let s=e.slice(Math.max(0,n.index-8),n.index);if(/\bnew\s+$/i.test(s))continue;let o=(n[1]||n[2]||n[3]||"").trim();o&&t.push(o);}return t}function li(e){return e.replace(/<script\b[^>]*>[\s\S]*?<\/script\s*>/gi,"")}function di(e,t){if(t==="dom-style")return true;let r=(()=>{try{return y.extname(new URL(e).pathname).toLowerCase()}catch{return ""}})();return !(r===".js"||r===".mjs"||r===".map"||t.includes("javascript"))}function bt(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Nn(e,t,r,n){let s=[];await $.mkdir(t,{recursive:true});let o=await $.readdir(e,{withFileTypes:true});for(let i of o){let u=y.join(e,i.name),a=y.join(t,i.name);if(i.isDirectory()){s.push(...await Nn(u,a,r,n));continue}await _n(u,a,r,n)&&s.push(a);}return s}async function pi(e,t){try{let[r,n]=await Promise.all([$.stat(e),$.stat(t)]);if(r.size!==n.size)return !1;let[s,o]=await Promise.all([$.readFile(e),$.readFile(t)]);return s.equals(o)}catch{return false}}async function _n(e,t,r,n){let s=y.resolve(t);return r.has(s)?(n.debug("Skipped post-crawl copy over crawled page output",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):await pi(e,t)?(n.debug("Skipped unchanged post-crawl copy target",{phase:"copy-extra-paths",sourcePath:e,targetPath:t}),false):(await $.mkdir(y.dirname(t),{recursive:true}),await $.copyFile(e,t),true)}async function mi(e,t,r){let n=Object.entries(e.postCrawlCopyMap||{}).map(([u,a])=>({sourcePath:u.trim(),prefix:String(a||"").trim()}));if(n.length===0)return [];let s=0,o=[],i=new Set([...t.donePages].map(u=>y.resolve(ge(e,u))));r.mark("copy-extra-paths");for(let u of n){let a=u.sourcePath,l=u.prefix;if(!a||!l){r.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:a,prefixPath:l});continue}let d=To(a);if(!d.resolvedPath){r.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:a,alias:d.alias||"",requiredEnv:d.requiredEnv||"",prefixPath:l});continue}let c=d.resolvedPath,m=bt(l);if(!m){r.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:a,prefixPath:l});continue}let h;try{h=await $.stat(c);}catch{r.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:a,sourceAbs:c,prefixPath:l});continue}let f=y.resolve(e.outputDir,m);if(h.isDirectory()){let w=await Nn(c,f,i,r);o.push(...w),s+=w.length,w.length>0&&r.progress(`Copied static directory to export: ${c} -> /${m}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${m}`,copiedItems:s,changedFiles:w.length});continue}let b=y.basename(c),p=l.endsWith("/")?y.join(f,b):f;await _n(c,p,i,r)&&(o.push(p),s++,r.progress(`Copied static file to export: ${c} -> /${bt(l.endsWith("/")?`${l}${b}`:l)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${bt(l.endsWith("/")?`${l}${b}`:l)}`,copiedItems:s}));}return r.endMark("copy-extra-paths",{mappedSources:n.length,copiedItems:s}),o}function qe(e,t,r,n,s){let o=new Set,i=we(r).replace(/\\\//g,"/"),u=s==="page-html"?li(i):i,a=di(t,s);if(a){for(let l of ci(u)){let d=V(e,l,t,n,`${s}:css-url`);d&&q(e,d)&&o.add(d.toString());}for(let l of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let d=V(e,l[1],t,n,`${s}:css-import`);d&&q(e,d)&&o.add(d.toString());}}if(a||s==="page-html")for(let l of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let d=V(e,l[1],t,n,`${s}:xml-stylesheet`);d&&q(e,d)&&o.add(d.toString());}if(s==="page-html")for(let l of Lr(i)){let d=l.startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l}`:l;if(er(d)){n.ignore(`${s}:serialized-url`,d,"looks like JavaScript/code fragment",t);continue}let c=V(e,d,t,n,`${s}:serialized-url`);c&&q(e,c)?o.add(c.toString()):c&&n.ignore("asset",c.toString(),"serialized URL did not pass safe asset path filter",t);}if(s!=="page-html"&&oi(t))for(let l of Dr(i)){if(er(l)||!/\.(?:js|mjs)(?:[?#]|$)/i.test(l))continue;let d=V(e,l,t,n,`${s}:relative-asset`);d&&q(e,d)&&o.add(d.toString());}return [...o]}function Hn(e,t,r,n){let s=new Set,o=we(r);for(let i of o.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let u=i[1],a=V(e,u,t,n,"page-link");a&&(jn(e,a)||Re(a)&&!Oe(a)&&!q(e,a)&&s.add(a.toString()));}return [...s]}async function wn(e,t,r,n,s,o,i=false,u=[]){let a=new Set(u),l=0,d=new Map,c=(m,h)=>{let f=[h],b=new Set;for(;f.length>0;){let p=f.pop();if(p===m)return true;b.has(p)||(b.add(p),f.push(...d.get(p)??[]));}return false};for(;r.sitemapQueue.length>0;){let m=r.sitemapQueue.shift();r.queuedSitemaps.delete(m);let h=se(m,n,"sitemap","sitemap-queue");if(!h)continue;let f=h.toString();if(r.doneSitemaps.has(f))continue;r.doneSitemaps.add(f),n.sitemap(`Fetching sitemap ${f}`,{url:f});let b=true;try{let p=await t.get(f,{timeout:6e4});if(b=!1,!p.ok()){if(n.skip("sitemap",f,p.status()),i&&!a.has(f))throw new Error(`Required child content-sync sitemap returned HTTP ${p.status()}: ${f}`);continue}let g=await p.body(),w=p.url()||f,k=se(w,n,"sitemap-response",f);if(!k)continue;let C=k.toString();r.successfulSitemaps.add(f),r.successfulSitemaps.add(C),a.has(f)&&(r.resolvedSitemapRoots.add(C),l+=1);let I=await nr(e,r.assetMap,C,g,r,n);G(s,e,I.outputPath),ne(s,I.outputPath);let A=g.toString("utf8");/<sitemapindex(?:\s|>)/i.test(A)?r.sitemapIndexUrls.add(C):r.sitemapLeafUrls.add(C);let R=qe(e,C,A,n,"sitemap");for(let E of R)W(e,r,E,n,C);let v=[];for(let E of ai(A)){let x=V(e,E.loc,C,n,"sitemap-loc",!0);if(!x){if(i)throw new Error(`Content-sync sitemap contains an inadmissible location: ${E.loc}`);continue}if(Ne(x)){if(c(C,x.toString()))throw new Error(`Sitemap index cycle detected: ${C} -> ${x.toString()}`);if(d.has(C)||d.set(C,new Set),d.get(C).add(x.toString()),v.push(x.toString()),i&&r.doneSitemaps.has(x.toString())&&!r.successfulSitemaps.has(x.toString()))throw new Error(`Required child content-sync sitemap was not resolved: ${x.toString()}`);Pt(e,r,x.toString(),n,C);}else if(i){if(Re(x)&&!q(e,x)){let z=o.requiredUrls.has(x.toString()),L=o.manifest.pages[x.toString()],P=!!L&&!!await Te(L.outputPath);if(!z&&!P)throw new Error(`Content-sync sitemap coverage is incomplete for ${x.toString()}.`)}n.ignore("sitemap-loc",x.toString(),"content-sync sitemap-only traversal does not enqueue urlset locations",C);}else q(e,x)?W(e,r,x.toString(),n,C):Re(x)?(E.lastmod&&(r.sitemapLastmodByPage[x.toString()]=E.lastmod),Me(e,r,x.toString(),n,C)):n.reject("sitemap-loc",x.toString(),"not page/sitemap/asset-like",C);}o.trackManifest&&(o.manifest.assets[C]={url:C,outputPath:I.outputPath,isText:!0,contentType:p.headers()["content-type"]??"application/xml",cacheControl:Zt(p.headers()),changeToken:null,tokenSource:null,discoveredAssets:[...new Set([...R,...v])],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0,rewriteComplete:!1},await Pe(e,o));}catch(p){n.error(`Failed sitemap ${f}`,{url:f,error:String(p)});let g=i&&a.has(f)&&b;if(i&&!g)throw p}}if(i&&a.size>0&&l===0)throw new Error(`None of the configured content-sync sitemap roots could be resolved: ${[...a].join(", ")}`)}async function gi(e,t,r,n,s,o,i){let u=V(e,n,e.sourceOrigin,s,"asset-fetch");if(!u||!q(e,u))return;let a=u.toString();if(r.doneAssets.has(a))return;let l=o.enabled||o.strictTargeted?o.previousManifest.assets[a]:void 0,d=o.trackManifest?await yt(e,t,o,a,s):null,c=d?.token??null,m=Nt(l?.changeToken,d),h=o.strictTargeted&&(!d?.supported||!d.token)&&_t(e.sourceOrigin,a),f=o.strictTargeted?o.manifest.assets[a]:void 0;if(f?.lastSeenRunId===o.runId&&await Te(f.outputPath)){r.doneAssets.add(a);for(let p of f.discoveredAssets||[])W(e,r,p,s,a);G(i,e,f.outputPath),Fe(i,e,f.outputPath,f.cacheControl),f.isText&&ne(i,f.outputPath),s.info(`Resumed completed content-sync asset ${a}`,{url:a,source:"content-sync-checkpoint"}),xe(r,s);return}if(l&&(m||h)&&await Te(l.outputPath)){o.manifest.assets[a]={...l,changeToken:c,tokenSource:d?.tokenSource??l.tokenSource,discoveredAssets:[...l.discoveredAssets||[]],lastSeenRunId:o.runId,unreferencedRuns:0},delete o.manifest.pages[a],r.doneAssets.add(a);for(let p of l.discoveredAssets||[])W(e,r,p,s,a);Fe(i,e,l.outputPath,l.cacheControl),s.info(`Reused unchanged trusted asset ${a}.`,{url:a,tokenSource:d?.tokenSource??"unknown",phase:"download-assets",source:m?"asset-change-token-match":"content-sync-release-fingerprint"}),xe(r,s);return}let b=(p,g={})=>{if(!l||o.strictTargeted)return false;o.manifest.assets[a]={...l,discoveredAssets:[...l.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0},delete o.manifest.pages[a],r.doneAssets.add(a);for(let w of l.discoveredAssets||[])W(e,r,w,s,a);return s.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...g,phase:"download-assets"}),xe(r,s),true};s.asset(`Fetching asset ${a}`,{url:a});try{let p=We(e,r.assetMap,a),g=p?await Te(p):null;if(p&&g&&!o.strictTargeted)try{let L=await B(`fetch asset headers ${a}`,6e4,()=>t.head(a,{timeout:6e4})),P=L.headers();if(L.ok()&&ni(g,P)){let j=await Dn(p);if(j!==null){let oe=L.url()||a,ee=se(oe,s,"asset-response",a);if(!ee)return;let ie=ee.toString(),T=await nr(e,r.assetMap,ie,j,r,s,P);T.wroteFile&&(G(i,e,T.outputPath),ce(T.outputPath)&&ne(i,T.outputPath));let F=Zt(P)||l?.cacheControl||null;Fe(i,e,T.outputPath,F),r.doneAssets.add(a),s.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:P["last-modified"]||"",contentLength:P["content-length"]||"",phase:"download-assets"}),xe(r,s);let D=rr(u,P),Z=l?.discoveredAssets?[...l.discoveredAssets]:[];if(D||l?.isText){let K=j.toString("utf8"),_e=qe(e,ie,K,s,"asset:cached");(_e.length>0||!l)&&(Z=_e);for(let tt of Z)W(e,r,tt,s,ie);}if(o.trackManifest){let K=D||!!l?.isText;o.manifest.assets[a]={url:a,outputPath:T.outputPath,isText:K,contentType:Gt(P)||l?.contentType||null,cacheControl:F,changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:Z,lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0,rewriteComplete:!K},delete o.manifest.pages[a],(o.enabled||o.strictTargeted)&&K&&await Pe(e,o);}return}}}catch{}let w=await B(`fetch asset ${a}`,6e4,()=>t.get(a,{timeout:6e4}));if(!w.ok()){if(b(`http-${w.status()}`,{status:w.status()}))return;if(o.strictTargeted)throw new Error(`Required content-sync asset returned HTTP ${w.status()}: ${a}`);s.skip("asset",a,w.status());return}let k=await B(`read asset response ${a}`,6e4,()=>w.body()),C=w.url()||a,I=se(C,s,"asset-response",a);if(!I)return;let A=I.toString(),R=w.headers(),v=await nr(e,r.assetMap,A,k,r,s,R);v.wroteFile&&(G(i,e,v.outputPath),ce(v.outputPath)&&ne(i,v.outputPath));let E=Zt(R);Fe(i,e,v.outputPath,E),r.doneAssets.add(a),xe(r,s);let x=rr(u,R),z=[];if(x){let L=k.toString("utf8");z=qe(e,A,L,s,`asset:${y.extname(u.pathname).toLowerCase()||R["content-type"]||"unknown"}`);for(let P of z)W(e,r,P,s,A);}o.trackManifest&&(o.manifest.assets[a]={url:a,outputPath:v.outputPath,isText:x,contentType:Gt(R),cacheControl:E,changeToken:d?.supported?d.token:null,tokenSource:d?.supported?d.tokenSource??null:null,discoveredAssets:z,lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,unreferencedRuns:0,rewriteComplete:!x},delete o.manifest.pages[a],(o.enabled||o.strictTargeted)&&x&&await Pe(e,o));}catch(p){if(b("fetch-error",{error:String(p)}))return;if(s.error(`Failed asset ${a}`,{url:a,error:String(p)}),o.strictTargeted)throw p}finally{r.doneAssets.has(a)||(r.doneAssets.add(a),xe(r,s));}}async function hi(e,t,r,n,s,o){let i=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),u=Array.from({length:i},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await gi(e,t,r,a,n,s,o));}});await Promise.all(u);}function Rt(e,t){let r=Y(e.outputDir,t);if(!r)throw new Error(`Remote export output is outside outputDir: ${t}`);return r}function fi(e,t){return rr(e,{"content-type":t})}function yn(e){return /(?:^|\/)index\.html?$|\.html?$/i.test(e)}async function wi(e,t,r,n,s,o,i,u,a){let l=Math.max(1,Math.min(100,a)),d=20;for(;r.assetQueue.length>0;){let c=r.assetQueue.splice(0,l*d);for(let b of c)r.queuedAssets.delete(b);await ir(e,t,s,c,n);let m=[];for(let b of c){let p=s.manifest.assets[b],g=p?Y(e.outputDir,p.outputPath):null,w=g?i.objects[g]:void 0,k=s.changeTokenCache.get(b)??null;if(!(s.enabled&&p&&w?.kind==="asset"&&w.sourceUrl===b&&(Nt(p.changeToken,k)||(!k?.supported||!k.token)&&_t(e.sourceOrigin,b)))||!p||!g||!w){m.push(b);continue}s.manifest.assets[b]={...p,changeToken:k?.token??null,tokenSource:k?.tokenSource??p.tokenSource,lastSeenRunId:s.runId,unreferencedRuns:0};for(let I of p.discoveredAssets||[])W(e,r,I,n,b);r.doneAssets.add(b),Fe(o,e,p.outputPath,p.cacheControl),n.info(`Reused unchanged S3-native asset ${b}.`,{url:b,relativePath:g,tokenSource:k?.tokenSource??"release-fingerprint",phase:"download-assets",execution:"lambda"}),xe(r,n);}if(m.length===0)continue;let h=[];for(let b=0;b<m.length;b+=d)h.push(m.slice(b,b+d));let f=await Promise.all(h.map(b=>u.fetch({urls:b,ignoreHttpsErrors:e.ignoreHttpsErrors,timeoutMs:6e4})));for(let b of f)for(let p of b.assets){let g=p.requestedUrl;try{if(p.outcome!=="fetched"||!p.object||!p.finalUrl){let E=p.error||`Remote asset fetch ended with ${p.outcome}.`;if(n.error(`Failed remote asset ${g}`,{url:g,status:p.httpStatus,outcome:p.outcome,error:E,execution:"lambda"}),s.strictTargeted)throw new Error(E);continue}let w=se(p.finalUrl,n,"remote-asset-response",g);if(!w)continue;let k=et(e,w.toString()),C=k.originalFilePath,I=k.originalPublicUrlWithSearch;if(k.preferQueryHashed&&k.hashedFilePath&&k.hashedPublicUrl&&Wn(r,k.originalFilePath,p.object.sha256)){let E=$n(e,r.assetMap,w.toString());C=E?.outputPath??k.hashedFilePath,I=E?.publicUrl??k.hashedPublicUrl;}tr(e,r.assetMap,w.toString(),I,k.originalPublicUrl),w.toString()!==g&&tr(e,r.assetMap,g,I,k.originalPublicUrl),qn(r,k.originalFilePath,p.object.sha256);let A=Rt(e,C),R=p.contentType?.trim()||"application/octet-stream",v=fi(w,R);De(i,{relativePath:A,sourceUrl:g,kind:"asset",contentType:R,cacheControl:p.cacheControl,isText:v,object:p.object}),G(o,e,C),v&&ne(o,C),Fe(o,e,C,p.cacheControl);for(let E of p.discoveredAssets)W(e,r,E,n,w.toString());r.stats.assetsSaved++,s.trackManifest&&(s.manifest.assets[g]={url:g,outputPath:C,isText:v,contentType:R,cacheControl:p.cacheControl,changeToken:null,tokenSource:null,discoveredAssets:[...p.discoveredAssets],lastCrawledAt:new Date().toISOString(),lastSeenRunId:s.runId,unreferencedRuns:0,rewriteComplete:!v},delete s.manifest.pages[g]),n.debug(`Remote asset fetch completed ${g}`,{url:g,finalUrl:w.toString(),relativePath:A,bytes:p.object.bytes,taskId:b.taskId,attempts:b.attempts,execution:"lambda"});}finally{r.doneAssets.add(g),xe(r,n);}}}}async function yi(e,t,r,n){for(let s of [...t.changedFiles].sort()){if(!un(r,s))continue;let o=y.join(e.outputDir,s),i=await $.readFile(o).catch(d=>{throw new Error(`Coordinator-owned output is missing before S3 staging: ${s}`,{cause:d})}),u=ce(o),a=String(Gr.lookup(s)||"")||(u?"text/plain; charset=utf-8":"application/octet-stream"),l=await n.writeWorkspaceObject({key:`${r.workspace.prefix}jobs/${r.jobId}/coordinator/${ue(s,40)}-${y.basename(s)}`,body:i,contentType:a,cacheControl:yn(s)?e.s3.htmlCacheControl:e.s3.assetCacheControl});De(r,{relativePath:s,sourceUrl:null,kind:"generated",contentType:a,cacheControl:yn(s)?e.s3.htmlCacheControl:e.s3.assetCacheControl,isText:u,object:l});}}function bi(e){let t=Math.ceil(e/1e3);return `${t} second${t===1?"":"s"}`}function Se(e){return Math.max(6e4,e.navigationTimeoutMs*2,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+15e3)}function Je(e){return Math.max(15e3,Math.min(6e4,Se(e)))}function Bn(e){return Math.max(15e3,e.readiness.timeoutMs+e.readiness.fallbackWaitMs+1e4)}function ur(e){return e.stats.pagesCompleted}function sr(e){let r=e.options?.browserProcess;return {pid:typeof r?.process?.pid=="number"?r.process.pid:null,kill:typeof r?.kill=="function"?r.kill.bind(r):null}}async function Jt(e,t,r){let n=sr(e);try{if(n.kill){await n.kill(),t.warn(`Force-killed crawl worker browser after ${r}`,{browserPid:n.pid,reason:r});return}if(n.pid!==null){process.kill(n.pid,"SIGKILL"),t.warn(`Sent SIGKILL to crawl worker browser after ${r}`,{browserPid:n.pid,reason:r});return}t.warn(`Could not resolve crawl worker browser PID after ${r}`,{reason:r});}catch(s){t.error(`Failed to force-kill crawl worker browser after ${r}`,{browserPid:n.pid,reason:r,error:String(s)});}}async function B(e,t,r){let n;try{return await Promise.race([r(),new Promise((s,o)=>{n=globalThis.setTimeout(()=>{o(new Ze(e,t));},t);})])}finally{n!==void 0&&globalThis.clearTimeout(n);}}var Ze=class extends Error{constructor(r,n){super(`${r} timed out after ${bi(n)}.`);this.operation=r;this.timeoutMs=n;this.name="OperationTimeoutError";}operation;timeoutMs};async function Si(e,t){let r=Bn(e);await t.evaluate(async n=>{await new Promise(s=>{let o=0,i=700,u=Date.now(),a=()=>{window.clearInterval(l),window.scrollTo(0,0),s();},l=window.setInterval(()=>{if(Date.now()-u>=n.maxDurationMs){a();return}window.scrollBy(0,i),o+=i;let d=document.body?.scrollHeight||0,c=document.documentElement?.scrollHeight||0,m=Math.max(d,c);o>=m+window.innerHeight&&a();},120);});},{maxDurationMs:r});}async function Pi(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Si(e,t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ri(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Vn(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ar(t,r))}async function vi(e,t,r,n,s=true){let o=new Set;if(s){let u=await t.evaluate(()=>{let a=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href","data-resource-url"],d=new Set,c=["srcset","data-srcset","data-lazy-srcset"],m=h=>{h.querySelectorAll("*").forEach(f=>{for(let p of l){let g=f.getAttribute(p);g&&a.add(g);}for(let p of c){let g=f.getAttribute(p);g&&g.split(",").forEach(w=>{let k=w.trim().split(/\s+/)[0];k&&a.add(k);});}let b=f.getAttribute("style");b&&d.add(b),f.shadowRoot&&m(f.shadowRoot);});};return m(document),{attrs:[...a],styles:[...d]}});for(let a of u.attrs)for(let l of ui(a)){let d=V(e,l,r,n,"dom-attr");d&&q(e,d)&&o.add(d.toString());}for(let a of u.styles)for(let l of qe(e,r,a,n,"dom-style"))o.add(l);}let i=await t.content();for(let u of qe(e,r,i,n,"page-html"))o.add(u);return [...o]}async function bn(e){return await e.evaluate(()=>{document.querySelectorAll("#wpsuite-consent-root").forEach(t=>t.remove()),document.documentElement.classList.remove("wpsuite-consent-scroll-locked"),document.body.classList.remove("wpsuite-consent-scroll-locked");}),e.content()}async function or(e,t,r,n,s){let o=ge(e,t);await fe(o,r,"utf8");let i=await $.stat(o);s?.debug(`Saved page output ${t}`,{phase:"save-pages",url:t,outputPath:o,bytes:i.size});let u=vt(n,o),a=Math.min(n.stats.pagesQueued,ur(n)+1);s&&u&&(n.stats.pagesSaved%10===0||n.stats.pagesSaved===1)&&s.progress(`Page progress: processed ${a}, rendered ${n.stats.pagesRendered}, saved ${n.stats.pagesSaved}, discovered ${n.stats.pagesQueued}.`,{donePages:a,claimedPages:n.donePages.size,pagesRendered:n.stats.pagesRendered,pagesSaved:n.stats.pagesSaved,pagesQueued:n.stats.pagesQueued,pageQueue:n.pageQueue.length,phase:"save-pages"});}function vt(e,t){let r=y.resolve(t);return e.savedPageOutputs.has(r)?false:(e.savedPageOutputs.add(r),e.stats.pagesSaved=e.savedPageOutputs.size,true)}async function kt(e,t,r,n,s){let o=Y(e.outputDir,n);if(!o)return;if(s){let u=s.objects[o];u?.kind==="page"&&u.sourceUrl===r&&u.object?.bucket===s.workspace.bucket&&typeof u.object.key=="string"&&u.object.key.startsWith(s.workspace.prefix)&&vt(t,n);return}(await Te(n).catch(()=>null))?.isFile()&&vt(t,n);}function zn(e){let t=ur(e);return t%5===0||t===1||t===e.stats.pagesQueued}function Qn(e,t){let r=ur(e);t.progress(`Render progress: processed ${r}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:r,claimedPages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function ki(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let n=new URL(String(r),window.location.href),s=new URL(window.location.href);return n.hash="",s.hash="",n.href===s.href}catch{return false}};try{let r=window.location.assign.bind(window.location),n=window.location.replace.bind(window.location),s=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:o=>{if(t(o)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",o??"");return}return r(o)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:o=>{if(t(o)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",o??"");return}return n(o)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Ci(e,t,r){let n=await e.newContext({viewport:t.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:t.ignoreHttpsErrors,...r?{javaScriptEnabled:false}:{}});return r||await ki(n),n}async function xi(e,t,r,n,s,o,i,u,a,l){for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let d=t.pageQueue.shift();if(!d)break;t.queuedPages.delete(d);try{let c=se(d,r,"page","page-queue");if(!c)continue;let m=c.toString();if(d=m,t.queuedPages.delete(d),t.donePages.has(d))continue;t.donePages.add(d),o.enabled&&o.seenPages.add(d);let h=Ct(e,d),f=Vn(e,c.pathname);if(!Re(c)||q(e,c)||Oe(c)){if(q(e,c)){r.info(`Remote worker redirected non-page URL to asset queue: ${d}`,{url:d,source:"remote-worker-guard"}),W(e,t,d,r,"remote-worker-guard");continue}r.reject("page",d,"guard rejected non-page URL before remote rendering","remote-worker");continue}let b=o.strictTargeted?o.manifest.pages[d]:void 0;if(b?.lastSeenRunId===o.runId&&await Te(b.outputPath)){o.seenPages.add(d);for(let v of b.discoveredAssets||[])W(e,t,v,r,d);G(i,e,b.outputPath),ne(i,b.outputPath),await kt(e,t,d,b.outputPath,l),r.info(`Resumed completed content-sync page ${d}`,{url:d,source:"content-sync-checkpoint"});continue}let p=await On(e,s,t,o,d,r);if(o.enabled&&n!=="single-url"&&n!=="content-sync"){let v=o.manifest.pages[d];if(p.action==="reuse"&&v){for(let E of v.discoveredAssets)W(e,t,E,r,d);for(let E of v.discoveredPages)Me(e,t,E,r,d);v.lastSeenRunId=o.runId,v.sitemapLastmod=Ee(t.sitemapLastmodByPage[d])??v.sitemapLastmod,p.changeToken?.supported&&(v.changeToken=p.changeToken.token,v.tokenSource=p.changeToken.tokenSource??v.tokenSource),await kt(e,t,d,v.outputPath,l),r.info(`Incremental reuse skipped unchanged page ${d}`,{url:d,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}r.page(`Rendering remotely ${d}`,{url:d,execution:"lambda"});let g=await a.render({sourceOrigin:e.sourceOrigin,url:d,options:{ignoreHttpsErrors:e.ignoreHttpsErrors,javaScriptEnabled:!f,viewport:e.viewport,navigationTimeoutMs:e.navigationTimeoutMs,autoScrollTimeoutMs:Bn(e),readiness:e.readiness},downloadHtml:!l}),w=g.page,k=w.finalUrl?se(w.finalUrl,r,"remote-final-response",d)?.toString():void 0;if(n==="content-sync"&&k&&k!==m){if(o.deleteOnMissingUrls.has(d)){await Ye(e,o,i,d,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${k}`,{url:d,finalResponseUrl:k,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${d} -> ${k}`)}let C=w.httpStatus;if(h){if(C!==404){r.error(`Generated 404 request path returned unexpected status for ${d}`,{url:d,expectedStatus:404,status:C,requestPath:e.generated404RequestPath,execution:"lambda"});continue}}else if(C!==null&&C>=400){if(n==="content-sync"&&(C===404||C===410)&&o.deleteOnMissingUrls.has(d)){await Ye(e,o,i,d,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${d}`,{url:d,status:C,source:"content-sync-tombstone"});continue}if(n==="content-sync")throw new Error(`Required content-sync route returned HTTP ${C}: ${d}`);r.skip("page",d,C,t.pageDiscoverySources.get(d));continue}t.stats.pagesRendered++;let I=new Set;if(!l){if(g.html===null)throw new Error("Remote render did not return HTML for local output mode.");for(let v of qe(e,m,g.html,r,"remote-page-html"))I.add(v);}for(let v of [...w.discoveredAssets,...w.networkUrls]){let E=V(e,v,m,r,"remote-render");E&&q(e,E)&&I.add(E.toString());}for(let v of I)W(e,t,v,r,d);let A=n!=="single-url"&&n!=="content-sync"&&!h?[...new Set([...!l&&g.html!==null?Hn(e,d,g.html,r):[],...w.discoveredPages])]:[];for(let v of A)Me(e,t,v,r,d);for(let v of w.cookies){let E=v.sameSite==="Strict"||v.sameSite==="Lax"||v.sameSite==="None"?v.sameSite:"Unspecified",x=!Number.isFinite(v.expires)||v.expires<=0;u.add({name:v.name,domain:v.domain,path:v.path||"/",expires:x?null:v.expires,session:x,secure:v.secure,httpOnly:v.httpOnly,sameSite:E,partitioned:v.partitioned===!0});}let R=ge(e,d);if(l){if(!w.html)throw new Error("Remote render completed without a workspace HTML object.");De(l,{relativePath:Rt(e,R),sourceUrl:d,kind:"page",contentType:w.contentType?.trim()||"text/html; charset=utf-8",cacheControl:e.s3.htmlCacheControl,isText:!0,object:w.html}),vt(t,R);}else {if(g.html===null)throw new Error("Remote render did not return HTML for local output mode.");await or(e,d,g.html,t,r);}G(i,e,R),ne(i,R),o.trackManifest&&(o.manifest.pages[d]={url:d,outputPath:R,contentType:w.contentType?.trim()||null,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:Ee(t.sitemapLastmodByPage[d])??null,discoveredPages:A,discoveredAssets:[...I],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,rewriteComplete:!1},(o.enabled||o.strictTargeted)&&await Pe(e,o)),r.debug(`Remote render completed ${d}`,{url:d,taskId:g.taskId,attempts:g.attempts,execution:"lambda"});}catch(c){if(r.error(`Failed remote page ${d}`,{url:d,error:String(c),execution:"lambda",...c instanceof te?{status:c.status,attempts:c.attempts}:{}}),c instanceof te){t.pageHttpFailures.set(d,c);continue}if(o.strictTargeted)throw c}finally{t.stats.pagesCompleted++,zn(t)&&Qn(t,r);}}}async function Ti(e,t,r,n,s,o,i,u){let a=await chromium.launch({headless:true}),l=sr(a);l.pid!==null&&r.debug(`Launched crawl worker browser PID ${l.pid}`,{browserPid:l.pid});try{for(;t.pageQueue.length>0;){a||(a=await chromium.launch({headless:!0}),l=sr(a),l.pid!==null&&r.debug(`Relaunched crawl worker browser PID ${l.pid}`,{browserPid:l.pid}));let d=a;if(e.maxPages>0&&t.donePages.size>=e.maxPages)break;let c=t.pageQueue.shift();if(!c)break;t.queuedPages.delete(c);let m=null,h=null,f=!1,b=null,p=[],g=new Set;try{let w=se(c,r,"page","page-queue");if(!w)continue;let k=w.toString();if(c=k,t.queuedPages.delete(c),t.donePages.has(c))continue;t.donePages.add(c),o.enabled&&o.seenPages.add(c);let C=Ct(e,c),I=Vn(e,w.pathname);if(!Re(w)||q(e,w)||Oe(w)){if(q(e,w)){r.info(`Worker redirected non-page URL to asset queue: ${c}`,{url:c,source:"worker-guard"}),W(e,t,c,r,"worker-guard");continue}r.reject("page",c,"guard rejected non-page URL before rendering","worker");continue}let A=o.strictTargeted?o.manifest.pages[c]:void 0;if(A?.lastSeenRunId===o.runId&&await Te(A.outputPath)){o.seenPages.add(c);for(let T of A.discoveredAssets||[])W(e,t,T,r,c);G(i,e,A.outputPath),ne(i,A.outputPath),await kt(e,t,c,A.outputPath),r.info(`Resumed completed content-sync page ${c}`,{url:c,source:"content-sync-checkpoint"});continue}let R=await On(e,s,t,o,c,r);if(o.enabled&&n!=="single-url"&&n!=="content-sync"){let T=o.manifest.pages[c];if(R.action==="reuse"&&T){for(let F of T.discoveredAssets)W(e,t,F,r,c);for(let F of T.discoveredPages)Me(e,t,F,r,c);T.lastSeenRunId=o.runId,T.sitemapLastmod=Ee(t.sitemapLastmodByPage[c])??T.sitemapLastmod,R.changeToken?.supported&&(T.changeToken=R.changeToken.token,T.tokenSource=R.changeToken.tokenSource??T.tokenSource),await kt(e,t,c,T.outputPath),r.info(`Incremental reuse skipped unchanged page ${c}`,{url:c,mode:"incremental",reason:R.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}m=await B(`create page context ${c}`,Se(e),()=>Ci(d,e,I)),h=await B(`create page ${c}`,Se(e),()=>m.newPage()),p=await m.cookies();let v=!1,E=new Set;await h.route("**/*",async T=>{let F=T.request();try{let D=V(e,F.url(),e.sourceOrigin,r,"network-request");D&&q(e,D)&&(E.add(D.toString()),W(e,t,D.toString(),r,c));}catch{}if(F.isNavigationRequest()&&F.frame()===h.mainFrame())try{let D=new URL(F.url()),Z=new URL(k);if(D.hash="",Z.hash="",D.href===Z.href){if(v){r.warn(`Blocked same-page navigation/reload for ${c}`,{url:c,requestUrl:F.url()});try{await T.abort("aborted");}catch(K){r.debug(`Ignored canceled route while blocking reload for ${c}`,{url:c,error:String(K)});}return}v=!0;}}catch{}try{await T.continue();}catch(D){r.debug(`Ignored canceled network route for ${c}`,{url:c,requestUrl:F.url(),error:String(D)});}}),h.on("response",T=>{try{let D=V(e,T.url(),e.sourceOrigin,r,"network-response");D&&q(e,D)&&(E.add(D.toString()),W(e,t,D.toString(),r,c));}catch{}let F=T.headerValues("set-cookie").then(D=>{for(let Z of D)u.add(_r(Z,T.url()));}).catch(()=>{}).then(()=>{});g.add(F),F.finally(()=>g.delete(F));}),r.page(`Rendering ${c}`,{url:c});let x=null,z=null;try{if(x=await h.goto(c,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),x&&x.ok())try{z=await B(`read response body ${c}`,Se(e),()=>x.text());}catch{z=null;}}catch(T){let F=t.pageHttpFailures.get(c);if(F){let D=(t.pageHttpAttempts.get(c)??0)+1;throw t.pageHttpAttempts.set(c,D),new te(c,F.status,D,`Retry navigation failed: ${String(T)}`)}r.warn(`Navigation issue for ${c}; saving current DOM if available`,{url:c,error:String(T)});}let L=x?.status();if(dt(L)){let T=(t.pageHttpAttempts.get(c)??0)+1;throw t.pageHttpAttempts.set(c,T),new te(c,L,T)}if(x)t.pageHttpFailures.delete(c);else if(t.pageHttpFailures.has(c)){let T=t.pageHttpFailures.get(c),F=(t.pageHttpAttempts.get(c)??0)+1;throw t.pageHttpAttempts.set(c,F),new te(c,T.status,F,"Retry navigation returned no HTTP response.")}if(n==="content-sync"&&x){let T=se(x.url(),r,"content-sync-final-response",c)?.toString();if(T&&T!==k){if(o.deleteOnMissingUrls.has(c)){await Ye(e,o,i,c,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive route redirected to ${T}`,{url:c,finalResponseUrl:T,source:"content-sync-tombstone"});continue}throw new Error(`Required content-sync route redirected outside its planned URL: ${c} -> ${T}`)}}if(C){let T=x?.status()??null;if(T!==404){r.error(`Generated 404 request path returned unexpected status for ${c}`,{url:c,expectedStatus:404,status:T,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${c}`,{url:c,status:T,outputPath:ge(e,c)});}else if(x&&!x.ok()){if(n==="content-sync"&&(x.status()===404||x.status()===410)&&o.deleteOnMissingUrls.has(c)){await Ye(e,o,i,c,o.deleteOnMissingUrls),r.info(`Removed stale content-sync archive page ${c}`,{url:c,status:x.status(),source:"content-sync-tombstone"}),await h.close();continue}if(n==="content-sync")throw new Error(`Required content-sync route returned HTTP ${x.status()}: ${c}`);r.skip("page",c,x.status(),t.pageDiscoverySources.get(c)),await h.close();continue}I?(r.info(`Rendering without JS execution for ${c}`,{url:c,mode:"no-js"}),await B(`wait for rendered page ${c}`,Se(e),()=>Ri(e,h))):await B(`wait for rendered page ${c}`,Se(e),()=>Pi(e,h)),t.stats.pagesRendered++;let P=await B(`extract rendered assets ${c}`,Se(e),()=>vi(e,h,k,r,!I));for(let T of P)W(e,t,T,r,c);let j=[...new Set([...P,...E])],oe=z??await B(`capture rendered html ${c}`,Se(e),()=>bn(h)),ee=n!=="single-url"&&n!=="content-sync"&&!C?Hn(e,c,oe,r):[];if(n!=="single-url"&&n!=="content-sync"&&!C)for(let T of ee)Me(e,t,T,r,c);await or(e,c,oe,t,r);let ie=ge(e,c);if(G(i,e,ie),ne(i,ie),o.trackManifest){let T=x?.headers();o.manifest.pages[c]={url:c,outputPath:ie,contentType:Gt(T),changeToken:R.changeToken?.supported?R.changeToken.token:null,tokenSource:R.changeToken?.supported?R.changeToken.tokenSource??null:null,sitemapLastmod:Ee(t.sitemapLastmodByPage[c])??null,discoveredPages:ee,discoveredAssets:j,lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,rewriteComplete:!1},(o.enabled||o.strictTargeted)&&await Pe(e,o);}}catch(w){if(w instanceof te){t.pageHttpFailures.set(c,w),w.attempts<Vt?(b=pt(w.attempts),r.warn(`Retrying page after HTTP ${w.status}: ${c}`,{url:c,status:w.status,attempt:w.attempts,maxAttempts:Vt,delayMs:b,reason:w.message,execution:"local"})):(t.pageHttpFailures.set(c,w),r.error(`Failed page after HTTP retries ${c}`,{url:c,status:w.status,attempts:w.attempts,error:String(w),execution:"local"}));continue}if(f=w instanceof Ze,o.strictTargeted)throw r.error(`Failed required content-sync page ${c}`,{url:c,error:String(w)}),w;if(f)r.warn(`Skipped partial DOM for timed-out page ${c}`,{url:c,error:String(w),recovery:"recycle-worker-browser"});else try{let k=h?await B(`capture partial html ${c}`,Se(e),()=>bn(h)):"";if(k&&k.trim()){await or(e,c,k,t,r);let C=ge(e,c);if(G(i,e,C),ne(i,C),o.trackManifest){let I=o.manifest.pages[c]??o.previousManifest.pages[c];o.manifest.pages[c]={url:c,outputPath:C,contentType:I?.contentType??null,changeToken:null,tokenSource:null,sitemapLastmod:Ee(t.sitemapLastmodByPage[c])??I?.sitemapLastmod??null,discoveredPages:I?.discoveredPages?[...I.discoveredPages]:[],discoveredAssets:I?.discoveredAssets?[...I.discoveredAssets]:[],lastCrawledAt:new Date().toISOString(),lastSeenRunId:o.runId,rewriteComplete:!1},(o.enabled||o.strictTargeted)&&await Pe(e,o);}r.warn(`Saved partial DOM for ${c}`,{url:c,error:String(w)});}}catch(k){k instanceof Ze&&(f=!0),r.error(`Could not save partial DOM for ${c}`,{url:c,error:String(k)});}r.error(`Failed page ${c}`,{url:c,error:String(w)});}finally{if(!f&&h&&await B(`drain network routes ${c}`,Je(e),()=>h.unrouteAll({behavior:"wait"})).catch(w=>{f=!0,r.warn(`Could not drain network routes for ${c}`,{url:c,browserPid:l.pid,error:String(w),recovery:"recycle-worker-browser"});}),f&&(a=null,await Jt(d,r,`page operation timeout for ${c}`),await d.close().catch(()=>{})),g.size>0&&await Promise.allSettled([...g]),!f&&m)try{let w=await m.cookies();u.addAll(Hr(p,w));}catch{}b===null&&(t.stats.pagesCompleted++,zn(t)&&Qn(t,r)),!f&&h&&await B(`close page ${c}`,Je(e),()=>h.close()).catch(w=>{f=!0,r.warn(`Timed out while closing page ${c}`,{url:c,browserPid:l.pid,error:String(w),recovery:"recycle-worker-browser"});}),!f&&m&&await B(`close page context ${c}`,Je(e),()=>m.close()).catch(w=>{f=!0,r.warn(`Timed out while closing page context for ${c}`,{url:c,browserPid:l.pid,error:String(w),recovery:"recycle-worker-browser"});}),f&&a===d&&(a=null,await Jt(d,r,`page cleanup timeout for ${c}`),await d.close().catch(()=>{})),b!==null&&(await setTimeout$1(b),t.donePages.delete(c),t.queuedPages.add(c),t.pageQueue.push(c));}}}finally{if(a){let d=a;await B("close worker browser",Je(e),()=>d.close()).catch(c=>(r.warn("Timed out while closing crawl worker browser",{browserPid:l.pid,error:String(c)}),Jt(d,r,"browser.close timeout")));}}}async function Ai(e){try{let t=await $.readFile(e),n=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!n)return [];try{let s=JSON.parse(n);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return n.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let o=JSON.parse(s);return o&&typeof o=="object"?o:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function Sn(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Ii(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Ei(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let n=Date.parse(String(r||""));if(Number.isFinite(n))return n}return 0}async function Kn(e){try{let t=await $.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Pn(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function Oi(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(n=>String(n?.role||"").trim()==="errors"&&String(n?.storedFileName||"").trim()!=="").sort((n,s)=>Pn(n)-Pn(s)).map(n=>y.join(e,String(n.storedFileName))):[];return [...new Set([...r,y.join(e,"errors.jsonl"),y.join(e,"errors.json"),y.join(e,"errors.jsonl.gz"),y.join(e,"errors.json.gz")])]}async function Rn(e){let t=await Kn(y.join(e,"job.json"));for(let r of Oi(e,t)){let n=await Ai(r);if(n.length>0)return n}return []}async function Mi(e){let t=y.join(e.logDir,"archive"),r;try{r=await $.readdir(t);}catch{return ""}let n="",s=0;for(let o of r){let i=y.join(t,o);if(!(await $.stat(i).catch(()=>null))?.isDirectory())continue;let a=await Kn(y.join(i,"job.json"));if(!Ii(a))continue;let l=Ei(a);l>=s&&(s=l,n=i);}return n}async function $i(e){let t=await Mi(e),r=t!==""?await Rn(t):await Rn(e.logDir),n=new Set;for(let s of r)if(Sn(s.error)||Sn(s.message)){let o=s.url;typeof o=="string"&&o&&n.add(o);}return [...n]}async function ji(e){let t=[];for(let r of e)if(r.startsWith("@")){let n=r.slice(1),s=await $.readFile(n,"utf8");t.push(...s.split(/\r?\n/).map(o=>o.trim()).filter(o=>o&&!o.startsWith("#")));}else t.push(r);return t}async function Ui(e){if(!e)throw new Error("Content-sync crawl requires an immutable impact plan.");let t=JSON.parse(await $.readFile(e,"utf8"));return fr(t),t}function Li(e,t){let r;try{r=new URL(e);}catch{return false}return t.archiveFamilies.some(n=>{let s=new URL(n.url);if(r.origin!==s.origin)return false;if(r.toString()===s.toString())return true;if(r.searchParams.has("paged")&&r.pathname===s.pathname)return /^\d+$/.test(r.searchParams.get("paged")||"");let o=s.pathname.replace(/\/+$/,""),i=r.pathname.slice(o.length).replace(/^\/+/,""),u=String(n.paginationBase||"page").replace(/^\/+|\/+$/g,"").replace(/[.*+?^${}()|[\]\\]/g,"\\$&");return r.pathname.startsWith(`${o}/`)&&new RegExp(`^${u}/\\d+/?$`).test(i)})}function Ke(e,t,r){let n=r==="page"?ge(e,t):et(e,t).originalFilePath;return Y(e.outputDir,n)}async function Ye(e,t,r,n,s){let o=t.manifest.pages[n];if(!o)return;let i=Y(e.outputDir,o.outputPath);if(!i)throw new Error(`Manifest-owned page path escaped output root: ${n}`);let u=y.resolve(o.outputPath);Object.entries(t.manifest.pages).some(([l,d])=>l!==n&&!s.has(l)&&y.resolve(d.outputPath)===u)||(await $.rm(o.outputPath,{force:true}),r.deletedFiles.add(i),r.revalidateFiles.delete(i)),delete t.manifest.pages[n];}async function Di(e,t,r,n,s){let o=t.manifest.assets[n];if(!o)return;let i=Y(e.outputDir,o.outputPath);if(!i)throw new Error(`Manifest-owned resource path escaped output root: ${n}`);let u=y.resolve(o.outputPath);Object.entries(t.manifest.assets).some(([l,d])=>l!==n&&!s.has(l)&&y.resolve(d.outputPath)===u)||(await $.rm(o.outputPath,{force:true}),r.deletedFiles.add(i)),delete t.manifest.assets[n];}function Fi(e,t){let r=new Set,n=[...t];for(;n.length>0;){let s=n.shift();if(r.has(s))continue;r.add(s);let o=e.assets[s];for(let i of o?.discoveredAssets??[])try{Ne(new URL(i))&&n.push(i);}catch{}}return r}async function Wi(e,t,r,n,s){let o=Fi(r.previousManifest,s);for(let i of o){if(t.successfulSitemaps.has(i))continue;let u=r.previousManifest.assets[i];if(!u)continue;let a=Y(e.outputDir,u.outputPath);if(!a)throw new Error(`Manifest-owned sitemap path escaped output root: ${i}`);await $.rm(u.outputPath,{force:true}),n.deletedFiles.add(a),delete r.manifest.assets[i];}await Pe(e,r);}async function qi(e){try{let t=await $.readFile(y.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Ni(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await $.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await $.unlink(y.join(e,r));}catch(n){if(n.code!=="ENOENT")throw n}}));}function vn(e,t,r,n,s){let o=V(e,r,e.sourceOrigin,n,"manual-url");o&&(Ne(o)?Pt(e,t,o.toString(),n,s):q(e,o)?W(e,t,o.toString(),n,s):Re(o)?Me(e,t,o.toString(),n,s):n.reject("manual-url",o.toString(),"not page/sitemap/asset-like",s));}async function Jn(e=process.argv.slice(2)){let t=[],r=await nt(void 0,{onRemotePublisherConfigDiagnostic:S=>{t.push(S);}}),n=wr(r.sitemapPaths),s=t.at(-1)??null,o=vo(e),i=o.mode==="content-sync"?await Ui(o.contentSyncPlanPath):null;if(o.resumeRewrite&&o.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let u=[];o.mode==="retry-timeouts"&&(u=await $i(r)),o.mode==="single-url"&&(u=await ji(o.urls)),i&&(u=[...i.directRenderUrls,...i.archiveFamilyUrls,...i.resourceRenderUrls]);let a=o.crawlMode==="incremental"&&o.mode==="full",l=a&&typeof r.subscriptionType=="string"&&r.subscriptionType.trim()!=="";if(i&&r.subscriptionType!=="PROFESSIONAL"&&r.subscriptionType!=="AGENCY")throw new Error("Content-sync requires an active Professional or Agency subscription.");let d=(o.mode==="full"||o.mode==="single-url"||o.mode==="content-sync")&&!o.resumeRewrite,c=o.mode!=="full"||o.resumeRewrite||l,m=c?await qi(r):{},h=l||o.mode==="single-url"||o.mode==="content-sync"?await Oo(r):An(),f=new Set(u),b=new Set;if(i){for(let S of i.archiveFamilies)if(S.foundPosts===0)for(let M of S.pageUrls)b.add(M);for(let S of Object.keys(h.pages))Li(S,i)&&!f.has(S)&&(u.push(S),b.add(S));}let p={enabled:l,trackManifest:d,manifest:Xt(h),previousManifest:Xt(h),runId:i?.jobId??`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map,pageChangeTokenPrimePromise:null,manifestSaveChain:Promise.resolve(),manifestPendingChanges:0,lastManifestSavedAt:Date.now(),strictTargeted:!!i,requiredUrls:f,deleteOnMissingUrls:b};await Ni(r.logDir);let g=new Ue(r.logDir,r.logLevel),w=o.remoteRender??r.remoteWorkers?.render?.enabled===true,k=rn({explicitPath:o.remoteWorkerConfigPath,runtimeDir:Xe()}),C=null,I=null,A=null,R=null,v=Math.max(1,r.concurrency),E=1,x=r.remoteWorkers?.rewrite?.enabled===true,z=r.remoteWorkers?.deploy?.enabled===true,L=w&&x&&z&&!o.resumeRewrite;if(!L&&o.mode==="full"&&!o.resumeRewrite&&await $.rm(ht(r),{force:true}),w&&!o.resumeRewrite||x){if(!k)throw new Error("Remote Lambda workers are enabled, but the runtime directory is unknown. Run through queue-runner with --runtime-dir, set STATIC_PUBLISHER_RUNTIME_DIR, or use --remote-worker-config for a diagnostic run.");let S=await Kr(k),M=false;if(A=new lt(S,{},{onProgress:O=>{g.progress(`Lambda ${O.operation} batch: ${O.completedItems+O.failedItems}/${O.totalItems}`,{remoteWorkerOperation:O.operation,remoteWorkerTaskId:O.taskId,remoteWorkerStatus:O.status,remoteWorkerCompletedItems:O.completedItems,remoteWorkerFailedItems:O.failedItems,remoteWorkerTotalItems:O.totalItems,remoteWorkerUpdatedAt:O.updatedAt});},onProgressError:O=>{M||(M=true,g.warn("Lambda progress status could not be read; task execution continues.",{error:O.message}));}}),L){if(!S.functions.asset)throw new Error("Full Lambda delegation requires an asset worker. Redeploy the Lambda worker stack and reinstall remote-workers.json.");let O=c?await dn(r):null;R=an(`crawl-${ue(p.runId,40)}`,S,O),I=new gt({client:A,jobId:R.jobId,maxAttempts:2}),g.info("S3-native Lambda data plane is enabled.",{execution:"lambda",assetWorker:S.functions.asset,workspaceBucket:S.workspace.bucket,workspacePrefix:S.workspace.prefix});}}if(w&&!o.resumeRewrite){if(!A)throw new Error("Remote render worker client was not initialized.");v=Math.max(1,Math.min(100,L?r.concurrency:r.remoteWorkers?.render?.concurrency??r.concurrency)),E=Math.max(1,Math.min(20,r.remoteWorkers?.render?.batchSize??5)),C=new mt({client:A,jobId:`crawl-${ue(p.runId,40)}`,maxAttempts:r.remoteWorkers?.render?.maxAttempts??2,batchSize:E,maxConcurrentBatches:v,onRetry:S=>{g.warn(`Retrying remote render for ${S.url}`,{url:S.url,attempt:S.attempt,maxAttempts:S.maxAttempts,reason:S.reason,status:S.status,delayMs:S.delayMs,execution:"lambda"});}}),g.info("Lambda page rendering is enabled.",{remoteWorkerConfigPath:y.resolve(k),concurrency:v,maxAttempts:r.remoteWorkers?.render?.maxAttempts??2,batchSize:E,execution:"lambda"});}if(s){let S={status:s.status,virtualAssetBaseUrl:s.virtualAssetBaseUrl,subscriptionType:s.subscriptionType??null,error:s.error,requests:s.requests,source:"remote-publisher-config"};s.status==="loaded"&&s.subscriptionType?g.info("Loaded remote WP Suite publisher configuration.",S):g.warn("Remote WP Suite publisher configuration is unavailable or has no active subscription type.",S);}if(a&&!l){let S="Incremental crawl requires an active remote publisher subscription; refusing to fall back to a destructive full crawl.";throw g.error(S,{requestedMode:o.crawlMode,subscriptionType:r.subscriptionType??null,source:"incremental-subscription-guard"}),await g.flush(),new Error(S)}let P={pageQueue:[],queuedPages:new Set,donePages:new Set,savedPageOutputs:new Set,pageHttpAttempts:new Map,pageHttpFailures:new Map,pageDiscoverySources:new Map,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,successfulSitemaps:new Set,resolvedSitemapRoots:new Set,sitemapIndexUrls:new Set,sitemapLeafUrls:new Set,sitemapLastmodByPage:{},assetMap:{...m},assetVariantDigestsByOriginalPath:new Map,stats:{pagesQueued:0,pagesRendered:0,pagesCompleted:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},j=Eo(),oe=new ut,ee=ko();if(ee&&!o.resumeRewrite&&await ct(ee,oe.artifact()),await gr(r),await Tt(r,{startedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir)}),!o.resumeRewrite&&r.pageCache?.purgeBeforeCrawl===true){g.mark("purge-page-cache");let S=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{let M=await sn(r,S);g.info("WordPress page cache was purged before crawl discovery.",{provider:M.provider,scope:M.scope,purgedAt:M.purgedAt??null}),g.endMark("purge-page-cache",{provider:M.provider,scope:M.scope});}catch(M){throw g.error("WordPress page-cache purge failed; crawl stopped before changing export output.",{error:M instanceof Error?M.message:String(M)}),await g.flush(),M}finally{await S.dispose();}}if(c||await $.rm(r.outputDir,{recursive:true,force:true}),await $.mkdir(r.outputDir,{recursive:true}),c||await Tt(r,{startedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir)}),i){let S=new Set([...i.deleteUrls,...p.deleteOnMissingUrls]);for(let O of i.deleteUrls)await Ye(r,p,j,O,S);let M=new Set(i.resourceDeleteUrls);for(let O of i.resourceDeleteUrls)await Di(r,p,j,O,M);}if(o.resumeRewrite)g.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:o.mode,crawlMode:o.crawlMode});else {let S=await request.newContext({ignoreHTTPSErrors:r.ignoreHttpsErrors});try{if(g.mark("discovery"),o.mode==="full"?(n.forEach(O=>Pt(r,P,O,g)),await wn(r,S,P,g,j,p),r.seedPaths.forEach(O=>vn(r,P,O,g,"seed-path")),r.generated404RequestPath&&Me(r,P,r.generated404RequestPath,g,"generated-404-request-path")):(u.forEach(O=>vn(r,P,O,g,"cli")),i?.sitemapUrls.forEach(O=>Pt(r,P,O,g,"content-sync-impact")),await wn(r,S,P,g,j,p,o.mode==="content-sync",i?.sitemapUrls??[]),i&&await Wi(r,P,p,j,i.sitemapUrls)),g.endMark("discovery",{pages:P.pageQueue.length,assets:P.assetQueue.length,sitemaps:P.sitemapQueue.length}),g.summary(`Queued ${P.pageQueue.length} pages, ${P.assetQueue.length} assets, ${P.sitemapQueue.length} sitemaps.`,{mode:o.mode,crawlMode:o.crawlMode,queuedPages:P.pageQueue.length,queuedAssets:P.assetQueue.length,queuedSitemaps:P.sitemapQueue.length}),o.mode==="retry-timeouts"&&P.pageQueue.length===0&&P.assetQueue.length===0&&P.sitemapQueue.length===0){g.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:o.mode,crawlMode:o.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await g.flush(),ee&&await ct(ee,oe.artifact());return}g.mark("render-pages");let M=Array.from({length:C?v*E:v},()=>C?xi(r,P,g,o.mode,S,p,j,oe,C,R??void 0):Ti(r,P,g,o.mode,S,p,j,oe));if(await Promise.all(M),P.pageHttpFailures.size>0&&await g.flush(),Jr(P.pageHttpFailures),ee&&await ct(ee,oe.artifact()),g.endMark("render-pages",{pages:P.stats.pagesCompleted}),g.mark("download-assets"),g.progress("Asset download phase started.",{phase:"download-assets",doneAssets:P.doneAssets.size,assetsQueued:P.stats.assetsQueued,assetQueue:P.assetQueue.length}),R&&I?await wi(r,S,P,g,p,j,R,I,r.concurrency):(await ir(r,S,p,[...P.assetQueue],g),await hi(r,S,P,g,p,j)),g.endMark("download-assets",{assets:P.doneAssets.size}),p.enabled&&r.maxPages===0&&(await Fo(r,P,p,g,j),o.mode==="full"&&await Wo(r,p,g,j)),o.mode==="full"){let O=await mi(r,P,g);for(let N of O)if(G(j,r,N),R&&A){let J=Rt(r,N),Ae=await $.readFile(N),Ie=ce(N),$e=String(Gr.lookup(J)||"")||(Ie?"text/plain; charset=utf-8":"application/octet-stream"),rt=await A.writeWorkspaceObject({key:`${R.workspace.prefix}jobs/${R.jobId}/copied/${ue(J,40)}-${y.basename(J)}`,body:Ae,contentType:$e});De(R,{relativePath:J,sourceUrl:null,kind:"generated",contentType:rt.contentType,cacheControl:null,isText:Ie,object:rt}),Ie&&ne(j,N);}}}finally{await B("dispose crawl request context",Je(r),()=>S.dispose()).catch(M=>{g.warn("Timed out while disposing crawl request context",{error:String(M)});});}}if(p.trackManifest&&await Pe(r,p,true),R)for(let S of j.deletedFiles)cn(R,S);let ie=JSON.stringify(P.assetMap,null,2);if(await $.writeFile(y.join(r.outputDir,"asset-map.json"),ie,"utf8"),R&&A){let S=await A.writeWorkspaceObject({key:`${R.workspace.prefix}jobs/${R.jobId}/generated/asset-map.json`,body:ie,contentType:"application/json; charset=utf-8"});De(R,{relativePath:"asset-map.json",sourceUrl:null,kind:"generated",contentType:"application/json; charset=utf-8",cacheControl:r.s3.assetCacheControl,isText:true,object:S});}JSON.stringify(m,null,2)!==JSON.stringify(P.assetMap,null,2)&&G(j,r,y.join(r.outputDir,"asset-map.json")),R&&A&&await yi(r,j,R,A);let T=o.mode==="single-url"||o.mode==="content-sync"?Qo(r,P):p.enabled&&!o.resumeRewrite?qo(r,p,j,m,P.assetMap):void 0;g.mark("rewrite-text");let F=o.mode==="single-url"||o.mode==="content-sync"?`Rewriting text files touched by URL crawl: 0/${T?.length??0}`:"Rewriting text files...";g.progress(F,{phase:"rewrite-text",index:0,totalFiles:T?.length,changedTextFiles:0}),await Kt(F);let D=Date.now(),Z={files:T,previousAssetMap:m,onStart:({totalFiles:S,workerCount:M})=>{g.info(`Rewrite phase scanning ${S} text files.`,{phase:"rewrite-text",totalFiles:S,workerCount:M});},onProgress:async({index:S,totalFiles:M,changedTextFiles:O,file:N,changed:J})=>{J&&G(j,r,y.join(r.outputDir,N)),g.checkpoint(`Rewriting text file ${S}/${M}`,{phase:"rewrite-text",index:S,totalFiles:M,changedTextFiles:O,file:N}),g.debug(`${J?"Rewrote":"Checked unchanged"} text file ${S}/${M}: ${N}`,{phase:"rewrite-text",index:S,totalFiles:M,changedTextFiles:O,file:N,changed:J});let Ae=Date.now();if(!(S===1||S===M||S%50===0||Ae-D>=5e3))return;D=Ae;let $e=`Rewriting text files: ${S}/${M}`;g.progress($e,{phase:"rewrite-text",index:S,totalFiles:M,changedTextFiles:O,file:N}),await Kt($e);}},K;if(R&&A){let S=T?.map(O=>Rt(r,y.isAbsolute(O)?O:y.join(r.outputDir,O))),M=Object.values(R.objects).filter(O=>O.isText&&(!S||S.includes(O.relativePath))).length;await Z.onStart?.({totalFiles:M,workerCount:Math.min(r.concurrency,M)}),K=await tn(r,R,P.assetMap,{client:A,jobId:R.jobId,concurrency:r.concurrency,batchSize:25,maxAttempts:2,...S?{relativePaths:S}:{},onProgress:Z.onProgress}),await ln(r,R);}else x&&A?K=await en(r,P.assetMap,{client:A,jobId:`crawl-${ue(p.runId,40)}`,concurrency:r.remoteWorkers?.rewrite?.concurrency??r.rewriteConcurrency,batchSize:r.remoteWorkers?.rewrite?.batchSize??25,maxAttempts:r.remoteWorkers?.rewrite?.maxAttempts??2,onStageProgress:async({index:S,totalFiles:M,file:O})=>{let N=Date.now();if(S!==1&&S!==M&&S%25!==0&&N-D<5e3)return;D=N;let J=`Staging text files for Lambda rewrite: ${S}/${M}`;g.progress(J,{phase:"rewrite-stage",index:S,totalFiles:M,file:O,execution:"lambda"}),await Kt(J);},options:Z}):K=await Er(r,P.assetMap,Z);p.trackManifest&&(No(p,T),await Pe(r,p,true)),g.endMark("rewrite-text",{changedTextFiles:K});let _e=i?[...j.deletedFiles].sort():p.enabled&&!o.resumeRewrite?await Ho(r,P,p,j,g,R):[],tt=[...new Set((T||[]).map(S=>Y(r.outputDir,y.isAbsolute(S)?S:y.join(r.outputDir,S))).filter(S=>S!==null))].sort();if(i){let S=[...j.changedFiles].sort(),M=new Set(S),O=new Set(i.directRenderUrls.map(U=>Ke(r,U,"page")).filter(U=>!!U&&M.has(U))),N=new Set(i.archiveFamilyUrls.map(U=>Ke(r,U,"page")).filter(U=>!!U&&M.has(U))),J=new Set(i.resourceRenderUrls.map(U=>Ke(r,U,"asset")).filter(U=>!!U&&M.has(U))),Ae=new Set([...P.sitemapLeafUrls].map(U=>Ke(r,U,"asset")).filter(U=>!!U&&M.has(U))),Ie=new Set([...P.sitemapIndexUrls].map(U=>Ke(r,U,"asset")).filter(U=>!!U&&M.has(U))),$e=new Set([...O,...N,...J,...Ae,...Ie]),rt=S.filter(U=>!$e.has(U)),Gn=[...new Set([...i.directRenderUrls,...i.archiveFamilyUrls,...P.resolvedSitemapRoots,...P.sitemapLeafUrls,...P.sitemapIndexUrls,...i.deleteUrls,...i.resourceRenderUrls,...i.resourceDeleteUrls].map(U=>{let cr=new URL(U);return `${cr.pathname}${cr.search}`}))].sort();await xt(r,{schemaVersion:2,generatedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir),runMode:"content-sync",crawlMode:"incremental",fullSyncRequired:false,changedFiles:S,deletedFiles:_e,rewriteTargets:tt,revalidateFiles:[...j.revalidateFiles].sort(),consumerId:i.consumerId,fromSequence:i.fromSequence,toSequence:i.toSequence,impactHash:i.impactHash,resolvedSitemapUrls:[...P.resolvedSitemapRoots],uploadGroups:{assets:rt,content:[...O].sort(),listings:[...N].sort(),resources:[...J].sort(),sitemapLeaves:[...Ae].sort(),sitemapIndexes:[...Ie].reverse()},invalidationPaths:Gn});}else await xt(r,{schemaVersion:1,generatedAt:new Date().toISOString(),outputDir:y.resolve(r.outputDir),runMode:o.mode,crawlMode:o.crawlMode,fullSyncRequired:!(p.enabled&&!o.resumeRewrite),changedFiles:p.enabled&&!o.resumeRewrite?[...j.changedFiles].sort():[],deletedFiles:p.enabled&&!o.resumeRewrite?_e:[],rewriteTargets:p.enabled&&!o.resumeRewrite?tt:[],revalidateFiles:[...j.revalidateFiles].sort()});await hr(r),g.summary(o.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${K} text files.`:`Done. Rendered ${P.stats.pagesRendered} pages, processed ${P.doneSitemaps.size} sitemaps and ${P.doneAssets.size} assets.`,{mode:o.mode,crawlMode:o.crawlMode,resumeRewrite:o.resumeRewrite,...P.stats,pagesRendered:P.stats.pagesRendered,donePages:P.stats.pagesCompleted,claimedPages:P.donePages.size,doneSitemaps:P.doneSitemaps.size,doneAssets:P.doneAssets.size,changedTextFiles:K}),await g.flush();}Jn().catch(async e=>{let t=ye(e instanceof Error?e.message:String(e));console.error(t);try{let n=(await nt().catch(()=>null))?.logDir??"logs",s=new Ue(n,"debug");s.error(`Unhandled error: ${t}`,e instanceof Error&&e.stack?{stack:ye(e.stack)}:void 0),await s.flush();}catch{}process.exit(1);});
|