@smart-cloud/publisher-exporter 1.0.17 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +4 -4
- package/dist/deploy.js +3 -3
- package/dist/invalidate.js +3 -3
- package/package.json +10 -10
- package/publisher.config.example.json +1 -0
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import S from'fs/promises';import
|
|
1
|
+
import {chromium}from'playwright';import S from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import bt from'crypto';import Ct from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';var gt=".deploy-plan.json",ht="deploy-plan.json";function ft(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Ae(e){let t=String(e||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !t||t==="."||t.startsWith("../")||t===".."?null:t}function ae(e){return [...new Set(e.map(Ae).filter(t=>t!==null))]}function mt(e,t){return {schemaVersion:1,generatedAt:String(t.generatedAt||new Date().toISOString()),outputDir:f.resolve(e.outputDir),runMode:t.runMode,crawlMode:t.crawlMode,fullSyncRequired:!!t.fullSyncRequired,changedFiles:ae(t.changedFiles||[]),deletedFiles:ae(t.deletedFiles||[]),rewriteTargets:ae(t.rewriteTargets||[])}}function wt(e){let t=f.join(f.resolve(e.outputDir),gt),r=ft();return r?[f.join(r,ht),t]:[t]}function V(e,t){let r=f.resolve(e),s=f.resolve(t),n=f.relative(r,s).replace(/\\/g,"/");return Ae(n)}async function ke(e,t){let r=mt(e,t);for(let s of wt(e))await S.mkdir(f.dirname(s),{recursive:true}),await S.writeFile(s,JSON.stringify(r,null,2),"utf8");}function Te(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function yt(e){let t=String(e??"").trim();if(!t)return "";let r=t;if(/^https?:\/\//i.test(r))try{r=new URL(r).pathname;}catch{return ""}if(r=r.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!r)return "";let s=r.endsWith("/"),n=r.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(n.length===0)return "";let i=`/${n.join("/")}`;return s&&f.extname(i)===""&&(i+="/"),i==="/"?"":i}function oe(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function St(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,o={};if(typeof i.targetOrigin=="string"){let a=Te(i.targetOrigin);a&&(o.targetOrigin=a);}let u=oe(i.extraReplacements);if(Object.keys(u).length>0&&(o.extraReplacements=u),i.s3&&typeof i.s3=="object"){let a={},c=i.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},c=i.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(l=>String(l??"").trim()).filter(l=>l.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}t[n]=o;}return t}function ue(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function vt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Me(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,ue(s,r))}async function le(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await S.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=Te(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.generated404RequestPath=yt(r.generated404RequestPath),r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=oe(r.extraReplacements),r.postCrawlCopyMap=oe(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=St(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=vt(),n=s?f.resolve(s,".."):"";return n?(r.outputDir=Me(n,r.outputDir,"export"),r.logDir=Me(n,r.logDir,"logs")):(f.isAbsolute(r.outputDir)||(r.outputDir=ue(r.outputDir,"export")),f.isAbsolute(r.logDir)||(r.logDir=ue(r.logDir,"logs"))),r}function Ie(e,t=10){return bt.createHash("sha1").update(e).digest("hex").slice(0,t)}async function ce(e){await S.mkdir(f.dirname(e),{recursive:true});}function O(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function j(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function Q(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function G(e){return e.replace(/\//g,"\\/")}function Le(e){return e.replace(/\//g,"\\\\/")}function K(e,t,r){if(!t)return;e[t]=r;let s=G(t),n=G(r);e[s]=n;let i=Le(t),o=Le(r);e[i]=o;let u=Q(t),a=Q(r);e[u]=a;let c=Q(s),l=Q(n);e[c]=l;let p=Q(i),d=Q(o);e[p]=d;}function de(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function De(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function Ee(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var kt=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Mt=new Set([".html",".htm"]),Oe="WPSuite.io Static Publisher",Tt=Oe.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Lt(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=f.extname(r).toLowerCase();return Mt.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function It(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Tt}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${Oe}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Qt(t,r));}updateCurrentEvent(t,r,s){let n=qt(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(N[this.level]>=N.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){Ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),N[this.level]>=N.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),N[this.level]>=N.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Wt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],Ht=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Jt=new Set(["",".html",".htm"]),Bt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),zt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Vt(e){let t=[],r="full",s="full",n=false,i=false;for(let u=0;u<e.length;u++){let a=e[u];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--resume-rewrite")n=true,i=true;else if(a==="--crawl-mode"){let o=e[++u];if(!o)throw new Error("--crawl-mode requires a value");s=o==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let o=e[++u];if(!o)throw new Error("--url requires a value");r="single-url",n=true,t.push(o);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let o=e[++u];if(!o)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${o}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Se(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.resolve(e):""}function Kt(){let e=Se();return e?h.resolve(e,".."):""}function Gt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?h.resolve(e):""}function Xt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Kt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Se(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Gt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?h.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:h.resolve(t)}}function et(e){let t=Se();return t?h.join(t,"crawl-manifest.json"):h.join(e.outputDir,".crawl-manifest.json")}function tt(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function Yt(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function O(e,t,r){let s=J(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function Ne(e,t,r){let s=J(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function X(e,t){e.rewriteTargets.add(h.resolve(t));}async function Zt(e){try{let t=await S.readFile(et(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return tt()}function qe(e){return JSON.parse(JSON.stringify(e))}async function er(e,t){let r=et(e);await S.mkdir(h.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}function Y(e){if(!e)return null;let t=e.trim();return t||null}function tr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],u=t[n];!i||!u||i===u||r.add(n);}return r}async function rr(e,t,r,s){let n=[...new Set(r.map(a=>a.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let u=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let a=await t.post(u,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!a.ok())return s.warn(`Change token lookup failed with HTTP ${a.status()}`,{endpoint:u,status:a.status()}),new Map;let o=await a.json().catch(()=>null);if(!o||!Array.isArray(o.items))return new Map;let c=new Map;for(let l of o.items)!l||typeof l.url!="string"||c.set(l.url,l);return c}catch(a){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:u,error:String(a)}),new Map}}async function he(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let u=(await rr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,u),u}async function sr(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await he(e,t,s,n,i):null};let u=s.manifest.pages[n];if(!u)return {action:"render",changeToken:await he(e,t,s,n,i)};let a=await he(e,t,s,n,i);if(a?.supported&&a.token)return u.changeToken===a.token?{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let o=Y(r.sitemapLastmodByPage[n]);return o&&u.sitemapLastmod&&u.sitemapLastmod===o?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function nr(e,t,r,s,n){let i=_e(r.previousManifest),u=0;for(let[d,p]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){Ne(n,e,p.outputPath);try{await S.unlink(p.outputPath);}catch(f){(f.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:p.outputPath,error:String(f)});}delete r.manifest.pages[d],u++;}let a=_e(r.manifest,r.runId),o=new Set,c=new Set;for(let d of a){for(let w of _(e,d))o.add(w);let p=r.manifest.assets[d]?.outputPath;p&&c.add(p);let f=Z(e,t.assetMap,d);f&&c.add(f);}for(let d of Object.keys(r.manifest.assets))a.has(d)||delete r.manifest.assets[d];let l=0,g=0;for(let d of i){let p=new Set,f=r.previousManifest.assets[d]?.outputPath;f&&p.add(f);let w=Z(e,t.assetMap,d);if(!a.has(d)&&w)p.add(w);else if(!a.has(d)){let y=be(e,d);p.add(y.originalFilePath),y.hashedFilePath&&p.add(y.hashedFilePath);}for(let y of p)if(!c.has(y)){Ne(n,e,y);try{await S.unlink(y),l++;}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:y,error:String(m)});}}for(let y of _(e,d))o.has(y)||Object.prototype.hasOwnProperty.call(t.assetMap,y)&&(delete t.assetMap[y],g++);}(u>0||l>0||g>0)&&s.info(`Incremental cleanup removed ${u} stale pages, ${l} stale assets, ${g} stale asset mappings`,{removedPages:u,removedAssets:l,removedAssetMappings:g,phase:"incremental-cleanup"});}function _e(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function We(e,t,r){for(let s of t)for(let n of _(e,s))if(r.has(n))return true;return false}function ir(e,t,r,s,n){let i=new Set(r.rewriteTargets),u=tr(s,n);if(u.size===0)return [...i];for(let a of Object.values(t.manifest.pages))We(e,a.discoveredAssets,u)&&i.add(h.resolve(a.outputPath));for(let[a,o]of Object.entries(t.manifest.assets)){o.isText&&We(e,o.discoveredAssets,u)&&i.add(h.resolve(o.outputPath));for(let c of _(e,a))if(u.has(c)){O(r,e,o.outputPath),o.isText&&i.add(h.resolve(o.outputPath));break}}return [...i]}function _(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function or(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let u of i)try{r.add(new URL(u).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return h.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function ar(e,t,r){for(let s of or(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function ur(e,t,r){for(let s of _(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return i}return null}function Z(e,t,r){let s=ur(e,t,r);return s||be(e,r).originalFilePath}function lr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?h.join(h.resolve(e),"queue-runner-heartbeat.json"):""}async function He(e){let t=lr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function cr(e,t){let r=new Set;for(let s of t.donePages)r.add(h.resolve(te(e,s)));for(let s of t.doneAssets){let n=Z(e,t.assetMap,s);!n||!E(n)||r.add(h.resolve(n));}return [...r]}var ee="re:",G=new Map;function dr(e){if(!e.startsWith(ee))return null;let t=G.get(e);if(t!==void 0)return t||null;let r=e.slice(ee.length).trim();if(!r)return G.set(e,false),null;try{let s=new RegExp(r);return G.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),G.set(e,false),null}}function ve(e,t){let r=dr(t);return r?r.test(e):t.startsWith(ee)?false:e.startsWith(t)}function pr(e,t){return t.find(r=>ve(e,r))}function st(e,t){let r=pr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(ee)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function gr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function hr(e,t){return gr(e).has(t.hostname)}function fr(e){let t=F(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=F(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function se(e){return zt.some(t=>e.pathname.startsWith(t))}function C(e,t,r=e.sourceOrigin,s,n="url"){let i=fr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let u=new URL(i,r);if(!["http:","https:"].includes(u.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!hr(e,u))return s?.ignore(n,u.toString(),"host not allowed",r),null;let a=st(e,u);if(a)return s?.reject(n,u.toString(),a,r),null;let o=new URL(e.sourceOrigin);return u.protocol=o.protocol,u.host=o.host,u.hash="",u}catch{return s?.reject(n,t,"invalid URL",r),null}}function it(e){return Ht.has(h.extname(e.pathname).toLowerCase())}function H(e){return se(e)?false:Jt.has(h.extname(e.pathname).toLowerCase())}function ne(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function R(e,t){return ne(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:it(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>ve(t.pathname,r)):false}function te(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":h.extname(s)||(s+="/index.html"),h.join(e.outputDir,s)}function be(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,u=h.join(e.outputDir,i),a=de(e.targetOrigin,i),o=`${a}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n};let c=h.extname(s),g=`${c?s.slice(0,-c.length):s}.${Ie(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:u,originalPublicUrl:a,originalPublicUrlWithSearch:o,preferQueryHashed:n,hashedPathname:g,hashedFilePath:h.join(e.outputDir,g),hashedPublicUrl:de(e.targetOrigin,g)}}function mr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),a=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{a.push(new URL(e.targetOrigin).origin);}catch{}for(let o of a)try{let c=new URL(o);t[c.origin+i.pathname+i.search]=s;}catch{}try{let o=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=o;}catch{}try{let o=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}try{let o=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=o;}catch{}}catch{}}async function Pe(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function wr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function at(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Pr(e,t){let r=at(t),s=wr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function yr(e,t){if(!t)return;let r=at(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function fe(e,t){let r=h.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Bt.has(r)||Wt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function me(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ye(e,t,r,s,n,i,u){let a=be(e,r),o=a.originalFilePath,c=a.originalPublicUrlWithSearch;if(a.preferQueryHashed&&a.hashedFilePath&&a.hashedPublicUrl){let p=ar(e,t,r);if(p)o=p.outputPath,c=p.publicUrl;else {let f=await Pe(a.originalFilePath);if(f!==null&&!f.equals(s)){let w=E(a.originalFilePath)?Buffer.from(V(s.toString("utf8"),e,t,a.originalFilePath),"utf8"):null;(w===null||!f.equals(w))&&(o=a.hashedFilePath,c=a.hashedPublicUrl);}}}mr(e,t,r,c,a.originalPublicUrl);let l=E(o)?Buffer.from(V(s.toString("utf8"),e,t,o),"utf8"):s,g=await Pe(o),d=false;return (g===null||!g.equals(l))&&(await ce(o),await S.writeFile(o,l),d=true),await yr(o,u),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:o,publicUrl:c,wroteFile:d}}function re(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"page");if(!u)return;if(!H(u)||it(u)){if(R(e,u)){s.info(`Seeded/non-page URL queued as asset: ${u.toString()}`,{url:u.toString(),source:n}),x(e,t,u.toString(),s,n);return}s.reject("page",u.toString(),se(u)?"asset/internal path cannot be page":"not page-like",n);return}let a=u.toString();!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function x(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"asset");if(!u)return;if(!R(e,u)){s.reject("asset",u.toString(),"not a safe asset path/prefix",n);return}let a=u.toString();!t.doneAssets.has(a)&&!t.queuedAssets.has(a)&&(t.queuedAssets.add(a),t.assetQueue.push(a),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Re(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,u=C(e,r,i,s,"sitemap");if(!u)return;if(!ne(u)){s.reject("sitemap",u.toString(),"not sitemap-like",n);return}let a=u.toString();!t.doneSitemaps.has(a)&&!t.queuedSitemaps.has(a)&&(t.queuedSitemaps.add(a),t.sitemapQueue.push(a),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Sr(e){let t=F(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:Y(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function vr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function br(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function we(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function ut(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=h.join(e,n.name),u=h.join(t,n.name);if(n.isDirectory()){r.push(...await ut(i,u));continue}await S.copyFile(i,u),r.push(u);}return r}async function Rr(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,u])=>({sourcePath:i.trim(),prefix:String(u||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let u=i.sourcePath,a=i.prefix;if(!u||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:u,prefixPath:a});continue}let o=Xt(u);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:u,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let c=o.resolvedPath,l=we(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:u,prefixPath:a});continue}let g;try{g=await S.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:u,sourceAbs:c,prefixPath:a});continue}let d=h.resolve(e.outputDir,l);if(g.isDirectory()){n.push(...await ut(c,d)),s++,t.progress(`Copied static directory to export: ${c} -> /${l}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${l}`,copiedItems:s});continue}let p=h.basename(c),f=a.endsWith("/")?h.join(d,p):d;await S.mkdir(h.dirname(f),{recursive:true}),await S.copyFile(c,f),n.push(f),s++,t.progress(`Copied static file to export: ${c} -> /${we(a.endsWith("/")?`${a}${p}`:a)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${we(a.endsWith("/")?`${a}${p}`:a)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function W(e,t,r,s,n){let i=new Set,u=F(r).replace(/\\\//g,"/");for(let c of br(u)){let l=C(e,c,t,s,`${n}:css-url`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let l=C(e,c[1],t,s,`${n}:css-import`);l&&R(e,l)&&i.add(l.toString());}for(let c of u.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let l=C(e,c[1],t,s,`${n}:xml-stylesheet`);l&&R(e,l)&&i.add(l.toString());}let a="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",o=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of u.matchAll(o)){let l=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(nt(l)){s.ignore(`${n}:serialized-url`,l,"looks like JavaScript/code fragment",t);continue}let g=C(e,l,t,s,`${n}:serialized-url`);g&&R(e,g)?i.add(g.toString()):g&&s.ignore("asset",g.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Cr(e,t,r,s){let n=new Set,i=F(r);for(let u of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let a=u[1],o=C(e,a,t,s,"page-link");o&&(st(e,o)||H(o)&&!se(o)&&!R(e,o)&&n.add(o.toString()));}return [...n]}async function Je(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let u=await i.body(),a=i.url()||n;await ye(e,r.assetMap,a,u,r,s);let o=u.toString("utf8");for(let c of W(e,a,o,s,"sitemap"))x(e,r,c,s,a);for(let c of Sr(o)){let l=C(e,c.loc,a,s,"sitemap-loc");l&&(ne(l)?Re(e,r,l.toString(),s,a):R(e,l)?x(e,r,l.toString(),s,a):H(l)?(c.lastmod&&(r.sitemapLastmodByPage[l.toString()]=c.lastmod),re(e,r,l.toString(),s,a)):s.reject("sitemap-loc",l.toString(),"not page/sitemap/asset-like",a));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function xr(e,t,r,s,n,i,u){let a=C(e,s,e.sourceOrigin,n,"asset-fetch");if(!a||!R(e,a))return;let o=a.toString();if(r.doneAssets.has(o))return;let c=i.enabled?i.previousManifest.assets[o]:void 0,l=(g,d={})=>{if(!c)return false;i.manifest.assets[o]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(o);for(let p of c.discoveredAssets||[])x(e,r,p,n,o);return n.warn(`Keeping previous incremental asset ${o} after fetch issue.`,{url:o,reason:g,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),me(r,n),true};n.asset(`Fetching asset ${o}`,{url:o});try{let g=Z(e,r.assetMap,o),d=g?await ot(g):null;if(g&&d)try{let v=await t.head(o,{timeout:6e4}),b=v.headers();if(v.ok()&&Pr(d,b)){let j=await Pe(g);if(j!==null){let T=v.url()||o,L=await ye(e,r.assetMap,T,j,r,n,b);L.wroteFile&&(O(u,e,L.outputPath),E(L.outputPath)&&X(u,L.outputPath)),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:g,lastModified:b["last-modified"]||"",contentLength:b["content-length"]||"",phase:"download-assets"}),me(r,n);let D=fe(a,b),ie=c?.discoveredAssets?[...c.discoveredAssets]:[];if(D||c?.isText){let ct=j.toString("utf8"),Ce=W(e,T,ct,n,"asset:cached");(Ce.length>0||!c)&&(ie=Ce);for(let dt of ie)x(e,r,dt,n,T);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:L.outputPath,isText:D||!!c?.isText,discoveredAssets:ie,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let p=await t.get(o,{timeout:6e4});if(!p.ok()){if(l(`http-${p.status()}`,{status:p.status()}))return;n.skip("asset",o,p.status());return}let f=await p.body(),w=p.url()||o,y=p.headers(),m=await ye(e,r.assetMap,w,f,r,n,y);m.wroteFile&&(O(u,e,m.outputPath),E(m.outputPath)&&X(u,m.outputPath)),r.doneAssets.add(o),me(r,n);let P=[];if(fe(a,y)){let v=f.toString("utf8");P=W(e,w,v,n,`asset:${h.extname(a.pathname).toLowerCase()||y["content-type"]||"unknown"}`);for(let b of P)x(e,r,b,n,w);}i.trackManifest&&(i.manifest.assets[o]={url:o,outputPath:m.outputPath,isText:fe(a,y),discoveredAssets:P,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){if(l("fetch-error",{error:String(g)}))return;n.error(`Failed asset ${o}`,{url:o,error:String(g)});}}async function Ar(e,t,r,s,n,i){let u=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),a=Array.from({length:u},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await xr(e,t,r,o,s,n,i));}});await Promise.all(a);}async function kr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Mr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await kr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Tr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Lr(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ve(t,r))}async function Ir(e,t,r,s,n=true){let i=new Set;if(n){let a=await t.evaluate(()=>{let o=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,g=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of c){let w=d.getAttribute(f);w&&o.add(w);}for(let f of g){let w=d.getAttribute(f);w&&w.split(",").forEach(y=>{let m=y.trim().split(/\s+/)[0];m&&o.add(m);});}let p=d.getAttribute("style");p&&l.add(p);}),{attrs:[...o],styles:[...l]}});for(let o of a.attrs)for(let c of vr(o)){let l=C(e,c,r,s,"dom-attr");l&&R(e,l)&&i.add(l.toString());}for(let o of a.styles)for(let c of W(e,r,o,s,"dom-style"))i.add(c);}let u=await t.content();for(let a of W(e,r,u,s,"page-html"))i.add(a);return [...i]}async function Be(e,t,r,s,n){let i=te(e,t);await ce(i),await S.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function ze(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ve(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Dr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Er(e,t,r,s,n,i,u){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Dr(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let g=new URL(l),d=Lr(e,g.pathname);if(!H(g)||R(e,g)||se(g)){if(R(e,g)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),x(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let p=await sr(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let m=i.manifest.pages[l];if(p.action==="reuse"&&m){for(let P of m.discoveredAssets)x(e,t,P,r,l);for(let P of m.discoveredPages)re(e,t,P,r,l);m.lastSeenRunId=i.runId,m.sitemapLastmod=Y(t.sitemapLastmodByPage[l])??m.sitemapLastmod,p.changeToken?.supported&&(m.changeToken=p.changeToken.token,m.tokenSource=p.changeToken.tokenSource??m.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:p.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),ze(t)&&Ve(t,r);continue}}let f=await(d?c.newPage():o.newPage()),w=!1,y=new Set;await f.route("**/*",async m=>{let P=m.request();if(P.isNavigationRequest()&&P.frame()===f.mainFrame())try{let v=new URL(P.url()),b=new URL(l);if(v.hash="",b.hash="",v.href===b.href){if(w){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:P.url()}),await m.abort("aborted");return}w=!0;}}catch{}await m.continue();}),f.on("response",m=>{try{let P=C(e,m.url(),e.sourceOrigin,void 0,"network-response");P&&R(e,P)&&(y.add(P.toString()),x(e,t,P.toString(),r,l));}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let m=null,P=null;try{if(m=await f.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{P=await m.text();}catch{P=null;}}catch(D){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(D)});}if(m&&!m.ok()){r.skip("page",l,m.status()),await f.close();continue}d?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Tr(e,f)):await Mr(e,f),t.stats.pagesRendered++,ze(t)&&Ve(t,r);let v=await Ir(e,f,l,r,!d);for(let D of v)x(e,t,D,r,l);let b=[...new Set([...v,...y])],j=P??await f.content(),T=s!=="single-url"?Cr(e,l,j,r):[];if(s!=="single-url")for(let D of T)re(e,t,D,r,l);await Be(e,l,j,t,r);let L=te(e,l);O(u,e,L),X(u,L),i.trackManifest&&(i.manifest.pages[l]={url:l,outputPath:L,changeToken:p.changeToken?.supported?p.changeToken.token:null,tokenSource:p.changeToken?.supported?p.changeToken.tokenSource??null:null,sitemapLastmod:Y(t.sitemapLastmodByPage[l])??null,discoveredPages:T,discoveredAssets:b,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let P=await f.content();if(P&&P.trim()){await Be(e,l,P,t,r);let v=te(e,l);O(u,e,v),X(u,v),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(m)});}}catch(P){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(P)});}r.error(`Failed page ${l}`,{url:l,error:String(m)});}finally{await f.close();}}}finally{await c.close(),await o.close(),await a.close();}}async function Fr(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ke(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Or(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function jr(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function lt(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ge(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function $r(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ge(s)-Ge(n)).map(s=>h.join(e,String(s.storedFileName))):[];return [...new Set([...r,h.join(e,"errors.jsonl"),h.join(e,"errors.json"),h.join(e,"errors.jsonl.gz"),h.join(e,"errors.json.gz")])]}async function Xe(e){let t=await lt(h.join(e,"job.json"));for(let r of $r(e,t)){let s=await Fr(r);if(s.length>0)return s}return []}async function Ur(e){let t=h.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let u=h.join(t,i);if(!(await S.stat(u).catch(()=>null))?.isDirectory())continue;let o=await lt(h.join(u,"job.json"));if(!Or(o))continue;let c=jr(o);c>=n&&(n=c,s=u);}return s}async function Qr(e){let t=await Ur(e),r=t!==""?await Xe(t):await Xe(e.logDir),s=new Set;for(let n of r)if(Ke(n.error)||Ke(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function qr(e){try{let t=await S.readFile(h.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function _r(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(h.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ye(e,t,r,s,n){let i=C(e,r,e.sourceOrigin,s,"manual-url");i&&(ne(i)?Re(e,t,i.toString(),s,n):R(e,i)?x(e,t,i.toString(),s,n):H(i)?re(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function Wr(){let e=await le(),t=Vt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await Qr(e)),t.mode==="single-url"&&(r=await Nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.mode==="full"&&!t.resumeRewrite,i=t.preserveOutput?await qr(e):{},u=s?await Zt(e):tt(),a={enabled:s,trackManifest:n,manifest:qe(u),previousManifest:qe(u),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await _r(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...i},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},c=Yt();t.preserveOutput||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let l=new q(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let w=await chromium.launch({headless:true}),y=await w.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),m=y.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(v=>Re(e,o,v,l)),await Je(e,m,o,l),e.seedPaths.forEach(v=>Ye(e,o,v,l,"seed-path"))):(r.forEach(v=>Ye(e,o,v,l,"cli")),await Je(e,m,o,l)),l.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),l.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let P=Array.from({length:Math.max(1,e.concurrency)},()=>Er(e,o,l,t.mode,m,a,c));if(await Promise.all(P),l.endMark("render-pages",{pages:o.donePages.size}),l.mark("download-assets"),await Ar(e,m,o,l,a,c),l.endMark("download-assets",{assets:o.doneAssets.size}),a.enabled&&e.maxPages===0&&await nr(e,o,a,l,c),t.mode==="full"){let v=await Rr(e,l);for(let b of v)O(c,e,b);}}finally{await y.close(),await w.close();}}a.trackManifest&&(a.manifest.updatedAt=new Date().toISOString(),await er(e,a.manifest)),await S.writeFile(h.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8"),JSON.stringify(i,null,2)!==JSON.stringify(o.assetMap,null,2)&&O(c,e,h.join(e.outputDir,"asset-map.json"));let g=t.mode==="single-url"?cr(e,o):a.enabled&&!t.resumeRewrite?ir(e,a,c,i,o.assetMap):void 0;l.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${g?.length??0}`:"Rewriting text files...";l.progress(d,{phase:"rewrite-text",index:0,totalFiles:g?.length,changedTextFiles:0}),await He(d);let p=Date.now(),f=await je(e,o.assetMap,{files:g,previousAssetMap:i,onProgress:async({index:w,totalFiles:y,changedTextFiles:m,file:P,changed:v})=>{v&&O(c,e,h.join(e.outputDir,P)),l.checkpoint(`Rewriting text file ${w}/${y}`,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:m,file:P});let b=Date.now();if(!(w===1||w===y||b-p>=5e3))return;p=b;let T=`Rewriting text files: ${w}/${y}`;l.progress(T,{phase:"rewrite-text",index:w,totalFiles:y,changedTextFiles:m,file:P}),await He(T);}});l.endMark("rewrite-text",{changedTextFiles:f}),await ke(e,{generatedAt:new Date().toISOString(),outputDir:h.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(a.enabled&&!t.resumeRewrite),changedFiles:a.enabled&&!t.resumeRewrite?[...c.changedFiles].sort():[],deletedFiles:a.enabled&&!t.resumeRewrite?[...c.deletedFiles].sort():[],rewriteTargets:a.enabled&&!t.resumeRewrite?[...new Set((g||[]).map(w=>J(e.outputDir,h.isAbsolute(w)?w:h.join(e.outputDir,w))).filter(w=>w!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${f} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:f}),await l.flush();}Wr().catch(async e=>{console.error(e);try{let r=(await le().catch(()=>null))?.logDir??"logs",s=new q(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Dt(e){let t=j(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function Et(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,G(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!r.length)return e;let s=i=>{let o=i;for(let[u,a]of r)o=o.split(u).join(a);return o},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,u,a)=>`${o}${s(u)}${a}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,u,a,c)=>Dt(a)?`${o}${s(a)}${c}`:i),n}function Ft(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function Ot(e,t,r){let s=Ft(t,r);if(!s)return e;let n=e;for(let i of kt){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");n=n.replace(u,`${s}${i}`).replace(a,`${s}${i}`);let c=i.replace(/\//g,"\\/"),l=s.replace(/\//g,"\\/"),p=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),m=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${l}${c}`).replace(g,`${l}${c}`).replace(m,`${l}${c}`);}return n}function ge(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:De(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${Ee(e.outputDir,r,n)}${i}`}function pe(e,t,r,s,n){K(e,r,ge(t,s,n));}function jt(e,t,r,s,n){if(!r||r===s)return;K(e,r,s);let i=ge(t,r,n),o=ge(t,s,n);K(e,i,o);}function X(e,t,r,s,n={}){let i=e,o={};pe(o,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,c]of Object.entries(t.extraReplacements))pe(o,t,a,c,s);for(let[a,c]of Object.entries(r))pe(o,t,a,c,s);for(let[a,c]of Object.entries(n)){let l=r[a];!l||l===c||jt(o,t,c,l,s);}let u=Object.entries(o).sort((a,c)=>c[0].length-a[0].length);for(let[a,c]of u)i=i.split(a).join(c);return t.urlRewriteMode!=="absolute"&&(i=Ot(i,t,s)),i=Et(i,u.map(([,a])=>a)),Lt(t,i,s)&&(i=It(i)),i}async function $t(e,t,r,s){let n=f.join(e.outputDir,s),i=f.extname(n).toLowerCase(),o;try{o=await S.readFile(n,"utf8");}catch(c){if(c.code!=="ENOENT")throw c;return {changed:false}}let a=X(o,e,t,i===".js"||i===".mjs"?void 0:n,r);return a!==o?(await S.writeFile(n,a,"utf8"),{changed:true}):{changed:false}}async function Ut(e,t){return await new Promise((r,s)=>{let n=a=>{if(u(),a?.error){s(new Error(a.error));return}r({changed:!!a?.changed});},i=a=>{u(),s(a);},o=a=>{u(),a!==0&&s(new Error(`Rewrite worker exited with code ${a}.`));},u=()=>{e.off("message",n),e.off("error",i),e.off("exit",o);};e.on("message",n),e.on("error",i),e.on("exit",o),e.postMessage({file:t});})}async function je(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>f.resolve(d)).filter(d=>d.startsWith(f.resolve(e.outputDir))).map(d=>f.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Ct(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&O(d)),i=0;if(n.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),u=Math.max(1,Math.min(o,n.length,availableParallelism())),a=0,c=0,l=null;if(u===1){for(let d of n){let g=await $t(e,t,r.previousAssetMap??{},d);g.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:d,changed:g.changed});}return i}let p=Array.from({length:u},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t,previousAssetMap:r.previousAssetMap??{}}});try{for(;!l;){let g=a;if(g>=n.length)return;a+=1;let m=n[g],h=await Ut(d,m);h.changed&&(i+=1),c+=1,await r.onProgress?.({index:c,totalFiles:n.length,changedTextFiles:i,file:m,changed:h.changed});}}catch(g){throw l=g instanceof Error?g:new Error(String(g)),l}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var N={error:0,warn:1,info:2,debug:3},$e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",Z=$e?f.join($e,"current-progress.json"):"",Ue=Promise.resolve();function Qe(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function Qt(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let o=r[n];if(typeof o=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(o,i);continue}r[n]=i;}return r}function qt(e,t,r){Z&&(Ue=Ue.then(async()=>{let s=new Date,n=s.toISOString();await S.mkdir(f.dirname(Z),{recursive:true});let i=await S.readFile(Z,"utf8").then(h=>JSON.parse(h)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},u=Qe(e),a=i?.currentStep||Qe(i?.source||""),c=i?.startedAt||n,l={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(a&&a!==u&&i?.stepStartedAt){let h=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));l[a]=(l[a]??0)+h,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(c).getTime())/1e3)),m={checkedAt:n,source:e,message:t,details:Qt(o,r??{}),startedAt:c,currentStep:u,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:l};await S.writeFile(Z,JSON.stringify(m,null,2),"utf8");}).catch(()=>{}));}async function Nt(e,t){await S.mkdir(f.dirname(e),{recursive:true}),await S.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function _t(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function Wt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:_t(e,r),level:e,message:t,details:r??{}}}var _=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await S.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>Nt(t,r));}updateCurrentEvent(t,r,s){let n=Wt(t,r,s);this.enqueueTask(async()=>{await S.mkdir(this.logDir,{recursive:true}),await S.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(N[this.level]>=N.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){qt("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),N[this.level]>=N.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),N[this.level]>=N.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var Jt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],zt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Bt=new Set(["",".html",".htm"]),Vt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),Gt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Kt(e){let t=[],r="full",s="full",n=false,i=false;for(let o=0;o<e.length;o++){let u=e[o];if(u==="--retry-timeouts")r="retry-timeouts",n=true;else if(u==="--resume")n=true;else if(u==="--resume-rewrite")n=true,i=true;else if(u==="--crawl-mode"){let a=e[++o];if(!a)throw new Error("--crawl-mode requires a value");s=a==="incremental"?"incremental":"full",n||=s==="incremental";}else if(u.startsWith("--crawl-mode="))s=u.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(u==="--incremental")s="incremental",n=true;else if(u==="--url"){let a=e[++o];if(!a)throw new Error("--url requires a value");r="single-url",n=true,t.push(a);}else if(u.startsWith("--url="))r="single-url",n=true,t.push(u.slice(6));else if(u==="--urls"){let a=e[++o];if(!a)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${a}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function Se(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function Xt(){let e=Se();return e?f.resolve(e,".."):""}function Zt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function Yt(e){let t=e.trim(),r=[{alias:"@storage-root",root:Xt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:Se(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Zt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function et(e){let t=Se();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}function tt(){return {schemaVersion:2,updatedAt:"",pages:{},assets:{}}}function er(){return {rewriteTargets:new Set,changedFiles:new Set,deletedFiles:new Set}}function $(e,t,r){let s=V(t.outputDir,r);s&&(e.changedFiles.add(s),e.deletedFiles.delete(s));}function qe(e,t,r){let s=V(t.outputDir,r);s&&(e.deletedFiles.add(s),e.changedFiles.delete(s));}function ee(e,t){e.rewriteTargets.add(f.resolve(t));}async function tr(e){try{let t=await S.readFile(et(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.pages&&typeof r.pages=="object"){if(r.schemaVersion===2&&r.assets&&typeof r.assets=="object")return r;if(r.schemaVersion===1)return {schemaVersion:2,updatedAt:String(r.updatedAt||""),pages:r.pages,assets:{}}}}catch{}return tt()}function Ne(e){return JSON.parse(JSON.stringify(e))}async function rr(e,t){let r=et(e);await S.mkdir(f.dirname(r),{recursive:true}),await S.writeFile(r,JSON.stringify(t,null,2),"utf8");}function te(e){if(!e)return null;let t=e.trim();return t||null}function sr(e,t){let r=new Set,s=new Set([...Object.keys(e),...Object.keys(t)]);for(let n of s){let i=e[n],o=t[n];!i||!o||i===o||r.add(n);}return r}async function nr(e,t,r,s){let n=[...new Set(r.map(u=>u.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let o=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let u=await t.post(o,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!u.ok())return s.warn(`Change token lookup failed with HTTP ${u.status()}`,{endpoint:o,status:u.status()}),new Map;let a=await u.json().catch(()=>null);if(!a||!Array.isArray(a.items))return new Map;let c=new Map;for(let l of a.items)!l||typeof l.url!="string"||c.set(l.url,l);return c}catch(u){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:o,error:String(u)}),new Map}}async function he(e,t,r,s,n){if(!r.enabled&&!r.trackManifest)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let o=(await nr(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,o),o}async function ir(e,t,r,s,n,i){if(at(e,n))return {action:"render",changeToken:null};if(!s.enabled)return {action:"render",changeToken:s.trackManifest?await he(e,t,s,n,i):null};let o=s.manifest.pages[n];if(!o)return {action:"render",changeToken:await he(e,t,s,n,i)};let u=await he(e,t,s,n,i);if(u?.supported&&u.token)return o.changeToken===u.token?{action:"reuse",changeToken:u}:{action:"render",changeToken:u};let a=te(r.sitemapLastmodByPage[n]);return a&&o.sitemapLastmod&&o.sitemapLastmod===a?{action:"reuse",changeToken:u}:{action:"render",changeToken:u}}async function ar(e,t,r,s,n){let i=_e(r.previousManifest),o=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){qe(n,e,g.outputPath);try{await S.unlink(g.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(m)});}delete r.manifest.pages[d],o++;}let u=_e(r.manifest,r.runId),a=new Set,c=new Set;for(let d of u){for(let h of H(e,d))a.add(h);let g=r.manifest.assets[d]?.outputPath;g&&c.add(g);let m=re(e,t.assetMap,d);m&&c.add(m);}for(let d of Object.keys(r.manifest.assets))u.has(d)||delete r.manifest.assets[d];let l=0,p=0;for(let d of i){let g=new Set,m=r.previousManifest.assets[d]?.outputPath;m&&g.add(m);let h=re(e,t.assetMap,d);if(!u.has(d)&&h)g.add(h);else if(!u.has(d)){let y=be(e,d);g.add(y.originalFilePath),y.hashedFilePath&&g.add(y.hashedFilePath);}for(let y of g)if(!c.has(y)){qe(n,e,y);try{await S.unlink(y),l++;}catch(v){(v.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:y,error:String(v)});}}for(let y of H(e,d))a.has(y)||Object.prototype.hasOwnProperty.call(t.assetMap,y)&&(delete t.assetMap[y],p++);}(o>0||l>0||p>0)&&s.info(`Incremental cleanup removed ${o} stale pages, ${l} stale assets, ${p} stale asset mappings`,{removedPages:o,removedAssets:l,removedAssetMappings:p,phase:"incremental-cleanup"});}function _e(e,t){let r=new Set;for(let s of Object.values(e.pages))for(let n of s.discoveredAssets||[]){let i=n.trim();i&&r.add(i);}for(let[s,n]of Object.entries(e.assets||{})){if(t&&n.lastSeenRunId!==t)continue;let i=s.trim();i&&r.add(i);}return r}function We(e,t,r){for(let s of t)for(let n of H(e,s))if(r.has(n))return true;return false}function or(e,t,r,s,n){let i=new Set(r.rewriteTargets),o=sr(s,n);if(o.size===0)return [...i];for(let u of Object.values(t.manifest.pages))We(e,u.discoveredAssets,o)&&i.add(f.resolve(u.outputPath));for(let[u,a]of Object.entries(t.manifest.assets)){a.isText&&We(e,a.discoveredAssets,o)&&i.add(f.resolve(a.outputPath));for(let c of H(e,u))if(o.has(c)){$(r,e,a.outputPath),a.isText&&i.add(f.resolve(a.outputPath));break}}return [...i]}function H(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function ur(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search);}catch{}return [...r]}function rt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function lr(e,t,r){for(let s of ur(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return {outputPath:i,publicUrl:n}}return null}function cr(e,t,r){for(let s of H(e,r)){let n=t[s];if(!n)continue;let i=rt(e,n);if(i)return i}return null}function re(e,t,r){let s=cr(e,t,r);return s||be(e,r).originalFilePath}function dr(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function He(e){let t=dr();if(t)try{let r=await S.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await S.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function pr(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(W(e,s)));for(let s of t.doneAssets){let n=re(e,t.assetMap,s);!n||!O(n)||r.add(f.resolve(n));}return [...r]}var se="re:",Y=new Map;function gr(e){if(!e.startsWith(se))return null;let t=Y.get(e);if(t!==void 0)return t||null;let r=e.slice(se.length).trim();if(!r)return Y.set(e,false),null;try{let s=new RegExp(r);return Y.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Y.set(e,false),null}}function ve(e,t){let r=gr(t);return r?r.test(e):t.startsWith(se)?false:e.startsWith(t)}function hr(e,t){return t.find(r=>ve(e,r))}function st(e,t){let r=hr(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(se)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function fr(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function mr(e,t){return fr(e).has(t.hostname)}function wr(e){let t=j(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function nt(e){let t=j(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function ne(e){return Gt.some(t=>e.pathname.startsWith(t))}function x(e,t,r=e.sourceOrigin,s,n="url"){let i=wr(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(nt(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let o=new URL(i,r);if(!["http:","https:"].includes(o.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!mr(e,o))return s?.ignore(n,o.toString(),"host not allowed",r),null;let u=st(e,o);if(u)return s?.reject(n,o.toString(),u,r),null;let a=new URL(e.sourceOrigin);return o.protocol=a.protocol,o.host=a.host,o.hash="",o}catch{return s?.reject(n,t,"invalid URL",r),null}}function it(e){return zt.has(f.extname(e.pathname).toLowerCase())}function B(e){return ne(e)?false:Bt.has(f.extname(e.pathname).toLowerCase())}function ie(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function R(e,t){return ie(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:it(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>ve(t.pathname,r)):false}function W(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function Pr(e){if(!e.generated404RequestPath)return null;try{return new URL(e.generated404RequestPath,e.sourceOrigin).toString()}catch{return null}}function at(e,t){let r=Pr(e);if(!r)return false;try{return new URL(t,e.sourceOrigin).toString()===r}catch{return false}}function be(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,o=f.join(e.outputDir,i),u=de(e.targetOrigin,i),a=`${u}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:o,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:n};let c=f.extname(s),p=`${c?s.slice(0,-c.length):s}.${Ie(r.search)}${c||".bin"}`;return {originalPathname:i,originalFilePath:o,originalPublicUrl:u,originalPublicUrlWithSearch:a,preferQueryHashed:n,hashedPathname:p,hashedFilePath:f.join(e.outputDir,p),hashedPublicUrl:de(e.targetOrigin,p)}}function yr(e,t,r,s,n){t[r]=s;try{let i=new URL(r),u=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{u.push(new URL(e.targetOrigin).origin);}catch{}for(let a of u)try{let c=new URL(a);t[c.origin+i.pathname+i.search]=s;}catch{}try{let a=(()=>{let c=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${c.pathname}${c.search}`})();t[i.pathname+i.search]=a;}catch{}try{let a=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}try{let a=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=a;}catch{}}catch{}}async function Pe(e){try{return await S.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function ot(e){try{return await S.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Sr(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function ut(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function vr(e,t){let r=ut(t),s=Sr(t),n=e?.mtime.getTime();return r!==null&&Math.abs((n??0)-r)>=1e3||s!==null&&e?.size!==s?false:r!==null||s!==null}async function br(e,t){if(!t)return;let r=ut(t);if(r===null)return;let s=await ot(e);if(!s)return;let n=new Date(r);await S.utimes(e,s.atime,n).catch(()=>{});}function fe(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return Vt.has(r)||Jt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function me(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ye(e,t,r,s,n,i,o){let u=be(e,r),a=u.originalFilePath,c=u.originalPublicUrlWithSearch;if(u.preferQueryHashed&&u.hashedFilePath&&u.hashedPublicUrl){let g=lr(e,t,r);if(g)a=g.outputPath,c=g.publicUrl;else {let m=await Pe(u.originalFilePath);if(m!==null&&!m.equals(s)){let h=O(u.originalFilePath)?Buffer.from(X(s.toString("utf8"),e,t,u.originalFilePath),"utf8"):null;(h===null||!m.equals(h))&&(a=u.hashedFilePath,c=u.hashedPublicUrl);}}}yr(e,t,r,c,u.originalPublicUrl);let l=O(a)?Buffer.from(X(s.toString("utf8"),e,t,a),"utf8"):s,p=await Pe(a),d=false;return (p===null||!p.equals(l))&&(await ce(a),await S.writeFile(a,l),d=true),await br(a,o),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"}),{outputPath:a,publicUrl:c,wroteFile:d}}function J(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=x(e,r,i,s,"page");if(!o)return;if(!B(o)||it(o)){if(R(e,o)){s.info(`Seeded/non-page URL queued as asset: ${o.toString()}`,{url:o.toString(),source:n}),A(e,t,o.toString(),s,n);return}s.reject("page",o.toString(),ne(o)?"asset/internal path cannot be page":"not page-like",n);return}let u=o.toString();!t.donePages.has(u)&&!t.queuedPages.has(u)&&(t.queuedPages.add(u),t.pageQueue.push(u),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function A(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=x(e,r,i,s,"asset");if(!o)return;if(!R(e,o)){s.reject("asset",o.toString(),"not a safe asset path/prefix",n);return}let u=o.toString();!t.doneAssets.has(u)&&!t.queuedAssets.has(u)&&(t.queuedAssets.add(u),t.assetQueue.push(u),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function Re(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=x(e,r,i,s,"sitemap");if(!o)return;if(!ie(o)){s.reject("sitemap",o.toString(),"not sitemap-like",n);return}let u=o.toString();!t.doneSitemaps.has(u)&&!t.queuedSitemaps.has(u)&&(t.queuedSitemaps.add(u),t.sitemapQueue.push(u),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Rr(e){let t=j(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:te(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Cr(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function xr(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function we(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function lt(e,t){let r=[];await S.mkdir(t,{recursive:true});let s=await S.readdir(e,{withFileTypes:true});for(let n of s){let i=f.join(e,n.name),o=f.join(t,n.name);if(n.isDirectory()){r.push(...await lt(i,o));continue}await S.copyFile(i,o),r.push(o);}return r}async function Ar(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([i,o])=>({sourcePath:i.trim(),prefix:String(o||"").trim()}));if(r.length===0)return [];let s=0,n=[];t.mark("copy-extra-paths");for(let i of r){let o=i.sourcePath,u=i.prefix;if(!o||!u){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:o,prefixPath:u});continue}let a=Yt(o);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:o,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:u});continue}let c=a.resolvedPath,l=we(u);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:o,prefixPath:u});continue}let p;try{p=await S.stat(c);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:o,sourceAbs:c,prefixPath:u});continue}let d=f.resolve(e.outputDir,l);if(p.isDirectory()){n.push(...await lt(c,d)),s++,t.progress(`Copied static directory to export: ${c} -> /${l}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${l}`,copiedItems:s});continue}let g=f.basename(c),m=u.endsWith("/")?f.join(d,g):d;await S.mkdir(f.dirname(m),{recursive:true}),await S.copyFile(c,m),n.push(m),s++,t.progress(`Copied static file to export: ${c} -> /${we(u.endsWith("/")?`${u}${g}`:u)}`,{phase:"copy-extra-paths",sourcePath:c,targetPath:`/${we(u.endsWith("/")?`${u}${g}`:u)}`,copiedItems:s});}return t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s}),n}function z(e,t,r,s,n){let i=new Set,o=j(r).replace(/\\\//g,"/");for(let c of xr(o)){let l=x(e,c,t,s,`${n}:css-url`);l&&R(e,l)&&i.add(l.toString());}for(let c of o.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let l=x(e,c[1],t,s,`${n}:css-import`);l&&R(e,l)&&i.add(l.toString());}for(let c of o.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let l=x(e,c[1],t,s,`${n}:xml-stylesheet`);l&&R(e,l)&&i.add(l.toString());}let u="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",a=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${u}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${u}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let c of o.matchAll(a)){let l=c[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${c[0]}`:c[0];if(nt(l)){s.ignore(`${n}:serialized-url`,l,"looks like JavaScript/code fragment",t);continue}let p=x(e,l,t,s,`${n}:serialized-url`);p&&R(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function kr(e,t,r,s){let n=new Set,i=j(r);for(let o of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let u=o[1],a=x(e,u,t,s,"page-link");a&&(st(e,a)||B(a)&&!ne(a)&&!R(e,a)&&n.add(a.toString()));}return [...n]}async function Je(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let o=await i.body(),u=i.url()||n;await ye(e,r.assetMap,u,o,r,s);let a=o.toString("utf8");for(let c of z(e,u,a,s,"sitemap"))A(e,r,c,s,u);for(let c of Rr(a)){let l=x(e,c.loc,u,s,"sitemap-loc");l&&(ie(l)?Re(e,r,l.toString(),s,u):R(e,l)?A(e,r,l.toString(),s,u):B(l)?(c.lastmod&&(r.sitemapLastmodByPage[l.toString()]=c.lastmod),J(e,r,l.toString(),s,u)):s.reject("sitemap-loc",l.toString(),"not page/sitemap/asset-like",u));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Mr(e,t,r,s,n,i,o){let u=x(e,s,e.sourceOrigin,n,"asset-fetch");if(!u||!R(e,u))return;let a=u.toString();if(r.doneAssets.has(a))return;let c=i.enabled?i.previousManifest.assets[a]:void 0,l=(p,d={})=>{if(!c)return false;i.manifest.assets[a]={...c,discoveredAssets:[...c.discoveredAssets||[]],lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId},r.doneAssets.add(a);for(let g of c.discoveredAssets||[])A(e,r,g,n,a);return n.warn(`Keeping previous incremental asset ${a} after fetch issue.`,{url:a,reason:p,preservedFromPreviousManifest:true,...d,phase:"download-assets"}),me(r,n),true};n.asset(`Fetching asset ${a}`,{url:a});try{let p=re(e,r.assetMap,a),d=p?await ot(p):null;if(p&&d)try{let w=await t.head(a,{timeout:6e4}),b=w.headers();if(w.ok()&&vr(d,b)){let L=await Pe(p);if(L!==null){let I=w.url()||a,E=await ye(e,r.assetMap,I,L,r,n,b);E.wroteFile&&($(o,e,E.outputPath),O(E.outputPath)&&ee(o,E.outputPath)),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:p,lastModified:b["last-modified"]||"",contentLength:b["content-length"]||"",phase:"download-assets"}),me(r,n);let U=fe(u,b),C=c?.discoveredAssets?[...c.discoveredAssets]:[];if(U||c?.isText){let dt=L.toString("utf8"),Ce=z(e,I,dt,n,"asset:cached");(Ce.length>0||!c)&&(C=Ce);for(let pt of C)A(e,r,pt,n,I);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:E.outputPath,isText:U||!!c?.isText,discoveredAssets:C,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});return}}}catch{}let g=await t.get(a,{timeout:6e4});if(!g.ok()){if(l(`http-${g.status()}`,{status:g.status()}))return;n.skip("asset",a,g.status());return}let m=await g.body(),h=g.url()||a,y=g.headers(),v=await ye(e,r.assetMap,h,m,r,n,y);v.wroteFile&&($(o,e,v.outputPath),O(v.outputPath)&&ee(o,v.outputPath)),r.doneAssets.add(a),me(r,n);let P=[];if(fe(u,y)){let w=m.toString("utf8");P=z(e,h,w,n,`asset:${f.extname(u.pathname).toLowerCase()||y["content-type"]||"unknown"}`);for(let b of P)A(e,r,b,n,h);}i.trackManifest&&(i.manifest.assets[a]={url:a,outputPath:v.outputPath,isText:fe(u,y),discoveredAssets:P,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(p){if(l("fetch-error",{error:String(p)}))return;n.error(`Failed asset ${a}`,{url:a,error:String(p)});}}async function Tr(e,t,r,s,n,i){let o=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),u=Array.from({length:o},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Mr(e,t,r,a,s,n,i));}});await Promise.all(u);}async function Lr(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Ir(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Lr(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Dr(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Er(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>ve(t,r))}async function Fr(e,t,r,s,n=true){let i=new Set;if(n){let u=await t.evaluate(()=>{let a=new Set,c=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],l=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let m of c){let h=d.getAttribute(m);h&&a.add(h);}for(let m of p){let h=d.getAttribute(m);h&&h.split(",").forEach(y=>{let v=y.trim().split(/\s+/)[0];v&&a.add(v);});}let g=d.getAttribute("style");g&&l.add(g);}),{attrs:[...a],styles:[...l]}});for(let a of u.attrs)for(let c of Cr(a)){let l=x(e,c,r,s,"dom-attr");l&&R(e,l)&&i.add(l.toString());}for(let a of u.styles)for(let c of z(e,r,a,s,"dom-style"))i.add(c);}let o=await t.content();for(let u of z(e,r,o,s,"page-html"))i.add(u);return [...i]}async function ze(e,t,r,s,n){let i=W(e,t);await ce(i),await S.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Be(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Ve(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function Or(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function jr(e,t,r,s,n,i,o){let u=await chromium.launch({headless:true}),a=await u.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=await u.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Or(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let p=new URL(l),d=at(e,l),g=Er(e,p.pathname);if(!B(p)||R(e,p)||ne(p)){if(R(e,p)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),A(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let m=await ir(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let P=i.manifest.pages[l];if(m.action==="reuse"&&P){for(let w of P.discoveredAssets)A(e,t,w,r,l);for(let w of P.discoveredPages)J(e,t,w,r,l);P.lastSeenRunId=i.runId,P.sitemapLastmod=te(t.sitemapLastmodByPage[l])??P.sitemapLastmod,m.changeToken?.supported&&(P.changeToken=m.changeToken.token,P.tokenSource=m.changeToken.tokenSource??P.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:m.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Be(t)&&Ve(t,r);continue}}let h=await(g?c.newPage():a.newPage()),y=!1,v=new Set;await h.route("**/*",async P=>{let w=P.request();if(w.isNavigationRequest()&&w.frame()===h.mainFrame())try{let b=new URL(w.url()),L=new URL(l);if(b.hash="",L.hash="",b.href===L.href){if(y){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:w.url()}),await P.abort("aborted");return}y=!0;}}catch{}await P.continue();}),h.on("response",P=>{try{let w=x(e,P.url(),e.sourceOrigin,void 0,"network-response");w&&R(e,w)&&(v.add(w.toString()),A(e,t,w.toString(),r,l));}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let P=null,w=null;try{if(P=await h.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),P&&P.ok())try{w=await P.text();}catch{w=null;}}catch(C){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(C)});}if(d){let C=P?.status()??null;if(C!==404){r.error(`Generated 404 request path returned unexpected status for ${l}`,{url:l,expectedStatus:404,status:C,requestPath:e.generated404RequestPath});continue}r.info(`Capturing rendered 404 page for ${l}`,{url:l,status:C,outputPath:W(e,l)});}else if(P&&!P.ok()){r.skip("page",l,P.status()),await h.close();continue}g?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Dr(e,h)):await Ir(e,h),t.stats.pagesRendered++,Be(t)&&Ve(t,r);let b=await Fr(e,h,l,r,!g);for(let C of b)A(e,t,C,r,l);let L=[...new Set([...b,...v])],I=w??await h.content(),E=s!=="single-url"&&!d?kr(e,l,I,r):[];if(s!=="single-url"&&!d)for(let C of E)J(e,t,C,r,l);await ze(e,l,I,t,r);let U=W(e,l);$(o,e,U),ee(o,U),i.trackManifest&&(i.manifest.pages[l]={url:l,outputPath:U,changeToken:m.changeToken?.supported?m.changeToken.token:null,tokenSource:m.changeToken?.supported?m.changeToken.tokenSource??null:null,sitemapLastmod:te(t.sitemapLastmodByPage[l])??null,discoveredPages:E,discoveredAssets:L,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(P){try{let w=await h.content();if(w&&w.trim()){await ze(e,l,w,t,r);let b=W(e,l);$(o,e,b),ee(o,b),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(P)});}}catch(w){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(w)});}r.error(`Failed page ${l}`,{url:l,error:String(P)});}finally{await h.close();}}}finally{await c.close(),await a.close(),await u.close();}}async function $r(e){try{let t=await S.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Ge(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Ur(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Qr(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function ct(e){try{let t=await S.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ke(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function qr(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ke(s)-Ke(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function Xe(e){let t=await ct(f.join(e,"job.json"));for(let r of qr(e,t)){let s=await $r(r);if(s.length>0)return s}return []}async function Nr(e){let t=f.join(e.logDir,"archive"),r;try{r=await S.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let o=f.join(t,i);if(!(await S.stat(o).catch(()=>null))?.isDirectory())continue;let a=await ct(f.join(o,"job.json"));if(!Ur(a))continue;let c=Qr(a);c>=n&&(n=c,s=o);}return s}async function _r(e){let t=await Nr(e),r=t!==""?await Xe(t):await Xe(e.logDir),s=new Set;for(let n of r)if(Ge(n.error)||Ge(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function Wr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await S.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function Hr(e){try{let t=await S.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function Jr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await S.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await S.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ze(e,t,r,s,n){let i=x(e,r,e.sourceOrigin,s,"manual-url");i&&(ie(i)?Re(e,t,i.toString(),s,n):R(e,i)?A(e,t,i.toString(),s,n):B(i)?J(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function zr(){let e=await le(),t=Kt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await _r(e)),t.mode==="single-url"&&(r=await Wr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=t.mode==="full"&&!t.resumeRewrite,i=t.preserveOutput?await Hr(e):{},o=s?await tr(e):tt(),u={enabled:s,trackManifest:n,manifest:Ne(o),previousManifest:Ne(o),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await Jr(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:{...i},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}},c=er();t.preserveOutput||await S.rm(e.outputDir,{recursive:true,force:true}),await S.mkdir(e.outputDir,{recursive:true});let l=new _(e.logDir,e.logLevel);if(t.resumeRewrite)l.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let h=await chromium.launch({headless:true}),y=await h.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),v=y.request;try{if(l.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(w=>Re(e,a,w,l)),await Je(e,v,a,l),e.seedPaths.forEach(w=>Ze(e,a,w,l,"seed-path")),e.generated404RequestPath&&J(e,a,e.generated404RequestPath,l,"generated-404-request-path")):(r.forEach(w=>Ze(e,a,w,l,"cli")),await Je(e,v,a,l)),l.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),l.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){l.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await l.flush();return}l.mark("render-pages");let P=Array.from({length:Math.max(1,e.concurrency)},()=>jr(e,a,l,t.mode,v,u,c));if(await Promise.all(P),l.endMark("render-pages",{pages:a.donePages.size}),l.mark("download-assets"),await Tr(e,v,a,l,u,c),l.endMark("download-assets",{assets:a.doneAssets.size}),u.enabled&&e.maxPages===0&&await ar(e,a,u,l,c),t.mode==="full"){let w=await Ar(e,l);for(let b of w)$(c,e,b);}}finally{await y.close(),await h.close();}}u.trackManifest&&(u.manifest.updatedAt=new Date().toISOString(),await rr(e,u.manifest)),await S.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8"),JSON.stringify(i,null,2)!==JSON.stringify(a.assetMap,null,2)&&$(c,e,f.join(e.outputDir,"asset-map.json"));let p=t.mode==="single-url"?pr(e,a):u.enabled&&!t.resumeRewrite?or(e,u,c,i,a.assetMap):void 0;l.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${p?.length??0}`:"Rewriting text files...";l.progress(d,{phase:"rewrite-text",index:0,totalFiles:p?.length,changedTextFiles:0}),await He(d);let g=Date.now(),m=await je(e,a.assetMap,{files:p,previousAssetMap:i,onProgress:async({index:h,totalFiles:y,changedTextFiles:v,file:P,changed:w})=>{w&&$(c,e,f.join(e.outputDir,P)),l.checkpoint(`Rewriting text file ${h}/${y}`,{phase:"rewrite-text",index:h,totalFiles:y,changedTextFiles:v,file:P});let b=Date.now();if(!(h===1||h===y||b-g>=5e3))return;g=b;let I=`Rewriting text files: ${h}/${y}`;l.progress(I,{phase:"rewrite-text",index:h,totalFiles:y,changedTextFiles:v,file:P}),await He(I);}});l.endMark("rewrite-text",{changedTextFiles:m}),await ke(e,{generatedAt:new Date().toISOString(),outputDir:f.resolve(e.outputDir),runMode:t.mode,crawlMode:t.crawlMode,fullSyncRequired:!(u.enabled&&!t.resumeRewrite),changedFiles:u.enabled&&!t.resumeRewrite?[...c.changedFiles].sort():[],deletedFiles:u.enabled&&!t.resumeRewrite?[...c.deletedFiles].sort():[],rewriteTargets:u.enabled&&!t.resumeRewrite?[...new Set((p||[]).map(h=>V(e.outputDir,f.isAbsolute(h)?h:f.join(e.outputDir,h))).filter(h=>h!==null))].sort():[]}),l.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${m} text files.`:`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:m}),await l.flush();}zr().catch(async e=>{console.error(e);try{let r=(await le().catch(()=>null))?.logDir??"logs",s=new _(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/dist/deploy.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {S3Client,ListObjectsV2Command,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {createHash}from'crypto';import
|
|
1
|
+
import {S3Client,ListObjectsV2Command,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {createHash}from'crypto';import nt from'fast-glob';import $ from'fs/promises';import S from'path';import $t from'mime-types';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ge(t){let r=t.trim();return r==="."?".":r.replace(/\/$/,"")}function Je(t){let r=String(t??"").trim();if(!r)return "";let e=r;if(/^https?:\/\//i.test(e))try{e=new URL(e).pathname;}catch{return ""}if(e=e.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!e)return "";let n=e.endsWith("/"),s=e.replace(/^\/+/,"").split("/").map(o=>o.replace(/[^A-Za-z0-9._-]/g,"")).filter(o=>o.length>0&&o!=="."&&o!=="..");if(s.length===0)return "";let i=`/${s.join("/")}`;return n&&S.extname(i)===""&&(i+="/"),i==="/"?"":i}function Q(t){return !t||typeof t!="object"?{}:Object.fromEntries(Object.entries(t).map(([r,e])=>[r.trim(),String(e??"")]).filter(([r])=>r.length>0))}function Ke(t){if(!t||typeof t!="object")return {};let r={};for(let[e,n]of Object.entries(t)){let s=e.trim();if(!s||!n||typeof n!="object")continue;let i=n,o={};if(typeof i.targetOrigin=="string"){let a=ge(i.targetOrigin);a&&(o.targetOrigin=a);}let u=Q(i.extraReplacements);if(Object.keys(u).length>0&&(o.extraReplacements=u),i.s3&&typeof i.s3=="object"){let a={},l=i.s3;typeof l.bucket=="string"&&(a.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(a.prefix=l.prefix.trim()),typeof l.region=="string"&&(a.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(a.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(a.assetCacheControl=l.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(i.cloudFront&&typeof i.cloudFront=="object"){let a={},l=i.cloudFront;typeof l.distributionId=="string"&&(a.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(a.invalidationPaths=l.invalidationPaths.map(g=>String(g??"").trim()).filter(g=>g.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}r[s]=o;}return r}function V(t,r){let n=String(t||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return r;let s=n.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return s.length>0?s.join("/"):r}function Be(){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return t.trim()?S.resolve(t):""}function ue(t,r,e){let n=String(r||"").trim();return n&&S.isAbsolute(n)?S.resolve(n):S.resolve(t,V(n,e))}async function G(){let t=process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await $.readFile(t,"utf8"),e=JSON.parse(r);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=ge(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=Je(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=Q(e.extraReplacements),e.postCrawlCopyMap=Q(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.scheduler||={enabled:false,timezone:"UTC",rules:[]},e.scheduler.enabled??=false,e.scheduler.timezone||="UTC",e.scheduler.rules||=[],e.deploymentProfiles=Ke(e.deploymentProfiles),e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.defaultDeploymentProfile=String(e.defaultDeploymentProfile??"").trim(),e.defaultDeploymentProfile&&!e.deploymentProfiles[e.defaultDeploymentProfile]&&(e.defaultDeploymentProfile="");let n=Be(),s=n?S.resolve(n,".."):"";return s?(e.outputDir=ue(s,e.outputDir,"export"),e.logDir=ue(s,e.logDir,"logs")):(S.isAbsolute(e.outputDir)||(e.outputDir=V(e.outputDir,"export")),S.isAbsolute(e.logDir)||(e.logDir=V(e.logDir,"logs"))),e}var qe=".deploy-plan.json",We="deploy-plan.json";function Qe(){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return t.trim()?S.resolve(t):""}function Ve(t){let r=String(t||"").replace(/\\/g,"/").replace(/^\/+/,"").trim();return !r||r==="."||r.startsWith("../")||r===".."?null:r}function Z(t){return [...new Set(t.map(Ve).filter(r=>r!==null))]}function Ge(t,r){return {schemaVersion:1,generatedAt:String(r.generatedAt||new Date().toISOString()),outputDir:S.resolve(t.outputDir),runMode:r.runMode,crawlMode:r.crawlMode,fullSyncRequired:!!r.fullSyncRequired,changedFiles:Z(r.changedFiles||[]),deletedFiles:Z(r.deletedFiles||[]),rewriteTargets:Z(r.rewriteTargets||[])}}function Ze(t){if(!t||typeof t!="object")return false;let r=t;return r.schemaVersion===1&&typeof r.generatedAt=="string"&&typeof r.outputDir=="string"&&(r.runMode==="full"||r.runMode==="retry-timeouts"||r.runMode==="single-url")&&(r.crawlMode==="full"||r.crawlMode==="incremental")&&typeof r.fullSyncRequired=="boolean"&&Array.isArray(r.changedFiles)&&Array.isArray(r.deletedFiles)&&Array.isArray(r.rewriteTargets)}function pe(t){let r=S.join(S.resolve(t.outputDir),qe),e=Qe();return e?[S.join(e,We),r]:[r]}async function de(t){let r=S.resolve(t.outputDir);for(let e of pe(t))try{let n=await $.readFile(e,"utf8"),s=JSON.parse(n);if(!Ze(s)||S.resolve(s.outputDir)!==r)continue;return Ge(t,s)}catch{}return null}async function fe(t){for(let r of pe(t))try{await $.unlink(r);}catch(e){if(e.code!=="ENOENT")throw e}}function me(t){let r="";for(let n=0;n<t.length;n++){let s=t[n];if(s==="--profile"){let i=(t[n+1]||"").trim();if(!i)throw new Error("Missing value for --profile.");r=i,n+=1;}else if(s.startsWith("--profile=")){let i=s.slice(10).trim();if(!i)throw new Error("Missing value for --profile.");r=i;}}let e=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return r||e||null}function Ye(t,r){return {...t,...r??{}}}function Xe(t,r){return {...t,...r??{},invalidationPaths:[...(r?.invalidationPaths??t.invalidationPaths)||[]]}}function he(t,r){let e=String(r??t.defaultDeploymentProfile??"").trim();if(!e)return {name:null,profile:null,config:t};let n=t.deploymentProfiles?.[e];if(!n)throw new Error(`Unknown deployment profile "${e}". Check deploymentProfiles in publisher.config.json.`);return {name:e,profile:n,config:{...t,targetOrigin:n.targetOrigin??t.targetOrigin,s3:Ye(t.s3,n.s3),cloudFront:Xe(t.cloudFront,n.cloudFront)}}}var we={error:0,warn:1,info:2,debug:3},ye=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",H=ye?S.join(ye,"current-progress.json"):"",Pe=Promise.resolve();function be(t){return t.includes("crawl")?"crawl":t.includes("deploy")?"deploy":t.includes("invalidate")?"invalidate":t}function et(t,r){let e={...t},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[s,i]of Object.entries(r)){let o=e[s];if(typeof o=="number"&&typeof i=="number"&&n.has(s)){e[s]=Math.max(o,i);continue}e[s]=i;}return e}function tt(t,r,e){H&&(Pe=Pe.then(async()=>{let n=new Date,s=n.toISOString();await $.mkdir(S.dirname(H),{recursive:true});let i=await $.readFile(H,"utf8").then(b=>JSON.parse(b)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},u=be(t),a=i?.currentStep||be(i?.source||""),l=i?.startedAt||s,g={...i?.stepDurationsSec??{}},m=i?.stepStartedAt||s;if(a&&a!==u&&i?.stepStartedAt){let b=Math.max(0,Math.round((n.getTime()-new Date(i.stepStartedAt).getTime())/1e3));g[a]=(g[a]??0)+b,m=s;}let c=Math.max(0,Math.round((n.getTime()-new Date(m).getTime())/1e3)),w=Math.max(0,Math.round((n.getTime()-new Date(l).getTime())/1e3)),P={checkedAt:s,source:t,message:r,details:et(o,e??{}),startedAt:l,currentStep:u,stepStartedAt:m,stepElapsedSec:c,totalElapsedSec:w,stepDurationsSec:g};await $.writeFile(H,JSON.stringify(P,null,2),"utf8");}).catch(()=>{}));}async function rt(t,r){await $.mkdir(S.dirname(t),{recursive:true}),await $.appendFile(t,`${JSON.stringify(r)}
|
|
2
2
|
`,"utf8");}var A=class{constructor(r,e,n="info"){this.logDir=r;this.logFile=e;this.level=n;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return S.join(this.logDir,this.logFile)}get errorsPath(){return S.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await $.mkdir(this.logDir,{recursive:true}),await $.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(r){this.writeQueue=this.writeQueue.then(r).catch(e=>{this.writeError=e instanceof Error?e:new Error(String(e));});}enqueueLine(r){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await $.appendFile(this.logPath,`${r}
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(r,e){this.enqueueTask(()=>
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(r,e){this.enqueueTask(()=>rt(r,e));}accepts(r){return we[r]<=we[this.level]}push(r,e,n){if(!this.accepts(r))return;let s=JSON.stringify({time:new Date().toISOString(),level:r,message:e,...n||{}});this.enqueueLine(s),r==="error"?console.error(`[ERROR] ${e}`,n?JSON.stringify(n):""):r==="warn"?console.warn(`[WARN] ${e}`,n?JSON.stringify(n):""):console.log(`[${r.toUpperCase().padEnd(5)}] ${e}`);}info(r,e){this.push("info",r,e);}progress(r,e){tt(this.logFile,r,e);}debug(r,e){this.push("debug",r,e);}warn(r,e){this.push("warn",r,e);}error(r,e){this.enqueueJsonLine(this.errorsPath,{message:r,...e||{}}),this.push("error",r,e);}mark(r){this.marks.set(r,Date.now());}endMark(r,e){let n=this.marks.get(r);if(!n)return;let s=Date.now()-n,i=Number((s/1e3).toFixed(2));this.push("info",`Timing ${r}: ${i}s`,{name:r,ms:s,seconds:i,...e||{}});}async flush(){let r=Date.now()-this.startedAt,e=Number((r/1e3).toFixed(2));if(this.push("info",`Total time: ${e}s`,{name:"total",ms:r,seconds:e}),await this.writeQueue,this.writeError)throw this.writeError}};function Se(t){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(S.extname(t).toLowerCase())}function xe(t){return t.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function F(t){return t.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function J(t){return t.replace(/\//g,"\\/")}function ve(t){return t.replace(/\//g,"\\\\/")}function K(t,r,e){if(!r)return;t[r]=e;let n=J(r),s=J(e);t[n]=s;let i=ve(r),o=ve(e);t[i]=o;let u=F(r),a=F(e);t[u]=a;let l=F(n),g=F(s);t[l]=g;let m=F(i),c=F(o);t[m]=c;}function De(t,r){try{return new URL(t,r==="."?"https://relative.invalid":r).pathname}catch{return t.startsWith("/")?t:`/${t.replace(/^\.\//,"")}`}}function $e(t,r,e){if(!r)return e;let n=S.dirname(S.resolve(r)),s=S.resolve(t,e.replace(/^\/+/,"")),i=S.relative(n,s).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var ot=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],at=new Set([".html",".htm"]),Ce="WPSuite.io Static Publisher",lt=Ce.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function ut(t,r,e){if(t.wpsuite?.subscriber===true||!e)return false;let n=S.extname(e).toLowerCase();return at.has(n)?/<head\b|<html\b|<!doctype html/i.test(r):false}function gt(t){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${lt}\\2)[^>]*\\/?>`,"i").test(t))return t;let r=`<meta name="generator" content="${Ce}" />`;if(t.match(/(\r?\n)([ \t]*)<\/head>/i))return t.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${r}$1$2</head>`);let n=t.includes(`\r
|
|
4
4
|
`)?`\r
|
|
5
5
|
`:`
|
|
6
|
-
`;return t.replace(/<head\b[^>]*>/i,s=>`${s}${n} ${r}`)}function gt(t){let r=xe(t).trim();return r.startsWith("{")||r.startsWith("[")||r.includes("\\/")||/"@context"|"@type"/.test(r)}function ct(t,r){let e=[...new Set(r)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!e.length)return t;let n=i=>{let o=i;for(let[u,a]of e)o=o.split(u).join(a);return o},s=t.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,u,a)=>`${o}${n(u)}${a}`);return s=s.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,u,a,l)=>gt(a)?`${o}${n(a)}${l}`:i),s}function pt(t,r){if(!r)return null;let e=S.dirname(S.resolve(r)),n=S.resolve(t.outputDir),s=S.relative(e,n).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s.endsWith("/")?s:`${s}/`):"./"}function dt(t,r,e){let n=pt(r,e);if(!n)return t;let s=t;for(let i of st){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");s=s.replace(u,`${n}${i}`).replace(a,`${n}${i}`);let l=i.replace(/\//g,"\\/"),g=n.replace(/\//g,"\\/"),m=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}\\\\/)+${m}`,"g"),w=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${m}`,"g"),P=new RegExp(`(?<![\\.\\\\])\\/${m}`,"g");s=s.replace(c,`${g}${l}`).replace(w,`${g}${l}`).replace(P,`${g}${l}`);}return s}function X(t,r,e){if(t.urlRewriteMode==="absolute")return r;let n=(()=>{try{return new URL(r,t.targetOrigin==="."?"https://relative.invalid":t.targetOrigin)}catch{return null}})(),s=n?n.pathname:De(r,t.targetOrigin),i=n?`${n.search}${n.hash}`:"";return t.urlRewriteMode==="root-relative"?`${s}${i}`:`${$e(t.outputDir,e,s)}${i}`}function Y(t,r,e,n,s){K(t,e,X(r,n,s));}function ft(t,r,e,n,s){if(!e||e===n)return;K(t,e,n);let i=X(r,e,s),o=X(r,n,s);K(t,i,o);}function mt(t,r,e,n,s={}){let i=t,o={};Y(o,r,r.sourceOrigin,r.targetOrigin,n);for(let[a,l]of Object.entries(r.extraReplacements))Y(o,r,a,l,n);for(let[a,l]of Object.entries(e))Y(o,r,a,l,n);for(let[a,l]of Object.entries(s)){let g=e[a];!g||g===l||ft(o,r,l,g,n);}let u=Object.entries(o).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of u)i=i.split(a).join(l);return r.urlRewriteMode!=="absolute"&&(i=dt(i,r,n)),i=ct(i,u.map(([,a])=>a)),lt(r,i,n)&&(i=ut(i)),i}async function ht(t,r,e,n){let s=S.join(t.outputDir,n),i=S.extname(s).toLowerCase(),o;try{o=await $.readFile(s,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=mt(o,t,r,i===".js"||i===".mjs"?void 0:s,e);return a!==o?(await $.writeFile(s,a,"utf8"),{changed:true}):{changed:false}}async function wt(t,r){return await new Promise((e,n)=>{let s=a=>{if(u(),a?.error){n(new Error(a.error));return}e({changed:!!a?.changed});},i=a=>{u(),n(a);},o=a=>{u(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},u=()=>{t.off("message",s),t.off("error",i),t.off("exit",o);};t.on("message",s),t.on("error",i),t.on("exit",o),t.postMessage({file:r});})}async function Ee(t,r,e={}){let s=(e.files?[...new Set(e.files)].map(c=>S.resolve(c)).filter(c=>c.startsWith(S.resolve(t.outputDir))).map(c=>S.relative(t.outputDir,c).replace(/\\/g,"/")).filter(c=>c.length>0):await rt(["**/*"],{cwd:t.outputDir,onlyFiles:true,dot:true})).filter(c=>c!=="asset-map.json"&&Se(c)),i=0;if(s.length===0)return 0;let o=Math.max(1,Number(t.rewriteConcurrency||t.assetDownloadConcurrency||1)),u=Math.max(1,Math.min(o,s.length,availableParallelism())),a=0,l=0,g=null;if(u===1){for(let c of s){let w=await ht(t,r,e.previousAssetMap??{},c);w.changed&&(i+=1),l+=1,await e.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:c,changed:w.changed});}return i}let m=Array.from({length:u},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:t,assetMap:r,previousAssetMap:e.previousAssetMap??{}}});try{for(;!g;){let w=a;if(w>=s.length)return;a+=1;let P=s[w],b=await wt(c,P);b.changed&&(i+=1),l+=1,await e.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:P,changed:b.changed});}}catch(w){throw g=w instanceof Error?w:new Error(String(w)),g}finally{await c.terminate().catch(()=>{});}});return await Promise.all(m),i}var Le="wpsuite-sha256",Fe="wpsuite-normalized-sha256",$t=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,oe=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;function Rt(t){return t&&(oe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{8,}$/.test(t)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(t)?"__alnum__":t)}function re(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:Rt(r)).join("")}function Ct(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:oe.test(r)?"__uuid__":/^[0-9a-f]{6,}$/i.test(r)?"__hex__":/^[0-9]{6,}$/.test(r)?"__num__":/^[a-z0-9_-]{6,}$/i.test(r)?"__id__":r).join("")}function B(t,r){return Array.isArray(t)?t.map(e=>B(e,r)):t&&typeof t=="object"?Object.fromEntries(Object.entries(t).map(([e,n])=>[e,B(n,e)])):typeof t!="string"?t:r&&$t.test(r)?Ct(t):oe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)||/^[0-9]{8,}$/.test(t)?re(t):t}function Et(t){return t.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function Mt(t,r){let e=r.trim();if((e.startsWith("{")||e.startsWith("[")||e.startsWith('"'))&&t.startsWith("data-"))try{let s=JSON.parse(e);return JSON.stringify(B(s))}catch{}if(t.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(e))try{let s=Buffer.from(e,"base64").toString("utf8"),i=JSON.parse(s),o=JSON.stringify(B(i));return Buffer.from(o,"utf8").toString("base64")}catch{}return r}function Lt(t,r){let e=t.toLowerCase();return e==="class"?r.split(/(\s+)/).map((n,s)=>s%2===1?n:re(n)).join(""):e==="id"||e==="for"||e==="aria-controls"||e==="aria-labelledby"||e==="aria-describedby"||e==="aria-owns"||e==="aria-activedescendant"||e==="name"?re(r):e==="href"||e==="src"?Et(r):Mt(e,r)}function Ft(t){return t.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(r,e,n)=>{if(e.startsWith("/"))return r;let s=n.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(i,o,u,a)=>{let l=Lt(o,a);return ` ${o}=${u}${l}${u}`});return `<${e}${s}>`})}function ee(t){let r=S.extname(t).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(r)||["robots.txt","sitemap.xml"].includes(S.basename(t))}function te(t,r){return [t.replace(/^\/+|\/+$/g,""),r.replace(/^\/+/,"")].filter(Boolean).join("/")}async function kt(t,r,e,n){let s=new Map,i,o=0;do{o++;let u=await t.send(new ListObjectsV2Command({Bucket:r,Prefix:e||void 0,ContinuationToken:i}));for(let a of u.Contents||[]){if(!a.Key)continue;let l=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),g=Number(a.Size??0);s.set(a.Key,{etag:l,size:g});}i=u.NextContinuationToken,n.debug(`Listed S3 keys page ${o}: ${s.size} keys so far`);}while(i);return s}async function It(t,r,e,n){if(n.has(e))return n.get(e)??null;try{let s=await t.send(new HeadObjectCommand({Bucket:r,Key:e})),i={etag:(s.ETag||"").replace(/^"|"$/g,"").toLowerCase(),size:Number(s.ContentLength??0)};return n.set(e,i),i}catch{return n.set(e,null),null}}async function ke(t,r,e,n){let s=[...new Set(Array.from(e).filter(Boolean))];if(s.length===0)return n.info("No stale S3 objects to delete."),[];n.info(`Deleting ${s.length} stale S3 object(s)\u2026`);for(let i=0;i<s.length;i+=1e3){let o=s.slice(i,i+1e3);if(o.length!==0){await t.send(new DeleteObjectsCommand({Bucket:r,Delete:{Objects:o.map(u=>({Key:u}))}})),n.info(`Deleted batch of ${o.length} object(s).`);for(let u of o)n.debug(` deleted: ${u}`);}}return s}async function jt(t,r,e,n,s){return ke(t,r,Array.from(e).filter(i=>!n.has(i)),s)}function Ot(t){return createHash("md5").update(t).digest("hex")}function Me(t){return createHash("sha256").update(t).digest("hex")}function ne(t){return createHash("sha256").update(Ft(t.toString("utf8"))).digest("hex")}async function Tt(t,r,e,n){if(n.has(e))return n.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[Le]||"").trim().toLowerCase()||null;return n.set(e,o),o}catch{return n.set(e,null),null}}async function _t(t,r,e,n,s){if(n.has(e))return n.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[Fe]||"").trim().toLowerCase();if(o)return n.set(e,o),o}catch{return n.set(e,null),null}if(s.has(e)){let i=s.get(e)??null;return n.set(e,i),i}try{let o=await(await t.send(new GetObjectCommand({Bucket:r,Key:e}))).Body?.transformToByteArray(),u=o?ne(Buffer.from(o)):null;return s.set(e,u),n.set(e,u),u}catch{return s.set(e,null),n.set(e,null),null}}function At(t){return t.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function Nt(t,r){let e=t,n=Object.entries(r).sort((s,i)=>i[0].length-s[0].length);for(let[s,i]of n)s&&(e=e.split(s).join(i));return e}async function Ut(t,r,e,n){let s=S.join(t,"asset-map.json"),i;try{i=await $.readFile(s,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(i);if(!o||typeof o!="object")return;let u=Object.fromEntries(Object.entries(o).map(([a,l])=>{let g=String(l??"");return r.urlRewriteMode==="absolute"&&r.targetOrigin&&e.targetOrigin&&r.targetOrigin!==e.targetOrigin&&(g=g.split(r.targetOrigin).join(e.targetOrigin)),g=Nt(g,n),[a,g]}));await $.writeFile(s,JSON.stringify(u,null,2),"utf8");}async function zt(t,r,e,n,s){let i=t.targetOrigin!==r.targetOrigin,o=Object.keys(n).length>0;if(!e||!i&&!o)return {outputDir:t.outputDir,cleanup:async()=>{}};if(i&&t.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${e}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let u=S.join(S.dirname(t.outputDir),`.deploy-profile-${At(e)}-${Date.now()}`);s.info(`Preparing deployment profile "${e}" from existing crawl output...`),await $.rm(u,{recursive:true,force:true}),await $.cp(t.outputDir,u,{recursive:true});let a={...t,sourceOrigin:t.targetOrigin,targetOrigin:r.targetOrigin,extraReplacements:n,outputDir:u},l=await Ee(a,{});return await Ut(u,t,r,n),s.info(`Prepared deployment profile "${e}" in ${u} (${l} rewritten text file(s)).`),{outputDir:u,cleanup:async()=>{await $.rm(u,{recursive:true,force:true});}}}async function Ht(){let t=await G(),r=me(process.argv.slice(2)),e=he(t,r),n=S.resolve(e.config.logDir||"logs"),s=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",i=s?S.join(S.resolve(s),"deploy-diff.json"):"",o=new A(n,"deploy.log.jsonl",e.config.logLevel??"info"),u=e.profile?.extraReplacements??{},a=async()=>{};try{let l=await zt(t,e.config,e.name,u,o);a=l.cleanup;let g={...e.config,outputDir:l.outputDir},m=g.s3SyncMode??"sdk-upload-delete",c=await de(t),w=l.outputDir!==t.outputDir&&(t.targetOrigin!==e.config.targetOrigin||Object.keys(u).length>0),P=!!c&&!c.fullSyncRequired&&!w;e.name&&o.info(`Using deployment profile: ${e.name}`),o.info(`Starting deploy \u2014 s3://${g.s3.bucket}/${g.s3.prefix||""} (region: ${g.s3.region})`),o.info(`S3 sync mode: ${m}`),c&&c.fullSyncRequired?o.info("Deploy plan requests a full sync; scanning the whole output tree."):c&&w?o.info("Ignoring incremental deploy plan because deployment-profile rewrites require a full sync."):P&&o.info(`Using incremental deploy plan: ${c.changedFiles.length} changed file(s), ${c.deletedFiles.length} deleted file(s).`),o.mark("deploy");let b=new S3Client({region:g.s3.region}),p=P?(await Promise.all(c.changedFiles.map(async f=>{try{return (await $.stat(S.join(g.outputDir,f))).isFile()?f:null}catch{return null}}))).filter(f=>f!==null):await rt(["**/*"],{cwd:g.outputDir,onlyFiles:!0,dot:!0}),Ie=P?new Set:new Set(p.map(f=>te(g.s3.prefix,f)));P?o.info(`Using ${p.length} planned changed file(s) from deploy plan for upload.`):o.info(`Found ${p.length} file(s) in output directory "${g.outputDir}".`);let v=0,y=0,C=0,W=[],k=[],q=[],U=[],I=new Map,je=new Map,Oe=new Map,Te=new Map,_e=new Map;P||(o.info("Listing existing S3 objects to detect unchanged files..."),I=await kt(b,g.s3.bucket,g.s3.prefix,o),o.info(`Found ${I.size} existing key(s) in S3.`));let Ae=async f=>I.has(f)?I.get(f)??null:P?It(b,g.s3.bucket,f,je):null;for(let f of p){let ae=S.join(g.outputDir,f),d=te(g.s3.prefix,f),j=Dt.lookup(f)||"application/octet-stream",Ne=ee(f)?g.s3.htmlCacheControl:g.s3.assetCacheControl,h=v+y+C+1;p.length<=200||h<=5||h%10===0||h===p.length?(o.progress(`Uploading [${h}/${p.length}] ${d} (${j})`,{file:f,key:d,index:h,totalFiles:p.length,contentType:j,mode:m}),o.info(`Uploading [${h}/${p.length}] ${d} (${j})`)):o.debug(`Uploading [${h}/${p.length}] ${d} (${j})`);try{let x=await $.readFile(ae),O=await Ae(d),T=null,L=null;if(O){let Ue=x.byteLength;if(O.size===Ue){let z=O.etag.includes("-");if(O.etag!==""&&!z){let ze=Ot(x);if(O.etag===ze){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:C,mode:m,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}T=Me(x);let le=await Tt(b,g.s3.bucket,d,Oe);if(le&&le===T){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:C,mode:m,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}if(ee(f)){L=ne(x);let z=await _t(b,g.s3.bucket,d,Te,_e);if(z&&z===L){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped semantically unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:C,mode:m,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped semantically unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}}T=T??Me(x),L=L??(ee(f)?ne(x):null),await new Upload({client:b,params:{Bucket:g.s3.bucket,Key:d,Body:x,ContentType:j,CacheControl:Ne,Metadata:{[Le]:T,...L?{[Fe]:L}:{}}}}).done(),v++,W.push(d),(v%25===0||v===p.length)&&(o.progress(`Uploaded ${v}/${p.length} file(s)\u2026`,{uploaded:v,skipped:y,failed:C,totalFiles:p.length,mode:m}),o.info(`Uploaded ${v}/${p.length} file(s)\u2026`));}catch(x){C++,q.push(d),o.error(`Failed to upload ${d}: ${x.message}`,{key:d,error:String(x)});}}o.info(`Upload complete: ${v} uploaded, ${y} skipped unchanged, ${C} failed.`),m==="sdk-upload-delete"?P?U=await ke(b,g.s3.bucket,c.deletedFiles.map(f=>te(g.s3.prefix,f)),o):U=await jt(b,g.s3.bucket,I.keys(),Ie,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion."),i&&(await $.mkdir(S.dirname(i),{recursive:!0}),await $.writeFile(i,JSON.stringify({generatedAt:new Date().toISOString(),mode:m,summary:{totalFiles:p.length,uploaded:W.length,skipped:k.length,failed:q.length,deleted:U.length,...e.name?{profile:e.name}:{}},uploadedKeys:W,skippedKeys:k,failedKeys:q,deletedKeys:U},null,2),"utf8")),await fe(t),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}Ht().catch(async t=>{console.error(t);try{let e=(await G().catch(()=>null))?.logDir??"logs",n=new A(e,"deploy.log.jsonl","debug");n.error(`Unhandled error: ${t instanceof Error?t.message:String(t)}`,t instanceof Error?{stack:t.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
|
6
|
+
`;return t.replace(/<head\b[^>]*>/i,s=>`${s}${n} ${r}`)}function ct(t){let r=xe(t).trim();return r.startsWith("{")||r.startsWith("[")||r.includes("\\/")||/"@context"|"@type"/.test(r)}function pt(t,r){let e=[...new Set(r)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,J(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!e.length)return t;let n=i=>{let o=i;for(let[u,a]of e)o=o.split(u).join(a);return o},s=t.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,u,a)=>`${o}${n(u)}${a}`);return s=s.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,u,a,l)=>ct(a)?`${o}${n(a)}${l}`:i),s}function dt(t,r){if(!r)return null;let e=S.dirname(S.resolve(r)),n=S.resolve(t.outputDir),s=S.relative(e,n).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s.endsWith("/")?s:`${s}/`):"./"}function ft(t,r,e){let n=dt(r,e);if(!n)return t;let s=t;for(let i of ot){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),u=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");s=s.replace(u,`${n}${i}`).replace(a,`${n}${i}`);let l=i.replace(/\//g,"\\/"),g=n.replace(/\//g,"\\/"),m=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),c=new RegExp(`(?:\\.{1,}\\\\/)+${m}`,"g"),w=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${m}`,"g"),P=new RegExp(`(?<![\\.\\\\])\\/${m}`,"g");s=s.replace(c,`${g}${l}`).replace(w,`${g}${l}`).replace(P,`${g}${l}`);}return s}function X(t,r,e){if(t.urlRewriteMode==="absolute")return r;let n=(()=>{try{return new URL(r,t.targetOrigin==="."?"https://relative.invalid":t.targetOrigin)}catch{return null}})(),s=n?n.pathname:De(r,t.targetOrigin),i=n?`${n.search}${n.hash}`:"";return t.urlRewriteMode==="root-relative"?`${s}${i}`:`${$e(t.outputDir,e,s)}${i}`}function Y(t,r,e,n,s){K(t,e,X(r,n,s));}function mt(t,r,e,n,s){if(!e||e===n)return;K(t,e,n);let i=X(r,e,s),o=X(r,n,s);K(t,i,o);}function ht(t,r,e,n,s={}){let i=t,o={};Y(o,r,r.sourceOrigin,r.targetOrigin,n);for(let[a,l]of Object.entries(r.extraReplacements))Y(o,r,a,l,n);for(let[a,l]of Object.entries(e))Y(o,r,a,l,n);for(let[a,l]of Object.entries(s)){let g=e[a];!g||g===l||mt(o,r,l,g,n);}let u=Object.entries(o).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of u)i=i.split(a).join(l);return r.urlRewriteMode!=="absolute"&&(i=ft(i,r,n)),i=pt(i,u.map(([,a])=>a)),ut(r,i,n)&&(i=gt(i)),i}async function wt(t,r,e,n){let s=S.join(t.outputDir,n),i=S.extname(s).toLowerCase(),o;try{o=await $.readFile(s,"utf8");}catch(l){if(l.code!=="ENOENT")throw l;return {changed:false}}let a=ht(o,t,r,i===".js"||i===".mjs"?void 0:s,e);return a!==o?(await $.writeFile(s,a,"utf8"),{changed:true}):{changed:false}}async function yt(t,r){return await new Promise((e,n)=>{let s=a=>{if(u(),a?.error){n(new Error(a.error));return}e({changed:!!a?.changed});},i=a=>{u(),n(a);},o=a=>{u(),a!==0&&n(new Error(`Rewrite worker exited with code ${a}.`));},u=()=>{t.off("message",s),t.off("error",i),t.off("exit",o);};t.on("message",s),t.on("error",i),t.on("exit",o),t.postMessage({file:r});})}async function Ee(t,r,e={}){let s=(e.files?[...new Set(e.files)].map(c=>S.resolve(c)).filter(c=>c.startsWith(S.resolve(t.outputDir))).map(c=>S.relative(t.outputDir,c).replace(/\\/g,"/")).filter(c=>c.length>0):await nt(["**/*"],{cwd:t.outputDir,onlyFiles:true,dot:true})).filter(c=>c!=="asset-map.json"&&Se(c)),i=0;if(s.length===0)return 0;let o=Math.max(1,Number(t.rewriteConcurrency||t.assetDownloadConcurrency||1)),u=Math.max(1,Math.min(o,s.length,availableParallelism())),a=0,l=0,g=null;if(u===1){for(let c of s){let w=await wt(t,r,e.previousAssetMap??{},c);w.changed&&(i+=1),l+=1,await e.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:c,changed:w.changed});}return i}let m=Array.from({length:u},async()=>{let c=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:t,assetMap:r,previousAssetMap:e.previousAssetMap??{}}});try{for(;!g;){let w=a;if(w>=s.length)return;a+=1;let P=s[w],b=await yt(c,P);b.changed&&(i+=1),l+=1,await e.onProgress?.({index:l,totalFiles:s.length,changedTextFiles:i,file:P,changed:b.changed});}}catch(w){throw g=w instanceof Error?w:new Error(String(w)),g}finally{await c.terminate().catch(()=>{});}});return await Promise.all(m),i}var Le="wpsuite-sha256",Fe="wpsuite-normalized-sha256",Rt=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,oe=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;function Ct(t){return t&&(oe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{8,}$/.test(t)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(t)?"__alnum__":t)}function re(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:Ct(r)).join("")}function Et(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:oe.test(r)?"__uuid__":/^[0-9a-f]{6,}$/i.test(r)?"__hex__":/^[0-9]{6,}$/.test(r)?"__num__":/^[a-z0-9_-]{6,}$/i.test(r)?"__id__":r).join("")}function B(t,r){return Array.isArray(t)?t.map(e=>B(e,r)):t&&typeof t=="object"?Object.fromEntries(Object.entries(t).map(([e,n])=>[e,B(n,e)])):typeof t!="string"?t:r&&Rt.test(r)?Et(t):oe.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)||/^[0-9]{8,}$/.test(t)?re(t):t}function Mt(t){return t.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function Lt(t,r){let e=r.trim();if((e.startsWith("{")||e.startsWith("[")||e.startsWith('"'))&&t.startsWith("data-"))try{let s=JSON.parse(e);return JSON.stringify(B(s))}catch{}if(t.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(e))try{let s=Buffer.from(e,"base64").toString("utf8"),i=JSON.parse(s),o=JSON.stringify(B(i));return Buffer.from(o,"utf8").toString("base64")}catch{}return r}function Ft(t,r){let e=t.toLowerCase();return e==="class"?r.split(/(\s+)/).map((n,s)=>s%2===1?n:re(n)).join(""):e==="id"||e==="for"||e==="aria-controls"||e==="aria-labelledby"||e==="aria-describedby"||e==="aria-owns"||e==="aria-activedescendant"||e==="name"?re(r):e==="href"||e==="src"?Mt(r):Lt(e,r)}function kt(t){return t.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(r,e,n)=>{if(e.startsWith("/"))return r;let s=n.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(i,o,u,a)=>{let l=Ft(o,a);return ` ${o}=${u}${l}${u}`});return `<${e}${s}>`})}function ee(t){let r=S.extname(t).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(r)||["robots.txt","sitemap.xml"].includes(S.basename(t))}function te(t,r){return [t.replace(/^\/+|\/+$/g,""),r.replace(/^\/+/,"")].filter(Boolean).join("/")}async function jt(t,r,e,n){let s=new Map,i,o=0;do{o++;let u=await t.send(new ListObjectsV2Command({Bucket:r,Prefix:e||void 0,ContinuationToken:i}));for(let a of u.Contents||[]){if(!a.Key)continue;let l=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),g=Number(a.Size??0);s.set(a.Key,{etag:l,size:g});}i=u.NextContinuationToken,n.debug(`Listed S3 keys page ${o}: ${s.size} keys so far`);}while(i);return s}async function It(t,r,e,n){if(n.has(e))return n.get(e)??null;try{let s=await t.send(new HeadObjectCommand({Bucket:r,Key:e})),i={etag:(s.ETag||"").replace(/^"|"$/g,"").toLowerCase(),size:Number(s.ContentLength??0)};return n.set(e,i),i}catch{return n.set(e,null),null}}async function ke(t,r,e,n){let s=[...new Set(Array.from(e).filter(Boolean))];if(s.length===0)return n.info("No stale S3 objects to delete."),[];n.info(`Deleting ${s.length} stale S3 object(s)\u2026`);for(let i=0;i<s.length;i+=1e3){let o=s.slice(i,i+1e3);if(o.length!==0){await t.send(new DeleteObjectsCommand({Bucket:r,Delete:{Objects:o.map(u=>({Key:u}))}})),n.info(`Deleted batch of ${o.length} object(s).`);for(let u of o)n.debug(` deleted: ${u}`);}}return s}async function Ot(t,r,e,n,s){return ke(t,r,Array.from(e).filter(i=>!n.has(i)),s)}function Tt(t){return createHash("md5").update(t).digest("hex")}function Me(t){return createHash("sha256").update(t).digest("hex")}function ne(t){return createHash("sha256").update(kt(t.toString("utf8"))).digest("hex")}async function _t(t,r,e,n){if(n.has(e))return n.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[Le]||"").trim().toLowerCase()||null;return n.set(e,o),o}catch{return n.set(e,null),null}}async function At(t,r,e,n,s){if(n.has(e))return n.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[Fe]||"").trim().toLowerCase();if(o)return n.set(e,o),o}catch{return n.set(e,null),null}if(s.has(e)){let i=s.get(e)??null;return n.set(e,i),i}try{let o=await(await t.send(new GetObjectCommand({Bucket:r,Key:e}))).Body?.transformToByteArray(),u=o?ne(Buffer.from(o)):null;return s.set(e,u),n.set(e,u),u}catch{return s.set(e,null),n.set(e,null),null}}function Nt(t){return t.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function zt(t,r){let e=t,n=Object.entries(r).sort((s,i)=>i[0].length-s[0].length);for(let[s,i]of n)s&&(e=e.split(s).join(i));return e}async function Ut(t,r,e,n){let s=S.join(t,"asset-map.json"),i;try{i=await $.readFile(s,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(i);if(!o||typeof o!="object")return;let u=Object.fromEntries(Object.entries(o).map(([a,l])=>{let g=String(l??"");return r.urlRewriteMode==="absolute"&&r.targetOrigin&&e.targetOrigin&&r.targetOrigin!==e.targetOrigin&&(g=g.split(r.targetOrigin).join(e.targetOrigin)),g=zt(g,n),[a,g]}));await $.writeFile(s,JSON.stringify(u,null,2),"utf8");}async function Ht(t,r,e,n,s){let i=t.targetOrigin!==r.targetOrigin,o=Object.keys(n).length>0;if(!e||!i&&!o)return {outputDir:t.outputDir,cleanup:async()=>{}};if(i&&t.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${e}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let u=S.join(S.dirname(t.outputDir),`.deploy-profile-${Nt(e)}-${Date.now()}`);s.info(`Preparing deployment profile "${e}" from existing crawl output...`),await $.rm(u,{recursive:true,force:true}),await $.cp(t.outputDir,u,{recursive:true});let a={...t,sourceOrigin:t.targetOrigin,targetOrigin:r.targetOrigin,extraReplacements:n,outputDir:u},l=await Ee(a,{});return await Ut(u,t,r,n),s.info(`Prepared deployment profile "${e}" in ${u} (${l} rewritten text file(s)).`),{outputDir:u,cleanup:async()=>{await $.rm(u,{recursive:true,force:true});}}}async function Jt(){let t=await G(),r=me(process.argv.slice(2)),e=he(t,r),n=S.resolve(e.config.logDir||"logs"),s=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",i=s?S.join(S.resolve(s),"deploy-diff.json"):"",o=new A(n,"deploy.log.jsonl",e.config.logLevel??"info"),u=e.profile?.extraReplacements??{},a=async()=>{};try{let l=await Ht(t,e.config,e.name,u,o);a=l.cleanup;let g={...e.config,outputDir:l.outputDir},m=g.s3SyncMode??"sdk-upload-delete",c=await de(t),w=l.outputDir!==t.outputDir&&(t.targetOrigin!==e.config.targetOrigin||Object.keys(u).length>0),P=!!c&&!c.fullSyncRequired&&!w;e.name&&o.info(`Using deployment profile: ${e.name}`),o.info(`Starting deploy \u2014 s3://${g.s3.bucket}/${g.s3.prefix||""} (region: ${g.s3.region})`),o.info(`S3 sync mode: ${m}`),c&&c.fullSyncRequired?o.info("Deploy plan requests a full sync; scanning the whole output tree."):c&&w?o.info("Ignoring incremental deploy plan because deployment-profile rewrites require a full sync."):P&&o.info(`Using incremental deploy plan: ${c.changedFiles.length} changed file(s), ${c.deletedFiles.length} deleted file(s).`),o.mark("deploy");let b=new S3Client({region:g.s3.region}),p=P?(await Promise.all(c.changedFiles.map(async f=>{try{return (await $.stat(S.join(g.outputDir,f))).isFile()?f:null}catch{return null}}))).filter(f=>f!==null):await nt(["**/*"],{cwd:g.outputDir,onlyFiles:!0,dot:!0}),je=P?new Set:new Set(p.map(f=>te(g.s3.prefix,f)));P?o.info(`Using ${p.length} planned changed file(s) from deploy plan for upload.`):o.info(`Found ${p.length} file(s) in output directory "${g.outputDir}".`);let v=0,y=0,E=0,q=[],k=[],W=[],z=[],j=new Map,Ie=new Map,Oe=new Map,Te=new Map,_e=new Map;P||(o.info("Listing existing S3 objects to detect unchanged files..."),j=await jt(b,g.s3.bucket,g.s3.prefix,o),o.info(`Found ${j.size} existing key(s) in S3.`));let Ae=async f=>j.has(f)?j.get(f)??null:P?It(b,g.s3.bucket,f,Ie):null;for(let f of p){let ae=S.join(g.outputDir,f),d=te(g.s3.prefix,f),I=$t.lookup(f)||"application/octet-stream",Ne=ee(f)?g.s3.htmlCacheControl:g.s3.assetCacheControl,h=v+y+E+1;p.length<=200||h<=5||h%10===0||h===p.length?(o.progress(`Uploading [${h}/${p.length}] ${d} (${I})`,{file:f,key:d,index:h,totalFiles:p.length,contentType:I,mode:m}),o.info(`Uploading [${h}/${p.length}] ${d} (${I})`)):o.debug(`Uploading [${h}/${p.length}] ${d} (${I})`);try{let x=await $.readFile(ae),O=await Ae(d),T=null,L=null;if(O){let ze=x.byteLength;if(O.size===ze){let U=O.etag.includes("-");if(O.etag!==""&&!U){let Ue=Tt(x);if(O.etag===Ue){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:E,mode:m,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}T=Me(x);let le=await _t(b,g.s3.bucket,d,Oe);if(le&&le===T){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:E,mode:m,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}if(ee(f)){L=ne(x);let U=await At(b,g.s3.bucket,d,Te,_e);if(U&&U===L){y++,p.length<=200||y<=5||y%25===0?(o.progress(`Skipped semantically unchanged [${h}/${p.length}] ${d}`,{file:f,key:d,index:h,totalFiles:p.length,uploaded:v,skipped:y,failed:E,mode:m,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${h}/${p.length}] ${d}`)):o.debug(`Skipped semantically unchanged [${h}/${p.length}] ${d}`),k.push(d);continue}}}T=T??Me(x),L=L??(ee(f)?ne(x):null),await new Upload({client:b,params:{Bucket:g.s3.bucket,Key:d,Body:x,ContentType:I,CacheControl:Ne,Metadata:{[Le]:T,...L?{[Fe]:L}:{}}}}).done(),v++,q.push(d),(v%25===0||v===p.length)&&(o.progress(`Uploaded ${v}/${p.length} file(s)\u2026`,{uploaded:v,skipped:y,failed:E,totalFiles:p.length,mode:m}),o.info(`Uploaded ${v}/${p.length} file(s)\u2026`));}catch(x){E++,W.push(d),o.error(`Failed to upload ${d}: ${x.message}`,{key:d,error:String(x)});}}o.info(`Upload complete: ${v} uploaded, ${y} skipped unchanged, ${E} failed.`),m==="sdk-upload-delete"?P?z=await ke(b,g.s3.bucket,c.deletedFiles.map(f=>te(g.s3.prefix,f)),o):z=await Ot(b,g.s3.bucket,j.keys(),je,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion."),i&&(await $.mkdir(S.dirname(i),{recursive:!0}),await $.writeFile(i,JSON.stringify({generatedAt:new Date().toISOString(),mode:m,summary:{totalFiles:p.length,uploaded:q.length,skipped:k.length,failed:W.length,deleted:z.length,...e.name?{profile:e.name}:{}},uploadedKeys:q,skippedKeys:k,failedKeys:W,deletedKeys:z},null,2),"utf8")),await fe(t),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}Jt().catch(async t=>{console.error(t);try{let e=(await G().catch(()=>null))?.logDir??"logs",n=new A(e,"deploy.log.jsonl","debug");n.error(`Unhandled error: ${t instanceof Error?t.message:String(t)}`,t instanceof Error?{stack:t.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
package/dist/invalidate.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {CloudFrontClient,CreateInvalidationCommand}from'@aws-sdk/client-cloudfront';import
|
|
2
|
-
`,"utf8");}var h=class{constructor(t,e,
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(t,e){this.enqueueTask(()=>_(t,e));}accepts(t){return L[t]<=L[this.level]}push(t,e,
|
|
1
|
+
import {CloudFrontClient,CreateInvalidationCommand}from'@aws-sdk/client-cloudfront';import c from'fs/promises';import g from'path';function S(i){let t=i.trim();return t==="."?".":t.replace(/\/$/,"")}function T(i){let t=String(i??"").trim();if(!t)return "";let e=t;if(/^https?:\/\//i.test(e))try{e=new URL(e).pathname;}catch{return ""}if(e=e.split(/[?#]/,1)[0]?.replace(/\\/g,"/").trim()??"",!e)return "";let n=e.endsWith("/"),o=e.replace(/^\/+/,"").split("/").map(l=>l.replace(/[^A-Za-z0-9._-]/g,"")).filter(l=>l.length>0&&l!=="."&&l!=="..");if(o.length===0)return "";let r=`/${o.join("/")}`;return n&&g.extname(r)===""&&(r+="/"),r==="/"?"":r}function P(i){return !i||typeof i!="object"?{}:Object.fromEntries(Object.entries(i).map(([t,e])=>[t.trim(),String(e??"")]).filter(([t])=>t.length>0))}function O(i){if(!i||typeof i!="object")return {};let t={};for(let[e,n]of Object.entries(i)){let o=e.trim();if(!o||!n||typeof n!="object")continue;let r=n,l={};if(typeof r.targetOrigin=="string"){let s=S(r.targetOrigin);s&&(l.targetOrigin=s);}let d=P(r.extraReplacements);if(Object.keys(d).length>0&&(l.extraReplacements=d),r.s3&&typeof r.s3=="object"){let s={},a=r.s3;typeof a.bucket=="string"&&(s.bucket=a.bucket.trim()),typeof a.prefix=="string"&&(s.prefix=a.prefix.trim()),typeof a.region=="string"&&(s.region=a.region.trim()),typeof a.htmlCacheControl=="string"&&(s.htmlCacheControl=a.htmlCacheControl.trim()),typeof a.assetCacheControl=="string"&&(s.assetCacheControl=a.assetCacheControl.trim()),Object.keys(s).length>0&&(l.s3=s);}if(r.cloudFront&&typeof r.cloudFront=="object"){let s={},a=r.cloudFront;typeof a.distributionId=="string"&&(s.distributionId=a.distributionId.trim()),Array.isArray(a.invalidationPaths)&&(s.invalidationPaths=a.invalidationPaths.map(u=>String(u??"").trim()).filter(u=>u.length>0)),Object.keys(s).length>0&&(l.cloudFront=s);}t[o]=l;}return t}function w(i,t){let n=String(i||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let o=n.split("/").map(r=>r.trim()).filter(r=>r.length>0&&r!=="."&&r!=="..");return o.length>0?o.join("/"):t}function M(){let i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return i.trim()?g.resolve(i):""}function b(i,t,e){let n=String(t||"").trim();return n&&g.isAbsolute(n)?g.resolve(n):g.resolve(i,w(n,e))}async function y(){let i=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await c.readFile(i,"utf8"),e=JSON.parse(t);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=S(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.generated404RequestPath=T(e.generated404RequestPath),e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=P(e.extraReplacements),e.postCrawlCopyMap=P(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.scheduler||={enabled:false,timezone:"UTC",rules:[]},e.scheduler.enabled??=false,e.scheduler.timezone||="UTC",e.scheduler.rules||=[],e.deploymentProfiles=O(e.deploymentProfiles),e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.defaultDeploymentProfile=String(e.defaultDeploymentProfile??"").trim(),e.defaultDeploymentProfile&&!e.deploymentProfiles[e.defaultDeploymentProfile]&&(e.defaultDeploymentProfile="");let n=M(),o=n?g.resolve(n,".."):"";return o?(e.outputDir=b(o,e.outputDir,"export"),e.logDir=b(o,e.logDir,"logs")):(g.isAbsolute(e.outputDir)||(e.outputDir=w(e.outputDir,"export")),g.isAbsolute(e.logDir)||(e.logDir=w(e.logDir,"logs"))),e}function D(i){let t="";for(let n=0;n<i.length;n++){let o=i[n];if(o==="--profile"){let r=(i[n+1]||"").trim();if(!r)throw new Error("Missing value for --profile.");t=r,n+=1;}else if(o.startsWith("--profile=")){let r=o.slice(10).trim();if(!r)throw new Error("Missing value for --profile.");t=r;}}let e=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return t||e||null}function $(i,t){return {...i,...t??{}}}function A(i,t){return {...i,...t??{},invalidationPaths:[...(t?.invalidationPaths??i.invalidationPaths)||[]]}}function C(i,t){let e=String(t??i.defaultDeploymentProfile??"").trim();if(!e)return {name:null,profile:null,config:i};let n=i.deploymentProfiles?.[e];if(!n)throw new Error(`Unknown deployment profile "${e}". Check deploymentProfiles in publisher.config.json.`);return {name:e,profile:n,config:{...i,targetOrigin:n.targetOrigin??i.targetOrigin,s3:$(i.s3,n.s3),cloudFront:A(i.cloudFront,n.cloudFront)}}}var L={error:0,warn:1,info:2,debug:3},I=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",f=I?g.join(I,"current-progress.json"):"",R=Promise.resolve();function F(i){return i.includes("crawl")?"crawl":i.includes("deploy")?"deploy":i.includes("invalidate")?"invalidate":i}function N(i,t){let e={...i},n=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[o,r]of Object.entries(t)){let l=e[o];if(typeof l=="number"&&typeof r=="number"&&n.has(o)){e[o]=Math.max(l,r);continue}e[o]=r;}return e}function q(i,t,e){f&&(R=R.then(async()=>{let n=new Date,o=n.toISOString();await c.mkdir(g.dirname(f),{recursive:true});let r=await c.readFile(f,"utf8").then(v=>JSON.parse(v)).catch(()=>null),l=r&&typeof r.details=="object"&&r.details?r.details:{},d=F(i),s=r?.currentStep||F(r?.source||""),a=r?.startedAt||o,u={...r?.stepDurationsSec??{}},p=r?.stepStartedAt||o;if(s&&s!==d&&r?.stepStartedAt){let v=Math.max(0,Math.round((n.getTime()-new Date(r.stepStartedAt).getTime())/1e3));u[s]=(u[s]??0)+v,p=o;}let k=Math.max(0,Math.round((n.getTime()-new Date(p).getTime())/1e3)),E=Math.max(0,Math.round((n.getTime()-new Date(a).getTime())/1e3)),x={checkedAt:o,source:i,message:t,details:N(l,e??{}),startedAt:a,currentStep:d,stepStartedAt:p,stepElapsedSec:k,totalElapsedSec:E,stepDurationsSec:u};await c.writeFile(f,JSON.stringify(x,null,2),"utf8");}).catch(()=>{}));}async function _(i,t){await c.mkdir(g.dirname(i),{recursive:true}),await c.appendFile(i,`${JSON.stringify(t)}
|
|
2
|
+
`,"utf8");}var h=class{constructor(t,e,n="info"){this.logDir=t;this.logFile=e;this.level=n;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return g.join(this.logDir,this.logFile)}get errorsPath(){return g.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await c.mkdir(this.logDir,{recursive:true}),await c.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(e=>{this.writeError=e instanceof Error?e:new Error(String(e));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await c.appendFile(this.logPath,`${t}
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(t,e){this.enqueueTask(()=>_(t,e));}accepts(t){return L[t]<=L[this.level]}push(t,e,n){if(!this.accepts(t))return;let o=JSON.stringify({time:new Date().toISOString(),level:t,message:e,...n||{}});this.enqueueLine(o),t==="error"?console.error(`[ERROR] ${e}`,n?JSON.stringify(n):""):t==="warn"?console.warn(`[WARN] ${e}`,n?JSON.stringify(n):""):console.log(`[${t.toUpperCase().padEnd(5)}] ${e}`);}info(t,e){this.push("info",t,e);}progress(t,e){q(this.logFile,t,e);}debug(t,e){this.push("debug",t,e);}warn(t,e){this.push("warn",t,e);}error(t,e){this.enqueueJsonLine(this.errorsPath,{message:t,...e||{}}),this.push("error",t,e);}mark(t){this.marks.set(t,Date.now());}endMark(t,e){let n=this.marks.get(t);if(!n)return;let o=Date.now()-n,r=Number((o/1e3).toFixed(2));this.push("info",`Timing ${t}: ${r}s`,{name:t,ms:o,seconds:r,...e||{}});}async flush(){let t=Date.now()-this.startedAt,e=Number((t/1e3).toFixed(2));if(this.push("info",`Total time: ${e}s`,{name:"total",ms:t,seconds:e}),await this.writeQueue,this.writeError)throw this.writeError}};async function Q(){let i=await y(),t=D(process.argv.slice(2)),e=C(i,t),n=e.config,o=n.logDir||"logs",r=new h(o,"invalidate.log.jsonl",n.logLevel??"info");if(e.name&&r.info(`Using deployment profile: ${e.name}`),!n.cloudFront.distributionId){r.info("No CloudFront distributionId configured; skipping invalidation."),await r.flush();return}let l=n.cloudFront.invalidationPaths.length>0?n.cloudFront.invalidationPaths:["/*"];r.info(`Starting CloudFront invalidation for distribution ${n.cloudFront.distributionId}`),r.info(`Invalidation paths (${l.length}): ${l.join(", ")}`),r.mark("invalidate");let d=new CloudFrontClient({region:"us-east-1"}),s=`smartcloud-static-publisher-${Date.now()}`;r.debug(`CallerReference: ${s}`);let a=await d.send(new CreateInvalidationCommand({DistributionId:n.cloudFront.distributionId,InvalidationBatch:{CallerReference:s,Paths:{Quantity:l.length,Items:l}}})),u=a.Invalidation?.Id??"(unknown)",p=a.Invalidation?.Status??"(unknown)";r.info(`CloudFront invalidation created. ID: ${u} Status: ${p}`),r.debug(`Full invalidation response: ${JSON.stringify(a.Invalidation)}`),r.endMark("invalidate"),await r.flush();}Q().catch(async i=>{console.error(i);try{let e=(await y().catch(()=>null))?.logDir??"logs",n=new h(e,"invalidate.log.jsonl","debug");n.error(`Unhandled error: ${i instanceof Error?i.message:String(i)}`,i instanceof Error?{stack:i.stack}:void 0),await n.flush();}catch{}process.exit(1);});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|
|
@@ -39,10 +39,10 @@
|
|
|
39
39
|
"release:npm": "WPSUITE_PREMIUM=true npm run build && npm publish --access=public"
|
|
40
40
|
},
|
|
41
41
|
"dependencies": {
|
|
42
|
-
"@smart-cloud/publisher-core": "^1.0
|
|
43
|
-
"@aws-sdk/client-cloudfront": "^3.
|
|
44
|
-
"@aws-sdk/client-s3": "^3.
|
|
45
|
-
"@aws-sdk/lib-storage": "^3.
|
|
42
|
+
"@smart-cloud/publisher-core": "^1.1.0",
|
|
43
|
+
"@aws-sdk/client-cloudfront": "^3.1051.0",
|
|
44
|
+
"@aws-sdk/client-s3": "^3.1051.0",
|
|
45
|
+
"@aws-sdk/lib-storage": "^3.1051.0",
|
|
46
46
|
"fast-glob": "^3.3.3",
|
|
47
47
|
"mime-types": "^3.0.2",
|
|
48
48
|
"playwright": "^1.60.0"
|
|
@@ -50,14 +50,14 @@
|
|
|
50
50
|
"devDependencies": {
|
|
51
51
|
"@eslint/js": "^10.0.1",
|
|
52
52
|
"@types/mime-types": "^3.0.1",
|
|
53
|
-
"@types/node": "^25.
|
|
54
|
-
"@typescript-eslint/eslint-plugin": "^8.59.
|
|
55
|
-
"@typescript-eslint/parser": "^8.59.
|
|
53
|
+
"@types/node": "^25.9.1",
|
|
54
|
+
"@typescript-eslint/eslint-plugin": "^8.59.4",
|
|
55
|
+
"@typescript-eslint/parser": "^8.59.4",
|
|
56
56
|
"esbuild": "^0.28.0",
|
|
57
|
-
"eslint": "^10.
|
|
57
|
+
"eslint": "^10.4.0",
|
|
58
58
|
"globals": "^17.6.0",
|
|
59
59
|
"tsup": "^8.5.1",
|
|
60
60
|
"typescript": "^6.0.3",
|
|
61
|
-
"typescript-eslint": "^8.59.
|
|
61
|
+
"typescript-eslint": "^8.59.4"
|
|
62
62
|
}
|
|
63
63
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
"targetOrigin": "https://wpsuite.io",
|
|
4
4
|
"outputDir": "export",
|
|
5
5
|
"seedPaths": ["/", "/gatey/", "/ai-kit/", "/flow/", "/blog/", "/docs/"],
|
|
6
|
+
"generated404RequestPath": "",
|
|
6
7
|
"allowedAssetHosts": ["wpsuite.local", "localhost", "dev.wpsuite.io"],
|
|
7
8
|
"blockedPathPrefixes": [
|
|
8
9
|
"/wp-admin",
|