@smart-cloud/publisher-exporter 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +4 -4
- package/dist/deploy.js +5 -5
- package/dist/queue-runner.js +3 -3
- package/dist/rewrite-worker.js +4 -0
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import P from'fs/promises';import
|
|
1
|
+
import {chromium}from'playwright';import P from'fs/promises';import w from'path';import _e from'crypto';import Je from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function B(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ne(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,a={};if(typeof i.targetOrigin=="string"){let u=ue(i.targetOrigin);u&&(a.targetOrigin=u);}let o=B(i.extraReplacements);if(Object.keys(o).length>0&&(a.extraReplacements=o),i.s3&&typeof i.s3=="object"){let u={},l=i.s3;typeof l.bucket=="string"&&(u.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(u.prefix=l.prefix.trim()),typeof l.region=="string"&&(u.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(u.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(u.assetCacheControl=l.assetCacheControl.trim()),Object.keys(u).length>0&&(a.s3=u);}if(i.cloudFront&&typeof i.cloudFront=="object"){let u={},l=i.cloudFront;typeof l.distributionId=="string"&&(u.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(u.invalidationPaths=l.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(u).length>0&&(a.cloudFront=u);}t[n]=a;}return t}function z(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function We(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function oe(e,t,r){let s=String(t||"").trim();return s&&w.isAbsolute(s)?w.resolve(s):w.resolve(e,z(s,r))}async function G(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await P.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=ue(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=B(r.extraReplacements),r.postCrawlCopyMap=B(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=Ne(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=We(),n=s?w.resolve(s,".."):"";return n?(r.outputDir=oe(n,r.outputDir,"export"),r.logDir=oe(n,r.logDir,"logs")):(w.isAbsolute(r.outputDir)||(r.outputDir=z(r.outputDir,"export")),w.isAbsolute(r.logDir)||(r.logDir=z(r.logDir,"logs"))),r}function ce(e,t=10){return _e.createHash("sha1").update(e).digest("hex").slice(0,t)}async function V(e){await P.mkdir(w.dirname(e),{recursive:true});}function O(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(w.extname(e).toLowerCase())}function L(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function M(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function F(e){return e.replace(/\//g,"\\/")}function le(e){return e.replace(/\//g,"\\\\/")}function de(e,t,r){if(!t)return;e[t]=r;let s=F(t),n=F(r);e[s]=n;let i=le(t),a=le(r);e[i]=a;let o=M(t),u=M(r);e[o]=u;let l=M(s),c=M(n);e[l]=c;let p=M(i),d=M(a);e[p]=d;}function K(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function pe(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function ge(e,t,r){if(!t)return r;let s=w.dirname(w.resolve(t)),n=w.resolve(e,r.replace(/^\/+/,"")),i=w.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Ge=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Ve=new Set([".html",".htm"]),he="WPSuite.io Static Publisher",Ke=he.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Xe(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=w.extname(r).toLowerCase();return Ve.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Ye(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Ke}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${he}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>it(t,r));}updateCurrentEvent(t,r,s){let n=ot(t,r,s);this.enqueueTask(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(E[this.level]>=E.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){nt("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),E[this.level]>=E.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),E[this.level]>=E.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var ut=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],lt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),ct=new Set(["",".html",".htm"]),dt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),pt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function gt(e){let t=[],r="full",s="full",n=false;for(let i=0;i<e.length;i++){let a=e[i];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--crawl-mode"){let o=e[++i];if(!o)throw new Error("--crawl-mode requires a value");s=o==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let o=e[++i];if(!o)throw new Error("--url requires a value");r="single-url",n=true,t.push(o);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let o=e[++i];if(!o)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${o}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n}}function te(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function mt(){let e=te();return e?f.resolve(e,".."):""}function ht(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function ft(e){let t=e.trim(),r=[{alias:"@storage-root",root:mt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:te(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:ht(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function je(e){let t=te();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}async function wt(e){try{let t=await P.readFile(je(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.schemaVersion===1&&r.pages&&typeof r.pages=="object")return r}catch{}return {schemaVersion:1,updatedAt:"",pages:{}}}function Se(e){return JSON.parse(JSON.stringify(e))}async function Pt(e,t){let r=je(e);await P.mkdir(f.dirname(r),{recursive:true}),await P.writeFile(r,JSON.stringify(t,null,2),"utf8");}function q(e){if(!e)return null;let t=e.trim();return t||null}async function yt(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:a,status:o.status()}),new Map;let l=await o.json().catch(()=>null);if(!l||!Array.isArray(l.items))return new Map;let u=new Map;for(let c of l.items)!c||typeof c.url!="string"||u.set(c.url,c);return u}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:String(o)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await yt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,a),a}async function St(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let a=s.manifest.pages[n];if(!a)return {action:"render",changeToken:await ve(e,t,s,n,i)};let o=await ve(e,t,s,n,i);if(o?.supported&&o.token)return a.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let l=q(r.sitemapLastmodByPage[n]);return l&&a.sitemapLastmod&&a.sitemapLastmod===l?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function vt(e,t,r,s){let n=be(r.previousManifest),i=0;for(let[d,g]of Object.entries(r.manifest.pages))if(!r.seenPages.has(d)){try{await P.unlink(g.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${d}`,{url:d,outputPath:g.outputPath,error:String(m)});}delete r.manifest.pages[d],i++;}let a=be(r.manifest),o=new Set,l=new Set;for(let d of a){for(let m of Re(e,d))o.add(m);let g=N(e,t.assetMap,d);g&&l.add(g);}let u=0,c=0;for(let d of n){if(a.has(d))continue;let g=new Set,m=N(e,t.assetMap,d);if(m)g.add(m);else {let h=ne(e,d);g.add(h.originalFilePath),h.hashedFilePath&&g.add(h.hashedFilePath);}for(let h of g)if(!l.has(h))try{await P.unlink(h),u++;}catch(p){(p.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${d}`,{url:d,outputPath:h,error:String(p)});}for(let h of Re(e,d))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],c++);}(i>0||u>0||c>0)&&s.info(`Incremental cleanup removed ${i} stale pages, ${u} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:u,removedAssetMappings:c,phase:"incremental-cleanup"});}function be(e){let t=new Set;for(let r of Object.values(e.pages))for(let s of r.discoveredAssets||[]){let n=s.trim();n&&t.add(n);}return t}function Re(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function bt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function N(e,t,r){let s=t[r];return s?bt(e,s):ne(e,r).originalFilePath}function Rt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function xe(e){let t=Rt();if(t)try{let r=await P.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await P.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function xt(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(se(e,s)));for(let s of t.doneAssets){let n=N(e,t.assetMap,s);!n||!F(n)||r.add(f.resolve(n));}return [...r]}var _="re:",Q=new Map;function Ct(e){if(!e.startsWith(_))return null;let t=Q.get(e);if(t!==void 0)return t||null;let r=e.slice(_.length).trim();if(!r)return Q.set(e,false),null;try{let s=new RegExp(r);return Q.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Q.set(e,false),null}}function re(e,t){let r=Ct(t);return r?r.test(e):t.startsWith(_)?false:e.startsWith(t)}function kt(e,t){return t.find(r=>re(e,r))}function De(e,t){let r=kt(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(_)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Lt(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Mt(e,t){return Lt(e).has(t.hostname)}function Tt(e){let t=L(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Oe(e){let t=L(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function H(e){return pt.some(t=>e.pathname.startsWith(t))}function v(e,t,r=e.sourceOrigin,s,n="url"){let i=Tt(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Oe(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let a=new URL(i,r);if(!["http:","https:"].includes(a.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Mt(e,a))return s?.ignore(n,a.toString(),"host not allowed",r),null;let o=De(e,a);if(o)return s?.reject(n,a.toString(),o,r),null;let l=new URL(e.sourceOrigin);return a.protocol=l.protocol,a.host=l.host,a.hash="",a}catch{return s?.reject(n,t,"invalid URL",r),null}}function Fe(e){return lt.has(f.extname(e.pathname).toLowerCase())}function D(e){return H(e)?false:ct.has(f.extname(e.pathname).toLowerCase())}function J(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function y(e,t){return J(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Fe(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>re(t.pathname,r)):false}function se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function ne(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=f.join(e.outputDir,i),o=K(e.targetOrigin,i),l=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:l,preferQueryHashed:n};let u=f.extname(s),d=`${u?s.slice(0,-u.length):s}.${ce(r.search)}${u||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:l,preferQueryHashed:n,hashedPathname:d,hashedFilePath:f.join(e.outputDir,d),hashedPublicUrl:K(e.targetOrigin,d)}}function At(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let l of o)try{let u=new URL(l);t[u.origin+i.pathname+i.search]=s;}catch{}try{let l=(()=>{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${u.pathname}${u.search}`})();t[i.pathname+i.search]=l;}catch{}try{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=l;}catch{}try{let l=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=l;}catch{}}catch{}}async function Z(e){try{return await P.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Ue(e){try{return await P.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Et(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Qe(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function It(e,t){let r=Qe(t),s=Et(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function $t(e,t){if(!t)return;let r=Qe(t);if(r===null)return;let s=await Ue(e);if(!s)return;let n=new Date(r);await P.utimes(e,s.atime,n).catch(()=>{});}function Ce(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return dt.has(r)||ut.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function ke(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ee(e,t,r,s,n,i,a){let o=ne(e,r),l=o.originalFilePath,u=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let d=await Z(o.originalFilePath);d!==null&&!d.equals(s)&&(l=o.hashedFilePath,u=o.hashedPublicUrl);}At(e,t,r,u,o.originalPublicUrl);let c=await Z(l);(c===null||!c.equals(s))&&(await V(l),await P.writeFile(l,s)),await $t(l,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"});}function W(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=v(e,r,i,s,"page");if(!a)return;if(!D(a)||Fe(a)){if(y(e,a)){s.info(`Seeded/non-page URL queued as asset: ${a.toString()}`,{url:a.toString(),source:n}),k(e,t,a.toString(),s,n);return}s.reject("page",a.toString(),H(a)?"asset/internal path cannot be page":"not page-like",n);return}let o=a.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=v(e,r,i,s,"asset");if(!a)return;if(!y(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let o=a.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ie(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=v(e,r,i,s,"sitemap");if(!a)return;if(!J(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let o=a.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function jt(e){let t=L(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:q(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Dt(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Ot(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Y(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Ft(e,t){await P.mkdir(t,{recursive:true});let r=await P.readdir(e,{withFileTypes:true});for(let s of r){let n=f.join(e,s.name),i=f.join(t,s.name);if(s.isDirectory()){await P.cp(n,i,{recursive:true,force:true});continue}await P.copyFile(n,i);}}async function Ut(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([n,i])=>({sourcePath:n.trim(),prefix:String(i||"").trim()}));if(r.length===0)return;let s=0;t.mark("copy-extra-paths");for(let n of r){let i=n.sourcePath,a=n.prefix;if(!i||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:i,prefixPath:a});continue}let o=ft(i);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:i,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let l=o.resolvedPath,u=Y(a);if(!u){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:i,prefixPath:a});continue}let c;try{c=await P.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:i,sourceAbs:l,prefixPath:a});continue}let d=f.resolve(e.outputDir,u);if(c.isDirectory()){await Ft(l,d),s++,t.progress(`Copied static directory to export: ${l} -> /${u}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${u}`,copiedItems:s});continue}let g=f.basename(l),m=a.endsWith("/")?f.join(d,g):d;await P.mkdir(f.dirname(m),{recursive:true}),await P.copyFile(l,m),s++,t.progress(`Copied static file to export: ${l} -> /${Y(a.endsWith("/")?`${a}${g}`:a)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${Y(a.endsWith("/")?`${a}${g}`:a)}`,copiedItems:s});}t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s});}function j(e,t,r,s,n){let i=new Set,a=L(r).replace(/\\\//g,"/");for(let u of Ot(a)){let c=v(e,u,t,s,`${n}:css-url`);c&&y(e,c)&&i.add(c.toString());}for(let u of a.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=v(e,u[1],t,s,`${n}:css-import`);c&&y(e,c)&&i.add(c.toString());}for(let u of a.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=v(e,u[1],t,s,`${n}:xml-stylesheet`);c&&y(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",l=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let u of a.matchAll(l)){let c=u[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${u[0]}`:u[0];if(Oe(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let d=v(e,c,t,s,`${n}:serialized-url`);d&&y(e,d)?i.add(d.toString()):d&&s.ignore("asset",d.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Qt(e,t,r,s){let n=new Set,i=L(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=a[1],l=v(e,o,t,s,"page-link");l&&(De(e,l)||D(l)&&!H(l)&&!y(e,l)&&n.add(l.toString()));}return [...n]}async function Le(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let a=await i.body(),o=i.url()||n;await ee(e,r.assetMap,o,a,r,s);let l=a.toString("utf8");for(let u of j(e,o,l,s,"sitemap"))k(e,r,u,s,o);for(let u of jt(l)){let c=v(e,u.loc,o,s,"sitemap-loc");c&&(J(c)?ie(e,r,c.toString(),s,o):y(e,c)?k(e,r,c.toString(),s,o):D(c)?(u.lastmod&&(r.sitemapLastmodByPage[c.toString()]=u.lastmod),W(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function qt(e,t,r,s,n){let i=v(e,s,e.sourceOrigin,n,"asset-fetch");if(!i||!y(e,i))return;let a=i.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let o=N(e,r.assetMap,a),l=o?await Ue(o):null;if(o&&l)try{let m=await t.head(a,{timeout:6e4}),h=m.headers();if(m.ok()&&It(l,h)){let p=await Z(o);if(p!==null){let w=m.url()||a;if(await ee(e,r.assetMap,w,p,r,n,h),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:o,lastModified:h["last-modified"]||"",contentLength:h["content-length"]||"",phase:"download-assets"}),ke(r,n),Ce(i,h)){let S=p.toString("utf8");for(let b of j(e,w,S,n,"asset:cached"))k(e,r,b,n,w);}return}}}catch{}let u=await t.get(a,{timeout:6e4});if(!u.ok()){n.skip("asset",a,u.status());return}let c=await u.body(),d=u.url()||a,g=u.headers();if(await ee(e,r.assetMap,d,c,r,n,g),r.doneAssets.add(a),ke(r,n),Ce(i,g)){let m=c.toString("utf8");for(let h of j(e,d,m,n,`asset:${f.extname(i.pathname).toLowerCase()||g["content-type"]||"unknown"}`))k(e,r,h,n,d);}}catch(o){n.error(`Failed asset ${a}`,{url:a,error:String(o)});}}}async function Nt(e,t,r,s){let n=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),i=Array.from({length:n},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await qt(e,t,r,a,s));}});await Promise.all(i);}async function _t(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Wt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await _t(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Ht(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Jt(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>re(t,r))}async function Bt(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let l=new Set,u=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,d=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(g=>{for(let h of u){let p=g.getAttribute(h);p&&l.add(p);}for(let h of d){let p=g.getAttribute(h);p&&p.split(",").forEach(w=>{let S=w.trim().split(/\s+/)[0];S&&l.add(S);});}let m=g.getAttribute("style");m&&c.add(m);}),{attrs:[...l],styles:[...c]}});for(let l of o.attrs)for(let u of Dt(l)){let c=v(e,u,r,s,"dom-attr");c&&y(e,c)&&i.add(c.toString());}for(let l of o.styles)for(let u of j(e,r,l,s,"dom-style"))i.add(u);}let a=await t.content();for(let o of j(e,r,a,s,"page-html"))i.add(o);return [...i]}async function Me(e,t,r,s,n){let i=se(e,t);await V(i),await P.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}async function zt(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Gt(e,t,r,s,n,i){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await zt(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let u=t.pageQueue.shift();if(!u)break;if(t.queuedPages.delete(u),t.donePages.has(u))continue;t.donePages.add(u),i.enabled&&i.seenPages.add(u);let c=new URL(u),d=Jt(e,c.pathname);if(!D(c)||y(e,c)||H(c)){if(y(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${u}`,{url:u,source:"worker-guard"}),k(e,t,u,r,"worker-guard");continue}r.reject("page",u,"guard rejected non-page URL before rendering","worker");continue}let g=await St(e,n,t,i,u,r);if(i.enabled&&s!=="single-url"){let p=i.manifest.pages[u];if(g.action==="reuse"&&p){for(let w of p.discoveredAssets)k(e,t,w,r,u);for(let w of p.discoveredPages)W(e,t,w,r,u);p.lastSeenRunId=i.runId,p.sitemapLastmod=q(t.sitemapLastmodByPage[u])??p.sitemapLastmod,g.changeToken?.supported&&(p.changeToken=g.changeToken.token,p.tokenSource=g.changeToken.tokenSource??p.tokenSource),r.info(`Incremental reuse skipped unchanged page ${u}`,{url:u,mode:"incremental",reason:g.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}let m=await(d?l.newPage():o.newPage()),h=!1;await m.route("**/*",async p=>{let w=p.request();if(w.isNavigationRequest()&&w.frame()===m.mainFrame())try{let S=new URL(w.url()),b=new URL(u);if(S.hash="",b.hash="",S.href===b.href){if(h){r.warn(`Blocked same-page navigation/reload for ${u}`,{url:u,requestUrl:w.url()}),await p.abort("aborted");return}h=!0;}}catch{}await p.continue();}),m.on("response",p=>{try{let w=v(e,p.url(),e.sourceOrigin,void 0,"network-response");w&&y(e,w)&&k(e,t,w.toString(),r,u);}catch{}}),r.page(`Rendering ${u}`,{url:u});try{let p=null,w=null;try{if(p=await m.goto(u,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),p&&p.ok())try{w=await p.text();}catch{w=null;}}catch(I){r.warn(`Navigation issue for ${u}; saving current DOM if available`,{url:u,error:String(I)});}if(p&&!p.ok()){r.skip("page",u,p.status()),await m.close();continue}d?(r.info(`Rendering without JS execution for ${u}`,{url:u,mode:"no-js"}),await Ht(e,m)):await Wt(e,m),t.stats.pagesRendered++,(t.stats.pagesRendered%5===0||t.stats.pagesRendered===1||t.stats.pagesRendered===t.stats.pagesQueued)&&r.progress(`Render progress: rendered ${t.stats.pagesRendered}, saved ${t.stats.pagesSaved}, discovered ${t.stats.pagesQueued}.`,{pagesRendered:t.stats.pagesRendered,pagesSaved:t.stats.pagesSaved,pagesQueued:t.stats.pagesQueued,phase:"render-pages"});let S=await Bt(e,m,u,r,!d);for(let I of S)k(e,t,I,r,u);let b=w??await m.content(),ae=s!=="single-url"?Qt(e,u,b,r):[];if(s!=="single-url")for(let I of ae)W(e,t,I,r,u);await Me(e,u,b,t,r),i.enabled&&(i.manifest.pages[u]={url:u,outputPath:se(e,u),changeToken:g.changeToken?.supported?g.changeToken.token:null,tokenSource:g.changeToken?.supported?g.changeToken.tokenSource??null:null,sitemapLastmod:q(t.sitemapLastmodByPage[u])??null,discoveredPages:ae,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(p){try{let w=await m.content();w&&w.trim()&&(await Me(e,u,w,t,r),r.warn(`Saved partial DOM for ${u}`,{url:u,error:String(p)}));}catch(w){r.error(`Could not save partial DOM for ${u}`,{url:u,error:String(w)});}r.error(`Failed page ${u}`,{url:u,error:String(p)});}finally{await m.close();}}}finally{await l.close(),await o.close(),await a.close();}}async function Te(e){try{let r=(await P.readFile(e,"utf8")).trim();if(!r)return [];try{let s=JSON.parse(r);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return r.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let n=JSON.parse(s);return n&&typeof n=="object"?n:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function Ae(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Vt(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function Kt(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function Xt(e){try{let t=await P.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}async function Ee(e){let t=await Te(f.join(e,"errors.jsonl"));return t.length>0?t:Te(f.join(e,"errors.json"))}async function Yt(e){let t=f.join(e.logDir,"archive"),r;try{r=await P.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=f.join(t,i);if(!(await P.stat(a).catch(()=>null))?.isDirectory())continue;let l=await Xt(f.join(a,"job.json"));if(!Vt(l))continue;let u=Kt(l);u>=n&&(n=u,s=a);}return s}async function Zt(e){let t=await Yt(e),r=t!==""?await Ee(t):await Ee(e.logDir),s=new Set;for(let n of r)if(Ae(n.error)||Ae(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function er(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await P.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function tr(e){try{let t=await P.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function rr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await P.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await P.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ie(e,t,r,s,n){let i=v(e,r,e.sourceOrigin,s,"manual-url");i&&(J(i)?ie(e,t,i.toString(),s,n):y(e,i)?k(e,t,i.toString(),s,n):D(i)?W(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function sr(){let e=await G(),t=gt(process.argv.slice(2)),r=[];t.mode==="retry-timeouts"&&(r=await Zt(e)),t.mode==="single-url"&&(r=await er(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s?await wt(e):{schemaVersion:1,updatedAt:"",pages:{}},i={enabled:s,manifest:Se(n),previousManifest:Se(n),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await rr(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:t.preserveOutput?await tr(e):{},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}};t.preserveOutput||await P.rm(e.outputDir,{recursive:true,force:true}),await P.mkdir(e.outputDir,{recursive:true});let o=new $(e.logDir,e.logLevel),l=await chromium.launch({headless:true}),u=await l.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=u.request;try{if(o.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(p=>ie(e,a,p,o)),await Le(e,c,a,o),e.seedPaths.forEach(p=>Ie(e,a,p,o,"seed-path"))):(r.forEach(p=>Ie(e,a,p,o,"cli")),await Le(e,c,a,o)),o.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),o.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){o.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await o.flush();return}o.mark("render-pages");let h=Array.from({length:Math.max(1,e.concurrency)},()=>Gt(e,a,o,t.mode,c,i));await Promise.all(h),o.endMark("render-pages",{pages:a.donePages.size}),o.mark("download-assets"),await Nt(e,c,a,o),o.endMark("download-assets",{assets:a.doneAssets.size}),i.enabled&&e.maxPages===0&&await vt(e,a,i,o),t.mode==="full"&&await Ut(e,o);}finally{await u.close(),await l.close();}i.enabled&&(i.manifest.updatedAt=new Date().toISOString(),await Pt(e,i.manifest)),await P.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8");let d=t.mode==="single-url"?xt(e,a):void 0;o.mark("rewrite-text");let g=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${d?.length??0}`:"Rewriting text files...";o.progress(g,{phase:"rewrite-text",index:0,totalFiles:d?.length,changedTextFiles:0}),await xe(g);let m=await fe(e,a.assetMap,{files:d,onProgress:async({index:h,totalFiles:p,changedTextFiles:w,file:S})=>{if(o.checkpoint(`Rewriting text file ${h}/${p}`,{phase:"rewrite-text",index:h,totalFiles:p,changedTextFiles:w,file:S}),h!==1&&h!==p&&h%25!==0)return;let b=`Rewriting text files: ${h}/${p}`;o.progress(b,{phase:"rewrite-text",index:h,totalFiles:p,changedTextFiles:w,file:S}),await xe(b);}});o.endMark("rewrite-text",{changedTextFiles:m}),o.summary(`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:m}),await o.flush();}sr().catch(async e=>{console.error(e);try{let r=(await G().catch(()=>null))?.logDir??"logs",s=new $(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Ze(e){let t=L(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function et(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,F(i)]).filter(([i,a])=>i!==a).sort((i,a)=>a[0].length-i[0].length);if(!r.length)return e;let s=i=>{let a=i;for(let[o,u]of r)a=a.split(o).join(u);return a},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,a,o,u)=>`${a}${s(o)}${u}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,a,o,u,l)=>Ze(u)?`${a}${s(u)}${l}`:i),n}function tt(e,t){if(!t)return null;let r=w.dirname(w.resolve(t)),s=w.resolve(e.outputDir),n=w.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function rt(e,t,r){let s=tt(t,r);if(!s)return e;let n=e;for(let i of Ge){let a=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),o=new RegExp(`(?:\\.{1,}/)+${a}`,"g"),u=new RegExp(`(?<!\\.)/${a}`,"g");n=n.replace(o,`${s}${i}`).replace(u,`${s}${i}`);let l=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),p=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),h=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${c}${l}`).replace(g,`${c}${l}`).replace(h,`${c}${l}`);}return n}function st(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:pe(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${ge(e.outputDir,r,n)}${i}`}function X(e,t,r,s,n){de(e,r,st(t,s,n));}function nt(e,t,r,s){let n=e,i={};X(i,t,t.sourceOrigin,t.targetOrigin,s);for(let[o,u]of Object.entries(t.extraReplacements))X(i,t,o,u,s);for(let[o,u]of Object.entries(r))X(i,t,o,u,s);let a=Object.entries(i).sort((o,u)=>u[0].length-o[0].length);for(let[o,u]of a)n=n.split(o).join(u);return t.urlRewriteMode!=="absolute"&&(n=rt(n,t,s)),n=et(n,a.map(([,o])=>o)),Xe(t,n,s)&&(n=Ye(n)),n}async function it(e,t,r){let s=w.join(e.outputDir,r),n=w.extname(s).toLowerCase(),i;try{i=await P.readFile(s,"utf8");}catch(u){if(u.code!=="ENOENT")throw u;return {changed:false}}let o=nt(i,e,t,n===".js"||n===".mjs"?void 0:s);return o!==i?(await P.writeFile(s,o,"utf8"),{changed:true}):{changed:false}}async function at(e,t){return await new Promise((r,s)=>{let n=u=>{if(o(),u?.error){s(new Error(u.error));return}r({changed:!!u?.changed});},i=u=>{o(),s(u);},a=u=>{o(),u!==0&&s(new Error(`Rewrite worker exited with code ${u}.`));},o=()=>{e.off("message",n),e.off("error",i),e.off("exit",a);};e.on("message",n),e.on("error",i),e.on("exit",a),e.postMessage({file:t});})}async function fe(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>w.resolve(d)).filter(d=>d.startsWith(w.resolve(e.outputDir))).map(d=>w.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Je(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&O(d)),i=0;if(n.length===0)return 0;let a=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),o=Math.max(1,Math.min(a,n.length,availableParallelism())),u=0,l=0,c=null;if(o===1){for(let d of n)(await it(e,t,d)).changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:d});return i}let p=Array.from({length:o},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t}});try{for(;!c;){let g=u;if(g>=n.length)return;u+=1;let h=n[g];(await at(d,h)).changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:h});}}catch(g){throw c=g instanceof Error?g:new Error(String(g)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var A={error:0,warn:1,info:2,debug:3},we=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",U=we?w.join(we,"current-progress.json"):"",Pe=Promise.resolve();function ye(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function ot(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let a=r[n];if(typeof a=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(a,i);continue}r[n]=i;}return r}function ut(e,t,r){U&&(Pe=Pe.then(async()=>{let s=new Date,n=s.toISOString();await P.mkdir(w.dirname(U),{recursive:true});let i=await P.readFile(U,"utf8").then(m=>JSON.parse(m)).catch(()=>null),a=i&&typeof i.details=="object"&&i.details?i.details:{},o=ye(e),u=i?.currentStep||ye(i?.source||""),l=i?.startedAt||n,c={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(u&&u!==o&&i?.stepStartedAt){let m=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[u]=(c[u]??0)+m,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),h={checkedAt:n,source:e,message:t,details:ot(a,r??{}),startedAt:l,currentStep:o,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:c};await P.writeFile(U,JSON.stringify(h,null,2),"utf8");}).catch(()=>{}));}async function lt(e,t){await P.mkdir(w.dirname(e),{recursive:true}),await P.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function ct(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function dt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:ct(e,r),level:e,message:t,details:r??{}}}var $=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return w.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return w.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return w.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return w.join(this.logDir,"ignored.jsonl")}get skippedPath(){return w.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return w.join(this.logDir,"errors.jsonl")}get timingsPath(){return w.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await P.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>lt(t,r));}updateCurrentEvent(t,r,s){let n=dt(t,r,s);this.enqueueTask(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(A[this.level]>=A.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),A[this.level]>=A.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),A[this.level]>=A.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var pt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],gt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),mt=new Set(["",".html",".htm"]),ht=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),ft=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function wt(e){let t=[],r="full",s="full",n=false,i=false;for(let a=0;a<e.length;a++){let o=e[a];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let u=e[++a];if(!u)throw new Error("--crawl-mode requires a value");s=u==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let u=e[++a];if(!u)throw new Error("--url requires a value");r="single-url",n=true,t.push(u);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let u=e[++a];if(!u)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${u}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function te(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function Pt(){let e=te();return e?w.resolve(e,".."):""}function yt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?w.resolve(e):""}function St(e){let t=e.trim(),r=[{alias:"@storage-root",root:Pt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:te(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:yt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?w.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:w.resolve(t)}}function De(e){let t=te();return t?w.join(t,"crawl-manifest.json"):w.join(e.outputDir,".crawl-manifest.json")}async function vt(e){try{let t=await P.readFile(De(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.schemaVersion===1&&r.pages&&typeof r.pages=="object")return r}catch{}return {schemaVersion:1,updatedAt:"",pages:{}}}function Se(e){return JSON.parse(JSON.stringify(e))}async function bt(e,t){let r=De(e);await P.mkdir(w.dirname(r),{recursive:true}),await P.writeFile(r,JSON.stringify(t,null,2),"utf8");}function q(e){if(!e)return null;let t=e.trim();return t||null}async function Rt(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:a,status:o.status()}),new Map;let u=await o.json().catch(()=>null);if(!u||!Array.isArray(u.items))return new Map;let l=new Map;for(let c of u.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:String(o)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await Rt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,a),a}async function xt(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let a=s.manifest.pages[n];if(!a)return {action:"render",changeToken:await ve(e,t,s,n,i)};let o=await ve(e,t,s,n,i);if(o?.supported&&o.token)return a.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let u=q(r.sitemapLastmodByPage[n]);return u&&a.sitemapLastmod&&a.sitemapLastmod===u?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function Ct(e,t,r,s){let n=be(r.previousManifest),i=0;for(let[p,d]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){try{await P.unlink(d.outputPath);}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:d.outputPath,error:String(g)});}delete r.manifest.pages[p],i++;}let a=be(r.manifest),o=new Set,u=new Set;for(let p of a){for(let g of Re(e,p))o.add(g);let d=N(e,t.assetMap,p);d&&u.add(d);}let l=0,c=0;for(let p of n){if(a.has(p))continue;let d=new Set,g=N(e,t.assetMap,p);if(g)d.add(g);else {let h=ne(e,p);d.add(h.originalFilePath),h.hashedFilePath&&d.add(h.hashedFilePath);}for(let h of d)if(!u.has(h))try{await P.unlink(h),l++;}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:h,error:String(m)});}for(let h of Re(e,p))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],c++);}(i>0||l>0||c>0)&&s.info(`Incremental cleanup removed ${i} stale pages, ${l} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:l,removedAssetMappings:c,phase:"incremental-cleanup"});}function be(e){let t=new Set;for(let r of Object.values(e.pages))for(let s of r.discoveredAssets||[]){let n=s.trim();n&&t.add(n);}return t}function Re(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function kt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return w.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function N(e,t,r){let s=t[r];return s?kt(e,s):ne(e,r).originalFilePath}function Lt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.join(w.resolve(e),"queue-runner-heartbeat.json"):""}async function xe(e){let t=Lt();if(t)try{let r=await P.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await P.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Et(e,t){let r=new Set;for(let s of t.donePages)r.add(w.resolve(se(e,s)));for(let s of t.doneAssets){let n=N(e,t.assetMap,s);!n||!O(n)||r.add(w.resolve(n));}return [...r]}var W="re:",Q=new Map;function Mt(e){if(!e.startsWith(W))return null;let t=Q.get(e);if(t!==void 0)return t||null;let r=e.slice(W.length).trim();if(!r)return Q.set(e,false),null;try{let s=new RegExp(r);return Q.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Q.set(e,false),null}}function re(e,t){let r=Mt(t);return r?r.test(e):t.startsWith(W)?false:e.startsWith(t)}function Tt(e,t){return t.find(r=>re(e,r))}function je(e,t){let r=Tt(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(W)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function At(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function It(e,t){return At(e).has(t.hostname)}function $t(e){let t=L(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Fe(e){let t=L(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function H(e){return ft.some(t=>e.pathname.startsWith(t))}function S(e,t,r=e.sourceOrigin,s,n="url"){let i=$t(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Fe(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let a=new URL(i,r);if(!["http:","https:"].includes(a.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!It(e,a))return s?.ignore(n,a.toString(),"host not allowed",r),null;let o=je(e,a);if(o)return s?.reject(n,a.toString(),o,r),null;let u=new URL(e.sourceOrigin);return a.protocol=u.protocol,a.host=u.host,a.hash="",a}catch{return s?.reject(n,t,"invalid URL",r),null}}function Oe(e){return gt.has(w.extname(e.pathname).toLowerCase())}function j(e){return H(e)?false:mt.has(w.extname(e.pathname).toLowerCase())}function J(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function y(e,t){return J(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Oe(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>re(t.pathname,r)):false}function se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":w.extname(s)||(s+="/index.html"),w.join(e.outputDir,s)}function ne(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=w.join(e.outputDir,i),o=K(e.targetOrigin,i),u=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:u,preferQueryHashed:n};let l=w.extname(s),p=`${l?s.slice(0,-l.length):s}.${ce(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:u,preferQueryHashed:n,hashedPathname:p,hashedFilePath:w.join(e.outputDir,p),hashedPublicUrl:K(e.targetOrigin,p)}}function Dt(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let u of o)try{let l=new URL(u);t[l.origin+i.pathname+i.search]=s;}catch{}try{let u=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=u;}catch{}try{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}try{let u=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}}catch{}}async function Z(e){try{return await P.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Ue(e){try{return await P.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function jt(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Qe(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Ft(e,t){let r=Qe(t),s=jt(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function Ot(e,t){if(!t)return;let r=Qe(t);if(r===null)return;let s=await Ue(e);if(!s)return;let n=new Date(r);await P.utimes(e,s.atime,n).catch(()=>{});}function Ce(e,t){let r=w.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return ht.has(r)||pt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function ke(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ee(e,t,r,s,n,i,a){let o=ne(e,r),u=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let p=await Z(o.originalFilePath);p!==null&&!p.equals(s)&&(u=o.hashedFilePath,l=o.hashedPublicUrl);}Dt(e,t,r,l,o.originalPublicUrl);let c=await Z(u);(c===null||!c.equals(s))&&(await V(u),await P.writeFile(u,s)),await Ot(u,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"});}function _(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"page");if(!a)return;if(!j(a)||Oe(a)){if(y(e,a)){s.info(`Seeded/non-page URL queued as asset: ${a.toString()}`,{url:a.toString(),source:n}),k(e,t,a.toString(),s,n);return}s.reject("page",a.toString(),H(a)?"asset/internal path cannot be page":"not page-like",n);return}let o=a.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"asset");if(!a)return;if(!y(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let o=a.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ie(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"sitemap");if(!a)return;if(!J(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let o=a.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Ut(e){let t=L(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:q(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Qt(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function qt(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Y(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Nt(e,t){await P.mkdir(t,{recursive:true});let r=await P.readdir(e,{withFileTypes:true});for(let s of r){let n=w.join(e,s.name),i=w.join(t,s.name);if(s.isDirectory()){await P.cp(n,i,{recursive:true,force:true});continue}await P.copyFile(n,i);}}async function Wt(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([n,i])=>({sourcePath:n.trim(),prefix:String(i||"").trim()}));if(r.length===0)return;let s=0;t.mark("copy-extra-paths");for(let n of r){let i=n.sourcePath,a=n.prefix;if(!i||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:i,prefixPath:a});continue}let o=St(i);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:i,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let u=o.resolvedPath,l=Y(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:i,prefixPath:a});continue}let c;try{c=await P.stat(u);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:i,sourceAbs:u,prefixPath:a});continue}let p=w.resolve(e.outputDir,l);if(c.isDirectory()){await Nt(u,p),s++,t.progress(`Copied static directory to export: ${u} -> /${l}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${l}`,copiedItems:s});continue}let d=w.basename(u),g=a.endsWith("/")?w.join(p,d):p;await P.mkdir(w.dirname(g),{recursive:true}),await P.copyFile(u,g),s++,t.progress(`Copied static file to export: ${u} -> /${Y(a.endsWith("/")?`${a}${d}`:a)}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${Y(a.endsWith("/")?`${a}${d}`:a)}`,copiedItems:s});}t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s});}function D(e,t,r,s,n){let i=new Set,a=L(r).replace(/\\\//g,"/");for(let l of qt(a)){let c=S(e,l,t,s,`${n}:css-url`);c&&y(e,c)&&i.add(c.toString());}for(let l of a.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=S(e,l[1],t,s,`${n}:css-import`);c&&y(e,c)&&i.add(c.toString());}for(let l of a.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=S(e,l[1],t,s,`${n}:xml-stylesheet`);c&&y(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",u=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of a.matchAll(u)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(Fe(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=S(e,c,t,s,`${n}:serialized-url`);p&&y(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function _t(e,t,r,s){let n=new Set,i=L(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=a[1],u=S(e,o,t,s,"page-link");u&&(je(e,u)||j(u)&&!H(u)&&!y(e,u)&&n.add(u.toString()));}return [...n]}async function Le(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let a=await i.body(),o=i.url()||n;await ee(e,r.assetMap,o,a,r,s);let u=a.toString("utf8");for(let l of D(e,o,u,s,"sitemap"))k(e,r,l,s,o);for(let l of Ut(u)){let c=S(e,l.loc,o,s,"sitemap-loc");c&&(J(c)?ie(e,r,c.toString(),s,o):y(e,c)?k(e,r,c.toString(),s,o):j(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),_(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Ht(e,t,r,s,n){let i=S(e,s,e.sourceOrigin,n,"asset-fetch");if(!i||!y(e,i))return;let a=i.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let o=N(e,r.assetMap,a),u=o?await Ue(o):null;if(o&&u)try{let g=await t.head(a,{timeout:6e4}),h=g.headers();if(g.ok()&&Ft(u,h)){let m=await Z(o);if(m!==null){let f=g.url()||a;if(await ee(e,r.assetMap,f,m,r,n,h),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:o,lastModified:h["last-modified"]||"",contentLength:h["content-length"]||"",phase:"download-assets"}),ke(r,n),Ce(i,h)){let v=m.toString("utf8");for(let b of D(e,f,v,n,"asset:cached"))k(e,r,b,n,f);}return}}}catch{}let l=await t.get(a,{timeout:6e4});if(!l.ok()){n.skip("asset",a,l.status());return}let c=await l.body(),p=l.url()||a,d=l.headers();if(await ee(e,r.assetMap,p,c,r,n,d),r.doneAssets.add(a),ke(r,n),Ce(i,d)){let g=c.toString("utf8");for(let h of D(e,p,g,n,`asset:${w.extname(i.pathname).toLowerCase()||d["content-type"]||"unknown"}`))k(e,r,h,n,p);}}catch(o){n.error(`Failed asset ${a}`,{url:a,error:String(o)});}}}async function Jt(e,t,r,s){let n=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),i=Array.from({length:n},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Ht(e,t,r,a,s));}});await Promise.all(i);}async function Bt(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function zt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Bt(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Gt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Vt(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>re(t,r))}async function Kt(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let u=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let h of l){let m=d.getAttribute(h);m&&u.add(m);}for(let h of p){let m=d.getAttribute(h);m&&m.split(",").forEach(f=>{let v=f.trim().split(/\s+/)[0];v&&u.add(v);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...u],styles:[...c]}});for(let u of o.attrs)for(let l of Qt(u)){let c=S(e,l,r,s,"dom-attr");c&&y(e,c)&&i.add(c.toString());}for(let u of o.styles)for(let l of D(e,r,u,s,"dom-style"))i.add(l);}let a=await t.content();for(let o of D(e,r,a,s,"page-html"))i.add(o);return [...i]}async function Ee(e,t,r,s,n){let i=se(e,t);await V(i),await P.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}async function Xt(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Yt(e,t,r,s,n,i){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),u=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Xt(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let c=new URL(l),p=Vt(e,c.pathname);if(!j(c)||y(e,c)||H(c)){if(y(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),k(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let d=await xt(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let m=i.manifest.pages[l];if(d.action==="reuse"&&m){for(let f of m.discoveredAssets)k(e,t,f,r,l);for(let f of m.discoveredPages)_(e,t,f,r,l);m.lastSeenRunId=i.runId,m.sitemapLastmod=q(t.sitemapLastmodByPage[l])??m.sitemapLastmod,d.changeToken?.supported&&(m.changeToken=d.changeToken.token,m.tokenSource=d.changeToken.tokenSource??m.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:d.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}let g=await(p?u.newPage():o.newPage()),h=!1;await g.route("**/*",async m=>{let f=m.request();if(f.isNavigationRequest()&&f.frame()===g.mainFrame())try{let v=new URL(f.url()),b=new URL(l);if(v.hash="",b.hash="",v.href===b.href){if(h){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:f.url()}),await m.abort("aborted");return}h=!0;}}catch{}await m.continue();}),g.on("response",m=>{try{let f=S(e,m.url(),e.sourceOrigin,void 0,"network-response");f&&y(e,f)&&k(e,t,f.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let m=null,f=null;try{if(m=await g.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{f=await m.text();}catch{f=null;}}catch(I){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(I)});}if(m&&!m.ok()){r.skip("page",l,m.status()),await g.close();continue}p?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Gt(e,g)):await zt(e,g),t.stats.pagesRendered++,(t.stats.pagesRendered%5===0||t.stats.pagesRendered===1||t.stats.pagesRendered===t.stats.pagesQueued)&&r.progress(`Render progress: rendered ${t.stats.pagesRendered}, saved ${t.stats.pagesSaved}, discovered ${t.stats.pagesQueued}.`,{pagesRendered:t.stats.pagesRendered,pagesSaved:t.stats.pagesSaved,pagesQueued:t.stats.pagesQueued,phase:"render-pages"});let v=await Kt(e,g,l,r,!p);for(let I of v)k(e,t,I,r,l);let b=f??await g.content(),ae=s!=="single-url"?_t(e,l,b,r):[];if(s!=="single-url")for(let I of ae)_(e,t,I,r,l);await Ee(e,l,b,t,r),i.enabled&&(i.manifest.pages[l]={url:l,outputPath:se(e,l),changeToken:d.changeToken?.supported?d.changeToken.token:null,tokenSource:d.changeToken?.supported?d.changeToken.tokenSource??null:null,sitemapLastmod:q(t.sitemapLastmodByPage[l])??null,discoveredPages:ae,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let f=await g.content();f&&f.trim()&&(await Ee(e,l,f,t,r),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(m)}));}catch(f){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(f)});}r.error(`Failed page ${l}`,{url:l,error:String(m)});}finally{await g.close();}}}finally{await u.close(),await o.close(),await a.close();}}async function Me(e){try{let r=(await P.readFile(e,"utf8")).trim();if(!r)return [];try{let s=JSON.parse(r);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return r.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let n=JSON.parse(s);return n&&typeof n=="object"?n:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function Te(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Zt(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function er(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function tr(e){try{let t=await P.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}async function Ae(e){let t=await Me(w.join(e,"errors.jsonl"));return t.length>0?t:Me(w.join(e,"errors.json"))}async function rr(e){let t=w.join(e.logDir,"archive"),r;try{r=await P.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=w.join(t,i);if(!(await P.stat(a).catch(()=>null))?.isDirectory())continue;let u=await tr(w.join(a,"job.json"));if(!Zt(u))continue;let l=er(u);l>=n&&(n=l,s=a);}return s}async function sr(e){let t=await rr(e),r=t!==""?await Ae(t):await Ae(e.logDir),s=new Set;for(let n of r)if(Te(n.error)||Te(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await P.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function ir(e){try{let t=await P.readFile(w.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ar(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await P.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await P.unlink(w.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ie(e,t,r,s,n){let i=S(e,r,e.sourceOrigin,s,"manual-url");i&&(J(i)?ie(e,t,i.toString(),s,n):y(e,i)?k(e,t,i.toString(),s,n):j(i)?_(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function or(){let e=await G(),t=wt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await sr(e)),t.mode==="single-url"&&(r=await nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s?await vt(e):{schemaVersion:1,updatedAt:"",pages:{}},i={enabled:s,manifest:Se(n),previousManifest:Se(n),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await ar(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:t.preserveOutput?await ir(e):{},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}};t.preserveOutput||await P.rm(e.outputDir,{recursive:true,force:true}),await P.mkdir(e.outputDir,{recursive:true});let o=new $(e.logDir,e.logLevel);if(t.resumeRewrite)o.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let d=await chromium.launch({headless:true}),g=await d.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),h=g.request;try{if(o.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(f=>ie(e,a,f,o)),await Le(e,h,a,o),e.seedPaths.forEach(f=>Ie(e,a,f,o,"seed-path"))):(r.forEach(f=>Ie(e,a,f,o,"cli")),await Le(e,h,a,o)),o.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),o.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){o.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await o.flush();return}o.mark("render-pages");let m=Array.from({length:Math.max(1,e.concurrency)},()=>Yt(e,a,o,t.mode,h,i));await Promise.all(m),o.endMark("render-pages",{pages:a.donePages.size}),o.mark("download-assets"),await Jt(e,h,a,o),o.endMark("download-assets",{assets:a.doneAssets.size}),i.enabled&&e.maxPages===0&&await Ct(e,a,i,o),t.mode==="full"&&await Wt(e,o);}finally{await g.close(),await d.close();}}i.enabled&&!t.resumeRewrite&&(i.manifest.updatedAt=new Date().toISOString(),await bt(e,i.manifest)),await P.writeFile(w.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8");let u=t.mode==="single-url"?Et(e,a):void 0;o.mark("rewrite-text");let l=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${u?.length??0}`:"Rewriting text files...";o.progress(l,{phase:"rewrite-text",index:0,totalFiles:u?.length,changedTextFiles:0}),await xe(l);let c=Date.now(),p=await fe(e,a.assetMap,{files:u,onProgress:async({index:d,totalFiles:g,changedTextFiles:h,file:m})=>{o.checkpoint(`Rewriting text file ${d}/${g}`,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m});let f=Date.now();if(!(d===1||d===g||f-c>=5e3))return;c=f;let b=`Rewriting text files: ${d}/${g}`;o.progress(b,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m}),await xe(b);}});o.endMark("rewrite-text",{changedTextFiles:p}),o.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${p} text files.`:`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:p}),await o.flush();}or().catch(async e=>{console.error(e);try{let r=(await G().catch(()=>null))?.logDir??"logs",s=new $(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/dist/deploy.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {S3Client,ListObjectsV2Command,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {spawn}from'child_process';import {createHash}from'crypto';import Ae from'fast-glob';import $ from'fs/promises';import v from'path';import
|
|
2
|
-
`,"utf8");}var
|
|
3
|
-
`,"utf8");});}enqueueJsonLine(r,e){this.enqueueTask(()=>Te(r,e));}accepts(r){return oe[r]<=oe[this.level]}push(r,e,
|
|
1
|
+
import {S3Client,ListObjectsV2Command,HeadObjectCommand,GetObjectCommand,DeleteObjectsCommand}from'@aws-sdk/client-s3';import {Upload}from'@aws-sdk/lib-storage';import {spawn}from'child_process';import {createHash}from'crypto';import Ae from'fast-glob';import $ from'fs/promises';import v from'path';import ot from'mime-types';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ne(t){let r=t.trim();return r==="."?".":r.replace(/\/$/,"")}function K(t){return !t||typeof t!="object"?{}:Object.fromEntries(Object.entries(t).map(([r,e])=>[r.trim(),String(e??"")]).filter(([r])=>r.length>0))}function Ie(t){if(!t||typeof t!="object")return {};let r={};for(let[e,i]of Object.entries(t)){let s=e.trim();if(!s||!i||typeof i!="object")continue;let n=i,o={};if(typeof n.targetOrigin=="string"){let a=ne(n.targetOrigin);a&&(o.targetOrigin=a);}let l=K(n.extraReplacements);if(Object.keys(l).length>0&&(o.extraReplacements=l),n.s3&&typeof n.s3=="object"){let a={},c=n.s3;typeof c.bucket=="string"&&(a.bucket=c.bucket.trim()),typeof c.prefix=="string"&&(a.prefix=c.prefix.trim()),typeof c.region=="string"&&(a.region=c.region.trim()),typeof c.htmlCacheControl=="string"&&(a.htmlCacheControl=c.htmlCacheControl.trim()),typeof c.assetCacheControl=="string"&&(a.assetCacheControl=c.assetCacheControl.trim()),Object.keys(a).length>0&&(o.s3=a);}if(n.cloudFront&&typeof n.cloudFront=="object"){let a={},c=n.cloudFront;typeof c.distributionId=="string"&&(a.distributionId=c.distributionId.trim()),Array.isArray(c.invalidationPaths)&&(a.invalidationPaths=c.invalidationPaths.map(u=>String(u??"").trim()).filter(u=>u.length>0)),Object.keys(a).length>0&&(o.cloudFront=a);}r[s]=o;}return r}function q(t,r){let i=String(t||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!i)return r;let s=i.split("/").map(n=>n.trim()).filter(n=>n.length>0&&n!=="."&&n!=="..");return s.length>0?s.join("/"):r}function Fe(){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return t.trim()?v.resolve(t):""}function re(t,r,e){let i=String(r||"").trim();return i&&v.isAbsolute(i)?v.resolve(i):v.resolve(t,q(i,e))}async function Q(){let t=process.env.PUBLISHER_CONFIG||"publisher.config.json",r=await $.readFile(t,"utf8"),e=JSON.parse(r);e.sourceOrigin=e.sourceOrigin.replace(/\/$/,""),e.targetOrigin=ne(e.targetOrigin),e.ignoreHttpsErrors??=false,e.outputDir=String(e.outputDir||"export").trim()||"export",e.urlRewriteMode||=e.targetOrigin==="."?"relative":"absolute",e.noJavaScriptRenderPathPrefixes||=[],e.seedPaths||=[],e.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],e.allowedAssetHosts||=[],e.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],e.blockedPathPrefixes||=[],e.blockedSearchFragments||=[],e.concurrency||=1,e.maxPages||=0,e.extraReplacements=K(e.extraReplacements),e.postCrawlCopyMap=K(e.postCrawlCopyMap),e.logDir=String(e.logDir||"logs").trim()||"logs",e.verbose??=false,e.logLevel||=e.verbose?"debug":"info",e.s3SyncMode||="sdk-upload-delete",e.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},e.readiness.timeoutMs??=1500,e.readiness.fallbackWaitMs??=1500,e.viewport||={width:1440,height:1200},e.navigationTimeoutMs||=3e4,e.scheduler||={enabled:false,timezone:"UTC",rules:[]},e.scheduler.enabled??=false,e.scheduler.timezone||="UTC",e.scheduler.rules||=[],e.deploymentProfiles=Ie(e.deploymentProfiles),e.assetDownloadConcurrency=Number(e.assetDownloadConcurrency)>0?Number(e.assetDownloadConcurrency):e.concurrency,e.rewriteConcurrency=Number(e.rewriteConcurrency)>0?Number(e.rewriteConcurrency):e.assetDownloadConcurrency,e.defaultDeploymentProfile=String(e.defaultDeploymentProfile??"").trim(),e.defaultDeploymentProfile&&!e.deploymentProfiles[e.defaultDeploymentProfile]&&(e.defaultDeploymentProfile="");let i=Fe(),s=i?v.resolve(i,".."):"";return s?(e.outputDir=re(s,e.outputDir,"export"),e.logDir=re(s,e.logDir,"logs")):(v.isAbsolute(e.outputDir)||(e.outputDir=q(e.outputDir,"export")),v.isAbsolute(e.logDir)||(e.logDir=q(e.logDir,"logs"))),e}function ie(t){let r="";for(let i=0;i<t.length;i++){let s=t[i];if(s==="--profile"){let n=(t[i+1]||"").trim();if(!n)throw new Error("Missing value for --profile.");r=n,i+=1;}else if(s.startsWith("--profile=")){let n=s.slice(10).trim();if(!n)throw new Error("Missing value for --profile.");r=n;}}let e=String(process.env.PUBLISHER_DEPLOY_PROFILE||process.env.PUBLISHER_DEPLOYMENT_PROFILE||"").trim();return r||e||null}function Me(t,r){return {...t,...r??{}}}function je(t,r){return {...t,...r??{},invalidationPaths:[...(r?.invalidationPaths??t.invalidationPaths)||[]]}}function se(t,r){let e=String(r??t.defaultDeploymentProfile??"").trim();if(!e)return {name:null,profile:null,config:t};let i=t.deploymentProfiles?.[e];if(!i)throw new Error(`Unknown deployment profile "${e}". Check deploymentProfiles in publisher.config.json.`);return {name:e,profile:i,config:{...t,targetOrigin:i.targetOrigin??t.targetOrigin,s3:Me(t.s3,i.s3),cloudFront:je(t.cloudFront,i.cloudFront)}}}var oe={error:0,warn:1,info:2,debug:3},ae=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",z=ae?v.join(ae,"current-progress.json"):"",le=Promise.resolve();function ue(t){return t.includes("crawl")?"crawl":t.includes("deploy")?"deploy":t.includes("invalidate")?"invalidate":t}function Oe(t,r){let e={...t},i=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[s,n]of Object.entries(r)){let o=e[s];if(typeof o=="number"&&typeof n=="number"&&i.has(s)){e[s]=Math.max(o,n);continue}e[s]=n;}return e}function _e(t,r,e){z&&(le=le.then(async()=>{let i=new Date,s=i.toISOString();await $.mkdir(v.dirname(z),{recursive:true});let n=await $.readFile(z,"utf8").then(w=>JSON.parse(w)).catch(()=>null),o=n&&typeof n.details=="object"&&n.details?n.details:{},l=ue(t),a=n?.currentStep||ue(n?.source||""),c=n?.startedAt||s,u={...n?.stepDurationsSec??{}},g=n?.stepStartedAt||s;if(a&&a!==l&&n?.stepStartedAt){let w=Math.max(0,Math.round((i.getTime()-new Date(n.stepStartedAt).getTime())/1e3));u[a]=(u[a]??0)+w,g=s;}let d=Math.max(0,Math.round((i.getTime()-new Date(g).getTime())/1e3)),p=Math.max(0,Math.round((i.getTime()-new Date(c).getTime())/1e3)),x={checkedAt:s,source:t,message:r,details:Oe(o,e??{}),startedAt:c,currentStep:l,stepStartedAt:g,stepElapsedSec:d,totalElapsedSec:p,stepDurationsSec:u};await $.writeFile(z,JSON.stringify(x,null,2),"utf8");}).catch(()=>{}));}async function Te(t,r){await $.mkdir(v.dirname(t),{recursive:true}),await $.appendFile(t,`${JSON.stringify(r)}
|
|
2
|
+
`,"utf8");}var j=class{constructor(r,e,i="info"){this.logDir=r;this.logFile=e;this.level=i;this.ensureLogFileReady();}logDir;logFile;level;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;get logPath(){return v.join(this.logDir,this.logFile)}get errorsPath(){return v.join(this.logDir,this.logFile.replace(".log.jsonl",".errors.jsonl"))}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await $.mkdir(this.logDir,{recursive:true}),await $.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(r){this.writeQueue=this.writeQueue.then(r).catch(e=>{this.writeError=e instanceof Error?e:new Error(String(e));});}enqueueLine(r){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await $.appendFile(this.logPath,`${r}
|
|
3
|
+
`,"utf8");});}enqueueJsonLine(r,e){this.enqueueTask(()=>Te(r,e));}accepts(r){return oe[r]<=oe[this.level]}push(r,e,i){if(!this.accepts(r))return;let s=JSON.stringify({time:new Date().toISOString(),level:r,message:e,...i||{}});this.enqueueLine(s),r==="error"?console.error(`[ERROR] ${e}`,i?JSON.stringify(i):""):r==="warn"?console.warn(`[WARN] ${e}`,i?JSON.stringify(i):""):console.log(`[${r.toUpperCase().padEnd(5)}] ${e}`);}info(r,e){this.push("info",r,e);}progress(r,e){_e(this.logFile,r,e);}debug(r,e){this.push("debug",r,e);}warn(r,e){this.push("warn",r,e);}error(r,e){this.enqueueJsonLine(this.errorsPath,{message:r,...e||{}}),this.push("error",r,e);}mark(r){this.marks.set(r,Date.now());}endMark(r,e){let i=this.marks.get(r);if(!i)return;let s=Date.now()-i,n=Number((s/1e3).toFixed(2));this.push("info",`Timing ${r}: ${n}s`,{name:r,ms:s,seconds:n,...e||{}});}async flush(){let r=Date.now()-this.startedAt,e=Number((r/1e3).toFixed(2));if(this.push("info",`Total time: ${e}s`,{name:"total",ms:r,seconds:e}),await this.writeQueue,this.writeError)throw this.writeError}};function ge(t){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(v.extname(t).toLowerCase())}function pe(t){return t.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function E(t){return t.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function N(t){return t.replace(/\//g,"\\/")}function ce(t){return t.replace(/\//g,"\\\\/")}function de(t,r,e){if(!r)return;t[r]=e;let i=N(r),s=N(e);t[i]=s;let n=ce(r),o=ce(e);t[n]=o;let l=E(r),a=E(e);t[l]=a;let c=E(i),u=E(s);t[c]=u;let g=E(n),d=E(o);t[g]=d;}function fe(t,r){try{return new URL(t,r==="."?"https://relative.invalid":r).pathname}catch{return t.startsWith("/")?t:`/${t.replace(/^\.\//,"")}`}}function me(t,r,e){if(!r)return e;let i=v.dirname(v.resolve(r)),s=v.resolve(t,e.replace(/^\/+/,"")),n=v.relative(i,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n):"."}var Ue=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],We=new Set([".html",".htm"]),we="WPSuite.io Static Publisher",He=we.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Je(t,r,e){if(t.wpsuite?.subscriber===true||!e)return false;let i=v.extname(e).toLowerCase();return We.has(i)?/<head\b|<html\b|<!doctype html/i.test(r):false}function Be(t){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${He}\\2)[^>]*\\/?>`,"i").test(t))return t;let r=`<meta name="generator" content="${we}" />`;if(t.match(/(\r?\n)([ \t]*)<\/head>/i))return t.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${r}$1$2</head>`);let i=t.includes(`\r
|
|
4
4
|
`)?`\r
|
|
5
5
|
`:`
|
|
6
|
-
`;return t.replace(/<head\b[^>]*>/i,s=>`${s}${
|
|
7
|
-
`).find(e=>e.trim()!=="")??t).trim()}function
|
|
6
|
+
`;return t.replace(/<head\b[^>]*>/i,s=>`${s}${i} ${r}`)}function Ke(t){let r=pe(t).trim();return r.startsWith("{")||r.startsWith("[")||r.includes("\\/")||/"@context"|"@type"/.test(r)}function qe(t,r){let e=[...new Set(r)].filter(n=>n.includes("/")&&!/\\+\//.test(n)).map(n=>[n,N(n)]).filter(([n,o])=>n!==o).sort((n,o)=>o[0].length-n[0].length);if(!e.length)return t;let i=n=>{let o=n;for(let[l,a]of e)o=o.split(l).join(a);return o},s=t.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(n,o,l,a)=>`${o}${i(l)}${a}`);return s=s.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(n,o,l,a,c)=>Ke(a)?`${o}${i(a)}${c}`:n),s}function Qe(t,r){if(!r)return null;let e=v.dirname(v.resolve(r)),i=v.resolve(t.outputDir),s=v.relative(e,i).replace(/\\/g,"/");return s?(s.startsWith(".")||(s=`./${s}`),s.endsWith("/")?s:`${s}/`):"./"}function Ve(t,r,e){let i=Qe(r,e);if(!i)return t;let s=t;for(let n of Ue){let o=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),l=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),a=new RegExp(`(?<!\\.)/${o}`,"g");s=s.replace(l,`${i}${n}`).replace(a,`${i}${n}`);let c=n.replace(/\//g,"\\/"),u=i.replace(/\//g,"\\/"),g=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${g}`,"g"),p=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${g}`,"g"),x=new RegExp(`(?<![\\.\\\\])\\/${g}`,"g");s=s.replace(d,`${u}${c}`).replace(p,`${u}${c}`).replace(x,`${u}${c}`);}return s}function Ge(t,r,e){if(t.urlRewriteMode==="absolute")return r;let i=(()=>{try{return new URL(r,t.targetOrigin==="."?"https://relative.invalid":t.targetOrigin)}catch{return null}})(),s=i?i.pathname:fe(r,t.targetOrigin),n=i?`${i.search}${i.hash}`:"";return t.urlRewriteMode==="root-relative"?`${s}${n}`:`${me(t.outputDir,e,s)}${n}`}function V(t,r,e,i,s){de(t,e,Ge(r,i,s));}function Ze(t,r,e,i){let s=t,n={};V(n,r,r.sourceOrigin,r.targetOrigin,i);for(let[l,a]of Object.entries(r.extraReplacements))V(n,r,l,a,i);for(let[l,a]of Object.entries(e))V(n,r,l,a,i);let o=Object.entries(n).sort((l,a)=>a[0].length-l[0].length);for(let[l,a]of o)s=s.split(l).join(a);return r.urlRewriteMode!=="absolute"&&(s=Ve(s,r,i)),s=qe(s,o.map(([,l])=>l)),Je(r,s,i)&&(s=Be(s)),s}async function Ye(t,r,e){let i=v.join(t.outputDir,e),s=v.extname(i).toLowerCase(),n;try{n=await $.readFile(i,"utf8");}catch(a){if(a.code!=="ENOENT")throw a;return {changed:false}}let l=Ze(n,t,r,s===".js"||s===".mjs"?void 0:i);return l!==n?(await $.writeFile(i,l,"utf8"),{changed:true}):{changed:false}}async function Xe(t,r){return await new Promise((e,i)=>{let s=a=>{if(l(),a?.error){i(new Error(a.error));return}e({changed:!!a?.changed});},n=a=>{l(),i(a);},o=a=>{l(),a!==0&&i(new Error(`Rewrite worker exited with code ${a}.`));},l=()=>{t.off("message",s),t.off("error",n),t.off("exit",o);};t.on("message",s),t.on("error",n),t.on("exit",o),t.postMessage({file:r});})}async function ye(t,r,e={}){let s=(e.files?[...new Set(e.files)].map(d=>v.resolve(d)).filter(d=>d.startsWith(v.resolve(t.outputDir))).map(d=>v.relative(t.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Ae(["**/*"],{cwd:t.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&ge(d)),n=0;if(s.length===0)return 0;let o=Math.max(1,Number(t.rewriteConcurrency||t.assetDownloadConcurrency||1)),l=Math.max(1,Math.min(o,s.length,availableParallelism())),a=0,c=0,u=null;if(l===1){for(let d of s)(await Ye(t,r,d)).changed&&(n+=1),c+=1,await e.onProgress?.({index:c,totalFiles:s.length,changedTextFiles:n,file:d});return n}let g=Array.from({length:l},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:t,assetMap:r}});try{for(;!u;){let p=a;if(p>=s.length)return;a+=1;let x=s[p];(await Xe(d,x)).changed&&(n+=1),c+=1,await e.onProgress?.({index:c,totalFiles:s.length,changedTextFiles:n,file:x});}}catch(p){throw u=p instanceof Error?p:new Error(String(p)),u}finally{await d.terminate().catch(()=>{});}});return await Promise.all(g),n}var $e="wpsuite-sha256",xe="wpsuite-normalized-sha256",at=/(?:^|[-_:.])(id|uid|nonce|token|hash|instance)(?:$|[-_:.])/i,ee=/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;function lt(t){return t&&(ee.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)?"__hex__":/^[0-9]{8,}$/.test(t)?"__num__":/^(?=.*[a-z])(?=.*\d)[a-z0-9]{8,}$/i.test(t)?"__alnum__":t)}function Z(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:lt(r)).join("")}function ut(t){return t.split(/([_:.\-/=?&#]+)/g).map((r,e)=>e%2===1?r:ee.test(r)?"__uuid__":/^[0-9a-f]{6,}$/i.test(r)?"__hex__":/^[0-9]{6,}$/.test(r)?"__num__":/^[a-z0-9_-]{6,}$/i.test(r)?"__id__":r).join("")}function U(t,r){return Array.isArray(t)?t.map(e=>U(e,r)):t&&typeof t=="object"?Object.fromEntries(Object.entries(t).map(([e,i])=>[e,U(i,e)])):typeof t!="string"?t:r&&at.test(r)?ut(t):ee.test(t)?"__uuid__":/^[0-9a-f]{6,}$/i.test(t)||/^[0-9]{8,}$/.test(t)?Z(t):t}function ct(t){return t.replace(/((?:https?:\/\/[^"'\s>]+|\/)(?:wp-content|wp-includes)\/[^"'\s>?]+)\?ver=[A-Za-z0-9._-]+(?=$|[#"'])/gi,"$1")}function gt(t,r){let e=r.trim();if((e.startsWith("{")||e.startsWith("[")||e.startsWith('"'))&&t.startsWith("data-"))try{let s=JSON.parse(e);return JSON.stringify(U(s))}catch{}if(t.startsWith("data-")&&/^[A-Za-z0-9+/=]+$/.test(e))try{let s=Buffer.from(e,"base64").toString("utf8"),n=JSON.parse(s),o=JSON.stringify(U(n));return Buffer.from(o,"utf8").toString("base64")}catch{}return r}function pt(t,r){let e=t.toLowerCase();return e==="class"?r.split(/(\s+)/).map((i,s)=>s%2===1?i:Z(i)).join(""):e==="id"||e==="for"||e==="aria-controls"||e==="aria-labelledby"||e==="aria-describedby"||e==="aria-owns"||e==="aria-activedescendant"||e==="name"?Z(r):e==="href"||e==="src"?ct(r):gt(e,r)}function dt(t){return t.replace(/<([a-z][^\s/>]*)([^<>]*?)>/gi,(r,e,i)=>{if(e.startsWith("/"))return r;let s=i.replace(/\s([:@a-zA-Z_][-:.a-zA-Z0-9_]*?)=(['"])([\s\S]*?)\2/g,(n,o,l,a)=>{let c=pt(o,a);return ` ${o}=${l}${c}${l}`});return `<${e}${s}>`})}function G(t){let r=v.extname(t).toLowerCase();return [".html",".htm",".xml",".xsl",".txt"].includes(r)||["robots.txt","sitemap.xml"].includes(v.basename(t))}function be(t,r){return [t.replace(/^\/+|\/+$/g,""),r.replace(/^\/+/,"")].filter(Boolean).join("/")}async function ft(t,r,e,i){let s=new Map,n,o=0;do{o++;let l=await t.send(new ListObjectsV2Command({Bucket:r,Prefix:e||void 0,ContinuationToken:n}));for(let a of l.Contents||[]){if(!a.Key)continue;let c=(a.ETag||"").replace(/^"|"$/g,"").toLowerCase(),u=Number(a.Size??0);s.set(a.Key,{etag:c,size:u});}n=l.NextContinuationToken,i.debug(`Listed S3 keys page ${o}: ${s.size} keys so far`);}while(n);return s}async function mt(t,r,e,i,s){let n=Array.from(e).filter(o=>!i.has(o));if(n.length===0)return s.info("No stale S3 objects to delete."),[];s.info(`Deleting ${n.length} stale S3 object(s)\u2026`);for(let o=0;o<n.length;o+=1e3){let l=n.slice(o,o+1e3);if(l.length!==0){await t.send(new DeleteObjectsCommand({Bucket:r,Delete:{Objects:l.map(a=>({Key:a}))}})),s.info(`Deleted batch of ${l.length} object(s).`);for(let a of l)s.debug(` deleted: ${a}`);}}return n}function ht(t){return createHash("md5").update(t).digest("hex")}function Pe(t){return createHash("sha256").update(t).digest("hex")}function Y(t){return createHash("sha256").update(dt(t.toString("utf8"))).digest("hex")}async function wt(t,r,e,i){if(i.has(e))return i.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[$e]||"").trim().toLowerCase()||null;return i.set(e,o),o}catch{return i.set(e,null),null}}async function yt(t,r,e,i,s){if(i.has(e))return i.get(e)??null;try{let o=((await t.send(new HeadObjectCommand({Bucket:r,Key:e}))).Metadata?.[xe]||"").trim().toLowerCase();if(o)return i.set(e,o),o}catch{return i.set(e,null),null}if(s.has(e)){let n=s.get(e)??null;return i.set(e,n),n}try{let o=await(await t.send(new GetObjectCommand({Bucket:r,Key:e}))).Body?.transformToByteArray(),l=o?Y(Buffer.from(o)):null;return s.set(e,l),i.set(e,l),l}catch{return s.set(e,null),i.set(e,null),null}}function bt(t){return (t.split(`
|
|
7
|
+
`).find(e=>e.trim()!=="")??t).trim()}function Pt(t){return t.replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/-+/g,"-").replace(/^-|-$/g,"")||"profile"}function vt(t,r){let e=t,i=Object.entries(r).sort((s,n)=>n[0].length-s[0].length);for(let[s,n]of i)s&&(e=e.split(s).join(n));return e}async function St(t,r,e,i){let s=v.join(t,"asset-map.json"),n;try{n=await $.readFile(s,"utf8");}catch(a){if(a.code==="ENOENT")return;throw a}let o=JSON.parse(n);if(!o||typeof o!="object")return;let l=Object.fromEntries(Object.entries(o).map(([a,c])=>{let u=String(c??"");return r.urlRewriteMode==="absolute"&&r.targetOrigin&&e.targetOrigin&&r.targetOrigin!==e.targetOrigin&&(u=u.split(r.targetOrigin).join(e.targetOrigin)),u=vt(u,i),[a,u]}));await $.writeFile(s,JSON.stringify(l,null,2),"utf8");}async function $t(t,r,e,i,s){let n=t.targetOrigin!==r.targetOrigin,o=Object.keys(i).length>0;if(!e||!n&&!o)return {outputDir:t.outputDir,cleanup:async()=>{}};if(n&&t.urlRewriteMode!=="absolute")throw new Error(`Deployment profile "${e}" changes targetOrigin, but the base crawl output must use urlRewriteMode "absolute" for multi-target redeploys.`);let l=v.join(v.dirname(t.outputDir),`.deploy-profile-${Pt(e)}-${Date.now()}`);s.info(`Preparing deployment profile "${e}" from existing crawl output...`),await $.rm(l,{recursive:true,force:true}),await $.cp(t.outputDir,l,{recursive:true});let a={...t,sourceOrigin:t.targetOrigin,targetOrigin:r.targetOrigin,extraReplacements:i,outputDir:l},c=await ye(a,{});return await St(l,t,r,i),s.info(`Prepared deployment profile "${e}" in ${l} (${c} rewritten text file(s)).`),{outputDir:l,cleanup:async()=>{await $.rm(l,{recursive:true,force:true});}}}async function xt(t,r,e){await new Promise((i,s)=>{let n=`${t.outputDir.replace(/\/+$/,"")}/`,o=`s3://${t.s3.bucket}/${t.s3.prefix.replace(/^\/+|\/+$/g,"")}`.replace(/\/$/,""),l=["s3"];r==="aws-s3-cp-recursive"?l.push("cp",n,o,"--recursive","--region",t.s3.region):(l.push("sync",n,o,"--region",t.s3.region),r==="aws-s3-sync-delete"&&l.push("--delete")),e.info(`Running AWS CLI: aws ${l.join(" ")}`);let a=spawn("aws",l,{stdio:["ignore","pipe","pipe"]}),c=Date.now(),u=setInterval(()=>{let g=Math.floor((Date.now()-c)/1e3);e.progress(`AWS CLI is still running (${g}s elapsed) \u2014 syncing/copying in progress...`,{mode:r,elapsedSec:g}),e.info(`AWS CLI is still running (${g}s elapsed) \u2014 syncing/copying in progress...`);},15e3);a.stdout.setEncoding("utf8"),a.stderr.setEncoding("utf8"),a.stdout.on("data",g=>{g.split(/\r?\n/).map(d=>d.trim()).filter(d=>d.length>0).forEach(d=>e.info(d));}),a.stderr.on("data",g=>{g.split(/\r?\n/).map(d=>d.trim()).filter(d=>d.length>0).forEach(d=>e.warn(d));}),a.on("error",g=>{clearInterval(u),s(new Error(`Failed to start aws CLI: ${g.message}`));}),a.on("close",g=>{if(clearInterval(u),g===0){i();return}s(new Error(`aws CLI exited with code ${g??-1}`));});});}async function Dt(){let t=await Q(),r=ie(process.argv.slice(2)),e=se(t,r),i=v.resolve(e.config.logDir||"logs"),s=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",n=s?v.join(v.resolve(s),"deploy-diff.json"):"",o=new j(i,"deploy.log.jsonl",e.config.logLevel??"info"),l=e.profile?.extraReplacements??{},a=async()=>{};try{let c=await $t(t,e.config,e.name,l,o);a=c.cleanup;let u={...e.config,outputDir:c.outputDir},g=u.s3SyncMode??"sdk-upload-delete";e.name&&o.info(`Using deployment profile: ${e.name}`),o.info(`Starting deploy \u2014 s3://${u.s3.bucket}/${u.s3.prefix||""} (region: ${u.s3.region})`),o.info(`S3 sync mode: ${g}`),o.mark("deploy");try{let y=await new Promise((B,f)=>{let P=spawn("aws",["--version"],{stdio:["ignore","pipe","pipe"]}),T="",m="";P.stdout.setEncoding("utf8"),P.stderr.setEncoding("utf8"),P.stdout.on("data",h=>{T+=h;}),P.stderr.on("data",h=>{m+=h;}),P.on("error",h=>f(h)),P.on("close",h=>{h===0?B(bt(T||m)):f(new Error(`aws --version exited with ${h??-1}`));});});o.debug(`AWS CLI detected: ${y}`);}catch(y){o.warn(`AWS CLI version check failed: ${y.message}`);}if(g==="aws-s3-sync-delete"||g==="aws-s3-sync"||g==="aws-s3-cp-recursive"){await xt(u,g,o),n&&(await $.mkdir(v.dirname(n),{recursive:!0}),await $.writeFile(n,JSON.stringify({generatedAt:new Date().toISOString(),mode:g,summary:{strategy:"cli-mode",note:"Detailed object-level diff is not collected in AWS CLI modes.",...e.name?{profile:e.name}:{}},uploadedKeys:[],skippedKeys:[],failedKeys:[],deletedKeys:[]},null,2),"utf8")),o.endMark("deploy"),await o.flush();return}let d=new S3Client({region:u.s3.region}),p=await Ae(["**/*"],{cwd:u.outputDir,onlyFiles:!0,dot:!0}),x=new Set(p.map(y=>be(u.s3.prefix,y)));o.info(`Found ${p.length} file(s) in output directory "${u.outputDir}".`);let w=0,b=0,C=0,W=[],k=[],H=[],J=[],_=new Map,De=new Map,Ce=new Map,Re=new Map;(g==="sdk-upload-delete"||g==="sdk-upload-only")&&(o.info("Listing existing S3 objects to detect unchanged files..."),_=await ft(d,u.s3.bucket,u.s3.prefix,o),o.info(`Found ${_.size} existing key(s) in S3.`));for(let y of p){let B=v.join(u.outputDir,y),f=be(u.s3.prefix,y),P=ot.lookup(y)||"application/octet-stream",T=G(y)?u.s3.htmlCacheControl:u.s3.assetCacheControl,m=w+b+C+1;p.length<=200||m<=5||m%10===0||m===p.length?(o.progress(`Uploading [${m}/${p.length}] ${f} (${P})`,{file:y,key:f,index:m,totalFiles:p.length,contentType:P,mode:g}),o.info(`Uploading [${m}/${p.length}] ${f} (${P})`)):o.debug(`Uploading [${m}/${p.length}] ${f} (${P})`);try{let h=await $.readFile(B),I=_.get(f),F=null,L=null;if(I){let Le=h.byteLength;if(I.size===Le){let A=I.etag.includes("-");if(I.etag!==""&&!A){let Ee=ht(h);if(I.etag===Ee){b++,p.length<=200||b<=5||b%25===0?(o.progress(`Skipped unchanged [${m}/${p.length}] ${f}`,{file:y,key:f,index:m,totalFiles:p.length,uploaded:w,skipped:b,failed:C,mode:g,reason:"etag-md5-match"}),o.info(`Skipped unchanged [${m}/${p.length}] ${f}`)):o.debug(`Skipped unchanged [${m}/${p.length}] ${f}`),k.push(f);continue}}F=Pe(h);let te=await wt(d,u.s3.bucket,f,De);if(te&&te===F){b++,p.length<=200||b<=5||b%25===0?(o.progress(`Skipped unchanged [${m}/${p.length}] ${f}`,{file:y,key:f,index:m,totalFiles:p.length,uploaded:w,skipped:b,failed:C,mode:g,reason:"metadata-sha256-match"}),o.info(`Skipped unchanged [${m}/${p.length}] ${f}`)):o.debug(`Skipped unchanged [${m}/${p.length}] ${f}`),k.push(f);continue}}if(G(y)){L=Y(h);let A=await yt(d,u.s3.bucket,f,Ce,Re);if(A&&A===L){b++,p.length<=200||b<=5||b%25===0?(o.progress(`Skipped semantically unchanged [${m}/${p.length}] ${f}`,{file:y,key:f,index:m,totalFiles:p.length,uploaded:w,skipped:b,failed:C,mode:g,reason:"normalized-html-match"}),o.info(`Skipped semantically unchanged [${m}/${p.length}] ${f}`)):o.debug(`Skipped semantically unchanged [${m}/${p.length}] ${f}`),k.push(f);continue}}}F=F??Pe(h),L=L??(G(y)?Y(h):null),await new Upload({client:d,params:{Bucket:u.s3.bucket,Key:f,Body:h,ContentType:P,CacheControl:T,Metadata:{[$e]:F,...L?{[xe]:L}:{}}}}).done(),w++,W.push(f),(w%25===0||w===p.length)&&(o.progress(`Uploaded ${w}/${p.length} file(s)\u2026`,{uploaded:w,skipped:b,failed:C,totalFiles:p.length,mode:g}),o.info(`Uploaded ${w}/${p.length} file(s)\u2026`));}catch(h){C++,H.push(f),o.error(`Failed to upload ${f}: ${h.message}`,{key:f,error:String(h)});}}o.info(`Upload complete: ${w} uploaded, ${b} skipped unchanged, ${C} failed.`),(g==="sdk-upload-delete"||g==="sdk-upload-only")&&(g==="sdk-upload-delete"?J=await mt(d,u.s3.bucket,_.keys(),x,o):o.info("Sync mode is sdk-upload-only \u2014 skipping stale object deletion.")),n&&(await $.mkdir(v.dirname(n),{recursive:!0}),await $.writeFile(n,JSON.stringify({generatedAt:new Date().toISOString(),mode:g,summary:{totalFiles:p.length,uploaded:W.length,skipped:k.length,failed:H.length,deleted:J.length,...e.name?{profile:e.name}:{}},uploadedKeys:W,skippedKeys:k,failedKeys:H,deletedKeys:J},null,2),"utf8")),o.endMark("deploy"),await o.flush();}finally{await a().catch(()=>{});}}Dt().catch(async t=>{console.error(t);try{let e=(await Q().catch(()=>null))?.logDir??"logs",i=new j(e,"deploy.log.jsonl","debug");i.error(`Unhandled error: ${t instanceof Error?t.message:String(t)}`,t instanceof Error?{stack:t.stack}:void 0),await i.flush();}catch{}process.exit(1);});
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import
|
|
2
|
-
`,"utf8");}catch{}}async function p(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await f(Z(e.runtimeDir),n);}catch{}}async function I(e){try{await g.unlink(B(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let s=(e.id||`${r}-${t+1}`).trim();if(!s)return null;let c=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:s,enabled:c,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function se(e){let t=await h(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((s,c)=>oe(s,c)).filter(s=>!!s):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t,r){return e.some(n=>n.command===t&&(n.url||"").trim()===(r||"").trim()&&(n.status===void 0||n.status==="queued"||n.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let a=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${a}`),0}let n=d.join(e.runtimeDir,"queue.json"),s=d.join(e.runtimeDir,"current-run.json"),c=Date.now(),i=0,u=[];await R(e.runtimeDir,async()=>{let o=await h(n,[]),a=await h(s,null),w=await h(T(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];a&&a.status==="running"&&m.unshift(a);for(let l of r.rules){if(!l.enabled)continue;let P=l.intervalMinutes*60*1e3,S=Math.floor(c/P),v=w.lastEnqueuedBucketByRuleId[l.id]??-1;if(S<=v||ue(m,l.command,l.url))continue;let q={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(q),m.push(q),u.push({job:q,rule:l}),w.lastEnqueuedBucketByRuleId[l.id]=S,i+=1;}i>0&&(await f(n,o),await f(T(e.runtimeDir),w));});for(let o of u)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await g.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await g.unlink(e);}catch{}}function le(e,t){let r=[d.join(e,"dist",`${t}.js`),d.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${d.join(e,"dist")} or ${e}`)}async function y(e,t,r,n,s,c,i){if(i){let a=await j(t,i);if(a)throw new J(a,r)}let u=le(e,r),o={...process.env,...c??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,s&&(o.PUBLISHER_CONFIG=s),await new Promise((a,w)=>{let m=null,l=null,P=null,S=false,v=spawn(process.execPath,[u,...n],{cwd:e,env:o,stdio:"inherit"}),q=()=>{l&&clearInterval(l),P&&clearTimeout(P);},U=async()=>{if(!(!i||m||S)){S=true;try{let b=await j(t,i);if(!b)return;m=b,v.kill("SIGTERM"),P=setTimeout(()=>{v.kill("SIGKILL");},1e4);}finally{S=false;}}};i&&(l=setInterval(()=>{U();},1e3)),v.on("error",b=>{q(),w(b);}),v.on("close",(b,x)=>{if(q(),m){w(new J(m,r));return}b===0?a():w(new Error(`${r} exited with code ${b??-1}${x?` (signal ${x})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function ge(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let s=(await h(t.configPath||"",null))?.wpsuite;return s&&(s.accountId||s.siteId||s.siteKey)?s:null}async function pe(e,t){return await ge(e,t)}function F(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,t}async function fe(e){let t=d.join(e.runtimeDir,"queue.json"),r=d.join(e.runtimeDir,"current-run.json"),n=await R(e.runtimeDir,async()=>{let s=await h(r,null);if(!s||s.status!=="running"&&s.status!=="queued")return null;let c=await h(t,[]),i=Array.isArray(c)?c.filter(o=>o?.id!==s.id):[],u=F(s);return await f(t,[u,...i]),await f(r,null),u});n&&(await E(e.runtimeDir,n.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function he(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let u=await pe(e,t);await r.assertActiveSubscriptionForIdentity(u);}let s=me(e),c=e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"crawl"}),await y(t.exporterDir,t.runtimeDir,"crawl",c,t.configPath,s,e.id),await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await y(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,s,e.id),await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await y(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="crawl")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"crawl"}),await y(t.exporterDir,t.runtimeDir,"crawl",c,t.configPath,s,e.id),{exitCode:0};if(e.command==="deploy")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await y(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="invalidate")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await y(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await y(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,s,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await y(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,s,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof J?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function L(e){return async function(r=process.argv.slice(2)){let n=z(r),s=d.join(n.runtimeDir,"export.lock"),c=d.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await p(n,"starting",{message:"queue-runner starting"});try{await ce(s);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await p(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await fe(n);let i=await ae(n,e),u=0;for(let o=0;o<n.maxJobs;o++){let a=await ye(n,e);if(a==="none"||(u+=1,a==="stopped"))break}if(u===0)await I(n.runtimeDir),await p(n,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await h(c,null),a=String(o?.status||"").trim();await p(n,a==="success"?"job-success":a==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...a==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await p(n,"error",{message:u}),await A(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await de(s);}}}async function we(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json");await R(e.runtimeDir,async()=>{let s=await h(r,[]),c=Array.isArray(s)?s.filter(i=>i?.id!==t.id):[];await f(r,[F(t),...c]),await f(n,null);});}async function ye(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json"),s=d.join(e.runtimeDir,"last-run.json"),c=new Date().toISOString(),i=await R(e.runtimeDir,async()=>{let m=await h(r,[]);if(!Array.isArray(m)||m.length===0)return null;let l={...m[0],status:"running",startedAt:c};return await f(r,m.slice(1)),await f(n,l),l});if(!i)return "none";await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:c,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await p(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await he(i,e,t),o=new Date().toISOString();if(u.stopped){let m={...i,status:"stopped",endedAt:o,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:u.stopRequest?.mode||"requeue",stoppedStep:u.stoppedStep||i.command};await we(e,i);try{let l=await k(m,e);m.logArchiveDir=l.archiveDir,m.logArchiveCreatedAt=l.archiveCreatedAt,m.logArchiveFileCount=l.archivedFiles.length;}catch(l){m.logArchiveError=l instanceof Error?l.message:String(l);}return await f(s,m),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job stop requested and requeued.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:u.stopRequest?.mode||"requeue",stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:u.stoppedStep||i.command,logArchiveDir:m.logArchiveDir||"",logArchiveCreatedAt:m.logArchiveCreatedAt||"",logArchiveFileCount:m.logArchiveFileCount??0,logArchiveError:m.logArchiveError||""}}),"stopped"}let a={...i,status:u.exitCode===0?"success":"failed",endedAt:o,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await f(n,null),await E(e.runtimeDir,i.id);try{let m=await k(a,e);a.logArchiveDir=m.archiveDir,a.logArchiveCreatedAt=m.archiveCreatedAt,a.logArchiveFileCount=m.archivedFiles.length;}catch(m){a.logArchiveError=m instanceof Error?m.message:String(m);}await f(s,a);let w=i.startedAt&&a.endedAt?Math.max(0,Math.round((new Date(a.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:a.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:a.id,command:a.command,message:a.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:a.endedAt||"",durationSec:w,deploymentProfile:a.deploymentProfile||"",exitCode:a.exitCode??null,error:a.error||"",logArchiveDir:a.logArchiveDir||"",logArchiveCreatedAt:a.logArchiveCreatedAt||"",logArchiveFileCount:a.logArchiveFileCount??0,logArchiveError:a.logArchiveError||""}}),"processed"}function ve(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function qe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let c=`${ve(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,u=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(c,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let a=await o.json().catch(()=>null);if(!a||!a.config||!a.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(u);}}var Ae={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:qe},N=L(Ae),Pe=N,je=N;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Pe().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
-
export{
|
|
1
|
+
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import p from'fs/promises';import {existsSync}from'fs';import l from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';var R=class extends Error{request;step;constructor(t,r){super(`Stop requested during ${r}.`),this.name="StopRequestedError",this.request=t,this.step=r;}},K="queue-mutation.lock",H=5e3,z=3e4;function G(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",r=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),n=process.env.PUBLISHER_CONFIG||"",u=1;for(let s=0;s<e.length;s++){let o=e[s];o==="--runtime-dir"?t=e[++s]||"":o.startsWith("--runtime-dir=")?t=o.slice(14):o==="--exporter-dir"?r=e[++s]||r:o.startsWith("--exporter-dir=")?r=o.slice(15):o==="--config"?n=e[++s]||"":o.startsWith("--config=")?n=o.slice(9):o==="--max-jobs"?u=Number.parseInt(e[++s]||"1",10):o.startsWith("--max-jobs=")&&(u=Number.parseInt(o.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let a=l.resolve(t),i=n?l.resolve(n):l.join(a,"config.json");return {runtimeDir:a,exporterDir:l.resolve(r),configPath:i,maxJobs:Number.isFinite(u)&&u>0?u:1}}async function h(e,t){try{let r=await p.readFile(e,"utf8");return JSON.parse(r)}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="ENOENT"){let u=r instanceof Error?r.message:String(r);console.warn(`[queue-runner] failed to read JSON ${e}: ${u}`);}return t}}async function w(e,t){await p.mkdir(l.dirname(e),{recursive:true}),await p.writeFile(e,JSON.stringify(t,null,2),"utf8");}function F(e){return l.join(e,K)}async function X(e){await new Promise(t=>setTimeout(t,e));}async function Y(e){let t=F(e),r=Date.now()+H;for(;;)try{await p.mkdir(l.dirname(t),{recursive:!0}),await p.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="EEXIST")throw n;let a=await p.stat(t).catch(()=>null);if(a&&Date.now()-a.mtimeMs>z){await p.unlink(t).catch(()=>{});continue}if(Date.now()>=r)throw new Error("Timed out acquiring queue mutation lock.",{cause:n});await X(50);}}async function Z(e){await p.unlink(F(e)).catch(()=>{});}async function J(e,t){await Y(e);try{return await t()}finally{await Z(e);}}function V(e){return l.join(e,"queue-runner-heartbeat.json")}function x(e){return l.join(e,"current-progress.json")}function k(e){return l.join(e,"scheduler-state.json")}function ee(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let u=n.split("/").map(a=>a.trim()).filter(a=>a.length>0&&a!=="."&&a!=="..");return u.length>0?u.join("/"):t}async function B(e){let t=await h(e.configPath||"",null),r=l.resolve(e.runtimeDir,".."),u=(typeof t?.logDir=="string"?t.logDir:"").trim();return u&&l.isAbsolute(u)?l.resolve(u):l.resolve(r,ee(u,"logs"))}function C(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function te(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function re(e){return [te(e.endedAt||e.startedAt),C(e.command||"job","job"),C(e.id||"job","job"),C(e.status||"finished","finished")].join("-")}function ne(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}async function Q(e,t){let r=await B(t),n=new Date().toISOString(),u=l.join(r,"archive",re(e)),a=[];await p.mkdir(u,{recursive:true});try{let s=await p.readdir(r,{withFileTypes:!0});for(let o of s)!o.isFile()||!ne(o.name)||(await p.copyFile(l.join(r,o.name),l.join(u,o.name)),a.push(o.name));}catch(s){if((s&&typeof s=="object"&&"code"in s?String(s.code||""):"")!=="ENOENT")throw s}let i=x(t.runtimeDir);return existsSync(i)&&(await p.copyFile(i,l.join(u,"current-progress.json")),a.push("current-progress.json")),await w(l.join(u,"job.json"),{archivedAt:n,archiveDir:u,logDir:r,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:a,job:e}),a.push("job.json"),{archiveDir:u,archiveCreatedAt:n,archivedFiles:a}}function ie(e){return l.join(e,"audit-events.jsonl")}function D(e){return l.join(e,"stop-request.json")}async function O(e){try{let t=await p.readFile(D(e),"utf8"),r=JSON.parse(t);return r&&typeof r=="object"?r:null}catch{return null}}async function j(e,t){let r=await O(e);if(!r)return null;let n=String(r.targetJobId||"").trim();return n&&n!==t?null:r}async function E(e,t){if(!t){await p.unlink(D(e)).catch(()=>{});return}let r=await O(e);if(!r)return;let n=String(r.targetJobId||"").trim();(!n||n===t)&&await p.unlink(D(e)).catch(()=>{});}async function P(e,t){try{let r={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await p.mkdir(e,{recursive:!0}),await p.appendFile(ie(e),`${JSON.stringify(r)}
|
|
2
|
+
`,"utf8");}catch{}}async function g(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await w(V(e.runtimeDir),n);}catch{}}async function I(e){try{await p.unlink(x(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let u=(e.id||`${r}-${t+1}`).trim();if(!u)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",s=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:u,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&s?{deploymentProfile:s}:{},...i?{url:i}:{}}}async function se(e){let t=await h(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((u,a)=>oe(u,a)).filter(u=>!!u):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t,r){return e.some(n=>n.command===t&&(n.url||"").trim()===(r||"").trim()&&(n.status===void 0||n.status==="queued"||n.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let c=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,s=[];await J(e.runtimeDir,async()=>{let o=await h(n,[]),c=await h(u,null),y=await h(k(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];c&&c.status==="running"&&m.unshift(c);for(let d of r.rules){if(!d.enabled)continue;let f=d.intervalMinutes*60*1e3,v=Math.floor(a/f),q=y.lastEnqueuedBucketByRuleId[d.id]??-1;if(v<=q||ue(m,d.command,d.url))continue;let A={id:randomUUID(),command:d.command,...(d.command==="publish"||d.command==="crawl")&&d.crawlMode?{crawlMode:d.crawlMode}:{},...(d.command==="publish"||d.command==="deploy"||d.command==="invalidate")&&d.deploymentProfile?{deploymentProfile:d.deploymentProfile}:{},...d.url?{url:d.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(A),m.push(A),s.push({job:A,rule:d}),y.lastEnqueuedBucketByRuleId[d.id]=v,i+=1;}i>0&&(await w(n,o),await w(k(e.runtimeDir),y));});for(let o of s)await P(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await p.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await p.unlink(e);}catch{}}function le(e,t){let r=[l.join(e,"dist",`${t}.js`),l.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${l.join(e,"dist")} or ${e}`)}async function S(e,t,r,n,u,a,i){if(i){let c=await j(t,i);if(c)throw new R(c,r)}let s=le(e,r),o={...process.env,...a??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,u&&(o.PUBLISHER_CONFIG=u),await new Promise((c,y)=>{let m=null,d=null,f=null,v=false,q=spawn(process.execPath,[s,...n],{cwd:e,env:o,stdio:"inherit"}),A=()=>{d&&clearInterval(d),f&&clearTimeout(f);},_=async()=>{if(!(!i||m||v)){v=true;try{let b=await j(t,i);if(!b)return;m=b,q.kill("SIGTERM"),f=setTimeout(()=>{q.kill("SIGKILL");},1e4);}finally{v=false;}}};i&&(d=setInterval(()=>{_();},1e3)),q.on("error",b=>{A(),y(b);}),q.on("close",(b,T)=>{if(A(),m){y(new R(m,r));return}b===0?c():y(new Error(`${r} exited with code ${b??-1}${T?` (signal ${T})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function pe(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let u=(await h(t.configPath||"",null))?.wpsuite;return u&&(u.accountId||u.siteId||u.siteKey)?u:null}async function ge(e,t){return await pe(e,t)}function L(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function fe(e,t){let r=await h(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let u=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(u)return u;let a=await B(e).catch(()=>"");if(a){let i=await h(l.join(a,"current-crawl-event.json"),null),s=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(s)return s}return t}function we(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function he(e){let t=l.join(e.runtimeDir,"queue.json"),r=l.join(e.runtimeDir,"current-run.json"),n=await J(e.runtimeDir,async()=>{let u=await h(r,null);if(!u||u.status!=="running"&&u.status!=="queued")return null;let a=await h(t,[]),i=Array.isArray(a)?a.filter(o=>o?.id!==u.id):[],s=L(u);return await w(t,[s,...i]),await w(r,null),s});n&&(await E(e.runtimeDir,n.id),await P(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function ye(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let s=await ge(e,t);await r.assertActiveSubscriptionForIdentity(s);}let u=me(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="crawl")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),{exitCode:0};if(e.command==="deploy")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="invalidate")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,u,e.id),{exitCode:0};if(e.command==="url"){let s=(e.url||"").trim();return s?(await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--url",s],t.configPath,u,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof R?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function N(e){return async function(r=process.argv.slice(2)){let n=G(r),u=l.join(n.runtimeDir,"export.lock"),a=l.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await g(n,"starting",{message:"queue-runner starting"});try{await ce(u);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await g(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await he(n);let i=await ae(n,e),s=0;for(let o=0;o<n.maxJobs;o++){let c=await be(n,e);if(c==="none"||(s+=1,c==="stopped"))break}if(s===0)await I(n.runtimeDir),await g(n,"idle",{processedJobs:s,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await h(a,null),c=String(o?.status||"").trim();await g(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:s,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...c==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`}:{}});}}catch(i){let s=i instanceof Error?i.message:String(i);throw await g(n,"error",{message:s}),await P(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:s}}),i}finally{await de(u);}}}async function Se(e,t,r){let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json");await J(e.runtimeDir,async()=>{let a=await h(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],s=L(t),o=we(t,r);o?s.resumeFromStep=o:delete s.resumeFromStep,await w(n,[s,...i]),await w(u,null);});}async function be(e,t){let r=l.join(e.runtimeDir,"queue.json"),n=l.join(e.runtimeDir,"current-run.json"),u=l.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await J(e.runtimeDir,async()=>{let m=await h(r,[]);if(!Array.isArray(m)||m.length===0)return null;let d={...m[0],status:"running",startedAt:a};return await w(r,m.slice(1)),await w(n,d),d});if(!i)return "none";await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await g(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let s=await ye(i,e,t),o=new Date().toISOString();if(s.stopped){let m=await fe(e,s.stoppedStep||i.command),d={...i,status:"stopped",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{},stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopMode:s.stopRequest?.mode||"requeue",stoppedStep:m};await Se(e,i,m);try{let f=await Q(d,e);d.logArchiveDir=f.archiveDir,d.logArchiveCreatedAt=f.archiveCreatedAt,d.logArchiveFileCount=f.archivedFiles.length;}catch(f){d.logArchiveError=f instanceof Error?f.message:String(f);}return await w(u,d),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job stop requested and requeued.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:s.stopRequest?.mode||"requeue",stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveDir:d.logArchiveDir||"",logArchiveCreatedAt:d.logArchiveCreatedAt||"",logArchiveFileCount:d.logArchiveFileCount??0,logArchiveError:d.logArchiveError||""}}),"stopped"}let c={...i,status:s.exitCode===0?"success":"failed",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{}};await w(n,null),await E(e.runtimeDir,i.id);try{let m=await Q(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await w(u,c);let y=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await P(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:y,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Ae(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function Pe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Ae(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,s=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let c=await o.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(s);}}var Ie={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:Pe},U=N(Ie),Re=U,Me=U;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Re().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
+
export{Me as default,Re as main,U as runQueueRunner};
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import F from'fs/promises';import f from'path';import {workerData,parentPort}from'worker_threads';import'fast-glob';import'os';import'crypto';function R(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function u(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function h(e){return e.replace(/\//g,"\\/")}function b(e){return e.replace(/\//g,"\\\\/")}function $(e,t,i){if(!t)return;e[t]=i;let s=h(t),r=h(i);e[s]=r;let n=b(t),o=b(i);e[n]=o;let a=u(t),l=u(i);e[a]=l;let c=u(s),g=u(r);e[c]=g;let p=u(n),w=u(o);e[p]=w;}function P(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function E(e,t,i){if(!t)return i;let s=f.dirname(f.resolve(t)),r=f.resolve(e,i.replace(/^\/+/,"")),n=f.relative(s,r).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n):"."}var D=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],T=new Set([".html",".htm"]),y="WPSuite.io Static Publisher",j=y.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function k(e,t,i){if(e.wpsuite?.subscriber===true||!i)return false;let s=f.extname(i).toLowerCase();return T.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function H(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${j}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${y}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
|
+
`)?`\r
|
|
3
|
+
`:`
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,r=>`${r}${s} ${t}`)}function O(e){let t=R(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function L(e,t){let i=[...new Set(t)].filter(n=>n.includes("/")&&!/\\+\//.test(n)).map(n=>[n,h(n)]).filter(([n,o])=>n!==o).sort((n,o)=>o[0].length-n[0].length);if(!i.length)return e;let s=n=>{let o=n;for(let[a,l]of i)o=o.split(a).join(l);return o},r=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(n,o,a,l)=>`${o}${s(a)}${l}`);return r=r.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(n,o,a,l,c)=>O(l)?`${o}${s(l)}${c}`:n),r}function U(e,t){if(!t)return null;let i=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),r=f.relative(i,s).replace(/\\/g,"/");return r?(r.startsWith(".")||(r=`./${r}`),r.endsWith("/")?r:`${r}/`):"./"}function A(e,t,i){let s=U(t,i);if(!s)return e;let r=e;for(let n of D){let o=n.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),a=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),l=new RegExp(`(?<!\\.)/${o}`,"g");r=r.replace(a,`${s}${n}`).replace(l,`${s}${n}`);let c=n.replace(/\//g,"\\/"),g=s.replace(/\//g,"\\/"),p=c.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),w=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),v=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),W=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");r=r.replace(w,`${g}${c}`).replace(v,`${g}${c}`).replace(W,`${g}${c}`);}return r}function N(e,t,i){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),r=s?s.pathname:P(t,e.targetOrigin),n=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${r}${n}`:`${E(e.outputDir,i,r)}${n}`}function x(e,t,i,s,r){$(e,i,N(t,s,r));}function S(e,t,i,s){let r=e,n={};x(n,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,l]of Object.entries(t.extraReplacements))x(n,t,a,l,s);for(let[a,l]of Object.entries(i))x(n,t,a,l,s);let o=Object.entries(n).sort((a,l)=>l[0].length-a[0].length);for(let[a,l]of o)r=r.split(a).join(l);return t.urlRewriteMode!=="absolute"&&(r=A(r,t,s)),r=L(r,o.map(([,a])=>a)),k(t,r,s)&&(r=H(r)),r}var{config:M,assetMap:I}=workerData;async function J(e){let t=f.join(M.outputDir,e),i=f.extname(t).toLowerCase(),s;try{s=await F.readFile(t,"utf8");}catch(o){if(o.code==="ENOENT")return false;throw o}let n=S(s,M,I,i===".js"||i===".mjs"?void 0:t);return n===s?false:(await F.writeFile(t,n,"utf8"),true)}if(!parentPort)throw new Error("rewrite worker requires a parent port");parentPort.on("message",async e=>{try{let t=await J(e.file);parentPort?.postMessage({changed:t});}catch(t){parentPort?.postMessage({changed:false,error:t instanceof Error?t.message:String(t)});}});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|