@smart-cloud/publisher-exporter 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawl.js +4 -4
- package/dist/queue-runner.js +3 -3
- package/package.json +1 -1
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import P from'fs/promises';import w from'path';import
|
|
1
|
+
import {chromium}from'playwright';import P from'fs/promises';import w from'path';import _e from'crypto';import Je from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function B(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function Ne(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,a={};if(typeof i.targetOrigin=="string"){let u=ue(i.targetOrigin);u&&(a.targetOrigin=u);}let o=B(i.extraReplacements);if(Object.keys(o).length>0&&(a.extraReplacements=o),i.s3&&typeof i.s3=="object"){let u={},l=i.s3;typeof l.bucket=="string"&&(u.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(u.prefix=l.prefix.trim()),typeof l.region=="string"&&(u.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(u.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(u.assetCacheControl=l.assetCacheControl.trim()),Object.keys(u).length>0&&(a.s3=u);}if(i.cloudFront&&typeof i.cloudFront=="object"){let u={},l=i.cloudFront;typeof l.distributionId=="string"&&(u.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(u.invalidationPaths=l.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(u).length>0&&(a.cloudFront=u);}t[n]=a;}return t}function z(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function We(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function oe(e,t,r){let s=String(t||"").trim();return s&&w.isAbsolute(s)?w.resolve(s):w.resolve(e,z(s,r))}async function G(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await P.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=ue(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=B(r.extraReplacements),r.postCrawlCopyMap=B(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=Ne(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=We(),n=s?w.resolve(s,".."):"";return n?(r.outputDir=oe(n,r.outputDir,"export"),r.logDir=oe(n,r.logDir,"logs")):(w.isAbsolute(r.outputDir)||(r.outputDir=z(r.outputDir,"export")),w.isAbsolute(r.logDir)||(r.logDir=z(r.logDir,"logs"))),r}function ce(e,t=10){return _e.createHash("sha1").update(e).digest("hex").slice(0,t)}async function V(e){await P.mkdir(w.dirname(e),{recursive:true});}function O(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(w.extname(e).toLowerCase())}function L(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function M(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function F(e){return e.replace(/\//g,"\\/")}function le(e){return e.replace(/\//g,"\\\\/")}function de(e,t,r){if(!t)return;e[t]=r;let s=F(t),n=F(r);e[s]=n;let i=le(t),a=le(r);e[i]=a;let o=M(t),u=M(r);e[o]=u;let l=M(s),c=M(n);e[l]=c;let p=M(i),d=M(a);e[p]=d;}function K(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function pe(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function ge(e,t,r){if(!t)return r;let s=w.dirname(w.resolve(t)),n=w.resolve(e,r.replace(/^\/+/,"")),i=w.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Ge=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Ve=new Set([".html",".htm"]),he="WPSuite.io Static Publisher",Ke=he.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function Xe(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=w.extname(r).toLowerCase();return Ve.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function Ye(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Ke}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${he}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>ct(t,r));}updateCurrentEvent(t,r,s){let n=pt(t,r,s);this.enqueueTask(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(I[this.level]>=I.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){lt("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),I[this.level]>=I.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),I[this.level]>=I.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var gt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],mt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),ht=new Set(["",".html",".htm"]),ft=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),wt=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function Pt(e){let t=[],r="full",s="full",n=false;for(let i=0;i<e.length;i++){let a=e[i];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--crawl-mode"){let o=e[++i];if(!o)throw new Error("--crawl-mode requires a value");s=o==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let o=e[++i];if(!o)throw new Error("--url requires a value");r="single-url",n=true,t.push(o);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let o=e[++i];if(!o)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${o}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n}}function re(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function yt(){let e=re();return e?w.resolve(e,".."):""}function St(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?w.resolve(e):""}function vt(e){let t=e.trim(),r=[{alias:"@storage-root",root:yt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:re(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:St(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?w.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:w.resolve(t)}}function je(e){let t=re();return t?w.join(t,"crawl-manifest.json"):w.join(e.outputDir,".crawl-manifest.json")}async function bt(e){try{let t=await P.readFile(je(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.schemaVersion===1&&r.pages&&typeof r.pages=="object")return r}catch{}return {schemaVersion:1,updatedAt:"",pages:{}}}function ve(e){return JSON.parse(JSON.stringify(e))}async function Rt(e,t){let r=je(e);await P.mkdir(w.dirname(r),{recursive:true}),await P.writeFile(r,JSON.stringify(t,null,2),"utf8");}function N(e){if(!e)return null;let t=e.trim();return t||null}async function xt(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:a,status:o.status()}),new Map;let l=await o.json().catch(()=>null);if(!l||!Array.isArray(l.items))return new Map;let u=new Map;for(let c of l.items)!c||typeof c.url!="string"||u.set(c.url,c);return u}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:String(o)}),new Map}}async function be(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await xt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,a),a}async function Ct(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let a=s.manifest.pages[n];if(!a)return {action:"render",changeToken:await be(e,t,s,n,i)};let o=await be(e,t,s,n,i);if(o?.supported&&o.token)return a.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let l=N(r.sitemapLastmodByPage[n]);return l&&a.sitemapLastmod&&a.sitemapLastmod===l?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function kt(e,t,r,s){let n=Re(r.previousManifest),i=0;for(let[p,d]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){try{await P.unlink(d.outputPath);}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:d.outputPath,error:String(m)});}delete r.manifest.pages[p],i++;}let a=Re(r.manifest),o=new Set,l=new Set;for(let p of a){for(let m of xe(e,p))o.add(m);let d=W(e,t.assetMap,p);d&&l.add(d);}let u=0,c=0;for(let p of n){if(a.has(p))continue;let d=new Set,m=W(e,t.assetMap,p);if(m)d.add(m);else {let f=ie(e,p);d.add(f.originalFilePath),f.hashedFilePath&&d.add(f.hashedFilePath);}for(let f of d)if(!l.has(f))try{await P.unlink(f),u++;}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:f,error:String(g)});}for(let f of xe(e,p))o.has(f)||Object.prototype.hasOwnProperty.call(t.assetMap,f)&&(delete t.assetMap[f],c++);}(i>0||u>0||c>0)&&s.info(`Incremental cleanup removed ${i} stale pages, ${u} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:u,removedAssetMappings:c,phase:"incremental-cleanup"});}function Re(e){let t=new Set;for(let r of Object.values(e.pages))for(let s of r.discoveredAssets||[]){let n=s.trim();n&&t.add(n);}return t}function xe(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Lt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return w.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function W(e,t,r){let s=t[r];return s?Lt(e,s):ie(e,r).originalFilePath}function Et(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.join(w.resolve(e),"queue-runner-heartbeat.json"):""}async function Ce(e){let t=Et();if(t)try{let r=await P.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await P.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Mt(e,t){let r=new Set;for(let s of t.donePages)r.add(w.resolve(ne(e,s)));for(let s of t.doneAssets){let n=W(e,t.assetMap,s);!n||!U(n)||r.add(w.resolve(n));}return [...r]}var _="re:",q=new Map;function Tt(e){if(!e.startsWith(_))return null;let t=q.get(e);if(t!==void 0)return t||null;let r=e.slice(_.length).trim();if(!r)return q.set(e,false),null;try{let s=new RegExp(r);return q.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),q.set(e,false),null}}function se(e,t){let r=Tt(t);return r?r.test(e):t.startsWith(_)?false:e.startsWith(t)}function At(e,t){return t.find(r=>se(e,r))}function Fe(e,t){let r=At(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(_)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function It(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function $t(e,t){return It(e).has(t.hostname)}function Dt(e){let t=L(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Oe(e){let t=L(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function J(e){return wt.some(t=>e.pathname.startsWith(t))}function v(e,t,r=e.sourceOrigin,s,n="url"){let i=Dt(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Oe(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let a=new URL(i,r);if(!["http:","https:"].includes(a.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!$t(e,a))return s?.ignore(n,a.toString(),"host not allowed",r),null;let o=Fe(e,a);if(o)return s?.reject(n,a.toString(),o,r),null;let l=new URL(e.sourceOrigin);return a.protocol=l.protocol,a.host=l.host,a.hash="",a}catch{return s?.reject(n,t,"invalid URL",r),null}}function Ue(e){return mt.has(w.extname(e.pathname).toLowerCase())}function F(e){return J(e)?false:ht.has(w.extname(e.pathname).toLowerCase())}function B(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function y(e,t){return B(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Ue(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>se(t.pathname,r)):false}function ne(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":w.extname(s)||(s+="/index.html"),w.join(e.outputDir,s)}function ie(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=w.join(e.outputDir,i),o=X(e.targetOrigin,i),l=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:l,preferQueryHashed:n};let u=w.extname(s),p=`${u?s.slice(0,-u.length):s}.${de(r.search)}${u||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:l,preferQueryHashed:n,hashedPathname:p,hashedFilePath:w.join(e.outputDir,p),hashedPublicUrl:X(e.targetOrigin,p)}}function jt(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let l of o)try{let u=new URL(l);t[u.origin+i.pathname+i.search]=s;}catch{}try{let l=(()=>{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${u.pathname}${u.search}`})();t[i.pathname+i.search]=l;}catch{}try{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=l;}catch{}try{let l=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=l;}catch{}}catch{}}async function ee(e){try{return await P.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Qe(e){try{return await P.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Ft(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function qe(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Ot(e,t){let r=qe(t),s=Ft(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function Ut(e,t){if(!t)return;let r=qe(t);if(r===null)return;let s=await Qe(e);if(!s)return;let n=new Date(r);await P.utimes(e,s.atime,n).catch(()=>{});}function ke(e,t){let r=w.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return ft.has(r)||gt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function Le(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function te(e,t,r,s,n,i,a){let o=ie(e,r),l=o.originalFilePath,u=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let p=await ee(o.originalFilePath);p!==null&&!p.equals(s)&&(l=o.hashedFilePath,u=o.hashedPublicUrl);}jt(e,t,r,u,o.originalPublicUrl);let c=await ee(l);(c===null||!c.equals(s))&&(await K(l),await P.writeFile(l,s)),await Ut(l,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"});}function H(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=v(e,r,i,s,"page");if(!a)return;if(!F(a)||Ue(a)){if(y(e,a)){s.info(`Seeded/non-page URL queued as asset: ${a.toString()}`,{url:a.toString(),source:n}),k(e,t,a.toString(),s,n);return}s.reject("page",a.toString(),J(a)?"asset/internal path cannot be page":"not page-like",n);return}let o=a.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=v(e,r,i,s,"asset");if(!a)return;if(!y(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let o=a.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ae(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=v(e,r,i,s,"sitemap");if(!a)return;if(!B(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let o=a.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Qt(e){let t=L(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:N(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function qt(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Nt(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Z(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Wt(e,t){await P.mkdir(t,{recursive:true});let r=await P.readdir(e,{withFileTypes:true});for(let s of r){let n=w.join(e,s.name),i=w.join(t,s.name);if(s.isDirectory()){await P.cp(n,i,{recursive:true,force:true});continue}await P.copyFile(n,i);}}async function _t(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([n,i])=>({sourcePath:n.trim(),prefix:String(i||"").trim()}));if(r.length===0)return;let s=0;t.mark("copy-extra-paths");for(let n of r){let i=n.sourcePath,a=n.prefix;if(!i||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:i,prefixPath:a});continue}let o=vt(i);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:i,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let l=o.resolvedPath,u=Z(a);if(!u){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:i,prefixPath:a});continue}let c;try{c=await P.stat(l);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:i,sourceAbs:l,prefixPath:a});continue}let p=w.resolve(e.outputDir,u);if(c.isDirectory()){await Wt(l,p),s++,t.progress(`Copied static directory to export: ${l} -> /${u}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${u}`,copiedItems:s});continue}let d=w.basename(l),m=a.endsWith("/")?w.join(p,d):p;await P.mkdir(w.dirname(m),{recursive:true}),await P.copyFile(l,m),s++,t.progress(`Copied static file to export: ${l} -> /${Z(a.endsWith("/")?`${a}${d}`:a)}`,{phase:"copy-extra-paths",sourcePath:l,targetPath:`/${Z(a.endsWith("/")?`${a}${d}`:a)}`,copiedItems:s});}t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s});}function j(e,t,r,s,n){let i=new Set,a=L(r).replace(/\\\//g,"/");for(let u of Nt(a)){let c=v(e,u,t,s,`${n}:css-url`);c&&y(e,c)&&i.add(c.toString());}for(let u of a.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=v(e,u[1],t,s,`${n}:css-import`);c&&y(e,c)&&i.add(c.toString());}for(let u of a.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=v(e,u[1],t,s,`${n}:xml-stylesheet`);c&&y(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",l=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let u of a.matchAll(l)){let c=u[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${u[0]}`:u[0];if(Oe(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=v(e,c,t,s,`${n}:serialized-url`);p&&y(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function Ht(e,t,r,s){let n=new Set,i=L(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=a[1],l=v(e,o,t,s,"page-link");l&&(Fe(e,l)||F(l)&&!J(l)&&!y(e,l)&&n.add(l.toString()));}return [...n]}async function Ee(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let a=await i.body(),o=i.url()||n;await te(e,r.assetMap,o,a,r,s);let l=a.toString("utf8");for(let u of j(e,o,l,s,"sitemap"))k(e,r,u,s,o);for(let u of Qt(l)){let c=v(e,u.loc,o,s,"sitemap-loc");c&&(B(c)?ae(e,r,c.toString(),s,o):y(e,c)?k(e,r,c.toString(),s,o):F(c)?(u.lastmod&&(r.sitemapLastmodByPage[c.toString()]=u.lastmod),H(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Jt(e,t,r,s,n){let i=v(e,s,e.sourceOrigin,n,"asset-fetch");if(!i||!y(e,i))return;let a=i.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let o=W(e,r.assetMap,a),l=o?await Qe(o):null;if(o&&l)try{let m=await t.head(a,{timeout:6e4}),f=m.headers();if(m.ok()&&Ot(l,f)){let g=await ee(o);if(g!==null){let h=m.url()||a;if(await te(e,r.assetMap,h,g,r,n,f),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:o,lastModified:f["last-modified"]||"",contentLength:f["content-length"]||"",phase:"download-assets"}),Le(r,n),ke(i,f)){let S=g.toString("utf8");for(let b of j(e,h,S,n,"asset:cached"))k(e,r,b,n,h);}return}}}catch{}let u=await t.get(a,{timeout:6e4});if(!u.ok()){n.skip("asset",a,u.status());return}let c=await u.body(),p=u.url()||a,d=u.headers();if(await te(e,r.assetMap,p,c,r,n,d),r.doneAssets.add(a),Le(r,n),ke(i,d)){let m=c.toString("utf8");for(let f of j(e,p,m,n,`asset:${w.extname(i.pathname).toLowerCase()||d["content-type"]||"unknown"}`))k(e,r,f,n,p);}}catch(o){n.error(`Failed asset ${a}`,{url:a,error:String(o)});}}}async function Bt(e,t,r,s){let n=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),i=Array.from({length:n},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Jt(e,t,r,a,s));}});await Promise.all(i);}async function zt(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Gt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await zt(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Vt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Kt(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>se(t,r))}async function Xt(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let l=new Set,u=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let f of u){let g=d.getAttribute(f);g&&l.add(g);}for(let f of p){let g=d.getAttribute(f);g&&g.split(",").forEach(h=>{let S=h.trim().split(/\s+/)[0];S&&l.add(S);});}let m=d.getAttribute("style");m&&c.add(m);}),{attrs:[...l],styles:[...c]}});for(let l of o.attrs)for(let u of qt(l)){let c=v(e,u,r,s,"dom-attr");c&&y(e,c)&&i.add(c.toString());}for(let l of o.styles)for(let u of j(e,r,l,s,"dom-style"))i.add(u);}let a=await t.content();for(let o of j(e,r,a,s,"page-html"))i.add(o);return [...i]}async function Me(e,t,r,s,n){let i=ne(e,t);await K(i),await P.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}async function Yt(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Zt(e,t,r,s,n,i){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),l=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Yt(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let u=t.pageQueue.shift();if(!u)break;if(t.queuedPages.delete(u),t.donePages.has(u))continue;t.donePages.add(u),i.enabled&&i.seenPages.add(u);let c=new URL(u),p=Kt(e,c.pathname);if(!F(c)||y(e,c)||J(c)){if(y(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${u}`,{url:u,source:"worker-guard"}),k(e,t,u,r,"worker-guard");continue}r.reject("page",u,"guard rejected non-page URL before rendering","worker");continue}let d=await Ct(e,n,t,i,u,r);if(i.enabled&&s!=="single-url"){let g=i.manifest.pages[u];if(d.action==="reuse"&&g){for(let h of g.discoveredAssets)k(e,t,h,r,u);for(let h of g.discoveredPages)H(e,t,h,r,u);g.lastSeenRunId=i.runId,g.sitemapLastmod=N(t.sitemapLastmodByPage[u])??g.sitemapLastmod,d.changeToken?.supported&&(g.changeToken=d.changeToken.token,g.tokenSource=d.changeToken.tokenSource??g.tokenSource),r.info(`Incremental reuse skipped unchanged page ${u}`,{url:u,mode:"incremental",reason:d.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}let m=await(p?l.newPage():o.newPage()),f=!1;await m.route("**/*",async g=>{let h=g.request();if(h.isNavigationRequest()&&h.frame()===m.mainFrame())try{let S=new URL(h.url()),b=new URL(u);if(S.hash="",b.hash="",S.href===b.href){if(f){r.warn(`Blocked same-page navigation/reload for ${u}`,{url:u,requestUrl:h.url()}),await g.abort("aborted");return}f=!0;}}catch{}await g.continue();}),m.on("response",g=>{try{let h=v(e,g.url(),e.sourceOrigin,void 0,"network-response");h&&y(e,h)&&k(e,t,h.toString(),r,u);}catch{}}),r.page(`Rendering ${u}`,{url:u});try{let g=null,h=null;try{if(g=await m.goto(u,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),g&&g.ok())try{h=await g.text();}catch{h=null;}}catch(M){r.warn(`Navigation issue for ${u}; saving current DOM if available`,{url:u,error:String(M)});}if(g&&!g.ok()){r.skip("page",u,g.status()),await m.close();continue}p?(r.info(`Rendering without JS execution for ${u}`,{url:u,mode:"no-js"}),await Vt(e,m)):await Gt(e,m),t.stats.pagesRendered++,(t.stats.pagesRendered%5===0||t.stats.pagesRendered===1||t.stats.pagesRendered===t.stats.pagesQueued)&&r.progress(`Render progress: rendered ${t.stats.pagesRendered}, saved ${t.stats.pagesSaved}, discovered ${t.stats.pagesQueued}.`,{pagesRendered:t.stats.pagesRendered,pagesSaved:t.stats.pagesSaved,pagesQueued:t.stats.pagesQueued,phase:"render-pages"});let S=await Xt(e,m,u,r,!p);for(let M of S)k(e,t,M,r,u);let b=h??await m.content(),$=s!=="single-url"?Ht(e,u,b,r):[];if(s!=="single-url")for(let M of $)H(e,t,M,r,u);await Me(e,u,b,t,r),i.enabled&&(i.manifest.pages[u]={url:u,outputPath:ne(e,u),changeToken:d.changeToken?.supported?d.changeToken.token:null,tokenSource:d.changeToken?.supported?d.changeToken.tokenSource??null:null,sitemapLastmod:N(t.sitemapLastmodByPage[u])??null,discoveredPages:$,discoveredAssets:S,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(g){try{let h=await m.content();h&&h.trim()&&(await Me(e,u,h,t,r),r.warn(`Saved partial DOM for ${u}`,{url:u,error:String(g)}));}catch(h){r.error(`Could not save partial DOM for ${u}`,{url:u,error:String(h)});}r.error(`Failed page ${u}`,{url:u,error:String(g)});}finally{await m.close();}}}finally{await l.close(),await o.close(),await a.close();}}async function Te(e){try{let r=(await P.readFile(e,"utf8")).trim();if(!r)return [];try{let s=JSON.parse(r);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return r.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let n=JSON.parse(s);return n&&typeof n=="object"?n:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function Ae(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function er(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function tr(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function rr(e){try{let t=await P.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}async function Ie(e){let t=await Te(w.join(e,"errors.jsonl"));return t.length>0?t:Te(w.join(e,"errors.json"))}async function sr(e){let t=w.join(e.logDir,"archive"),r;try{r=await P.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=w.join(t,i);if(!(await P.stat(a).catch(()=>null))?.isDirectory())continue;let l=await rr(w.join(a,"job.json"));if(!er(l))continue;let u=tr(l);u>=n&&(n=u,s=a);}return s}async function nr(e){let t=await sr(e),r=t!==""?await Ie(t):await Ie(e.logDir),s=new Set;for(let n of r)if(Ae(n.error)||Ae(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function ir(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await P.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function ar(e){try{let t=await P.readFile(w.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function or(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await P.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await P.unlink(w.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function $e(e,t,r,s,n){let i=v(e,r,e.sourceOrigin,s,"manual-url");i&&(B(i)?ae(e,t,i.toString(),s,n):y(e,i)?k(e,t,i.toString(),s,n):F(i)?H(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function ur(){let e=await V(),t=Pt(process.argv.slice(2)),r=[];t.mode==="retry-timeouts"&&(r=await nr(e)),t.mode==="single-url"&&(r=await ir(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s?await bt(e):{schemaVersion:1,updatedAt:"",pages:{}},i={enabled:s,manifest:ve(n),previousManifest:ve(n),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await or(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:t.preserveOutput?await ar(e):{},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}};t.preserveOutput||await P.rm(e.outputDir,{recursive:true,force:true}),await P.mkdir(e.outputDir,{recursive:true});let o=new D(e.logDir,e.logLevel),l=await chromium.launch({headless:true}),u=await l.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),c=u.request;try{if(o.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(h=>ae(e,a,h,o)),await Ee(e,c,a,o),e.seedPaths.forEach(h=>$e(e,a,h,o,"seed-path"))):(r.forEach(h=>$e(e,a,h,o,"cli")),await Ee(e,c,a,o)),o.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),o.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){o.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await o.flush();return}o.mark("render-pages");let g=Array.from({length:Math.max(1,e.concurrency)},()=>Zt(e,a,o,t.mode,c,i));await Promise.all(g),o.endMark("render-pages",{pages:a.donePages.size}),o.mark("download-assets"),await Bt(e,c,a,o),o.endMark("download-assets",{assets:a.doneAssets.size}),i.enabled&&e.maxPages===0&&await kt(e,a,i,o),t.mode==="full"&&await _t(e,o);}finally{await u.close(),await l.close();}i.enabled&&(i.manifest.updatedAt=new Date().toISOString(),await Rt(e,i.manifest)),await P.writeFile(w.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8");let p=t.mode==="single-url"?Mt(e,a):void 0;o.mark("rewrite-text");let d=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${p?.length??0}`:"Rewriting text files...";o.progress(d,{phase:"rewrite-text",index:0,totalFiles:p?.length,changedTextFiles:0}),await Ce(d);let m=Date.now(),f=await we(e,a.assetMap,{files:p,onProgress:async({index:g,totalFiles:h,changedTextFiles:S,file:b})=>{o.checkpoint(`Rewriting text file ${g}/${h}`,{phase:"rewrite-text",index:g,totalFiles:h,changedTextFiles:S,file:b});let $=Date.now();if(!(g===1||g===h||$-m>=5e3))return;m=$;let oe=`Rewriting text files: ${g}/${h}`;o.progress(oe,{phase:"rewrite-text",index:g,totalFiles:h,changedTextFiles:S,file:b}),await Ce(oe);}});o.endMark("rewrite-text",{changedTextFiles:f}),o.summary(`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:f}),await o.flush();}ur().catch(async e=>{console.error(e);try{let r=(await V().catch(()=>null))?.logDir??"logs",s=new D(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function Ze(e){let t=L(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function et(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,F(i)]).filter(([i,a])=>i!==a).sort((i,a)=>a[0].length-i[0].length);if(!r.length)return e;let s=i=>{let a=i;for(let[o,u]of r)a=a.split(o).join(u);return a},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,a,o,u)=>`${a}${s(o)}${u}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,a,o,u,l)=>Ze(u)?`${a}${s(u)}${l}`:i),n}function tt(e,t){if(!t)return null;let r=w.dirname(w.resolve(t)),s=w.resolve(e.outputDir),n=w.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function rt(e,t,r){let s=tt(t,r);if(!s)return e;let n=e;for(let i of Ge){let a=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),o=new RegExp(`(?:\\.{1,}/)+${a}`,"g"),u=new RegExp(`(?<!\\.)/${a}`,"g");n=n.replace(o,`${s}${i}`).replace(u,`${s}${i}`);let l=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),p=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),h=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${c}${l}`).replace(g,`${c}${l}`).replace(h,`${c}${l}`);}return n}function st(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:pe(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${ge(e.outputDir,r,n)}${i}`}function X(e,t,r,s,n){de(e,r,st(t,s,n));}function nt(e,t,r,s){let n=e,i={};X(i,t,t.sourceOrigin,t.targetOrigin,s);for(let[o,u]of Object.entries(t.extraReplacements))X(i,t,o,u,s);for(let[o,u]of Object.entries(r))X(i,t,o,u,s);let a=Object.entries(i).sort((o,u)=>u[0].length-o[0].length);for(let[o,u]of a)n=n.split(o).join(u);return t.urlRewriteMode!=="absolute"&&(n=rt(n,t,s)),n=et(n,a.map(([,o])=>o)),Xe(t,n,s)&&(n=Ye(n)),n}async function it(e,t,r){let s=w.join(e.outputDir,r),n=w.extname(s).toLowerCase(),i;try{i=await P.readFile(s,"utf8");}catch(u){if(u.code!=="ENOENT")throw u;return {changed:false}}let o=nt(i,e,t,n===".js"||n===".mjs"?void 0:s);return o!==i?(await P.writeFile(s,o,"utf8"),{changed:true}):{changed:false}}async function at(e,t){return await new Promise((r,s)=>{let n=u=>{if(o(),u?.error){s(new Error(u.error));return}r({changed:!!u?.changed});},i=u=>{o(),s(u);},a=u=>{o(),u!==0&&s(new Error(`Rewrite worker exited with code ${u}.`));},o=()=>{e.off("message",n),e.off("error",i),e.off("exit",a);};e.on("message",n),e.on("error",i),e.on("exit",a),e.postMessage({file:t});})}async function fe(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>w.resolve(d)).filter(d=>d.startsWith(w.resolve(e.outputDir))).map(d=>w.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Je(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&O(d)),i=0;if(n.length===0)return 0;let a=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),o=Math.max(1,Math.min(a,n.length,availableParallelism())),u=0,l=0,c=null;if(o===1){for(let d of n)(await it(e,t,d)).changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:d});return i}let p=Array.from({length:o},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t}});try{for(;!c;){let g=u;if(g>=n.length)return;u+=1;let h=n[g];(await at(d,h)).changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:h});}}catch(g){throw c=g instanceof Error?g:new Error(String(g)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var A={error:0,warn:1,info:2,debug:3},we=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",U=we?w.join(we,"current-progress.json"):"",Pe=Promise.resolve();function ye(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function ot(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let a=r[n];if(typeof a=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(a,i);continue}r[n]=i;}return r}function ut(e,t,r){U&&(Pe=Pe.then(async()=>{let s=new Date,n=s.toISOString();await P.mkdir(w.dirname(U),{recursive:true});let i=await P.readFile(U,"utf8").then(m=>JSON.parse(m)).catch(()=>null),a=i&&typeof i.details=="object"&&i.details?i.details:{},o=ye(e),u=i?.currentStep||ye(i?.source||""),l=i?.startedAt||n,c={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(u&&u!==o&&i?.stepStartedAt){let m=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[u]=(c[u]??0)+m,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),h={checkedAt:n,source:e,message:t,details:ot(a,r??{}),startedAt:l,currentStep:o,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:c};await P.writeFile(U,JSON.stringify(h,null,2),"utf8");}).catch(()=>{}));}async function lt(e,t){await P.mkdir(w.dirname(e),{recursive:true}),await P.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function ct(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function dt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:ct(e,r),level:e,message:t,details:r??{}}}var $=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return w.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return w.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return w.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return w.join(this.logDir,"ignored.jsonl")}get skippedPath(){return w.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return w.join(this.logDir,"errors.jsonl")}get timingsPath(){return w.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await P.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>lt(t,r));}updateCurrentEvent(t,r,s){let n=dt(t,r,s);this.enqueueTask(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(A[this.level]>=A.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),A[this.level]>=A.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),A[this.level]>=A.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var pt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],gt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),mt=new Set(["",".html",".htm"]),ht=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),ft=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function wt(e){let t=[],r="full",s="full",n=false,i=false;for(let a=0;a<e.length;a++){let o=e[a];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let u=e[++a];if(!u)throw new Error("--crawl-mode requires a value");s=u==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let u=e[++a];if(!u)throw new Error("--url requires a value");r="single-url",n=true,t.push(u);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let u=e[++a];if(!u)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${u}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function te(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function Pt(){let e=te();return e?w.resolve(e,".."):""}function yt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?w.resolve(e):""}function St(e){let t=e.trim(),r=[{alias:"@storage-root",root:Pt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:te(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:yt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?w.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:w.resolve(t)}}function De(e){let t=te();return t?w.join(t,"crawl-manifest.json"):w.join(e.outputDir,".crawl-manifest.json")}async function vt(e){try{let t=await P.readFile(De(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.schemaVersion===1&&r.pages&&typeof r.pages=="object")return r}catch{}return {schemaVersion:1,updatedAt:"",pages:{}}}function Se(e){return JSON.parse(JSON.stringify(e))}async function bt(e,t){let r=De(e);await P.mkdir(w.dirname(r),{recursive:true}),await P.writeFile(r,JSON.stringify(t,null,2),"utf8");}function q(e){if(!e)return null;let t=e.trim();return t||null}async function Rt(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:a,status:o.status()}),new Map;let u=await o.json().catch(()=>null);if(!u||!Array.isArray(u.items))return new Map;let l=new Map;for(let c of u.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:String(o)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await Rt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,a),a}async function xt(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let a=s.manifest.pages[n];if(!a)return {action:"render",changeToken:await ve(e,t,s,n,i)};let o=await ve(e,t,s,n,i);if(o?.supported&&o.token)return a.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let u=q(r.sitemapLastmodByPage[n]);return u&&a.sitemapLastmod&&a.sitemapLastmod===u?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function Ct(e,t,r,s){let n=be(r.previousManifest),i=0;for(let[p,d]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){try{await P.unlink(d.outputPath);}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:d.outputPath,error:String(g)});}delete r.manifest.pages[p],i++;}let a=be(r.manifest),o=new Set,u=new Set;for(let p of a){for(let g of Re(e,p))o.add(g);let d=N(e,t.assetMap,p);d&&u.add(d);}let l=0,c=0;for(let p of n){if(a.has(p))continue;let d=new Set,g=N(e,t.assetMap,p);if(g)d.add(g);else {let h=ne(e,p);d.add(h.originalFilePath),h.hashedFilePath&&d.add(h.hashedFilePath);}for(let h of d)if(!u.has(h))try{await P.unlink(h),l++;}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:h,error:String(m)});}for(let h of Re(e,p))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],c++);}(i>0||l>0||c>0)&&s.info(`Incremental cleanup removed ${i} stale pages, ${l} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:l,removedAssetMappings:c,phase:"incremental-cleanup"});}function be(e){let t=new Set;for(let r of Object.values(e.pages))for(let s of r.discoveredAssets||[]){let n=s.trim();n&&t.add(n);}return t}function Re(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function kt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return w.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function N(e,t,r){let s=t[r];return s?kt(e,s):ne(e,r).originalFilePath}function Lt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.join(w.resolve(e),"queue-runner-heartbeat.json"):""}async function xe(e){let t=Lt();if(t)try{let r=await P.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await P.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Et(e,t){let r=new Set;for(let s of t.donePages)r.add(w.resolve(se(e,s)));for(let s of t.doneAssets){let n=N(e,t.assetMap,s);!n||!O(n)||r.add(w.resolve(n));}return [...r]}var W="re:",Q=new Map;function Mt(e){if(!e.startsWith(W))return null;let t=Q.get(e);if(t!==void 0)return t||null;let r=e.slice(W.length).trim();if(!r)return Q.set(e,false),null;try{let s=new RegExp(r);return Q.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Q.set(e,false),null}}function re(e,t){let r=Mt(t);return r?r.test(e):t.startsWith(W)?false:e.startsWith(t)}function Tt(e,t){return t.find(r=>re(e,r))}function je(e,t){let r=Tt(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(W)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function At(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function It(e,t){return At(e).has(t.hostname)}function $t(e){let t=L(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Fe(e){let t=L(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function H(e){return ft.some(t=>e.pathname.startsWith(t))}function S(e,t,r=e.sourceOrigin,s,n="url"){let i=$t(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Fe(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let a=new URL(i,r);if(!["http:","https:"].includes(a.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!It(e,a))return s?.ignore(n,a.toString(),"host not allowed",r),null;let o=je(e,a);if(o)return s?.reject(n,a.toString(),o,r),null;let u=new URL(e.sourceOrigin);return a.protocol=u.protocol,a.host=u.host,a.hash="",a}catch{return s?.reject(n,t,"invalid URL",r),null}}function Oe(e){return gt.has(w.extname(e.pathname).toLowerCase())}function j(e){return H(e)?false:mt.has(w.extname(e.pathname).toLowerCase())}function J(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function y(e,t){return J(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Oe(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>re(t.pathname,r)):false}function se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":w.extname(s)||(s+="/index.html"),w.join(e.outputDir,s)}function ne(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=w.join(e.outputDir,i),o=K(e.targetOrigin,i),u=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:u,preferQueryHashed:n};let l=w.extname(s),p=`${l?s.slice(0,-l.length):s}.${ce(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:u,preferQueryHashed:n,hashedPathname:p,hashedFilePath:w.join(e.outputDir,p),hashedPublicUrl:K(e.targetOrigin,p)}}function Dt(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let u of o)try{let l=new URL(u);t[l.origin+i.pathname+i.search]=s;}catch{}try{let u=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=u;}catch{}try{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}try{let u=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}}catch{}}async function Z(e){try{return await P.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Ue(e){try{return await P.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function jt(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Qe(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Ft(e,t){let r=Qe(t),s=jt(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function Ot(e,t){if(!t)return;let r=Qe(t);if(r===null)return;let s=await Ue(e);if(!s)return;let n=new Date(r);await P.utimes(e,s.atime,n).catch(()=>{});}function Ce(e,t){let r=w.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return ht.has(r)||pt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function ke(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ee(e,t,r,s,n,i,a){let o=ne(e,r),u=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let p=await Z(o.originalFilePath);p!==null&&!p.equals(s)&&(u=o.hashedFilePath,l=o.hashedPublicUrl);}Dt(e,t,r,l,o.originalPublicUrl);let c=await Z(u);(c===null||!c.equals(s))&&(await V(u),await P.writeFile(u,s)),await Ot(u,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"});}function _(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"page");if(!a)return;if(!j(a)||Oe(a)){if(y(e,a)){s.info(`Seeded/non-page URL queued as asset: ${a.toString()}`,{url:a.toString(),source:n}),k(e,t,a.toString(),s,n);return}s.reject("page",a.toString(),H(a)?"asset/internal path cannot be page":"not page-like",n);return}let o=a.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"asset");if(!a)return;if(!y(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let o=a.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ie(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"sitemap");if(!a)return;if(!J(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let o=a.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Ut(e){let t=L(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:q(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Qt(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function qt(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Y(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Nt(e,t){await P.mkdir(t,{recursive:true});let r=await P.readdir(e,{withFileTypes:true});for(let s of r){let n=w.join(e,s.name),i=w.join(t,s.name);if(s.isDirectory()){await P.cp(n,i,{recursive:true,force:true});continue}await P.copyFile(n,i);}}async function Wt(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([n,i])=>({sourcePath:n.trim(),prefix:String(i||"").trim()}));if(r.length===0)return;let s=0;t.mark("copy-extra-paths");for(let n of r){let i=n.sourcePath,a=n.prefix;if(!i||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:i,prefixPath:a});continue}let o=St(i);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:i,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let u=o.resolvedPath,l=Y(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:i,prefixPath:a});continue}let c;try{c=await P.stat(u);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:i,sourceAbs:u,prefixPath:a});continue}let p=w.resolve(e.outputDir,l);if(c.isDirectory()){await Nt(u,p),s++,t.progress(`Copied static directory to export: ${u} -> /${l}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${l}`,copiedItems:s});continue}let d=w.basename(u),g=a.endsWith("/")?w.join(p,d):p;await P.mkdir(w.dirname(g),{recursive:true}),await P.copyFile(u,g),s++,t.progress(`Copied static file to export: ${u} -> /${Y(a.endsWith("/")?`${a}${d}`:a)}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${Y(a.endsWith("/")?`${a}${d}`:a)}`,copiedItems:s});}t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s});}function D(e,t,r,s,n){let i=new Set,a=L(r).replace(/\\\//g,"/");for(let l of qt(a)){let c=S(e,l,t,s,`${n}:css-url`);c&&y(e,c)&&i.add(c.toString());}for(let l of a.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=S(e,l[1],t,s,`${n}:css-import`);c&&y(e,c)&&i.add(c.toString());}for(let l of a.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=S(e,l[1],t,s,`${n}:xml-stylesheet`);c&&y(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",u=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of a.matchAll(u)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(Fe(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=S(e,c,t,s,`${n}:serialized-url`);p&&y(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function _t(e,t,r,s){let n=new Set,i=L(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=a[1],u=S(e,o,t,s,"page-link");u&&(je(e,u)||j(u)&&!H(u)&&!y(e,u)&&n.add(u.toString()));}return [...n]}async function Le(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let a=await i.body(),o=i.url()||n;await ee(e,r.assetMap,o,a,r,s);let u=a.toString("utf8");for(let l of D(e,o,u,s,"sitemap"))k(e,r,l,s,o);for(let l of Ut(u)){let c=S(e,l.loc,o,s,"sitemap-loc");c&&(J(c)?ie(e,r,c.toString(),s,o):y(e,c)?k(e,r,c.toString(),s,o):j(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),_(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Ht(e,t,r,s,n){let i=S(e,s,e.sourceOrigin,n,"asset-fetch");if(!i||!y(e,i))return;let a=i.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let o=N(e,r.assetMap,a),u=o?await Ue(o):null;if(o&&u)try{let g=await t.head(a,{timeout:6e4}),h=g.headers();if(g.ok()&&Ft(u,h)){let m=await Z(o);if(m!==null){let f=g.url()||a;if(await ee(e,r.assetMap,f,m,r,n,h),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:o,lastModified:h["last-modified"]||"",contentLength:h["content-length"]||"",phase:"download-assets"}),ke(r,n),Ce(i,h)){let v=m.toString("utf8");for(let b of D(e,f,v,n,"asset:cached"))k(e,r,b,n,f);}return}}}catch{}let l=await t.get(a,{timeout:6e4});if(!l.ok()){n.skip("asset",a,l.status());return}let c=await l.body(),p=l.url()||a,d=l.headers();if(await ee(e,r.assetMap,p,c,r,n,d),r.doneAssets.add(a),ke(r,n),Ce(i,d)){let g=c.toString("utf8");for(let h of D(e,p,g,n,`asset:${w.extname(i.pathname).toLowerCase()||d["content-type"]||"unknown"}`))k(e,r,h,n,p);}}catch(o){n.error(`Failed asset ${a}`,{url:a,error:String(o)});}}}async function Jt(e,t,r,s){let n=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),i=Array.from({length:n},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Ht(e,t,r,a,s));}});await Promise.all(i);}async function Bt(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function zt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Bt(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Gt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Vt(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>re(t,r))}async function Kt(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let u=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let h of l){let m=d.getAttribute(h);m&&u.add(m);}for(let h of p){let m=d.getAttribute(h);m&&m.split(",").forEach(f=>{let v=f.trim().split(/\s+/)[0];v&&u.add(v);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...u],styles:[...c]}});for(let u of o.attrs)for(let l of Qt(u)){let c=S(e,l,r,s,"dom-attr");c&&y(e,c)&&i.add(c.toString());}for(let u of o.styles)for(let l of D(e,r,u,s,"dom-style"))i.add(l);}let a=await t.content();for(let o of D(e,r,a,s,"page-html"))i.add(o);return [...i]}async function Ee(e,t,r,s,n){let i=se(e,t);await V(i),await P.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}async function Xt(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Yt(e,t,r,s,n,i){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),u=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Xt(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let c=new URL(l),p=Vt(e,c.pathname);if(!j(c)||y(e,c)||H(c)){if(y(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),k(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let d=await xt(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let m=i.manifest.pages[l];if(d.action==="reuse"&&m){for(let f of m.discoveredAssets)k(e,t,f,r,l);for(let f of m.discoveredPages)_(e,t,f,r,l);m.lastSeenRunId=i.runId,m.sitemapLastmod=q(t.sitemapLastmodByPage[l])??m.sitemapLastmod,d.changeToken?.supported&&(m.changeToken=d.changeToken.token,m.tokenSource=d.changeToken.tokenSource??m.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:d.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}let g=await(p?u.newPage():o.newPage()),h=!1;await g.route("**/*",async m=>{let f=m.request();if(f.isNavigationRequest()&&f.frame()===g.mainFrame())try{let v=new URL(f.url()),b=new URL(l);if(v.hash="",b.hash="",v.href===b.href){if(h){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:f.url()}),await m.abort("aborted");return}h=!0;}}catch{}await m.continue();}),g.on("response",m=>{try{let f=S(e,m.url(),e.sourceOrigin,void 0,"network-response");f&&y(e,f)&&k(e,t,f.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let m=null,f=null;try{if(m=await g.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{f=await m.text();}catch{f=null;}}catch(I){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(I)});}if(m&&!m.ok()){r.skip("page",l,m.status()),await g.close();continue}p?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Gt(e,g)):await zt(e,g),t.stats.pagesRendered++,(t.stats.pagesRendered%5===0||t.stats.pagesRendered===1||t.stats.pagesRendered===t.stats.pagesQueued)&&r.progress(`Render progress: rendered ${t.stats.pagesRendered}, saved ${t.stats.pagesSaved}, discovered ${t.stats.pagesQueued}.`,{pagesRendered:t.stats.pagesRendered,pagesSaved:t.stats.pagesSaved,pagesQueued:t.stats.pagesQueued,phase:"render-pages"});let v=await Kt(e,g,l,r,!p);for(let I of v)k(e,t,I,r,l);let b=f??await g.content(),ae=s!=="single-url"?_t(e,l,b,r):[];if(s!=="single-url")for(let I of ae)_(e,t,I,r,l);await Ee(e,l,b,t,r),i.enabled&&(i.manifest.pages[l]={url:l,outputPath:se(e,l),changeToken:d.changeToken?.supported?d.changeToken.token:null,tokenSource:d.changeToken?.supported?d.changeToken.tokenSource??null:null,sitemapLastmod:q(t.sitemapLastmodByPage[l])??null,discoveredPages:ae,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let f=await g.content();f&&f.trim()&&(await Ee(e,l,f,t,r),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(m)}));}catch(f){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(f)});}r.error(`Failed page ${l}`,{url:l,error:String(m)});}finally{await g.close();}}}finally{await u.close(),await o.close(),await a.close();}}async function Me(e){try{let r=(await P.readFile(e,"utf8")).trim();if(!r)return [];try{let s=JSON.parse(r);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return r.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let n=JSON.parse(s);return n&&typeof n=="object"?n:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function Te(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Zt(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function er(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function tr(e){try{let t=await P.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}async function Ae(e){let t=await Me(w.join(e,"errors.jsonl"));return t.length>0?t:Me(w.join(e,"errors.json"))}async function rr(e){let t=w.join(e.logDir,"archive"),r;try{r=await P.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=w.join(t,i);if(!(await P.stat(a).catch(()=>null))?.isDirectory())continue;let u=await tr(w.join(a,"job.json"));if(!Zt(u))continue;let l=er(u);l>=n&&(n=l,s=a);}return s}async function sr(e){let t=await rr(e),r=t!==""?await Ae(t):await Ae(e.logDir),s=new Set;for(let n of r)if(Te(n.error)||Te(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await P.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function ir(e){try{let t=await P.readFile(w.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ar(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await P.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await P.unlink(w.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ie(e,t,r,s,n){let i=S(e,r,e.sourceOrigin,s,"manual-url");i&&(J(i)?ie(e,t,i.toString(),s,n):y(e,i)?k(e,t,i.toString(),s,n):j(i)?_(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function or(){let e=await G(),t=wt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await sr(e)),t.mode==="single-url"&&(r=await nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s?await vt(e):{schemaVersion:1,updatedAt:"",pages:{}},i={enabled:s,manifest:Se(n),previousManifest:Se(n),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await ar(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:t.preserveOutput?await ir(e):{},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}};t.preserveOutput||await P.rm(e.outputDir,{recursive:true,force:true}),await P.mkdir(e.outputDir,{recursive:true});let o=new $(e.logDir,e.logLevel);if(t.resumeRewrite)o.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let d=await chromium.launch({headless:true}),g=await d.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),h=g.request;try{if(o.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(f=>ie(e,a,f,o)),await Le(e,h,a,o),e.seedPaths.forEach(f=>Ie(e,a,f,o,"seed-path"))):(r.forEach(f=>Ie(e,a,f,o,"cli")),await Le(e,h,a,o)),o.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),o.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){o.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await o.flush();return}o.mark("render-pages");let m=Array.from({length:Math.max(1,e.concurrency)},()=>Yt(e,a,o,t.mode,h,i));await Promise.all(m),o.endMark("render-pages",{pages:a.donePages.size}),o.mark("download-assets"),await Jt(e,h,a,o),o.endMark("download-assets",{assets:a.doneAssets.size}),i.enabled&&e.maxPages===0&&await Ct(e,a,i,o),t.mode==="full"&&await Wt(e,o);}finally{await g.close(),await d.close();}}i.enabled&&!t.resumeRewrite&&(i.manifest.updatedAt=new Date().toISOString(),await bt(e,i.manifest)),await P.writeFile(w.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8");let u=t.mode==="single-url"?Et(e,a):void 0;o.mark("rewrite-text");let l=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${u?.length??0}`:"Rewriting text files...";o.progress(l,{phase:"rewrite-text",index:0,totalFiles:u?.length,changedTextFiles:0}),await xe(l);let c=Date.now(),p=await fe(e,a.assetMap,{files:u,onProgress:async({index:d,totalFiles:g,changedTextFiles:h,file:m})=>{o.checkpoint(`Rewriting text file ${d}/${g}`,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m});let f=Date.now();if(!(d===1||d===g||f-c>=5e3))return;c=f;let b=`Rewriting text files: ${d}/${g}`;o.progress(b,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m}),await xe(b);}});o.endMark("rewrite-text",{changedTextFiles:p}),o.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${p} text files.`:`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:p}),await o.flush();}or().catch(async e=>{console.error(e);try{let r=(await G().catch(()=>null))?.logDir??"logs",s=new $(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import
|
|
2
|
-
`,"utf8");}catch{}}async function p(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await f(Z(e.runtimeDir),n);}catch{}}async function I(e){try{await g.unlink(B(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let s=(e.id||`${r}-${t+1}`).trim();if(!s)return null;let c=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:s,enabled:c,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function se(e){let t=await h(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((s,c)=>oe(s,c)).filter(s=>!!s):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t,r){return e.some(n=>n.command===t&&(n.url||"").trim()===(r||"").trim()&&(n.status===void 0||n.status==="queued"||n.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let a=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${a}`),0}let n=d.join(e.runtimeDir,"queue.json"),s=d.join(e.runtimeDir,"current-run.json"),c=Date.now(),i=0,u=[];await R(e.runtimeDir,async()=>{let o=await h(n,[]),a=await h(s,null),w=await h(T(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];a&&a.status==="running"&&m.unshift(a);for(let l of r.rules){if(!l.enabled)continue;let P=l.intervalMinutes*60*1e3,S=Math.floor(c/P),v=w.lastEnqueuedBucketByRuleId[l.id]??-1;if(S<=v||ue(m,l.command,l.url))continue;let q={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(q),m.push(q),u.push({job:q,rule:l}),w.lastEnqueuedBucketByRuleId[l.id]=S,i+=1;}i>0&&(await f(n,o),await f(T(e.runtimeDir),w));});for(let o of u)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await g.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await g.unlink(e);}catch{}}function le(e,t){let r=[d.join(e,"dist",`${t}.js`),d.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${d.join(e,"dist")} or ${e}`)}async function y(e,t,r,n,s,c,i){if(i){let a=await j(t,i);if(a)throw new J(a,r)}let u=le(e,r),o={...process.env,...c??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,s&&(o.PUBLISHER_CONFIG=s),await new Promise((a,w)=>{let m=null,l=null,P=null,S=false,v=spawn(process.execPath,[u,...n],{cwd:e,env:o,stdio:"inherit"}),q=()=>{l&&clearInterval(l),P&&clearTimeout(P);},U=async()=>{if(!(!i||m||S)){S=true;try{let b=await j(t,i);if(!b)return;m=b,v.kill("SIGTERM"),P=setTimeout(()=>{v.kill("SIGKILL");},1e4);}finally{S=false;}}};i&&(l=setInterval(()=>{U();},1e3)),v.on("error",b=>{q(),w(b);}),v.on("close",(b,x)=>{if(q(),m){w(new J(m,r));return}b===0?a():w(new Error(`${r} exited with code ${b??-1}${x?` (signal ${x})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function ge(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let s=(await h(t.configPath||"",null))?.wpsuite;return s&&(s.accountId||s.siteId||s.siteKey)?s:null}async function pe(e,t){return await ge(e,t)}function F(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,t}async function fe(e){let t=d.join(e.runtimeDir,"queue.json"),r=d.join(e.runtimeDir,"current-run.json"),n=await R(e.runtimeDir,async()=>{let s=await h(r,null);if(!s||s.status!=="running"&&s.status!=="queued")return null;let c=await h(t,[]),i=Array.isArray(c)?c.filter(o=>o?.id!==s.id):[],u=F(s);return await f(t,[u,...i]),await f(r,null),u});n&&(await E(e.runtimeDir,n.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function he(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let u=await pe(e,t);await r.assertActiveSubscriptionForIdentity(u);}let s=me(e),c=e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"crawl"}),await y(t.exporterDir,t.runtimeDir,"crawl",c,t.configPath,s,e.id),await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await y(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,s,e.id),await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await y(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="crawl")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"crawl"}),await y(t.exporterDir,t.runtimeDir,"crawl",c,t.configPath,s,e.id),{exitCode:0};if(e.command==="deploy")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await y(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="invalidate")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await y(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await y(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,s,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await p(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await y(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,s,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof J?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function L(e){return async function(r=process.argv.slice(2)){let n=z(r),s=d.join(n.runtimeDir,"export.lock"),c=d.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await p(n,"starting",{message:"queue-runner starting"});try{await ce(s);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await p(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await fe(n);let i=await ae(n,e),u=0;for(let o=0;o<n.maxJobs;o++){let a=await ye(n,e);if(a==="none"||(u+=1,a==="stopped"))break}if(u===0)await I(n.runtimeDir),await p(n,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await h(c,null),a=String(o?.status||"").trim();await p(n,a==="success"?"job-success":a==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...a==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await p(n,"error",{message:u}),await A(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await de(s);}}}async function we(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json");await R(e.runtimeDir,async()=>{let s=await h(r,[]),c=Array.isArray(s)?s.filter(i=>i?.id!==t.id):[];await f(r,[F(t),...c]),await f(n,null);});}async function ye(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json"),s=d.join(e.runtimeDir,"last-run.json"),c=new Date().toISOString(),i=await R(e.runtimeDir,async()=>{let m=await h(r,[]);if(!Array.isArray(m)||m.length===0)return null;let l={...m[0],status:"running",startedAt:c};return await f(r,m.slice(1)),await f(n,l),l});if(!i)return "none";await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:c,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await p(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await he(i,e,t),o=new Date().toISOString();if(u.stopped){let m={...i,status:"stopped",endedAt:o,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:u.stopRequest?.mode||"requeue",stoppedStep:u.stoppedStep||i.command};await we(e,i);try{let l=await k(m,e);m.logArchiveDir=l.archiveDir,m.logArchiveCreatedAt=l.archiveCreatedAt,m.logArchiveFileCount=l.archivedFiles.length;}catch(l){m.logArchiveError=l instanceof Error?l.message:String(l);}return await f(s,m),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job stop requested and requeued.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:u.stopRequest?.mode||"requeue",stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:u.stoppedStep||i.command,logArchiveDir:m.logArchiveDir||"",logArchiveCreatedAt:m.logArchiveCreatedAt||"",logArchiveFileCount:m.logArchiveFileCount??0,logArchiveError:m.logArchiveError||""}}),"stopped"}let a={...i,status:u.exitCode===0?"success":"failed",endedAt:o,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await f(n,null),await E(e.runtimeDir,i.id);try{let m=await k(a,e);a.logArchiveDir=m.archiveDir,a.logArchiveCreatedAt=m.archiveCreatedAt,a.logArchiveFileCount=m.archivedFiles.length;}catch(m){a.logArchiveError=m instanceof Error?m.message:String(m);}await f(s,a);let w=i.startedAt&&a.endedAt?Math.max(0,Math.round((new Date(a.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:a.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:a.id,command:a.command,message:a.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:a.endedAt||"",durationSec:w,deploymentProfile:a.deploymentProfile||"",exitCode:a.exitCode??null,error:a.error||"",logArchiveDir:a.logArchiveDir||"",logArchiveCreatedAt:a.logArchiveCreatedAt||"",logArchiveFileCount:a.logArchiveFileCount??0,logArchiveError:a.logArchiveError||""}}),"processed"}function ve(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function qe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let c=`${ve(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,u=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(c,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let a=await o.json().catch(()=>null);if(!a||!a.config||!a.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(u);}}var Ae={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:qe},N=L(Ae),Pe=N,je=N;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Pe().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
-
export{
|
|
1
|
+
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import p from'fs/promises';import {existsSync}from'fs';import l from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';var R=class extends Error{request;step;constructor(t,r){super(`Stop requested during ${r}.`),this.name="StopRequestedError",this.request=t,this.step=r;}},K="queue-mutation.lock",H=5e3,z=3e4;function G(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",r=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),n=process.env.PUBLISHER_CONFIG||"",u=1;for(let s=0;s<e.length;s++){let o=e[s];o==="--runtime-dir"?t=e[++s]||"":o.startsWith("--runtime-dir=")?t=o.slice(14):o==="--exporter-dir"?r=e[++s]||r:o.startsWith("--exporter-dir=")?r=o.slice(15):o==="--config"?n=e[++s]||"":o.startsWith("--config=")?n=o.slice(9):o==="--max-jobs"?u=Number.parseInt(e[++s]||"1",10):o.startsWith("--max-jobs=")&&(u=Number.parseInt(o.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let a=l.resolve(t),i=n?l.resolve(n):l.join(a,"config.json");return {runtimeDir:a,exporterDir:l.resolve(r),configPath:i,maxJobs:Number.isFinite(u)&&u>0?u:1}}async function h(e,t){try{let r=await p.readFile(e,"utf8");return JSON.parse(r)}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="ENOENT"){let u=r instanceof Error?r.message:String(r);console.warn(`[queue-runner] failed to read JSON ${e}: ${u}`);}return t}}async function w(e,t){await p.mkdir(l.dirname(e),{recursive:true}),await p.writeFile(e,JSON.stringify(t,null,2),"utf8");}function F(e){return l.join(e,K)}async function X(e){await new Promise(t=>setTimeout(t,e));}async function Y(e){let t=F(e),r=Date.now()+H;for(;;)try{await p.mkdir(l.dirname(t),{recursive:!0}),await p.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="EEXIST")throw n;let a=await p.stat(t).catch(()=>null);if(a&&Date.now()-a.mtimeMs>z){await p.unlink(t).catch(()=>{});continue}if(Date.now()>=r)throw new Error("Timed out acquiring queue mutation lock.",{cause:n});await X(50);}}async function Z(e){await p.unlink(F(e)).catch(()=>{});}async function J(e,t){await Y(e);try{return await t()}finally{await Z(e);}}function V(e){return l.join(e,"queue-runner-heartbeat.json")}function x(e){return l.join(e,"current-progress.json")}function k(e){return l.join(e,"scheduler-state.json")}function ee(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let u=n.split("/").map(a=>a.trim()).filter(a=>a.length>0&&a!=="."&&a!=="..");return u.length>0?u.join("/"):t}async function B(e){let t=await h(e.configPath||"",null),r=l.resolve(e.runtimeDir,".."),u=(typeof t?.logDir=="string"?t.logDir:"").trim();return u&&l.isAbsolute(u)?l.resolve(u):l.resolve(r,ee(u,"logs"))}function C(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function te(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function re(e){return [te(e.endedAt||e.startedAt),C(e.command||"job","job"),C(e.id||"job","job"),C(e.status||"finished","finished")].join("-")}function ne(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}async function Q(e,t){let r=await B(t),n=new Date().toISOString(),u=l.join(r,"archive",re(e)),a=[];await p.mkdir(u,{recursive:true});try{let s=await p.readdir(r,{withFileTypes:!0});for(let o of s)!o.isFile()||!ne(o.name)||(await p.copyFile(l.join(r,o.name),l.join(u,o.name)),a.push(o.name));}catch(s){if((s&&typeof s=="object"&&"code"in s?String(s.code||""):"")!=="ENOENT")throw s}let i=x(t.runtimeDir);return existsSync(i)&&(await p.copyFile(i,l.join(u,"current-progress.json")),a.push("current-progress.json")),await w(l.join(u,"job.json"),{archivedAt:n,archiveDir:u,logDir:r,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:a,job:e}),a.push("job.json"),{archiveDir:u,archiveCreatedAt:n,archivedFiles:a}}function ie(e){return l.join(e,"audit-events.jsonl")}function D(e){return l.join(e,"stop-request.json")}async function O(e){try{let t=await p.readFile(D(e),"utf8"),r=JSON.parse(t);return r&&typeof r=="object"?r:null}catch{return null}}async function j(e,t){let r=await O(e);if(!r)return null;let n=String(r.targetJobId||"").trim();return n&&n!==t?null:r}async function E(e,t){if(!t){await p.unlink(D(e)).catch(()=>{});return}let r=await O(e);if(!r)return;let n=String(r.targetJobId||"").trim();(!n||n===t)&&await p.unlink(D(e)).catch(()=>{});}async function P(e,t){try{let r={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await p.mkdir(e,{recursive:!0}),await p.appendFile(ie(e),`${JSON.stringify(r)}
|
|
2
|
+
`,"utf8");}catch{}}async function g(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await w(V(e.runtimeDir),n);}catch{}}async function I(e){try{await p.unlink(x(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let u=(e.id||`${r}-${t+1}`).trim();if(!u)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",s=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:u,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&s?{deploymentProfile:s}:{},...i?{url:i}:{}}}async function se(e){let t=await h(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((u,a)=>oe(u,a)).filter(u=>!!u):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t,r){return e.some(n=>n.command===t&&(n.url||"").trim()===(r||"").trim()&&(n.status===void 0||n.status==="queued"||n.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let c=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,s=[];await J(e.runtimeDir,async()=>{let o=await h(n,[]),c=await h(u,null),y=await h(k(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];c&&c.status==="running"&&m.unshift(c);for(let d of r.rules){if(!d.enabled)continue;let f=d.intervalMinutes*60*1e3,v=Math.floor(a/f),q=y.lastEnqueuedBucketByRuleId[d.id]??-1;if(v<=q||ue(m,d.command,d.url))continue;let A={id:randomUUID(),command:d.command,...(d.command==="publish"||d.command==="crawl")&&d.crawlMode?{crawlMode:d.crawlMode}:{},...(d.command==="publish"||d.command==="deploy"||d.command==="invalidate")&&d.deploymentProfile?{deploymentProfile:d.deploymentProfile}:{},...d.url?{url:d.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(A),m.push(A),s.push({job:A,rule:d}),y.lastEnqueuedBucketByRuleId[d.id]=v,i+=1;}i>0&&(await w(n,o),await w(k(e.runtimeDir),y));});for(let o of s)await P(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await p.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await p.unlink(e);}catch{}}function le(e,t){let r=[l.join(e,"dist",`${t}.js`),l.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${l.join(e,"dist")} or ${e}`)}async function S(e,t,r,n,u,a,i){if(i){let c=await j(t,i);if(c)throw new R(c,r)}let s=le(e,r),o={...process.env,...a??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,u&&(o.PUBLISHER_CONFIG=u),await new Promise((c,y)=>{let m=null,d=null,f=null,v=false,q=spawn(process.execPath,[s,...n],{cwd:e,env:o,stdio:"inherit"}),A=()=>{d&&clearInterval(d),f&&clearTimeout(f);},_=async()=>{if(!(!i||m||v)){v=true;try{let b=await j(t,i);if(!b)return;m=b,q.kill("SIGTERM"),f=setTimeout(()=>{q.kill("SIGKILL");},1e4);}finally{v=false;}}};i&&(d=setInterval(()=>{_();},1e3)),q.on("error",b=>{A(),y(b);}),q.on("close",(b,T)=>{if(A(),m){y(new R(m,r));return}b===0?c():y(new Error(`${r} exited with code ${b??-1}${T?` (signal ${T})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function pe(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let u=(await h(t.configPath||"",null))?.wpsuite;return u&&(u.accountId||u.siteId||u.siteKey)?u:null}async function ge(e,t){return await pe(e,t)}function L(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function fe(e,t){let r=await h(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let u=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(u)return u;let a=await B(e).catch(()=>"");if(a){let i=await h(l.join(a,"current-crawl-event.json"),null),s=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(s)return s}return t}function we(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function he(e){let t=l.join(e.runtimeDir,"queue.json"),r=l.join(e.runtimeDir,"current-run.json"),n=await J(e.runtimeDir,async()=>{let u=await h(r,null);if(!u||u.status!=="running"&&u.status!=="queued")return null;let a=await h(t,[]),i=Array.isArray(a)?a.filter(o=>o?.id!==u.id):[],s=L(u);return await w(t,[s,...i]),await w(r,null),s});n&&(await E(e.runtimeDir,n.id),await P(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function ye(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let s=await ge(e,t);await r.assertActiveSubscriptionForIdentity(s);}let u=me(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="crawl")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),{exitCode:0};if(e.command==="deploy")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="invalidate")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,u,e.id),{exitCode:0};if(e.command==="url"){let s=(e.url||"").trim();return s?(await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--url",s],t.configPath,u,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof R?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function N(e){return async function(r=process.argv.slice(2)){let n=G(r),u=l.join(n.runtimeDir,"export.lock"),a=l.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await g(n,"starting",{message:"queue-runner starting"});try{await ce(u);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await g(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await he(n);let i=await ae(n,e),s=0;for(let o=0;o<n.maxJobs;o++){let c=await be(n,e);if(c==="none"||(s+=1,c==="stopped"))break}if(s===0)await I(n.runtimeDir),await g(n,"idle",{processedJobs:s,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await h(a,null),c=String(o?.status||"").trim();await g(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:s,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...c==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`}:{}});}}catch(i){let s=i instanceof Error?i.message:String(i);throw await g(n,"error",{message:s}),await P(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:s}}),i}finally{await de(u);}}}async function Se(e,t,r){let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json");await J(e.runtimeDir,async()=>{let a=await h(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],s=L(t),o=we(t,r);o?s.resumeFromStep=o:delete s.resumeFromStep,await w(n,[s,...i]),await w(u,null);});}async function be(e,t){let r=l.join(e.runtimeDir,"queue.json"),n=l.join(e.runtimeDir,"current-run.json"),u=l.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await J(e.runtimeDir,async()=>{let m=await h(r,[]);if(!Array.isArray(m)||m.length===0)return null;let d={...m[0],status:"running",startedAt:a};return await w(r,m.slice(1)),await w(n,d),d});if(!i)return "none";await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await g(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let s=await ye(i,e,t),o=new Date().toISOString();if(s.stopped){let m=await fe(e,s.stoppedStep||i.command),d={...i,status:"stopped",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{},stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopMode:s.stopRequest?.mode||"requeue",stoppedStep:m};await Se(e,i,m);try{let f=await Q(d,e);d.logArchiveDir=f.archiveDir,d.logArchiveCreatedAt=f.archiveCreatedAt,d.logArchiveFileCount=f.archivedFiles.length;}catch(f){d.logArchiveError=f instanceof Error?f.message:String(f);}return await w(u,d),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job stop requested and requeued.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:s.stopRequest?.mode||"requeue",stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveDir:d.logArchiveDir||"",logArchiveCreatedAt:d.logArchiveCreatedAt||"",logArchiveFileCount:d.logArchiveFileCount??0,logArchiveError:d.logArchiveError||""}}),"stopped"}let c={...i,status:s.exitCode===0?"success":"failed",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{}};await w(n,null),await E(e.runtimeDir,i.id);try{let m=await Q(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await w(u,c);let y=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await P(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:y,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Ae(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function Pe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Ae(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,s=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let c=await o.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(s);}}var Ie={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:Pe},U=N(Ie),Re=U,Me=U;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Re().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
+
export{Me as default,Re as main,U as runQueueRunner};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|