@smart-cloud/publisher-exporter 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -2
- package/dist/cli.js +4 -2
- package/dist/crawl.js +4 -4
- package/dist/prune-logs.js +14 -0
- package/dist/queue-runner.js +3 -3
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -59,6 +59,15 @@ publisher-exporter queue-runner \
|
|
|
59
59
|
|
|
60
60
|
Direct CLI invocation from cron is the recommended setup.
|
|
61
61
|
|
|
62
|
+
When the runtime config contains enabled scheduler rules and the active queue-runner policy allows scheduler auto-enqueue, queue-runner evaluates those rules once at startup before draining queued jobs.
|
|
63
|
+
|
|
64
|
+
- Scheduler rules are read from `runtime/config.json` and their last interval buckets are persisted in `runtime/scheduler-state.json`.
|
|
65
|
+
- Scheduler only auto-enqueues jobs into the runtime queue. It does not replace cron, systemd timers, or Windows Task Scheduler.
|
|
66
|
+
- A 1-minute external runner tick is the recommended cadence.
|
|
67
|
+
- Supported scheduled commands are `publish`, `crawl`, `deploy`, `invalidate`, `retry-timeouts`, and `url`.
|
|
68
|
+
- The scheduler timezone value is currently informational for operations context; interval matching is based on elapsed minute buckets checked at each queue-runner start.
|
|
69
|
+
- If an equivalent queued or running job already exists for the same command, crawl mode, deployment profile, and URL, that rule is skipped for the current interval bucket.
|
|
70
|
+
|
|
62
71
|
Same-host Linux cron example:
|
|
63
72
|
|
|
64
73
|
```cron
|
|
@@ -70,6 +79,7 @@ RUNTIME_PATH=/var/www/site/wp-content/uploads/smartcloud-static-publisher/runtim
|
|
|
70
79
|
LOG_PATH=/var/www/site/wp-content/uploads/smartcloud-static-publisher/logs
|
|
71
80
|
|
|
72
81
|
* * * * * /usr/bin/flock -n /tmp/static-publisher.cron.lock publisher-exporter queue-runner --runtime-dir "$RUNTIME_PATH" --max-jobs 1 >> "$LOG_PATH/queue-runner-cron.log" 2>&1
|
|
82
|
+
17 3 * * * publisher-exporter prune-logs --runtime-dir "$RUNTIME_PATH" --older-than-days 30 >> "$LOG_PATH/prune-logs-cron.log" 2>&1
|
|
73
83
|
```
|
|
74
84
|
|
|
75
85
|
If WordPress and the runner are on different machines but share the same mounted publisher storage, keep `outputDir` and `logDir` storage-relative in WordPress admin and point cron at the crawler host's local mount path:
|
|
@@ -89,9 +99,11 @@ If you want a stable user-local launcher that survives Node version changes unde
|
|
|
89
99
|
|
|
90
100
|
After each finished, failed, or stopped job, queue-runner copies the exporter-generated working logs into `"<logDir>/archive/<timestamp-command-jobId-status>/"` and includes a `job.json` summary plus the latest `current-progress.json` snapshot when available. The live root log files remain the current working set and may be overwritten by the next job.
|
|
91
101
|
|
|
92
|
-
|
|
102
|
+
Archived files are gzip-compressed per file and listed in `job.json`. WordPress Audit Log rows can download those archived artifacts through the authenticated REST API.
|
|
103
|
+
|
|
104
|
+
`retry-timeouts` uses the newest archived full `crawl` or `publish` job log snapshot as its timeout source. It prefers the manifest-backed archived `errors.*` artifact when available and falls back to older uncompressed archive layouts.
|
|
93
105
|
|
|
94
|
-
|
|
106
|
+
Prune old archive directories with `publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30`. Add that command to daily cron or another retention scheduler that matches your log policy.
|
|
95
107
|
|
|
96
108
|
## WordPress Integration
|
|
97
109
|
|
package/dist/cli.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {spawn}from'child_process';import {existsSync}from'fs';import o from'path';import {fileURLToPath}from'url';var
|
|
2
|
+
import {spawn}from'child_process';import {existsSync}from'fs';import o from'path';import {fileURLToPath}from'url';var c=o.dirname(fileURLToPath(import.meta.url)),i=o.resolve(c,"..");function w(){console.log(`publisher-exporter <command> [args]
|
|
3
3
|
|
|
4
4
|
Commands:
|
|
5
5
|
crawl Run the crawl phase
|
|
@@ -7,6 +7,7 @@ Commands:
|
|
|
7
7
|
invalidate Request CloudFront invalidation
|
|
8
8
|
publish Run crawl, deploy, then invalidate
|
|
9
9
|
queue-runner Process queued jobs from runtime JSON
|
|
10
|
+
prune-logs Delete archived log directories older than a retention window
|
|
10
11
|
install-browsers Install Playwright Chromium into the current/shared browsers path
|
|
11
12
|
help Show this help
|
|
12
13
|
|
|
@@ -14,5 +15,6 @@ Examples:
|
|
|
14
15
|
publisher-exporter crawl --crawl-mode incremental
|
|
15
16
|
publisher-exporter deploy --profile prod
|
|
16
17
|
publisher-exporter queue-runner --runtime-dir /srv/site/runtime --max-jobs 1
|
|
18
|
+
publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30
|
|
17
19
|
publisher-exporter install-browsers
|
|
18
|
-
`);}function p(r){let e=o.join(
|
|
20
|
+
`);}function p(r){let e=o.join(c,`${r}.js`);if(!existsSync(e))throw new Error(`Missing CLI entrypoint: ${e}`);return e}function f(){let r=[o.join(i,"node_modules","playwright","cli.js"),o.join(i,"node_modules","playwright-core","cli.js")];for(let e of r)if(existsSync(e))return e;throw new Error("Playwright CLI was not found under this package. Run `npm install` first.")}function t(r,e){return new Promise((n,s)=>{let a=spawn(process.execPath,[r,...e],{cwd:i,env:process.env,stdio:"inherit"});a.on("error",s),a.on("close",(d,l)=>{if(l){s(new Error(`Command terminated by signal ${l}.`));return}n(d??1);});})}async function g(r){for(let e of ["crawl","deploy","invalidate"]){let n=await t(p(e),r);if(n!==0)return n}return 0}async function y(){let[r="help",...e]=process.argv.slice(2);if(["help","--help","-h"].includes(r)){w();return}if(r==="publish"){process.exitCode=await g(e);return}if(r==="install-browsers"){let n=e.length>0?e:["install","chromium"];process.exitCode=await t(f(),n);return}if(r==="crawl"||r==="deploy"||r==="invalidate"||r==="queue-runner"||r==="prune-logs"){process.exitCode=await t(p(r),e);return}throw new Error(`Unknown command: ${r}`)}y().catch(r=>{console.error(r instanceof Error?r.message:String(r)),process.exit(1);});
|
package/dist/crawl.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {chromium}from'playwright';import P from'fs/promises';import
|
|
1
|
+
import {chromium}from'playwright';import P from'fs/promises';import f from'path';import {gunzipSync}from'zlib';import Be from'crypto';import Ge from'fast-glob';import {availableParallelism}from'os';import {Worker}from'worker_threads';function ue(e){let t=e.trim();return t==="."?".":t.replace(/\/$/,"")}function B(e){return !e||typeof e!="object"?{}:Object.fromEntries(Object.entries(e).map(([t,r])=>[t.trim(),String(r??"")]).filter(([t])=>t.length>0))}function He(e){if(!e||typeof e!="object")return {};let t={};for(let[r,s]of Object.entries(e)){let n=r.trim();if(!n||!s||typeof s!="object")continue;let i=s,o={};if(typeof i.targetOrigin=="string"){let u=ue(i.targetOrigin);u&&(o.targetOrigin=u);}let a=B(i.extraReplacements);if(Object.keys(a).length>0&&(o.extraReplacements=a),i.s3&&typeof i.s3=="object"){let u={},l=i.s3;typeof l.bucket=="string"&&(u.bucket=l.bucket.trim()),typeof l.prefix=="string"&&(u.prefix=l.prefix.trim()),typeof l.region=="string"&&(u.region=l.region.trim()),typeof l.htmlCacheControl=="string"&&(u.htmlCacheControl=l.htmlCacheControl.trim()),typeof l.assetCacheControl=="string"&&(u.assetCacheControl=l.assetCacheControl.trim()),Object.keys(u).length>0&&(o.s3=u);}if(i.cloudFront&&typeof i.cloudFront=="object"){let u={},l=i.cloudFront;typeof l.distributionId=="string"&&(u.distributionId=l.distributionId.trim()),Array.isArray(l.invalidationPaths)&&(u.invalidationPaths=l.invalidationPaths.map(c=>String(c??"").trim()).filter(c=>c.length>0)),Object.keys(u).length>0&&(o.cloudFront=u);}t[n]=o;}return t}function z(e,t){let s=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!s)return t;let n=s.split("/").map(i=>i.trim()).filter(i=>i.length>0&&i!=="."&&i!=="..");return n.length>0?n.join("/"):t}function Je(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function ae(e,t,r){let s=String(t||"").trim();return s&&f.isAbsolute(s)?f.resolve(s):f.resolve(e,z(s,r))}async function G(){let e=process.env.PUBLISHER_CONFIG||"publisher.config.json",t=await P.readFile(e,"utf8"),r=JSON.parse(t);r.sourceOrigin=r.sourceOrigin.replace(/\/$/,""),r.targetOrigin=ue(r.targetOrigin),r.ignoreHttpsErrors??=false,r.outputDir=String(r.outputDir||"export").trim()||"export",r.urlRewriteMode||=r.targetOrigin==="."?"relative":"absolute",r.noJavaScriptRenderPathPrefixes||=[],r.seedPaths||=[],r.sitemapPaths||=["/sitemap_index.xml","/sitemap.xml"],r.allowedAssetHosts||=[],r.assetPathPrefixes||=["/wp-content/","/wp-includes/","/static/","/assets/","/build/","/_next/","/docs/","/sitemap","/robots.txt","/llms.txt"],r.blockedPathPrefixes||=[],r.blockedSearchFragments||=[],r.concurrency||=1,r.maxPages||=0,r.extraReplacements=B(r.extraReplacements),r.postCrawlCopyMap=B(r.postCrawlCopyMap),r.logDir=String(r.logDir||"logs").trim()||"logs",r.verbose??=false,r.logLevel||=r.verbose?"debug":"info",r.s3SyncMode||="sdk-upload-delete",r.readiness||={waitForSelector:null,waitForFunction:null,timeoutMs:1500,fallbackWaitMs:1500},r.readiness.timeoutMs??=1500,r.readiness.fallbackWaitMs??=1500,r.viewport||={width:1440,height:1200},r.navigationTimeoutMs||=3e4,r.scheduler||={enabled:false,timezone:"UTC",rules:[]},r.scheduler.enabled??=false,r.scheduler.timezone||="UTC",r.scheduler.rules||=[],r.deploymentProfiles=He(r.deploymentProfiles),r.assetDownloadConcurrency=Number(r.assetDownloadConcurrency)>0?Number(r.assetDownloadConcurrency):r.concurrency,r.rewriteConcurrency=Number(r.rewriteConcurrency)>0?Number(r.rewriteConcurrency):r.assetDownloadConcurrency,r.defaultDeploymentProfile=String(r.defaultDeploymentProfile??"").trim(),r.defaultDeploymentProfile&&!r.deploymentProfiles[r.defaultDeploymentProfile]&&(r.defaultDeploymentProfile="");let s=Je(),n=s?f.resolve(s,".."):"";return n?(r.outputDir=ae(n,r.outputDir,"export"),r.logDir=ae(n,r.logDir,"logs")):(f.isAbsolute(r.outputDir)||(r.outputDir=z(r.outputDir,"export")),f.isAbsolute(r.logDir)||(r.logDir=z(r.logDir,"logs"))),r}function ce(e,t=10){return Be.createHash("sha1").update(e).digest("hex").slice(0,t)}async function V(e){await P.mkdir(f.dirname(e),{recursive:true});}function O(e){return [".html",".htm",".css",".js",".mjs",".json",".xml",".xsl",".txt",".svg",".map",".enc",".jws"].includes(f.extname(e).toLowerCase())}function L(e){return e.replace(/"/g,'"').replace(/"/g,'"').replace(/'/g,"'").replace(/'/g,"'").replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function E(e){return e.replace(/&/g,"&").replace(/"/g,""").replace(/'/g,"'").replace(/</g,"<").replace(/>/g,">")}function F(e){return e.replace(/\//g,"\\/")}function le(e){return e.replace(/\//g,"\\\\/")}function de(e,t,r){if(!t)return;e[t]=r;let s=F(t),n=F(r);e[s]=n;let i=le(t),o=le(r);e[i]=o;let a=E(t),u=E(r);e[a]=u;let l=E(s),c=E(n);e[l]=c;let p=E(i),d=E(o);e[p]=d;}function K(e,t){let r=t.startsWith("/")?t:`/${t}`,s=(e||"").replace(/\/$/,"");return !s||s==="."||s==="/"?r:`${s}${r}`}function pe(e,t){try{return new URL(e,t==="."?"https://relative.invalid":t).pathname}catch{return e.startsWith("/")?e:`/${e.replace(/^\.\//,"")}`}}function ge(e,t,r){if(!t)return r;let s=f.dirname(f.resolve(t)),n=f.resolve(e,r.replace(/^\/+/,"")),i=f.relative(s,n).replace(/\\/g,"/");return i?(i.startsWith(".")||(i=`./${i}`),i):"."}var Xe=["wp-content/","wp-includes/","wp-admin/","wp-json/","_next/"],Ye=new Set([".html",".htm"]),he="WPSuite.io Static Publisher",Ze=he.replace(/[.*+?^${}()|[\]\\]/g,"\\$&");function et(e,t,r){if(e.wpsuite?.subscriber===true||!r)return false;let s=f.extname(r).toLowerCase();return Ye.has(s)?/<head\b|<html\b|<!doctype html/i.test(t):false}function tt(e){if(new RegExp(`<meta\\b(?=[^>]*\\bname=(["'])generator\\1)(?=[^>]*\\bcontent=(["'])${Ze}\\2)[^>]*\\/?>`,"i").test(e))return e;let t=`<meta name="generator" content="${he}" />`;if(e.match(/(\r?\n)([ \t]*)<\/head>/i))return e.replace(/(\r?\n)([ \t]*)<\/head>/i,`$1$2${t}$1$2</head>`);let s=e.includes(`\r
|
|
2
2
|
`)?`\r
|
|
3
3
|
`:`
|
|
4
|
-
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function
|
|
5
|
-
`,"utf8");}function
|
|
6
|
-
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>lt(t,r));}updateCurrentEvent(t,r,s){let n=dt(t,r,s);this.enqueueTask(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(A[this.level]>=A.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){ut("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),A[this.level]>=A.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),A[this.level]>=A.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var pt=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],gt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),mt=new Set(["",".html",".htm"]),ht=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),ft=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function wt(e){let t=[],r="full",s="full",n=false,i=false;for(let a=0;a<e.length;a++){let o=e[a];if(o==="--retry-timeouts")r="retry-timeouts",n=true;else if(o==="--resume")n=true;else if(o==="--resume-rewrite")n=true,i=true;else if(o==="--crawl-mode"){let u=e[++a];if(!u)throw new Error("--crawl-mode requires a value");s=u==="incremental"?"incremental":"full",n||=s==="incremental";}else if(o.startsWith("--crawl-mode="))s=o.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(o==="--incremental")s="incremental",n=true;else if(o==="--url"){let u=e[++a];if(!u)throw new Error("--url requires a value");r="single-url",n=true,t.push(u);}else if(o.startsWith("--url="))r="single-url",n=true,t.push(o.slice(6));else if(o==="--urls"){let u=e[++a];if(!u)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${u}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function te(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.resolve(e):""}function Pt(){let e=te();return e?w.resolve(e,".."):""}function yt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?w.resolve(e):""}function St(e){let t=e.trim(),r=[{alias:"@storage-root",root:Pt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:te(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:yt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?w.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:w.resolve(t)}}function De(e){let t=te();return t?w.join(t,"crawl-manifest.json"):w.join(e.outputDir,".crawl-manifest.json")}async function vt(e){try{let t=await P.readFile(De(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.schemaVersion===1&&r.pages&&typeof r.pages=="object")return r}catch{}return {schemaVersion:1,updatedAt:"",pages:{}}}function Se(e){return JSON.parse(JSON.stringify(e))}async function bt(e,t){let r=De(e);await P.mkdir(w.dirname(r),{recursive:true}),await P.writeFile(r,JSON.stringify(t,null,2),"utf8");}function q(e){if(!e)return null;let t=e.trim();return t||null}async function Rt(e,t,r,s){let n=[...new Set(r.map(o=>o.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let a=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let o=await t.post(a,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!o.ok())return s.warn(`Change token lookup failed with HTTP ${o.status()}`,{endpoint:a,status:o.status()}),new Map;let u=await o.json().catch(()=>null);if(!u||!Array.isArray(u.items))return new Map;let l=new Map;for(let c of u.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(o){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:a,error:String(o)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let a=(await Rt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,a),a}async function xt(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let a=s.manifest.pages[n];if(!a)return {action:"render",changeToken:await ve(e,t,s,n,i)};let o=await ve(e,t,s,n,i);if(o?.supported&&o.token)return a.changeToken===o.token?{action:"reuse",changeToken:o}:{action:"render",changeToken:o};let u=q(r.sitemapLastmodByPage[n]);return u&&a.sitemapLastmod&&a.sitemapLastmod===u?{action:"reuse",changeToken:o}:{action:"render",changeToken:o}}async function Ct(e,t,r,s){let n=be(r.previousManifest),i=0;for(let[p,d]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){try{await P.unlink(d.outputPath);}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:d.outputPath,error:String(g)});}delete r.manifest.pages[p],i++;}let a=be(r.manifest),o=new Set,u=new Set;for(let p of a){for(let g of Re(e,p))o.add(g);let d=N(e,t.assetMap,p);d&&u.add(d);}let l=0,c=0;for(let p of n){if(a.has(p))continue;let d=new Set,g=N(e,t.assetMap,p);if(g)d.add(g);else {let h=ne(e,p);d.add(h.originalFilePath),h.hashedFilePath&&d.add(h.hashedFilePath);}for(let h of d)if(!u.has(h))try{await P.unlink(h),l++;}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:h,error:String(m)});}for(let h of Re(e,p))o.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],c++);}(i>0||l>0||c>0)&&s.info(`Incremental cleanup removed ${i} stale pages, ${l} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:l,removedAssetMappings:c,phase:"incremental-cleanup"});}function be(e){let t=new Set;for(let r of Object.values(e.pages))for(let s of r.discoveredAssets||[]){let n=s.trim();n&&t.add(n);}return t}function Re(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let a of i)try{r.add(new URL(a).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function kt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return w.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function N(e,t,r){let s=t[r];return s?kt(e,s):ne(e,r).originalFilePath}function Lt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?w.join(w.resolve(e),"queue-runner-heartbeat.json"):""}async function xe(e){let t=Lt();if(t)try{let r=await P.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await P.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function Et(e,t){let r=new Set;for(let s of t.donePages)r.add(w.resolve(se(e,s)));for(let s of t.doneAssets){let n=N(e,t.assetMap,s);!n||!O(n)||r.add(w.resolve(n));}return [...r]}var W="re:",Q=new Map;function Mt(e){if(!e.startsWith(W))return null;let t=Q.get(e);if(t!==void 0)return t||null;let r=e.slice(W.length).trim();if(!r)return Q.set(e,false),null;try{let s=new RegExp(r);return Q.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Q.set(e,false),null}}function re(e,t){let r=Mt(t);return r?r.test(e):t.startsWith(W)?false:e.startsWith(t)}function Tt(e,t){return t.find(r=>re(e,r))}function je(e,t){let r=Tt(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(W)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function At(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function It(e,t){return At(e).has(t.hostname)}function $t(e){let t=L(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Fe(e){let t=L(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function H(e){return ft.some(t=>e.pathname.startsWith(t))}function S(e,t,r=e.sourceOrigin,s,n="url"){let i=$t(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Fe(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let a=new URL(i,r);if(!["http:","https:"].includes(a.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!It(e,a))return s?.ignore(n,a.toString(),"host not allowed",r),null;let o=je(e,a);if(o)return s?.reject(n,a.toString(),o,r),null;let u=new URL(e.sourceOrigin);return a.protocol=u.protocol,a.host=u.host,a.hash="",a}catch{return s?.reject(n,t,"invalid URL",r),null}}function Oe(e){return gt.has(w.extname(e.pathname).toLowerCase())}function j(e){return H(e)?false:mt.has(w.extname(e.pathname).toLowerCase())}function J(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function y(e,t){return J(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Oe(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>re(t.pathname,r)):false}function se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":w.extname(s)||(s+="/index.html"),w.join(e.outputDir,s)}function ne(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,a=w.join(e.outputDir,i),o=K(e.targetOrigin,i),u=`${o}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:u,preferQueryHashed:n};let l=w.extname(s),p=`${l?s.slice(0,-l.length):s}.${ce(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:a,originalPublicUrl:o,originalPublicUrlWithSearch:u,preferQueryHashed:n,hashedPathname:p,hashedFilePath:w.join(e.outputDir,p),hashedPublicUrl:K(e.targetOrigin,p)}}function Dt(e,t,r,s,n){t[r]=s;try{let i=new URL(r),o=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{o.push(new URL(e.targetOrigin).origin);}catch{}for(let u of o)try{let l=new URL(u);t[l.origin+i.pathname+i.search]=s;}catch{}try{let u=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=u;}catch{}try{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}try{let u=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}}catch{}}async function Z(e){try{return await P.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Ue(e){try{return await P.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function jt(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function Qe(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Ft(e,t){let r=Qe(t),s=jt(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function Ot(e,t){if(!t)return;let r=Qe(t);if(r===null)return;let s=await Ue(e);if(!s)return;let n=new Date(r);await P.utimes(e,s.atime,n).catch(()=>{});}function Ce(e,t){let r=w.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return ht.has(r)||pt.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function ke(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ee(e,t,r,s,n,i,a){let o=ne(e,r),u=o.originalFilePath,l=o.originalPublicUrlWithSearch;if(o.preferQueryHashed&&o.hashedFilePath&&o.hashedPublicUrl){let p=await Z(o.originalFilePath);p!==null&&!p.equals(s)&&(u=o.hashedFilePath,l=o.hashedPublicUrl);}Dt(e,t,r,l,o.originalPublicUrl);let c=await Z(u);(c===null||!c.equals(s))&&(await V(u),await P.writeFile(u,s)),await Ot(u,a),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"});}function _(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"page");if(!a)return;if(!j(a)||Oe(a)){if(y(e,a)){s.info(`Seeded/non-page URL queued as asset: ${a.toString()}`,{url:a.toString(),source:n}),k(e,t,a.toString(),s,n);return}s.reject("page",a.toString(),H(a)?"asset/internal path cannot be page":"not page-like",n);return}let o=a.toString();!t.donePages.has(o)&&!t.queuedPages.has(o)&&(t.queuedPages.add(o),t.pageQueue.push(o),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"asset");if(!a)return;if(!y(e,a)){s.reject("asset",a.toString(),"not a safe asset path/prefix",n);return}let o=a.toString();!t.doneAssets.has(o)&&!t.queuedAssets.has(o)&&(t.queuedAssets.add(o),t.assetQueue.push(o),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ie(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,a=S(e,r,i,s,"sitemap");if(!a)return;if(!J(a)){s.reject("sitemap",a.toString(),"not sitemap-like",n);return}let o=a.toString();!t.doneSitemaps.has(o)&&!t.queuedSitemaps.has(o)&&(t.queuedSitemaps.add(o),t.sitemapQueue.push(o),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Ut(e){let t=L(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:q(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function Qt(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function qt(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Y(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Nt(e,t){await P.mkdir(t,{recursive:true});let r=await P.readdir(e,{withFileTypes:true});for(let s of r){let n=w.join(e,s.name),i=w.join(t,s.name);if(s.isDirectory()){await P.cp(n,i,{recursive:true,force:true});continue}await P.copyFile(n,i);}}async function Wt(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([n,i])=>({sourcePath:n.trim(),prefix:String(i||"").trim()}));if(r.length===0)return;let s=0;t.mark("copy-extra-paths");for(let n of r){let i=n.sourcePath,a=n.prefix;if(!i||!a){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:i,prefixPath:a});continue}let o=St(i);if(!o.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:i,alias:o.alias||"",requiredEnv:o.requiredEnv||"",prefixPath:a});continue}let u=o.resolvedPath,l=Y(a);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:i,prefixPath:a});continue}let c;try{c=await P.stat(u);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:i,sourceAbs:u,prefixPath:a});continue}let p=w.resolve(e.outputDir,l);if(c.isDirectory()){await Nt(u,p),s++,t.progress(`Copied static directory to export: ${u} -> /${l}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${l}`,copiedItems:s});continue}let d=w.basename(u),g=a.endsWith("/")?w.join(p,d):p;await P.mkdir(w.dirname(g),{recursive:true}),await P.copyFile(u,g),s++,t.progress(`Copied static file to export: ${u} -> /${Y(a.endsWith("/")?`${a}${d}`:a)}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${Y(a.endsWith("/")?`${a}${d}`:a)}`,copiedItems:s});}t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s});}function D(e,t,r,s,n){let i=new Set,a=L(r).replace(/\\\//g,"/");for(let l of qt(a)){let c=S(e,l,t,s,`${n}:css-url`);c&&y(e,c)&&i.add(c.toString());}for(let l of a.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=S(e,l[1],t,s,`${n}:css-import`);c&&y(e,c)&&i.add(c.toString());}for(let l of a.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=S(e,l[1],t,s,`${n}:xml-stylesheet`);c&&y(e,c)&&i.add(c.toString());}let o="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",u=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${o}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of a.matchAll(u)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(Fe(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=S(e,c,t,s,`${n}:serialized-url`);p&&y(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function _t(e,t,r,s){let n=new Set,i=L(r);for(let a of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let o=a[1],u=S(e,o,t,s,"page-link");u&&(je(e,u)||j(u)&&!H(u)&&!y(e,u)&&n.add(u.toString()));}return [...n]}async function Le(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let a=await i.body(),o=i.url()||n;await ee(e,r.assetMap,o,a,r,s);let u=a.toString("utf8");for(let l of D(e,o,u,s,"sitemap"))k(e,r,l,s,o);for(let l of Ut(u)){let c=S(e,l.loc,o,s,"sitemap-loc");c&&(J(c)?ie(e,r,c.toString(),s,o):y(e,c)?k(e,r,c.toString(),s,o):j(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),_(e,r,c.toString(),s,o)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",o));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Ht(e,t,r,s,n){let i=S(e,s,e.sourceOrigin,n,"asset-fetch");if(!i||!y(e,i))return;let a=i.toString();if(!r.doneAssets.has(a)){n.asset(`Fetching asset ${a}`,{url:a});try{let o=N(e,r.assetMap,a),u=o?await Ue(o):null;if(o&&u)try{let g=await t.head(a,{timeout:6e4}),h=g.headers();if(g.ok()&&Ft(u,h)){let m=await Z(o);if(m!==null){let f=g.url()||a;if(await ee(e,r.assetMap,f,m,r,n,h),r.doneAssets.add(a),n.info(`Reused existing asset ${a} based on response headers.`,{url:a,filePath:o,lastModified:h["last-modified"]||"",contentLength:h["content-length"]||"",phase:"download-assets"}),ke(r,n),Ce(i,h)){let v=m.toString("utf8");for(let b of D(e,f,v,n,"asset:cached"))k(e,r,b,n,f);}return}}}catch{}let l=await t.get(a,{timeout:6e4});if(!l.ok()){n.skip("asset",a,l.status());return}let c=await l.body(),p=l.url()||a,d=l.headers();if(await ee(e,r.assetMap,p,c,r,n,d),r.doneAssets.add(a),ke(r,n),Ce(i,d)){let g=c.toString("utf8");for(let h of D(e,p,g,n,`asset:${w.extname(i.pathname).toLowerCase()||d["content-type"]||"unknown"}`))k(e,r,h,n,p);}}catch(o){n.error(`Failed asset ${a}`,{url:a,error:String(o)});}}}async function Jt(e,t,r,s){let n=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),i=Array.from({length:n},async()=>{for(;r.assetQueue.length>0;){let a=r.assetQueue.shift();a&&(r.queuedAssets.delete(a),await Ht(e,t,r,a,s));}});await Promise.all(i);}async function Bt(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function zt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Bt(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Gt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Vt(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>re(t,r))}async function Kt(e,t,r,s,n=true){let i=new Set;if(n){let o=await t.evaluate(()=>{let u=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let h of l){let m=d.getAttribute(h);m&&u.add(m);}for(let h of p){let m=d.getAttribute(h);m&&m.split(",").forEach(f=>{let v=f.trim().split(/\s+/)[0];v&&u.add(v);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...u],styles:[...c]}});for(let u of o.attrs)for(let l of Qt(u)){let c=S(e,l,r,s,"dom-attr");c&&y(e,c)&&i.add(c.toString());}for(let u of o.styles)for(let l of D(e,r,u,s,"dom-style"))i.add(l);}let a=await t.content();for(let o of D(e,r,a,s,"page-html"))i.add(o);return [...i]}async function Ee(e,t,r,s,n){let i=se(e,t);await V(i),await P.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}async function Xt(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function Yt(e,t,r,s,n,i){let a=await chromium.launch({headless:true}),o=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),u=await a.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await Xt(o);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let c=new URL(l),p=Vt(e,c.pathname);if(!j(c)||y(e,c)||H(c)){if(y(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),k(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let d=await xt(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let m=i.manifest.pages[l];if(d.action==="reuse"&&m){for(let f of m.discoveredAssets)k(e,t,f,r,l);for(let f of m.discoveredPages)_(e,t,f,r,l);m.lastSeenRunId=i.runId,m.sitemapLastmod=q(t.sitemapLastmodByPage[l])??m.sitemapLastmod,d.changeToken?.supported&&(m.changeToken=d.changeToken.token,m.tokenSource=d.changeToken.tokenSource??m.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:d.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"});continue}}let g=await(p?u.newPage():o.newPage()),h=!1;await g.route("**/*",async m=>{let f=m.request();if(f.isNavigationRequest()&&f.frame()===g.mainFrame())try{let v=new URL(f.url()),b=new URL(l);if(v.hash="",b.hash="",v.href===b.href){if(h){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:f.url()}),await m.abort("aborted");return}h=!0;}}catch{}await m.continue();}),g.on("response",m=>{try{let f=S(e,m.url(),e.sourceOrigin,void 0,"network-response");f&&y(e,f)&&k(e,t,f.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let m=null,f=null;try{if(m=await g.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{f=await m.text();}catch{f=null;}}catch(I){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(I)});}if(m&&!m.ok()){r.skip("page",l,m.status()),await g.close();continue}p?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Gt(e,g)):await zt(e,g),t.stats.pagesRendered++,(t.stats.pagesRendered%5===0||t.stats.pagesRendered===1||t.stats.pagesRendered===t.stats.pagesQueued)&&r.progress(`Render progress: rendered ${t.stats.pagesRendered}, saved ${t.stats.pagesSaved}, discovered ${t.stats.pagesQueued}.`,{pagesRendered:t.stats.pagesRendered,pagesSaved:t.stats.pagesSaved,pagesQueued:t.stats.pagesQueued,phase:"render-pages"});let v=await Kt(e,g,l,r,!p);for(let I of v)k(e,t,I,r,l);let b=f??await g.content(),ae=s!=="single-url"?_t(e,l,b,r):[];if(s!=="single-url")for(let I of ae)_(e,t,I,r,l);await Ee(e,l,b,t,r),i.enabled&&(i.manifest.pages[l]={url:l,outputPath:se(e,l),changeToken:d.changeToken?.supported?d.changeToken.token:null,tokenSource:d.changeToken?.supported?d.changeToken.tokenSource??null:null,sitemapLastmod:q(t.sitemapLastmodByPage[l])??null,discoveredPages:ae,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let f=await g.content();f&&f.trim()&&(await Ee(e,l,f,t,r),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(m)}));}catch(f){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(f)});}r.error(`Failed page ${l}`,{url:l,error:String(m)});}finally{await g.close();}}}finally{await u.close(),await o.close(),await a.close();}}async function Me(e){try{let r=(await P.readFile(e,"utf8")).trim();if(!r)return [];try{let s=JSON.parse(r);if(Array.isArray(s))return s;if(s&&typeof s=="object")return [s]}catch{return r.split(/\r?\n/).map(s=>s.trim()).filter(Boolean).map(s=>{try{let n=JSON.parse(s);return n&&typeof n=="object"?n:null}catch{return null}}).filter(s=>s!==null)}return []}catch{return []}}function Te(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function Zt(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function er(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function tr(e){try{let t=await P.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}async function Ae(e){let t=await Me(w.join(e,"errors.jsonl"));return t.length>0?t:Me(w.join(e,"errors.json"))}async function rr(e){let t=w.join(e.logDir,"archive"),r;try{r=await P.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let a=w.join(t,i);if(!(await P.stat(a).catch(()=>null))?.isDirectory())continue;let u=await tr(w.join(a,"job.json"));if(!Zt(u))continue;let l=er(u);l>=n&&(n=l,s=a);}return s}async function sr(e){let t=await rr(e),r=t!==""?await Ae(t):await Ae(e.logDir),s=new Set;for(let n of r)if(Te(n.error)||Te(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function nr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await P.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function ir(e){try{let t=await P.readFile(w.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function ar(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await P.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await P.unlink(w.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function Ie(e,t,r,s,n){let i=S(e,r,e.sourceOrigin,s,"manual-url");i&&(J(i)?ie(e,t,i.toString(),s,n):y(e,i)?k(e,t,i.toString(),s,n):j(i)?_(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function or(){let e=await G(),t=wt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await sr(e)),t.mode==="single-url"&&(r=await nr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s?await vt(e):{schemaVersion:1,updatedAt:"",pages:{}},i={enabled:s,manifest:Se(n),previousManifest:Se(n),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await ar(e.logDir);let a={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:t.preserveOutput?await ir(e):{},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}};t.preserveOutput||await P.rm(e.outputDir,{recursive:true,force:true}),await P.mkdir(e.outputDir,{recursive:true});let o=new $(e.logDir,e.logLevel);if(t.resumeRewrite)o.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let d=await chromium.launch({headless:true}),g=await d.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),h=g.request;try{if(o.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(f=>ie(e,a,f,o)),await Le(e,h,a,o),e.seedPaths.forEach(f=>Ie(e,a,f,o,"seed-path"))):(r.forEach(f=>Ie(e,a,f,o,"cli")),await Le(e,h,a,o)),o.endMark("discovery",{pages:a.pageQueue.length,assets:a.assetQueue.length,sitemaps:a.sitemapQueue.length}),o.summary(`Queued ${a.pageQueue.length} pages, ${a.assetQueue.length} assets, ${a.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:a.pageQueue.length,queuedAssets:a.assetQueue.length,queuedSitemaps:a.sitemapQueue.length}),t.mode==="retry-timeouts"&&a.pageQueue.length===0&&a.assetQueue.length===0&&a.sitemapQueue.length===0){o.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await o.flush();return}o.mark("render-pages");let m=Array.from({length:Math.max(1,e.concurrency)},()=>Yt(e,a,o,t.mode,h,i));await Promise.all(m),o.endMark("render-pages",{pages:a.donePages.size}),o.mark("download-assets"),await Jt(e,h,a,o),o.endMark("download-assets",{assets:a.doneAssets.size}),i.enabled&&e.maxPages===0&&await Ct(e,a,i,o),t.mode==="full"&&await Wt(e,o);}finally{await g.close(),await d.close();}}i.enabled&&!t.resumeRewrite&&(i.manifest.updatedAt=new Date().toISOString(),await bt(e,i.manifest)),await P.writeFile(w.join(e.outputDir,"asset-map.json"),JSON.stringify(a.assetMap,null,2),"utf8");let u=t.mode==="single-url"?Et(e,a):void 0;o.mark("rewrite-text");let l=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${u?.length??0}`:"Rewriting text files...";o.progress(l,{phase:"rewrite-text",index:0,totalFiles:u?.length,changedTextFiles:0}),await xe(l);let c=Date.now(),p=await fe(e,a.assetMap,{files:u,onProgress:async({index:d,totalFiles:g,changedTextFiles:h,file:m})=>{o.checkpoint(`Rewriting text file ${d}/${g}`,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m});let f=Date.now();if(!(d===1||d===g||f-c>=5e3))return;c=f;let b=`Rewriting text files: ${d}/${g}`;o.progress(b,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m}),await xe(b);}});o.endMark("rewrite-text",{changedTextFiles:p}),o.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${p} text files.`:`Done. Rendered ${a.stats.pagesRendered} pages, processed ${a.doneSitemaps.size} sitemaps, downloaded ${a.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...a.stats,pagesRendered:a.stats.pagesRendered,donePages:a.donePages.size,doneSitemaps:a.doneSitemaps.size,doneAssets:a.doneAssets.size,changedTextFiles:p}),await o.flush();}or().catch(async e=>{console.error(e);try{let r=(await G().catch(()=>null))?.logDir??"logs",s=new $(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
4
|
+
`;return e.replace(/<head\b[^>]*>/i,n=>`${n}${s} ${t}`)}function rt(e){let t=L(e).trim();return t.startsWith("{")||t.startsWith("[")||t.includes("\\/")||/"@context"|"@type"/.test(t)}function st(e,t){let r=[...new Set(t)].filter(i=>i.includes("/")&&!/\\+\//.test(i)).map(i=>[i,F(i)]).filter(([i,o])=>i!==o).sort((i,o)=>o[0].length-i[0].length);if(!r.length)return e;let s=i=>{let o=i;for(let[a,u]of r)o=o.split(a).join(u);return o},n=e.replace(/(<script\b[^>]*\btype=["']application\/(?:ld\+)?json["'][^>]*>)([\s\S]*?)(<\/script>)/gi,(i,o,a,u)=>`${o}${s(a)}${u}`);return n=n.replace(/(<meta\b[^>]*\bcontent=(['"]))([\s\S]*?)(\2[^>]*>)/gi,(i,o,a,u,l)=>rt(u)?`${o}${s(u)}${l}`:i),n}function nt(e,t){if(!t)return null;let r=f.dirname(f.resolve(t)),s=f.resolve(e.outputDir),n=f.relative(r,s).replace(/\\/g,"/");return n?(n.startsWith(".")||(n=`./${n}`),n.endsWith("/")?n:`${n}/`):"./"}function it(e,t,r){let s=nt(t,r);if(!s)return e;let n=e;for(let i of Xe){let o=i.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),a=new RegExp(`(?:\\.{1,}/)+${o}`,"g"),u=new RegExp(`(?<!\\.)/${o}`,"g");n=n.replace(a,`${s}${i}`).replace(u,`${s}${i}`);let l=i.replace(/\//g,"\\/"),c=s.replace(/\//g,"\\/"),p=l.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),d=new RegExp(`(?:\\.{1,}\\\\/)+${p}`,"g"),g=new RegExp(`(?:\\.{1,}(?:\\\\/|\\\\))+${p}`,"g"),h=new RegExp(`(?<![\\.\\\\])\\/${p}`,"g");n=n.replace(d,`${c}${l}`).replace(g,`${c}${l}`).replace(h,`${c}${l}`);}return n}function ot(e,t,r){if(e.urlRewriteMode==="absolute")return t;let s=(()=>{try{return new URL(t,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin)}catch{return null}})(),n=s?s.pathname:pe(t,e.targetOrigin),i=s?`${s.search}${s.hash}`:"";return e.urlRewriteMode==="root-relative"?`${n}${i}`:`${ge(e.outputDir,r,n)}${i}`}function X(e,t,r,s,n){de(e,r,ot(t,s,n));}function at(e,t,r,s){let n=e,i={};X(i,t,t.sourceOrigin,t.targetOrigin,s);for(let[a,u]of Object.entries(t.extraReplacements))X(i,t,a,u,s);for(let[a,u]of Object.entries(r))X(i,t,a,u,s);let o=Object.entries(i).sort((a,u)=>u[0].length-a[0].length);for(let[a,u]of o)n=n.split(a).join(u);return t.urlRewriteMode!=="absolute"&&(n=it(n,t,s)),n=st(n,o.map(([,a])=>a)),et(t,n,s)&&(n=tt(n)),n}async function ut(e,t,r){let s=f.join(e.outputDir,r),n=f.extname(s).toLowerCase(),i;try{i=await P.readFile(s,"utf8");}catch(u){if(u.code!=="ENOENT")throw u;return {changed:false}}let a=at(i,e,t,n===".js"||n===".mjs"?void 0:s);return a!==i?(await P.writeFile(s,a,"utf8"),{changed:true}):{changed:false}}async function lt(e,t){return await new Promise((r,s)=>{let n=u=>{if(a(),u?.error){s(new Error(u.error));return}r({changed:!!u?.changed});},i=u=>{a(),s(u);},o=u=>{a(),u!==0&&s(new Error(`Rewrite worker exited with code ${u}.`));},a=()=>{e.off("message",n),e.off("error",i),e.off("exit",o);};e.on("message",n),e.on("error",i),e.on("exit",o),e.postMessage({file:t});})}async function fe(e,t,r={}){let n=(r.files?[...new Set(r.files)].map(d=>f.resolve(d)).filter(d=>d.startsWith(f.resolve(e.outputDir))).map(d=>f.relative(e.outputDir,d).replace(/\\/g,"/")).filter(d=>d.length>0):await Ge(["**/*"],{cwd:e.outputDir,onlyFiles:true,dot:true})).filter(d=>d!=="asset-map.json"&&O(d)),i=0;if(n.length===0)return 0;let o=Math.max(1,Number(e.rewriteConcurrency||e.assetDownloadConcurrency||1)),a=Math.max(1,Math.min(o,n.length,availableParallelism())),u=0,l=0,c=null;if(a===1){for(let d of n)(await ut(e,t,d)).changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:d});return i}let p=Array.from({length:a},async()=>{let d=new Worker(new URL("./rewrite-worker.js",import.meta.url),{workerData:{config:e,assetMap:t}});try{for(;!c;){let g=u;if(g>=n.length)return;u+=1;let h=n[g];(await lt(d,h)).changed&&(i+=1),l+=1,await r.onProgress?.({index:l,totalFiles:n.length,changedTextFiles:i,file:h});}}catch(g){throw c=g instanceof Error?g:new Error(String(g)),c}finally{await d.terminate().catch(()=>{});}});return await Promise.all(p),i}var T={error:0,warn:1,info:2,debug:3},we=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",U=we?f.join(we,"current-progress.json"):"",Pe=Promise.resolve();function ye(e){return e.includes("crawl")?"crawl":e.includes("deploy")?"deploy":e.includes("invalidate")?"invalidate":e}function ct(e,t){let r={...e},s=new Set(["pagesQueued","pagesRendered","assetsQueued","sitemapsQueued","pagesDiscovered","assetsDiscovered","sitemapsDiscovered","pagesSaved","assetsSaved","donePages","doneAssets","doneSitemaps","changedTextFiles","uploaded","failed","index","totalFiles","pageQueue","assetQueue","sitemapQueue"]);for(let[n,i]of Object.entries(t)){let o=r[n];if(typeof o=="number"&&typeof i=="number"&&s.has(n)){r[n]=Math.max(o,i);continue}r[n]=i;}return r}function dt(e,t,r){U&&(Pe=Pe.then(async()=>{let s=new Date,n=s.toISOString();await P.mkdir(f.dirname(U),{recursive:true});let i=await P.readFile(U,"utf8").then(m=>JSON.parse(m)).catch(()=>null),o=i&&typeof i.details=="object"&&i.details?i.details:{},a=ye(e),u=i?.currentStep||ye(i?.source||""),l=i?.startedAt||n,c={...i?.stepDurationsSec??{}},p=i?.stepStartedAt||n;if(u&&u!==a&&i?.stepStartedAt){let m=Math.max(0,Math.round((s.getTime()-new Date(i.stepStartedAt).getTime())/1e3));c[u]=(c[u]??0)+m,p=n;}let d=Math.max(0,Math.round((s.getTime()-new Date(p).getTime())/1e3)),g=Math.max(0,Math.round((s.getTime()-new Date(l).getTime())/1e3)),h={checkedAt:n,source:e,message:t,details:ct(o,r??{}),startedAt:l,currentStep:a,stepStartedAt:p,stepElapsedSec:d,totalElapsedSec:g,stepDurationsSec:c};await P.writeFile(U,JSON.stringify(h,null,2),"utf8");}).catch(()=>{}));}async function pt(e,t){await P.mkdir(f.dirname(e),{recursive:true}),await P.appendFile(e,`${JSON.stringify(t)}
|
|
5
|
+
`,"utf8");}function gt(e,t){let r=typeof t?.phase=="string"?t.phase.trim():"";return r||(e==="page"?"render-pages":e==="asset"?"download-assets":e==="sitemap"?"discovery":e==="timing"&&typeof t?.name=="string"&&t.name.trim()||"crawl")}function mt(e,t,r){return {checkedAt:new Date().toISOString(),currentStep:gt(e,r),level:e,message:t,details:r??{}}}var $=class{constructor(t,r="info"){this.logDir=t;this.level=typeof r=="boolean"?r?"debug":"info":r,this.ensureLogFileReady();}logDir;startedAt=Date.now();marks=new Map;initPromise=null;writeQueue=Promise.resolve();writeError=null;level;get logPath(){return f.join(this.logDir,"crawl.log.jsonl")}get currentEventPath(){return f.join(this.logDir,"current-crawl-event.json")}get rejectedPath(){return f.join(this.logDir,"rejected.jsonl")}get ignoredPath(){return f.join(this.logDir,"ignored.jsonl")}get skippedPath(){return f.join(this.logDir,"skipped-http.jsonl")}get errorsPath(){return f.join(this.logDir,"errors.jsonl")}get timingsPath(){return f.join(this.logDir,"timings.jsonl")}ensureLogFileReady(){return this.initPromise||(this.initPromise=(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.logPath,"","utf8");})()),this.initPromise}enqueueTask(t){this.writeQueue=this.writeQueue.then(t).catch(r=>{this.writeError=r instanceof Error?r:new Error(String(r));});}enqueueLine(t){this.enqueueTask(async()=>{await this.ensureLogFileReady(),await P.appendFile(this.logPath,`${t}
|
|
6
|
+
`,"utf8");});}enqueueJsonLine(t,r){this.enqueueTask(()=>pt(t,r));}updateCurrentEvent(t,r,s){let n=mt(t,r,s);this.enqueueTask(async()=>{await P.mkdir(this.logDir,{recursive:true}),await P.writeFile(this.currentEventPath,JSON.stringify(n,null,2),"utf8");});}push(t,r,s){let n=JSON.stringify({time:new Date().toISOString(),level:t,message:r,...s||{}});this.enqueueLine(n),this.updateCurrentEvent(t,r,s),t==="error"?console.error(r,s||""):t==="warn"?console.warn(r,s||""):(T[this.level]>=T.debug||["summary","page","sitemap","asset","timing"].includes(t))&&console.log(r);}info(t,r){this.push("info",t,r);}progress(t,r){dt("crawl.log.jsonl",t,r),this.updateCurrentEvent("progress",t,r);}checkpoint(t,r){this.updateCurrentEvent("checkpoint",t,r);}page(t,r){this.push("page",t,r);}sitemap(t,r){this.push("sitemap",t,r);}asset(t,r){this.push("asset",t,r);}warn(t,r){this.push("warn",t,r);}error(t,r){this.enqueueJsonLine(this.errorsPath,{message:t,...r||{}}),this.push("error",t,r);}summary(t,r){this.push("summary",t,r);}mark(t){this.marks.set(t,Date.now());}endMark(t,r){let s=this.marks.get(t);if(!s)return;let n=Date.now()-s,i={name:t,ms:n,seconds:Number((n/1e3).toFixed(2)),...r||{}};this.enqueueJsonLine(this.timingsPath,i),this.push("timing",`Timing ${t}: ${i.seconds}s`,i);}reject(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.rejectedPath,i),T[this.level]>=T.debug&&this.push("reject",`Rejected ${t} ${r}: ${s}`,i);}ignore(t,r,s,n){let i={kind:t,url:r,reason:s,source:n};this.enqueueJsonLine(this.ignoredPath,i),T[this.level]>=T.debug&&this.push("ignore",`Ignored ${t} ${r}: ${s}`,i);}skip(t,r,s,n){let i={kind:t,url:r,status:s,source:n};this.enqueueJsonLine(this.skippedPath,i),this.push("warn",`Skipped ${t} ${r}: HTTP ${s}`,i);}async flush(){let t=Date.now()-this.startedAt,r={name:"total",ms:t,seconds:Number((t/1e3).toFixed(2))};if(this.enqueueJsonLine(this.timingsPath,r),await this.writeQueue,this.writeError)throw this.writeError}};var ft=["text/css","javascript","image/","font/","application/font","application/json","application/xml","text/xml","application/rss+xml","application/atom+xml","application/xslt+xml","text/xsl","image/svg+xml","application/octet-stream"],wt=new Set([".css",".js",".mjs",".json",".map",".xml",".xsl",".rss",".atom",".txt",".enc",".jws",".png",".jpg",".jpeg",".gif",".webp",".avif",".svg",".ico",".woff",".woff2",".ttf",".otf",".eot",".pdf",".mp4",".webm"]),Pt=new Set(["",".html",".htm"]),yt=new Set([".css",".xml",".xsl",".svg",".json",".html",".htm",".txt",".enc",".jws"]),St=["/wp-content/","/wp-includes/","/wp-admin/","/static/","/assets/","/build/","/_next/"];function vt(e){let t=[],r="full",s="full",n=false,i=false;for(let o=0;o<e.length;o++){let a=e[o];if(a==="--retry-timeouts")r="retry-timeouts",n=true;else if(a==="--resume")n=true;else if(a==="--resume-rewrite")n=true,i=true;else if(a==="--crawl-mode"){let u=e[++o];if(!u)throw new Error("--crawl-mode requires a value");s=u==="incremental"?"incremental":"full",n||=s==="incremental";}else if(a.startsWith("--crawl-mode="))s=a.slice(13)==="incremental"?"incremental":"full",n||=s==="incremental";else if(a==="--incremental")s="incremental",n=true;else if(a==="--url"){let u=e[++o];if(!u)throw new Error("--url requires a value");r="single-url",n=true,t.push(u);}else if(a.startsWith("--url="))r="single-url",n=true,t.push(a.slice(6));else if(a==="--urls"){let u=e[++o];if(!u)throw new Error("--urls requires a file path");r="single-url",n=true,t.push(`@${u}`);}}return {mode:r,crawlMode:s,urls:t,preserveOutput:n,resumeRewrite:i}}function te(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.resolve(e):""}function bt(){let e=te();return e?f.resolve(e,".."):""}function Rt(){let e=process.env.STATIC_PUBLISHER_WP_ROOT||process.env.WPSUITE_STATIC_PUBLISHER_WP_ROOT||"";return e.trim()?f.resolve(e):""}function xt(e){let t=e.trim(),r=[{alias:"@storage-root",root:bt(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@runtime",root:te(),requiredEnv:"STATIC_PUBLISHER_RUNTIME_DIR or WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR"},{alias:"@wp-root",root:Rt(),requiredEnv:"STATIC_PUBLISHER_WP_ROOT or WPSUITE_STATIC_PUBLISHER_WP_ROOT"}];for(let s of r){if(t!==s.alias&&!t.startsWith(`${s.alias}/`))continue;if(!s.root)return {resolvedPath:null,alias:s.alias,requiredEnv:s.requiredEnv};let n=t.slice(s.alias.length).replace(/^\/+/,"");return {resolvedPath:n?f.resolve(s.root,n):s.root,alias:s.alias}}return {resolvedPath:f.resolve(t)}}function Fe(e){let t=te();return t?f.join(t,"crawl-manifest.json"):f.join(e.outputDir,".crawl-manifest.json")}async function Ct(e){try{let t=await P.readFile(Fe(e),"utf8"),r=JSON.parse(t);if(r&&typeof r=="object"&&r.schemaVersion===1&&r.pages&&typeof r.pages=="object")return r}catch{}return {schemaVersion:1,updatedAt:"",pages:{}}}function Se(e){return JSON.parse(JSON.stringify(e))}async function kt(e,t){let r=Fe(e);await P.mkdir(f.dirname(r),{recursive:true}),await P.writeFile(r,JSON.stringify(t,null,2),"utf8");}function N(e){if(!e)return null;let t=e.trim();return t||null}async function Lt(e,t,r,s){let n=[...new Set(r.map(a=>a.trim()).filter(Boolean))],i=String(e.wpsuite?.siteKey||"").trim();if(n.length===0||!i)return new Map;let o=new URL("/wp-json/smartcloud-static-publisher/v1/change-tokens",e.sourceOrigin).toString();try{let a=await t.post(o,{timeout:3e4,failOnStatusCode:!1,headers:{"content-type":"application/json","x-site-key":i},data:JSON.stringify({urls:n})});if(!a.ok())return s.warn(`Change token lookup failed with HTTP ${a.status()}`,{endpoint:o,status:a.status()}),new Map;let u=await a.json().catch(()=>null);if(!u||!Array.isArray(u.items))return new Map;let l=new Map;for(let c of u.items)!c||typeof c.url!="string"||l.set(c.url,c);return l}catch(a){return s.warn("Change token lookup failed; falling back to sitemap metadata",{endpoint:o,error:String(a)}),new Map}}async function ve(e,t,r,s,n){if(!r.enabled)return null;if(r.changeTokenCache.has(s))return r.changeTokenCache.get(s)??null;let o=(await Lt(e,t,[s],n)).get(s)??null;return r.changeTokenCache.set(s,o),o}async function At(e,t,r,s,n,i){if(!s.enabled)return {action:"render",changeToken:null};let o=s.manifest.pages[n];if(!o)return {action:"render",changeToken:await ve(e,t,s,n,i)};let a=await ve(e,t,s,n,i);if(a?.supported&&a.token)return o.changeToken===a.token?{action:"reuse",changeToken:a}:{action:"render",changeToken:a};let u=N(r.sitemapLastmodByPage[n]);return u&&o.sitemapLastmod&&o.sitemapLastmod===u?{action:"reuse",changeToken:a}:{action:"render",changeToken:a}}async function Et(e,t,r,s){let n=be(r.previousManifest),i=0;for(let[p,d]of Object.entries(r.manifest.pages))if(!r.seenPages.has(p)){try{await P.unlink(d.outputPath);}catch(g){(g.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental page output for ${p}`,{url:p,outputPath:d.outputPath,error:String(g)});}delete r.manifest.pages[p],i++;}let o=be(r.manifest),a=new Set,u=new Set;for(let p of o){for(let g of Re(e,p))a.add(g);let d=q(e,t.assetMap,p);d&&u.add(d);}let l=0,c=0;for(let p of n){if(o.has(p))continue;let d=new Set,g=q(e,t.assetMap,p);if(g)d.add(g);else {let h=ne(e,p);d.add(h.originalFilePath),h.hashedFilePath&&d.add(h.hashedFilePath);}for(let h of d)if(!u.has(h))try{await P.unlink(h),l++;}catch(m){(m.code||"")!=="ENOENT"&&s.warn(`Failed to remove stale incremental asset output for ${p}`,{url:p,outputPath:h,error:String(m)});}for(let h of Re(e,p))a.has(h)||Object.prototype.hasOwnProperty.call(t.assetMap,h)&&(delete t.assetMap[h],c++);}(i>0||l>0||c>0)&&s.info(`Incremental cleanup removed ${i} stale pages, ${l} stale assets, ${c} stale asset mappings`,{removedPages:i,removedAssets:l,removedAssetMappings:c,phase:"incremental-cleanup"});}function be(e){let t=new Set;for(let r of Object.values(e.pages))for(let s of r.discoveredAssets||[]){let n=s.trim();n&&t.add(n);}return t}function Re(e,t){let r=new Set([t]);try{let s=new URL(t),i=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{i.push(new URL(e.targetOrigin).origin);}catch{}for(let o of i)try{r.add(new URL(o).origin+s.pathname+s.search);}catch{}r.add(s.pathname+s.search),r.add(s.pathname);}catch{}return [...r]}function Mt(e,t){let r=e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/"?e.targetOrigin:"https://relative.invalid/";try{let s=new URL(t,r);return f.join(e.outputDir,decodeURIComponent(s.pathname))}catch{return null}}function q(e,t,r){let s=t[r];return s?Mt(e,s):ne(e,r).originalFilePath}function Tt(){let e=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"";return e.trim()?f.join(f.resolve(e),"queue-runner-heartbeat.json"):""}async function xe(e){let t=Tt();if(t)try{let r=await P.readFile(t,"utf8"),s=JSON.parse(r);if(!s||typeof s!="object")return;await P.writeFile(t,JSON.stringify({...s,checkedAt:new Date().toISOString(),status:"running",currentStep:"rewrite",message:e},null,2),"utf8");}catch{}}function It(e,t){let r=new Set;for(let s of t.donePages)r.add(f.resolve(se(e,s)));for(let s of t.doneAssets){let n=q(e,t.assetMap,s);!n||!O(n)||r.add(f.resolve(n));}return [...r]}var W="re:",Q=new Map;function $t(e){if(!e.startsWith(W))return null;let t=Q.get(e);if(t!==void 0)return t||null;let r=e.slice(W.length).trim();if(!r)return Q.set(e,false),null;try{let s=new RegExp(r);return Q.set(e,s),s}catch{return console.warn(`[crawl] Ignoring invalid path matcher regex: ${e}`),Q.set(e,false),null}}function re(e,t){let r=$t(t);return r?r.test(e):t.startsWith(W)?false:e.startsWith(t)}function jt(e,t){return t.find(r=>re(e,r))}function Oe(e,t){let r=jt(t.pathname,e.blockedPathPrefixes);if(r)return r.startsWith(W)?`blocked path regex: ${r}`:`blocked path prefix: ${r}`;let s=e.blockedSearchFragments.find(n=>t.search.includes(n));return s?`blocked search fragment: ${s}`:null}function Dt(e){let t=new URL(e.sourceOrigin),r=new Set([t.hostname,...e.allowedAssetHosts]);if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{r.add(new URL(e.targetOrigin).hostname);}catch{}for(let s of Object.keys(e.extraReplacements||{}))try{r.add(new URL(s).hostname);}catch{}return r}function Ft(e,t){return Dt(e).has(t.hostname)}function Ot(e){let t=L(e.trim()).replace(/\\\//g,"/").replace(/^['"]|['"]$/g,"").trim();return t=t.replace(/[)]+$/g,"").trim(),t=t.replace(/;.*$/g,"").trim(),t}function Ue(e){let t=L(e).replace(/\\\//g,"/");return /[{}]|\bwindow\.|\blocation\.|\bincludes\(|\?\?null|\+|%7B|%7D|%22|<|>|\s/.test(t)}function H(e){return St.some(t=>e.pathname.startsWith(t))}function S(e,t,r=e.sourceOrigin,s,n="url"){let i=Ot(t);if(!i||i.startsWith("data:")||i.startsWith("blob:")||i.startsWith("mailto:")||i.startsWith("tel:")||i.startsWith("#"))return s?.ignore(n,t,"empty or unsupported scheme",r),null;if(Ue(i))return s?.ignore(n,t,"looks like JavaScript/code fragment",r),null;try{let o=new URL(i,r);if(!["http:","https:"].includes(o.protocol))return s?.ignore(n,t,"unsupported protocol: ",r),null;if(!Ft(e,o))return s?.ignore(n,o.toString(),"host not allowed",r),null;let a=Oe(e,o);if(a)return s?.reject(n,o.toString(),a,r),null;let u=new URL(e.sourceOrigin);return o.protocol=u.protocol,o.host=u.host,o.hash="",o}catch{return s?.reject(n,t,"invalid URL",r),null}}function Qe(e){return wt.has(f.extname(e.pathname).toLowerCase())}function D(e){return H(e)?false:Pt.has(f.extname(e.pathname).toLowerCase())}function J(e){let t=e.pathname.toLowerCase();return t.endsWith(".xml")&&(t.includes("sitemap")||t.endsWith("/sitemap.xml"))}function y(e,t){return J(t)||t.pathname==="/robots.txt"||t.pathname==="/llms.txt"||t.pathname.toLowerCase().endsWith(".xsl")?true:Qe(t)?e.assetPathPrefixes.length===0?true:e.assetPathPrefixes.some(r=>re(t.pathname,r)):false}function se(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);return s.endsWith("/")?s+="index.html":f.extname(s)||(s+="/index.html"),f.join(e.outputDir,s)}function ne(e,t){let r=new URL(t),s=decodeURIComponent(r.pathname);s.endsWith("/")&&(s+="index.html");let n=!!r.search,i=s,o=f.join(e.outputDir,i),a=K(e.targetOrigin,i),u=`${a}${r.search}`;if(!n)return {originalPathname:i,originalFilePath:o,originalPublicUrl:a,originalPublicUrlWithSearch:u,preferQueryHashed:n};let l=f.extname(s),p=`${l?s.slice(0,-l.length):s}.${ce(r.search)}${l||".bin"}`;return {originalPathname:i,originalFilePath:o,originalPublicUrl:a,originalPublicUrlWithSearch:u,preferQueryHashed:n,hashedPathname:p,hashedFilePath:f.join(e.outputDir,p),hashedPublicUrl:K(e.targetOrigin,p)}}function Ut(e,t,r,s,n){t[r]=s;try{let i=new URL(r),a=[new URL(e.sourceOrigin).origin,...Object.keys(e.extraReplacements||{})];if(e.targetOrigin&&e.targetOrigin!=="."&&e.targetOrigin!=="/")try{a.push(new URL(e.targetOrigin).origin);}catch{}for(let u of a)try{let l=new URL(u);t[l.origin+i.pathname+i.search]=s;}catch{}try{let u=(()=>{let l=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin);return `${l.pathname}${l.search}`})();t[i.pathname+i.search]=u;}catch{}try{let u=new URL(s,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}try{let u=new URL(n,e.targetOrigin==="."?"https://relative.invalid":e.targetOrigin).pathname;t[i.pathname]=u;}catch{}}catch{}}async function Z(e){try{return await P.readFile(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}async function Ne(e){try{return await P.stat(e)}catch(t){if(t.code==="ENOENT")return null;throw t}}function Qt(e){let t=String(e["content-length"]||"").trim();if(!t)return null;let r=Number.parseInt(t,10);return Number.isFinite(r)&&r>=0?r:null}function qe(e){let t=String(e["last-modified"]||"").trim();if(!t)return null;let r=Date.parse(t);return Number.isFinite(r)?r:null}function Nt(e,t){let r=qe(t),s=Qt(t),n=e.mtime.getTime();return r!==null&&Math.abs(n-r)>=1e3||s!==null&&e.size!==s?false:r!==null||s!==null}async function qt(e,t){if(!t)return;let r=qe(t);if(r===null)return;let s=await Ne(e);if(!s)return;let n=new Date(r);await P.utimes(e,s.atime,n).catch(()=>{});}function Ce(e,t){let r=f.extname(e.pathname).toLowerCase(),s=t["content-type"]||"";return yt.has(r)||ft.some(n=>s.includes(n))&&!s.startsWith("image/")&&!s.includes("font")}function ke(e,t){(e.doneAssets.size%25===0||e.doneAssets.size===1||e.doneAssets.size===e.stats.assetsQueued)&&t.progress(`Asset download progress: downloaded ${e.doneAssets.size}, discovered ${e.stats.assetsQueued}.`,{doneAssets:e.doneAssets.size,assetsQueued:e.stats.assetsQueued,phase:"download-assets"});}async function ee(e,t,r,s,n,i,o){let a=ne(e,r),u=a.originalFilePath,l=a.originalPublicUrlWithSearch;if(a.preferQueryHashed&&a.hashedFilePath&&a.hashedPublicUrl){let p=await Z(a.originalFilePath);p!==null&&!p.equals(s)&&(u=a.hashedFilePath,l=a.hashedPublicUrl);}Ut(e,t,r,l,a.originalPublicUrl);let c=await Z(u);(c===null||!c.equals(s))&&(await V(u),await P.writeFile(u,s)),await qt(u,o),n.stats.assetsSaved++,i&&(n.stats.assetsSaved%50===0||n.stats.assetsSaved===1)&&i.progress(`Asset progress: saved ${n.stats.assetsSaved}, discovered ${n.stats.assetsQueued}, pending ${n.assetQueue.length}.`,{assetsSaved:n.stats.assetsSaved,assetsQueued:n.stats.assetsQueued,assetQueue:n.assetQueue.length,phase:"download-assets"});}function _(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=S(e,r,i,s,"page");if(!o)return;if(!D(o)||Qe(o)){if(y(e,o)){s.info(`Seeded/non-page URL queued as asset: ${o.toString()}`,{url:o.toString(),source:n}),k(e,t,o.toString(),s,n);return}s.reject("page",o.toString(),H(o)?"asset/internal path cannot be page":"not page-like",n);return}let a=o.toString();!t.donePages.has(a)&&!t.queuedPages.has(a)&&(t.queuedPages.add(a),t.pageQueue.push(a),t.stats.pagesQueued++,(t.stats.pagesQueued%25===0||t.stats.pagesQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,pageQueue:t.pageQueue.length,phase:"discovery"}));}function k(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=S(e,r,i,s,"asset");if(!o)return;if(!y(e,o)){s.reject("asset",o.toString(),"not a safe asset path/prefix",n);return}let a=o.toString();!t.doneAssets.has(a)&&!t.queuedAssets.has(a)&&(t.queuedAssets.add(a),t.assetQueue.push(a),t.stats.assetsQueued++,(t.stats.assetsQueued%100===0||t.stats.assetsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,assetQueue:t.assetQueue.length,phase:"discovery"}));}function ie(e,t,r,s,n){let i=n&&/^https?:\/\//i.test(n)?n:e.sourceOrigin,o=S(e,r,i,s,"sitemap");if(!o)return;if(!J(o)){s.reject("sitemap",o.toString(),"not sitemap-like",n);return}let a=o.toString();!t.doneSitemaps.has(a)&&!t.queuedSitemaps.has(a)&&(t.queuedSitemaps.add(a),t.sitemapQueue.push(a),t.stats.sitemapsQueued++,(t.stats.sitemapsQueued%10===0||t.stats.sitemapsQueued===1)&&s.progress(`Discovery progress: pages ${t.stats.pagesQueued}, assets ${t.stats.assetsQueued}, sitemaps ${t.stats.sitemapsQueued}.`,{pagesQueued:t.stats.pagesQueued,assetsQueued:t.stats.assetsQueued,sitemapsQueued:t.stats.sitemapsQueued,sitemapQueue:t.sitemapQueue.length,phase:"discovery"}));}function Wt(e){let t=L(e),r=[...t.matchAll(/<url>\s*([\s\S]*?)\s*<\/url>/gi)].map(s=>s[1]);return r.length>0?r.map(s=>{let n=s.match(/<loc>\s*([^<]+?)\s*<\/loc>/i);if(!n?.[1])return null;let i=s.match(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/i);return {loc:n[1].trim(),lastmod:N(i?.[1]??null)}}).filter(s=>!!s):[...t.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map(s=>({loc:s[1].trim(),lastmod:null})).filter(s=>s.loc)}function _t(e){return e.split(",").map(t=>t.trim().split(/\s+/)[0]).filter(Boolean)}function Ht(e){let t=[],r=/url\(\s*(?:"([^"]+)"|'([^']+)'|([^)]*?))\s*\)/gi;for(let s of e.matchAll(r)){let n=(s[1]||s[2]||s[3]||"").trim();n&&t.push(n);}return t}function Y(e){return e.trim().replace(/\\/g,"/").replace(/^https?:\/\/[^/]+/i,"").replace(/^\/+/,"")}async function Jt(e,t){await P.mkdir(t,{recursive:true});let r=await P.readdir(e,{withFileTypes:true});for(let s of r){let n=f.join(e,s.name),i=f.join(t,s.name);if(s.isDirectory()){await P.cp(n,i,{recursive:true,force:true});continue}await P.copyFile(n,i);}}async function Bt(e,t){let r=Object.entries(e.postCrawlCopyMap||{}).map(([n,i])=>({sourcePath:n.trim(),prefix:String(i||"").trim()}));if(r.length===0)return;let s=0;t.mark("copy-extra-paths");for(let n of r){let i=n.sourcePath,o=n.prefix;if(!i||!o){t.warn("Skipped post-crawl copy mapping with empty key/value",{sourcePath:i,prefixPath:o});continue}let a=xt(i);if(!a.resolvedPath){t.warn("Skipped post-crawl copy source because alias root is not configured",{sourcePath:i,alias:a.alias||"",requiredEnv:a.requiredEnv||"",prefixPath:o});continue}let u=a.resolvedPath,l=Y(o);if(!l){t.warn("Skipped post-crawl copy mapping with invalid export prefix",{sourcePath:i,prefixPath:o});continue}let c;try{c=await P.stat(u);}catch{t.warn("Skipped post-crawl copy source because it does not exist",{sourcePath:i,sourceAbs:u,prefixPath:o});continue}let p=f.resolve(e.outputDir,l);if(c.isDirectory()){await Jt(u,p),s++,t.progress(`Copied static directory to export: ${u} -> /${l}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${l}`,copiedItems:s});continue}let d=f.basename(u),g=o.endsWith("/")?f.join(p,d):p;await P.mkdir(f.dirname(g),{recursive:true}),await P.copyFile(u,g),s++,t.progress(`Copied static file to export: ${u} -> /${Y(o.endsWith("/")?`${o}${d}`:o)}`,{phase:"copy-extra-paths",sourcePath:u,targetPath:`/${Y(o.endsWith("/")?`${o}${d}`:o)}`,copiedItems:s});}t.endMark("copy-extra-paths",{mappedSources:r.length,copiedItems:s});}function j(e,t,r,s,n){let i=new Set,o=L(r).replace(/\\\//g,"/");for(let l of Ht(o)){let c=S(e,l,t,s,`${n}:css-url`);c&&y(e,c)&&i.add(c.toString());}for(let l of o.matchAll(/@import\s+(?:url\()?\s*['"]?([^'"\s;]+)['"]?\s*\)?/gi)){let c=S(e,l[1],t,s,`${n}:css-import`);c&&y(e,c)&&i.add(c.toString());}for(let l of o.matchAll(/<\?xml-stylesheet[^>]+href=["']([^"']+)["'][^>]*\?>/gi)){let c=S(e,l[1],t,s,`${n}:xml-stylesheet`);c&&y(e,c)&&i.add(c.toString());}let a="(?:css|js|mjs|json|map|xml|xsl|rss|atom|txt|enc|jws|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf|otf|eot|pdf|mp4|webm)",u=new RegExp(`(?:https?:)?//[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?|(?<!\\.)/[^\\s'"<>\\);]+\\.${a}(?:\\?[^\\s'"<>\\);]*)?`,"gi");for(let l of o.matchAll(u)){let c=l[0].startsWith("//")?`${new URL(e.sourceOrigin).protocol}${l[0]}`:l[0];if(Ue(c)){s.ignore(`${n}:serialized-url`,c,"looks like JavaScript/code fragment",t);continue}let p=S(e,c,t,s,`${n}:serialized-url`);p&&y(e,p)?i.add(p.toString()):p&&s.ignore("asset",p.toString(),"serialized URL did not pass safe asset path filter",t);}return [...i]}function zt(e,t,r,s){let n=new Set,i=L(r);for(let o of i.matchAll(/href=["']([^"'#\s][^"']*?)["']/gi)){let a=o[1],u=S(e,a,t,s,"page-link");u&&(Oe(e,u)||D(u)&&!H(u)&&!y(e,u)&&n.add(u.toString()));}return [...n]}async function Le(e,t,r,s){for(;r.sitemapQueue.length>0;){let n=r.sitemapQueue.shift();if(r.queuedSitemaps.delete(n),!r.doneSitemaps.has(n)){r.doneSitemaps.add(n),s.sitemap(`Fetching sitemap ${n}`,{url:n});try{let i=await t.get(n,{timeout:6e4});if(!i.ok()){s.skip("sitemap",n,i.status());continue}let o=await i.body(),a=i.url()||n;await ee(e,r.assetMap,a,o,r,s);let u=o.toString("utf8");for(let l of j(e,a,u,s,"sitemap"))k(e,r,l,s,a);for(let l of Wt(u)){let c=S(e,l.loc,a,s,"sitemap-loc");c&&(J(c)?ie(e,r,c.toString(),s,a):y(e,c)?k(e,r,c.toString(),s,a):D(c)?(l.lastmod&&(r.sitemapLastmodByPage[c.toString()]=l.lastmod),_(e,r,c.toString(),s,a)):s.reject("sitemap-loc",c.toString(),"not page/sitemap/asset-like",a));}}catch(i){s.error(`Failed sitemap ${n}`,{url:n,error:String(i)});}}}}async function Gt(e,t,r,s,n){let i=S(e,s,e.sourceOrigin,n,"asset-fetch");if(!i||!y(e,i))return;let o=i.toString();if(!r.doneAssets.has(o)){n.asset(`Fetching asset ${o}`,{url:o});try{let a=q(e,r.assetMap,o),u=a?await Ne(a):null;if(a&&u)try{let g=await t.head(o,{timeout:6e4}),h=g.headers();if(g.ok()&&Nt(u,h)){let m=await Z(a);if(m!==null){let w=g.url()||o;if(await ee(e,r.assetMap,w,m,r,n,h),r.doneAssets.add(o),n.info(`Reused existing asset ${o} based on response headers.`,{url:o,filePath:a,lastModified:h["last-modified"]||"",contentLength:h["content-length"]||"",phase:"download-assets"}),ke(r,n),Ce(i,h)){let v=m.toString("utf8");for(let b of j(e,w,v,n,"asset:cached"))k(e,r,b,n,w);}return}}}catch{}let l=await t.get(o,{timeout:6e4});if(!l.ok()){n.skip("asset",o,l.status());return}let c=await l.body(),p=l.url()||o,d=l.headers();if(await ee(e,r.assetMap,p,c,r,n,d),r.doneAssets.add(o),ke(r,n),Ce(i,d)){let g=c.toString("utf8");for(let h of j(e,p,g,n,`asset:${f.extname(i.pathname).toLowerCase()||d["content-type"]||"unknown"}`))k(e,r,h,n,p);}}catch(a){n.error(`Failed asset ${o}`,{url:o,error:String(a)});}}}async function Vt(e,t,r,s){let n=Math.max(1,Number(e.assetDownloadConcurrency||e.concurrency||1)),i=Array.from({length:n},async()=>{for(;r.assetQueue.length>0;){let o=r.assetQueue.shift();o&&(r.queuedAssets.delete(o),await Gt(e,t,r,o,s));}});await Promise.all(i);}async function Kt(e){await e.evaluate(async()=>{await new Promise(t=>{let r=0,s=700,n=window.setInterval(()=>{window.scrollBy(0,s),r+=s,r>=document.body.scrollHeight+window.innerHeight&&(window.clearInterval(n),window.scrollTo(0,0),t());},120);});});}async function Xt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForFunction&&await t.waitForFunction(e.readiness.waitForFunction,void 0,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await Kt(t),await t.waitForTimeout(e.readiness.fallbackWaitMs);}async function Yt(e,t){await t.waitForLoadState("domcontentloaded",{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForLoadState("load",{timeout:e.readiness.timeoutMs}).catch(()=>{}),e.readiness.waitForSelector&&await t.waitForSelector(e.readiness.waitForSelector,{timeout:e.readiness.timeoutMs}).catch(()=>{}),await t.waitForTimeout(e.readiness.fallbackWaitMs);}function Zt(e,t){return e.noJavaScriptRenderPathPrefixes.some(r=>re(t,r))}async function er(e,t,r,s,n=true){let i=new Set;if(n){let a=await t.evaluate(()=>{let u=new Set,l=["href","src","poster","data-src","data-lazy-src","data-original","data-bg","data-background","data-href"],c=new Set,p=["srcset","data-srcset","data-lazy-srcset"];return document.querySelectorAll("*").forEach(d=>{for(let h of l){let m=d.getAttribute(h);m&&u.add(m);}for(let h of p){let m=d.getAttribute(h);m&&m.split(",").forEach(w=>{let v=w.trim().split(/\s+/)[0];v&&u.add(v);});}let g=d.getAttribute("style");g&&c.add(g);}),{attrs:[...u],styles:[...c]}});for(let u of a.attrs)for(let l of _t(u)){let c=S(e,l,r,s,"dom-attr");c&&y(e,c)&&i.add(c.toString());}for(let u of a.styles)for(let l of j(e,r,u,s,"dom-style"))i.add(l);}let o=await t.content();for(let a of j(e,r,o,s,"page-html"))i.add(a);return [...i]}async function Ae(e,t,r,s,n){let i=se(e,t);await V(i),await P.writeFile(i,r,"utf8"),s.stats.pagesSaved++,n&&(s.stats.pagesSaved%10===0||s.stats.pagesSaved===1)&&n.progress(`Page progress: processed ${s.donePages.size}, rendered ${s.stats.pagesRendered}, saved ${s.stats.pagesSaved}, discovered ${s.stats.pagesQueued}.`,{donePages:s.donePages.size,pagesRendered:s.stats.pagesRendered,pagesSaved:s.stats.pagesSaved,pagesQueued:s.stats.pagesQueued,pageQueue:s.pageQueue.length,phase:"save-pages"});}function Ee(e){return e.donePages.size%5===0||e.donePages.size===1||e.donePages.size===e.stats.pagesQueued}function Me(e,t){t.progress(`Render progress: processed ${e.donePages.size}, rendered ${e.stats.pagesRendered}, saved ${e.stats.pagesSaved}, discovered ${e.stats.pagesQueued}.`,{donePages:e.donePages.size,pagesRendered:e.stats.pagesRendered,pagesSaved:e.stats.pagesSaved,pagesQueued:e.stats.pagesQueued,phase:"render-pages"});}async function tr(e){await e.addInitScript(()=>{Object.defineProperty(window,"__WPSUITE_STATIC_EXPORT__",{value:true,writable:false,configurable:true});let t=r=>{if(r==null||String(r)==="")return true;try{let s=new URL(String(r),window.location.href),n=new URL(window.location.href);return s.hash="",n.hash="",s.href===n.href}catch{return false}};try{let r=window.location.assign.bind(window.location),s=window.location.replace.bind(window.location),n=window.location.reload.bind(window.location);Object.defineProperty(window.location,"assign",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.assign",i??"");return}return r(i)}}),Object.defineProperty(window.location,"replace",{configurable:!0,value:i=>{if(t(i)){console.warn("[smartcloud-static-publisher] blocked same-page location.replace",i??"");return}return s(i)}}),Object.defineProperty(window.location,"reload",{configurable:!0,value:()=>{console.warn("[smartcloud-static-publisher] blocked location.reload");}});}catch{}});}async function rr(e,t,r,s,n,i){let o=await chromium.launch({headless:true}),a=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",ignoreHTTPSErrors:e.ignoreHttpsErrors}),u=await o.newContext({viewport:e.viewport,userAgent:"WPSuiteStaticPublisher/0.8 Playwright SitemapOnly",javaScriptEnabled:false,ignoreHTTPSErrors:e.ignoreHttpsErrors});await tr(a);try{for(;t.pageQueue.length>0&&!(e.maxPages>0&&t.donePages.size>=e.maxPages);){let l=t.pageQueue.shift();if(!l)break;if(t.queuedPages.delete(l),t.donePages.has(l))continue;t.donePages.add(l),i.enabled&&i.seenPages.add(l);let c=new URL(l),p=Zt(e,c.pathname);if(!D(c)||y(e,c)||H(c)){if(y(e,c)){r.info(`Worker redirected non-page URL to asset queue: ${l}`,{url:l,source:"worker-guard"}),k(e,t,l,r,"worker-guard");continue}r.reject("page",l,"guard rejected non-page URL before rendering","worker");continue}let d=await At(e,n,t,i,l,r);if(i.enabled&&s!=="single-url"){let m=i.manifest.pages[l];if(d.action==="reuse"&&m){for(let w of m.discoveredAssets)k(e,t,w,r,l);for(let w of m.discoveredPages)_(e,t,w,r,l);m.lastSeenRunId=i.runId,m.sitemapLastmod=N(t.sitemapLastmodByPage[l])??m.sitemapLastmod,d.changeToken?.supported&&(m.changeToken=d.changeToken.token,m.tokenSource=d.changeToken.tokenSource??m.tokenSource),r.info(`Incremental reuse skipped unchanged page ${l}`,{url:l,mode:"incremental",reason:d.changeToken?.supported===!0?"change-token-match":"sitemap-lastmod-match"}),Ee(t)&&Me(t,r);continue}}let g=await(p?u.newPage():a.newPage()),h=!1;await g.route("**/*",async m=>{let w=m.request();if(w.isNavigationRequest()&&w.frame()===g.mainFrame())try{let v=new URL(w.url()),b=new URL(l);if(v.hash="",b.hash="",v.href===b.href){if(h){r.warn(`Blocked same-page navigation/reload for ${l}`,{url:l,requestUrl:w.url()}),await m.abort("aborted");return}h=!0;}}catch{}await m.continue();}),g.on("response",m=>{try{let w=S(e,m.url(),e.sourceOrigin,void 0,"network-response");w&&y(e,w)&&k(e,t,w.toString(),r,l);}catch{}}),r.page(`Rendering ${l}`,{url:l});try{let m=null,w=null;try{if(m=await g.goto(l,{waitUntil:"domcontentloaded",timeout:e.navigationTimeoutMs}),m&&m.ok())try{w=await m.text();}catch{w=null;}}catch(I){r.warn(`Navigation issue for ${l}; saving current DOM if available`,{url:l,error:String(I)});}if(m&&!m.ok()){r.skip("page",l,m.status()),await g.close();continue}p?(r.info(`Rendering without JS execution for ${l}`,{url:l,mode:"no-js"}),await Yt(e,g)):await Xt(e,g),t.stats.pagesRendered++,Ee(t)&&Me(t,r);let v=await er(e,g,l,r,!p);for(let I of v)k(e,t,I,r,l);let b=w??await g.content(),oe=s!=="single-url"?zt(e,l,b,r):[];if(s!=="single-url")for(let I of oe)_(e,t,I,r,l);await Ae(e,l,b,t,r),i.enabled&&(i.manifest.pages[l]={url:l,outputPath:se(e,l),changeToken:d.changeToken?.supported?d.changeToken.token:null,tokenSource:d.changeToken?.supported?d.changeToken.tokenSource??null:null,sitemapLastmod:N(t.sitemapLastmodByPage[l])??null,discoveredPages:oe,discoveredAssets:v,lastCrawledAt:new Date().toISOString(),lastSeenRunId:i.runId});}catch(m){try{let w=await g.content();w&&w.trim()&&(await Ae(e,l,w,t,r),r.warn(`Saved partial DOM for ${l}`,{url:l,error:String(m)}));}catch(w){r.error(`Could not save partial DOM for ${l}`,{url:l,error:String(w)});}r.error(`Failed page ${l}`,{url:l,error:String(m)});}finally{await g.close();}}}finally{await u.close(),await a.close(),await o.close();}}async function sr(e){try{let t=await P.readFile(e),s=(e.endsWith(".gz")?gunzipSync(t):t).toString("utf8").trim();if(!s)return [];try{let n=JSON.parse(s);if(Array.isArray(n))return n;if(n&&typeof n=="object")return [n]}catch{return s.split(/\r?\n/).map(n=>n.trim()).filter(Boolean).map(n=>{try{let i=JSON.parse(n);return i&&typeof i=="object"?i:null}catch{return null}}).filter(n=>n!==null)}return []}catch{return []}}function Te(e){let t=String(e||"").toLowerCase();return t.includes("timeout")||t.includes("timed out")||t.includes("navigation timeout")}function nr(e){let t=String(e?.job?.command||"").trim(),r=String(e?.job?.crawlMode||"full").trim();return t!=="publish"&&t!=="crawl"?false:r!=="incremental"}function ir(e){let t=[e?.archivedAt,e?.job?.endedAt,e?.job?.startedAt];for(let r of t){let s=Date.parse(String(r||""));if(Number.isFinite(s))return s}return 0}async function We(e){try{let t=await P.readFile(e,"utf8"),r=JSON.parse(t);return r&&typeof r=="object"&&!Array.isArray(r)?r:null}catch{return null}}function Ie(e){let t=String(e.originalFileName||"").toLowerCase();return t.endsWith("errors.jsonl")?0:t.endsWith("errors.json")?1:2}function or(e,t){let r=Array.isArray(t?.artifacts)?t.artifacts.filter(s=>String(s?.role||"").trim()==="errors"&&String(s?.storedFileName||"").trim()!=="").sort((s,n)=>Ie(s)-Ie(n)).map(s=>f.join(e,String(s.storedFileName))):[];return [...new Set([...r,f.join(e,"errors.jsonl"),f.join(e,"errors.json"),f.join(e,"errors.jsonl.gz"),f.join(e,"errors.json.gz")])]}async function $e(e){let t=await We(f.join(e,"job.json"));for(let r of or(e,t)){let s=await sr(r);if(s.length>0)return s}return []}async function ar(e){let t=f.join(e.logDir,"archive"),r;try{r=await P.readdir(t);}catch{return ""}let s="",n=0;for(let i of r){let o=f.join(t,i);if(!(await P.stat(o).catch(()=>null))?.isDirectory())continue;let u=await We(f.join(o,"job.json"));if(!nr(u))continue;let l=ir(u);l>=n&&(n=l,s=o);}return s}async function ur(e){let t=await ar(e),r=t!==""?await $e(t):await $e(e.logDir),s=new Set;for(let n of r)if(Te(n.error)||Te(n.message)){let i=n.url;typeof i=="string"&&i&&s.add(i);}return [...s]}async function lr(e){let t=[];for(let r of e)if(r.startsWith("@")){let s=r.slice(1),n=await P.readFile(s,"utf8");t.push(...n.split(/\r?\n/).map(i=>i.trim()).filter(i=>i&&!i.startsWith("#")));}else t.push(r);return t}async function cr(e){try{let t=await P.readFile(f.join(e.outputDir,"asset-map.json"),"utf8");return JSON.parse(t)}catch{return {}}}async function dr(e){let t=["crawl.log.jsonl","current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"];await P.mkdir(e,{recursive:true}),await Promise.all(t.map(async r=>{try{await P.unlink(f.join(e,r));}catch(s){if(s.code!=="ENOENT")throw s}}));}function je(e,t,r,s,n){let i=S(e,r,e.sourceOrigin,s,"manual-url");i&&(J(i)?ie(e,t,i.toString(),s,n):y(e,i)?k(e,t,i.toString(),s,n):D(i)?_(e,t,i.toString(),s,n):s.reject("manual-url",i.toString(),"not page/sitemap/asset-like",n));}async function pr(){let e=await G(),t=vt(process.argv.slice(2));if(t.resumeRewrite&&t.mode!=="full")throw new Error("--resume-rewrite is only supported for full crawl/publish jobs.");let r=[];t.mode==="retry-timeouts"&&(r=await ur(e)),t.mode==="single-url"&&(r=await lr(t.urls));let s=t.crawlMode==="incremental"&&t.mode==="full",n=s?await Ct(e):{schemaVersion:1,updatedAt:"",pages:{}},i={enabled:s,manifest:Se(n),previousManifest:Se(n),runId:`${Date.now()}`,seenPages:new Set,changeTokenCache:new Map};await dr(e.logDir);let o={pageQueue:[],queuedPages:new Set,donePages:new Set,assetQueue:[],queuedAssets:new Set,doneAssets:new Set,sitemapQueue:[],queuedSitemaps:new Set,doneSitemaps:new Set,sitemapLastmodByPage:{},assetMap:t.preserveOutput?await cr(e):{},stats:{pagesQueued:0,pagesRendered:0,assetsQueued:0,sitemapsQueued:0,assetsSaved:0,pagesSaved:0}};t.preserveOutput||await P.rm(e.outputDir,{recursive:true,force:true}),await P.mkdir(e.outputDir,{recursive:true});let a=new $(e.logDir,e.logLevel);if(t.resumeRewrite)a.info("Resuming final rewrite from existing output.",{phase:"rewrite-text",mode:t.mode,crawlMode:t.crawlMode});else {let d=await chromium.launch({headless:true}),g=await d.newContext({ignoreHTTPSErrors:e.ignoreHttpsErrors}),h=g.request;try{if(a.mark("discovery"),t.mode==="full"?(e.sitemapPaths.forEach(w=>ie(e,o,w,a)),await Le(e,h,o,a),e.seedPaths.forEach(w=>je(e,o,w,a,"seed-path"))):(r.forEach(w=>je(e,o,w,a,"cli")),await Le(e,h,o,a)),a.endMark("discovery",{pages:o.pageQueue.length,assets:o.assetQueue.length,sitemaps:o.sitemapQueue.length}),a.summary(`Queued ${o.pageQueue.length} pages, ${o.assetQueue.length} assets, ${o.sitemapQueue.length} sitemaps.`,{mode:t.mode,crawlMode:t.crawlMode,queuedPages:o.pageQueue.length,queuedAssets:o.assetQueue.length,queuedSitemaps:o.sitemapQueue.length}),t.mode==="retry-timeouts"&&o.pageQueue.length===0&&o.assetQueue.length===0&&o.sitemapQueue.length===0){a.summary("No timed-out URLs were queued. Skipping retry crawl.",{mode:t.mode,crawlMode:t.crawlMode,queuedPages:0,queuedAssets:0,queuedSitemaps:0}),await a.flush();return}a.mark("render-pages");let m=Array.from({length:Math.max(1,e.concurrency)},()=>rr(e,o,a,t.mode,h,i));await Promise.all(m),a.endMark("render-pages",{pages:o.donePages.size}),a.mark("download-assets"),await Vt(e,h,o,a),a.endMark("download-assets",{assets:o.doneAssets.size}),i.enabled&&e.maxPages===0&&await Et(e,o,i,a),t.mode==="full"&&await Bt(e,a);}finally{await g.close(),await d.close();}}i.enabled&&!t.resumeRewrite&&(i.manifest.updatedAt=new Date().toISOString(),await kt(e,i.manifest)),await P.writeFile(f.join(e.outputDir,"asset-map.json"),JSON.stringify(o.assetMap,null,2),"utf8");let u=t.mode==="single-url"?It(e,o):void 0;a.mark("rewrite-text");let l=t.mode==="single-url"?`Rewriting text files touched by URL crawl: 0/${u?.length??0}`:"Rewriting text files...";a.progress(l,{phase:"rewrite-text",index:0,totalFiles:u?.length,changedTextFiles:0}),await xe(l);let c=Date.now(),p=await fe(e,o.assetMap,{files:u,onProgress:async({index:d,totalFiles:g,changedTextFiles:h,file:m})=>{a.checkpoint(`Rewriting text file ${d}/${g}`,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m});let w=Date.now();if(!(d===1||d===g||w-c>=5e3))return;c=w;let b=`Rewriting text files: ${d}/${g}`;a.progress(b,{phase:"rewrite-text",index:d,totalFiles:g,changedTextFiles:h,file:m}),await xe(b);}});a.endMark("rewrite-text",{changedTextFiles:p}),a.summary(t.resumeRewrite?`Done. Resumed final rewrite over existing output and changed ${p} text files.`:`Done. Rendered ${o.stats.pagesRendered} pages, processed ${o.doneSitemaps.size} sitemaps, downloaded ${o.doneAssets.size} assets.`,{mode:t.mode,crawlMode:t.crawlMode,resumeRewrite:t.resumeRewrite,...o.stats,pagesRendered:o.stats.pagesRendered,donePages:o.donePages.size,doneSitemaps:o.doneSitemaps.size,doneAssets:o.doneAssets.size,changedTextFiles:p}),await a.flush();}pr().catch(async e=>{console.error(e);try{let r=(await G().catch(()=>null))?.logDir??"logs",s=new $(r,"debug");s.error(`Unhandled error: ${e instanceof Error?e.message:String(e)}`,e instanceof Error?{stack:e.stack}:void 0),await s.flush();}catch{}process.exit(1);});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import u from'fs/promises';import a from'path';function f(){console.log(`publisher-exporter prune-logs [options]
|
|
2
|
+
|
|
3
|
+
Options:
|
|
4
|
+
--runtime-dir <path> Runtime directory that contains config.json
|
|
5
|
+
--config <path> Explicit config.json path
|
|
6
|
+
--log-dir <path> Override the resolved log directory
|
|
7
|
+
--older-than-days <days> Delete archive directories older than this many days (default: 30)
|
|
8
|
+
--dry-run Print matching archive directories without deleting them
|
|
9
|
+
--help Show this help
|
|
10
|
+
|
|
11
|
+
Examples:
|
|
12
|
+
publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30
|
|
13
|
+
publisher-exporter prune-logs --log-dir /srv/site/logs --older-than-days 14 --dry-run
|
|
14
|
+
`);}function m(e){let i=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",t=process.env.PUBLISHER_CONFIG||"",n="",o=30,s=false;for(let l=0;l<e.length;l++){let r=e[l];(r==="--help"||r==="-h")&&(f(),process.exit(0)),r==="--runtime-dir"?i=e[++l]||"":r.startsWith("--runtime-dir=")?i=r.slice(14):r==="--config"?t=e[++l]||"":r.startsWith("--config=")?t=r.slice(9):r==="--log-dir"?n=e[++l]||"":r.startsWith("--log-dir=")?n=r.slice(10):r==="--older-than-days"?o=Number.parseInt(e[++l]||"30",10):r.startsWith("--older-than-days=")?o=Number.parseInt(r.slice(18),10):r==="--dry-run"&&(s=true);}if(!n&&!i&&!t)throw new Error("Missing log context. Use --runtime-dir, --config, --log-dir, or set STATIC_PUBLISHER_RUNTIME_DIR.");if(!Number.isFinite(o)||o<0)throw new Error("--older-than-days must be a non-negative integer.");let c=i?a.resolve(i):"",d=t?a.resolve(t):c?a.join(c,"config.json"):"";return {runtimeDir:c,configPath:d,logDir:n?a.resolve(n):"",olderThanDays:o,dryRun:s}}async function h(e,i){try{let t=await u.readFile(e,"utf8");return JSON.parse(t)}catch{return i}}function p(e,i){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return i;let o=n.split("/").map(s=>s.trim()).filter(s=>s.length>0&&s!=="."&&s!=="..");return o.length>0?o.join("/"):i}async function y(e){if(e.logDir)return e.logDir;let i=await h(e.configPath,null),n=(typeof i?.logDir=="string"?i.logDir:"").trim();if(n&&a.isAbsolute(n))return a.resolve(n);let o=e.runtimeDir?a.resolve(e.runtimeDir,".."):a.resolve(a.dirname(e.configPath),"..");return a.resolve(o,p(n,"logs"))}function D(e){let i=typeof e?.archivedAt=="string"?e.archivedAt:"",t=e?.job&&typeof e.job=="object"?e.job:null,n=[i,typeof t?.endedAt=="string"?t.endedAt:"",typeof t?.startedAt=="string"?t.startedAt:""];for(let o of n){let s=Date.parse(o);if(Number.isFinite(s))return s}return 0}async function v(e){let i=await h(a.join(e,"job.json"),null),t=D(i);return t>0?t:(await u.stat(e).catch(()=>null))?.mtimeMs??0}async function R(){let e=m(process.argv.slice(2)),i=await y(e),t=a.join(i,"archive");if(!(await u.stat(t).catch(()=>null))?.isDirectory()){console.log(`No archive directory found at ${t}.`);return}let o=Date.now()-e.olderThanDays*24*60*60*1e3,s=await u.readdir(t,{withFileTypes:true}),c=0,d=0;for(let l of s){if(!l.isDirectory())continue;let r=a.join(t,l.name),g=await v(r);if(!(g<=0||g>=o)){if(c+=1,e.dryRun){console.log(`[dry-run] ${r}`);continue}await u.rm(r,{recursive:true,force:true}),d+=1,console.log(`Deleted ${r}`);}}if(e.dryRun){console.log(`Dry run complete. ${c} archive director${c===1?"y":"ies"} older than ${e.olderThanDays} day(s) matched.`);return}console.log(`Deleted ${d} archive director${d===1?"y":"ies"} older than ${e.olderThanDays} day(s).`);}R().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exit(1);});
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import
|
|
2
|
-
`,"utf8");}catch{}}async function g(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await w(V(e.runtimeDir),n);}catch{}}async function I(e){try{await p.unlink(x(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let u=(e.id||`${r}-${t+1}`).trim();if(!u)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",s=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:u,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&s?{deploymentProfile:s}:{},...i?{url:i}:{}}}async function se(e){let t=await h(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((u,a)=>oe(u,a)).filter(u=>!!u):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t,r){return e.some(n=>n.command===t&&(n.url||"").trim()===(r||"").trim()&&(n.status===void 0||n.status==="queued"||n.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let c=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,s=[];await J(e.runtimeDir,async()=>{let o=await h(n,[]),c=await h(u,null),y=await h(k(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];c&&c.status==="running"&&m.unshift(c);for(let d of r.rules){if(!d.enabled)continue;let f=d.intervalMinutes*60*1e3,v=Math.floor(a/f),q=y.lastEnqueuedBucketByRuleId[d.id]??-1;if(v<=q||ue(m,d.command,d.url))continue;let A={id:randomUUID(),command:d.command,...(d.command==="publish"||d.command==="crawl")&&d.crawlMode?{crawlMode:d.crawlMode}:{},...(d.command==="publish"||d.command==="deploy"||d.command==="invalidate")&&d.deploymentProfile?{deploymentProfile:d.deploymentProfile}:{},...d.url?{url:d.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(A),m.push(A),s.push({job:A,rule:d}),y.lastEnqueuedBucketByRuleId[d.id]=v,i+=1;}i>0&&(await w(n,o),await w(k(e.runtimeDir),y));});for(let o of s)await P(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await p.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await p.unlink(e);}catch{}}function le(e,t){let r=[l.join(e,"dist",`${t}.js`),l.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${l.join(e,"dist")} or ${e}`)}async function S(e,t,r,n,u,a,i){if(i){let c=await j(t,i);if(c)throw new R(c,r)}let s=le(e,r),o={...process.env,...a??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,u&&(o.PUBLISHER_CONFIG=u),await new Promise((c,y)=>{let m=null,d=null,f=null,v=false,q=spawn(process.execPath,[s,...n],{cwd:e,env:o,stdio:"inherit"}),A=()=>{d&&clearInterval(d),f&&clearTimeout(f);},_=async()=>{if(!(!i||m||v)){v=true;try{let b=await j(t,i);if(!b)return;m=b,q.kill("SIGTERM"),f=setTimeout(()=>{q.kill("SIGKILL");},1e4);}finally{v=false;}}};i&&(d=setInterval(()=>{_();},1e3)),q.on("error",b=>{A(),y(b);}),q.on("close",(b,T)=>{if(A(),m){y(new R(m,r));return}b===0?c():y(new Error(`${r} exited with code ${b??-1}${T?` (signal ${T})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function pe(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let u=(await h(t.configPath||"",null))?.wpsuite;return u&&(u.accountId||u.siteId||u.siteKey)?u:null}async function ge(e,t){return await pe(e,t)}function L(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function fe(e,t){let r=await h(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let u=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(u)return u;let a=await B(e).catch(()=>"");if(a){let i=await h(l.join(a,"current-crawl-event.json"),null),s=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(s)return s}return t}function we(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function he(e){let t=l.join(e.runtimeDir,"queue.json"),r=l.join(e.runtimeDir,"current-run.json"),n=await J(e.runtimeDir,async()=>{let u=await h(r,null);if(!u||u.status!=="running"&&u.status!=="queued")return null;let a=await h(t,[]),i=Array.isArray(a)?a.filter(o=>o?.id!==u.id):[],s=L(u);return await w(t,[s,...i]),await w(r,null),s});n&&(await E(e.runtimeDir,n.id),await P(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function ye(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let s=await ge(e,t);await r.assertActiveSubscriptionForIdentity(s);}let u=me(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="crawl")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),{exitCode:0};if(e.command==="deploy")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="invalidate")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,u,e.id),{exitCode:0};if(e.command==="url"){let s=(e.url||"").trim();return s?(await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--url",s],t.configPath,u,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof R?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function N(e){return async function(r=process.argv.slice(2)){let n=G(r),u=l.join(n.runtimeDir,"export.lock"),a=l.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await g(n,"starting",{message:"queue-runner starting"});try{await ce(u);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await g(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await he(n);let i=await ae(n,e),s=0;for(let o=0;o<n.maxJobs;o++){let c=await be(n,e);if(c==="none"||(s+=1,c==="stopped"))break}if(s===0)await I(n.runtimeDir),await g(n,"idle",{processedJobs:s,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await h(a,null),c=String(o?.status||"").trim();await g(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:s,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...c==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`}:{}});}}catch(i){let s=i instanceof Error?i.message:String(i);throw await g(n,"error",{message:s}),await P(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:s}}),i}finally{await de(u);}}}async function Se(e,t,r){let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json");await J(e.runtimeDir,async()=>{let a=await h(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],s=L(t),o=we(t,r);o?s.resumeFromStep=o:delete s.resumeFromStep,await w(n,[s,...i]),await w(u,null);});}async function be(e,t){let r=l.join(e.runtimeDir,"queue.json"),n=l.join(e.runtimeDir,"current-run.json"),u=l.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await J(e.runtimeDir,async()=>{let m=await h(r,[]);if(!Array.isArray(m)||m.length===0)return null;let d={...m[0],status:"running",startedAt:a};return await w(r,m.slice(1)),await w(n,d),d});if(!i)return "none";await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await g(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let s=await ye(i,e,t),o=new Date().toISOString();if(s.stopped){let m=await fe(e,s.stoppedStep||i.command),d={...i,status:"stopped",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{},stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopMode:s.stopRequest?.mode||"requeue",stoppedStep:m};await Se(e,i,m);try{let f=await Q(d,e);d.logArchiveDir=f.archiveDir,d.logArchiveCreatedAt=f.archiveCreatedAt,d.logArchiveFileCount=f.archivedFiles.length;}catch(f){d.logArchiveError=f instanceof Error?f.message:String(f);}return await w(u,d),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job stop requested and requeued.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:s.stopRequest?.mode||"requeue",stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveDir:d.logArchiveDir||"",logArchiveCreatedAt:d.logArchiveCreatedAt||"",logArchiveFileCount:d.logArchiveFileCount??0,logArchiveError:d.logArchiveError||""}}),"stopped"}let c={...i,status:s.exitCode===0?"success":"failed",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{}};await w(n,null),await E(e.runtimeDir,i.id);try{let m=await Q(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await w(u,c);let y=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await P(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:y,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Ae(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function Pe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Ae(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,s=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let c=await o.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(s);}}var Ie={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:Pe},U=N(Ie),Re=U,Me=U;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Re().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
-
export{
|
|
1
|
+
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import g from'fs/promises';import {existsSync,createReadStream,createWriteStream}from'fs';import d from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';import {pipeline}from'stream/promises';import {createGzip}from'zlib';var J=class extends Error{request;step;constructor(t,r){super(`Stop requested during ${r}.`),this.name="StopRequestedError",this.request=t,this.step=r;}},V="queue-mutation.lock",Y=5e3,Z=3e4;function ee(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",r=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),n=process.env.PUBLISHER_CONFIG||"",o=1;for(let u=0;u<e.length;u++){let s=e[u];s==="--runtime-dir"?t=e[++u]||"":s.startsWith("--runtime-dir=")?t=s.slice(14):s==="--exporter-dir"?r=e[++u]||r:s.startsWith("--exporter-dir=")?r=s.slice(15):s==="--config"?n=e[++u]||"":s.startsWith("--config=")?n=s.slice(9):s==="--max-jobs"?o=Number.parseInt(e[++u]||"1",10):s.startsWith("--max-jobs=")&&(o=Number.parseInt(s.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let a=d.resolve(t),i=n?d.resolve(n):d.join(a,"config.json");return {runtimeDir:a,exporterDir:d.resolve(r),configPath:i,maxJobs:Number.isFinite(o)&&o>0?o:1}}async function S(e,t){try{let r=await g.readFile(e,"utf8");return JSON.parse(r)}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="ENOENT"){let o=r instanceof Error?r.message:String(r);console.warn(`[queue-runner] failed to read JSON ${e}: ${o}`);}return t}}async function y(e,t){await g.mkdir(d.dirname(e),{recursive:true}),await g.writeFile(e,JSON.stringify(t,null,2),"utf8");}function B(e){return d.join(e,V)}async function te(e){await new Promise(t=>setTimeout(t,e));}async function re(e){let t=B(e),r=Date.now()+Y;for(;;)try{await g.mkdir(d.dirname(t),{recursive:!0}),await g.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="EEXIST")throw n;let a=await g.stat(t).catch(()=>null);if(a&&Date.now()-a.mtimeMs>Z){await g.unlink(t).catch(()=>{});continue}if(Date.now()>=r)throw new Error("Timed out acquiring queue mutation lock.",{cause:n});await te(50);}}async function ne(e){await g.unlink(B(e)).catch(()=>{});}async function R(e,t){await re(e);try{return await t()}finally{await ne(e);}}function ie(e){return d.join(e,"queue-runner-heartbeat.json")}function x(e){return d.join(e,"current-progress.json")}function T(e){return d.join(e,"scheduler-state.json")}function oe(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let o=n.split("/").map(a=>a.trim()).filter(a=>a.length>0&&a!=="."&&a!=="..");return o.length>0?o.join("/"):t}async function L(e){let t=await S(e.configPath||"",null),r=d.resolve(e.runtimeDir,".."),o=(typeof t?.logDir=="string"?t.logDir:"").trim();return o&&d.isAbsolute(o)?d.resolve(o):d.resolve(r,oe(o,"logs"))}function C(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function se(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function ue(e){return [se(e.endedAt||e.startedAt),C(e.command||"job","job"),C(e.id||"job","job"),C(e.status||"finished","finished")].join("-")}function ae(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}function ce(e){return e==="errors.json"||e==="errors.jsonl"||e.endsWith(".errors.jsonl")?"errors":e==="ignored.json"||e==="ignored.jsonl"?"ignored":e==="rejected.json"||e==="rejected.jsonl"?"rejected":e==="skipped-http.json"||e==="skipped-http.jsonl"?"skipped-http":e==="timings.json"||e==="timings.jsonl"?"timings":e==="current-progress.json"?"current-progress":e==="current-crawl-event.json"?"current-crawl-event":e.endsWith(".log.jsonl")?e.replace(/\.log\.jsonl$/i,"-log"):e.replace(/\.[^.]+$/u,"")||"artifact"}function de(e){return e.endsWith(".gz")?"application/gzip":e.endsWith(".jsonl")?"application/x-ndjson":e.endsWith(".json")?"application/json":"text/plain"}async function k(e,t,r){let n=`${r}.gz`,o=d.join(t,n);await pipeline(createReadStream(e),createGzip({level:9}),createWriteStream(o));let[a,i]=await Promise.all([g.stat(e),g.stat(o)]);return {role:ce(r),originalFileName:r,storedFileName:n,compressed:true,compression:"gzip",contentType:de(n),originalSize:a.size,storedSize:i.size}}async function M(e,t){let r=await L(t),n=new Date().toISOString(),o=ue(e),a=d.join(r,"archive",o),i=[],u=[];await g.mkdir(a,{recursive:true});try{let p=await g.readdir(r,{withFileTypes:!0});for(let m of p){if(!m.isFile()||!ae(m.name))continue;let l=await k(d.join(r,m.name),a,m.name);u.push(l),i.push(l.storedFileName);}}catch(p){if((p&&typeof p=="object"&&"code"in p?String(p.code||""):"")!=="ENOENT")throw p}let s=x(t.runtimeDir);if(existsSync(s)){let p=await k(s,a,"current-progress.json");u.push(p),i.push(p.storedFileName);}let c=[...i,"job.json"];return await y(d.join(a,"job.json"),{manifestVersion:1,archivedAt:n,archiveKey:o,archiveDir:a,logDir:r,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:c,artifacts:u,job:e}),{archiveKey:o,archiveDir:a,archiveCreatedAt:n,archivedFiles:c,artifacts:u}}function le(e){return d.join(e,"audit-events.jsonl")}function D(e){return d.join(e,"stop-request.json")}async function O(e){try{let t=await g.readFile(D(e),"utf8"),r=JSON.parse(t);return r&&typeof r=="object"?r:null}catch{return null}}async function Q(e,t){let r=await O(e);if(!r)return null;let n=String(r.targetJobId||"").trim();return n&&n!==t?null:r}async function E(e,t){if(!t){await g.unlink(D(e)).catch(()=>{});return}let r=await O(e);if(!r)return;let n=String(r.targetJobId||"").trim();(!n||n===t)&&await g.unlink(D(e)).catch(()=>{});}async function P(e,t){try{let r={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await g.mkdir(e,{recursive:!0}),await g.appendFile(le(e),`${JSON.stringify(r)}
|
|
2
|
+
`,"utf8");}catch{}}async function w(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await y(ie(e.runtimeDir),n);}catch{}}async function I(e){try{await g.unlink(x(e));}catch{}}function me(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let o=(e.id||`${r}-${t+1}`).trim();if(!o)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",s=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:o,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:s}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function pe(e){let t=await S(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((o,a)=>me(o,a)).filter(o=>!!o):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ge(e,t){let r=(t.url||"").trim(),n=t.crawlMode||"full",o=(t.deploymentProfile||"").trim();return e.some(a=>a.command===t.command&&(a.url||"").trim()===r&&(a.crawlMode||"full")===n&&(a.deploymentProfile||"").trim()===o&&(a.status===void 0||a.status==="queued"||a.status==="running"))}async function fe(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await pe(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(s){let c=s instanceof Error?s.message:String(s);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=d.join(e.runtimeDir,"queue.json"),o=d.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,u=[];await R(e.runtimeDir,async()=>{let s=await S(n,[]),c=await S(o,null),p=await S(T(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...s];c&&c.status==="running"&&m.unshift(c);for(let l of r.rules){if(!l.enabled)continue;let f=l.intervalMinutes*60*1e3,h=Math.floor(a/f),A=p.lastEnqueuedBucketByRuleId[l.id]??-1;if(h<=A||ge(m,{command:l.command,url:l.url,crawlMode:l.crawlMode,deploymentProfile:l.deploymentProfile}))continue;let q={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};s.push(q),m.push(q),u.push({job:q,rule:l}),p.lastEnqueuedBucketByRuleId[l.id]=h,i+=1;}i>0&&(await y(n,s),await y(T(e.runtimeDir),p));});for(let s of u)await P(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:s.job.id,command:s.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:s.rule.id,intervalMinutes:s.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:s.rule.deploymentProfile||"",url:s.rule.url||""}});return i}async function he(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await g.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function we(e){try{await g.unlink(e);}catch{}}function ye(e,t){let r=[d.join(e,"dist",`${t}.js`),d.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${d.join(e,"dist")} or ${e}`)}async function b(e,t,r,n,o,a,i){if(i){let c=await Q(t,i);if(c)throw new J(c,r)}let u=ye(e,r),s={...process.env,...a??{}};return s.STATIC_PUBLISHER_RUNTIME_DIR=t,o&&(s.PUBLISHER_CONFIG=o),await new Promise((c,p)=>{let m=null,l=null,f=null,h=false,A=spawn(process.execPath,[u,...n],{cwd:e,env:s,stdio:"inherit"}),q=()=>{l&&clearInterval(l),f&&clearTimeout(f);},$=async()=>{if(!(!i||m||h)){h=true;try{let v=await Q(t,i);if(!v)return;m=v,A.kill("SIGTERM"),f=setTimeout(()=>{A.kill("SIGKILL");},1e4);}finally{h=false;}}};i&&(l=setInterval(()=>{$();},1e3)),A.on("error",v=>{q(),p(v);}),A.on("close",(v,j)=>{if(q(),m){p(new J(m,r));return}v===0?c():p(new Error(`${r} exited with code ${v??-1}${j?` (signal ${j})`:""}`));});}),0}function Se(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function be(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let o=(await S(t.configPath||"",null))?.wpsuite;return o&&(o.accountId||o.siteId||o.siteKey)?o:null}async function ve(e,t){return await be(e,t)}function U(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function Ae(e,t){let r=await S(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let o=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(o)return o;let a=await L(e).catch(()=>"");if(a){let i=await S(d.join(a,"current-crawl-event.json"),null),u=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(u)return u}return t}function qe(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function Pe(e){let t=d.join(e.runtimeDir,"queue.json"),r=d.join(e.runtimeDir,"current-run.json"),n=await R(e.runtimeDir,async()=>{let o=await S(r,null);if(!o||o.status!=="running"&&o.status!=="queued")return null;let a=await S(t,[]),i=Array.isArray(a)?a.filter(s=>s?.id!==o.id):[],u=U(o);return await y(t,[u,...i]),await y(r,null),u});n&&(await E(e.runtimeDir,n.id),await P(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function Ie(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let u=await ve(e,t);await r.assertActiveSubscriptionForIdentity(u);}let o=Se(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await b(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,o,e.id),await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await b(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await b(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="crawl")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await b(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,o,e.id),{exitCode:0};if(e.command==="deploy")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await b(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="invalidate")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await b(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,o,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await b(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,o,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await b(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,o,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof J?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function W(e){return async function(r=process.argv.slice(2)){let n=ee(r),o=d.join(n.runtimeDir,"export.lock"),a=d.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await w(n,"starting",{message:"queue-runner starting"});try{await he(o);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await w(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await Pe(n);let i=await fe(n,e),u=0;for(let s=0;s<n.maxJobs;s++){let c=await Re(n,e);if(c==="none"||(u+=1,c==="stopped"))break}if(u===0)await I(n.runtimeDir),await w(n,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let s=await S(a,null),c=String(s?.status||"").trim();await w(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:s?.id,lastJobCommand:s?.command,lastJobStatus:s?.status,lastJobExitCode:s?.exitCode,lastJobError:s?.error,...c==="stopped"?{currentStep:s?.stoppedStep,stopRequestedAt:s?.stopRequestedAt,stopRequestedByLogin:s?.stopRequestedByLogin,stopRequestedMode:s?.stopMode,lastStoppedStep:s?.stoppedStep,message:s?.stopMode==="requeue"?`Job stopped during ${s?.stoppedStep||s?.command||"active step"} and requeued.`:`Job stopped during ${s?.stoppedStep||s?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await w(n,"error",{message:u}),await P(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await we(o);}}}async function Je(e,t,r){let n=d.join(e.runtimeDir,"queue.json"),o=d.join(e.runtimeDir,"current-run.json");await R(e.runtimeDir,async()=>{let a=await S(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],u=U(t),s=qe(t,r);s?u.resumeFromStep=s:delete u.resumeFromStep,await y(n,[u,...i]),await y(o,null);});}async function Re(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json"),o=d.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await R(e.runtimeDir,async()=>{let m=await S(r,[]);if(!Array.isArray(m)||m.length===0)return null;let l={...m[0],status:"running",startedAt:a};return await y(r,m.slice(1)),await y(n,l),l});if(!i)return "none";await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await w(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await Ie(i,e,t),s=new Date().toISOString();if(u.stopped){let m=await Ae(e,u.stoppedStep||i.command),l=u.stopRequest?.mode==="requeue"?"requeue":"stop",f={...i,status:"stopped",endedAt:s,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:l,stoppedStep:m};l==="requeue"?await Je(e,i,m):await y(n,null);try{let h=await M(f,e);f.logArchiveDir=h.archiveDir,f.logArchiveCreatedAt=h.archiveCreatedAt,f.logArchiveFileCount=h.archivedFiles.length;}catch(h){f.logArchiveError=h instanceof Error?h.message:String(h);}return await y(o,f),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:l==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:s,deploymentProfile:i.deploymentProfile||"",stopMode:l,stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveKey:f.logArchiveError||!f.logArchiveDir?"":d.basename(f.logArchiveDir),logArchiveDir:f.logArchiveDir||"",logArchiveCreatedAt:f.logArchiveCreatedAt||"",logArchiveFileCount:f.logArchiveFileCount??0,logArchiveError:f.logArchiveError||""}}),"stopped"}let c={...i,status:u.exitCode===0?"success":"failed",endedAt:s,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await y(n,null),await E(e.runtimeDir,i.id);try{let m=await M(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await y(o,c);let p=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await P(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:p,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveKey:c.logArchiveError||!c.logArchiveDir?"":d.basename(c.logArchiveDir),logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Ee(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function xe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Ee(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,u=setTimeout(()=>i.abort(),12e3);try{let s=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!s.ok)throw new Error(`Subscription check failed with HTTP ${s.status}`);let c=await s.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(u);}}var je={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:xe},_=W(je),Te=_,Ke=_;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Te().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
+
export{Ke as default,Te as main,_ as runQueueRunner};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.5",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|
|
@@ -33,6 +33,7 @@
|
|
|
33
33
|
"crawl:urls": "node dist/crawl.js --urls",
|
|
34
34
|
"deploy": "node dist/deploy.js",
|
|
35
35
|
"invalidate": "node dist/invalidate.js",
|
|
36
|
+
"logs:prune": "node dist/prune-logs.js",
|
|
36
37
|
"queue:run": "node dist/queue-runner.js",
|
|
37
38
|
"queue:drain": "node dist/queue-runner.js --max-jobs=100",
|
|
38
39
|
"queue:cron": "bash ./scripts/queue-runner-cron.sh",
|