@smart-cloud/publisher-exporter 1.0.3 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/queue-runner.js +3 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -59,6 +59,15 @@ publisher-exporter queue-runner \
|
|
|
59
59
|
|
|
60
60
|
Direct CLI invocation from cron is the recommended setup.
|
|
61
61
|
|
|
62
|
+
When the runtime config contains enabled scheduler rules and the active queue-runner policy allows scheduler auto-enqueue, queue-runner evaluates those rules once at startup before draining queued jobs.
|
|
63
|
+
|
|
64
|
+
- Scheduler rules are read from `runtime/config.json` and their last interval buckets are persisted in `runtime/scheduler-state.json`.
|
|
65
|
+
- Scheduler only auto-enqueues jobs into the runtime queue. It does not replace cron, systemd timers, or Windows Task Scheduler.
|
|
66
|
+
- A 1-minute external runner tick is the recommended cadence.
|
|
67
|
+
- Supported scheduled commands are `publish`, `crawl`, `deploy`, `invalidate`, `retry-timeouts`, and `url`.
|
|
68
|
+
- The scheduler timezone value is currently informational for operations context; interval matching is based on elapsed minute buckets checked at each queue-runner start.
|
|
69
|
+
- If an equivalent queued or running job already exists for the same command, crawl mode, deployment profile, and URL, that rule is skipped for the current interval bucket.
|
|
70
|
+
|
|
62
71
|
Same-host Linux cron example:
|
|
63
72
|
|
|
64
73
|
```cron
|
package/dist/queue-runner.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import p from'fs/promises';import {existsSync}from'fs';import
|
|
2
|
-
`,"utf8");}catch{}}async function g(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await w(V(e.runtimeDir),n);}catch{}}async function I(e){try{await p.unlink(x(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let u=(e.id||`${r}-${t+1}`).trim();if(!u)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",s=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:u,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&s?{deploymentProfile:s}:{},...i?{url:i}:{}}}async function se(e){let t=await h(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((u,a)=>oe(u,a)).filter(u=>!!u):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t,r){return e.some(n=>n.command===t&&(n.url||"").trim()===(r||"").trim()&&(n.status===void 0||n.status==="queued"||n.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let c=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,s=[];await J(e.runtimeDir,async()=>{let o=await h(n,[]),c=await h(u,null),y=await h(k(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];c&&c.status==="running"&&m.unshift(c);for(let d of r.rules){if(!d.enabled)continue;let f=d.intervalMinutes*60*1e3,v=Math.floor(a/f),q=y.lastEnqueuedBucketByRuleId[d.id]??-1;if(v<=q||ue(m,d.command,d.url))continue;let A={id:randomUUID(),command:d.command,...(d.command==="publish"||d.command==="crawl")&&d.crawlMode?{crawlMode:d.crawlMode}:{},...(d.command==="publish"||d.command==="deploy"||d.command==="invalidate")&&d.deploymentProfile?{deploymentProfile:d.deploymentProfile}:{},...d.url?{url:d.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(A),m.push(A),s.push({job:A,rule:d}),y.lastEnqueuedBucketByRuleId[d.id]=v,i+=1;}i>0&&(await w(n,o),await w(k(e.runtimeDir),y));});for(let o of s)await P(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await p.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await p.unlink(e);}catch{}}function le(e,t){let r=[l.join(e,"dist",`${t}.js`),l.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${l.join(e,"dist")} or ${e}`)}async function S(e,t,r,n,u,a,i){if(i){let c=await j(t,i);if(c)throw new R(c,r)}let s=le(e,r),o={...process.env,...a??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,u&&(o.PUBLISHER_CONFIG=u),await new Promise((c,y)=>{let m=null,d=null,f=null,v=false,q=spawn(process.execPath,[s,...n],{cwd:e,env:o,stdio:"inherit"}),A=()=>{d&&clearInterval(d),f&&clearTimeout(f);},_=async()=>{if(!(!i||m||v)){v=true;try{let b=await j(t,i);if(!b)return;m=b,q.kill("SIGTERM"),f=setTimeout(()=>{q.kill("SIGKILL");},1e4);}finally{v=false;}}};i&&(d=setInterval(()=>{_();},1e3)),q.on("error",b=>{A(),y(b);}),q.on("close",(b,T)=>{if(A(),m){y(new R(m,r));return}b===0?c():y(new Error(`${r} exited with code ${b??-1}${T?` (signal ${T})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function pe(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let u=(await h(t.configPath||"",null))?.wpsuite;return u&&(u.accountId||u.siteId||u.siteKey)?u:null}async function ge(e,t){return await pe(e,t)}function L(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function fe(e,t){let r=await h(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let u=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(u)return u;let a=await B(e).catch(()=>"");if(a){let i=await h(l.join(a,"current-crawl-event.json"),null),s=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(s)return s}return t}function we(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function he(e){let t=l.join(e.runtimeDir,"queue.json"),r=l.join(e.runtimeDir,"current-run.json"),n=await J(e.runtimeDir,async()=>{let u=await h(r,null);if(!u||u.status!=="running"&&u.status!=="queued")return null;let a=await h(t,[]),i=Array.isArray(a)?a.filter(o=>o?.id!==u.id):[],s=L(u);return await w(t,[s,...i]),await w(r,null),s});n&&(await E(e.runtimeDir,n.id),await P(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function ye(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let s=await ge(e,t);await r.assertActiveSubscriptionForIdentity(s);}let u=me(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="crawl")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await S(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,u,e.id),{exitCode:0};if(e.command==="deploy")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await S(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="invalidate")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await S(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,u,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,u,e.id),{exitCode:0};if(e.command==="url"){let s=(e.url||"").trim();return s?(await g(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await S(t.exporterDir,t.runtimeDir,"crawl",["--url",s],t.configPath,u,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof R?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function N(e){return async function(r=process.argv.slice(2)){let n=G(r),u=l.join(n.runtimeDir,"export.lock"),a=l.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await g(n,"starting",{message:"queue-runner starting"});try{await ce(u);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await g(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await he(n);let i=await ae(n,e),s=0;for(let o=0;o<n.maxJobs;o++){let c=await be(n,e);if(c==="none"||(s+=1,c==="stopped"))break}if(s===0)await I(n.runtimeDir),await g(n,"idle",{processedJobs:s,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await h(a,null),c=String(o?.status||"").trim();await g(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:s,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...c==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`}:{}});}}catch(i){let s=i instanceof Error?i.message:String(i);throw await g(n,"error",{message:s}),await P(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:s}}),i}finally{await de(u);}}}async function Se(e,t,r){let n=l.join(e.runtimeDir,"queue.json"),u=l.join(e.runtimeDir,"current-run.json");await J(e.runtimeDir,async()=>{let a=await h(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],s=L(t),o=we(t,r);o?s.resumeFromStep=o:delete s.resumeFromStep,await w(n,[s,...i]),await w(u,null);});}async function be(e,t){let r=l.join(e.runtimeDir,"queue.json"),n=l.join(e.runtimeDir,"current-run.json"),u=l.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await J(e.runtimeDir,async()=>{let m=await h(r,[]);if(!Array.isArray(m)||m.length===0)return null;let d={...m[0],status:"running",startedAt:a};return await w(r,m.slice(1)),await w(n,d),d});if(!i)return "none";await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await g(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let s=await ye(i,e,t),o=new Date().toISOString();if(s.stopped){let m=await fe(e,s.stoppedStep||i.command),d={...i,status:"stopped",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{},stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopMode:s.stopRequest?.mode||"requeue",stoppedStep:m};await Se(e,i,m);try{let f=await Q(d,e);d.logArchiveDir=f.archiveDir,d.logArchiveCreatedAt=f.archiveCreatedAt,d.logArchiveFileCount=f.archivedFiles.length;}catch(f){d.logArchiveError=f instanceof Error?f.message:String(f);}return await w(u,d),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await P(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job stop requested and requeued.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:s.stopRequest?.mode||"requeue",stopRequestedAt:s.stopRequest?.requestedAt||"",stopRequestedByLogin:s.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof s.stopRequest?.requestedByUserId=="number"?s.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveDir:d.logArchiveDir||"",logArchiveCreatedAt:d.logArchiveCreatedAt||"",logArchiveFileCount:d.logArchiveFileCount??0,logArchiveError:d.logArchiveError||""}}),"stopped"}let c={...i,status:s.exitCode===0?"success":"failed",endedAt:o,exitCode:s.exitCode,...s.error?{error:s.error}:{}};await w(n,null),await E(e.runtimeDir,i.id);try{let m=await Q(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await w(u,c);let y=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await P(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:y,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Ae(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function Pe(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Ae(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,s=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let c=await o.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(s);}}var Ie={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:Pe},U=N(Ie),Re=U,Me=U;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Re().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
-
export{
|
|
1
|
+
import {pathToFileURL}from'url';import {PLUGIN_KEY}from'@smart-cloud/publisher-core/constants';import p from'fs/promises';import {existsSync}from'fs';import d from'path';import {spawn}from'child_process';import {randomUUID}from'crypto';var J=class extends Error{request;step;constructor(t,r){super(`Stop requested during ${r}.`),this.name="StopRequestedError",this.request=t,this.step=r;}},K="queue-mutation.lock",H=5e3,z=3e4;function G(e){let t=process.env.STATIC_PUBLISHER_RUNTIME_DIR||process.env.WPSUITE_STATIC_PUBLISHER_RUNTIME_DIR||"",r=process.env.STATIC_PUBLISHER_EXPORTER_DIR||process.cwd(),n=process.env.PUBLISHER_CONFIG||"",s=1;for(let u=0;u<e.length;u++){let o=e[u];o==="--runtime-dir"?t=e[++u]||"":o.startsWith("--runtime-dir=")?t=o.slice(14):o==="--exporter-dir"?r=e[++u]||r:o.startsWith("--exporter-dir=")?r=o.slice(15):o==="--config"?n=e[++u]||"":o.startsWith("--config=")?n=o.slice(9):o==="--max-jobs"?s=Number.parseInt(e[++u]||"1",10):o.startsWith("--max-jobs=")&&(s=Number.parseInt(o.slice(11),10));}if(!t)throw new Error("Missing runtime dir. Use --runtime-dir or STATIC_PUBLISHER_RUNTIME_DIR.");let a=d.resolve(t),i=n?d.resolve(n):d.join(a,"config.json");return {runtimeDir:a,exporterDir:d.resolve(r),configPath:i,maxJobs:Number.isFinite(s)&&s>0?s:1}}async function y(e,t){try{let r=await p.readFile(e,"utf8");return JSON.parse(r)}catch(r){if((r&&typeof r=="object"&&"code"in r?String(r.code||""):"")!=="ENOENT"){let s=r instanceof Error?r.message:String(r);console.warn(`[queue-runner] failed to read JSON ${e}: ${s}`);}return t}}async function h(e,t){await p.mkdir(d.dirname(e),{recursive:true}),await p.writeFile(e,JSON.stringify(t,null,2),"utf8");}function F(e){return d.join(e,K)}async function X(e){await new Promise(t=>setTimeout(t,e));}async function Y(e){let t=F(e),r=Date.now()+H;for(;;)try{await p.mkdir(d.dirname(t),{recursive:!0}),await p.writeFile(t,JSON.stringify({pid:process.pid,createdAt:new Date().toISOString()},null,2),{encoding:"utf8",flag:"wx"});return}catch(n){if((n&&typeof n=="object"&&"code"in n?String(n.code||""):"")!=="EEXIST")throw n;let a=await p.stat(t).catch(()=>null);if(a&&Date.now()-a.mtimeMs>z){await p.unlink(t).catch(()=>{});continue}if(Date.now()>=r)throw new Error("Timed out acquiring queue mutation lock.",{cause:n});await X(50);}}async function Z(e){await p.unlink(F(e)).catch(()=>{});}async function R(e,t){await Y(e);try{return await t()}finally{await Z(e);}}function V(e){return d.join(e,"queue-runner-heartbeat.json")}function x(e){return d.join(e,"current-progress.json")}function k(e){return d.join(e,"scheduler-state.json")}function ee(e,t){let n=String(e||"").replace(/\\/g,"/").trim().replace(/^\/+|\/+$/g,"");if(!n)return t;let s=n.split("/").map(a=>a.trim()).filter(a=>a.length>0&&a!=="."&&a!=="..");return s.length>0?s.join("/"):t}async function B(e){let t=await y(e.configPath||"",null),r=d.resolve(e.runtimeDir,".."),s=(typeof t?.logDir=="string"?t.logDir:"").trim();return s&&d.isAbsolute(s)?d.resolve(s):d.resolve(r,ee(s,"logs"))}function C(e,t){return String(e||"").trim().replace(/[^a-zA-Z0-9._-]+/g,"-").replace(/^-+|-+$/g,"")||t}function te(e){let t=e?new Date(e):new Date;return (Number.isNaN(t.getTime())?new Date:t).toISOString().replace(/[-:]/g,"").replace(/\.\d{3}Z$/,"Z")}function re(e){return [te(e.endedAt||e.startedAt),C(e.command||"job","job"),C(e.id||"job","job"),C(e.status||"finished","finished")].join("-")}function ne(e){return e.endsWith(".log.jsonl")||e.endsWith(".errors.jsonl")?true:["current-crawl-event.json","rejected.jsonl","ignored.jsonl","skipped-http.jsonl","errors.jsonl","timings.jsonl","rejected.json","ignored.json","skipped-http.json","errors.json","timings.json"].includes(e)}async function M(e,t){let r=await B(t),n=new Date().toISOString(),s=d.join(r,"archive",re(e)),a=[];await p.mkdir(s,{recursive:true});try{let u=await p.readdir(r,{withFileTypes:!0});for(let o of u)!o.isFile()||!ne(o.name)||(await p.copyFile(d.join(r,o.name),d.join(s,o.name)),a.push(o.name));}catch(u){if((u&&typeof u=="object"&&"code"in u?String(u.code||""):"")!=="ENOENT")throw u}let i=x(t.runtimeDir);return existsSync(i)&&(await p.copyFile(i,d.join(s,"current-progress.json")),a.push("current-progress.json")),await h(d.join(s,"job.json"),{archivedAt:n,archiveDir:s,logDir:r,runtimeDir:t.runtimeDir,exporterDir:t.exporterDir,configPath:t.configPath||"",archivedFiles:a,job:e}),a.push("job.json"),{archiveDir:s,archiveCreatedAt:n,archivedFiles:a}}function ie(e){return d.join(e,"audit-events.jsonl")}function D(e){return d.join(e,"stop-request.json")}async function O(e){try{let t=await p.readFile(D(e),"utf8"),r=JSON.parse(t);return r&&typeof r=="object"?r:null}catch{return null}}async function Q(e,t){let r=await O(e);if(!r)return null;let n=String(r.targetJobId||"").trim();return n&&n!==t?null:r}async function E(e,t){if(!t){await p.unlink(D(e)).catch(()=>{});return}let r=await O(e);if(!r)return;let n=String(r.targetJobId||"").trim();(!n||n===t)&&await p.unlink(D(e)).catch(()=>{});}async function A(e,t){try{let r={occurredAt:t.occurredAt||new Date().toISOString(),status:t.status||"info",actorSource:t.actorSource||"queue-runner",...t};await p.mkdir(e,{recursive:!0}),await p.appendFile(ie(e),`${JSON.stringify(r)}
|
|
2
|
+
`,"utf8");}catch{}}async function w(e,t,r){let n={checkedAt:new Date().toISOString(),status:t,pid:process.pid,nodePath:process.execPath,nodeVersion:process.version,runtimeDir:e.runtimeDir,exporterDir:e.exporterDir,...r??{}};try{await h(V(e.runtimeDir),n);}catch{}}async function I(e){try{await p.unlink(x(e));}catch{}}function oe(e,t){let r=e.command;if(r!=="publish"&&r!=="crawl"&&r!=="deploy"&&r!=="invalidate"&&r!=="retry-timeouts"&&r!=="url")return null;let n=Number.parseInt(String(e.intervalMinutes??"0"),10);if(!Number.isFinite(n)||n<1)return null;let s=(e.id||`${r}-${t+1}`).trim();if(!s)return null;let a=e.enabled!==false,i=typeof e.url=="string"?e.url.trim():"",u=typeof e.deploymentProfile=="string"?e.deploymentProfile.trim():"",o=(r==="publish"||r==="crawl")&&e.crawlMode==="incremental"?"incremental":"full";return r==="url"&&!i?null:{id:s,enabled:a,command:r,intervalMinutes:n,...r==="publish"||r==="crawl"?{crawlMode:o}:{},...(r==="publish"||r==="deploy"||r==="invalidate")&&u?{deploymentProfile:u}:{},...i?{url:i}:{}}}async function se(e){let t=await y(e.configPath||"",null),r=t?.scheduler,n=Array.isArray(r?.rules)?r.rules.map((s,a)=>oe(s,a)).filter(s=>!!s):[];return {enabled:!!r?.enabled,timezone:typeof r?.timezone=="string"&&r.timezone.trim()?r.timezone.trim():"UTC",rules:n,wpsuite:t?.wpsuite}}function ue(e,t){let r=(t.url||"").trim(),n=t.crawlMode||"full",s=(t.deploymentProfile||"").trim();return e.some(a=>a.command===t.command&&(a.url||"").trim()===r&&(a.crawlMode||"full")===n&&(a.deploymentProfile||"").trim()===s&&(a.status===void 0||a.status==="queued"||a.status==="running"))}async function ae(e,t){if(!t.allowSchedulerAutoEnqueue)return 0;let r=await se(e);if(!r.enabled||r.rules.length===0)return 0;if(t.assertActiveSubscriptionForIdentity)try{await t.assertActiveSubscriptionForIdentity(r.wpsuite??null);}catch(o){let c=o instanceof Error?o.message:String(o);return console.warn(`[queue-runner] scheduler auto-enqueue skipped: ${c}`),0}let n=d.join(e.runtimeDir,"queue.json"),s=d.join(e.runtimeDir,"current-run.json"),a=Date.now(),i=0,u=[];await R(e.runtimeDir,async()=>{let o=await y(n,[]),c=await y(s,null),S=await y(k(e.runtimeDir),{lastEnqueuedBucketByRuleId:{}}),m=[...o];c&&c.status==="running"&&m.unshift(c);for(let l of r.rules){if(!l.enabled)continue;let g=l.intervalMinutes*60*1e3,f=Math.floor(a/g),q=S.lastEnqueuedBucketByRuleId[l.id]??-1;if(f<=q||ue(m,{command:l.command,url:l.url,crawlMode:l.crawlMode,deploymentProfile:l.deploymentProfile}))continue;let P={id:randomUUID(),command:l.command,...(l.command==="publish"||l.command==="crawl")&&l.crawlMode?{crawlMode:l.crawlMode}:{},...(l.command==="publish"||l.command==="deploy"||l.command==="invalidate")&&l.deploymentProfile?{deploymentProfile:l.deploymentProfile}:{},...l.url?{url:l.url}:{},enqueueSource:"scheduler",...r.wpsuite?{wpsuite:r.wpsuite}:{},status:"queued",createdAt:new Date().toISOString(),createdBy:0};o.push(P),m.push(P),u.push({job:P,rule:l}),S.lastEnqueuedBucketByRuleId[l.id]=f,i+=1;}i>0&&(await h(n,o),await h(k(e.runtimeDir),S));});for(let o of u)await A(e.runtimeDir,{eventType:"job-created",status:"queued",actorSource:"queue-runner-scheduler",jobId:o.job.id,command:o.job.command,message:"Scheduler auto-enqueued a job.",details:{ruleId:o.rule.id,intervalMinutes:o.rule.intervalMinutes,timezone:r.timezone,deploymentProfile:o.rule.deploymentProfile||"",url:o.rule.url||""}});return i}async function ce(e){let t={pid:process.pid,startedAt:new Date().toISOString()};await p.writeFile(e,JSON.stringify(t,null,2),{encoding:"utf8",flag:"wx"});}async function de(e){try{await p.unlink(e);}catch{}}function le(e,t){let r=[d.join(e,"dist",`${t}.js`),d.join(e,`${t}.js`)];for(let n of r)if(existsSync(n))return n;throw new Error(`Cannot find ${t}.js in ${d.join(e,"dist")} or ${e}`)}async function b(e,t,r,n,s,a,i){if(i){let c=await Q(t,i);if(c)throw new J(c,r)}let u=le(e,r),o={...process.env,...a??{}};return o.STATIC_PUBLISHER_RUNTIME_DIR=t,s&&(o.PUBLISHER_CONFIG=s),await new Promise((c,S)=>{let m=null,l=null,g=null,f=false,q=spawn(process.execPath,[u,...n],{cwd:e,env:o,stdio:"inherit"}),P=()=>{l&&clearInterval(l),g&&clearTimeout(g);},_=async()=>{if(!(!i||m||f)){f=true;try{let v=await Q(t,i);if(!v)return;m=v,q.kill("SIGTERM"),g=setTimeout(()=>{q.kill("SIGKILL");},1e4);}finally{f=false;}}};i&&(l=setInterval(()=>{_();},1e3)),q.on("error",v=>{P(),S(v);}),q.on("close",(v,T)=>{if(P(),m){S(new J(m,r));return}v===0?c():S(new Error(`${r} exited with code ${v??-1}${T?` (signal ${T})`:""}`));});}),0}function me(e){let t=e.awsTempCreds;if(!t)return {};let r={};return typeof t.accessKeyId=="string"&&t.accessKeyId.trim()!==""&&(r.AWS_ACCESS_KEY_ID=t.accessKeyId.trim()),typeof t.secretAccessKey=="string"&&t.secretAccessKey.trim()!==""&&(r.AWS_SECRET_ACCESS_KEY=t.secretAccessKey.trim()),typeof t.sessionToken=="string"&&t.sessionToken.trim()!==""&&(r.AWS_SESSION_TOKEN=t.sessionToken.trim()),r}async function pe(e,t){let r=e.wpsuite;if(r&&(r.accountId||r.siteId||r.siteKey))return r;let s=(await y(t.configPath||"",null))?.wpsuite;return s&&(s.accountId||s.siteId||s.siteKey)?s:null}async function ge(e,t){return await pe(e,t)}function L(e){let t={...e,status:"queued"};return delete t.startedAt,delete t.endedAt,delete t.exitCode,delete t.error,delete t.stopRequestedAt,delete t.stopRequestedByUserId,delete t.stopRequestedByLogin,delete t.stopMode,delete t.stoppedStep,t}async function fe(e,t){let r=await y(x(e.runtimeDir),null),n=r&&r.details&&typeof r.details.phase=="string"?r.details.phase.trim():"";if(n)return n;let s=r&&typeof r.currentStep=="string"?r.currentStep.trim():"";if(s)return s;let a=await B(e).catch(()=>"");if(a){let i=await y(d.join(a,"current-crawl-event.json"),null),u=i&&typeof i.currentStep=="string"?i.currentStep.trim():"";if(u)return u}return t}function we(e,t){if((e.command==="publish"||e.command==="crawl")&&t==="rewrite-text")return "rewrite-text"}async function he(e){let t=d.join(e.runtimeDir,"queue.json"),r=d.join(e.runtimeDir,"current-run.json"),n=await R(e.runtimeDir,async()=>{let s=await y(r,null);if(!s||s.status!=="running"&&s.status!=="queued")return null;let a=await y(t,[]),i=Array.isArray(a)?a.filter(o=>o?.id!==s.id):[],u=L(s);return await h(t,[u,...i]),await h(r,null),u});n&&(await E(e.runtimeDir,n.id),await A(e.runtimeDir,{eventType:"job-recovered",status:"queued",actorSource:"queue-runner",jobId:n.id,command:n.command,message:"Recovered stale current-run entry back into queue."}));}async function ye(e,t,r){try{if((r.shouldEnforceSubscriptionOnJobExecution?.(e)??r.enforceSubscriptionOnJobExecution)&&r.assertActiveSubscriptionForIdentity){let u=await ge(e,t);await r.assertActiveSubscriptionForIdentity(u);}let s=me(e),a=e.resumeFromStep==="rewrite-text"?["--resume-rewrite"]:e.crawlMode==="incremental"?["--crawl-mode","incremental"]:[],i=e.deploymentProfile?["--profile",e.deploymentProfile]:[];if(e.command==="publish")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await b(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,s,e.id),await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await b(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,s,e.id),await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await b(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="crawl")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:e.resumeFromStep==="rewrite-text"?"rewrite-text":"crawl"}),await b(t.exporterDir,t.runtimeDir,"crawl",a,t.configPath,s,e.id),{exitCode:0};if(e.command==="deploy")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"deploy"}),await b(t.exporterDir,t.runtimeDir,"deploy",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="invalidate")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"invalidate"}),await b(t.exporterDir,t.runtimeDir,"invalidate",i,t.configPath,s,e.id),{exitCode:0};if(e.command==="retry-timeouts")return await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"retry-timeouts"}),await b(t.exporterDir,t.runtimeDir,"crawl",["--retry-timeouts"],t.configPath,s,e.id),{exitCode:0};if(e.command==="url"){let u=(e.url||"").trim();return u?(await w(t,"running",{currentJobId:e.id,currentJobCommand:e.command,currentStep:"url"}),await b(t.exporterDir,t.runtimeDir,"crawl",["--url",u],t.configPath,s,e.id),{exitCode:0}):{exitCode:2,error:"Missing url for command 'url'"}}return {exitCode:2,error:`Unsupported command: ${e.command}`}}catch(n){return n instanceof J?{exitCode:130,error:"Job stop requested.",stopped:true,stopRequest:n.request,stoppedStep:n.step}:{exitCode:1,error:n instanceof Error?n.message:String(n)}}}function N(e){return async function(r=process.argv.slice(2)){let n=G(r),s=d.join(n.runtimeDir,"export.lock"),a=d.join(n.runtimeDir,"last-run.json");await I(n.runtimeDir),await w(n,"starting",{message:"queue-runner starting"});try{await ce(s);}catch(i){if((i&&typeof i=="object"&&"code"in i?String(i.code||""):"")!=="EEXIST")throw i;console.log("[queue-runner] lock active, skipping"),await w(n,"lock-active",{message:"lock active, skipped this cron tick"});return}try{await he(n);let i=await ae(n,e),u=0;for(let o=0;o<n.maxJobs;o++){let c=await be(n,e);if(c==="none"||(u+=1,c==="stopped"))break}if(u===0)await I(n.runtimeDir),await w(n,"idle",{processedJobs:u,schedulerEnqueued:i,message:"no queued jobs"});else {await I(n.runtimeDir);let o=await y(a,null),c=String(o?.status||"").trim();await w(n,c==="success"?"job-success":c==="stopped"?"job-stopped":"job-failed",{processedJobs:u,schedulerEnqueued:i,lastJobId:o?.id,lastJobCommand:o?.command,lastJobStatus:o?.status,lastJobExitCode:o?.exitCode,lastJobError:o?.error,...c==="stopped"?{currentStep:o?.stoppedStep,stopRequestedAt:o?.stopRequestedAt,stopRequestedByLogin:o?.stopRequestedByLogin,stopRequestedMode:o?.stopMode,lastStoppedStep:o?.stoppedStep,message:o?.stopMode==="requeue"?`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and requeued.`:`Job stopped during ${o?.stoppedStep||o?.command||"active step"} and left out of queue.`}:{}});}}catch(i){let u=i instanceof Error?i.message:String(i);throw await w(n,"error",{message:u}),await A(n.runtimeDir,{eventType:"queue-runner-error",status:"failed",actorSource:"queue-runner",message:"Unhandled queue-runner error.",details:{error:u}}),i}finally{await de(s);}}}async function Se(e,t,r){let n=d.join(e.runtimeDir,"queue.json"),s=d.join(e.runtimeDir,"current-run.json");await R(e.runtimeDir,async()=>{let a=await y(n,[]),i=Array.isArray(a)?a.filter(c=>c?.id!==t.id):[],u=L(t),o=we(t,r);o?u.resumeFromStep=o:delete u.resumeFromStep,await h(n,[u,...i]),await h(s,null);});}async function be(e,t){let r=d.join(e.runtimeDir,"queue.json"),n=d.join(e.runtimeDir,"current-run.json"),s=d.join(e.runtimeDir,"last-run.json"),a=new Date().toISOString(),i=await R(e.runtimeDir,async()=>{let m=await y(r,[]);if(!Array.isArray(m)||m.length===0)return null;let l={...m[0],status:"running",startedAt:a};return await h(r,m.slice(1)),await h(n,l),l});if(!i)return "none";await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-started",status:"running",actorSource:"queue-runner",jobId:i.id,command:i.command,message:"Job execution started.",details:{createdAt:i.createdAt||"",startedAt:a,createdBy:i.createdBy??null,deploymentProfile:i.deploymentProfile||"",queuedWithTempAwsCreds:!!i.awsTempCreds}}),await w(e,"running",{currentJobId:i.id,currentJobCommand:i.command,currentStep:i.command});let u=await ye(i,e,t),o=new Date().toISOString();if(u.stopped){let m=await fe(e,u.stoppedStep||i.command),l=u.stopRequest?.mode==="requeue"?"requeue":"stop",g={...i,status:"stopped",endedAt:o,exitCode:u.exitCode,...u.error?{error:u.error}:{},stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopMode:l,stoppedStep:m};l==="requeue"?await Se(e,i,m):await h(n,null);try{let f=await M(g,e);g.logArchiveDir=f.archiveDir,g.logArchiveCreatedAt=f.archiveCreatedAt,g.logArchiveFileCount=f.archivedFiles.length;}catch(f){g.logArchiveError=f instanceof Error?f.message:String(f);}return await h(s,g),await E(e.runtimeDir,i.id),await I(e.runtimeDir),await A(e.runtimeDir,{eventType:"job-run-stopped",status:"stopped",actorSource:"queue-runner",jobId:i.id,command:i.command,message:l==="requeue"?"Job stop requested and requeued.":"Job stop requested and removed from active execution without requeue.",details:{startedAt:i.startedAt||"",endedAt:o,deploymentProfile:i.deploymentProfile||"",stopMode:l,stopRequestedAt:u.stopRequest?.requestedAt||"",stopRequestedByLogin:u.stopRequest?.requestedByLogin||"",stopRequestedByUserId:typeof u.stopRequest?.requestedByUserId=="number"?u.stopRequest.requestedByUserId:null,stoppedStep:m,logArchiveDir:g.logArchiveDir||"",logArchiveCreatedAt:g.logArchiveCreatedAt||"",logArchiveFileCount:g.logArchiveFileCount??0,logArchiveError:g.logArchiveError||""}}),"stopped"}let c={...i,status:u.exitCode===0?"success":"failed",endedAt:o,exitCode:u.exitCode,...u.error?{error:u.error}:{}};await h(n,null),await E(e.runtimeDir,i.id);try{let m=await M(c,e);c.logArchiveDir=m.archiveDir,c.logArchiveCreatedAt=m.archiveCreatedAt,c.logArchiveFileCount=m.archivedFiles.length;}catch(m){c.logArchiveError=m instanceof Error?m.message:String(m);}await h(s,c);let S=i.startedAt&&c.endedAt?Math.max(0,Math.round((new Date(c.endedAt).getTime()-new Date(i.startedAt).getTime())/1e3)):void 0;return await A(e.runtimeDir,{eventType:"job-run-finished",status:c.status==="success"?"success":"failed",actorSource:"queue-runner",jobId:c.id,command:c.command,message:c.status==="success"?"Job execution finished successfully.":"Job execution finished with failure.",details:{startedAt:i.startedAt||"",endedAt:c.endedAt||"",durationSec:S,deploymentProfile:c.deploymentProfile||"",exitCode:c.exitCode??null,error:c.error||"",logArchiveDir:c.logArchiveDir||"",logArchiveCreatedAt:c.logArchiveCreatedAt||"",logArchiveFileCount:c.logArchiveFileCount??0,logArchiveError:c.logArchiveError||""}}),"processed"}function Pe(e){let t=(e.apiBase||"").trim();if(t)return t.replace(/\/$/,"");let r=(process.env.WPSUITE_API_BASE_URL||"").trim();return r?r.replace(/\/$/,""):"https://api.wpsuite.io"}async function Ae(e){if(!e)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let t=String(e.accountId||"").trim(),r=String(e.siteId||"").trim(),n=String(e.siteKey||"").trim();if(!t||!r||!n)throw new Error("Missing WPSuite identity (accountId/siteId/siteKey). Link the site before running business logic.");let a=`${Pe(e)}/account/${encodeURIComponent(t)}/site/${encodeURIComponent(r)}/license`,i=new AbortController,u=setTimeout(()=>i.abort(),12e3);try{let o=await fetch(a,{method:"GET",headers:{Accept:"application/json","X-Site-Key":n,"X-Plugin":PLUGIN_KEY},signal:i.signal});if(!o.ok)throw new Error(`Subscription check failed with HTTP ${o.status}`);let c=await o.json().catch(()=>null);if(!c||!c.config||!c.jws)throw new Error("No active subscription payload received from wpsuite API.")}finally{clearTimeout(u);}}var Ie={allowSchedulerAutoEnqueue:true,enforceSubscriptionOnJobExecution:false,shouldEnforceSubscriptionOnJobExecution:e=>e.enqueueSource==="scheduler"||e.createdBy===0||e.crawlMode==="incremental",assertActiveSubscriptionForIdentity:Ae},U=N(Ie),Je=U,je=U;process.argv[1]&&import.meta.url===pathToFileURL(process.argv[1]).href&&Je().catch(e=>{console.error(e),process.exit(1);});
|
|
3
|
+
export{je as default,Je as main,U as runQueueRunner};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smart-cloud/publisher-exporter",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.4",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "Headless Playwright static publisher for WordPress/Elementor sites with sitemap-only page discovery, strict asset capture, escaped URL rewrite, structured logs, and targeted retry modes.",
|