@smart-cloud/publisher-exporter 1.1.60 → 1.1.62

package/README.md

@@ -171,6 +171,85 @@ Archived files are gzip-compressed per file and listed in `job.json`. WordPress
 
 Prune old archive directories with `publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30`. Add that command to daily cron or another retention scheduler that matches your log policy.
 
+## Optional Lambda workers
+
+The exporter includes protocol-v1 building blocks for delegating bounded
+render, text-rewrite, and S3 deploy-copy tasks to container-image Lambda
+functions. Large payloads do not travel in the Lambda response: workers write
+HTML and detailed results below a configured S3 workspace prefix and return a
+small `resultKey` pointer. Reusing the same job/task ID is idempotent.
+
+The companion CDK application is maintained separately at
+[`smartcloudsol/static-publisher-lambda-workers`](https://github.com/smartcloudsol/static-publisher-lambda-workers).
+directory. Its deploy script emits a `remote-workers.json` containing the
+versioned Lambda alias ARNs, workspace bucket/prefix, region, and generated
+assumable role. The file contains no static AWS access keys.
+
+Install that output with owner-only permissions at
+`<site-runtime>/remote-workers.json`, then validate the role, aliases, and
+function runtimes:
+
+```bash
+publisher-exporter remote-worker-health \
+  --runtime-dir /srv/site/runtime
+```
+
+Protocol v1 accepts 1-20 same-origin URLs per render task. Start with one URL
+per invocation so retries stay small and Lambda concurrency supplies the
+parallelism. The Lambda response is not the rendered body; read the returned
+S3 result object using the same temporary role credentials.
+
+Enable the desired Lambda phases in the Static Publisher WordPress admin. The saved
+runtime configuration contains only the operational settings:
+
+```json
+{
+  "remoteWorkers": {
+    "render": {
+      "enabled": true,
+      "concurrency": 4,
+      "maxAttempts": 2
+    },
+    "rewrite": {
+      "enabled": true,
+      "concurrency": 8,
+      "batchSize": 25,
+      "maxAttempts": 2
+    },
+    "deploy": {
+      "enabled": true,
+      "concurrency": 8,
+      "batchSize": 50,
+      "maxAttempts": 2
+    }
+  }
+}
+```
+
+The normal queue runner already supplies the runtime directory, so cron needs
+no Lambda-specific environment variables. A one-off diagnostic crawl can still
+override the generated file explicitly:
+
+```bash
+STATIC_PUBLISHER_RUNTIME_DIR=/srv/site/runtime \
+  publisher-exporter crawl --remote-render
+```
+
+Use `--local-render` to override an enabled configuration for a diagnostic run.
+The crawl coordinator keeps sitemap discovery, incremental decisions, asset
+downloads, manifests, and deploy planning local. Each admitted
+page is rendered by one Lambda task, and its HTML is read from S3 with length
+and SHA-256 verification. Failed/deferred tasks receive a new task ID when
+retried. Final text files are staged in the workspace, rewritten in bounded
+Lambda batches, checksum-verified, and restored to the local export tree.
+Deployment stages finalized objects with their cache and checksum metadata;
+deploy workers then check or copy them directly from S3 to the allowlisted
+target. Only explicit deploy-plan tombstones may be deleted. Each completed
+batch updates phase-scoped runtime progress and adds a structured batch summary
+to `deploy.log.jsonl`; debug logging also records each returned object result.
+There is no silent local fallback for any
+enabled remote phase.
+
 ## Troubleshooting
 
 - If exported HTML still contains old rewritten URLs after a rewrite fix, run `crawl --resume-rewrite`. This reprocesses every text file in the current output tree and does not depend on earlier `rewriteComplete: true` entries in `crawl-manifest.json`.

package/dist/cli.js

@@ -1,23 +1,31 @@
 #!/usr/bin/env node
-import {spawn}from'child_process';import {existsSync,readFileSync}from'fs';import o from'path';import {fileURLToPath}from'url';var p=o.dirname(fileURLToPath(import.meta.url)),l=o.resolve(p,"..");function f(){try{let n=readFileSync(o.join(l,"package.json"),"utf8"),e=JSON.parse(n);if(typeof e.version=="string"&&e.version.trim())return e.version.trim()}catch{}return "unknown"}function g(){console.log(`publisher-exporter <command> [args]
+import {spawn}from'child_process';import {existsSync,readFileSync}from'fs';import o from'path';import {fileURLToPath}from'url';var p=o.dirname(fileURLToPath(import.meta.url)),l=o.resolve(p,"..");function f(){try{let e=readFileSync(o.join(l,"package.json"),"utf8"),r=JSON.parse(e);if(typeof r.version=="string"&&r.version.trim())return r.version.trim()}catch{}return "unknown"}function g(){console.log(`publisher-exporter <command> [args]
 
 Commands:
-  crawl               Run the crawl phase
-  deploy              Deploy the prepared output
-  invalidate          Request CloudFront invalidation
-  publish             Run crawl, deploy, then invalidate
-  content-sync        Run an internal claimed content-sync job
-  queue-runner        Process queued jobs from runtime JSON
-  prune-logs          Delete archived log directories older than a retention window
-  install-browsers    Install Playwright Chromium into the current/shared browsers path
-  version             Show the installed CLI version
-  help                Show this help
+  crawl                 Run the crawl phase
+  deploy                Deploy the prepared output
+  invalidate            Request CloudFront invalidation
+  publish               Run crawl, deploy, then invalidate
+  content-sync          Run an internal claimed content-sync job
+  queue-runner          Process queued jobs from runtime JSON
+  prune-logs            Delete archived log directories older than a retention window
+  remote-worker-health  Invoke health checks on all configured Lambda workers
+  install-browsers      Install Playwright Chromium into the current/shared browsers path
+  version               Show the installed CLI version
+  help                  Show this help
+
+Crawl worker options:
+  --remote-render                 Delegate Playwright page rendering to Lambda
+  --local-render                  Force local Playwright rendering
+  --remote-worker-config <path>   Diagnostic override for runtime/remote-workers.json
 
 Examples:
   publisher-exporter crawl --crawl-mode incremental
+  publisher-exporter crawl --remote-render
   publisher-exporter deploy --profile prod
   publisher-exporter queue-runner --runtime-dir /srv/site/runtime --max-jobs 1
   publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30
+  publisher-exporter remote-worker-health --runtime-dir /srv/site/runtime
   publisher-exporter install-browsers
   publisher-exporter --version
-`);}function v(){console.log(f());}function a(n){let e=o.join(p,`${n}.js`);if(!existsSync(e))throw new Error(`Missing CLI entrypoint: ${e}`);return e}function y(){let n=[o.join(l,"node_modules","playwright","cli.js"),o.join(l,"node_modules","playwright-core","cli.js")];for(let e of n)if(existsSync(e))return e;throw new Error("Playwright CLI was not found under this package. Run `npm install` first.")}function i(n,e){return new Promise((r,s)=>{let t=spawn(process.execPath,[n,...e],{cwd:l,env:process.env,stdio:"inherit"});t.on("error",s),t.on("close",(d,c)=>{if(c){s(new Error(`Command terminated by signal ${c}.`));return}r(d??1);});})}async function b(n){let e=`direct-publish-${Date.now()}`,r=await i(a("content-sync"),["--capture-baseline-cutoffs",e]);if(r!==0)return r;for(let s of ["crawl","deploy","invalidate"]){let t=await i(a(s),n);if(t!==0)return t}return i(a("content-sync"),["--establish-baselines",e])}async function x(){let[n="help",...e]=process.argv.slice(2);if(["version","--version","-v"].includes(n)){v();return}if(["help","--help","-h"].includes(n)){g();return}if(n==="publish"){process.exitCode=await b(e);return}if(n==="install-browsers"){let r=e.length>0?e:["install","chromium"];process.exitCode=await i(y(),r);return}if(n==="crawl"||n==="deploy"||n==="invalidate"||n==="queue-runner"||n==="prune-logs"||n==="content-sync"){process.exitCode=await i(a(n),e);return}throw new Error(`Unknown command: ${n}`)}x().catch(n=>{console.error(n instanceof Error?n.message:String(n)),process.exit(1);});
+`);}function v(){console.log(f());}function a(e){let r=o.join(p,`${e}.js`);if(!existsSync(r))throw new Error(`Missing CLI entrypoint: ${r}`);return r}function y(){let e=[o.join(l,"node_modules","playwright","cli.js"),o.join(l,"node_modules","playwright-core","cli.js")];for(let r of e)if(existsSync(r))return r;throw new Error("Playwright CLI was not found under this package. Run `npm install` first.")}function i(e,r){return new Promise((n,s)=>{let t=spawn(process.execPath,[e,...r],{cwd:l,env:process.env,stdio:"inherit"});t.on("error",s),t.on("close",(d,c)=>{if(c){s(new Error(`Command terminated by signal ${c}.`));return}n(d??1);});})}async function b(e){let r=`direct-publish-${Date.now()}`,n=await i(a("content-sync"),["--capture-baseline-cutoffs",r]);if(n!==0)return n;for(let s of ["crawl","deploy","invalidate"]){let t=await i(a(s),e);if(t!==0)return t}return i(a("content-sync"),["--establish-baselines",r])}async function x(){let[e="help",...r]=process.argv.slice(2);if(["version","--version","-v"].includes(e)){v();return}if(["help","--help","-h"].includes(e)){g();return}if(e==="publish"){process.exitCode=await b(r);return}if(e==="install-browsers"){let n=r.length>0?r:["install","chromium"];process.exitCode=await i(y(),n);return}if(e==="crawl"||e==="deploy"||e==="invalidate"||e==="queue-runner"||e==="prune-logs"||e==="remote-worker-health"||e==="content-sync"){process.exitCode=await i(a(e),r);return}throw new Error(`Unknown command: ${e}`)}x().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exit(1);});