@smart-cloud/publisher-exporter 1.1.61 → 1.1.62

package/README.md

@@ -179,17 +179,19 @@ functions. Large payloads do not travel in the Lambda response: workers write
 HTML and detailed results below a configured S3 workspace prefix and return a
 small `resultKey` pointer. Reusing the same job/task ID is idempotent.
 
-The companion CDK application is in the repository `infrastructure/`
+The companion CDK application is maintained separately at
+[`smartcloudsol/static-publisher-lambda-workers`](https://github.com/smartcloudsol/static-publisher-lambda-workers).
 directory. Its deploy script emits a `remote-workers.json` containing the
 versioned Lambda alias ARNs, workspace bucket/prefix, region, and generated
 assumable role. The file contains no static AWS access keys.
 
-Copy that output to the exporter host with owner-only permissions and validate
-the role, aliases, and function runtimes:
+Install that output with owner-only permissions at
+`<site-runtime>/remote-workers.json`, then validate the role, aliases, and
+function runtimes:
 
 ```bash
 publisher-exporter remote-worker-health \
-  --config /etc/wpsuite/remote-workers.json
+  --runtime-dir /srv/site/runtime
 ```
 
 Protocol v1 accepts 1-20 same-origin URLs per render task. Start with one URL
@@ -197,11 +199,56 @@ per invocation so retries stay small and Lambda concurrency supplies the
 parallelism. The Lambda response is not the rendered body; read the returned
 S3 result object using the same temporary role credentials.
 
-This change establishes the deployable workers, client, health check, and S3
-contract. The normal `crawl`, `rewrite`, and `deploy` commands retain their
-local behavior until their coordinator paths are explicitly configured to
-delegate work. There is deliberately no automatic local fallback after a
-remote worker error.
+Enable the desired Lambda phases in the Static Publisher WordPress admin. The saved
+runtime configuration contains only the operational settings:
+
+```json
+{
+  "remoteWorkers": {
+    "render": {
+      "enabled": true,
+      "concurrency": 4,
+      "maxAttempts": 2
+    },
+    "rewrite": {
+      "enabled": true,
+      "concurrency": 8,
+      "batchSize": 25,
+      "maxAttempts": 2
+    },
+    "deploy": {
+      "enabled": true,
+      "concurrency": 8,
+      "batchSize": 50,
+      "maxAttempts": 2
+    }
+  }
+}
+```
+
+The normal queue runner already supplies the runtime directory, so cron needs
+no Lambda-specific environment variables. A one-off diagnostic crawl can still
+override the generated file explicitly:
+
+```bash
+STATIC_PUBLISHER_RUNTIME_DIR=/srv/site/runtime \
+  publisher-exporter crawl --remote-render
+```
+
+Use `--local-render` to override an enabled configuration for a diagnostic run.
+The crawl coordinator keeps sitemap discovery, incremental decisions, asset
+downloads, manifests, and deploy planning local. Each admitted
+page is rendered by one Lambda task, and its HTML is read from S3 with length
+and SHA-256 verification. Failed/deferred tasks receive a new task ID when
+retried. Final text files are staged in the workspace, rewritten in bounded
+Lambda batches, checksum-verified, and restored to the local export tree.
+Deployment stages finalized objects with their cache and checksum metadata;
+deploy workers then check or copy them directly from S3 to the allowlisted
+target. Only explicit deploy-plan tombstones may be deleted. Each completed
+batch updates phase-scoped runtime progress and adds a structured batch summary
+to `deploy.log.jsonl`; debug logging also records each returned object result.
+There is no silent local fallback for any
+enabled remote phase.
 
 ## Troubleshooting
 

package/dist/cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import {spawn}from'child_process';import {existsSync,readFileSync}from'fs';import o from'path';import {fileURLToPath}from'url';var p=o.dirname(fileURLToPath(import.meta.url)),l=o.resolve(p,"..");function f(){try{let e=readFileSync(o.join(l,"package.json"),"utf8"),n=JSON.parse(e);if(typeof n.version=="string"&&n.version.trim())return n.version.trim()}catch{}return "unknown"}function g(){console.log(`publisher-exporter <command> [args]
+import {spawn}from'child_process';import {existsSync,readFileSync}from'fs';import o from'path';import {fileURLToPath}from'url';var p=o.dirname(fileURLToPath(import.meta.url)),l=o.resolve(p,"..");function f(){try{let e=readFileSync(o.join(l,"package.json"),"utf8"),r=JSON.parse(e);if(typeof r.version=="string"&&r.version.trim())return r.version.trim()}catch{}return "unknown"}function g(){console.log(`publisher-exporter <command> [args]
 
 Commands:
   crawl                 Run the crawl phase
@@ -14,12 +14,18 @@ Commands:
   version               Show the installed CLI version
   help                  Show this help
 
+Crawl worker options:
+  --remote-render                 Delegate Playwright page rendering to Lambda
+  --local-render                  Force local Playwright rendering
+  --remote-worker-config <path>   Diagnostic override for runtime/remote-workers.json
+
 Examples:
   publisher-exporter crawl --crawl-mode incremental
+  publisher-exporter crawl --remote-render
   publisher-exporter deploy --profile prod
   publisher-exporter queue-runner --runtime-dir /srv/site/runtime --max-jobs 1
   publisher-exporter prune-logs --runtime-dir /srv/site/runtime --older-than-days 30
-  publisher-exporter remote-worker-health --config ./remote-workers.json
+  publisher-exporter remote-worker-health --runtime-dir /srv/site/runtime
   publisher-exporter install-browsers
   publisher-exporter --version
-`);}function v(){console.log(f());}function a(e){let n=o.join(p,`${e}.js`);if(!existsSync(n))throw new Error(`Missing CLI entrypoint: ${n}`);return n}function y(){let e=[o.join(l,"node_modules","playwright","cli.js"),o.join(l,"node_modules","playwright-core","cli.js")];for(let n of e)if(existsSync(n))return n;throw new Error("Playwright CLI was not found under this package. Run `npm install` first.")}function i(e,n){return new Promise((r,s)=>{let t=spawn(process.execPath,[e,...n],{cwd:l,env:process.env,stdio:"inherit"});t.on("error",s),t.on("close",(d,c)=>{if(c){s(new Error(`Command terminated by signal ${c}.`));return}r(d??1);});})}async function b(e){let n=`direct-publish-${Date.now()}`,r=await i(a("content-sync"),["--capture-baseline-cutoffs",n]);if(r!==0)return r;for(let s of ["crawl","deploy","invalidate"]){let t=await i(a(s),e);if(t!==0)return t}return i(a("content-sync"),["--establish-baselines",n])}async function x(){let[e="help",...n]=process.argv.slice(2);if(["version","--version","-v"].includes(e)){v();return}if(["help","--help","-h"].includes(e)){g();return}if(e==="publish"){process.exitCode=await b(n);return}if(e==="install-browsers"){let r=n.length>0?n:["install","chromium"];process.exitCode=await i(y(),r);return}if(e==="crawl"||e==="deploy"||e==="invalidate"||e==="queue-runner"||e==="prune-logs"||e==="remote-worker-health"||e==="content-sync"){process.exitCode=await i(a(e),n);return}throw new Error(`Unknown command: ${e}`)}x().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exit(1);});
+`);}function v(){console.log(f());}function a(e){let r=o.join(p,`${e}.js`);if(!existsSync(r))throw new Error(`Missing CLI entrypoint: ${r}`);return r}function y(){let e=[o.join(l,"node_modules","playwright","cli.js"),o.join(l,"node_modules","playwright-core","cli.js")];for(let r of e)if(existsSync(r))return r;throw new Error("Playwright CLI was not found under this package. Run `npm install` first.")}function i(e,r){return new Promise((n,s)=>{let t=spawn(process.execPath,[e,...r],{cwd:l,env:process.env,stdio:"inherit"});t.on("error",s),t.on("close",(d,c)=>{if(c){s(new Error(`Command terminated by signal ${c}.`));return}n(d??1);});})}async function b(e){let r=`direct-publish-${Date.now()}`,n=await i(a("content-sync"),["--capture-baseline-cutoffs",r]);if(n!==0)return n;for(let s of ["crawl","deploy","invalidate"]){let t=await i(a(s),e);if(t!==0)return t}return i(a("content-sync"),["--establish-baselines",r])}async function x(){let[e="help",...r]=process.argv.slice(2);if(["version","--version","-v"].includes(e)){v();return}if(["help","--help","-h"].includes(e)){g();return}if(e==="publish"){process.exitCode=await b(r);return}if(e==="install-browsers"){let n=r.length>0?r:["install","chromium"];process.exitCode=await i(y(),n);return}if(e==="crawl"||e==="deploy"||e==="invalidate"||e==="queue-runner"||e==="prune-logs"||e==="remote-worker-health"||e==="content-sync"){process.exitCode=await i(a(e),r);return}throw new Error(`Unknown command: ${e}`)}x().catch(e=>{console.error(e instanceof Error?e.message:String(e)),process.exit(1);});