@lobu/worker 7.1.0 → 7.2.0

package/src/openclaw/worker.ts

@@ -54,6 +54,12 @@ import {
   resolveModelRef,
 } from "./model-resolver";
 import { checkSandboxLeak } from "./sandbox-leak";
+import {
+  clearSnapshots,
+  hydrateFromSnapshot,
+  type TerminalStatus,
+  writeSnapshot,
+} from "./transcript-snapshot";
 import {
   loadPlugins,
   runPluginHooks,
@@ -275,6 +281,20 @@ export class OpenClawWorker implements WorkerExecutor {
   public workerTransport: WorkerTransport;
   private config: WorkerConfig;
   private progressProcessor: OpenClawProgressProcessor;
+  /**
+   * Terminal status for the current run, used by `cleanup()` to discriminate
+   * the snapshot row. Defaults to `failed` (pessimistic) so an early crash
+   * before any return-path assignment is recorded as a failure, not silently
+   * accepted as a completion. Set to `completed` only on the success path
+   * in `execute()`. Resets on every `execute()` invocation.
+   */
+  private terminalStatus: TerminalStatus = "failed";
+  /**
+   * Path to the OpenClaw session file for the current run. Captured in
+   * `runAISession()` (where SessionManager opens it) so `cleanup()` can
+   * read it back for the snapshot without re-deriving the path.
+   */
+  private sessionFilePath: string | null = null;
 
   constructor(config: WorkerConfig) {
     this.config = config;
@@ -314,6 +334,33 @@ export class OpenClawWorker implements WorkerExecutor {
    */
   async execute(): Promise<void> {
     const executeStartTime = Date.now();
+    // Reset terminal status for this run. Defaults to `failed` (pessimistic);
+    // assigned to `completed` only on the success path below. SESSION_TIMEOUT
+    // throws and is reassigned in the catch block.
+    this.terminalStatus = "failed";
+
+    // Fail loud when snapshot mode is enabled but the per-run scope the
+    // gateway is supposed to provide hasn't reached this job. A silent
+    // skip in cleanup() would hide a configuration bug across many
+    // turns; throwing here surfaces it on the first turn and the runs
+    // queue's retry path handles re-delivery. Codex round 2 quality
+    // win D on PR #865.
+    //
+    // Phase 5: snapshot is the default; setting LOBU_SESSION_STORE=file
+    // opts out (legacy / local-dev path that keeps reading session.jsonl
+    // straight off disk without writing to Postgres).
+    if (process.env.LOBU_SESSION_STORE !== "file") {
+      if (typeof this.config.runId !== "number") {
+        throw new Error(
+          "Snapshot mode (LOBU_SESSION_STORE != 'file') but WorkerConfig.runId is missing — runs-queue dispatch did not stamp runId on the job payload"
+        );
+      }
+      if (!this.config.runJobToken) {
+        throw new Error(
+          "Snapshot mode (LOBU_SESSION_STORE != 'file') but WorkerConfig.runJobToken is missing — MessageConsumer did not mint a per-run worker token"
+        );
+      }
+    }
 
     try {
       this.progressProcessor.reset();
@@ -468,6 +515,10 @@ export class OpenClawWorker implements WorkerExecutor {
       this.workerTransport.setModuleData(moduleData);
 
       if (result.success) {
+        // Snapshot writer in cleanup() reads this to discriminate the row.
+        // Hydrate skips non-completed snapshots, so getting this right is
+        // what stops a failed turn from poisoning the next attempt.
+        this.terminalStatus = "completed";
         const outputSnapshot = this.progressProcessor.getOutputSnapshot();
         const hintGatewayUrl = process.env.DISPATCHER_URL;
         const hintWorkerToken = process.env.WORKER_TOKEN;
@@ -521,6 +572,12 @@ export class OpenClawWorker implements WorkerExecutor {
         const isTimeout = result.exitCode === 124;
 
         if (isTimeout) {
+          // Mark the snapshot as `timeout` instead of `failed` so operators
+          // can distinguish runaway agents from genuine failures in the
+          // dashboard. The catch block below sees `SESSION_TIMEOUT` and
+          // keeps this assignment intact (it only forces `failed` on
+          // exceptions that aren't already marked).
+          this.terminalStatus = "timeout";
           logger.info(
             `Session timed out (exit code 124) - will be retried automatically, not showing error to user`
           );
@@ -551,6 +608,55 @@ export class OpenClawWorker implements WorkerExecutor {
   }
 
   async cleanup(): Promise<void> {
+    // Snapshot the post-run session.jsonl to Postgres so the next worker
+    // (possibly on a different pod) can hydrate from it. Hydrate filters
+    // `terminal_status='completed'`, so we ONLY POST on the success path
+    // — writing `failed`/`timeout`/`cancelled` rows is pure network
+    // waste (codex round 2 quality win C on PR #865). Default-on in
+    // Phase 5; LOBU_SESSION_STORE=file opts out for legacy/local-dev.
+    //
+    // The runs queue has already moved this run to a terminal state by
+    // the time cleanup() fires (sse-client.ts:865 finally block runs
+    // after execute() returns). We POST in the worker's own dying
+    // breath; the gateway-side advisory lock held by the spawner is
+    // released when the subprocess exits, so by the next claim's boot
+    // this snapshot is the visible "latest" row.
+    if (
+      process.env.LOBU_SESSION_STORE !== "file" &&
+      this.sessionFilePath &&
+      this.terminalStatus === "completed"
+    ) {
+      const gatewayUrl = process.env.DISPATCHER_URL;
+      const runId = this.config.runId;
+      // Per-run JWT minted by the gateway's MessageConsumer alongside
+      // `runId`. The snapshot route requires `tokenData.runId ===
+      // body.runId`, so the deployment-lifetime WORKER_TOKEN cannot be
+      // used here — it would carry no `runId` and the route would 403.
+      // Codex round 2 finding A.
+      const runJobToken = this.config.runJobToken;
+      if (gatewayUrl && runJobToken && typeof runId === "number") {
+        await writeSnapshot({
+          sessionFile: this.sessionFilePath,
+          gatewayUrl,
+          workerToken: runJobToken,
+          terminalStatus: this.terminalStatus,
+          runId,
+        });
+      } else if (gatewayUrl) {
+        // Missing per-run scope (legacy direct-enqueue path or token
+        // mint failure on the gateway). Skip the snapshot rather than
+        // risk a mis-attributed row; the next run will hydrate from
+        // the previous completed snapshot the next time a normal
+        // runs-queue dispatch comes through.
+        logger.warn(
+          `Skipping transcript snapshot: ${
+            typeof runId !== "number"
+              ? "WorkerConfig.runId is missing"
+              : "WorkerConfig.runJobToken is missing"
+          } (legacy enqueue path)`
+        );
+      }
+    }
     logger.info("Worker cleanup completed");
   }
 
@@ -855,12 +961,56 @@ export class OpenClawWorker implements WorkerExecutor {
     await fs.mkdir(path.join(workspaceDir, ".openclaw"), { recursive: true });
 
     const sessionFile = path.join(workspaceDir, ".openclaw", "session.jsonl");
+    // Capture for cleanup() — it reads the file back to write the snapshot
+    // at terminal time. Set unconditionally so file-mode opt-outs
+    // still get a defined value (snapshot writer no-ops when
+    // LOBU_SESSION_STORE=file).
+    this.sessionFilePath = sessionFile;
     const providerStateFile = path.join(
       workspaceDir,
       ".openclaw",
       "provider.json"
     );
 
+    // Hydrate from the latest completed Postgres snapshot BEFORE the
+    // provider-state check or SessionManager.open(). Phase 5: snapshot
+    // mode is the default; LOBU_SESSION_STORE=file opts out and keeps
+    // the legacy file-only behaviour for local-dev / single-replica
+    // self-hosters.
+    //
+    // Order matters: hydrate → provider check (may unlink) →
+    // SessionManager.open(). The provider-change unlink at line ~925 still
+    // does the right thing after hydrate: it drops the file we just wrote
+    // and SessionManager creates a fresh one, exactly like a first-turn
+    // boot. The next snapshot will have its own run_id, so the historical
+    // PG rows remain readable without poisoning the new conversation
+    // (hydrate would only resurrect them if a subsequent run completes
+    // successfully and overwrites the latest pointer).
+    if (process.env.LOBU_SESSION_STORE !== "file") {
+      const gatewayUrl = process.env.DISPATCHER_URL;
+      const workerToken = process.env.WORKER_TOKEN;
+      if (gatewayUrl && workerToken) {
+        try {
+          await hydrateFromSnapshot({
+            sessionFile,
+            gatewayUrl,
+            workerToken,
+          });
+        } catch (err) {
+          // Hydrate failure is non-fatal — fall back to whatever's on disk.
+          // Worst case the worker boots without history and the user re-
+          // grounds the conversation. Better than refusing to start.
+          logger.warn(
+            `Snapshot hydrate failed; continuing with local session file: ${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      } else {
+        logger.warn(
+          "Snapshot mode active (LOBU_SESSION_STORE != 'file') but DISPATCHER_URL or WORKER_TOKEN missing; snapshot disabled"
+        );
+      }
+    }
+
     // Detect provider change and reset session if needed
     let sessionSummary: string | undefined;
     try {
@@ -1401,6 +1551,21 @@ Use it when the user references past discussions or you need context.`);
           // File may not exist
         }
 
+        // Also purge the Postgres snapshots for this (org, agent, conv)
+        // — in snapshot mode (the Phase 5 default) the next worker boot
+        // would otherwise rehydrate from the now-flushed conversation
+        // and the user-visible "Starting fresh" would be a lie. Best-
+        // effort: a failure here is logged but doesn't block the reset
+        // since the local unlink already happened and the snapshot
+        // helper is a no-op in file mode.
+        if (process.env.LOBU_SESSION_STORE !== "file") {
+          const gatewayUrl = process.env.DISPATCHER_URL;
+          const workerToken = process.env.WORKER_TOKEN;
+          if (gatewayUrl && workerToken) {
+            await clearSnapshots({ gatewayUrl, workerToken });
+          }
+        }
+
         // Send visible confirmation to user
         await onProgress({
           type: "output",