la-machina-engine 0.19.3 → 0.19.6

package/dist/index.cjs

@@ -10980,6 +10980,38 @@ var RunStateManager = class {
     await this.write(next);
     return next;
   }
+  /**
+   * Merge async lifecycle timing fields into the durable state.
+   * Existing timestamps are preserved unless explicitly overwritten
+   * by the caller.
+   */
+  async patchAsyncTiming(runId, nodeId, patch) {
+    const current = await this.read(runId, nodeId);
+    if (current === null) {
+      throw new Error(`RunStateManager.patchAsyncTiming: no state found for ${runId}/${nodeId}`);
+    }
+    const next = {
+      ...current,
+      asyncTiming: { ...current.asyncTiming ?? {}, ...patch }
+    };
+    await this.write(next);
+    return next;
+  }
+  async appendManualWebhookRetry(runId, nodeId, row) {
+    const current = await this.read(runId, nodeId);
+    if (current === null) {
+      throw new Error(
+        `RunStateManager.appendManualWebhookRetry: no state found for ${runId}/${nodeId}`
+      );
+    }
+    const retries = current.manualWebhookRetries ?? [];
+    const next = {
+      ...current,
+      manualWebhookRetries: [...retries, row]
+    };
+    await this.write(next);
+    return next;
+  }
   /**
    * Update just the heartbeat + progress (cheap, called every turn).
    */
@@ -11716,6 +11748,7 @@ ${inputJson}
    * uses fire-and-forget Promises which won't survive Worker request exit.
    */
   async start(options) {
+    const startCalledAt = Date.now();
     const runId = options.runId ?? `run_${randomUUID()}`;
     const storage = await this.buildStorage();
     const stateManager = new RunStateManager(storage.workspace);
@@ -11727,19 +11760,62 @@ ${inputJson}
       deliveries: []
     } : void 0;
     const initial = RunStateManager.initial(runId, options.nodeId, webhook);
-    await stateManager.write(initial);
+    await stateManager.write({
+      ...initial,
+      asyncTiming: { startCalledAt }
+    });
+    await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+      initialStateWrittenAt: Date.now()
+    });
     const handoffEnabled = this.config.runner !== void 0;
+    await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+      backgroundScheduledAt: Date.now()
+    });
     this.backgroundExecutor.schedule(runId, async (signal) => {
+      await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+        backgroundStartedAt: Date.now()
+      });
+      if (signal.aborted) return;
       await stateManager.update(runId, options.nodeId, { status: "running" });
       try {
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          runCallStartedAt: Date.now()
+        });
         const response = await this.run({ ...options, runId }, { handoffToRunner: handoffEnabled });
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          runCallCompletedAt: Date.now()
+        });
         if (signal.aborted) return;
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          handoffStartedAt: Date.now()
+        });
         const postHandoff = await this.maybeHandoffToRunner(runId, options.nodeId, response);
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          handoffCompletedAt: Date.now(),
+          finalizeStartedAt: Date.now()
+        });
+        if (await this.runnerAlreadyWroteTerminal(stateManager, runId, options.nodeId, postHandoff)) {
+          await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+            finalizeCompletedAt: Date.now(),
+            backgroundCompletedAt: Date.now()
+          });
+          return;
+        }
         await stateManager.finalize(runId, options.nodeId, postHandoff);
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          finalizeCompletedAt: Date.now()
+        });
         await this.maybeFireWebhook(stateManager, runId, options.nodeId, postHandoff);
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          backgroundCompletedAt: Date.now()
+        });
       } catch (err) {
         if (signal.aborted) return;
         const errorMsg = err instanceof Error ? err.message : String(err);
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          backgroundFailedAt: Date.now(),
+          errorMessage: errorMsg
+        });
         const failResponse = {
           runId,
           status: "failed",
@@ -11748,8 +11824,17 @@ ${inputJson}
           errors: [{ code: "RUN_FAILED", message: errorMsg }],
           timestamp: Date.now()
         };
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          finalizeStartedAt: Date.now()
+        });
         await stateManager.finalize(runId, options.nodeId, failResponse);
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          finalizeCompletedAt: Date.now()
+        });
         await this.maybeFireWebhook(stateManager, runId, options.nodeId, failResponse);
+        await this.recordAsyncTiming(stateManager, runId, options.nodeId, {
+          backgroundCompletedAt: Date.now()
+        });
       }
     });
     return { runId, nodeId: options.nodeId, status: "queued" };
@@ -11759,6 +11844,7 @@ ${inputJson}
    * dispatched via the background executor. Returns immediately.
    */
   async resumeAsync(options) {
+    const startCalledAt = Date.now();
     const storage = await this.buildStorage();
     const stateManager = new RunStateManager(storage.workspace);
     let nodeId = options.nodeId;
@@ -11780,21 +11866,74 @@ ${inputJson}
       lastHeartbeat: Date.now(),
       response: null,
       // clear stale paused response so getStatus returns provisional
+      asyncTiming: {
+        ...existing.asyncTiming ?? {},
+        startCalledAt
+      },
       ...webhook !== void 0 ? { webhook } : {}
-    } : { ...RunStateManager.initial(options.runId, nodeId, webhook), status: "running" };
+    } : {
+      ...RunStateManager.initial(options.runId, nodeId, webhook),
+      status: "running",
+      asyncTiming: { startCalledAt }
+    };
     await stateManager.write(next);
+    await this.recordAsyncTiming(stateManager, options.runId, nodeId, {
+      initialStateWrittenAt: Date.now()
+    });
     const resumeNodeId = nodeId;
     const handoffEnabled = this.config.runner !== void 0;
+    await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+      backgroundScheduledAt: Date.now()
+    });
     this.backgroundExecutor.schedule(options.runId, async (signal) => {
+      await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+        backgroundStartedAt: Date.now()
+      });
+      if (signal.aborted) return;
       try {
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          runCallStartedAt: Date.now()
+        });
         const response = await this.resume(options, { handoffToRunner: handoffEnabled });
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          runCallCompletedAt: Date.now()
+        });
         if (signal.aborted) return;
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          handoffStartedAt: Date.now()
+        });
         const postHandoff = await this.maybeHandoffToRunner(options.runId, resumeNodeId, response);
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          handoffCompletedAt: Date.now(),
+          finalizeStartedAt: Date.now()
+        });
+        if (await this.runnerAlreadyWroteTerminal(
+          stateManager,
+          options.runId,
+          resumeNodeId,
+          postHandoff
+        )) {
+          await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+            finalizeCompletedAt: Date.now(),
+            backgroundCompletedAt: Date.now()
+          });
+          return;
+        }
         await stateManager.finalize(options.runId, resumeNodeId, postHandoff);
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          finalizeCompletedAt: Date.now()
+        });
         await this.maybeFireWebhook(stateManager, options.runId, resumeNodeId, postHandoff);
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          backgroundCompletedAt: Date.now()
+        });
       } catch (err) {
         if (signal.aborted) return;
         const errorMsg = err instanceof Error ? err.message : String(err);
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          backgroundFailedAt: Date.now(),
+          errorMessage: errorMsg
+        });
         const failResponse = {
           runId: options.runId,
           status: "failed",
@@ -11803,8 +11942,17 @@ ${inputJson}
           errors: [{ code: "RESUME_FAILED", message: errorMsg }],
           timestamp: Date.now()
         };
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          finalizeStartedAt: Date.now()
+        });
         await stateManager.finalize(options.runId, resumeNodeId, failResponse);
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          finalizeCompletedAt: Date.now()
+        });
         await this.maybeFireWebhook(stateManager, options.runId, resumeNodeId, failResponse);
+        await this.recordAsyncTiming(stateManager, options.runId, resumeNodeId, {
+          backgroundCompletedAt: Date.now()
+        });
       }
     });
     return { runId: options.runId, nodeId, status: "running" };
@@ -11837,7 +11985,15 @@ ${inputJson}
         timestamp: Date.now()
       };
     }
-    if (state.response !== null) return state.response;
+    if (state.response !== null) {
+      return state.asyncTiming === void 0 ? state.response : {
+        ...state.response,
+        meta: {
+          ...state.response.meta,
+          asyncTiming: state.asyncTiming
+        }
+      };
+    }
     return {
       runId: state.runId,
       status: state.status,
@@ -11847,7 +12003,8 @@ ${inputJson}
         turns: state.progress.turns,
         tokensUsed: state.progress.tokensUsed,
         activity: state.progress.currentActivity,
-        ...state.progress.lastTool !== void 0 ? { lastTool: state.progress.lastTool } : {}
+        ...state.progress.lastTool !== void 0 ? { lastTool: state.progress.lastTool } : {},
+        ...state.asyncTiming !== void 0 ? { asyncTiming: state.asyncTiming } : {}
       },
       errors: [],
       timestamp: state.lastHeartbeat
@@ -11856,6 +12013,11 @@ ${inputJson}
   /**
    * Poll until the run reaches a terminal state (done | failed | paused |
    * cancelled) or the timeout expires. Returns the final EngineResponse.
+   *
+   * For async paused runs, wait until the background wrapper has completed
+   * its post-finalize bookkeeping before returning. Otherwise a caller can
+   * immediately resume while the previous wrapper is still writing timing
+   * diagnostics, allowing a stale paused state to overwrite the resumed run.
    */
   async waitFor(runId, opts = {}) {
     const pollInterval = opts.pollIntervalMs ?? 1e3;
@@ -11863,9 +12025,10 @@ ${inputJson}
     const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : Infinity;
     for (; ; ) {
       const resp = await this.getStatus(runId, opts.nodeId);
-      if (resp.status === "done" || resp.status === "failed" || resp.status === "paused") {
+      if (resp.status === "done" || resp.status === "failed") {
         return resp;
       }
+      if (resp.status === "paused" && this.pausedRunIsQuiesced(resp, opts)) return resp;
       if (Date.now() >= deadline) {
         return {
           runId,
@@ -11881,6 +12044,12 @@ ${inputJson}
       await new Promise((r) => setTimeout(r, pollInterval));
     }
   }
+  pausedRunIsQuiesced(resp, opts) {
+    if (resp.meta.pauseReason === "handoff_to_runner") return opts.waitForRunnerHandoff !== true;
+    const timing = resp.meta.asyncTiming;
+    if (timing === void 0) return true;
+    return timing.backgroundCompletedAt !== void 0 || timing.backgroundFailedAt !== void 0;
+  }
   /**
    * Cancel an async run. Aborts the background executor and marks the
    * state as cancelled. Idempotent — safe to call on already-terminal runs.
@@ -11937,14 +12106,31 @@ ${inputJson}
     if (original === void 0) {
       throw new Error(`retryWebhook: delivery ${deliveryId} not found`);
     }
-    await this.dispatchWebhookWithRetries(
-      stateManager,
-      runId,
-      targetNodeId,
-      original.event,
-      state.response,
-      1
-    );
+    const retryStartedAt = Date.now();
+    try {
+      await this.dispatchWebhookWithRetries(
+        stateManager,
+        runId,
+        targetNodeId,
+        original.event,
+        state.response,
+        1
+      );
+      await this.recordManualWebhookRetry(stateManager, runId, targetNodeId, {
+        deliveryId,
+        startedAt: retryStartedAt,
+        completedAt: Date.now()
+      });
+    } catch (err) {
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      await this.recordManualWebhookRetry(stateManager, runId, targetNodeId, {
+        deliveryId,
+        startedAt: retryStartedAt,
+        completedAt: Date.now(),
+        errorMessage
+      });
+      throw err;
+    }
   }
   /**
    * Scan all runs for stale heartbeats and mark them as failed. Clients
@@ -11981,6 +12167,12 @@ ${inputJson}
     return orphaned;
   }
   // ---------- runner handoff (Plan 019) ----------
+  async runnerAlreadyWroteTerminal(stateManager, runId, nodeId, response) {
+    if (response.status !== "paused") return false;
+    if (response.meta.pauseReason !== "handoff_to_runner") return false;
+    const latest = await stateManager.read(runId, nodeId);
+    return latest?.status === "done" || latest?.status === "failed" || latest?.status === "cancelled";
+  }
   /**
    * When the response indicates the run paused for runner handoff, POST
    * `{ runId }` to the configured runner URL. On success, return the
@@ -12046,7 +12238,22 @@ ${inputJson}
     }
   }
   // ---------- webhook helpers ----------
+  async recordAsyncTiming(stateManager, runId, nodeId, patch) {
+    try {
+      await stateManager.patchAsyncTiming(runId, nodeId, patch);
+    } catch {
+    }
+  }
+  async recordManualWebhookRetry(stateManager, runId, nodeId, row) {
+    try {
+      await stateManager.appendManualWebhookRetry(runId, nodeId, row);
+    } catch {
+    }
+  }
   async maybeFireWebhook(stateManager, runId, nodeId, response) {
+    await this.recordAsyncTiming(stateManager, runId, nodeId, {
+      webhookCheckStartedAt: Date.now()
+    });
     const state = await stateManager.read(runId, nodeId);
     if (state === null || state.webhook === void 0) return;
     const event = response.status === "done" ? "done" : response.status === "paused" ? "paused" : "failed";
@@ -12057,12 +12264,18 @@ ${inputJson}
     const state = await stateManager.read(runId, nodeId);
     if (state === null || state.webhook === void 0) return;
     const hook = state.webhook;
+    await this.recordAsyncTiming(stateManager, runId, nodeId, {
+      webhookDispatchStartedAt: Date.now()
+    });
     let attempt = startAttempt;
     while (attempt <= MAX_ATTEMPTS) {
       const delay = RETRY_DELAYS_MS[attempt - 1] ?? 0;
       if (delay > 0) {
         await new Promise((r) => setTimeout(r, delay));
       }
+      await this.recordAsyncTiming(stateManager, runId, nodeId, {
+        webhookHttpStartedAt: Date.now()
+      });
       const result = await this.webhookDispatcher.deliver({
         url: hook.url,
         event,
@@ -12071,6 +12284,9 @@ ${inputJson}
         ...hook.headers !== void 0 ? { headers: hook.headers } : {},
         attempt
       });
+      await this.recordAsyncTiming(stateManager, runId, nodeId, {
+        webhookHttpCompletedAt: Date.now()
+      });
       const latest = await stateManager.read(runId, nodeId);
       if (latest !== null && latest.webhook !== void 0) {
         const updated = {
@@ -12078,10 +12294,21 @@ ${inputJson}
           deliveries: [...latest.webhook.deliveries, result.delivery]
         };
         await stateManager.update(runId, nodeId, { webhook: updated });
+        await this.recordAsyncTiming(stateManager, runId, nodeId, {
+          webhookStatePersistedAt: Date.now()
+        });
+      }
+      if (!result.shouldRetry) {
+        await this.recordAsyncTiming(stateManager, runId, nodeId, {
+          webhookDispatchCompletedAt: Date.now()
+        });
+        return;
       }
-      if (!result.shouldRetry) return;
       attempt += 1;
     }
+    await this.recordAsyncTiming(stateManager, runId, nodeId, {
+      webhookDispatchCompletedAt: Date.now()
+    });
   }
   /**
    * Shut down engine-owned background resources — currently just the