@axlsdk/axl 0.27.0 → 0.27.1

package/dist/index.cjs

@@ -602,7 +602,7 @@ var sharedGuard = typeof installedGuard === "object" && installedGuard !== null
 var buildPath = typeof __AXL_BUILD_PATH__ === "string" ? __AXL_BUILD_PATH__ : "<source>";
 var thisCopy = {
   path: buildPath.startsWith("file:") ? (0, import_node_url.fileURLToPath)(buildPath) : buildPath,
-  version: true ? "0.27.0" : "unknown"
+  version: true ? "0.27.1" : "unknown"
 };
 function warnCrossCopy(other, joined) {
   if (other === thisCopy) return;
@@ -17280,7 +17280,7 @@ ${appendedSchemaText}`;
         `ctx.checkpoint name "${name}" is reserved \u2014 names starting with "__auto/" (case-insensitive) are used by the runtime for ask/spawn/race/parallel/map auto-checkpointing.`
       );
     }
-    return this._checkpoint(name, fn);
+    return this._checkpoint(name, fn, { saveRequired: true });
   }
   /**
    * Internal checkpoint implementation shared by both the public
@@ -17310,7 +17310,17 @@ ${appendedSchemaText}`;
       return saved;
     }
     const result = await fn();
-    await this.stateStore.saveCheckpoint(this.executionId, name, result);
+    if (!options?.saveRequired && this.autoCheckpointCounters.saveFailed) return result;
+    try {
+      await this.stateStore.saveCheckpoint(this.executionId, name, result);
+    } catch (err) {
+      if (options?.saveRequired) throw err;
+      this.autoCheckpointCounters.saveFailed = true;
+      console.error(
+        `[axl] Failed to save checkpoint "${name}" for executionId=${this.executionId}; a replay would run this step and later ones again: ${err instanceof Error ? err.message : String(err)}`
+      );
+      return result;
+    }
     this.emitEvent({
       type: "checkpoint_save",
       ...agentStamp,
@@ -20231,11 +20241,18 @@ var Session = class _Session {
     try {
       const completion = axlStream.promise.then(
         (result) => this.commitHistory(history, result).catch((err) => {
-          this.runtime.emit("error", {
-            type: "session_history_save_failed",
-            sessionId: this.sessionId,
-            error: err instanceof Error ? err.message : String(err)
-          });
+          const error = err instanceof Error ? err.message : String(err);
+          try {
+            this.runtime.emit("error", {
+              type: "session_history_save_failed",
+              sessionId: this.sessionId,
+              error
+            });
+          } catch {
+            console.error(
+              `[axl] Failed to save session history for sessionId=${this.sessionId}: ${error}`
+            );
+          }
         }),
         () => void 0
       );
@@ -22302,7 +22319,11 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
         observation
       });
       if (this.stateStore.deleteCheckpoints) {
-        await this.stateStore.deleteCheckpoints(execInfo.executionId);
+        await this.stateStore.deleteCheckpoints(execInfo.executionId).catch((err) => {
+          console.error(
+            `[axl] Failed to delete checkpoints for executionId=${execInfo.executionId}: ` + (err instanceof Error ? err.message : String(err))
+          );
+        });
       }
       span?.setAttribute("axl.workflow.cost", execInfo.totalCost);
       span?.setAttribute("axl.workflow.duration", execInfo.duration);
@@ -22374,7 +22395,10 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
       const chain = this.stateStore.saveExecution(snapshot).then(() => {
         return this.streamingFlusher?.finalize(id).catch(() => {
         });
-      }).catch(() => {
+      }).catch((err) => {
+        console.error(
+          `[axl] Failed to save execution history for executionId=${id}: ` + (err instanceof Error ? err.message : String(err))
+        );
       }).then(() => void 0);
       this.persistInflight.add(chain);
       const pending = { snapshot, settled: chain };
@@ -24797,12 +24821,108 @@ function warnIfRedisExecutionRetentionUnset(opts) {
     warnExecutionRetentionUnset("RedisStore");
   }
 }
+var CLOSE_TIMEOUT_MS = 2e3;
+function reconnectDelayMs(retries) {
+  return Math.min(2 ** retries * 50, 2e3) + Math.floor(Math.random() * 200);
+}
+function abandonConnection(client) {
+  try {
+    client.destroy?.();
+  } catch {
+  }
+  client.unref?.();
+}
+function errorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+function connectionEventReporter(client, onConnectionEvent) {
+  if (onConnectionEvent) {
+    const reportFailure = (error) => console.error(`[axl] RedisStore: onConnectionEvent failed: ${errorMessage(error)}`);
+    return (event) => {
+      try {
+        Promise.resolve(onConnectionEvent(event)).catch(reportFailure);
+      } catch (error) {
+        reportFailure(error);
+      }
+    };
+  }
+  let outageLogged = false;
+  return (event) => {
+    if (event.type === "reconnected") {
+      outageLogged = false;
+      console.warn("[axl] RedisStore: reconnected to Redis.");
+    } else if (!outageLogged) {
+      outageLogged = client.isReady === false;
+      console.warn(
+        `[axl] RedisStore: Redis connection lost (${event.error.message}). Store calls fail until it reconnects, which happens automatically.`
+      );
+    }
+  };
+}
+async function connectClient(createClient, opts) {
+  let established = false;
+  const client = createClient({
+    ...opts.url ? { url: opts.url } : {},
+    // A command accepted just before a drop rejects instead of waiting for
+    // the reconnect.
+    disableOfflineQueue: true,
+    socket: { reconnectStrategy: (retries) => established ? reconnectDelayMs(retries) : false }
+  });
+  const reportConnectionEvent = connectionEventReporter(client, opts.onConnectionEvent);
+  client.on("error", (error) => {
+    if (!established || client.isOpen === false) return;
+    reportConnectionEvent({
+      type: "error",
+      error: error instanceof Error ? error : new Error(String(error))
+    });
+  });
+  client.on("connect", () => {
+    if (client.isOpen === false) client.unref?.();
+  });
+  client.on("ready", () => {
+    if (!established) {
+      established = true;
+    } else if (client.isOpen !== false) {
+      reportConnectionEvent({ type: "reconnected" });
+    }
+  });
+  await client.connect();
+  return client;
+}
 var RedisStore = class _RedisStore {
-  constructor(client, keyPrefix, ttls) {
-    this.client = client;
+  constructor(connection, keyPrefix, ttls) {
+    this.connection = connection;
     this.keyPrefix = keyPrefix;
     this.ttls = ttls;
   }
+  /**
+   * Whether the store can send commands right now, for a health check or for
+   * telling an outage from other failures when a call rejects. `false` while
+   * reconnecting and after `close()`.
+   */
+  get isConnected() {
+    return this.connection.isOpen !== false && this.connection.isReady !== false;
+  }
+  /**
+   * The client for sending a command, failing immediately while disconnected.
+   *
+   * Every command goes through here because node-redis's `disableOfflineQueue`
+   * rejects single commands but not `multi().exec()`, which would wait for a
+   * reconnect attempt to fail or succeed. Nothing may `await` between
+   * `client.multi()` and `.exec()`, or a transaction could get past this check.
+   *
+   * A closed store isn't "unavailable": it falls through to node-redis's own
+   * "The client is closed" rejection.
+   */
+  get client() {
+    if (this.connection.isOpen !== false && this.connection.isReady === false) {
+      throw new AxlError(
+        "REDIS_UNAVAILABLE",
+        "RedisStore: not connected to Redis, so the command was not sent. The store reconnects automatically; retry once Redis is back."
+      );
+    }
+    return this.connection;
+  }
   /**
    * History kinds with a prune running in this process. A save skips its
    * prune while another one of the same kind is in flight: the running one
@@ -24872,8 +24992,7 @@ var RedisStore = class _RedisStore {
       if (err instanceof Error && err.message.includes("createClient")) throw err;
       throw new Error("redis is required for RedisStore. Install it with: npm install redis");
     }
-    const client = opts.url ? createClient({ url: opts.url }) : createClient();
-    await client.connect();
+    const client = await connectClient(createClient, opts);
     const resolveCategory = (override) => {
       if (override === null) return null;
       if (typeof override === "number") return override;
@@ -24897,8 +25016,13 @@ var RedisStore = class _RedisStore {
     const store = new _RedisStore(client, keyPrefix, effectiveTtls);
     warnIfRedisExecutionRetentionUnset(opts);
     if (!opts.skipMigration) {
-      await store.backfillExecutionIndex();
-      await store.backfillEvalIndex();
+      try {
+        await store.backfillExecutionIndex();
+        await store.backfillEvalIndex();
+      } catch (error) {
+        abandonConnection(client);
+        throw error;
+      }
     }
     return store;
   }
@@ -25167,7 +25291,7 @@ var RedisStore = class _RedisStore {
       }
     }
     if (dead.length > 0) {
-      this.client.sRem(this.pendingExecSetKey(), dead).catch((err) => {
+      this.connection.sRem(this.pendingExecSetKey(), dead).catch((err) => {
         console.error(
           `[axl] RedisStore: failed to prune ${dead.length} stale pending-exec ids: ` + (err instanceof Error ? err.message : String(err))
         );
@@ -25213,7 +25337,7 @@ var RedisStore = class _RedisStore {
     }
     const ttl = this.ttlFor("memory");
     if (ttl !== void 0) {
-      this.client.expire(this.memoryKey(scope), ttl).catch((err) => {
+      this.connection.expire(this.memoryKey(scope), ttl).catch((err) => {
         console.error(
           `[axl] RedisStore: failed to set TTL on migrated memory key ${this.memoryKey(scope)}: ` + (err instanceof Error ? err.message : String(err))
         );
@@ -25505,6 +25629,7 @@ var RedisStore = class _RedisStore {
     try {
       await this.pruneHistory(index);
     } catch (err) {
+      if (this.connection.isReady === false) return;
       warnDiagnosticOnce(
         `redis_history_prune_failed\0${index.kind}`,
         `RedisStore: removing expired ${index.label} index entries failed (${err instanceof Error ? err.message : String(err)}). The record was saved; the next save retries the cleanup. Listings stay correct meanwhile. The cleanup runs a Lua script, so the Redis user needs permission to run EVAL.`,
@@ -25709,10 +25834,31 @@ var RedisStore = class _RedisStore {
    * must not get a rejected promise for it. `MemoryStore.close()` and
    * better-sqlite3's `close()` are both no-ops the second time; node-redis
    * instead rejects with "The client is closed", so absorb that here.
+   *
+   * Bounded, so a shutdown can't hang on Redis. `QUIT` lets queued commands
+   * finish on a healthy connection, but node-redis never settles it if the
+   * socket closes first, and a stalled server may never answer it. While
+   * disconnected it couldn't be sent at all.
    */
   async close() {
-    if (this.client.isOpen === false) return;
-    await this.client.quit();
+    if (this.connection.isOpen === false) return;
+    if (this.connection.isReady === false) {
+      abandonConnection(this.connection);
+      return;
+    }
+    let timer;
+    try {
+      await Promise.race([
+        this.connection.quit(),
+        new Promise((_resolve, reject) => {
+          timer = setTimeout(() => reject(new Error("QUIT timed out")), CLOSE_TIMEOUT_MS);
+        })
+      ]);
+    } catch {
+      abandonConnection(this.connection);
+    } finally {
+      clearTimeout(timer);
+    }
   }
   async deleteCheckpoints(executionId) {
     await this.client.del(this.checkpointKey(executionId));