@axlsdk/axl 0.26.0 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -602,7 +602,7 @@ var sharedGuard = typeof installedGuard === "object" && installedGuard !== null
602
602
  var buildPath = typeof __AXL_BUILD_PATH__ === "string" ? __AXL_BUILD_PATH__ : "<source>";
603
603
  var thisCopy = {
604
604
  path: buildPath.startsWith("file:") ? (0, import_node_url.fileURLToPath)(buildPath) : buildPath,
605
- version: true ? "0.26.0" : "unknown"
605
+ version: true ? "0.27.1" : "unknown"
606
606
  };
607
607
  function warnCrossCopy(other, joined) {
608
608
  if (other === thisCopy) return;
@@ -12869,8 +12869,17 @@ function detectDroppedRefinements(schema) {
12869
12869
  var DOCS_URL = "docs/observability.md#schema-diagnostics";
12870
12870
  var MAX_WARNED_KEYS = 2048;
12871
12871
  var warnedDiagnosticKeys = /* @__PURE__ */ new Set();
12872
+ var silencedDepth = 0;
12873
+ function withDiagnosticsSilenced(fn) {
12874
+ silencedDepth++;
12875
+ try {
12876
+ return fn();
12877
+ } finally {
12878
+ silencedDepth--;
12879
+ }
12880
+ }
12872
12881
  function warnDiagnosticOnce(key, message, silent, docs = DOCS_URL) {
12873
- if (silent || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
12882
+ if (silent || silencedDepth > 0 || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
12874
12883
  if (warnedDiagnosticKeys.has(key)) return;
12875
12884
  if (warnedDiagnosticKeys.size >= MAX_WARNED_KEYS) warnedDiagnosticKeys.clear();
12876
12885
  warnedDiagnosticKeys.add(key);
@@ -17271,7 +17280,7 @@ ${appendedSchemaText}`;
17271
17280
  `ctx.checkpoint name "${name}" is reserved \u2014 names starting with "__auto/" (case-insensitive) are used by the runtime for ask/spawn/race/parallel/map auto-checkpointing.`
17272
17281
  );
17273
17282
  }
17274
- return this._checkpoint(name, fn);
17283
+ return this._checkpoint(name, fn, { saveRequired: true });
17275
17284
  }
17276
17285
  /**
17277
17286
  * Internal checkpoint implementation shared by both the public
@@ -17301,7 +17310,17 @@ ${appendedSchemaText}`;
17301
17310
  return saved;
17302
17311
  }
17303
17312
  const result = await fn();
17304
- await this.stateStore.saveCheckpoint(this.executionId, name, result);
17313
+ if (!options?.saveRequired && this.autoCheckpointCounters.saveFailed) return result;
17314
+ try {
17315
+ await this.stateStore.saveCheckpoint(this.executionId, name, result);
17316
+ } catch (err) {
17317
+ if (options?.saveRequired) throw err;
17318
+ this.autoCheckpointCounters.saveFailed = true;
17319
+ console.error(
17320
+ `[axl] Failed to save checkpoint "${name}" for executionId=${this.executionId}; a replay would run this step and later ones again: ${err instanceof Error ? err.message : String(err)}`
17321
+ );
17322
+ return result;
17323
+ }
17305
17324
  this.emitEvent({
17306
17325
  type: "checkpoint_save",
17307
17326
  ...agentStamp,
@@ -18934,10 +18953,135 @@ var import_node_async_hooks4 = require("async_hooks");
18934
18953
  var import_node_fs3 = require("fs");
18935
18954
  var import_node_path3 = require("path");
18936
18955
 
18956
+ // src/state/retention.ts
18957
+ var PRUNE_BATCH = 100;
18958
+ var DEFAULT_MEMORY_MAX_EXECUTIONS = 1e3;
18959
+ function warnExecutionRetentionUnset(storeName) {
18960
+ warnDiagnosticOnce(
18961
+ `execution_retention_unset\0${storeName}`,
18962
+ `${storeName} keeps execution history forever because no retention is set; set ttls.executionHistory${storeName === "RedisStore" ? " (or defaultTtl)" : ""} to a number of seconds, or ttls.executionHistory to null to keep it forever without this warning.`,
18963
+ void 0,
18964
+ "docs/migration/execution-history-retention.md#silencing-the-retention-warning"
18965
+ );
18966
+ }
18967
+ function resolveTtl(storeName, category, value) {
18968
+ if (value === void 0 || value === null) return null;
18969
+ if (typeof value !== "number") {
18970
+ throw new AxlError(
18971
+ "INVALID_CONFIG",
18972
+ `${storeName}: ttls.${category} must be a positive number of seconds or null; got ${typeof value}`
18973
+ );
18974
+ }
18975
+ if (!Number.isFinite(value) || value <= 0) {
18976
+ throw new AxlError(
18977
+ "INVALID_CONFIG",
18978
+ `${storeName}: ttls.${category} must be a positive number of seconds or null; got ${value}`
18979
+ );
18980
+ }
18981
+ return value;
18982
+ }
18983
+ function resolveHistoryTtls(storeName, ttls) {
18984
+ if (ttls !== void 0 && (ttls === null || typeof ttls !== "object")) {
18985
+ throw new AxlError(
18986
+ "INVALID_CONFIG",
18987
+ `${storeName}: ttls must be an object; got ${ttls === null ? "null" : typeof ttls}`
18988
+ );
18989
+ }
18990
+ return {
18991
+ executionHistory: resolveTtl(storeName, "executionHistory", ttls?.executionHistory),
18992
+ evalHistory: resolveTtl(storeName, "evalHistory", ttls?.evalHistory)
18993
+ };
18994
+ }
18995
+ function expiryFrom(now, ttlSeconds) {
18996
+ return ttlSeconds === null ? null : now + Math.round(ttlSeconds * 1e3);
18997
+ }
18998
+ function isExpired(expiresAt, now) {
18999
+ return expiresAt !== null && now >= expiresAt;
19000
+ }
19001
+ var DEFAULT_RETENTION_BATCH_SIZE = 500;
19002
+ function resolveHistoryRetentionOptions(storeName, options) {
19003
+ const method = `${storeName}.applyHistoryRetention()`;
19004
+ if (options === null || typeof options !== "object") {
19005
+ throw new AxlError("INVALID_ARGUMENT", `${method} options must be an object`);
19006
+ }
19007
+ const { kind, batchSize, cursor } = options;
19008
+ if (kind !== "executions" && kind !== "evals") {
19009
+ throw new AxlError(
19010
+ "INVALID_ARGUMENT",
19011
+ `${method} kind must be 'executions' or 'evals'; got ${String(kind)}`
19012
+ );
19013
+ }
19014
+ if (batchSize !== void 0 && !(typeof batchSize === "number" && Number.isSafeInteger(batchSize) && batchSize > 0)) {
19015
+ throw new AxlError(
19016
+ "INVALID_ARGUMENT",
19017
+ `${method} batchSize must be a positive integer; got ${String(batchSize)}`
19018
+ );
19019
+ }
19020
+ let parsedCursor;
19021
+ if (cursor !== void 0 && cursor !== null) {
19022
+ const candidate = cursor;
19023
+ if (typeof candidate !== "object" || typeof candidate.time !== "number" || !Number.isFinite(candidate.time) || typeof candidate.id !== "string") {
19024
+ throw new AxlError(
19025
+ "INVALID_ARGUMENT",
19026
+ `${method} cursor must be the { time, id } cursor returned by the previous call`
19027
+ );
19028
+ }
19029
+ parsedCursor = { time: candidate.time, id: candidate.id };
19030
+ }
19031
+ return {
19032
+ kind,
19033
+ batchSize: batchSize ?? DEFAULT_RETENTION_BATCH_SIZE,
19034
+ cursor: parsedCursor
19035
+ };
19036
+ }
19037
+ function retentionForUntimedRecord(lastSavedAt, ttlSeconds, now) {
19038
+ const expiresAt = lastSavedAt + Math.round(ttlSeconds * 1e3);
19039
+ return isExpired(expiresAt, now) ? { expired: true } : { expired: false, expiresAt };
19040
+ }
19041
+ function executionLastSavedAt(record) {
19042
+ return typeof record.completedAt === "number" && Number.isFinite(record.completedAt) ? record.completedAt : record.startedAt;
19043
+ }
19044
+ function compareExecutionsNewestFirst(a, b) {
19045
+ if (a.startedAt !== b.startedAt) return b.startedAt - a.startedAt;
19046
+ return compareCodePoints(b.executionId, a.executionId);
19047
+ }
19048
+ function compareRetentionPositions(a, b) {
19049
+ if (a.time !== b.time) return a.time - b.time;
19050
+ return compareCodePoints(a.id, b.id);
19051
+ }
19052
+ function compareCodePoints(a, b) {
19053
+ const len = Math.min(a.length, b.length);
19054
+ for (let i = 0; i < len; i++) {
19055
+ const x = a.charCodeAt(i);
19056
+ const y = b.charCodeAt(i);
19057
+ if (x !== y) return codePointRank(x) - codePointRank(y);
19058
+ }
19059
+ return a.length - b.length;
19060
+ }
19061
+ function codePointRank(unit) {
19062
+ if (unit >= 57344) return unit - 2048;
19063
+ if (unit >= 55296) return unit + 8192;
19064
+ return unit;
19065
+ }
19066
+
18937
19067
  // src/state/memory.ts
18938
19068
  var import_node_fs = require("fs");
18939
19069
  var import_node_path = require("path");
18940
19070
  var import_node_os = require("os");
19071
+ function compareOldestFirst(a, b) {
19072
+ return compareExecutionsNewestFirst(b, a);
19073
+ }
19074
+ function resolveMaxExecutions(value) {
19075
+ if (value === void 0) return DEFAULT_MEMORY_MAX_EXECUTIONS;
19076
+ if (value === Infinity) return Infinity;
19077
+ if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
19078
+ throw new AxlError(
19079
+ "INVALID_CONFIG",
19080
+ `MemoryStore: maxExecutions must be a positive integer or Infinity; got ${typeof value === "number" ? value : typeof value}`
19081
+ );
19082
+ }
19083
+ return value;
19084
+ }
18941
19085
  var AWAIT_HUMAN_TEMP_DIR = (0, import_node_path.join)((0, import_node_os.tmpdir)(), "axl-memory-store");
18942
19086
  var AWAIT_HUMAN_TEMP_FILE = (0, import_node_path.join)(AWAIT_HUMAN_TEMP_DIR, "await-human-state.json");
18943
19087
  var MemoryStore = class {
@@ -18947,9 +19091,32 @@ var MemoryStore = class {
18947
19091
  decisions = /* @__PURE__ */ new Map();
18948
19092
  executionStates = /* @__PURE__ */ new Map();
18949
19093
  memories = /* @__PURE__ */ new Map();
19094
+ /**
19095
+ * Iteration order is last-save order: a save deletes and re-inserts its key.
19096
+ * With one retention per store and a clock that does not run backwards, that
19097
+ * is also expiry order, which lets the prune stop at the first live record.
19098
+ */
18950
19099
  executionHistory = /* @__PURE__ */ new Map();
19100
+ /** Every key in `executionHistory`, sorted oldest first in listing order. */
19101
+ executionOrder = [];
19102
+ /**
19103
+ * Iteration order is first-write order (a re-save keeps the key's position
19104
+ * and its expiry), which is also expiry order for the same reason as above.
19105
+ */
18951
19106
  evalHistory = /* @__PURE__ */ new Map();
18952
- constructor() {
19107
+ ttls;
19108
+ maxExecutions;
19109
+ now;
19110
+ constructor(options) {
19111
+ this.ttls = resolveHistoryTtls("MemoryStore", options?.ttls);
19112
+ this.maxExecutions = resolveMaxExecutions(options?.maxExecutions);
19113
+ if (options?.now !== void 0 && typeof options.now !== "function") {
19114
+ throw new AxlError(
19115
+ "INVALID_CONFIG",
19116
+ "MemoryStore: now must be a function returning epoch milliseconds"
19117
+ );
19118
+ }
19119
+ this.now = options?.now ?? (() => Date.now());
18953
19120
  this.loadPersistedState();
18954
19121
  }
18955
19122
  async saveCheckpoint(executionId, name, data) {
@@ -19046,23 +19213,43 @@ var MemoryStore = class {
19046
19213
  }
19047
19214
  // ── Execution History ──────────────────────────────────────────────
19048
19215
  async saveExecution(execution) {
19049
- const normalized = normalizeStoredExecution({
19050
- ...execution,
19051
- eventSchemaVersion: getExecutionEventSchemaVersion(execution)
19216
+ const now = this.now();
19217
+ const normalized = structuredClone(
19218
+ normalizeStoredExecution({
19219
+ ...execution,
19220
+ eventSchemaVersion: getExecutionEventSchemaVersion(execution)
19221
+ })
19222
+ );
19223
+ this.removeExecutionRecord(normalized.executionId);
19224
+ this.executionHistory.set(normalized.executionId, {
19225
+ record: normalized,
19226
+ expiresAt: expiryFrom(now, this.ttls.executionHistory)
19052
19227
  });
19053
- this.executionHistory.set(execution.executionId, structuredClone(normalized));
19228
+ this.insertExecutionOrder(normalized);
19229
+ this.pruneExpiredExecutions(now);
19230
+ this.evictOverflowExecutions();
19054
19231
  }
19055
19232
  async getExecution(executionId) {
19056
- const exec = this.executionHistory.get(executionId);
19057
- return exec ? normalizeStoredExecution(structuredClone(exec)) : null;
19058
- }
19059
- async listExecutions(limit) {
19060
- const sorted = [...this.executionHistory.values()].sort((a, b) => b.startedAt - a.startedAt);
19061
- const result = limit ? sorted.slice(0, limit) : sorted;
19062
- return result.map((e) => normalizeStoredExecution(structuredClone(e)));
19233
+ const stored = this.executionHistory.get(executionId);
19234
+ if (!stored || isExpired(stored.expiresAt, this.now())) return null;
19235
+ return normalizeStoredExecution(structuredClone(stored.record));
19236
+ }
19237
+ /** Supports `listExecutions(limit, { cursor })`. */
19238
+ historyPaging = true;
19239
+ async listExecutions(limit, options) {
19240
+ const now = this.now();
19241
+ const max = limit ? limit : Infinity;
19242
+ const result = [];
19243
+ const start = options?.cursor !== void 0 ? this.executionOrderLowerBound(options.cursor) - 1 : this.executionOrder.length - 1;
19244
+ for (let i = start; i >= 0 && result.length < max; i--) {
19245
+ const stored = this.executionHistory.get(this.executionOrder[i].executionId);
19246
+ if (isExpired(stored.expiresAt, now)) continue;
19247
+ result.push(normalizeStoredExecution(structuredClone(stored.record)));
19248
+ }
19249
+ return result;
19063
19250
  }
19064
19251
  async deleteExecution(executionId) {
19065
- const removedHistory = this.executionHistory.delete(executionId);
19252
+ const removedHistory = this.removeExecutionRecord(executionId);
19066
19253
  const removedCheckpoints = this.checkpoints.delete(executionId);
19067
19254
  const removedState = this.executionStates.delete(executionId);
19068
19255
  const removedStreaming = this.streamingEvents.delete(executionId);
@@ -19073,26 +19260,193 @@ var MemoryStore = class {
19073
19260
  return removedHistory || removedCheckpoints || removedState || removedStreaming || removedDecision;
19074
19261
  }
19075
19262
  // ── Eval History ──────────────────────────────────────────────────
19263
+ /**
19264
+ * Write an eval history row. Retention counts from the row's first write:
19265
+ * re-saving a live row keeps its expiry (and its position in expiry order).
19266
+ * A row that has already expired is replaced as a new one.
19267
+ */
19076
19268
  async saveEvalResult(entry) {
19077
- this.evalHistory.set(entry.id, structuredClone(entry));
19269
+ const now = this.now();
19270
+ const existing = this.evalHistory.get(entry.id);
19271
+ if (existing && !isExpired(existing.expiresAt, now)) {
19272
+ this.evalHistory.set(entry.id, {
19273
+ entry: structuredClone(entry),
19274
+ expiresAt: existing.expiresAt
19275
+ });
19276
+ } else {
19277
+ this.evalHistory.delete(entry.id);
19278
+ this.evalHistory.set(entry.id, {
19279
+ entry: structuredClone(entry),
19280
+ expiresAt: expiryFrom(now, this.ttls.evalHistory)
19281
+ });
19282
+ }
19283
+ this.pruneExpiredEvals(now);
19078
19284
  }
19079
- /** Update-only write: see `StateStore.updateEvalResult`. */
19285
+ /** Update-only write: see `StateStore.updateEvalResult`. Never changes retention. */
19080
19286
  async updateEvalResult(entry) {
19081
- if (!this.evalHistory.has(entry.id)) return false;
19082
- this.evalHistory.set(entry.id, structuredClone(entry));
19287
+ const existing = this.evalHistory.get(entry.id);
19288
+ if (!existing || isExpired(existing.expiresAt, this.now())) return false;
19289
+ this.evalHistory.set(entry.id, {
19290
+ entry: structuredClone(entry),
19291
+ expiresAt: existing.expiresAt
19292
+ });
19083
19293
  return true;
19084
19294
  }
19085
19295
  async listEvalResults(limit) {
19086
- const sorted = [...this.evalHistory.values()].sort((a, b) => b.timestamp - a.timestamp);
19296
+ const now = this.now();
19297
+ const sorted = [...this.evalHistory.values()].filter((stored) => !isExpired(stored.expiresAt, now)).map((stored) => stored.entry).sort((a, b) => b.timestamp - a.timestamp);
19087
19298
  const result = limit ? sorted.slice(0, limit) : sorted;
19088
19299
  return result.map((e) => structuredClone(e));
19089
19300
  }
19090
19301
  async deleteEvalResult(id) {
19091
19302
  return this.evalHistory.delete(id);
19092
19303
  }
19093
- /** In-memory history never expires on its own — only existence is meaningful. */
19304
+ /** Existence plus the expiry stored at the row's first write, if any. */
19094
19305
  async getEvalRetention(id) {
19095
- return { exists: this.evalHistory.has(id) };
19306
+ const stored = this.evalHistory.get(id);
19307
+ if (!stored || isExpired(stored.expiresAt, this.now())) return { exists: false };
19308
+ return stored.expiresAt === null ? { exists: true } : { exists: true, expiresAt: stored.expiresAt };
19309
+ }
19310
+ // ── History maintenance ───────────────────────────────────────────
19311
+ /**
19312
+ * See `StateStore.applyHistoryRetention`. Every record this store holds was
19313
+ * saved under the retention it was constructed with, so in practice a run
19314
+ * finds nothing to expire or arm; the method exists so the same maintenance
19315
+ * code works against every built-in store. There is no separate index, so
19316
+ * `indexed` and `deadIndexEntries` are always `0`.
19317
+ *
19318
+ * An armed record keeps its place in last-save order, which is also where
19319
+ * its expiry (last save + retention) belongs, so the prune-on-save order
19320
+ * still holds.
19321
+ */
19322
+ async applyHistoryRetention(options) {
19323
+ const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("MemoryStore", options);
19324
+ const now = this.now();
19325
+ let expired = 0;
19326
+ let armed = 0;
19327
+ let positions;
19328
+ if (kind === "executions") {
19329
+ let start = 0;
19330
+ if (cursor !== void 0) {
19331
+ const key = { startedAt: cursor.time, executionId: cursor.id };
19332
+ start = this.executionOrderLowerBound(key);
19333
+ const at = this.executionOrder[start];
19334
+ if (at !== void 0 && compareOldestFirst(at, key) === 0) start++;
19335
+ }
19336
+ positions = this.executionOrder.slice(start, start + batchSize + 1).map((k) => ({ time: k.startedAt, id: k.executionId }));
19337
+ } else {
19338
+ positions = [...this.evalHistory.values()].map((stored) => ({ time: stored.entry.timestamp, id: stored.entry.id })).filter((p) => cursor === void 0 || compareRetentionPositions(cursor, p) < 0).sort(compareRetentionPositions).slice(0, batchSize + 1);
19339
+ }
19340
+ const batch = positions.slice(0, batchSize);
19341
+ const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
19342
+ if (ttl !== null) {
19343
+ for (const { id } of batch) {
19344
+ if (kind === "executions") {
19345
+ const stored = this.executionHistory.get(id);
19346
+ if (!stored || stored.expiresAt !== null) continue;
19347
+ const decision = retentionForUntimedRecord(executionLastSavedAt(stored.record), ttl, now);
19348
+ if (decision.expired) {
19349
+ this.expireExecutionRecord(id);
19350
+ expired++;
19351
+ } else {
19352
+ stored.expiresAt = decision.expiresAt;
19353
+ armed++;
19354
+ }
19355
+ } else {
19356
+ const stored = this.evalHistory.get(id);
19357
+ if (!stored || stored.expiresAt !== null) continue;
19358
+ const decision = retentionForUntimedRecord(stored.entry.timestamp, ttl, now);
19359
+ if (decision.expired) {
19360
+ this.evalHistory.delete(id);
19361
+ expired++;
19362
+ } else {
19363
+ stored.expiresAt = decision.expiresAt;
19364
+ armed++;
19365
+ }
19366
+ }
19367
+ }
19368
+ }
19369
+ return {
19370
+ expired,
19371
+ armed,
19372
+ indexed: 0,
19373
+ deadIndexEntries: 0,
19374
+ cursor: positions.length > batchSize ? batch[batch.length - 1] : null
19375
+ };
19376
+ }
19377
+ // ── History retention internals ───────────────────────────────────
19378
+ /** Index of the first order key not older than `key` (binary search). */
19379
+ executionOrderLowerBound(key) {
19380
+ let lo = 0;
19381
+ let hi = this.executionOrder.length;
19382
+ while (lo < hi) {
19383
+ const mid = lo + hi >>> 1;
19384
+ if (compareOldestFirst(this.executionOrder[mid], key) < 0) lo = mid + 1;
19385
+ else hi = mid;
19386
+ }
19387
+ return lo;
19388
+ }
19389
+ insertExecutionOrder(record) {
19390
+ const key = { startedAt: record.startedAt, executionId: record.executionId };
19391
+ this.executionOrder.splice(this.executionOrderLowerBound(key), 0, key);
19392
+ }
19393
+ /** Remove a history record and its listing-index key. Returns whether one existed. */
19394
+ removeExecutionRecord(executionId) {
19395
+ const stored = this.executionHistory.get(executionId);
19396
+ if (!stored) return false;
19397
+ this.executionHistory.delete(executionId);
19398
+ const index = this.executionOrderLowerBound(stored.record);
19399
+ if (this.executionOrder[index]?.executionId === executionId) {
19400
+ this.executionOrder.splice(index, 1);
19401
+ }
19402
+ return true;
19403
+ }
19404
+ /**
19405
+ * A run leaving history through retention (expiry or `maxExecutions`) takes
19406
+ * its leftover per-run data with it: checkpoints (kept today when a run
19407
+ * fails), saved execution state and any streaming buffer. Pending decisions
19408
+ * are not touched. The runtime never writes execution state, and a run is
19409
+ * only saved to history once it has finished, so nothing still running
19410
+ * depends on what is removed here.
19411
+ */
19412
+ expireExecutionRecord(executionId) {
19413
+ this.removeExecutionRecord(executionId);
19414
+ this.checkpoints.delete(executionId);
19415
+ const state = this.executionStates.get(executionId);
19416
+ this.executionStates.delete(executionId);
19417
+ this.streamingEvents.delete(executionId);
19418
+ if (state?.status === "waiting") this.persistAwaitHumanState();
19419
+ }
19420
+ /**
19421
+ * Remove up to `PRUNE_BATCH` expired execution records, oldest save first.
19422
+ * Stops at the first live record: Map order is last-save order, which is
19423
+ * expiry order while the clock moves forward. If the clock steps back, a
19424
+ * few expired records wait for a later save; reads still filter them.
19425
+ */
19426
+ pruneExpiredExecutions(now) {
19427
+ if (this.ttls.executionHistory === null) return;
19428
+ let removed = 0;
19429
+ for (const [id, stored] of this.executionHistory) {
19430
+ if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
19431
+ this.expireExecutionRecord(id);
19432
+ removed++;
19433
+ }
19434
+ }
19435
+ /** Evict the oldest records in listing order until the count bound holds. */
19436
+ evictOverflowExecutions() {
19437
+ while (this.executionHistory.size > this.maxExecutions) {
19438
+ this.expireExecutionRecord(this.executionOrder[0].executionId);
19439
+ }
19440
+ }
19441
+ /** Remove up to `PRUNE_BATCH` expired eval rows, oldest first write first. */
19442
+ pruneExpiredEvals(now) {
19443
+ if (this.ttls.evalHistory === null) return;
19444
+ let removed = 0;
19445
+ for (const [id, stored] of this.evalHistory) {
19446
+ if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
19447
+ this.evalHistory.delete(id);
19448
+ removed++;
19449
+ }
19096
19450
  }
19097
19451
  // ── Sessions (Studio introspection) ─────────────────────────────────
19098
19452
  async listSessions() {
@@ -19170,6 +19524,7 @@ var MemoryStore = class {
19170
19524
  };
19171
19525
 
19172
19526
  // src/state/sqlite.ts
19527
+ var LIVE = "(expires_at IS NULL OR expires_at > ?)";
19173
19528
  function safeJsonParse(data) {
19174
19529
  try {
19175
19530
  return JSON.parse(data);
@@ -19206,7 +19561,17 @@ function rowToExecutionInfo(row) {
19206
19561
  }
19207
19562
  var SQLiteStore = class {
19208
19563
  db;
19209
- constructor(filePath) {
19564
+ ttls;
19565
+ now;
19566
+ constructor(filePath, options) {
19567
+ this.ttls = resolveHistoryTtls("SQLiteStore", options?.ttls);
19568
+ if (options?.now !== void 0 && typeof options.now !== "function") {
19569
+ throw new AxlError(
19570
+ "INVALID_CONFIG",
19571
+ "SQLiteStore: now must be a function returning epoch milliseconds"
19572
+ );
19573
+ }
19574
+ this.now = options?.now ?? (() => Date.now());
19210
19575
  let BetterSqlite3;
19211
19576
  try {
19212
19577
  BetterSqlite3 = require("better-sqlite3");
@@ -19219,6 +19584,7 @@ var SQLiteStore = class {
19219
19584
  this.db.pragma("journal_mode = WAL");
19220
19585
  this.migrate();
19221
19586
  this.initTables();
19587
+ if (options?.ttls?.executionHistory === void 0) warnExecutionRetentionUnset("SQLiteStore");
19222
19588
  }
19223
19589
  /**
19224
19590
  * Idempotent schema migration. Tracks version via `PRAGMA user_version`
@@ -19246,6 +19612,11 @@ var SQLiteStore = class {
19246
19612
  * Existing rows remain `NULL`, the documented v1 sentinel.
19247
19613
  * - v5 (observation completeness): add `execution_history.observation`.
19248
19614
  * Existing rows remain `NULL`, meaning they predate the contract.
19615
+ * - v6 (history retention): add nullable `expires_at` (epoch ms) to
19616
+ * `execution_history` and `eval_history`. Existing rows remain `NULL`
19617
+ * (never expire), so an upgrade deletes nothing. The single-column
19618
+ * `started_at` index is replaced by `(started_at, execution_id)`, which
19619
+ * serves the tie-safe listing order.
19249
19620
  *
19250
19621
  * Applied BEFORE `initTables()` so the subsequent `CREATE TABLE
19251
19622
  * IF NOT EXISTS` runs against the post-migration column name. Fresh
@@ -19263,7 +19634,7 @@ var SQLiteStore = class {
19263
19634
  * transactional re-read costs nothing.
19264
19635
  */
19265
19636
  migrate() {
19266
- const TARGET_VERSION = 5;
19637
+ const TARGET_VERSION = 6;
19267
19638
  const current = this.db.pragma("user_version", { simple: true });
19268
19639
  if (current >= TARGET_VERSION) return;
19269
19640
  this.db.exec("BEGIN IMMEDIATE");
@@ -19303,6 +19674,15 @@ var SQLiteStore = class {
19303
19674
  this.db.exec("ALTER TABLE execution_history ADD COLUMN observation TEXT");
19304
19675
  }
19305
19676
  }
19677
+ if (committed < 6) {
19678
+ for (const table of ["execution_history", "eval_history"]) {
19679
+ const cols = this.db.pragma(`table_info(${table})`);
19680
+ if (cols.length > 0 && !cols.some((c) => c.name === "expires_at")) {
19681
+ this.db.exec(`ALTER TABLE ${table} ADD COLUMN expires_at INTEGER`);
19682
+ }
19683
+ }
19684
+ this.db.exec("DROP INDEX IF EXISTS idx_exec_history_started");
19685
+ }
19306
19686
  this.db.pragma(`user_version = ${TARGET_VERSION}`);
19307
19687
  this.db.exec("COMMIT");
19308
19688
  } catch (err) {
@@ -19368,18 +19748,28 @@ var SQLiteStore = class {
19368
19748
  events TEXT NOT NULL,
19369
19749
  metadata TEXT,
19370
19750
  event_schema_version INTEGER,
19371
- observation TEXT
19751
+ observation TEXT,
19752
+ expires_at INTEGER
19372
19753
  );
19373
19754
 
19374
19755
  CREATE TABLE IF NOT EXISTS eval_history (
19375
19756
  id TEXT PRIMARY KEY,
19376
19757
  eval_name TEXT NOT NULL,
19377
19758
  timestamp INTEGER NOT NULL,
19378
- data TEXT NOT NULL
19759
+ data TEXT NOT NULL,
19760
+ expires_at INTEGER
19379
19761
  );
19380
19762
 
19381
- CREATE INDEX IF NOT EXISTS idx_exec_history_started ON execution_history (started_at DESC);
19763
+ CREATE INDEX IF NOT EXISTS idx_exec_history_started_id
19764
+ ON execution_history (started_at DESC, execution_id DESC);
19765
+ CREATE INDEX IF NOT EXISTS idx_exec_history_expires ON execution_history (expires_at);
19382
19766
  CREATE INDEX IF NOT EXISTS idx_eval_history_timestamp ON eval_history (timestamp DESC);
19767
+ CREATE INDEX IF NOT EXISTS idx_eval_history_expires ON eval_history (expires_at);
19768
+
19769
+ -- Superseded by idx_exec_history_started_id in v6. Dropped here as well as in
19770
+ -- migrate(): a pre-v6 process reopening a v6 file re-creates it, and
19771
+ -- migrate() never runs again once user_version is 6.
19772
+ DROP INDEX IF EXISTS idx_exec_history_started;
19383
19773
  `);
19384
19774
  }
19385
19775
  // ── Checkpoints ────────────────────────────────────────────────────────
@@ -19492,30 +19882,71 @@ var SQLiteStore = class {
19492
19882
  ...execution,
19493
19883
  eventSchemaVersion: getExecutionEventSchemaVersion(execution)
19494
19884
  });
19885
+ const now = this.now();
19886
+ this.db.transaction(() => {
19887
+ this.db.prepare(
19888
+ "INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, events, metadata, event_schema_version, observation, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
19889
+ ).run(
19890
+ normalized.executionId,
19891
+ normalized.workflow,
19892
+ normalized.status,
19893
+ normalized.totalCost,
19894
+ normalized.startedAt,
19895
+ normalized.completedAt ?? null,
19896
+ normalized.duration,
19897
+ normalized.error ?? null,
19898
+ JSON.stringify(normalized.events),
19899
+ normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
19900
+ normalized.eventSchemaVersion,
19901
+ normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null,
19902
+ expiryFrom(now, this.ttls.executionHistory)
19903
+ );
19904
+ this.pruneExpiredExecutions(now);
19905
+ })();
19906
+ }
19907
+ /**
19908
+ * Remove up to `PRUNE_BATCH` expired execution-history rows, together with
19909
+ * each run's leftover checkpoints and saved execution state, inside the
19910
+ * caller's transaction. Pending decisions are not touched. The runtime
19911
+ * never writes execution state, and a run is only saved to history once it
19912
+ * has finished, so nothing still running depends on what is removed here.
19913
+ */
19914
+ pruneExpiredExecutions(now) {
19915
+ const ids = this.db.prepare(
19916
+ "SELECT execution_id FROM execution_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?"
19917
+ ).all(now, PRUNE_BATCH).map((row) => row.execution_id);
19918
+ if (ids.length === 0) return;
19919
+ const idList = JSON.stringify(ids);
19920
+ for (const table of ["execution_history", "checkpoints", "execution_state"]) {
19921
+ this.db.prepare(`DELETE FROM ${table} WHERE execution_id IN (SELECT value FROM json_each(?))`).run(idList);
19922
+ }
19923
+ }
19924
+ /** Remove up to `PRUNE_BATCH` expired eval-history rows (caller's transaction). */
19925
+ pruneExpiredEvals(now) {
19495
19926
  this.db.prepare(
19496
- "INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, events, metadata, event_schema_version, observation) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
19497
- ).run(
19498
- normalized.executionId,
19499
- normalized.workflow,
19500
- normalized.status,
19501
- normalized.totalCost,
19502
- normalized.startedAt,
19503
- normalized.completedAt ?? null,
19504
- normalized.duration,
19505
- normalized.error ?? null,
19506
- JSON.stringify(normalized.events),
19507
- normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
19508
- normalized.eventSchemaVersion,
19509
- normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null
19510
- );
19927
+ "DELETE FROM eval_history WHERE rowid IN (SELECT rowid FROM eval_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?)"
19928
+ ).run(now, PRUNE_BATCH);
19511
19929
  }
19512
19930
  async getExecution(executionId) {
19513
- const row = this.db.prepare("SELECT * FROM execution_history WHERE execution_id = ?").get(executionId);
19931
+ const row = this.db.prepare(`SELECT * FROM execution_history WHERE execution_id = ? AND ${LIVE}`).get(executionId, this.now());
19514
19932
  return row ? rowToExecutionInfo(row) : null;
19515
19933
  }
19516
- async listExecutions(limit) {
19517
- const sql = limit ? "SELECT * FROM execution_history ORDER BY started_at DESC LIMIT ?" : "SELECT * FROM execution_history ORDER BY started_at DESC";
19518
- const rows = limit ? this.db.prepare(sql).all(limit) : this.db.prepare(sql).all();
19934
+ /** Supports `listExecutions(limit, { cursor })`. */
19935
+ historyPaging = true;
19936
+ async listExecutions(limit, options) {
19937
+ const cursor = options?.cursor;
19938
+ const params = [this.now()];
19939
+ let sql = `SELECT * FROM execution_history WHERE ${LIVE}`;
19940
+ if (cursor !== void 0) {
19941
+ sql += " AND (started_at < ? OR (started_at = ? AND execution_id < ?))";
19942
+ params.push(cursor.startedAt, cursor.startedAt, cursor.executionId);
19943
+ }
19944
+ sql += " ORDER BY started_at DESC, execution_id DESC";
19945
+ if (limit) {
19946
+ sql += " LIMIT ?";
19947
+ params.push(limit);
19948
+ }
19949
+ const rows = this.db.prepare(sql).all(...params);
19519
19950
  return rows.map(rowToExecutionInfo);
19520
19951
  }
19521
19952
  async deleteExecution(executionId) {
@@ -19529,24 +19960,38 @@ var SQLiteStore = class {
19529
19960
  return tx();
19530
19961
  }
19531
19962
  // ── Eval History ────────────────────────────────────────────────────────
19963
+ /**
19964
+ * Write an eval history row. Retention counts from the row's first write:
19965
+ * re-saving a live row keeps its `expires_at` (including none, for a row
19966
+ * written without retention). A row that has already expired is replaced
19967
+ * as a new one.
19968
+ */
19532
19969
  async saveEvalResult(entry) {
19533
- this.db.prepare(
19534
- "INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data) VALUES (?, ?, ?, ?)"
19535
- ).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data));
19970
+ const now = this.now();
19971
+ this.db.transaction(() => {
19972
+ const existing = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(entry.id, now);
19973
+ const expiresAt = existing ? existing.expires_at : expiryFrom(now, this.ttls.evalHistory);
19974
+ this.db.prepare(
19975
+ "INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data, expires_at) VALUES (?, ?, ?, ?, ?)"
19976
+ ).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data), expiresAt);
19977
+ this.pruneExpiredEvals(now);
19978
+ })();
19536
19979
  }
19537
19980
  /**
19538
19981
  * Update-only write: see `StateStore.updateEvalResult`. `UPDATE ... WHERE id`
19539
- * touches nothing when the row is gone, where `INSERT OR REPLACE` would
19540
- * resurrect it — and SQLite history has no expiry, so a resurrection here is
19541
- * permanent.
19982
+ * touches nothing when the row is gone or expired, where `INSERT OR REPLACE`
19983
+ * would resurrect it. `expires_at` is left as it was.
19542
19984
  */
19543
19985
  async updateEvalResult(entry) {
19544
- const result = this.db.prepare("UPDATE eval_history SET eval_name = ?, timestamp = ?, data = ? WHERE id = ?").run(entry.eval, entry.timestamp, JSON.stringify(entry.data), entry.id);
19986
+ const result = this.db.prepare(
19987
+ `UPDATE eval_history SET eval_name = ?, timestamp = ?, data = ? WHERE id = ? AND ${LIVE}`
19988
+ ).run(entry.eval, entry.timestamp, JSON.stringify(entry.data), entry.id, this.now());
19545
19989
  return result.changes > 0;
19546
19990
  }
19547
19991
  async listEvalResults(limit) {
19548
- const sql = limit ? "SELECT * FROM eval_history ORDER BY timestamp DESC LIMIT ?" : "SELECT * FROM eval_history ORDER BY timestamp DESC";
19549
- const rows = limit ? this.db.prepare(sql).all(limit) : this.db.prepare(sql).all();
19992
+ const base = `SELECT * FROM eval_history WHERE ${LIVE} ORDER BY timestamp DESC`;
19993
+ const now = this.now();
19994
+ const rows = limit ? this.db.prepare(`${base} LIMIT ?`).all(now, limit) : this.db.prepare(base).all(now);
19550
19995
  return rows.map((r) => ({
19551
19996
  id: r.id,
19552
19997
  eval: r.eval_name,
@@ -19558,10 +20003,75 @@ var SQLiteStore = class {
19558
20003
  const result = this.db.prepare("DELETE FROM eval_history WHERE id = ?").run(id);
19559
20004
  return result.changes > 0;
19560
20005
  }
19561
- /** SQLite history has no automatic expiry — only existence is meaningful. */
20006
+ /** Existence plus the expiry stored at the row's first write, if any. */
19562
20007
  async getEvalRetention(id) {
19563
- const row = this.db.prepare("SELECT 1 FROM eval_history WHERE id = ?").get(id);
19564
- return { exists: row !== void 0 };
20008
+ const row = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(id, this.now());
20009
+ if (!row) return { exists: false };
20010
+ return row.expires_at === null ? { exists: true } : { exists: true, expiresAt: row.expires_at };
20011
+ }
20012
+ // ── History maintenance ─────────────────────────────────────────────────
20013
+ /**
20014
+ * See `StateStore.applyHistoryRetention`. One batch runs in one immediate
20015
+ * transaction, so no other writer can change a row between this call
20016
+ * reading it and acting on it. Only rows whose `expires_at` is `NULL` are
20017
+ * touched. An expired execution takes its checkpoints and saved execution
20018
+ * state with it, as a normal expiry does. There is no separate index, so
20019
+ * `indexed` and `deadIndexEntries` are always `0`.
20020
+ */
20021
+ async applyHistoryRetention(options) {
20022
+ const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("SQLiteStore", options);
20023
+ const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
20024
+ const table = kind === "executions" ? { name: "execution_history", id: "execution_id", time: "started_at" } : { name: "eval_history", id: "id", time: "timestamp" };
20025
+ const lastSaved = kind === "executions" ? "completed_at" : "NULL";
20026
+ const now = this.now();
20027
+ return this.db.transaction(() => {
20028
+ const params = [];
20029
+ let sql = `SELECT ${table.id} AS id, ${table.time} AS time, ${lastSaved} AS last_saved, expires_at FROM ${table.name}`;
20030
+ if (cursor !== void 0) {
20031
+ sql += ` WHERE ${table.time} > ? OR (${table.time} = ? AND ${table.id} > ?)`;
20032
+ params.push(cursor.time, cursor.time, cursor.id);
20033
+ }
20034
+ sql += ` ORDER BY ${table.time}, ${table.id} LIMIT ?`;
20035
+ params.push(batchSize + 1);
20036
+ const rows = this.db.prepare(sql).all(...params);
20037
+ const batch = rows.slice(0, batchSize);
20038
+ const toExpire = [];
20039
+ let armed = 0;
20040
+ if (ttl !== null) {
20041
+ const arm = this.db.prepare(
20042
+ `UPDATE ${table.name} SET expires_at = ? WHERE ${table.id} = ? AND expires_at IS NULL`
20043
+ );
20044
+ for (const row of batch) {
20045
+ if (row.expires_at !== null) continue;
20046
+ const lastSavedAt = kind === "executions" ? executionLastSavedAt({ startedAt: row.time, completedAt: row.last_saved }) : row.time;
20047
+ const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
20048
+ if (decision.expired) toExpire.push(row.id);
20049
+ else armed += arm.run(decision.expiresAt, row.id).changes;
20050
+ }
20051
+ }
20052
+ let expired = 0;
20053
+ if (toExpire.length > 0) {
20054
+ const idList = JSON.stringify(toExpire);
20055
+ expired = this.db.prepare(
20056
+ `DELETE FROM ${table.name} WHERE expires_at IS NULL AND ${table.id} IN (SELECT value FROM json_each(?))`
20057
+ ).run(idList).changes;
20058
+ if (kind === "executions") {
20059
+ for (const side of ["checkpoints", "execution_state"]) {
20060
+ this.db.prepare(
20061
+ `DELETE FROM ${side} WHERE execution_id IN (SELECT value FROM json_each(?))`
20062
+ ).run(idList);
20063
+ }
20064
+ }
20065
+ }
20066
+ const last = batch[batch.length - 1];
20067
+ return {
20068
+ expired,
20069
+ armed,
20070
+ indexed: 0,
20071
+ deadIndexEntries: 0,
20072
+ cursor: rows.length > batchSize ? { time: last.time, id: last.id } : null
20073
+ };
20074
+ }).immediate();
19565
20075
  }
19566
20076
  // ── Sessions (Studio introspection) ────────────────────────────────────
19567
20077
  async listSessions() {
@@ -19731,11 +20241,18 @@ var Session = class _Session {
19731
20241
  try {
19732
20242
  const completion = axlStream.promise.then(
19733
20243
  (result) => this.commitHistory(history, result).catch((err) => {
19734
- this.runtime.emit("error", {
19735
- type: "session_history_save_failed",
19736
- sessionId: this.sessionId,
19737
- error: err instanceof Error ? err.message : String(err)
19738
- });
20244
+ const error = err instanceof Error ? err.message : String(err);
20245
+ try {
20246
+ this.runtime.emit("error", {
20247
+ type: "session_history_save_failed",
20248
+ sessionId: this.sessionId,
20249
+ error
20250
+ });
20251
+ } catch {
20252
+ console.error(
20253
+ `[axl] Failed to save session history for sessionId=${this.sessionId}: ${error}`
20254
+ );
20255
+ }
19739
20256
  }),
19740
20257
  () => void 0
19741
20258
  );
@@ -20901,6 +21418,9 @@ function hashInput(input) {
20901
21418
  }
20902
21419
  var DEFAULT_MAX_EVENTS_PER_EXECUTION = 5e4;
20903
21420
  var DEFAULT_STREAMING_BATCH_SIZE = 100;
21421
+ var DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE = 1e3;
21422
+ var LARGE_EXECUTION_LISTING = 1e3;
21423
+ var EXECUTION_SCAN_PAGE_SIZE = 100;
20904
21424
  var DEFAULT_STREAMING_BATCH_INTERVAL = 1e3;
20905
21425
  var DEFAULT_ARTIFACT_SWEEP_MS = 6e4;
20906
21426
  var DEFAULT_ARTIFACT_LEASE_MS = 3e5;
@@ -21113,6 +21633,78 @@ function assertNoRemovedObservationCallbacks(options) {
21113
21633
  `runtime.createContext() no longer accepts ${removed.join(", ")}. Observe ctx.events or use runtime.stream() instead. See docs/migration/stream-first-observation.md.`
21114
21634
  );
21115
21635
  }
21636
+ var RecentRunsCache = class {
21637
+ constructor(capacity) {
21638
+ this.capacity = capacity;
21639
+ }
21640
+ entries = /* @__PURE__ */ new Map();
21641
+ get size() {
21642
+ return this.entries.size;
21643
+ }
21644
+ get(executionId) {
21645
+ const entry = this.entries.get(executionId);
21646
+ if (entry) {
21647
+ this.entries.delete(executionId);
21648
+ this.entries.set(executionId, entry);
21649
+ }
21650
+ return entry;
21651
+ }
21652
+ set(executionId, execution) {
21653
+ this.entries.delete(executionId);
21654
+ this.entries.set(executionId, execution);
21655
+ while (this.entries.size > this.capacity) {
21656
+ const oldest = this.entries.keys().next().value;
21657
+ this.entries.delete(oldest);
21658
+ }
21659
+ }
21660
+ delete(executionId) {
21661
+ return this.entries.delete(executionId);
21662
+ }
21663
+ values() {
21664
+ return [...this.entries.values()];
21665
+ }
21666
+ };
21667
+ function resolveRecentExecutionsCacheSize(requested) {
21668
+ if (requested === void 0) return DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE;
21669
+ if (typeof requested !== "number" || !Number.isInteger(requested) || requested < 0) {
21670
+ throw new AxlError(
21671
+ "INVALID_CONFIG",
21672
+ `config.state.recentExecutionsCacheSize must be a non-negative integer; got ${String(requested)}`
21673
+ );
21674
+ }
21675
+ return requested;
21676
+ }
21677
+ function parseGetExecutionsOptions(options) {
21678
+ if (options === void 0) return {};
21679
+ if (options === null || typeof options !== "object") {
21680
+ throw new AxlError(
21681
+ "INVALID_ARGUMENT",
21682
+ `getExecutions() options must be an object; got ${options === null ? "null" : typeof options}`
21683
+ );
21684
+ }
21685
+ const { limit, cursor } = options;
21686
+ if (limit !== void 0 && limit !== Infinity && !(typeof limit === "number" && Number.isInteger(limit) && limit > 0)) {
21687
+ throw new AxlError(
21688
+ "INVALID_ARGUMENT",
21689
+ `getExecutions() limit must be a positive integer; got ${String(limit)}`
21690
+ );
21691
+ }
21692
+ let parsedCursor;
21693
+ if (cursor !== void 0) {
21694
+ const candidate = cursor;
21695
+ if (candidate === null || typeof candidate !== "object" || typeof candidate.startedAt !== "number" || !Number.isFinite(candidate.startedAt) || typeof candidate.executionId !== "string") {
21696
+ throw new AxlError(
21697
+ "INVALID_ARGUMENT",
21698
+ "getExecutions() cursor must be { startedAt: number; executionId: string }, built from the last item of the previous page"
21699
+ );
21700
+ }
21701
+ parsedCursor = { startedAt: candidate.startedAt, executionId: candidate.executionId };
21702
+ }
21703
+ return {
21704
+ ...limit === void 0 || limit === Infinity ? {} : { limit },
21705
+ ...parsedCursor ? { cursor: parsedCursor } : {}
21706
+ };
21707
+ }
21116
21708
  var costScopeStorage = new import_node_async_hooks4.AsyncLocalStorage();
21117
21709
  function forwardAbortSignal(external, internal) {
21118
21710
  if (!external) return () => {
@@ -21159,8 +21751,6 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21159
21751
  mcpManager;
21160
21752
  memoryManager;
21161
21753
  spanManager = new NoopSpanManager();
21162
- historicalExecutions = /* @__PURE__ */ new Map();
21163
- historicalExecutionsLoadPromise = null;
21164
21754
  /** Per-session in-process serialization. Each entry is a chain of pending
21165
21755
  * work for that session id — `Session.send` / `Session.stream` await the
21166
21756
  * prior entry before reading history, so concurrent calls produce a
@@ -21196,12 +21786,34 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21196
21786
  * a delete from resurrecting the (intentionally-removed) row. Entries are
21197
21787
  * removed once the would-be persist has been short-circuited. */
21198
21788
  pendingDeletedExecutions = /* @__PURE__ */ new Set();
21199
- /** In-flight `persistExecution` chains (save → finalize). `shutdown()`
21200
- * awaits these before closing the StateStore so a workflow whose abort
21789
+ /** In-flight `persistExecution` chains (save → finalize) and crash-
21790
+ * recovery saves. `shutdown()` awaits these before closing the StateStore so a workflow whose abort
21201
21791
  * triggers a persist-during-shutdown isn't racing a closed connection.
21202
21792
  * The Set is mutated by the chain's own `.finally` so entries clear as
21203
21793
  * soon as the work settles. */
21204
21794
  persistInflight = /* @__PURE__ */ new Set();
21795
+ /** Finished runs whose history save has not settled, keyed by execution
21796
+ * id. Reads serve the snapshot until the save lands, so a run is visible
21797
+ * the moment it finishes (read-your-writes). `deleteExecution()` awaits
21798
+ * the entry before deleting from the store, so a save that lands late
21799
+ * cannot re-create a deleted run. Each entry removes itself when its save
21800
+ * settles (success or failure), but only if it is still the current entry
21801
+ * for that id, so the map holds only in-flight saves. */
21802
+ pendingSaves = /* @__PURE__ */ new Map();
21803
+ /** Recently finished runs, kept only when the store cannot save, list, or
21804
+ * fetch execution history itself (and the cache size is not `0`).
21805
+ * `undefined` for every built-in store: execution history is read through
21806
+ * the store on every call and never cached here. */
21807
+ recentRuns;
21808
+ /** Execution ids `recoverIncompleteStreams()` is working on, each with a
21809
+ * promise that settles when recovery is done with that id.
21810
+ * `deleteExecution()` awaits it, so a recovered record cannot land after
21811
+ * the delete. */
21812
+ recoveringExecutions = /* @__PURE__ */ new Map();
21813
+ /** Execution ids with a `deleteExecution()` in progress, with a count of
21814
+ * concurrent deletes. Recovery skips them: their store sweep removes the
21815
+ * streaming buffer it would otherwise turn back into a record. */
21816
+ deletingExecutions = /* @__PURE__ */ new Map();
21205
21817
  /** Resolved diagnostic artifact backend, when `diagnostics.artifacts` is set. */
21206
21818
  artifactStore;
21207
21819
  artifactLeaseMs = DEFAULT_ARTIFACT_LEASE_MS;
@@ -21235,6 +21847,13 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21235
21847
  this.providerRegistry = new ProviderRegistry();
21236
21848
  this.transcriptionProviderRegistry = new TranscriptionProviderRegistry();
21237
21849
  this.stateStore = this.createStateStore();
21850
+ const recentRunsSize = resolveRecentExecutionsCacheSize(
21851
+ this.config.state?.recentExecutionsCacheSize
21852
+ );
21853
+ const storeKeepsHistory = Boolean(
21854
+ this.stateStore.saveExecution && this.stateStore.listExecutions
21855
+ );
21856
+ this.recentRuns = !storeKeepsHistory && recentRunsSize > 0 ? new RecentRunsCache(recentRunsSize) : void 0;
21238
21857
  const requested = this.config.state?.maxEventsPerExecution;
21239
21858
  if (requested === void 0) {
21240
21859
  this.maxEventsPerExecution = DEFAULT_MAX_EVENTS_PER_EXECUTION;
@@ -21640,11 +22259,15 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21640
22259
  const storeOption = this.config.state?.store ?? "memory";
21641
22260
  if (typeof storeOption !== "string") return storeOption;
21642
22261
  switch (storeOption) {
21643
- case "sqlite":
21644
- return new SQLiteStore(this.config.state?.sqlite?.path ?? "./data/axl.db");
22262
+ case "sqlite": {
22263
+ const create = () => new SQLiteStore(this.config.state?.sqlite?.path ?? "./data/axl.db", {
22264
+ ttls: this.config.state?.sqlite?.ttls
22265
+ });
22266
+ return this.config.diagnostics?.silent ? withDiagnosticsSilenced(create) : create();
22267
+ }
21645
22268
  case "memory":
21646
22269
  default:
21647
- return new MemoryStore();
22270
+ return new MemoryStore(this.config.state?.memory);
21648
22271
  }
21649
22272
  }
21650
22273
  /**
@@ -21696,7 +22319,11 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21696
22319
  observation
21697
22320
  });
21698
22321
  if (this.stateStore.deleteCheckpoints) {
21699
- await this.stateStore.deleteCheckpoints(execInfo.executionId);
22322
+ await this.stateStore.deleteCheckpoints(execInfo.executionId).catch((err) => {
22323
+ console.error(
22324
+ `[axl] Failed to delete checkpoints for executionId=${execInfo.executionId}: ` + (err instanceof Error ? err.message : String(err))
22325
+ );
22326
+ });
21700
22327
  }
21701
22328
  span?.setAttribute("axl.workflow.cost", execInfo.totalCost);
21702
22329
  span?.setAttribute("axl.workflow.duration", execInfo.duration);
@@ -21732,13 +22359,15 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21732
22359
  }
21733
22360
  /**
21734
22361
  * Persist a completed/failed execution to the state store (fire-and-forget)
21735
- * and move it from the active executions map to the historical cache.
22362
+ * and remove it from the active executions map. Until the save settles the
22363
+ * run is served from `pendingSaves`; afterwards from the store (or, for a
22364
+ * store that cannot keep history, from `recentRuns`).
21736
22365
  *
21737
22366
  * Resurrection guard: if `runtime.deleteExecution(id)` was called while
21738
22367
  * this execution was still running, the id will be in
21739
- * `pendingDeletedExecutions`. In that case we skip the save AND the
21740
- * historical-cache write, and finalize the streaming buffer (if any) so
21741
- * the buffer doesn't outlive the delete.
22368
+ * `pendingDeletedExecutions`. In that case we skip the save AND every
22369
+ * in-process record of the run, and finalize the streaming buffer (if any)
22370
+ * so the buffer doesn't outlive the delete.
21742
22371
  */
21743
22372
  persistExecution(execInfo) {
21744
22373
  const id = execInfo.executionId;
@@ -21766,17 +22395,25 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21766
22395
  const chain = this.stateStore.saveExecution(snapshot).then(() => {
21767
22396
  return this.streamingFlusher?.finalize(id).catch(() => {
21768
22397
  });
21769
- }).catch(() => {
22398
+ }).catch((err) => {
22399
+ console.error(
22400
+ `[axl] Failed to save execution history for executionId=${id}: ` + (err instanceof Error ? err.message : String(err))
22401
+ );
21770
22402
  }).then(() => void 0);
21771
22403
  this.persistInflight.add(chain);
21772
- chain.finally(() => this.persistInflight.delete(chain));
22404
+ const pending = { snapshot, settled: chain };
22405
+ this.pendingSaves.set(id, pending);
22406
+ chain.finally(() => {
22407
+ this.persistInflight.delete(chain);
22408
+ if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
22409
+ });
21773
22410
  } else if (this.streamingFlusher) {
21774
22411
  const chain = this.streamingFlusher.finalize(id).catch(() => {
21775
22412
  });
21776
22413
  this.persistInflight.add(chain);
21777
22414
  chain.finally(() => this.persistInflight.delete(chain));
21778
22415
  }
21779
- this.historicalExecutions.set(id, snapshot);
22416
+ this.recentRuns?.set(id, snapshot);
21780
22417
  this.executions.delete(id);
21781
22418
  }
21782
22419
  /** Register a workflow with the runtime. */
@@ -21902,7 +22539,6 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21902
22539
  }
21903
22540
  return result;
21904
22541
  }
21905
- /** Get all execution info (running + completed + historical). */
21906
22542
  /** Normalize an execution loaded from a `StateStore` so the
21907
22543
  * historical event union holds for downstream consumers
21908
22544
  * (Studio aggregators, REST routes, redaction). Custom or legacy stores
@@ -21922,27 +22558,78 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
21922
22558
  }
21923
22559
  return normalizeStoredExecution(exec);
21924
22560
  }
21925
- async getExecutions() {
21926
- if (!this.historicalExecutionsLoadPromise && this.stateStore.listExecutions) {
21927
- this.historicalExecutionsLoadPromise = this.stateStore.listExecutions().then((stored) => {
21928
- for (const exec of stored) {
21929
- if (!this.executions.has(exec.executionId) && !this.historicalExecutions.has(exec.executionId)) {
21930
- this.historicalExecutions.set(exec.executionId, this.normalizeStoredExecution(exec));
21931
- }
21932
- }
21933
- }).catch((error) => {
21934
- this.historicalExecutionsLoadPromise = null;
21935
- throw error;
21936
- });
21937
- }
21938
- if (this.historicalExecutionsLoadPromise) {
21939
- await this.historicalExecutionsLoadPromise;
21940
- }
21941
- const merged = new Map(this.historicalExecutions);
21942
- for (const [id, exec] of this.executions) {
21943
- merged.set(id, exec);
22561
+ /**
22562
+ * List executions, newest first: running runs, runs whose save is still in
22563
+ * flight, and the history the state store holds. Ordered by `startedAt`,
22564
+ * then `executionId`, both descending.
22565
+ *
22566
+ * With no options, returns every retained run. Pass `limit` for one page,
22567
+ * and `cursor` (built from the last item of the previous page) for the
22568
+ * page after it. Each call reads through the store, so runs finished or
22569
+ * deleted by other runtimes sharing the store are reflected immediately.
22570
+ * A call without `limit` that returns more than 1 000 runs warns once per
22571
+ * process (silenced by `diagnostics.silent`); pass `limit: Infinity` to ask
22572
+ * for everything without the warning.
22573
+ *
22574
+ * @throws {AxlError} `INVALID_ARGUMENT` for a `limit` that is not a positive
22575
+ * integer (`Infinity` means no limit) or a malformed `cursor`.
22576
+ */
22577
+ async getExecutions(options) {
22578
+ const { limit, cursor } = parseGetExecutionsOptions(options);
22579
+ const isAfterCursor = (exec) => cursor === void 0 || compareExecutionsNewestFirst(exec, cursor) > 0;
22580
+ const live = new Map(this.executions);
22581
+ for (const [id, { snapshot }] of this.pendingSaves) {
22582
+ if (!live.has(id)) live.set(id, snapshot);
22583
+ }
22584
+ const recent = this.recentRunsServeListing() ? this.recentRuns.values() : [];
22585
+ const stored = await this.listStoredExecutions({
22586
+ limit,
22587
+ cursor,
22588
+ overlap: live.size + recent.length
22589
+ });
22590
+ const merged = /* @__PURE__ */ new Map();
22591
+ for (const [id, exec] of live) {
22592
+ if (isAfterCursor(exec)) merged.set(id, exec);
22593
+ }
22594
+ for (const exec of stored) {
22595
+ const id = exec.executionId;
22596
+ if (!live.has(id) && !merged.has(id)) merged.set(id, exec);
22597
+ }
22598
+ for (const exec of recent) {
22599
+ const id = exec.executionId;
22600
+ if (!live.has(id) && !merged.has(id) && isAfterCursor(exec)) merged.set(id, exec);
22601
+ }
22602
+ const sorted = [...merged.values()].sort(compareExecutionsNewestFirst);
22603
+ if (limit !== void 0) return sorted.slice(0, limit);
22604
+ if (options?.limit === void 0 && sorted.length > LARGE_EXECUTION_LISTING) {
22605
+ warnDiagnosticOnce(
22606
+ "large_execution_listing",
22607
+ `runtime.getExecutions() without a limit returned ${sorted.length} runs and reads the whole history on every call; pass { limit, cursor } to read it a page at a time.`,
22608
+ this.config.diagnostics?.silent,
22609
+ "docs/migration/execution-history-retention.md#page-through-large-histories"
22610
+ );
21944
22611
  }
21945
- return [...merged.values()].sort((a, b) => b.startedAt - a.startedAt);
22612
+ return sorted;
22613
+ }
22614
+ /**
22615
+ * Read one page of execution history from the store: the records strictly
22616
+ * after `cursor`, newest first, at least `limit + overlap` of them when the
22617
+ * store holds that many (all of them when `limit` is unset).
22618
+ *
22619
+ * Only a store that declares `historyPaging` is handed the cursor. Any
22620
+ * other store is listed in full and paged here: filtering the output of a
22621
+ * store that ignored the cursor would return its first page again, and an
22622
+ * empty page after that.
22623
+ */
22624
+ async listStoredExecutions(page) {
22625
+ const store = this.stateStore;
22626
+ if (!store.listExecutions) return [];
22627
+ const wanted = page.limit === void 0 ? void 0 : page.limit + page.overlap;
22628
+ const rows = store.historyPaging === true ? await store.listExecutions(wanted, { cursor: page.cursor }) : await store.listExecutions();
22629
+ const cursor = page.cursor;
22630
+ const after = cursor === void 0 ? rows : rows.filter((row) => compareExecutionsNewestFirst(row, cursor) > 0);
22631
+ const window = wanted === void 0 ? after : [...after].sort(compareExecutionsNewestFirst).slice(0, wanted);
22632
+ return window.map((row) => this.normalizeStoredExecution(row));
21946
22633
  }
21947
22634
  /**
21948
22635
  * Create a WorkflowContext for ad-hoc use (evals, tool testing, prototyping).
@@ -22373,20 +23060,25 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
22373
23060
  * Recover executions whose process died mid-run when `state.persist:
22374
23061
  * 'streaming'` was configured. Synthesizes a partial `ExecutionInfo`
22375
23062
  * from the streaming buffer for each — `status: 'failed'`, error
22376
- * `'process terminated'` — registers them in the historical cache so
22377
- * they show up in `getExecutions()`, persists them via the store's
22378
- * `saveExecution`, and finalizes the streaming buffer.
23063
+ * `'process terminated'` — persists them via the store's
23064
+ * `saveExecution` (so they show up in `getExecutions()`), and finalizes
23065
+ * the streaming buffer.
22379
23066
  *
22380
23067
  * Idempotent — if the streaming buffer is already gone (or
22381
23068
  * `listStreamingExecutions` is empty), this is a no-op. Returns the
22382
23069
  * list of recovered `ExecutionInfo`s for caller introspection.
22383
23070
  *
22384
- * Wire this into your process startup AFTER `runtime.getExecutions()`
22385
- * has lazy-loaded the historical cache. Order matters: completed
22386
- * executions already in the store should not be overwritten by a
22387
- * stale streaming buffer that the previous process didn't get to
22388
- * finalize. The implementation checks the historical store before
22389
- * synthesizing, so a re-run is safe.
23071
+ * A run whose final record already exists is never recovered: for each
23072
+ * buffered id this asks the store's `getExecution(id)` and, when a record
23073
+ * is there, only drops the leftover buffer. It does not list history, so
23074
+ * its cost does not grow with the size of the store. A store without
23075
+ * `getExecution` cannot be checked, and every buffered run is recovered.
23076
+ *
23077
+ * Wire this into process startup, before the process accepts new work:
23078
+ * a run live in another process looks the same as a crashed one.
23079
+ * `deleteExecution(id)` called while recovery is working on that id waits
23080
+ * for recovery to finish with it, so the recovered record cannot outlive
23081
+ * the delete.
22390
23082
  */
22391
23083
  async recoverIncompleteStreams() {
22392
23084
  if (!this.stateStore.listStreamingExecutions || !this.stateStore.getStreamingEvents) {
@@ -22394,73 +23086,119 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
22394
23086
  }
22395
23087
  const ids = await this.stateStore.listStreamingExecutions();
22396
23088
  if (ids.length === 0) return [];
22397
- await this.getExecutions();
22398
23089
  const recovered = [];
22399
23090
  for (const id of ids) {
22400
23091
  if (this.executions.has(id) || this.streamableExecutionIds.has(id)) {
22401
23092
  continue;
22402
23093
  }
22403
- const existing = this.historicalExecutions.get(id);
22404
- if (existing) {
22405
- await this.stateStore.finalizeStreamingEvents?.(id);
22406
- continue;
22407
- }
22408
- const events = await this.stateStore.getStreamingEvents(id);
22409
- if (events.length === 0) {
22410
- await this.stateStore.finalizeStreamingEvents?.(id);
22411
- continue;
22412
- }
22413
- const eventSchemaVersion = getEventSchemaVersion(events[0]);
22414
- for (const event of events) {
22415
- if (getEventSchemaVersion(event) !== eventSchemaVersion) {
22416
- throw new Error(`Streaming buffer ${id} contains mixed event schema versions`);
22417
- }
23094
+ if (this.deletingExecutions.has(id) || this.recoveringExecutions.has(id)) continue;
23095
+ let finishRecovery;
23096
+ this.recoveringExecutions.set(
23097
+ id,
23098
+ new Promise((resolve2) => {
23099
+ finishRecovery = resolve2;
23100
+ })
23101
+ );
23102
+ try {
23103
+ const result = await this.recoverStreamingExecution(id);
23104
+ if (result) recovered.push(result);
23105
+ } finally {
23106
+ this.recoveringExecutions.delete(id);
23107
+ finishRecovery();
22418
23108
  }
22419
- const boundedEvents = boundRecoveredEvents(events, this.maxEventsPerExecution);
22420
- const workflowStartEvent = boundedEvents.find((e) => e.type === "workflow_start");
22421
- const lastEvent = boundedEvents[boundedEvents.length - 1];
22422
- const startedAt = workflowStartEvent?.timestamp ?? boundedEvents[0]?.timestamp ?? Date.now();
22423
- const completedAt = lastEvent.timestamp ?? startedAt;
22424
- const workflowName = workflowStartEvent?.workflow ?? RECOVERED_UNKNOWN_WORKFLOW;
22425
- const totalCost = boundedEvents.reduce((sum, e) => sum + eventCostContribution(e), 0);
22426
- const unpriced = boundedEvents.some(isUnpricedLeaf);
22427
- const executionBase = {
22428
- executionId: id,
22429
- workflow: workflowName,
22430
- status: "failed",
22431
- events: boundedEvents,
22432
- totalCost,
22433
- unpriced,
22434
- startedAt,
22435
- completedAt,
22436
- duration: completedAt - startedAt,
22437
- error: "process terminated (recovered from streaming buffer)"
22438
- };
22439
- const synthesized = eventSchemaVersion === 2 ? {
22440
- ...executionBase,
22441
- eventSchemaVersion: 2,
22442
- events: boundedEvents,
22443
- observation: { complete: false, reason: "process_interrupted" }
22444
- } : {
22445
- ...executionBase,
22446
- eventSchemaVersion: 1,
22447
- events: boundedEvents
23109
+ }
23110
+ return recovered;
23111
+ }
23112
+ /** Recover one buffered execution. Returns the synthesized record, or
23113
+ * `undefined` when there was nothing to recover. */
23114
+ async recoverStreamingExecution(id) {
23115
+ if (!this.stateStore.getStreamingEvents) return void 0;
23116
+ const existing = await this.stateStore.getExecution?.(id);
23117
+ if (existing) {
23118
+ await this.stateStore.finalizeStreamingEvents?.(id);
23119
+ return void 0;
23120
+ }
23121
+ const events = await this.stateStore.getStreamingEvents(id);
23122
+ if (events.length === 0) {
23123
+ await this.stateStore.finalizeStreamingEvents?.(id);
23124
+ return void 0;
23125
+ }
23126
+ const eventSchemaVersion = getEventSchemaVersion(events[0]);
23127
+ for (const event of events) {
23128
+ if (getEventSchemaVersion(event) !== eventSchemaVersion) {
23129
+ throw new Error(`Streaming buffer ${id} contains mixed event schema versions`);
23130
+ }
23131
+ }
23132
+ const boundedEvents = boundRecoveredEvents(events, this.maxEventsPerExecution);
23133
+ const workflowStartEvent = boundedEvents.find((e) => e.type === "workflow_start");
23134
+ const lastEvent = boundedEvents[boundedEvents.length - 1];
23135
+ const startedAt = workflowStartEvent?.timestamp ?? boundedEvents[0]?.timestamp ?? Date.now();
23136
+ const completedAt = lastEvent.timestamp ?? startedAt;
23137
+ const workflowName = workflowStartEvent?.workflow ?? RECOVERED_UNKNOWN_WORKFLOW;
23138
+ const totalCost = boundedEvents.reduce((sum, e) => sum + eventCostContribution(e), 0);
23139
+ const unpriced = boundedEvents.some(isUnpricedLeaf);
23140
+ const executionBase = {
23141
+ executionId: id,
23142
+ workflow: workflowName,
23143
+ status: "failed",
23144
+ events: boundedEvents,
23145
+ totalCost,
23146
+ unpriced,
23147
+ startedAt,
23148
+ completedAt,
23149
+ duration: completedAt - startedAt,
23150
+ error: "process terminated (recovered from streaming buffer)"
23151
+ };
23152
+ const synthesized = eventSchemaVersion === 2 ? {
23153
+ ...executionBase,
23154
+ eventSchemaVersion: 2,
23155
+ events: boundedEvents,
23156
+ observation: { complete: false, reason: "process_interrupted" }
23157
+ } : {
23158
+ ...executionBase,
23159
+ eventSchemaVersion: 1,
23160
+ events: boundedEvents
23161
+ };
23162
+ const saveExecution = this.stateStore.saveExecution;
23163
+ if (saveExecution) {
23164
+ const save = (async () => saveExecution.call(this.stateStore, synthesized))();
23165
+ const pending = {
23166
+ snapshot: synthesized,
23167
+ settled: save.then(
23168
+ () => void 0,
23169
+ () => void 0
23170
+ )
22448
23171
  };
22449
- if (this.stateStore.saveExecution) {
22450
- try {
22451
- await this.stateStore.saveExecution(synthesized);
22452
- } catch (err) {
22453
- console.error(
22454
- `[axl] recoverIncompleteStreams: failed to save synthesized execution ${id}: ${err instanceof Error ? err.message : String(err)}. Streaming buffer left in place for next recovery attempt.`
22455
- );
22456
- continue;
22457
- }
23172
+ this.pendingSaves.set(id, pending);
23173
+ this.persistInflight.add(pending.settled);
23174
+ try {
23175
+ await save;
23176
+ } catch (err) {
23177
+ console.error(
23178
+ `[axl] recoverIncompleteStreams: failed to save synthesized execution ${id}: ${err instanceof Error ? err.message : String(err)}. Streaming buffer left in place for next recovery attempt.`
23179
+ );
23180
+ return void 0;
23181
+ } finally {
23182
+ this.persistInflight.delete(pending.settled);
23183
+ if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
22458
23184
  }
22459
- this.historicalExecutions.set(id, synthesized);
22460
- await this.stateStore.finalizeStreamingEvents?.(id);
22461
- recovered.push(synthesized);
22462
23185
  }
22463
- return recovered;
23186
+ this.recentRuns?.set(id, synthesized);
23187
+ const finalize = Promise.resolve(this.stateStore.finalizeStreamingEvents?.(id)).then(
23188
+ () => void 0,
23189
+ (err) => {
23190
+ console.error(
23191
+ `[axl] recoverIncompleteStreams: saved execution ${id} but failed to drop its streaming buffer: ${err instanceof Error ? err.message : String(err)}. The next recovery removes it.`
23192
+ );
23193
+ }
23194
+ );
23195
+ this.persistInflight.add(finalize);
23196
+ try {
23197
+ await finalize;
23198
+ } finally {
23199
+ this.persistInflight.delete(finalize);
23200
+ }
23201
+ return synthesized;
22464
23202
  }
22465
23203
  /** Abort a running execution by its ID. */
22466
23204
  abort(executionId) {
@@ -22470,21 +23208,57 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
22470
23208
  this.abortControllers.delete(executionId);
22471
23209
  }
22472
23210
  }
22473
- /** Get execution details by ID. */
23211
+ /**
23212
+ * Get execution details by ID. Looks in order at running runs, runs whose
23213
+ * save is still in flight, the state store (its `getExecution`, or a scan
23214
+ * of its listing when it has `listExecutions` but no `getExecution`), and
23215
+ * (only for a store that cannot list or save history) the recent-runs
23216
+ * cache. Nothing read from the store is cached, so a run deleted or
23217
+ * expired elsewhere is gone on the next call, and every run
23218
+ * `getExecutions()` lists can be fetched here.
23219
+ */
22474
23220
  async getExecution(executionId) {
22475
- const inMemory = this.executions.get(executionId);
22476
- if (inMemory) return inMemory;
22477
- const cached = this.historicalExecutions.get(executionId);
22478
- if (cached) return cached;
22479
- if (this.stateStore.getExecution) {
22480
- const stored = await this.stateStore.getExecution(executionId);
22481
- if (stored) {
22482
- const normalized = this.normalizeStoredExecution(stored);
22483
- this.historicalExecutions.set(executionId, normalized);
22484
- return normalized;
22485
- }
23221
+ const active = this.executions.get(executionId);
23222
+ if (active) return active;
23223
+ const pending = this.pendingSaves.get(executionId);
23224
+ if (pending) return pending.snapshot;
23225
+ const stored = this.stateStore.getExecution ? await this.stateStore.getExecution(executionId) : await this.findInStoreListing(executionId);
23226
+ if (stored) return this.normalizeStoredExecution(stored);
23227
+ return this.recentRunsServeReads() ? this.recentRuns.get(executionId) : void 0;
23228
+ }
23229
+ /**
23230
+ * By-id read for a store that lists but cannot fetch by id. A paging store
23231
+ * is scanned page by page with the cursor, stopping at the match; any
23232
+ * other store is listed once in full. Returns `null` when the store cannot
23233
+ * list or does not hold the run.
23234
+ */
23235
+ async findInStoreListing(executionId) {
23236
+ const store = this.stateStore;
23237
+ if (!store.listExecutions) return null;
23238
+ if (store.historyPaging !== true) {
23239
+ return (await store.listExecutions()).find((e) => e.executionId === executionId) ?? null;
22486
23240
  }
22487
- return void 0;
23241
+ let cursor;
23242
+ for (; ; ) {
23243
+ const page = await store.listExecutions(EXECUTION_SCAN_PAGE_SIZE, { cursor });
23244
+ const found = page.find((e) => e.executionId === executionId);
23245
+ if (found) return found;
23246
+ if (page.length === 0) return null;
23247
+ const last = [...page].sort(compareExecutionsNewestFirst)[page.length - 1];
23248
+ if (cursor && compareExecutionsNewestFirst(last, cursor) <= 0) return null;
23249
+ cursor = { startedAt: last.startedAt, executionId: last.executionId };
23250
+ }
23251
+ }
23252
+ /** The recent-runs cache stands in for listing when the store cannot list
23253
+ * runs, or never receives them because it cannot save. */
23254
+ recentRunsServeListing() {
23255
+ return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.listExecutions);
23256
+ }
23257
+ /** The recent-runs cache stands in for by-id reads when the store cannot
23258
+ * fetch or list runs (it exists only then), or never receives them because
23259
+ * it cannot save. */
23260
+ recentRunsServeReads() {
23261
+ return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.getExecution);
22488
23262
  }
22489
23263
  /** Save an eval result to history. */
22490
23264
  async saveEvalResult(entry) {
@@ -22517,11 +23291,16 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
22517
23291
  }
22518
23292
  /**
22519
23293
  * Delete an execution from history by id. Removes from the active map (if
22520
- * still running), the historical cache, the configured StateStore, AND
23294
+ * still running), the recent-runs cache, the configured StateStore, AND
22521
23295
  * any execution-scoped side state — pending awaitHuman decisions,
22522
23296
  * resolver maps, streaming buffer registration, in-flight abort controller.
22523
- * Returns true if an entry was actually removed from any of the three
22524
- * primary surfaces (active map, historical cache, store).
23297
+ * Returns true if the run was running, had a save in flight, or was
23298
+ * removed from the recent-runs cache or the store.
23299
+ *
23300
+ * If the run has already finished but its history save has not landed
23301
+ * yet, this method waits for that save (whether it succeeds or fails)
23302
+ * before deleting, so the save cannot re-create the deleted record. It
23303
+ * likewise waits for `recoverIncompleteStreams()` to finish with the id.
22525
23304
  *
22526
23305
  * In-flight handling: if the execution is still running, this method
22527
23306
  * **aborts** it (via the registered abort controller) AND marks the id for
@@ -22546,7 +23325,18 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
22546
23325
  * (`defaultTtl` / `ttls.executionHistory`).
22547
23326
  */
22548
23327
  async deleteExecution(id) {
23328
+ this.deletingExecutions.set(id, (this.deletingExecutions.get(id) ?? 0) + 1);
23329
+ try {
23330
+ return await this.deleteExecutionWhileMarked(id);
23331
+ } finally {
23332
+ const remaining = (this.deletingExecutions.get(id) ?? 1) - 1;
23333
+ if (remaining > 0) this.deletingExecutions.set(id, remaining);
23334
+ else this.deletingExecutions.delete(id);
23335
+ }
23336
+ }
23337
+ async deleteExecutionWhileMarked(id) {
22549
23338
  const wasActive = this.executions.has(id);
23339
+ const hadPendingSave = this.pendingSaves.has(id);
22550
23340
  const pendingDecisionResolver = this.pendingDecisionResolvers.get(id);
22551
23341
  const hadPendingDecision = pendingDecisionResolver !== void 0;
22552
23342
  if (wasActive) {
@@ -22564,22 +23354,25 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
22564
23354
  this.pendingDecisionResolvers.delete(id);
22565
23355
  }
22566
23356
  }
23357
+ await this.pendingSaves.get(id)?.settled;
23358
+ await this.recoveringExecutions.get(id);
22567
23359
  const existing = await this.getExecution(id);
22568
23360
  const workflow2 = existing?.workflow;
22569
23361
  const removedFromActive = this.executions.delete(id);
22570
- const removedFromHistorical = this.historicalExecutions.delete(id);
23362
+ const removedFromRecent = this.recentRuns?.delete(id) ?? false;
22571
23363
  let removedFromStore = false;
22572
23364
  if (this.stateStore.deleteExecution) {
22573
23365
  removedFromStore = await this.stateStore.deleteExecution(id);
22574
23366
  }
23367
+ const removed = removedFromActive || hadPendingSave || removedFromRecent || removedFromStore;
22575
23368
  this.emit("execution_deleted", {
22576
23369
  executionId: id,
22577
23370
  workflow: workflow2,
22578
23371
  wasActive,
22579
23372
  hadPendingDecision,
22580
- removed: removedFromActive || removedFromHistorical || removedFromStore
23373
+ removed
22581
23374
  });
22582
- return removedFromActive || removedFromHistorical || removedFromStore;
23375
+ return removed;
22583
23376
  }
22584
23377
  /**
22585
23378
  * Delete an eval history entry by id. Removes from in-memory cache and
@@ -23997,12 +24790,153 @@ async function* stringStreamFromEvents(source, opts) {
23997
24790
  }
23998
24791
 
23999
24792
  // src/state/redis.ts
24793
+ var PRUNE_HISTORY_INDEX_SCRIPT = `
24794
+ local now = tonumber(ARGV[1])
24795
+ local removed = 0
24796
+ for i = 2, #ARGV do
24797
+ local id = ARGV[i]
24798
+ local pttl = redis.call('PTTL', KEYS[i + 2])
24799
+ if pttl == -2 then
24800
+ redis.call('SREM', KEYS[1], id)
24801
+ redis.call('ZREM', KEYS[2], id)
24802
+ redis.call('ZREM', KEYS[3], id)
24803
+ removed = removed + 1
24804
+ elseif pttl == -1 then
24805
+ redis.call('ZREM', KEYS[3], id)
24806
+ else
24807
+ redis.call('ZADD', KEYS[3], string.format('%d', now + pttl), id)
24808
+ end
24809
+ end
24810
+ return removed
24811
+ `;
24812
+ function compareMembers(a, b) {
24813
+ return Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8"));
24814
+ }
24815
+ function compareNewestFirst(a, b) {
24816
+ if (a.score !== b.score) return b.score - a.score;
24817
+ return compareMembers(b.id, a.id);
24818
+ }
24819
+ function warnIfRedisExecutionRetentionUnset(opts) {
24820
+ if (opts.ttls?.executionHistory === void 0 && typeof opts.defaultTtl !== "number") {
24821
+ warnExecutionRetentionUnset("RedisStore");
24822
+ }
24823
+ }
24824
+ var CLOSE_TIMEOUT_MS = 2e3;
24825
+ function reconnectDelayMs(retries) {
24826
+ return Math.min(2 ** retries * 50, 2e3) + Math.floor(Math.random() * 200);
24827
+ }
24828
+ function abandonConnection(client) {
24829
+ try {
24830
+ client.destroy?.();
24831
+ } catch {
24832
+ }
24833
+ client.unref?.();
24834
+ }
24835
+ function errorMessage(error) {
24836
+ return error instanceof Error ? error.message : String(error);
24837
+ }
24838
+ function connectionEventReporter(client, onConnectionEvent) {
24839
+ if (onConnectionEvent) {
24840
+ const reportFailure = (error) => console.error(`[axl] RedisStore: onConnectionEvent failed: ${errorMessage(error)}`);
24841
+ return (event) => {
24842
+ try {
24843
+ Promise.resolve(onConnectionEvent(event)).catch(reportFailure);
24844
+ } catch (error) {
24845
+ reportFailure(error);
24846
+ }
24847
+ };
24848
+ }
24849
+ let outageLogged = false;
24850
+ return (event) => {
24851
+ if (event.type === "reconnected") {
24852
+ outageLogged = false;
24853
+ console.warn("[axl] RedisStore: reconnected to Redis.");
24854
+ } else if (!outageLogged) {
24855
+ outageLogged = client.isReady === false;
24856
+ console.warn(
24857
+ `[axl] RedisStore: Redis connection lost (${event.error.message}). Store calls fail until it reconnects, which happens automatically.`
24858
+ );
24859
+ }
24860
+ };
24861
+ }
24862
+ async function connectClient(createClient, opts) {
24863
+ let established = false;
24864
+ const client = createClient({
24865
+ ...opts.url ? { url: opts.url } : {},
24866
+ // A command accepted just before a drop rejects instead of waiting for
24867
+ // the reconnect.
24868
+ disableOfflineQueue: true,
24869
+ socket: { reconnectStrategy: (retries) => established ? reconnectDelayMs(retries) : false }
24870
+ });
24871
+ const reportConnectionEvent = connectionEventReporter(client, opts.onConnectionEvent);
24872
+ client.on("error", (error) => {
24873
+ if (!established || client.isOpen === false) return;
24874
+ reportConnectionEvent({
24875
+ type: "error",
24876
+ error: error instanceof Error ? error : new Error(String(error))
24877
+ });
24878
+ });
24879
+ client.on("connect", () => {
24880
+ if (client.isOpen === false) client.unref?.();
24881
+ });
24882
+ client.on("ready", () => {
24883
+ if (!established) {
24884
+ established = true;
24885
+ } else if (client.isOpen !== false) {
24886
+ reportConnectionEvent({ type: "reconnected" });
24887
+ }
24888
+ });
24889
+ await client.connect();
24890
+ return client;
24891
+ }
24000
24892
  var RedisStore = class _RedisStore {
24001
- constructor(client, keyPrefix, ttls) {
24002
- this.client = client;
24893
+ constructor(connection, keyPrefix, ttls) {
24894
+ this.connection = connection;
24003
24895
  this.keyPrefix = keyPrefix;
24004
24896
  this.ttls = ttls;
24005
24897
  }
24898
+ /**
24899
+ * Whether the store can send commands right now, for a health check or for
24900
+ * telling an outage from other failures when a call rejects. `false` while
24901
+ * reconnecting and after `close()`.
24902
+ */
24903
+ get isConnected() {
24904
+ return this.connection.isOpen !== false && this.connection.isReady !== false;
24905
+ }
24906
+ /**
24907
+ * The client for sending a command, failing immediately while disconnected.
24908
+ *
24909
+ * Every command goes through here because node-redis's `disableOfflineQueue`
24910
+ * rejects single commands but not `multi().exec()`, which would wait for a
24911
+ * reconnect attempt to fail or succeed. Nothing may `await` between
24912
+ * `client.multi()` and `.exec()`, or a transaction could get past this check.
24913
+ *
24914
+ * A closed store isn't "unavailable": it falls through to node-redis's own
24915
+ * "The client is closed" rejection.
24916
+ */
24917
+ get client() {
24918
+ if (this.connection.isOpen !== false && this.connection.isReady === false) {
24919
+ throw new AxlError(
24920
+ "REDIS_UNAVAILABLE",
24921
+ "RedisStore: not connected to Redis, so the command was not sent. The store reconnects automatically; retry once Redis is back."
24922
+ );
24923
+ }
24924
+ return this.connection;
24925
+ }
24926
+ /**
24927
+ * History kinds with a prune running in this process. A save skips its
24928
+ * prune while another one of the same kind is in flight: the running one
24929
+ * already covers the oldest expired entries. Created lazily so the
24930
+ * instance needs no constructor state.
24931
+ */
24932
+ prunesInFlight;
24933
+ /**
24934
+ * Supports `listExecutions(limit, { cursor })`. A getter rather than a
24935
+ * field so it lives on the prototype.
24936
+ */
24937
+ get historyPaging() {
24938
+ return true;
24939
+ }
24006
24940
  /**
24007
24941
  * Resolve the TTL for a given storage category. Returns `undefined` when
24008
24942
  * no TTL applies (category is `null` in resolved config, the value is
@@ -24058,8 +24992,7 @@ var RedisStore = class _RedisStore {
24058
24992
  if (err instanceof Error && err.message.includes("createClient")) throw err;
24059
24993
  throw new Error("redis is required for RedisStore. Install it with: npm install redis");
24060
24994
  }
24061
- const client = opts.url ? createClient({ url: opts.url }) : createClient();
24062
- await client.connect();
24995
+ const client = await connectClient(createClient, opts);
24063
24996
  const resolveCategory = (override) => {
24064
24997
  if (override === null) return null;
24065
24998
  if (typeof override === "number") return override;
@@ -24081,9 +25014,15 @@ var RedisStore = class _RedisStore {
24081
25014
  streamingEvents: resolveStreamingEvents(opts.ttls?.streamingEvents)
24082
25015
  };
24083
25016
  const store = new _RedisStore(client, keyPrefix, effectiveTtls);
25017
+ warnIfRedisExecutionRetentionUnset(opts);
24084
25018
  if (!opts.skipMigration) {
24085
- await store.backfillExecutionIndex();
24086
- await store.backfillEvalIndex();
25019
+ try {
25020
+ await store.backfillExecutionIndex();
25021
+ await store.backfillEvalIndex();
25022
+ } catch (error) {
25023
+ abandonConnection(client);
25024
+ throw error;
25025
+ }
24087
25026
  }
24088
25027
  return store;
24089
25028
  }
@@ -24192,13 +25131,9 @@ var RedisStore = class _RedisStore {
24192
25131
  execHistoryKey(executionId) {
24193
25132
  return `${this.keyPrefix}exec-history:${executionId}`;
24194
25133
  }
24195
- // NOTE for future implementer of #3 (TTL config): when the executionHistory
24196
- // and evalHistory data blobs gain TTLs, eviction MUST be driven by
24197
- // ZREMRANGEBYSCORE on the ZSET below — NOT by SREM on these legacy ID sets.
24198
- // The slow-path fallback (`listHistoryByZset`) reads from the SET when
24199
- // `zsetCard < setCard`, so SET-based eviction would race with the fallback
24200
- // and surface dead IDs whose data has been TTL'd away. ZSET eviction +
24201
- // dual-write `SREM` is the clean path; doc the assumption then.
25134
+ // The legacy id set and the sorted set below always change together (save,
25135
+ // delete, prune), so the slow-path guard `ZCARD >= SCARD` in
25136
+ // `listHistoryByZset` and backfill stays meaningful.
24202
25137
  execHistorySetKey() {
24203
25138
  return `${this.keyPrefix}exec-history-ids`;
24204
25139
  }
@@ -24221,6 +25156,36 @@ var RedisStore = class _RedisStore {
24221
25156
  evalHistoryZsetKey() {
24222
25157
  return `${this.keyPrefix}eval-history-z`;
24223
25158
  }
25159
+ /** Keys and blob parsing for one kind of history. See {@link HistoryIndex}. */
25160
+ historyIndex(kind) {
25161
+ if (kind === "execution") {
25162
+ return {
25163
+ kind,
25164
+ label: "execution-history",
25165
+ ids: this.execHistorySetKey(),
25166
+ time: this.execHistoryZsetKey(),
25167
+ expiry: `${this.keyPrefix}exec-history-exp`,
25168
+ dataKey: (id) => this.execHistoryKey(id),
25169
+ lastSavedAtOf: (raw) => {
25170
+ const record = JSON.parse(raw);
25171
+ return typeof record.startedAt === "number" ? executionLastSavedAt(record) : null;
25172
+ }
25173
+ };
25174
+ }
25175
+ return {
25176
+ kind,
25177
+ label: "eval-history",
25178
+ ids: this.evalHistorySetKey(),
25179
+ time: this.evalHistoryZsetKey(),
25180
+ expiry: `${this.keyPrefix}eval-history-exp`,
25181
+ dataKey: (id) => this.evalHistoryKey(id),
25182
+ // Eval retention counts from the first write, which is the row's timestamp.
25183
+ lastSavedAtOf: (raw) => {
25184
+ const timestamp = JSON.parse(raw).timestamp;
25185
+ return typeof timestamp === "number" ? timestamp : null;
25186
+ }
25187
+ };
25188
+ }
24224
25189
  memoryKey(scope) {
24225
25190
  return `${this.keyPrefix}memory:${scope}`;
24226
25191
  }
@@ -24326,7 +25291,7 @@ var RedisStore = class _RedisStore {
24326
25291
  }
24327
25292
  }
24328
25293
  if (dead.length > 0) {
24329
- this.client.sRem(this.pendingExecSetKey(), dead).catch((err) => {
25294
+ this.connection.sRem(this.pendingExecSetKey(), dead).catch((err) => {
24330
25295
  console.error(
24331
25296
  `[axl] RedisStore: failed to prune ${dead.length} stale pending-exec ids: ` + (err instanceof Error ? err.message : String(err))
24332
25297
  );
@@ -24372,7 +25337,7 @@ var RedisStore = class _RedisStore {
24372
25337
  }
24373
25338
  const ttl = this.ttlFor("memory");
24374
25339
  if (ttl !== void 0) {
24375
- this.client.expire(this.memoryKey(scope), ttl).catch((err) => {
25340
+ this.connection.expire(this.memoryKey(scope), ttl).catch((err) => {
24376
25341
  console.error(
24377
25342
  `[axl] RedisStore: failed to set TTL on migrated memory key ${this.memoryKey(scope)}: ` + (err instanceof Error ? err.message : String(err))
24378
25343
  );
@@ -24392,31 +25357,35 @@ var RedisStore = class _RedisStore {
24392
25357
  async saveExecution(execution) {
24393
25358
  const ttl = this.ttlFor("executionHistory");
24394
25359
  const setOptions = ttl !== void 0 ? { EX: ttl } : void 0;
25360
+ const index = this.historyIndex("execution");
25361
+ const id = execution.executionId;
24395
25362
  const storedExecution = normalizeStoredExecution({
24396
25363
  ...execution,
24397
25364
  eventSchemaVersion: getExecutionEventSchemaVersion(execution)
24398
25365
  });
24399
- await this.client.multi().sAdd(this.execHistorySetKey(), execution.executionId).set(this.execHistoryKey(execution.executionId), JSON.stringify(storedExecution), setOptions).zAdd(this.execHistoryZsetKey(), {
24400
- score: execution.startedAt,
24401
- value: execution.executionId
24402
- }).exec();
25366
+ const tx = this.client.multi().sAdd(index.ids, id).set(index.dataKey(id), JSON.stringify(storedExecution), setOptions).zAdd(index.time, { score: execution.startedAt, value: id });
25367
+ if (ttl !== void 0) tx.zAdd(index.expiry, { score: Date.now() + ttl * 1e3, value: id });
25368
+ else tx.zRem(index.expiry, id);
25369
+ await tx.exec();
25370
+ await this.pruneHistoryAfterSave(index);
24403
25371
  }
24404
25372
  async getExecution(executionId) {
24405
25373
  const raw = await this.client.get(this.execHistoryKey(executionId));
24406
25374
  return raw ? normalizeStoredExecution(JSON.parse(raw)) : null;
24407
25375
  }
24408
- async listExecutions(limit) {
25376
+ async listExecutions(limit, options) {
25377
+ const cursor = options?.cursor;
24409
25378
  const executions = await this.listHistoryByZset(
24410
- this.execHistorySetKey(),
24411
- this.execHistoryZsetKey(),
24412
- (id) => this.execHistoryKey(id),
24413
- (a, b) => b.startedAt - a.startedAt,
24414
- limit
25379
+ this.historyIndex("execution"),
25380
+ (e) => ({ score: e.startedAt, id: e.executionId }),
25381
+ limit,
25382
+ cursor !== void 0 ? { score: cursor.startedAt, id: cursor.executionId } : void 0
24415
25383
  );
24416
25384
  return executions.map(normalizeStoredExecution);
24417
25385
  }
24418
25386
  async deleteExecution(executionId) {
24419
- const results = await this.client.multi().sRem(this.execHistorySetKey(), executionId).del(this.execHistoryKey(executionId)).zRem(this.execHistoryZsetKey(), executionId).del(this.checkpointKey(executionId)).del(this.executionStateKey(executionId)).sRem(this.pendingExecSetKey(), executionId).del(this.streamingEventsKey(executionId)).sRem(this.streamingIdsKey(), executionId).hDel(this.decisionsKey(), executionId).exec();
25387
+ const index = this.historyIndex("execution");
25388
+ const results = await this.client.multi().sRem(index.ids, executionId).del(index.dataKey(executionId)).zRem(index.time, executionId).zRem(index.expiry, executionId).del(this.checkpointKey(executionId)).del(this.executionStateKey(executionId)).sRem(this.pendingExecSetKey(), executionId).del(this.streamingEventsKey(executionId)).sRem(this.streamingIdsKey(), executionId).hDel(this.decisionsKey(), executionId).exec();
24420
25389
  return results.some((result) => typeof result === "number" && result > 0);
24421
25390
  }
24422
25391
  // ── Eval History ────────────────────────────────────────────────────
@@ -24441,18 +25410,31 @@ var RedisStore = class _RedisStore {
24441
25410
  * lifetime, so the race costs at most one window's length, never the window
24442
25411
  * itself. It also carries no Redis version floor, where `KEEPTTL` needs 6.0.
24443
25412
  *
24444
- * The index entries carry no TTL and age out lazily, as with `saveExecution`.
25413
+ * As with `saveExecution`, a timed row is also scored in the expiry sorted
25414
+ * set (by the expiry it carries, so a re-save keeps its score), an untimed
25415
+ * one is taken out of it, and a bounded prune follows the save.
24445
25416
  */
24446
25417
  async saveEvalResult(entry) {
24447
25418
  const ttl = this.ttlFor("evalHistory");
24448
- const key = this.evalHistoryKey(entry.id);
25419
+ const index = this.historyIndex("eval");
25420
+ const key = index.dataKey(entry.id);
24449
25421
  let setOptions;
25422
+ let expiresAt;
24450
25423
  if (ttl !== void 0) {
24451
25424
  const remaining = await this.client.pTTL(key);
24452
- if (remaining === -2) setOptions = { EX: ttl };
24453
- else if (remaining > 0) setOptions = { PX: remaining };
25425
+ if (remaining === -2) {
25426
+ setOptions = { EX: ttl };
25427
+ expiresAt = Date.now() + ttl * 1e3;
25428
+ } else if (remaining > 0) {
25429
+ setOptions = { PX: remaining };
25430
+ expiresAt = Date.now() + remaining;
25431
+ }
24454
25432
  }
24455
- await this.client.multi().sAdd(this.evalHistorySetKey(), entry.id).set(key, JSON.stringify(entry), setOptions).zAdd(this.evalHistoryZsetKey(), { score: entry.timestamp, value: entry.id }).exec();
25433
+ const tx = this.client.multi().sAdd(index.ids, entry.id).set(key, JSON.stringify(entry), setOptions).zAdd(index.time, { score: entry.timestamp, value: entry.id });
25434
+ if (expiresAt !== void 0) tx.zAdd(index.expiry, { score: expiresAt, value: entry.id });
25435
+ else tx.zRem(index.expiry, entry.id);
25436
+ await tx.exec();
25437
+ await this.pruneHistoryAfterSave(index);
24456
25438
  }
24457
25439
  /**
24458
25440
  * Update-only write: see `StateStore.updateEvalResult`.
@@ -24492,15 +25474,14 @@ var RedisStore = class _RedisStore {
24492
25474
  }
24493
25475
  async listEvalResults(limit) {
24494
25476
  return this.listHistoryByZset(
24495
- this.evalHistorySetKey(),
24496
- this.evalHistoryZsetKey(),
24497
- (id) => this.evalHistoryKey(id),
24498
- (a, b) => b.timestamp - a.timestamp,
25477
+ this.historyIndex("eval"),
25478
+ (e) => ({ score: e.timestamp, id: e.id }),
24499
25479
  limit
24500
25480
  );
24501
25481
  }
24502
25482
  async deleteEvalResult(id) {
24503
- const [, deletedCount] = await this.client.multi().sRem(this.evalHistorySetKey(), id).del(this.evalHistoryKey(id)).zRem(this.evalHistoryZsetKey(), id).exec();
25483
+ const index = this.historyIndex("eval");
25484
+ const [, deletedCount] = await this.client.multi().sRem(index.ids, id).del(index.dataKey(id)).zRem(index.time, id).zRem(index.expiry, id).exec();
24504
25485
  const deleted = typeof deletedCount === "number" ? deletedCount : 0;
24505
25486
  return deleted > 0;
24506
25487
  }
@@ -24524,50 +25505,292 @@ var RedisStore = class _RedisStore {
24524
25505
  return { exists: true, expiresAt: Date.now() + ttl };
24525
25506
  }
24526
25507
  /**
24527
- * Shared list-by-sorted-set implementation. Uses ZREVRANGEBYSCORE +
24528
- * MGET when the ZSET is populated (post-migration). Falls back to the
24529
- * legacy SET + N×GET + JS-sort path when the ZSET is empty or smaller
24530
- * than the SET (covers users that constructed with `skipMigration: true`
24531
- * before calling backfill).
25508
+ * Shared list-by-sorted-set implementation, newest first by
25509
+ * `(score DESC, member DESC)`, starting strictly after `cursor` when given.
25510
+ *
25511
+ * Fast path (the time sorted set is at least as large as the id set): walk
25512
+ * the sorted set in chunks and MGET each chunk's blobs. Index entries whose
25513
+ * blob is gone (expired, or deleted before the prune caught up) are skipped,
25514
+ * and the walk keeps reading until it has `limit` live records or the set
25515
+ * runs out, however many dead entries are in the way. Each chunk restarts
25516
+ * at the last member read, by score and member, rather than at an offset,
25517
+ * so entries added or removed meanwhile cannot shift the walk.
25518
+ *
25519
+ * Slow path (the time sorted set is behind the id set: backfill has not
25520
+ * run): read every id with GET, then sort and apply the cursor in process.
24532
25521
  */
24533
- async listHistoryByZset(setKey, zsetKey, dataKey, sortCmp, limit) {
24534
- const setSize = await this.client.sCard(setKey);
25522
+ async listHistoryByZset(index, positionOf, limit, cursor) {
25523
+ const setSize = await this.client.sCard(index.ids);
24535
25524
  if (setSize === 0) return [];
24536
- const zsetSize = await this.client.zCard(zsetKey);
24537
- if (zsetSize >= setSize) {
24538
- const fetchCount = limit !== void 0 ? Math.max(limit * 2, limit + 5) : void 0;
24539
- const ids2 = await this.client.zRange(zsetKey, "+inf", "-inf", {
25525
+ const zsetSize = await this.client.zCard(index.time);
25526
+ const max = limit ? limit : void 0;
25527
+ if (zsetSize < setSize) {
25528
+ const ids = await this.client.sMembers(index.ids);
25529
+ const entries = [];
25530
+ for (const id of ids) {
25531
+ const raw = await this.client.get(index.dataKey(id));
25532
+ if (raw == null) continue;
25533
+ try {
25534
+ entries.push(JSON.parse(raw));
25535
+ } catch {
25536
+ }
25537
+ }
25538
+ const ordered = entries.filter((e) => cursor === void 0 || compareNewestFirst(cursor, positionOf(e)) < 0).sort((a, b) => compareNewestFirst(positionOf(a), positionOf(b)));
25539
+ return max !== void 0 ? ordered.slice(0, max) : ordered;
25540
+ }
25541
+ const results = [];
25542
+ const seen = /* @__PURE__ */ new Set();
25543
+ let bound = cursor;
25544
+ let rangeMax = cursor !== void 0 ? cursor.score : "+inf";
25545
+ let anchor;
25546
+ let grow = 1;
25547
+ const read = (offset, count2) => (
25548
+ // ZRANGE BY SCORE REV: start is the highest score, stop the lowest.
25549
+ this.client.zRangeWithScores(index.time, rangeMax, "-inf", {
24540
25550
  BY: "SCORE",
24541
25551
  REV: true,
24542
- ...fetchCount !== void 0 ? { LIMIT: { offset: 0, count: fetchCount } } : {}
25552
+ ...count2 !== void 0 ? { LIMIT: { offset, count: count2 } } : {}
25553
+ })
25554
+ );
25555
+ for (; ; ) {
25556
+ const remaining = max !== void 0 ? max - results.length : void 0;
25557
+ const want = remaining !== void 0 ? Math.max(remaining * 2, remaining + 5) * grow : void 0;
25558
+ let offset = 0;
25559
+ let count2 = want;
25560
+ let members;
25561
+ if (anchor !== void 0 && want !== void 0) {
25562
+ offset = anchor.pos;
25563
+ count2 = want + 1;
25564
+ members = await read(offset, count2);
25565
+ if (members[0]?.value !== anchor.id) {
25566
+ offset = 0;
25567
+ count2 = anchor.pos + 1 + want;
25568
+ members = await read(offset, count2);
25569
+ }
25570
+ } else {
25571
+ members = await read(offset, count2);
25572
+ }
25573
+ const fresh = members.filter(
25574
+ (m) => !seen.has(m.value) && (bound === void 0 || compareNewestFirst(bound, { score: m.score, id: m.value }) < 0)
25575
+ );
25576
+ if (fresh.length > 0) {
25577
+ const raws = await this.client.mGet(fresh.map((m) => index.dataKey(m.value)));
25578
+ for (let i = 0; i < fresh.length; i++) {
25579
+ seen.add(fresh[i].value);
25580
+ const raw = raws[i];
25581
+ if (raw == null) continue;
25582
+ try {
25583
+ results.push(JSON.parse(raw));
25584
+ } catch {
25585
+ continue;
25586
+ }
25587
+ if (max !== void 0 && results.length >= max) return results;
25588
+ }
25589
+ const lastFresh = fresh[fresh.length - 1];
25590
+ bound = { score: lastFresh.score, id: lastFresh.value };
25591
+ grow = 1;
25592
+ } else {
25593
+ grow *= 2;
25594
+ }
25595
+ if (count2 === void 0 || members.length < count2) return results;
25596
+ const last = members[members.length - 1];
25597
+ if (last.score === rangeMax) {
25598
+ anchor = { id: last.value, score: last.score, pos: offset + members.length - 1 };
25599
+ } else {
25600
+ rangeMax = last.score;
25601
+ const before = members.filter((m) => m.score === last.score).length - 1;
25602
+ anchor = { id: last.value, score: last.score, pos: before };
25603
+ }
25604
+ }
25605
+ }
25606
+ // ── History index cleanup ──────────────────────────────────────────────
25607
+ //
25608
+ // Index members cannot carry a TTL, so when a history record expires its
25609
+ // entries in the id set and the time sorted set stay behind. Every history
25610
+ // save runs one bounded prune of its kind: it reads up to PRUNE_BATCH ids
25611
+ // whose expiry has passed from the expiry sorted set, then runs
25612
+ // PRUNE_HISTORY_INDEX_SCRIPT over them. The script checks each record's blob
25613
+ // and updates the indexes in one atomic step, so a save or delete landing
25614
+ // between the read and the script cannot be undone by it: a record saved
25615
+ // again is seen live and stays listed, and a deleted one is already gone.
25616
+ // The script only removes index entries and re-scores expiry entries; it
25617
+ // never adds a listing entry or writes a blob, so it cannot bring back
25618
+ // deleted data. Several instances pruning at once is safe.
25619
+ /**
25620
+ * Run one prune of `index`'s kind after a save. A prune failure does not
25621
+ * fail the save, whose MULTI already committed the record: it is reported
25622
+ * once per process and the next save of that kind tries again. That also
25623
+ * covers a Redis user whose ACL denies `EVAL`.
25624
+ */
25625
+ async pruneHistoryAfterSave(index) {
25626
+ const inFlight = this.prunesInFlight ??= /* @__PURE__ */ new Set();
25627
+ if (inFlight.has(index.kind)) return;
25628
+ inFlight.add(index.kind);
25629
+ try {
25630
+ await this.pruneHistory(index);
25631
+ } catch (err) {
25632
+ if (this.connection.isReady === false) return;
25633
+ warnDiagnosticOnce(
25634
+ `redis_history_prune_failed\0${index.kind}`,
25635
+ `RedisStore: removing expired ${index.label} index entries failed (${err instanceof Error ? err.message : String(err)}). The record was saved; the next save retries the cleanup. Listings stay correct meanwhile. The cleanup runs a Lua script, so the Redis user needs permission to run EVAL.`,
25636
+ void 0,
25637
+ "docs/api-reference.md#redisstore-options"
25638
+ );
25639
+ } finally {
25640
+ inFlight.delete(index.kind);
25641
+ }
25642
+ }
25643
+ async pruneHistory(index) {
25644
+ const candidates = await this.client.zRange(index.expiry, "-inf", Date.now(), {
25645
+ BY: "SCORE",
25646
+ LIMIT: { offset: 0, count: PRUNE_BATCH }
25647
+ });
25648
+ if (candidates.length > 0) await this.runPruneScript(index, candidates);
25649
+ }
25650
+ /**
25651
+ * Run {@link PRUNE_HISTORY_INDEX_SCRIPT} over `ids`, at most PRUNE_BATCH
25652
+ * per call so one script never blocks the server for long.
25653
+ */
25654
+ async runPruneScript(index, ids) {
25655
+ for (let start = 0; start < ids.length; start += PRUNE_BATCH) {
25656
+ const chunk = ids.slice(start, start + PRUNE_BATCH);
25657
+ await this.client.eval(PRUNE_HISTORY_INDEX_SCRIPT, {
25658
+ keys: [index.ids, index.time, index.expiry, ...chunk.map((id) => index.dataKey(id))],
25659
+ arguments: [String(Date.now()), ...chunk]
24543
25660
  });
24544
- if (ids2.length === 0) return [];
24545
- const keys = ids2.map(dataKey);
24546
- const raws = await this.client.mGet(keys);
24547
- const entries2 = [];
24548
- for (const raw of raws) {
24549
- if (raw == null) continue;
25661
+ }
25662
+ }
25663
+ // ── History maintenance ────────────────────────────────────────────────
25664
+ /**
25665
+ * See `StateStore.applyHistoryRetention`. One call:
25666
+ *
25667
+ * 1. On the first call of a walk (no cursor), backfills the time sorted
25668
+ * set from the id set if it is behind (see `backfillExecutionIndex`).
25669
+ * 2. Reads the next `batchSize` ids from the time sorted set, oldest first
25670
+ * by `(score, member)`, strictly after the cursor. The read restarts at
25671
+ * the cursor's position by value, never by offset.
25672
+ * 3. MGETs their blobs and reads each live blob's PTTL.
25673
+ * 4. In one MULTI, for each live blob:
25674
+ * - a TTL (`PTTL > 0`): `ZADD NX` to the expiry set at now + PTTL. A
25675
+ * record already in the expiry set keeps its entry (`indexed` counts
25676
+ * only new entries).
25677
+ * - no TTL (`-1`), with a retention configured: `PEXPIREAT … NX` at last
25678
+ * save + retention. `NX` applies only to a key that still has no TTL,
25679
+ * so a record re-saved meanwhile keeps the fresh window its save gave
25680
+ * it. A window that has already passed uses a timestamp in the past,
25681
+ * which makes Redis delete the key, again only if it is still untimed.
25682
+ * 5. Adds each newly armed record to the expiry set (`ZADD NX`).
25683
+ * 6. Runs the prune script (`PRUNE_HISTORY_INDEX_SCRIPT`) over ids whose
25684
+ * blob is gone and ids just deleted in step 4. It checks each blob
25685
+ * atomically, so a record re-saved at any point stays listed and a
25686
+ * deleted one is never re-indexed.
25687
+ *
25688
+ * Blobs are never written. A record deleted while the call runs can leave
25689
+ * an entry in the expiry set only; the next prune removes it.
25690
+ */
25691
+ async applyHistoryRetention(options) {
25692
+ const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("RedisStore", options);
25693
+ const index = this.historyIndex(kind === "executions" ? "execution" : "eval");
25694
+ const ttl = this.ttlFor(kind === "executions" ? "executionHistory" : "evalHistory");
25695
+ if (cursor === void 0) {
25696
+ if (kind === "executions") await this.backfillExecutionIndex();
25697
+ else await this.backfillEvalIndex();
25698
+ }
25699
+ const positions = await this.readRetentionBatch(index, cursor, batchSize + 1);
25700
+ const batch = positions.slice(0, batchSize);
25701
+ const result = {
25702
+ expired: 0,
25703
+ armed: 0,
25704
+ indexed: 0,
25705
+ deadIndexEntries: 0,
25706
+ cursor: positions.length > batchSize ? { time: batch[batch.length - 1].score, id: batch[batch.length - 1].value } : null
25707
+ };
25708
+ if (batch.length === 0) return result;
25709
+ const ids = batch.map((m) => m.value);
25710
+ const raws = await this.client.mGet(ids.map((id) => index.dataKey(id)));
25711
+ const dead = ids.filter((_, i) => raws[i] == null);
25712
+ const present = ids.flatMap((id, i) => {
25713
+ const raw = raws[i];
25714
+ return raw == null ? [] : [{ id, raw }];
25715
+ });
25716
+ const pttls = await Promise.all(present.map((r) => this.client.pTTL(index.dataKey(r.id))));
25717
+ const now = Date.now();
25718
+ const tx = this.client.multi();
25719
+ const queued = [];
25720
+ for (let i = 0; i < present.length; i++) {
25721
+ const { id, raw } = present[i];
25722
+ const pttl = pttls[i];
25723
+ if (pttl === -2) {
25724
+ dead.push(id);
25725
+ } else if (pttl > 0) {
25726
+ tx.zAdd(index.expiry, { score: now + pttl, value: id }, { condition: "NX" });
25727
+ queued.push({ action: "index", id });
25728
+ } else if (pttl === -1 && ttl !== void 0) {
25729
+ let lastSavedAt;
24550
25730
  try {
24551
- entries2.push(JSON.parse(raw));
25731
+ lastSavedAt = index.lastSavedAtOf(raw);
24552
25732
  } catch {
25733
+ lastSavedAt = null;
25734
+ }
25735
+ if (lastSavedAt === null) continue;
25736
+ const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
25737
+ if (decision.expired) {
25738
+ tx.pExpireAt(index.dataKey(id), 1, "NX");
25739
+ queued.push({ action: "expire", id });
25740
+ } else {
25741
+ tx.pExpireAt(index.dataKey(id), decision.expiresAt, "NX");
25742
+ queued.push({ action: "arm", id, expiresAt: decision.expiresAt });
24553
25743
  }
24554
- if (limit !== void 0 && entries2.length >= limit) break;
24555
25744
  }
24556
- return entries2;
24557
25745
  }
24558
- const ids = await this.client.sMembers(setKey);
24559
- if (ids.length === 0) return [];
24560
- const entries = [];
24561
- for (const id of ids) {
24562
- const raw = await this.client.get(dataKey(id));
24563
- if (raw == null) continue;
24564
- try {
24565
- entries.push(JSON.parse(raw));
24566
- } catch {
25746
+ const expiredIds = [];
25747
+ const armed = [];
25748
+ if (queued.length > 0) {
25749
+ const replies = await tx.exec();
25750
+ for (let i = 0; i < queued.length; i++) {
25751
+ if (Number(replies[i]) !== 1) continue;
25752
+ const entry = queued[i];
25753
+ if (entry.action === "index") result.indexed++;
25754
+ else if (entry.action === "expire") expiredIds.push(entry.id);
25755
+ else armed.push(entry);
24567
25756
  }
24568
25757
  }
24569
- entries.sort(sortCmp);
24570
- return limit ? entries.slice(0, limit) : entries;
25758
+ if (armed.length > 0) {
25759
+ const indexTx = this.client.multi();
25760
+ for (const { id, expiresAt } of armed) {
25761
+ indexTx.zAdd(index.expiry, { score: expiresAt, value: id }, { condition: "NX" });
25762
+ }
25763
+ await indexTx.exec();
25764
+ }
25765
+ result.armed = armed.length;
25766
+ result.expired = expiredIds.length;
25767
+ result.deadIndexEntries = dead.length;
25768
+ if (dead.length > 0 || expiredIds.length > 0) {
25769
+ await this.runPruneScript(index, [...dead, ...expiredIds]);
25770
+ }
25771
+ return result;
25772
+ }
25773
+ /**
25774
+ * Up to `count` members of the time sorted set strictly after `cursor`,
25775
+ * oldest first by `(score, member)`. The members sharing the cursor's score
25776
+ * are read in full and filtered by member; the rest start just above that
25777
+ * score. Neither read uses an offset, so entries removed or added
25778
+ * meanwhile cannot shift the walk.
25779
+ */
25780
+ async readRetentionBatch(index, cursor, count2) {
25781
+ if (cursor === void 0) {
25782
+ return this.client.zRangeWithScores(index.time, "-inf", "+inf", {
25783
+ BY: "SCORE",
25784
+ LIMIT: { offset: 0, count: count2 }
25785
+ });
25786
+ }
25787
+ const ties = (await this.client.zRangeWithScores(index.time, cursor.time, cursor.time, { BY: "SCORE" })).filter((m) => compareMembers(m.value, cursor.id) > 0);
25788
+ if (ties.length >= count2) return ties.slice(0, count2);
25789
+ const rest = await this.client.zRangeWithScores(index.time, `(${cursor.time}`, "+inf", {
25790
+ BY: "SCORE",
25791
+ LIMIT: { offset: 0, count: count2 - ties.length }
25792
+ });
25793
+ return [...ties, ...rest];
24571
25794
  }
24572
25795
  // ── Sessions (Studio introspection) ────────────────────────────────────
24573
25796
  async listSessions() {
@@ -24611,10 +25834,31 @@ var RedisStore = class _RedisStore {
24611
25834
  * must not get a rejected promise for it. `MemoryStore.close()` and
24612
25835
  * better-sqlite3's `close()` are both no-ops the second time; node-redis
24613
25836
  * instead rejects with "The client is closed", so absorb that here.
25837
+ *
25838
+ * Bounded, so a shutdown can't hang on Redis. `QUIT` lets queued commands
25839
+ * finish on a healthy connection, but node-redis never settles it if the
25840
+ * socket closes first, and a stalled server may never answer it. While
25841
+ * disconnected it couldn't be sent at all.
24614
25842
  */
24615
25843
  async close() {
24616
- if (this.client.isOpen === false) return;
24617
- await this.client.quit();
25844
+ if (this.connection.isOpen === false) return;
25845
+ if (this.connection.isReady === false) {
25846
+ abandonConnection(this.connection);
25847
+ return;
25848
+ }
25849
+ let timer;
25850
+ try {
25851
+ await Promise.race([
25852
+ this.connection.quit(),
25853
+ new Promise((_resolve, reject) => {
25854
+ timer = setTimeout(() => reject(new Error("QUIT timed out")), CLOSE_TIMEOUT_MS);
25855
+ })
25856
+ ]);
25857
+ } catch {
25858
+ abandonConnection(this.connection);
25859
+ } finally {
25860
+ clearTimeout(timer);
25861
+ }
24618
25862
  }
24619
25863
  async deleteCheckpoints(executionId) {
24620
25864
  await this.client.del(this.checkpointKey(executionId));