@axlsdk/axl 0.26.0 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -348,7 +348,7 @@ var sharedGuard = typeof installedGuard === "object" && installedGuard !== null
348
348
  var buildPath = typeof __AXL_BUILD_PATH__ === "string" ? __AXL_BUILD_PATH__ : "<source>";
349
349
  var thisCopy = {
350
350
  path: buildPath.startsWith("file:") ? fileURLToPath(buildPath) : buildPath,
351
- version: true ? "0.26.0" : "unknown"
351
+ version: true ? "0.27.1" : "unknown"
352
352
  };
353
353
  function warnCrossCopy(other, joined) {
354
354
  if (other === thisCopy) return;
@@ -12615,8 +12615,17 @@ function detectDroppedRefinements(schema) {
12615
12615
  var DOCS_URL = "docs/observability.md#schema-diagnostics";
12616
12616
  var MAX_WARNED_KEYS = 2048;
12617
12617
  var warnedDiagnosticKeys = /* @__PURE__ */ new Set();
12618
+ var silencedDepth = 0;
12619
+ function withDiagnosticsSilenced(fn) {
12620
+ silencedDepth++;
12621
+ try {
12622
+ return fn();
12623
+ } finally {
12624
+ silencedDepth--;
12625
+ }
12626
+ }
12618
12627
  function warnDiagnosticOnce(key, message, silent, docs = DOCS_URL) {
12619
- if (silent || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
12628
+ if (silent || silencedDepth > 0 || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
12620
12629
  if (warnedDiagnosticKeys.has(key)) return;
12621
12630
  if (warnedDiagnosticKeys.size >= MAX_WARNED_KEYS) warnedDiagnosticKeys.clear();
12622
12631
  warnedDiagnosticKeys.add(key);
@@ -17017,7 +17026,7 @@ ${appendedSchemaText}`;
17017
17026
  `ctx.checkpoint name "${name}" is reserved \u2014 names starting with "__auto/" (case-insensitive) are used by the runtime for ask/spawn/race/parallel/map auto-checkpointing.`
17018
17027
  );
17019
17028
  }
17020
- return this._checkpoint(name, fn);
17029
+ return this._checkpoint(name, fn, { saveRequired: true });
17021
17030
  }
17022
17031
  /**
17023
17032
  * Internal checkpoint implementation shared by both the public
@@ -17047,7 +17056,17 @@ ${appendedSchemaText}`;
17047
17056
  return saved;
17048
17057
  }
17049
17058
  const result = await fn();
17050
- await this.stateStore.saveCheckpoint(this.executionId, name, result);
17059
+ if (!options?.saveRequired && this.autoCheckpointCounters.saveFailed) return result;
17060
+ try {
17061
+ await this.stateStore.saveCheckpoint(this.executionId, name, result);
17062
+ } catch (err) {
17063
+ if (options?.saveRequired) throw err;
17064
+ this.autoCheckpointCounters.saveFailed = true;
17065
+ console.error(
17066
+ `[axl] Failed to save checkpoint "${name}" for executionId=${this.executionId}; a replay would run this step and later ones again: ${err instanceof Error ? err.message : String(err)}`
17067
+ );
17068
+ return result;
17069
+ }
17051
17070
  this.emitEvent({
17052
17071
  type: "checkpoint_save",
17053
17072
  ...agentStamp,
@@ -18680,10 +18699,135 @@ import { AsyncLocalStorage as AsyncLocalStorage4 } from "async_hooks";
18680
18699
  import { appendFileSync, mkdirSync as mkdirSync2 } from "fs";
18681
18700
  import { dirname } from "path";
18682
18701
 
18702
+ // src/state/retention.ts
18703
+ var PRUNE_BATCH = 100;
18704
+ var DEFAULT_MEMORY_MAX_EXECUTIONS = 1e3;
18705
+ function warnExecutionRetentionUnset(storeName) {
18706
+ warnDiagnosticOnce(
18707
+ `execution_retention_unset\0${storeName}`,
18708
+ `${storeName} keeps execution history forever because no retention is set; set ttls.executionHistory${storeName === "RedisStore" ? " (or defaultTtl)" : ""} to a number of seconds, or ttls.executionHistory to null to keep it forever without this warning.`,
18709
+ void 0,
18710
+ "docs/migration/execution-history-retention.md#silencing-the-retention-warning"
18711
+ );
18712
+ }
18713
+ function resolveTtl(storeName, category, value) {
18714
+ if (value === void 0 || value === null) return null;
18715
+ if (typeof value !== "number") {
18716
+ throw new AxlError(
18717
+ "INVALID_CONFIG",
18718
+ `${storeName}: ttls.${category} must be a positive number of seconds or null; got ${typeof value}`
18719
+ );
18720
+ }
18721
+ if (!Number.isFinite(value) || value <= 0) {
18722
+ throw new AxlError(
18723
+ "INVALID_CONFIG",
18724
+ `${storeName}: ttls.${category} must be a positive number of seconds or null; got ${value}`
18725
+ );
18726
+ }
18727
+ return value;
18728
+ }
18729
+ function resolveHistoryTtls(storeName, ttls) {
18730
+ if (ttls !== void 0 && (ttls === null || typeof ttls !== "object")) {
18731
+ throw new AxlError(
18732
+ "INVALID_CONFIG",
18733
+ `${storeName}: ttls must be an object; got ${ttls === null ? "null" : typeof ttls}`
18734
+ );
18735
+ }
18736
+ return {
18737
+ executionHistory: resolveTtl(storeName, "executionHistory", ttls?.executionHistory),
18738
+ evalHistory: resolveTtl(storeName, "evalHistory", ttls?.evalHistory)
18739
+ };
18740
+ }
18741
+ function expiryFrom(now, ttlSeconds) {
18742
+ return ttlSeconds === null ? null : now + Math.round(ttlSeconds * 1e3);
18743
+ }
18744
+ function isExpired(expiresAt, now) {
18745
+ return expiresAt !== null && now >= expiresAt;
18746
+ }
18747
+ var DEFAULT_RETENTION_BATCH_SIZE = 500;
18748
+ function resolveHistoryRetentionOptions(storeName, options) {
18749
+ const method = `${storeName}.applyHistoryRetention()`;
18750
+ if (options === null || typeof options !== "object") {
18751
+ throw new AxlError("INVALID_ARGUMENT", `${method} options must be an object`);
18752
+ }
18753
+ const { kind, batchSize, cursor } = options;
18754
+ if (kind !== "executions" && kind !== "evals") {
18755
+ throw new AxlError(
18756
+ "INVALID_ARGUMENT",
18757
+ `${method} kind must be 'executions' or 'evals'; got ${String(kind)}`
18758
+ );
18759
+ }
18760
+ if (batchSize !== void 0 && !(typeof batchSize === "number" && Number.isSafeInteger(batchSize) && batchSize > 0)) {
18761
+ throw new AxlError(
18762
+ "INVALID_ARGUMENT",
18763
+ `${method} batchSize must be a positive integer; got ${String(batchSize)}`
18764
+ );
18765
+ }
18766
+ let parsedCursor;
18767
+ if (cursor !== void 0 && cursor !== null) {
18768
+ const candidate = cursor;
18769
+ if (typeof candidate !== "object" || typeof candidate.time !== "number" || !Number.isFinite(candidate.time) || typeof candidate.id !== "string") {
18770
+ throw new AxlError(
18771
+ "INVALID_ARGUMENT",
18772
+ `${method} cursor must be the { time, id } cursor returned by the previous call`
18773
+ );
18774
+ }
18775
+ parsedCursor = { time: candidate.time, id: candidate.id };
18776
+ }
18777
+ return {
18778
+ kind,
18779
+ batchSize: batchSize ?? DEFAULT_RETENTION_BATCH_SIZE,
18780
+ cursor: parsedCursor
18781
+ };
18782
+ }
18783
+ function retentionForUntimedRecord(lastSavedAt, ttlSeconds, now) {
18784
+ const expiresAt = lastSavedAt + Math.round(ttlSeconds * 1e3);
18785
+ return isExpired(expiresAt, now) ? { expired: true } : { expired: false, expiresAt };
18786
+ }
18787
+ function executionLastSavedAt(record) {
18788
+ return typeof record.completedAt === "number" && Number.isFinite(record.completedAt) ? record.completedAt : record.startedAt;
18789
+ }
18790
+ function compareExecutionsNewestFirst(a, b) {
18791
+ if (a.startedAt !== b.startedAt) return b.startedAt - a.startedAt;
18792
+ return compareCodePoints(b.executionId, a.executionId);
18793
+ }
18794
+ function compareRetentionPositions(a, b) {
18795
+ if (a.time !== b.time) return a.time - b.time;
18796
+ return compareCodePoints(a.id, b.id);
18797
+ }
18798
+ function compareCodePoints(a, b) {
18799
+ const len = Math.min(a.length, b.length);
18800
+ for (let i = 0; i < len; i++) {
18801
+ const x = a.charCodeAt(i);
18802
+ const y = b.charCodeAt(i);
18803
+ if (x !== y) return codePointRank(x) - codePointRank(y);
18804
+ }
18805
+ return a.length - b.length;
18806
+ }
18807
+ function codePointRank(unit) {
18808
+ if (unit >= 57344) return unit - 2048;
18809
+ if (unit >= 55296) return unit + 8192;
18810
+ return unit;
18811
+ }
18812
+
18683
18813
  // src/state/memory.ts
18684
18814
  import { writeFileSync, readFileSync, existsSync, mkdirSync, unlinkSync } from "fs";
18685
18815
  import { join } from "path";
18686
18816
  import { tmpdir } from "os";
18817
+ function compareOldestFirst(a, b) {
18818
+ return compareExecutionsNewestFirst(b, a);
18819
+ }
18820
+ function resolveMaxExecutions(value) {
18821
+ if (value === void 0) return DEFAULT_MEMORY_MAX_EXECUTIONS;
18822
+ if (value === Infinity) return Infinity;
18823
+ if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
18824
+ throw new AxlError(
18825
+ "INVALID_CONFIG",
18826
+ `MemoryStore: maxExecutions must be a positive integer or Infinity; got ${typeof value === "number" ? value : typeof value}`
18827
+ );
18828
+ }
18829
+ return value;
18830
+ }
18687
18831
  var AWAIT_HUMAN_TEMP_DIR = join(tmpdir(), "axl-memory-store");
18688
18832
  var AWAIT_HUMAN_TEMP_FILE = join(AWAIT_HUMAN_TEMP_DIR, "await-human-state.json");
18689
18833
  var MemoryStore = class {
@@ -18693,9 +18837,32 @@ var MemoryStore = class {
18693
18837
  decisions = /* @__PURE__ */ new Map();
18694
18838
  executionStates = /* @__PURE__ */ new Map();
18695
18839
  memories = /* @__PURE__ */ new Map();
18840
+ /**
18841
+ * Iteration order is last-save order: a save deletes and re-inserts its key.
18842
+ * With one retention per store and a clock that does not run backwards, that
18843
+ * is also expiry order, which lets the prune stop at the first live record.
18844
+ */
18696
18845
  executionHistory = /* @__PURE__ */ new Map();
18846
+ /** Every key in `executionHistory`, sorted oldest first in listing order. */
18847
+ executionOrder = [];
18848
+ /**
18849
+ * Iteration order is first-write order (a re-save keeps the key's position
18850
+ * and its expiry), which is also expiry order for the same reason as above.
18851
+ */
18697
18852
  evalHistory = /* @__PURE__ */ new Map();
18698
- constructor() {
18853
+ ttls;
18854
+ maxExecutions;
18855
+ now;
18856
+ constructor(options) {
18857
+ this.ttls = resolveHistoryTtls("MemoryStore", options?.ttls);
18858
+ this.maxExecutions = resolveMaxExecutions(options?.maxExecutions);
18859
+ if (options?.now !== void 0 && typeof options.now !== "function") {
18860
+ throw new AxlError(
18861
+ "INVALID_CONFIG",
18862
+ "MemoryStore: now must be a function returning epoch milliseconds"
18863
+ );
18864
+ }
18865
+ this.now = options?.now ?? (() => Date.now());
18699
18866
  this.loadPersistedState();
18700
18867
  }
18701
18868
  async saveCheckpoint(executionId, name, data) {
@@ -18792,23 +18959,43 @@ var MemoryStore = class {
18792
18959
  }
18793
18960
  // ── Execution History ──────────────────────────────────────────────
18794
18961
  async saveExecution(execution) {
18795
- const normalized = normalizeStoredExecution({
18796
- ...execution,
18797
- eventSchemaVersion: getExecutionEventSchemaVersion(execution)
18962
+ const now = this.now();
18963
+ const normalized = structuredClone(
18964
+ normalizeStoredExecution({
18965
+ ...execution,
18966
+ eventSchemaVersion: getExecutionEventSchemaVersion(execution)
18967
+ })
18968
+ );
18969
+ this.removeExecutionRecord(normalized.executionId);
18970
+ this.executionHistory.set(normalized.executionId, {
18971
+ record: normalized,
18972
+ expiresAt: expiryFrom(now, this.ttls.executionHistory)
18798
18973
  });
18799
- this.executionHistory.set(execution.executionId, structuredClone(normalized));
18974
+ this.insertExecutionOrder(normalized);
18975
+ this.pruneExpiredExecutions(now);
18976
+ this.evictOverflowExecutions();
18800
18977
  }
18801
18978
  async getExecution(executionId) {
18802
- const exec = this.executionHistory.get(executionId);
18803
- return exec ? normalizeStoredExecution(structuredClone(exec)) : null;
18804
- }
18805
- async listExecutions(limit) {
18806
- const sorted = [...this.executionHistory.values()].sort((a, b) => b.startedAt - a.startedAt);
18807
- const result = limit ? sorted.slice(0, limit) : sorted;
18808
- return result.map((e) => normalizeStoredExecution(structuredClone(e)));
18979
+ const stored = this.executionHistory.get(executionId);
18980
+ if (!stored || isExpired(stored.expiresAt, this.now())) return null;
18981
+ return normalizeStoredExecution(structuredClone(stored.record));
18982
+ }
18983
+ /** Supports `listExecutions(limit, { cursor })`. */
18984
+ historyPaging = true;
18985
+ async listExecutions(limit, options) {
18986
+ const now = this.now();
18987
+ const max = limit ? limit : Infinity;
18988
+ const result = [];
18989
+ const start = options?.cursor !== void 0 ? this.executionOrderLowerBound(options.cursor) - 1 : this.executionOrder.length - 1;
18990
+ for (let i = start; i >= 0 && result.length < max; i--) {
18991
+ const stored = this.executionHistory.get(this.executionOrder[i].executionId);
18992
+ if (isExpired(stored.expiresAt, now)) continue;
18993
+ result.push(normalizeStoredExecution(structuredClone(stored.record)));
18994
+ }
18995
+ return result;
18809
18996
  }
18810
18997
  async deleteExecution(executionId) {
18811
- const removedHistory = this.executionHistory.delete(executionId);
18998
+ const removedHistory = this.removeExecutionRecord(executionId);
18812
18999
  const removedCheckpoints = this.checkpoints.delete(executionId);
18813
19000
  const removedState = this.executionStates.delete(executionId);
18814
19001
  const removedStreaming = this.streamingEvents.delete(executionId);
@@ -18819,26 +19006,193 @@ var MemoryStore = class {
18819
19006
  return removedHistory || removedCheckpoints || removedState || removedStreaming || removedDecision;
18820
19007
  }
18821
19008
  // ── Eval History ──────────────────────────────────────────────────
19009
+ /**
19010
+ * Write an eval history row. Retention counts from the row's first write:
19011
+ * re-saving a live row keeps its expiry (and its position in expiry order).
19012
+ * A row that has already expired is replaced as a new one.
19013
+ */
18822
19014
  async saveEvalResult(entry) {
18823
- this.evalHistory.set(entry.id, structuredClone(entry));
19015
+ const now = this.now();
19016
+ const existing = this.evalHistory.get(entry.id);
19017
+ if (existing && !isExpired(existing.expiresAt, now)) {
19018
+ this.evalHistory.set(entry.id, {
19019
+ entry: structuredClone(entry),
19020
+ expiresAt: existing.expiresAt
19021
+ });
19022
+ } else {
19023
+ this.evalHistory.delete(entry.id);
19024
+ this.evalHistory.set(entry.id, {
19025
+ entry: structuredClone(entry),
19026
+ expiresAt: expiryFrom(now, this.ttls.evalHistory)
19027
+ });
19028
+ }
19029
+ this.pruneExpiredEvals(now);
18824
19030
  }
18825
- /** Update-only write: see `StateStore.updateEvalResult`. */
19031
+ /** Update-only write: see `StateStore.updateEvalResult`. Never changes retention. */
18826
19032
  async updateEvalResult(entry) {
18827
- if (!this.evalHistory.has(entry.id)) return false;
18828
- this.evalHistory.set(entry.id, structuredClone(entry));
19033
+ const existing = this.evalHistory.get(entry.id);
19034
+ if (!existing || isExpired(existing.expiresAt, this.now())) return false;
19035
+ this.evalHistory.set(entry.id, {
19036
+ entry: structuredClone(entry),
19037
+ expiresAt: existing.expiresAt
19038
+ });
18829
19039
  return true;
18830
19040
  }
18831
19041
  async listEvalResults(limit) {
18832
- const sorted = [...this.evalHistory.values()].sort((a, b) => b.timestamp - a.timestamp);
19042
+ const now = this.now();
19043
+ const sorted = [...this.evalHistory.values()].filter((stored) => !isExpired(stored.expiresAt, now)).map((stored) => stored.entry).sort((a, b) => b.timestamp - a.timestamp);
18833
19044
  const result = limit ? sorted.slice(0, limit) : sorted;
18834
19045
  return result.map((e) => structuredClone(e));
18835
19046
  }
18836
19047
  async deleteEvalResult(id) {
18837
19048
  return this.evalHistory.delete(id);
18838
19049
  }
18839
- /** In-memory history never expires on its own — only existence is meaningful. */
19050
+ /** Existence plus the expiry stored at the row's first write, if any. */
18840
19051
  async getEvalRetention(id) {
18841
- return { exists: this.evalHistory.has(id) };
19052
+ const stored = this.evalHistory.get(id);
19053
+ if (!stored || isExpired(stored.expiresAt, this.now())) return { exists: false };
19054
+ return stored.expiresAt === null ? { exists: true } : { exists: true, expiresAt: stored.expiresAt };
19055
+ }
19056
+ // ── History maintenance ───────────────────────────────────────────
19057
+ /**
19058
+ * See `StateStore.applyHistoryRetention`. Every record this store holds was
19059
+ * saved under the retention it was constructed with, so in practice a run
19060
+ * finds nothing to expire or arm; the method exists so the same maintenance
19061
+ * code works against every built-in store. There is no separate index, so
19062
+ * `indexed` and `deadIndexEntries` are always `0`.
19063
+ *
19064
+ * An armed record keeps its place in last-save order, which is also where
19065
+ * its expiry (last save + retention) belongs, so the prune-on-save order
19066
+ * still holds.
19067
+ */
19068
+ async applyHistoryRetention(options) {
19069
+ const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("MemoryStore", options);
19070
+ const now = this.now();
19071
+ let expired = 0;
19072
+ let armed = 0;
19073
+ let positions;
19074
+ if (kind === "executions") {
19075
+ let start = 0;
19076
+ if (cursor !== void 0) {
19077
+ const key = { startedAt: cursor.time, executionId: cursor.id };
19078
+ start = this.executionOrderLowerBound(key);
19079
+ const at = this.executionOrder[start];
19080
+ if (at !== void 0 && compareOldestFirst(at, key) === 0) start++;
19081
+ }
19082
+ positions = this.executionOrder.slice(start, start + batchSize + 1).map((k) => ({ time: k.startedAt, id: k.executionId }));
19083
+ } else {
19084
+ positions = [...this.evalHistory.values()].map((stored) => ({ time: stored.entry.timestamp, id: stored.entry.id })).filter((p) => cursor === void 0 || compareRetentionPositions(cursor, p) < 0).sort(compareRetentionPositions).slice(0, batchSize + 1);
19085
+ }
19086
+ const batch = positions.slice(0, batchSize);
19087
+ const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
19088
+ if (ttl !== null) {
19089
+ for (const { id } of batch) {
19090
+ if (kind === "executions") {
19091
+ const stored = this.executionHistory.get(id);
19092
+ if (!stored || stored.expiresAt !== null) continue;
19093
+ const decision = retentionForUntimedRecord(executionLastSavedAt(stored.record), ttl, now);
19094
+ if (decision.expired) {
19095
+ this.expireExecutionRecord(id);
19096
+ expired++;
19097
+ } else {
19098
+ stored.expiresAt = decision.expiresAt;
19099
+ armed++;
19100
+ }
19101
+ } else {
19102
+ const stored = this.evalHistory.get(id);
19103
+ if (!stored || stored.expiresAt !== null) continue;
19104
+ const decision = retentionForUntimedRecord(stored.entry.timestamp, ttl, now);
19105
+ if (decision.expired) {
19106
+ this.evalHistory.delete(id);
19107
+ expired++;
19108
+ } else {
19109
+ stored.expiresAt = decision.expiresAt;
19110
+ armed++;
19111
+ }
19112
+ }
19113
+ }
19114
+ }
19115
+ return {
19116
+ expired,
19117
+ armed,
19118
+ indexed: 0,
19119
+ deadIndexEntries: 0,
19120
+ cursor: positions.length > batchSize ? batch[batch.length - 1] : null
19121
+ };
19122
+ }
19123
+ // ── History retention internals ───────────────────────────────────
19124
+ /** Index of the first order key not older than `key` (binary search). */
19125
+ executionOrderLowerBound(key) {
19126
+ let lo = 0;
19127
+ let hi = this.executionOrder.length;
19128
+ while (lo < hi) {
19129
+ const mid = lo + hi >>> 1;
19130
+ if (compareOldestFirst(this.executionOrder[mid], key) < 0) lo = mid + 1;
19131
+ else hi = mid;
19132
+ }
19133
+ return lo;
19134
+ }
19135
+ insertExecutionOrder(record) {
19136
+ const key = { startedAt: record.startedAt, executionId: record.executionId };
19137
+ this.executionOrder.splice(this.executionOrderLowerBound(key), 0, key);
19138
+ }
19139
+ /** Remove a history record and its listing-index key. Returns whether one existed. */
19140
+ removeExecutionRecord(executionId) {
19141
+ const stored = this.executionHistory.get(executionId);
19142
+ if (!stored) return false;
19143
+ this.executionHistory.delete(executionId);
19144
+ const index = this.executionOrderLowerBound(stored.record);
19145
+ if (this.executionOrder[index]?.executionId === executionId) {
19146
+ this.executionOrder.splice(index, 1);
19147
+ }
19148
+ return true;
19149
+ }
19150
+ /**
19151
+ * A run leaving history through retention (expiry or `maxExecutions`) takes
19152
+ * its leftover per-run data with it: checkpoints (kept today when a run
19153
+ * fails), saved execution state and any streaming buffer. Pending decisions
19154
+ * are not touched. The runtime never writes execution state, and a run is
19155
+ * only saved to history once it has finished, so nothing still running
19156
+ * depends on what is removed here.
19157
+ */
19158
+ expireExecutionRecord(executionId) {
19159
+ this.removeExecutionRecord(executionId);
19160
+ this.checkpoints.delete(executionId);
19161
+ const state = this.executionStates.get(executionId);
19162
+ this.executionStates.delete(executionId);
19163
+ this.streamingEvents.delete(executionId);
19164
+ if (state?.status === "waiting") this.persistAwaitHumanState();
19165
+ }
19166
+ /**
19167
+ * Remove up to `PRUNE_BATCH` expired execution records, oldest save first.
19168
+ * Stops at the first live record: Map order is last-save order, which is
19169
+ * expiry order while the clock moves forward. If the clock steps back, a
19170
+ * few expired records wait for a later save; reads still filter them.
19171
+ */
19172
+ pruneExpiredExecutions(now) {
19173
+ if (this.ttls.executionHistory === null) return;
19174
+ let removed = 0;
19175
+ for (const [id, stored] of this.executionHistory) {
19176
+ if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
19177
+ this.expireExecutionRecord(id);
19178
+ removed++;
19179
+ }
19180
+ }
19181
+ /** Evict the oldest records in listing order until the count bound holds. */
19182
+ evictOverflowExecutions() {
19183
+ while (this.executionHistory.size > this.maxExecutions) {
19184
+ this.expireExecutionRecord(this.executionOrder[0].executionId);
19185
+ }
19186
+ }
19187
+ /** Remove up to `PRUNE_BATCH` expired eval rows, oldest first write first. */
19188
+ pruneExpiredEvals(now) {
19189
+ if (this.ttls.evalHistory === null) return;
19190
+ let removed = 0;
19191
+ for (const [id, stored] of this.evalHistory) {
19192
+ if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
19193
+ this.evalHistory.delete(id);
19194
+ removed++;
19195
+ }
18842
19196
  }
18843
19197
  // ── Sessions (Studio introspection) ─────────────────────────────────
18844
19198
  async listSessions() {
@@ -18916,6 +19270,7 @@ var MemoryStore = class {
18916
19270
  };
18917
19271
 
18918
19272
  // src/state/sqlite.ts
19273
+ var LIVE = "(expires_at IS NULL OR expires_at > ?)";
18919
19274
  function safeJsonParse(data) {
18920
19275
  try {
18921
19276
  return JSON.parse(data);
@@ -18952,7 +19307,17 @@ function rowToExecutionInfo(row) {
18952
19307
  }
18953
19308
  var SQLiteStore = class {
18954
19309
  db;
18955
- constructor(filePath) {
19310
+ ttls;
19311
+ now;
19312
+ constructor(filePath, options) {
19313
+ this.ttls = resolveHistoryTtls("SQLiteStore", options?.ttls);
19314
+ if (options?.now !== void 0 && typeof options.now !== "function") {
19315
+ throw new AxlError(
19316
+ "INVALID_CONFIG",
19317
+ "SQLiteStore: now must be a function returning epoch milliseconds"
19318
+ );
19319
+ }
19320
+ this.now = options?.now ?? (() => Date.now());
18956
19321
  let BetterSqlite3;
18957
19322
  try {
18958
19323
  BetterSqlite3 = __require("better-sqlite3");
@@ -18965,6 +19330,7 @@ var SQLiteStore = class {
18965
19330
  this.db.pragma("journal_mode = WAL");
18966
19331
  this.migrate();
18967
19332
  this.initTables();
19333
+ if (options?.ttls?.executionHistory === void 0) warnExecutionRetentionUnset("SQLiteStore");
18968
19334
  }
18969
19335
  /**
18970
19336
  * Idempotent schema migration. Tracks version via `PRAGMA user_version`
@@ -18992,6 +19358,11 @@ var SQLiteStore = class {
18992
19358
  * Existing rows remain `NULL`, the documented v1 sentinel.
18993
19359
  * - v5 (observation completeness): add `execution_history.observation`.
18994
19360
  * Existing rows remain `NULL`, meaning they predate the contract.
19361
+ * - v6 (history retention): add nullable `expires_at` (epoch ms) to
19362
+ * `execution_history` and `eval_history`. Existing rows remain `NULL`
19363
+ * (never expire), so an upgrade deletes nothing. The single-column
19364
+ * `started_at` index is replaced by `(started_at, execution_id)`, which
19365
+ * serves the tie-safe listing order.
18995
19366
  *
18996
19367
  * Applied BEFORE `initTables()` so the subsequent `CREATE TABLE
18997
19368
  * IF NOT EXISTS` runs against the post-migration column name. Fresh
@@ -19009,7 +19380,7 @@ var SQLiteStore = class {
19009
19380
  * transactional re-read costs nothing.
19010
19381
  */
19011
19382
  migrate() {
19012
- const TARGET_VERSION = 5;
19383
+ const TARGET_VERSION = 6;
19013
19384
  const current = this.db.pragma("user_version", { simple: true });
19014
19385
  if (current >= TARGET_VERSION) return;
19015
19386
  this.db.exec("BEGIN IMMEDIATE");
@@ -19049,6 +19420,15 @@ var SQLiteStore = class {
19049
19420
  this.db.exec("ALTER TABLE execution_history ADD COLUMN observation TEXT");
19050
19421
  }
19051
19422
  }
19423
+ if (committed < 6) {
19424
+ for (const table of ["execution_history", "eval_history"]) {
19425
+ const cols = this.db.pragma(`table_info(${table})`);
19426
+ if (cols.length > 0 && !cols.some((c) => c.name === "expires_at")) {
19427
+ this.db.exec(`ALTER TABLE ${table} ADD COLUMN expires_at INTEGER`);
19428
+ }
19429
+ }
19430
+ this.db.exec("DROP INDEX IF EXISTS idx_exec_history_started");
19431
+ }
19052
19432
  this.db.pragma(`user_version = ${TARGET_VERSION}`);
19053
19433
  this.db.exec("COMMIT");
19054
19434
  } catch (err) {
@@ -19114,18 +19494,28 @@ var SQLiteStore = class {
19114
19494
  events TEXT NOT NULL,
19115
19495
  metadata TEXT,
19116
19496
  event_schema_version INTEGER,
19117
- observation TEXT
19497
+ observation TEXT,
19498
+ expires_at INTEGER
19118
19499
  );
19119
19500
 
19120
19501
  CREATE TABLE IF NOT EXISTS eval_history (
19121
19502
  id TEXT PRIMARY KEY,
19122
19503
  eval_name TEXT NOT NULL,
19123
19504
  timestamp INTEGER NOT NULL,
19124
- data TEXT NOT NULL
19505
+ data TEXT NOT NULL,
19506
+ expires_at INTEGER
19125
19507
  );
19126
19508
 
19127
- CREATE INDEX IF NOT EXISTS idx_exec_history_started ON execution_history (started_at DESC);
19509
+ CREATE INDEX IF NOT EXISTS idx_exec_history_started_id
19510
+ ON execution_history (started_at DESC, execution_id DESC);
19511
+ CREATE INDEX IF NOT EXISTS idx_exec_history_expires ON execution_history (expires_at);
19128
19512
  CREATE INDEX IF NOT EXISTS idx_eval_history_timestamp ON eval_history (timestamp DESC);
19513
+ CREATE INDEX IF NOT EXISTS idx_eval_history_expires ON eval_history (expires_at);
19514
+
19515
+ -- Superseded by idx_exec_history_started_id in v6. Dropped here as well as in
19516
+ -- migrate(): a pre-v6 process reopening a v6 file re-creates it, and
19517
+ -- migrate() never runs again once user_version is 6.
19518
+ DROP INDEX IF EXISTS idx_exec_history_started;
19129
19519
  `);
19130
19520
  }
19131
19521
  // ── Checkpoints ────────────────────────────────────────────────────────
@@ -19238,30 +19628,71 @@ var SQLiteStore = class {
19238
19628
  ...execution,
19239
19629
  eventSchemaVersion: getExecutionEventSchemaVersion(execution)
19240
19630
  });
19631
+ const now = this.now();
19632
+ this.db.transaction(() => {
19633
+ this.db.prepare(
19634
+ "INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, events, metadata, event_schema_version, observation, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
19635
+ ).run(
19636
+ normalized.executionId,
19637
+ normalized.workflow,
19638
+ normalized.status,
19639
+ normalized.totalCost,
19640
+ normalized.startedAt,
19641
+ normalized.completedAt ?? null,
19642
+ normalized.duration,
19643
+ normalized.error ?? null,
19644
+ JSON.stringify(normalized.events),
19645
+ normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
19646
+ normalized.eventSchemaVersion,
19647
+ normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null,
19648
+ expiryFrom(now, this.ttls.executionHistory)
19649
+ );
19650
+ this.pruneExpiredExecutions(now);
19651
+ })();
19652
+ }
19653
+ /**
19654
+ * Remove up to `PRUNE_BATCH` expired execution-history rows, together with
19655
+ * each run's leftover checkpoints and saved execution state, inside the
19656
+ * caller's transaction. Pending decisions are not touched. The runtime
19657
+ * never writes execution state, and a run is only saved to history once it
19658
+ * has finished, so nothing still running depends on what is removed here.
19659
+ */
19660
+ pruneExpiredExecutions(now) {
19661
+ const ids = this.db.prepare(
19662
+ "SELECT execution_id FROM execution_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?"
19663
+ ).all(now, PRUNE_BATCH).map((row) => row.execution_id);
19664
+ if (ids.length === 0) return;
19665
+ const idList = JSON.stringify(ids);
19666
+ for (const table of ["execution_history", "checkpoints", "execution_state"]) {
19667
+ this.db.prepare(`DELETE FROM ${table} WHERE execution_id IN (SELECT value FROM json_each(?))`).run(idList);
19668
+ }
19669
+ }
19670
+ /** Remove up to `PRUNE_BATCH` expired eval-history rows (caller's transaction). */
19671
+ pruneExpiredEvals(now) {
19241
19672
  this.db.prepare(
19242
- "INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, events, metadata, event_schema_version, observation) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
19243
- ).run(
19244
- normalized.executionId,
19245
- normalized.workflow,
19246
- normalized.status,
19247
- normalized.totalCost,
19248
- normalized.startedAt,
19249
- normalized.completedAt ?? null,
19250
- normalized.duration,
19251
- normalized.error ?? null,
19252
- JSON.stringify(normalized.events),
19253
- normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
19254
- normalized.eventSchemaVersion,
19255
- normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null
19256
- );
19673
+ "DELETE FROM eval_history WHERE rowid IN (SELECT rowid FROM eval_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?)"
19674
+ ).run(now, PRUNE_BATCH);
19257
19675
  }
19258
19676
  async getExecution(executionId) {
19259
- const row = this.db.prepare("SELECT * FROM execution_history WHERE execution_id = ?").get(executionId);
19677
+ const row = this.db.prepare(`SELECT * FROM execution_history WHERE execution_id = ? AND ${LIVE}`).get(executionId, this.now());
19260
19678
  return row ? rowToExecutionInfo(row) : null;
19261
19679
  }
19262
- async listExecutions(limit) {
19263
- const sql = limit ? "SELECT * FROM execution_history ORDER BY started_at DESC LIMIT ?" : "SELECT * FROM execution_history ORDER BY started_at DESC";
19264
- const rows = limit ? this.db.prepare(sql).all(limit) : this.db.prepare(sql).all();
19680
+ /** Supports `listExecutions(limit, { cursor })`. */
19681
+ historyPaging = true;
19682
+ async listExecutions(limit, options) {
19683
+ const cursor = options?.cursor;
19684
+ const params = [this.now()];
19685
+ let sql = `SELECT * FROM execution_history WHERE ${LIVE}`;
19686
+ if (cursor !== void 0) {
19687
+ sql += " AND (started_at < ? OR (started_at = ? AND execution_id < ?))";
19688
+ params.push(cursor.startedAt, cursor.startedAt, cursor.executionId);
19689
+ }
19690
+ sql += " ORDER BY started_at DESC, execution_id DESC";
19691
+ if (limit) {
19692
+ sql += " LIMIT ?";
19693
+ params.push(limit);
19694
+ }
19695
+ const rows = this.db.prepare(sql).all(...params);
19265
19696
  return rows.map(rowToExecutionInfo);
19266
19697
  }
19267
19698
  async deleteExecution(executionId) {
@@ -19275,24 +19706,38 @@ var SQLiteStore = class {
19275
19706
  return tx();
19276
19707
  }
19277
19708
  // ── Eval History ────────────────────────────────────────────────────────
19709
+ /**
19710
+ * Write an eval history row. Retention counts from the row's first write:
19711
+ * re-saving a live row keeps its `expires_at` (including none, for a row
19712
+ * written without retention). A row that has already expired is replaced
19713
+ * as a new one.
19714
+ */
19278
19715
  async saveEvalResult(entry) {
19279
- this.db.prepare(
19280
- "INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data) VALUES (?, ?, ?, ?)"
19281
- ).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data));
19716
+ const now = this.now();
19717
+ this.db.transaction(() => {
19718
+ const existing = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(entry.id, now);
19719
+ const expiresAt = existing ? existing.expires_at : expiryFrom(now, this.ttls.evalHistory);
19720
+ this.db.prepare(
19721
+ "INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data, expires_at) VALUES (?, ?, ?, ?, ?)"
19722
+ ).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data), expiresAt);
19723
+ this.pruneExpiredEvals(now);
19724
+ })();
19282
19725
  }
19283
19726
  /**
19284
19727
  * Update-only write: see `StateStore.updateEvalResult`. `UPDATE ... WHERE id`
19285
- * touches nothing when the row is gone, where `INSERT OR REPLACE` would
19286
- * resurrect it — and SQLite history has no expiry, so a resurrection here is
19287
- * permanent.
19728
+ * touches nothing when the row is gone or expired, where `INSERT OR REPLACE`
19729
+ * would resurrect it. `expires_at` is left as it was.
19288
19730
  */
19289
19731
  async updateEvalResult(entry) {
19290
- const result = this.db.prepare("UPDATE eval_history SET eval_name = ?, timestamp = ?, data = ? WHERE id = ?").run(entry.eval, entry.timestamp, JSON.stringify(entry.data), entry.id);
19732
+ const result = this.db.prepare(
19733
+ `UPDATE eval_history SET eval_name = ?, timestamp = ?, data = ? WHERE id = ? AND ${LIVE}`
19734
+ ).run(entry.eval, entry.timestamp, JSON.stringify(entry.data), entry.id, this.now());
19291
19735
  return result.changes > 0;
19292
19736
  }
19293
19737
  async listEvalResults(limit) {
19294
- const sql = limit ? "SELECT * FROM eval_history ORDER BY timestamp DESC LIMIT ?" : "SELECT * FROM eval_history ORDER BY timestamp DESC";
19295
- const rows = limit ? this.db.prepare(sql).all(limit) : this.db.prepare(sql).all();
19738
+ const base = `SELECT * FROM eval_history WHERE ${LIVE} ORDER BY timestamp DESC`;
19739
+ const now = this.now();
19740
+ const rows = limit ? this.db.prepare(`${base} LIMIT ?`).all(now, limit) : this.db.prepare(base).all(now);
19296
19741
  return rows.map((r) => ({
19297
19742
  id: r.id,
19298
19743
  eval: r.eval_name,
@@ -19304,10 +19749,75 @@ var SQLiteStore = class {
19304
19749
  const result = this.db.prepare("DELETE FROM eval_history WHERE id = ?").run(id);
19305
19750
  return result.changes > 0;
19306
19751
  }
19307
- /** SQLite history has no automatic expiry — only existence is meaningful. */
19752
+ /** Existence plus the expiry stored at the row's first write, if any. */
19308
19753
  async getEvalRetention(id) {
19309
- const row = this.db.prepare("SELECT 1 FROM eval_history WHERE id = ?").get(id);
19310
- return { exists: row !== void 0 };
19754
+ const row = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(id, this.now());
19755
+ if (!row) return { exists: false };
19756
+ return row.expires_at === null ? { exists: true } : { exists: true, expiresAt: row.expires_at };
19757
+ }
19758
+ // ── History maintenance ─────────────────────────────────────────────────
19759
+ /**
19760
+ * See `StateStore.applyHistoryRetention`. One batch runs in one immediate
19761
+ * transaction, so no other writer can change a row between this call
19762
+ * reading it and acting on it. Only rows whose `expires_at` is `NULL` are
19763
+ * touched. An expired execution takes its checkpoints and saved execution
19764
+ * state with it, as a normal expiry does. There is no separate index, so
19765
+ * `indexed` and `deadIndexEntries` are always `0`.
19766
+ */
19767
+ async applyHistoryRetention(options) {
19768
+ const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("SQLiteStore", options);
19769
+ const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
19770
+ const table = kind === "executions" ? { name: "execution_history", id: "execution_id", time: "started_at" } : { name: "eval_history", id: "id", time: "timestamp" };
19771
+ const lastSaved = kind === "executions" ? "completed_at" : "NULL";
19772
+ const now = this.now();
19773
+ return this.db.transaction(() => {
19774
+ const params = [];
19775
+ let sql = `SELECT ${table.id} AS id, ${table.time} AS time, ${lastSaved} AS last_saved, expires_at FROM ${table.name}`;
19776
+ if (cursor !== void 0) {
19777
+ sql += ` WHERE ${table.time} > ? OR (${table.time} = ? AND ${table.id} > ?)`;
19778
+ params.push(cursor.time, cursor.time, cursor.id);
19779
+ }
19780
+ sql += ` ORDER BY ${table.time}, ${table.id} LIMIT ?`;
19781
+ params.push(batchSize + 1);
19782
+ const rows = this.db.prepare(sql).all(...params);
19783
+ const batch = rows.slice(0, batchSize);
19784
+ const toExpire = [];
19785
+ let armed = 0;
19786
+ if (ttl !== null) {
19787
+ const arm = this.db.prepare(
19788
+ `UPDATE ${table.name} SET expires_at = ? WHERE ${table.id} = ? AND expires_at IS NULL`
19789
+ );
19790
+ for (const row of batch) {
19791
+ if (row.expires_at !== null) continue;
19792
+ const lastSavedAt = kind === "executions" ? executionLastSavedAt({ startedAt: row.time, completedAt: row.last_saved }) : row.time;
19793
+ const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
19794
+ if (decision.expired) toExpire.push(row.id);
19795
+ else armed += arm.run(decision.expiresAt, row.id).changes;
19796
+ }
19797
+ }
19798
+ let expired = 0;
19799
+ if (toExpire.length > 0) {
19800
+ const idList = JSON.stringify(toExpire);
19801
+ expired = this.db.prepare(
19802
+ `DELETE FROM ${table.name} WHERE expires_at IS NULL AND ${table.id} IN (SELECT value FROM json_each(?))`
19803
+ ).run(idList).changes;
19804
+ if (kind === "executions") {
19805
+ for (const side of ["checkpoints", "execution_state"]) {
19806
+ this.db.prepare(
19807
+ `DELETE FROM ${side} WHERE execution_id IN (SELECT value FROM json_each(?))`
19808
+ ).run(idList);
19809
+ }
19810
+ }
19811
+ }
19812
+ const last = batch[batch.length - 1];
19813
+ return {
19814
+ expired,
19815
+ armed,
19816
+ indexed: 0,
19817
+ deadIndexEntries: 0,
19818
+ cursor: rows.length > batchSize ? { time: last.time, id: last.id } : null
19819
+ };
19820
+ }).immediate();
19311
19821
  }
19312
19822
  // ── Sessions (Studio introspection) ────────────────────────────────────
19313
19823
  async listSessions() {
@@ -19477,11 +19987,18 @@ var Session = class _Session {
19477
19987
  try {
19478
19988
  const completion = axlStream.promise.then(
19479
19989
  (result) => this.commitHistory(history, result).catch((err) => {
19480
- this.runtime.emit("error", {
19481
- type: "session_history_save_failed",
19482
- sessionId: this.sessionId,
19483
- error: err instanceof Error ? err.message : String(err)
19484
- });
19990
+ const error = err instanceof Error ? err.message : String(err);
19991
+ try {
19992
+ this.runtime.emit("error", {
19993
+ type: "session_history_save_failed",
19994
+ sessionId: this.sessionId,
19995
+ error
19996
+ });
19997
+ } catch {
19998
+ console.error(
19999
+ `[axl] Failed to save session history for sessionId=${this.sessionId}: ${error}`
20000
+ );
20001
+ }
19485
20002
  }),
19486
20003
  () => void 0
19487
20004
  );
@@ -20656,6 +21173,9 @@ function hashInput(input) {
20656
21173
  }
20657
21174
  var DEFAULT_MAX_EVENTS_PER_EXECUTION = 5e4;
20658
21175
  var DEFAULT_STREAMING_BATCH_SIZE = 100;
21176
+ var DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE = 1e3;
21177
+ var LARGE_EXECUTION_LISTING = 1e3;
21178
+ var EXECUTION_SCAN_PAGE_SIZE = 100;
20659
21179
  var DEFAULT_STREAMING_BATCH_INTERVAL = 1e3;
20660
21180
  var DEFAULT_ARTIFACT_SWEEP_MS = 6e4;
20661
21181
  var DEFAULT_ARTIFACT_LEASE_MS = 3e5;
@@ -20868,6 +21388,78 @@ function assertNoRemovedObservationCallbacks(options) {
20868
21388
  `runtime.createContext() no longer accepts ${removed.join(", ")}. Observe ctx.events or use runtime.stream() instead. See docs/migration/stream-first-observation.md.`
20869
21389
  );
20870
21390
  }
21391
+ var RecentRunsCache = class {
21392
+ constructor(capacity) {
21393
+ this.capacity = capacity;
21394
+ }
21395
+ entries = /* @__PURE__ */ new Map();
21396
+ get size() {
21397
+ return this.entries.size;
21398
+ }
21399
+ get(executionId) {
21400
+ const entry = this.entries.get(executionId);
21401
+ if (entry) {
21402
+ this.entries.delete(executionId);
21403
+ this.entries.set(executionId, entry);
21404
+ }
21405
+ return entry;
21406
+ }
21407
+ set(executionId, execution) {
21408
+ this.entries.delete(executionId);
21409
+ this.entries.set(executionId, execution);
21410
+ while (this.entries.size > this.capacity) {
21411
+ const oldest = this.entries.keys().next().value;
21412
+ this.entries.delete(oldest);
21413
+ }
21414
+ }
21415
+ delete(executionId) {
21416
+ return this.entries.delete(executionId);
21417
+ }
21418
+ values() {
21419
+ return [...this.entries.values()];
21420
+ }
21421
+ };
21422
+ function resolveRecentExecutionsCacheSize(requested) {
21423
+ if (requested === void 0) return DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE;
21424
+ if (typeof requested !== "number" || !Number.isInteger(requested) || requested < 0) {
21425
+ throw new AxlError(
21426
+ "INVALID_CONFIG",
21427
+ `config.state.recentExecutionsCacheSize must be a non-negative integer; got ${String(requested)}`
21428
+ );
21429
+ }
21430
+ return requested;
21431
+ }
21432
+ function parseGetExecutionsOptions(options) {
21433
+ if (options === void 0) return {};
21434
+ if (options === null || typeof options !== "object") {
21435
+ throw new AxlError(
21436
+ "INVALID_ARGUMENT",
21437
+ `getExecutions() options must be an object; got ${options === null ? "null" : typeof options}`
21438
+ );
21439
+ }
21440
+ const { limit, cursor } = options;
21441
+ if (limit !== void 0 && limit !== Infinity && !(typeof limit === "number" && Number.isInteger(limit) && limit > 0)) {
21442
+ throw new AxlError(
21443
+ "INVALID_ARGUMENT",
21444
+ `getExecutions() limit must be a positive integer; got ${String(limit)}`
21445
+ );
21446
+ }
21447
+ let parsedCursor;
21448
+ if (cursor !== void 0) {
21449
+ const candidate = cursor;
21450
+ if (candidate === null || typeof candidate !== "object" || typeof candidate.startedAt !== "number" || !Number.isFinite(candidate.startedAt) || typeof candidate.executionId !== "string") {
21451
+ throw new AxlError(
21452
+ "INVALID_ARGUMENT",
21453
+ "getExecutions() cursor must be { startedAt: number; executionId: string }, built from the last item of the previous page"
21454
+ );
21455
+ }
21456
+ parsedCursor = { startedAt: candidate.startedAt, executionId: candidate.executionId };
21457
+ }
21458
+ return {
21459
+ ...limit === void 0 || limit === Infinity ? {} : { limit },
21460
+ ...parsedCursor ? { cursor: parsedCursor } : {}
21461
+ };
21462
+ }
20871
21463
  var costScopeStorage = new AsyncLocalStorage4();
20872
21464
  function forwardAbortSignal(external, internal) {
20873
21465
  if (!external) return () => {
@@ -20914,8 +21506,6 @@ var AxlRuntime = class extends EventEmitter2 {
20914
21506
  mcpManager;
20915
21507
  memoryManager;
20916
21508
  spanManager = new NoopSpanManager();
20917
- historicalExecutions = /* @__PURE__ */ new Map();
20918
- historicalExecutionsLoadPromise = null;
20919
21509
  /** Per-session in-process serialization. Each entry is a chain of pending
20920
21510
  * work for that session id — `Session.send` / `Session.stream` await the
20921
21511
  * prior entry before reading history, so concurrent calls produce a
@@ -20951,12 +21541,34 @@ var AxlRuntime = class extends EventEmitter2 {
20951
21541
  * a delete from resurrecting the (intentionally-removed) row. Entries are
20952
21542
  * removed once the would-be persist has been short-circuited. */
20953
21543
  pendingDeletedExecutions = /* @__PURE__ */ new Set();
20954
- /** In-flight `persistExecution` chains (save → finalize). `shutdown()`
20955
- * awaits these before closing the StateStore so a workflow whose abort
21544
+ /** In-flight `persistExecution` chains (save → finalize) and crash-
21545
+ * recovery saves. `shutdown()` awaits these before closing the StateStore so a workflow whose abort
20956
21546
  * triggers a persist-during-shutdown isn't racing a closed connection.
20957
21547
  * The Set is mutated by the chain's own `.finally` so entries clear as
20958
21548
  * soon as the work settles. */
20959
21549
  persistInflight = /* @__PURE__ */ new Set();
21550
+ /** Finished runs whose history save has not settled, keyed by execution
21551
+ * id. Reads serve the snapshot until the save lands, so a run is visible
21552
+ * the moment it finishes (read-your-writes). `deleteExecution()` awaits
21553
+ * the entry before deleting from the store, so a save that lands late
21554
+ * cannot re-create a deleted run. Each entry removes itself when its save
21555
+ * settles (success or failure), but only if it is still the current entry
21556
+ * for that id, so the map holds only in-flight saves. */
21557
+ pendingSaves = /* @__PURE__ */ new Map();
21558
+ /** Recently finished runs, kept only when the store cannot save, list, or
21559
+ * fetch execution history itself (and the cache size is not `0`).
21560
+ * `undefined` for every built-in store: execution history is read through
21561
+ * the store on every call and never cached here. */
21562
+ recentRuns;
21563
+ /** Execution ids `recoverIncompleteStreams()` is working on, each with a
21564
+ * promise that settles when recovery is done with that id.
21565
+ * `deleteExecution()` awaits it, so a recovered record cannot land after
21566
+ * the delete. */
21567
+ recoveringExecutions = /* @__PURE__ */ new Map();
21568
+ /** Execution ids with a `deleteExecution()` in progress, with a count of
21569
+ * concurrent deletes. Recovery skips them: their store sweep removes the
21570
+ * streaming buffer it would otherwise turn back into a record. */
21571
+ deletingExecutions = /* @__PURE__ */ new Map();
20960
21572
  /** Resolved diagnostic artifact backend, when `diagnostics.artifacts` is set. */
20961
21573
  artifactStore;
20962
21574
  artifactLeaseMs = DEFAULT_ARTIFACT_LEASE_MS;
@@ -20990,6 +21602,13 @@ var AxlRuntime = class extends EventEmitter2 {
20990
21602
  this.providerRegistry = new ProviderRegistry();
20991
21603
  this.transcriptionProviderRegistry = new TranscriptionProviderRegistry();
20992
21604
  this.stateStore = this.createStateStore();
21605
+ const recentRunsSize = resolveRecentExecutionsCacheSize(
21606
+ this.config.state?.recentExecutionsCacheSize
21607
+ );
21608
+ const storeKeepsHistory = Boolean(
21609
+ this.stateStore.saveExecution && this.stateStore.listExecutions
21610
+ );
21611
+ this.recentRuns = !storeKeepsHistory && recentRunsSize > 0 ? new RecentRunsCache(recentRunsSize) : void 0;
20993
21612
  const requested = this.config.state?.maxEventsPerExecution;
20994
21613
  if (requested === void 0) {
20995
21614
  this.maxEventsPerExecution = DEFAULT_MAX_EVENTS_PER_EXECUTION;
@@ -21395,11 +22014,15 @@ var AxlRuntime = class extends EventEmitter2 {
21395
22014
  const storeOption = this.config.state?.store ?? "memory";
21396
22015
  if (typeof storeOption !== "string") return storeOption;
21397
22016
  switch (storeOption) {
21398
- case "sqlite":
21399
- return new SQLiteStore(this.config.state?.sqlite?.path ?? "./data/axl.db");
22017
+ case "sqlite": {
22018
+ const create = () => new SQLiteStore(this.config.state?.sqlite?.path ?? "./data/axl.db", {
22019
+ ttls: this.config.state?.sqlite?.ttls
22020
+ });
22021
+ return this.config.diagnostics?.silent ? withDiagnosticsSilenced(create) : create();
22022
+ }
21400
22023
  case "memory":
21401
22024
  default:
21402
- return new MemoryStore();
22025
+ return new MemoryStore(this.config.state?.memory);
21403
22026
  }
21404
22027
  }
21405
22028
  /**
@@ -21451,7 +22074,11 @@ var AxlRuntime = class extends EventEmitter2 {
21451
22074
  observation
21452
22075
  });
21453
22076
  if (this.stateStore.deleteCheckpoints) {
21454
- await this.stateStore.deleteCheckpoints(execInfo.executionId);
22077
+ await this.stateStore.deleteCheckpoints(execInfo.executionId).catch((err) => {
22078
+ console.error(
22079
+ `[axl] Failed to delete checkpoints for executionId=${execInfo.executionId}: ` + (err instanceof Error ? err.message : String(err))
22080
+ );
22081
+ });
21455
22082
  }
21456
22083
  span?.setAttribute("axl.workflow.cost", execInfo.totalCost);
21457
22084
  span?.setAttribute("axl.workflow.duration", execInfo.duration);
@@ -21487,13 +22114,15 @@ var AxlRuntime = class extends EventEmitter2 {
21487
22114
  }
21488
22115
  /**
21489
22116
  * Persist a completed/failed execution to the state store (fire-and-forget)
21490
- * and move it from the active executions map to the historical cache.
22117
+ * and remove it from the active executions map. Until the save settles the
22118
+ * run is served from `pendingSaves`; afterwards from the store (or, for a
22119
+ * store that cannot keep history, from `recentRuns`).
21491
22120
  *
21492
22121
  * Resurrection guard: if `runtime.deleteExecution(id)` was called while
21493
22122
  * this execution was still running, the id will be in
21494
- * `pendingDeletedExecutions`. In that case we skip the save AND the
21495
- * historical-cache write, and finalize the streaming buffer (if any) so
21496
- * the buffer doesn't outlive the delete.
22123
+ * `pendingDeletedExecutions`. In that case we skip the save AND every
22124
+ * in-process record of the run, and finalize the streaming buffer (if any)
22125
+ * so the buffer doesn't outlive the delete.
21497
22126
  */
21498
22127
  persistExecution(execInfo) {
21499
22128
  const id = execInfo.executionId;
@@ -21521,17 +22150,25 @@ var AxlRuntime = class extends EventEmitter2 {
21521
22150
  const chain = this.stateStore.saveExecution(snapshot).then(() => {
21522
22151
  return this.streamingFlusher?.finalize(id).catch(() => {
21523
22152
  });
21524
- }).catch(() => {
22153
+ }).catch((err) => {
22154
+ console.error(
22155
+ `[axl] Failed to save execution history for executionId=${id}: ` + (err instanceof Error ? err.message : String(err))
22156
+ );
21525
22157
  }).then(() => void 0);
21526
22158
  this.persistInflight.add(chain);
21527
- chain.finally(() => this.persistInflight.delete(chain));
22159
+ const pending = { snapshot, settled: chain };
22160
+ this.pendingSaves.set(id, pending);
22161
+ chain.finally(() => {
22162
+ this.persistInflight.delete(chain);
22163
+ if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
22164
+ });
21528
22165
  } else if (this.streamingFlusher) {
21529
22166
  const chain = this.streamingFlusher.finalize(id).catch(() => {
21530
22167
  });
21531
22168
  this.persistInflight.add(chain);
21532
22169
  chain.finally(() => this.persistInflight.delete(chain));
21533
22170
  }
21534
- this.historicalExecutions.set(id, snapshot);
22171
+ this.recentRuns?.set(id, snapshot);
21535
22172
  this.executions.delete(id);
21536
22173
  }
21537
22174
  /** Register a workflow with the runtime. */
@@ -21657,7 +22294,6 @@ var AxlRuntime = class extends EventEmitter2 {
21657
22294
  }
21658
22295
  return result;
21659
22296
  }
21660
- /** Get all execution info (running + completed + historical). */
21661
22297
  /** Normalize an execution loaded from a `StateStore` so the
21662
22298
  * historical event union holds for downstream consumers
21663
22299
  * (Studio aggregators, REST routes, redaction). Custom or legacy stores
@@ -21677,27 +22313,78 @@ var AxlRuntime = class extends EventEmitter2 {
21677
22313
  }
21678
22314
  return normalizeStoredExecution(exec);
21679
22315
  }
21680
- async getExecutions() {
21681
- if (!this.historicalExecutionsLoadPromise && this.stateStore.listExecutions) {
21682
- this.historicalExecutionsLoadPromise = this.stateStore.listExecutions().then((stored) => {
21683
- for (const exec of stored) {
21684
- if (!this.executions.has(exec.executionId) && !this.historicalExecutions.has(exec.executionId)) {
21685
- this.historicalExecutions.set(exec.executionId, this.normalizeStoredExecution(exec));
21686
- }
21687
- }
21688
- }).catch((error) => {
21689
- this.historicalExecutionsLoadPromise = null;
21690
- throw error;
21691
- });
21692
- }
21693
- if (this.historicalExecutionsLoadPromise) {
21694
- await this.historicalExecutionsLoadPromise;
21695
- }
21696
- const merged = new Map(this.historicalExecutions);
21697
- for (const [id, exec] of this.executions) {
21698
- merged.set(id, exec);
22316
+ /**
22317
+ * List executions, newest first: running runs, runs whose save is still in
22318
+ * flight, and the history the state store holds. Ordered by `startedAt`,
22319
+ * then `executionId`, both descending.
22320
+ *
22321
+ * With no options, returns every retained run. Pass `limit` for one page,
22322
+ * and `cursor` (built from the last item of the previous page) for the
22323
+ * page after it. Each call reads through the store, so runs finished or
22324
+ * deleted by other runtimes sharing the store are reflected immediately.
22325
+ * A call without `limit` that returns more than 1 000 runs warns once per
22326
+ * process (silenced by `diagnostics.silent`); pass `limit: Infinity` to ask
22327
+ * for everything without the warning.
22328
+ *
22329
+ * @throws {AxlError} `INVALID_ARGUMENT` for a `limit` that is not a positive
22330
+ * integer (`Infinity` means no limit) or a malformed `cursor`.
22331
+ */
22332
+ async getExecutions(options) {
22333
+ const { limit, cursor } = parseGetExecutionsOptions(options);
22334
+ const isAfterCursor = (exec) => cursor === void 0 || compareExecutionsNewestFirst(exec, cursor) > 0;
22335
+ const live = new Map(this.executions);
22336
+ for (const [id, { snapshot }] of this.pendingSaves) {
22337
+ if (!live.has(id)) live.set(id, snapshot);
22338
+ }
22339
+ const recent = this.recentRunsServeListing() ? this.recentRuns.values() : [];
22340
+ const stored = await this.listStoredExecutions({
22341
+ limit,
22342
+ cursor,
22343
+ overlap: live.size + recent.length
22344
+ });
22345
+ const merged = /* @__PURE__ */ new Map();
22346
+ for (const [id, exec] of live) {
22347
+ if (isAfterCursor(exec)) merged.set(id, exec);
22348
+ }
22349
+ for (const exec of stored) {
22350
+ const id = exec.executionId;
22351
+ if (!live.has(id) && !merged.has(id)) merged.set(id, exec);
22352
+ }
22353
+ for (const exec of recent) {
22354
+ const id = exec.executionId;
22355
+ if (!live.has(id) && !merged.has(id) && isAfterCursor(exec)) merged.set(id, exec);
22356
+ }
22357
+ const sorted = [...merged.values()].sort(compareExecutionsNewestFirst);
22358
+ if (limit !== void 0) return sorted.slice(0, limit);
22359
+ if (options?.limit === void 0 && sorted.length > LARGE_EXECUTION_LISTING) {
22360
+ warnDiagnosticOnce(
22361
+ "large_execution_listing",
22362
+ `runtime.getExecutions() without a limit returned ${sorted.length} runs and reads the whole history on every call; pass { limit, cursor } to read it a page at a time.`,
22363
+ this.config.diagnostics?.silent,
22364
+ "docs/migration/execution-history-retention.md#page-through-large-histories"
22365
+ );
21699
22366
  }
21700
- return [...merged.values()].sort((a, b) => b.startedAt - a.startedAt);
22367
+ return sorted;
22368
+ }
22369
+ /**
22370
+ * Read one page of execution history from the store: the records strictly
22371
+ * after `cursor`, newest first, at least `limit + overlap` of them when the
22372
+ * store holds that many (all of them when `limit` is unset).
22373
+ *
22374
+ * Only a store that declares `historyPaging` is handed the cursor. Any
22375
+ * other store is listed in full and paged here: filtering the output of a
22376
+ * store that ignored the cursor would return its first page again, and an
22377
+ * empty page after that.
22378
+ */
22379
+ async listStoredExecutions(page) {
22380
+ const store = this.stateStore;
22381
+ if (!store.listExecutions) return [];
22382
+ const wanted = page.limit === void 0 ? void 0 : page.limit + page.overlap;
22383
+ const rows = store.historyPaging === true ? await store.listExecutions(wanted, { cursor: page.cursor }) : await store.listExecutions();
22384
+ const cursor = page.cursor;
22385
+ const after = cursor === void 0 ? rows : rows.filter((row) => compareExecutionsNewestFirst(row, cursor) > 0);
22386
+ const window = wanted === void 0 ? after : [...after].sort(compareExecutionsNewestFirst).slice(0, wanted);
22387
+ return window.map((row) => this.normalizeStoredExecution(row));
21701
22388
  }
21702
22389
  /**
21703
22390
  * Create a WorkflowContext for ad-hoc use (evals, tool testing, prototyping).
@@ -22128,20 +22815,25 @@ var AxlRuntime = class extends EventEmitter2 {
22128
22815
  * Recover executions whose process died mid-run when `state.persist:
22129
22816
  * 'streaming'` was configured. Synthesizes a partial `ExecutionInfo`
22130
22817
  * from the streaming buffer for each — `status: 'failed'`, error
22131
- * `'process terminated'` — registers them in the historical cache so
22132
- * they show up in `getExecutions()`, persists them via the store's
22133
- * `saveExecution`, and finalizes the streaming buffer.
22818
+ * `'process terminated'` — persists them via the store's
22819
+ * `saveExecution` (so they show up in `getExecutions()`), and finalizes
22820
+ * the streaming buffer.
22134
22821
  *
22135
22822
  * Idempotent — if the streaming buffer is already gone (or
22136
22823
  * `listStreamingExecutions` is empty), this is a no-op. Returns the
22137
22824
  * list of recovered `ExecutionInfo`s for caller introspection.
22138
22825
  *
22139
- * Wire this into your process startup AFTER `runtime.getExecutions()`
22140
- * has lazy-loaded the historical cache. Order matters: completed
22141
- * executions already in the store should not be overwritten by a
22142
- * stale streaming buffer that the previous process didn't get to
22143
- * finalize. The implementation checks the historical store before
22144
- * synthesizing, so a re-run is safe.
22826
+ * A run whose final record already exists is never recovered: for each
22827
+ * buffered id this asks the store's `getExecution(id)` and, when a record
22828
+ * is there, only drops the leftover buffer. It does not list history, so
22829
+ * its cost does not grow with the size of the store. A store without
22830
+ * `getExecution` cannot be checked, and every buffered run is recovered.
22831
+ *
22832
+ * Wire this into process startup, before the process accepts new work:
22833
+ * a run live in another process looks the same as a crashed one.
22834
+ * `deleteExecution(id)` called while recovery is working on that id waits
22835
+ * for recovery to finish with it, so the recovered record cannot outlive
22836
+ * the delete.
22145
22837
  */
22146
22838
  async recoverIncompleteStreams() {
22147
22839
  if (!this.stateStore.listStreamingExecutions || !this.stateStore.getStreamingEvents) {
@@ -22149,73 +22841,119 @@ var AxlRuntime = class extends EventEmitter2 {
22149
22841
  }
22150
22842
  const ids = await this.stateStore.listStreamingExecutions();
22151
22843
  if (ids.length === 0) return [];
22152
- await this.getExecutions();
22153
22844
  const recovered = [];
22154
22845
  for (const id of ids) {
22155
22846
  if (this.executions.has(id) || this.streamableExecutionIds.has(id)) {
22156
22847
  continue;
22157
22848
  }
22158
- const existing = this.historicalExecutions.get(id);
22159
- if (existing) {
22160
- await this.stateStore.finalizeStreamingEvents?.(id);
22161
- continue;
22162
- }
22163
- const events = await this.stateStore.getStreamingEvents(id);
22164
- if (events.length === 0) {
22165
- await this.stateStore.finalizeStreamingEvents?.(id);
22166
- continue;
22167
- }
22168
- const eventSchemaVersion = getEventSchemaVersion(events[0]);
22169
- for (const event of events) {
22170
- if (getEventSchemaVersion(event) !== eventSchemaVersion) {
22171
- throw new Error(`Streaming buffer ${id} contains mixed event schema versions`);
22172
- }
22849
+ if (this.deletingExecutions.has(id) || this.recoveringExecutions.has(id)) continue;
22850
+ let finishRecovery;
22851
+ this.recoveringExecutions.set(
22852
+ id,
22853
+ new Promise((resolve2) => {
22854
+ finishRecovery = resolve2;
22855
+ })
22856
+ );
22857
+ try {
22858
+ const result = await this.recoverStreamingExecution(id);
22859
+ if (result) recovered.push(result);
22860
+ } finally {
22861
+ this.recoveringExecutions.delete(id);
22862
+ finishRecovery();
22173
22863
  }
22174
- const boundedEvents = boundRecoveredEvents(events, this.maxEventsPerExecution);
22175
- const workflowStartEvent = boundedEvents.find((e) => e.type === "workflow_start");
22176
- const lastEvent = boundedEvents[boundedEvents.length - 1];
22177
- const startedAt = workflowStartEvent?.timestamp ?? boundedEvents[0]?.timestamp ?? Date.now();
22178
- const completedAt = lastEvent.timestamp ?? startedAt;
22179
- const workflowName = workflowStartEvent?.workflow ?? RECOVERED_UNKNOWN_WORKFLOW;
22180
- const totalCost = boundedEvents.reduce((sum, e) => sum + eventCostContribution(e), 0);
22181
- const unpriced = boundedEvents.some(isUnpricedLeaf);
22182
- const executionBase = {
22183
- executionId: id,
22184
- workflow: workflowName,
22185
- status: "failed",
22186
- events: boundedEvents,
22187
- totalCost,
22188
- unpriced,
22189
- startedAt,
22190
- completedAt,
22191
- duration: completedAt - startedAt,
22192
- error: "process terminated (recovered from streaming buffer)"
22193
- };
22194
- const synthesized = eventSchemaVersion === 2 ? {
22195
- ...executionBase,
22196
- eventSchemaVersion: 2,
22197
- events: boundedEvents,
22198
- observation: { complete: false, reason: "process_interrupted" }
22199
- } : {
22200
- ...executionBase,
22201
- eventSchemaVersion: 1,
22202
- events: boundedEvents
22864
+ }
22865
+ return recovered;
22866
+ }
22867
+ /** Recover one buffered execution. Returns the synthesized record, or
22868
+ * `undefined` when there was nothing to recover. */
22869
+ async recoverStreamingExecution(id) {
22870
+ if (!this.stateStore.getStreamingEvents) return void 0;
22871
+ const existing = await this.stateStore.getExecution?.(id);
22872
+ if (existing) {
22873
+ await this.stateStore.finalizeStreamingEvents?.(id);
22874
+ return void 0;
22875
+ }
22876
+ const events = await this.stateStore.getStreamingEvents(id);
22877
+ if (events.length === 0) {
22878
+ await this.stateStore.finalizeStreamingEvents?.(id);
22879
+ return void 0;
22880
+ }
22881
+ const eventSchemaVersion = getEventSchemaVersion(events[0]);
22882
+ for (const event of events) {
22883
+ if (getEventSchemaVersion(event) !== eventSchemaVersion) {
22884
+ throw new Error(`Streaming buffer ${id} contains mixed event schema versions`);
22885
+ }
22886
+ }
22887
+ const boundedEvents = boundRecoveredEvents(events, this.maxEventsPerExecution);
22888
+ const workflowStartEvent = boundedEvents.find((e) => e.type === "workflow_start");
22889
+ const lastEvent = boundedEvents[boundedEvents.length - 1];
22890
+ const startedAt = workflowStartEvent?.timestamp ?? boundedEvents[0]?.timestamp ?? Date.now();
22891
+ const completedAt = lastEvent.timestamp ?? startedAt;
22892
+ const workflowName = workflowStartEvent?.workflow ?? RECOVERED_UNKNOWN_WORKFLOW;
22893
+ const totalCost = boundedEvents.reduce((sum, e) => sum + eventCostContribution(e), 0);
22894
+ const unpriced = boundedEvents.some(isUnpricedLeaf);
22895
+ const executionBase = {
22896
+ executionId: id,
22897
+ workflow: workflowName,
22898
+ status: "failed",
22899
+ events: boundedEvents,
22900
+ totalCost,
22901
+ unpriced,
22902
+ startedAt,
22903
+ completedAt,
22904
+ duration: completedAt - startedAt,
22905
+ error: "process terminated (recovered from streaming buffer)"
22906
+ };
22907
+ const synthesized = eventSchemaVersion === 2 ? {
22908
+ ...executionBase,
22909
+ eventSchemaVersion: 2,
22910
+ events: boundedEvents,
22911
+ observation: { complete: false, reason: "process_interrupted" }
22912
+ } : {
22913
+ ...executionBase,
22914
+ eventSchemaVersion: 1,
22915
+ events: boundedEvents
22916
+ };
22917
+ const saveExecution = this.stateStore.saveExecution;
22918
+ if (saveExecution) {
22919
+ const save = (async () => saveExecution.call(this.stateStore, synthesized))();
22920
+ const pending = {
22921
+ snapshot: synthesized,
22922
+ settled: save.then(
22923
+ () => void 0,
22924
+ () => void 0
22925
+ )
22203
22926
  };
22204
- if (this.stateStore.saveExecution) {
22205
- try {
22206
- await this.stateStore.saveExecution(synthesized);
22207
- } catch (err) {
22208
- console.error(
22209
- `[axl] recoverIncompleteStreams: failed to save synthesized execution ${id}: ${err instanceof Error ? err.message : String(err)}. Streaming buffer left in place for next recovery attempt.`
22210
- );
22211
- continue;
22212
- }
22927
+ this.pendingSaves.set(id, pending);
22928
+ this.persistInflight.add(pending.settled);
22929
+ try {
22930
+ await save;
22931
+ } catch (err) {
22932
+ console.error(
22933
+ `[axl] recoverIncompleteStreams: failed to save synthesized execution ${id}: ${err instanceof Error ? err.message : String(err)}. Streaming buffer left in place for next recovery attempt.`
22934
+ );
22935
+ return void 0;
22936
+ } finally {
22937
+ this.persistInflight.delete(pending.settled);
22938
+ if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
22213
22939
  }
22214
- this.historicalExecutions.set(id, synthesized);
22215
- await this.stateStore.finalizeStreamingEvents?.(id);
22216
- recovered.push(synthesized);
22217
22940
  }
22218
- return recovered;
22941
+ this.recentRuns?.set(id, synthesized);
22942
+ const finalize = Promise.resolve(this.stateStore.finalizeStreamingEvents?.(id)).then(
22943
+ () => void 0,
22944
+ (err) => {
22945
+ console.error(
22946
+ `[axl] recoverIncompleteStreams: saved execution ${id} but failed to drop its streaming buffer: ${err instanceof Error ? err.message : String(err)}. The next recovery removes it.`
22947
+ );
22948
+ }
22949
+ );
22950
+ this.persistInflight.add(finalize);
22951
+ try {
22952
+ await finalize;
22953
+ } finally {
22954
+ this.persistInflight.delete(finalize);
22955
+ }
22956
+ return synthesized;
22219
22957
  }
22220
22958
  /** Abort a running execution by its ID. */
22221
22959
  abort(executionId) {
@@ -22225,21 +22963,57 @@ var AxlRuntime = class extends EventEmitter2 {
22225
22963
  this.abortControllers.delete(executionId);
22226
22964
  }
22227
22965
  }
22228
- /** Get execution details by ID. */
22966
+ /**
22967
+ * Get execution details by ID. Looks in order at running runs, runs whose
22968
+ * save is still in flight, the state store (its `getExecution`, or a scan
22969
+ * of its listing when it has `listExecutions` but no `getExecution`), and
22970
+ * (only for a store that cannot list or save history) the recent-runs
22971
+ * cache. Nothing read from the store is cached, so a run deleted or
22972
+ * expired elsewhere is gone on the next call, and every run
22973
+ * `getExecutions()` lists can be fetched here.
22974
+ */
22229
22975
  async getExecution(executionId) {
22230
- const inMemory = this.executions.get(executionId);
22231
- if (inMemory) return inMemory;
22232
- const cached = this.historicalExecutions.get(executionId);
22233
- if (cached) return cached;
22234
- if (this.stateStore.getExecution) {
22235
- const stored = await this.stateStore.getExecution(executionId);
22236
- if (stored) {
22237
- const normalized = this.normalizeStoredExecution(stored);
22238
- this.historicalExecutions.set(executionId, normalized);
22239
- return normalized;
22240
- }
22976
+ const active = this.executions.get(executionId);
22977
+ if (active) return active;
22978
+ const pending = this.pendingSaves.get(executionId);
22979
+ if (pending) return pending.snapshot;
22980
+ const stored = this.stateStore.getExecution ? await this.stateStore.getExecution(executionId) : await this.findInStoreListing(executionId);
22981
+ if (stored) return this.normalizeStoredExecution(stored);
22982
+ return this.recentRunsServeReads() ? this.recentRuns.get(executionId) : void 0;
22983
+ }
22984
+ /**
22985
+ * By-id read for a store that lists but cannot fetch by id. A paging store
22986
+ * is scanned page by page with the cursor, stopping at the match; any
22987
+ * other store is listed once in full. Returns `null` when the store cannot
22988
+ * list or does not hold the run.
22989
+ */
22990
+ async findInStoreListing(executionId) {
22991
+ const store = this.stateStore;
22992
+ if (!store.listExecutions) return null;
22993
+ if (store.historyPaging !== true) {
22994
+ return (await store.listExecutions()).find((e) => e.executionId === executionId) ?? null;
22241
22995
  }
22242
- return void 0;
22996
+ let cursor;
22997
+ for (; ; ) {
22998
+ const page = await store.listExecutions(EXECUTION_SCAN_PAGE_SIZE, { cursor });
22999
+ const found = page.find((e) => e.executionId === executionId);
23000
+ if (found) return found;
23001
+ if (page.length === 0) return null;
23002
+ const last = [...page].sort(compareExecutionsNewestFirst)[page.length - 1];
23003
+ if (cursor && compareExecutionsNewestFirst(last, cursor) <= 0) return null;
23004
+ cursor = { startedAt: last.startedAt, executionId: last.executionId };
23005
+ }
23006
+ }
23007
+ /** The recent-runs cache stands in for listing when the store cannot list
23008
+ * runs, or never receives them because it cannot save. */
23009
+ recentRunsServeListing() {
23010
+ return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.listExecutions);
23011
+ }
23012
+ /** The recent-runs cache stands in for by-id reads when the store cannot
23013
+ * fetch or list runs (it exists only then), or never receives them because
23014
+ * it cannot save. */
23015
+ recentRunsServeReads() {
23016
+ return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.getExecution);
22243
23017
  }
22244
23018
  /** Save an eval result to history. */
22245
23019
  async saveEvalResult(entry) {
@@ -22272,11 +23046,16 @@ var AxlRuntime = class extends EventEmitter2 {
22272
23046
  }
22273
23047
  /**
22274
23048
  * Delete an execution from history by id. Removes from the active map (if
22275
- * still running), the historical cache, the configured StateStore, AND
23049
+ * still running), the recent-runs cache, the configured StateStore, AND
22276
23050
  * any execution-scoped side state — pending awaitHuman decisions,
22277
23051
  * resolver maps, streaming buffer registration, in-flight abort controller.
22278
- * Returns true if an entry was actually removed from any of the three
22279
- * primary surfaces (active map, historical cache, store).
23052
+ * Returns true if the run was running, had a save in flight, or was
23053
+ * removed from the recent-runs cache or the store.
23054
+ *
23055
+ * If the run has already finished but its history save has not landed
23056
+ * yet, this method waits for that save (whether it succeeds or fails)
23057
+ * before deleting, so the save cannot re-create the deleted record. It
23058
+ * likewise waits for `recoverIncompleteStreams()` to finish with the id.
22280
23059
  *
22281
23060
  * In-flight handling: if the execution is still running, this method
22282
23061
  * **aborts** it (via the registered abort controller) AND marks the id for
@@ -22301,7 +23080,18 @@ var AxlRuntime = class extends EventEmitter2 {
22301
23080
  * (`defaultTtl` / `ttls.executionHistory`).
22302
23081
  */
22303
23082
  async deleteExecution(id) {
23083
+ this.deletingExecutions.set(id, (this.deletingExecutions.get(id) ?? 0) + 1);
23084
+ try {
23085
+ return await this.deleteExecutionWhileMarked(id);
23086
+ } finally {
23087
+ const remaining = (this.deletingExecutions.get(id) ?? 1) - 1;
23088
+ if (remaining > 0) this.deletingExecutions.set(id, remaining);
23089
+ else this.deletingExecutions.delete(id);
23090
+ }
23091
+ }
23092
+ async deleteExecutionWhileMarked(id) {
22304
23093
  const wasActive = this.executions.has(id);
23094
+ const hadPendingSave = this.pendingSaves.has(id);
22305
23095
  const pendingDecisionResolver = this.pendingDecisionResolvers.get(id);
22306
23096
  const hadPendingDecision = pendingDecisionResolver !== void 0;
22307
23097
  if (wasActive) {
@@ -22319,22 +23109,25 @@ var AxlRuntime = class extends EventEmitter2 {
22319
23109
  this.pendingDecisionResolvers.delete(id);
22320
23110
  }
22321
23111
  }
23112
+ await this.pendingSaves.get(id)?.settled;
23113
+ await this.recoveringExecutions.get(id);
22322
23114
  const existing = await this.getExecution(id);
22323
23115
  const workflow2 = existing?.workflow;
22324
23116
  const removedFromActive = this.executions.delete(id);
22325
- const removedFromHistorical = this.historicalExecutions.delete(id);
23117
+ const removedFromRecent = this.recentRuns?.delete(id) ?? false;
22326
23118
  let removedFromStore = false;
22327
23119
  if (this.stateStore.deleteExecution) {
22328
23120
  removedFromStore = await this.stateStore.deleteExecution(id);
22329
23121
  }
23122
+ const removed = removedFromActive || hadPendingSave || removedFromRecent || removedFromStore;
22330
23123
  this.emit("execution_deleted", {
22331
23124
  executionId: id,
22332
23125
  workflow: workflow2,
22333
23126
  wasActive,
22334
23127
  hadPendingDecision,
22335
- removed: removedFromActive || removedFromHistorical || removedFromStore
23128
+ removed
22336
23129
  });
22337
- return removedFromActive || removedFromHistorical || removedFromStore;
23130
+ return removed;
22338
23131
  }
22339
23132
  /**
22340
23133
  * Delete an eval history entry by id. Removes from in-memory cache and
@@ -23752,12 +24545,153 @@ async function* stringStreamFromEvents(source, opts) {
23752
24545
  }
23753
24546
 
23754
24547
  // src/state/redis.ts
24548
+ var PRUNE_HISTORY_INDEX_SCRIPT = `
24549
+ local now = tonumber(ARGV[1])
24550
+ local removed = 0
24551
+ for i = 2, #ARGV do
24552
+ local id = ARGV[i]
24553
+ local pttl = redis.call('PTTL', KEYS[i + 2])
24554
+ if pttl == -2 then
24555
+ redis.call('SREM', KEYS[1], id)
24556
+ redis.call('ZREM', KEYS[2], id)
24557
+ redis.call('ZREM', KEYS[3], id)
24558
+ removed = removed + 1
24559
+ elseif pttl == -1 then
24560
+ redis.call('ZREM', KEYS[3], id)
24561
+ else
24562
+ redis.call('ZADD', KEYS[3], string.format('%d', now + pttl), id)
24563
+ end
24564
+ end
24565
+ return removed
24566
+ `;
24567
+ function compareMembers(a, b) {
24568
+ return Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8"));
24569
+ }
24570
+ function compareNewestFirst(a, b) {
24571
+ if (a.score !== b.score) return b.score - a.score;
24572
+ return compareMembers(b.id, a.id);
24573
+ }
24574
+ function warnIfRedisExecutionRetentionUnset(opts) {
24575
+ if (opts.ttls?.executionHistory === void 0 && typeof opts.defaultTtl !== "number") {
24576
+ warnExecutionRetentionUnset("RedisStore");
24577
+ }
24578
+ }
24579
+ var CLOSE_TIMEOUT_MS = 2e3;
24580
+ function reconnectDelayMs(retries) {
24581
+ return Math.min(2 ** retries * 50, 2e3) + Math.floor(Math.random() * 200);
24582
+ }
24583
+ function abandonConnection(client) {
24584
+ try {
24585
+ client.destroy?.();
24586
+ } catch {
24587
+ }
24588
+ client.unref?.();
24589
+ }
24590
+ function errorMessage(error) {
24591
+ return error instanceof Error ? error.message : String(error);
24592
+ }
24593
+ function connectionEventReporter(client, onConnectionEvent) {
24594
+ if (onConnectionEvent) {
24595
+ const reportFailure = (error) => console.error(`[axl] RedisStore: onConnectionEvent failed: ${errorMessage(error)}`);
24596
+ return (event) => {
24597
+ try {
24598
+ Promise.resolve(onConnectionEvent(event)).catch(reportFailure);
24599
+ } catch (error) {
24600
+ reportFailure(error);
24601
+ }
24602
+ };
24603
+ }
24604
+ let outageLogged = false;
24605
+ return (event) => {
24606
+ if (event.type === "reconnected") {
24607
+ outageLogged = false;
24608
+ console.warn("[axl] RedisStore: reconnected to Redis.");
24609
+ } else if (!outageLogged) {
24610
+ outageLogged = client.isReady === false;
24611
+ console.warn(
24612
+ `[axl] RedisStore: Redis connection lost (${event.error.message}). Store calls fail until it reconnects, which happens automatically.`
24613
+ );
24614
+ }
24615
+ };
24616
+ }
24617
+ async function connectClient(createClient, opts) {
24618
+ let established = false;
24619
+ const client = createClient({
24620
+ ...opts.url ? { url: opts.url } : {},
24621
+ // A command accepted just before a drop rejects instead of waiting for
24622
+ // the reconnect.
24623
+ disableOfflineQueue: true,
24624
+ socket: { reconnectStrategy: (retries) => established ? reconnectDelayMs(retries) : false }
24625
+ });
24626
+ const reportConnectionEvent = connectionEventReporter(client, opts.onConnectionEvent);
24627
+ client.on("error", (error) => {
24628
+ if (!established || client.isOpen === false) return;
24629
+ reportConnectionEvent({
24630
+ type: "error",
24631
+ error: error instanceof Error ? error : new Error(String(error))
24632
+ });
24633
+ });
24634
+ client.on("connect", () => {
24635
+ if (client.isOpen === false) client.unref?.();
24636
+ });
24637
+ client.on("ready", () => {
24638
+ if (!established) {
24639
+ established = true;
24640
+ } else if (client.isOpen !== false) {
24641
+ reportConnectionEvent({ type: "reconnected" });
24642
+ }
24643
+ });
24644
+ await client.connect();
24645
+ return client;
24646
+ }
23755
24647
  var RedisStore = class _RedisStore {
23756
- constructor(client, keyPrefix, ttls) {
23757
- this.client = client;
24648
+ constructor(connection, keyPrefix, ttls) {
24649
+ this.connection = connection;
23758
24650
  this.keyPrefix = keyPrefix;
23759
24651
  this.ttls = ttls;
23760
24652
  }
24653
+ /**
24654
+ * Whether the store can send commands right now, for a health check or for
24655
+ * telling an outage from other failures when a call rejects. `false` while
24656
+ * reconnecting and after `close()`.
24657
+ */
24658
+ get isConnected() {
24659
+ return this.connection.isOpen !== false && this.connection.isReady !== false;
24660
+ }
24661
+ /**
24662
+ * The client for sending a command, failing immediately while disconnected.
24663
+ *
24664
+ * Every command goes through here because node-redis's `disableOfflineQueue`
24665
+ * rejects single commands but not `multi().exec()`, which would wait for a
24666
+ * reconnect attempt to fail or succeed. Nothing may `await` between
24667
+ * `client.multi()` and `.exec()`, or a transaction could get past this check.
24668
+ *
24669
+ * A closed store isn't "unavailable": it falls through to node-redis's own
24670
+ * "The client is closed" rejection.
24671
+ */
24672
+ get client() {
24673
+ if (this.connection.isOpen !== false && this.connection.isReady === false) {
24674
+ throw new AxlError(
24675
+ "REDIS_UNAVAILABLE",
24676
+ "RedisStore: not connected to Redis, so the command was not sent. The store reconnects automatically; retry once Redis is back."
24677
+ );
24678
+ }
24679
+ return this.connection;
24680
+ }
24681
+ /**
24682
+ * History kinds with a prune running in this process. A save skips its
24683
+ * prune while another one of the same kind is in flight: the running one
24684
+ * already covers the oldest expired entries. Created lazily so the
24685
+ * instance needs no constructor state.
24686
+ */
24687
+ prunesInFlight;
24688
+ /**
24689
+ * Supports `listExecutions(limit, { cursor })`. A getter rather than a
24690
+ * field so it lives on the prototype.
24691
+ */
24692
+ get historyPaging() {
24693
+ return true;
24694
+ }
23761
24695
  /**
23762
24696
  * Resolve the TTL for a given storage category. Returns `undefined` when
23763
24697
  * no TTL applies (category is `null` in resolved config, the value is
@@ -23813,8 +24747,7 @@ var RedisStore = class _RedisStore {
23813
24747
  if (err instanceof Error && err.message.includes("createClient")) throw err;
23814
24748
  throw new Error("redis is required for RedisStore. Install it with: npm install redis");
23815
24749
  }
23816
- const client = opts.url ? createClient({ url: opts.url }) : createClient();
23817
- await client.connect();
24750
+ const client = await connectClient(createClient, opts);
23818
24751
  const resolveCategory = (override) => {
23819
24752
  if (override === null) return null;
23820
24753
  if (typeof override === "number") return override;
@@ -23836,9 +24769,15 @@ var RedisStore = class _RedisStore {
23836
24769
  streamingEvents: resolveStreamingEvents(opts.ttls?.streamingEvents)
23837
24770
  };
23838
24771
  const store = new _RedisStore(client, keyPrefix, effectiveTtls);
24772
+ warnIfRedisExecutionRetentionUnset(opts);
23839
24773
  if (!opts.skipMigration) {
23840
- await store.backfillExecutionIndex();
23841
- await store.backfillEvalIndex();
24774
+ try {
24775
+ await store.backfillExecutionIndex();
24776
+ await store.backfillEvalIndex();
24777
+ } catch (error) {
24778
+ abandonConnection(client);
24779
+ throw error;
24780
+ }
23842
24781
  }
23843
24782
  return store;
23844
24783
  }
@@ -23947,13 +24886,9 @@ var RedisStore = class _RedisStore {
23947
24886
  execHistoryKey(executionId) {
23948
24887
  return `${this.keyPrefix}exec-history:${executionId}`;
23949
24888
  }
23950
- // NOTE for future implementer of #3 (TTL config): when the executionHistory
23951
- // and evalHistory data blobs gain TTLs, eviction MUST be driven by
23952
- // ZREMRANGEBYSCORE on the ZSET below — NOT by SREM on these legacy ID sets.
23953
- // The slow-path fallback (`listHistoryByZset`) reads from the SET when
23954
- // `zsetCard < setCard`, so SET-based eviction would race with the fallback
23955
- // and surface dead IDs whose data has been TTL'd away. ZSET eviction +
23956
- // dual-write `SREM` is the clean path; doc the assumption then.
24889
+ // The legacy id set and the sorted set below always change together (save,
24890
+ // delete, prune), so the slow-path guard `ZCARD >= SCARD` in
24891
+ // `listHistoryByZset` and backfill stays meaningful.
23957
24892
  execHistorySetKey() {
23958
24893
  return `${this.keyPrefix}exec-history-ids`;
23959
24894
  }
@@ -23976,6 +24911,36 @@ var RedisStore = class _RedisStore {
23976
24911
  evalHistoryZsetKey() {
23977
24912
  return `${this.keyPrefix}eval-history-z`;
23978
24913
  }
24914
+ /** Keys and blob parsing for one kind of history. See {@link HistoryIndex}. */
24915
+ historyIndex(kind) {
24916
+ if (kind === "execution") {
24917
+ return {
24918
+ kind,
24919
+ label: "execution-history",
24920
+ ids: this.execHistorySetKey(),
24921
+ time: this.execHistoryZsetKey(),
24922
+ expiry: `${this.keyPrefix}exec-history-exp`,
24923
+ dataKey: (id) => this.execHistoryKey(id),
24924
+ lastSavedAtOf: (raw) => {
24925
+ const record = JSON.parse(raw);
24926
+ return typeof record.startedAt === "number" ? executionLastSavedAt(record) : null;
24927
+ }
24928
+ };
24929
+ }
24930
+ return {
24931
+ kind,
24932
+ label: "eval-history",
24933
+ ids: this.evalHistorySetKey(),
24934
+ time: this.evalHistoryZsetKey(),
24935
+ expiry: `${this.keyPrefix}eval-history-exp`,
24936
+ dataKey: (id) => this.evalHistoryKey(id),
24937
+ // Eval retention counts from the first write, which is the row's timestamp.
24938
+ lastSavedAtOf: (raw) => {
24939
+ const timestamp = JSON.parse(raw).timestamp;
24940
+ return typeof timestamp === "number" ? timestamp : null;
24941
+ }
24942
+ };
24943
+ }
23979
24944
  memoryKey(scope) {
23980
24945
  return `${this.keyPrefix}memory:${scope}`;
23981
24946
  }
@@ -24081,7 +25046,7 @@ var RedisStore = class _RedisStore {
24081
25046
  }
24082
25047
  }
24083
25048
  if (dead.length > 0) {
24084
- this.client.sRem(this.pendingExecSetKey(), dead).catch((err) => {
25049
+ this.connection.sRem(this.pendingExecSetKey(), dead).catch((err) => {
24085
25050
  console.error(
24086
25051
  `[axl] RedisStore: failed to prune ${dead.length} stale pending-exec ids: ` + (err instanceof Error ? err.message : String(err))
24087
25052
  );
@@ -24127,7 +25092,7 @@ var RedisStore = class _RedisStore {
24127
25092
  }
24128
25093
  const ttl = this.ttlFor("memory");
24129
25094
  if (ttl !== void 0) {
24130
- this.client.expire(this.memoryKey(scope), ttl).catch((err) => {
25095
+ this.connection.expire(this.memoryKey(scope), ttl).catch((err) => {
24131
25096
  console.error(
24132
25097
  `[axl] RedisStore: failed to set TTL on migrated memory key ${this.memoryKey(scope)}: ` + (err instanceof Error ? err.message : String(err))
24133
25098
  );
@@ -24147,31 +25112,35 @@ var RedisStore = class _RedisStore {
24147
25112
  async saveExecution(execution) {
24148
25113
  const ttl = this.ttlFor("executionHistory");
24149
25114
  const setOptions = ttl !== void 0 ? { EX: ttl } : void 0;
25115
+ const index = this.historyIndex("execution");
25116
+ const id = execution.executionId;
24150
25117
  const storedExecution = normalizeStoredExecution({
24151
25118
  ...execution,
24152
25119
  eventSchemaVersion: getExecutionEventSchemaVersion(execution)
24153
25120
  });
24154
- await this.client.multi().sAdd(this.execHistorySetKey(), execution.executionId).set(this.execHistoryKey(execution.executionId), JSON.stringify(storedExecution), setOptions).zAdd(this.execHistoryZsetKey(), {
24155
- score: execution.startedAt,
24156
- value: execution.executionId
24157
- }).exec();
25121
+ const tx = this.client.multi().sAdd(index.ids, id).set(index.dataKey(id), JSON.stringify(storedExecution), setOptions).zAdd(index.time, { score: execution.startedAt, value: id });
25122
+ if (ttl !== void 0) tx.zAdd(index.expiry, { score: Date.now() + ttl * 1e3, value: id });
25123
+ else tx.zRem(index.expiry, id);
25124
+ await tx.exec();
25125
+ await this.pruneHistoryAfterSave(index);
24158
25126
  }
24159
25127
  async getExecution(executionId) {
24160
25128
  const raw = await this.client.get(this.execHistoryKey(executionId));
24161
25129
  return raw ? normalizeStoredExecution(JSON.parse(raw)) : null;
24162
25130
  }
24163
- async listExecutions(limit) {
25131
+ async listExecutions(limit, options) {
25132
+ const cursor = options?.cursor;
24164
25133
  const executions = await this.listHistoryByZset(
24165
- this.execHistorySetKey(),
24166
- this.execHistoryZsetKey(),
24167
- (id) => this.execHistoryKey(id),
24168
- (a, b) => b.startedAt - a.startedAt,
24169
- limit
25134
+ this.historyIndex("execution"),
25135
+ (e) => ({ score: e.startedAt, id: e.executionId }),
25136
+ limit,
25137
+ cursor !== void 0 ? { score: cursor.startedAt, id: cursor.executionId } : void 0
24170
25138
  );
24171
25139
  return executions.map(normalizeStoredExecution);
24172
25140
  }
24173
25141
  async deleteExecution(executionId) {
24174
- const results = await this.client.multi().sRem(this.execHistorySetKey(), executionId).del(this.execHistoryKey(executionId)).zRem(this.execHistoryZsetKey(), executionId).del(this.checkpointKey(executionId)).del(this.executionStateKey(executionId)).sRem(this.pendingExecSetKey(), executionId).del(this.streamingEventsKey(executionId)).sRem(this.streamingIdsKey(), executionId).hDel(this.decisionsKey(), executionId).exec();
25142
+ const index = this.historyIndex("execution");
25143
+ const results = await this.client.multi().sRem(index.ids, executionId).del(index.dataKey(executionId)).zRem(index.time, executionId).zRem(index.expiry, executionId).del(this.checkpointKey(executionId)).del(this.executionStateKey(executionId)).sRem(this.pendingExecSetKey(), executionId).del(this.streamingEventsKey(executionId)).sRem(this.streamingIdsKey(), executionId).hDel(this.decisionsKey(), executionId).exec();
24175
25144
  return results.some((result) => typeof result === "number" && result > 0);
24176
25145
  }
24177
25146
  // ── Eval History ────────────────────────────────────────────────────
@@ -24196,18 +25165,31 @@ var RedisStore = class _RedisStore {
24196
25165
  * lifetime, so the race costs at most one window's length, never the window
24197
25166
  * itself. It also carries no Redis version floor, where `KEEPTTL` needs 6.0.
24198
25167
  *
24199
- * The index entries carry no TTL and age out lazily, as with `saveExecution`.
25168
+ * As with `saveExecution`, a timed row is also scored in the expiry sorted
25169
+ * set (by the expiry it carries, so a re-save keeps its score), an untimed
25170
+ * one is taken out of it, and a bounded prune follows the save.
24200
25171
  */
24201
25172
  async saveEvalResult(entry) {
24202
25173
  const ttl = this.ttlFor("evalHistory");
24203
- const key = this.evalHistoryKey(entry.id);
25174
+ const index = this.historyIndex("eval");
25175
+ const key = index.dataKey(entry.id);
24204
25176
  let setOptions;
25177
+ let expiresAt;
24205
25178
  if (ttl !== void 0) {
24206
25179
  const remaining = await this.client.pTTL(key);
24207
- if (remaining === -2) setOptions = { EX: ttl };
24208
- else if (remaining > 0) setOptions = { PX: remaining };
25180
+ if (remaining === -2) {
25181
+ setOptions = { EX: ttl };
25182
+ expiresAt = Date.now() + ttl * 1e3;
25183
+ } else if (remaining > 0) {
25184
+ setOptions = { PX: remaining };
25185
+ expiresAt = Date.now() + remaining;
25186
+ }
24209
25187
  }
24210
- await this.client.multi().sAdd(this.evalHistorySetKey(), entry.id).set(key, JSON.stringify(entry), setOptions).zAdd(this.evalHistoryZsetKey(), { score: entry.timestamp, value: entry.id }).exec();
25188
+ const tx = this.client.multi().sAdd(index.ids, entry.id).set(key, JSON.stringify(entry), setOptions).zAdd(index.time, { score: entry.timestamp, value: entry.id });
25189
+ if (expiresAt !== void 0) tx.zAdd(index.expiry, { score: expiresAt, value: entry.id });
25190
+ else tx.zRem(index.expiry, entry.id);
25191
+ await tx.exec();
25192
+ await this.pruneHistoryAfterSave(index);
24211
25193
  }
24212
25194
  /**
24213
25195
  * Update-only write: see `StateStore.updateEvalResult`.
@@ -24247,15 +25229,14 @@ var RedisStore = class _RedisStore {
24247
25229
  }
24248
25230
  async listEvalResults(limit) {
24249
25231
  return this.listHistoryByZset(
24250
- this.evalHistorySetKey(),
24251
- this.evalHistoryZsetKey(),
24252
- (id) => this.evalHistoryKey(id),
24253
- (a, b) => b.timestamp - a.timestamp,
25232
+ this.historyIndex("eval"),
25233
+ (e) => ({ score: e.timestamp, id: e.id }),
24254
25234
  limit
24255
25235
  );
24256
25236
  }
24257
25237
  async deleteEvalResult(id) {
24258
- const [, deletedCount] = await this.client.multi().sRem(this.evalHistorySetKey(), id).del(this.evalHistoryKey(id)).zRem(this.evalHistoryZsetKey(), id).exec();
25238
+ const index = this.historyIndex("eval");
25239
+ const [, deletedCount] = await this.client.multi().sRem(index.ids, id).del(index.dataKey(id)).zRem(index.time, id).zRem(index.expiry, id).exec();
24259
25240
  const deleted = typeof deletedCount === "number" ? deletedCount : 0;
24260
25241
  return deleted > 0;
24261
25242
  }
@@ -24279,50 +25260,292 @@ var RedisStore = class _RedisStore {
24279
25260
  return { exists: true, expiresAt: Date.now() + ttl };
24280
25261
  }
24281
25262
  /**
24282
- * Shared list-by-sorted-set implementation. Uses ZREVRANGEBYSCORE +
24283
- * MGET when the ZSET is populated (post-migration). Falls back to the
24284
- * legacy SET + N×GET + JS-sort path when the ZSET is empty or smaller
24285
- * than the SET (covers users that constructed with `skipMigration: true`
24286
- * before calling backfill).
25263
+ * Shared list-by-sorted-set implementation, newest first by
25264
+ * `(score DESC, member DESC)`, starting strictly after `cursor` when given.
25265
+ *
25266
+ * Fast path (the time sorted set is at least as large as the id set): walk
25267
+ * the sorted set in chunks and MGET each chunk's blobs. Index entries whose
25268
+ * blob is gone (expired, or deleted before the prune caught up) are skipped,
25269
+ * and the walk keeps reading until it has `limit` live records or the set
25270
+ * runs out, however many dead entries are in the way. Each chunk restarts
25271
+ * at the last member read, by score and member, rather than at an offset,
25272
+ * so entries added or removed meanwhile cannot shift the walk.
25273
+ *
25274
+ * Slow path (the time sorted set is behind the id set: backfill has not
25275
+ * run): read every id with GET, then sort and apply the cursor in process.
24287
25276
  */
24288
- async listHistoryByZset(setKey, zsetKey, dataKey, sortCmp, limit) {
24289
- const setSize = await this.client.sCard(setKey);
25277
+ async listHistoryByZset(index, positionOf, limit, cursor) {
25278
+ const setSize = await this.client.sCard(index.ids);
24290
25279
  if (setSize === 0) return [];
24291
- const zsetSize = await this.client.zCard(zsetKey);
24292
- if (zsetSize >= setSize) {
24293
- const fetchCount = limit !== void 0 ? Math.max(limit * 2, limit + 5) : void 0;
24294
- const ids2 = await this.client.zRange(zsetKey, "+inf", "-inf", {
25280
+ const zsetSize = await this.client.zCard(index.time);
25281
+ const max = limit ? limit : void 0;
25282
+ if (zsetSize < setSize) {
25283
+ const ids = await this.client.sMembers(index.ids);
25284
+ const entries = [];
25285
+ for (const id of ids) {
25286
+ const raw = await this.client.get(index.dataKey(id));
25287
+ if (raw == null) continue;
25288
+ try {
25289
+ entries.push(JSON.parse(raw));
25290
+ } catch {
25291
+ }
25292
+ }
25293
+ const ordered = entries.filter((e) => cursor === void 0 || compareNewestFirst(cursor, positionOf(e)) < 0).sort((a, b) => compareNewestFirst(positionOf(a), positionOf(b)));
25294
+ return max !== void 0 ? ordered.slice(0, max) : ordered;
25295
+ }
25296
+ const results = [];
25297
+ const seen = /* @__PURE__ */ new Set();
25298
+ let bound = cursor;
25299
+ let rangeMax = cursor !== void 0 ? cursor.score : "+inf";
25300
+ let anchor;
25301
+ let grow = 1;
25302
+ const read = (offset, count2) => (
25303
+ // ZRANGE BY SCORE REV: start is the highest score, stop the lowest.
25304
+ this.client.zRangeWithScores(index.time, rangeMax, "-inf", {
24295
25305
  BY: "SCORE",
24296
25306
  REV: true,
24297
- ...fetchCount !== void 0 ? { LIMIT: { offset: 0, count: fetchCount } } : {}
25307
+ ...count2 !== void 0 ? { LIMIT: { offset, count: count2 } } : {}
25308
+ })
25309
+ );
25310
+ for (; ; ) {
25311
+ const remaining = max !== void 0 ? max - results.length : void 0;
25312
+ const want = remaining !== void 0 ? Math.max(remaining * 2, remaining + 5) * grow : void 0;
25313
+ let offset = 0;
25314
+ let count2 = want;
25315
+ let members;
25316
+ if (anchor !== void 0 && want !== void 0) {
25317
+ offset = anchor.pos;
25318
+ count2 = want + 1;
25319
+ members = await read(offset, count2);
25320
+ if (members[0]?.value !== anchor.id) {
25321
+ offset = 0;
25322
+ count2 = anchor.pos + 1 + want;
25323
+ members = await read(offset, count2);
25324
+ }
25325
+ } else {
25326
+ members = await read(offset, count2);
25327
+ }
25328
+ const fresh = members.filter(
25329
+ (m) => !seen.has(m.value) && (bound === void 0 || compareNewestFirst(bound, { score: m.score, id: m.value }) < 0)
25330
+ );
25331
+ if (fresh.length > 0) {
25332
+ const raws = await this.client.mGet(fresh.map((m) => index.dataKey(m.value)));
25333
+ for (let i = 0; i < fresh.length; i++) {
25334
+ seen.add(fresh[i].value);
25335
+ const raw = raws[i];
25336
+ if (raw == null) continue;
25337
+ try {
25338
+ results.push(JSON.parse(raw));
25339
+ } catch {
25340
+ continue;
25341
+ }
25342
+ if (max !== void 0 && results.length >= max) return results;
25343
+ }
25344
+ const lastFresh = fresh[fresh.length - 1];
25345
+ bound = { score: lastFresh.score, id: lastFresh.value };
25346
+ grow = 1;
25347
+ } else {
25348
+ grow *= 2;
25349
+ }
25350
+ if (count2 === void 0 || members.length < count2) return results;
25351
+ const last = members[members.length - 1];
25352
+ if (last.score === rangeMax) {
25353
+ anchor = { id: last.value, score: last.score, pos: offset + members.length - 1 };
25354
+ } else {
25355
+ rangeMax = last.score;
25356
+ const before = members.filter((m) => m.score === last.score).length - 1;
25357
+ anchor = { id: last.value, score: last.score, pos: before };
25358
+ }
25359
+ }
25360
+ }
25361
+ // ── History index cleanup ──────────────────────────────────────────────
25362
+ //
25363
+ // Index members cannot carry a TTL, so when a history record expires its
25364
+ // entries in the id set and the time sorted set stay behind. Every history
25365
+ // save runs one bounded prune of its kind: it reads up to PRUNE_BATCH ids
25366
+ // whose expiry has passed from the expiry sorted set, then runs
25367
+ // PRUNE_HISTORY_INDEX_SCRIPT over them. The script checks each record's blob
25368
+ // and updates the indexes in one atomic step, so a save or delete landing
25369
+ // between the read and the script cannot be undone by it: a record saved
25370
+ // again is seen live and stays listed, and a deleted one is already gone.
25371
+ // The script only removes index entries and re-scores expiry entries; it
25372
+ // never adds a listing entry or writes a blob, so it cannot bring back
25373
+ // deleted data. Several instances pruning at once is safe.
25374
+ /**
25375
+ * Run one prune of `index`'s kind after a save. A prune failure does not
25376
+ * fail the save, whose MULTI already committed the record: it is reported
25377
+ * once per process and the next save of that kind tries again. That also
25378
+ * covers a Redis user whose ACL denies `EVAL`.
25379
+ */
25380
+ async pruneHistoryAfterSave(index) {
25381
+ const inFlight = this.prunesInFlight ??= /* @__PURE__ */ new Set();
25382
+ if (inFlight.has(index.kind)) return;
25383
+ inFlight.add(index.kind);
25384
+ try {
25385
+ await this.pruneHistory(index);
25386
+ } catch (err) {
25387
+ if (this.connection.isReady === false) return;
25388
+ warnDiagnosticOnce(
25389
+ `redis_history_prune_failed\0${index.kind}`,
25390
+ `RedisStore: removing expired ${index.label} index entries failed (${err instanceof Error ? err.message : String(err)}). The record was saved; the next save retries the cleanup. Listings stay correct meanwhile. The cleanup runs a Lua script, so the Redis user needs permission to run EVAL.`,
25391
+ void 0,
25392
+ "docs/api-reference.md#redisstore-options"
25393
+ );
25394
+ } finally {
25395
+ inFlight.delete(index.kind);
25396
+ }
25397
+ }
25398
+ async pruneHistory(index) {
25399
+ const candidates = await this.client.zRange(index.expiry, "-inf", Date.now(), {
25400
+ BY: "SCORE",
25401
+ LIMIT: { offset: 0, count: PRUNE_BATCH }
25402
+ });
25403
+ if (candidates.length > 0) await this.runPruneScript(index, candidates);
25404
+ }
25405
+ /**
25406
+ * Run {@link PRUNE_HISTORY_INDEX_SCRIPT} over `ids`, at most PRUNE_BATCH
25407
+ * per call so one script never blocks the server for long.
25408
+ */
25409
+ async runPruneScript(index, ids) {
25410
+ for (let start = 0; start < ids.length; start += PRUNE_BATCH) {
25411
+ const chunk = ids.slice(start, start + PRUNE_BATCH);
25412
+ await this.client.eval(PRUNE_HISTORY_INDEX_SCRIPT, {
25413
+ keys: [index.ids, index.time, index.expiry, ...chunk.map((id) => index.dataKey(id))],
25414
+ arguments: [String(Date.now()), ...chunk]
24298
25415
  });
24299
- if (ids2.length === 0) return [];
24300
- const keys = ids2.map(dataKey);
24301
- const raws = await this.client.mGet(keys);
24302
- const entries2 = [];
24303
- for (const raw of raws) {
24304
- if (raw == null) continue;
25416
+ }
25417
+ }
25418
+ // ── History maintenance ────────────────────────────────────────────────
25419
+ /**
25420
+ * See `StateStore.applyHistoryRetention`. One call:
25421
+ *
25422
+ * 1. On the first call of a walk (no cursor), backfills the time sorted
25423
+ * set from the id set if it is behind (see `backfillExecutionIndex`).
25424
+ * 2. Reads the next `batchSize` ids from the time sorted set, oldest first
25425
+ * by `(score, member)`, strictly after the cursor. The read restarts at
25426
+ * the cursor's position by value, never by offset.
25427
+ * 3. MGETs their blobs and reads each live blob's PTTL.
25428
+ * 4. In one MULTI, for each live blob:
25429
+ * - a TTL (`PTTL > 0`): `ZADD NX` to the expiry set at now + PTTL. A
25430
+ * record already in the expiry set keeps its entry (`indexed` counts
25431
+ * only new entries).
25432
+ * - no TTL (`-1`), with a retention configured: `PEXPIREAT … NX` at last
25433
+ * save + retention. `NX` applies only to a key that still has no TTL,
25434
+ * so a record re-saved meanwhile keeps the fresh window its save gave
25435
+ * it. A window that has already passed uses a timestamp in the past,
25436
+ * which makes Redis delete the key, again only if it is still untimed.
25437
+ * 5. Adds each newly armed record to the expiry set (`ZADD NX`).
25438
+ * 6. Runs the prune script (`PRUNE_HISTORY_INDEX_SCRIPT`) over ids whose
25439
+ * blob is gone and ids just deleted in step 4. It checks each blob
25440
+ * atomically, so a record re-saved at any point stays listed and a
25441
+ * deleted one is never re-indexed.
25442
+ *
25443
+ * Blobs are never written. A record deleted while the call runs can leave
25444
+ * an entry in the expiry set only; the next prune removes it.
25445
+ */
25446
+ async applyHistoryRetention(options) {
25447
+ const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("RedisStore", options);
25448
+ const index = this.historyIndex(kind === "executions" ? "execution" : "eval");
25449
+ const ttl = this.ttlFor(kind === "executions" ? "executionHistory" : "evalHistory");
25450
+ if (cursor === void 0) {
25451
+ if (kind === "executions") await this.backfillExecutionIndex();
25452
+ else await this.backfillEvalIndex();
25453
+ }
25454
+ const positions = await this.readRetentionBatch(index, cursor, batchSize + 1);
25455
+ const batch = positions.slice(0, batchSize);
25456
+ const result = {
25457
+ expired: 0,
25458
+ armed: 0,
25459
+ indexed: 0,
25460
+ deadIndexEntries: 0,
25461
+ cursor: positions.length > batchSize ? { time: batch[batch.length - 1].score, id: batch[batch.length - 1].value } : null
25462
+ };
25463
+ if (batch.length === 0) return result;
25464
+ const ids = batch.map((m) => m.value);
25465
+ const raws = await this.client.mGet(ids.map((id) => index.dataKey(id)));
25466
+ const dead = ids.filter((_, i) => raws[i] == null);
25467
+ const present = ids.flatMap((id, i) => {
25468
+ const raw = raws[i];
25469
+ return raw == null ? [] : [{ id, raw }];
25470
+ });
25471
+ const pttls = await Promise.all(present.map((r) => this.client.pTTL(index.dataKey(r.id))));
25472
+ const now = Date.now();
25473
+ const tx = this.client.multi();
25474
+ const queued = [];
25475
+ for (let i = 0; i < present.length; i++) {
25476
+ const { id, raw } = present[i];
25477
+ const pttl = pttls[i];
25478
+ if (pttl === -2) {
25479
+ dead.push(id);
25480
+ } else if (pttl > 0) {
25481
+ tx.zAdd(index.expiry, { score: now + pttl, value: id }, { condition: "NX" });
25482
+ queued.push({ action: "index", id });
25483
+ } else if (pttl === -1 && ttl !== void 0) {
25484
+ let lastSavedAt;
24305
25485
  try {
24306
- entries2.push(JSON.parse(raw));
25486
+ lastSavedAt = index.lastSavedAtOf(raw);
24307
25487
  } catch {
25488
+ lastSavedAt = null;
25489
+ }
25490
+ if (lastSavedAt === null) continue;
25491
+ const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
25492
+ if (decision.expired) {
25493
+ tx.pExpireAt(index.dataKey(id), 1, "NX");
25494
+ queued.push({ action: "expire", id });
25495
+ } else {
25496
+ tx.pExpireAt(index.dataKey(id), decision.expiresAt, "NX");
25497
+ queued.push({ action: "arm", id, expiresAt: decision.expiresAt });
24308
25498
  }
24309
- if (limit !== void 0 && entries2.length >= limit) break;
24310
25499
  }
24311
- return entries2;
24312
25500
  }
24313
- const ids = await this.client.sMembers(setKey);
24314
- if (ids.length === 0) return [];
24315
- const entries = [];
24316
- for (const id of ids) {
24317
- const raw = await this.client.get(dataKey(id));
24318
- if (raw == null) continue;
24319
- try {
24320
- entries.push(JSON.parse(raw));
24321
- } catch {
25501
+ const expiredIds = [];
25502
+ const armed = [];
25503
+ if (queued.length > 0) {
25504
+ const replies = await tx.exec();
25505
+ for (let i = 0; i < queued.length; i++) {
25506
+ if (Number(replies[i]) !== 1) continue;
25507
+ const entry = queued[i];
25508
+ if (entry.action === "index") result.indexed++;
25509
+ else if (entry.action === "expire") expiredIds.push(entry.id);
25510
+ else armed.push(entry);
24322
25511
  }
24323
25512
  }
24324
- entries.sort(sortCmp);
24325
- return limit ? entries.slice(0, limit) : entries;
25513
+ if (armed.length > 0) {
25514
+ const indexTx = this.client.multi();
25515
+ for (const { id, expiresAt } of armed) {
25516
+ indexTx.zAdd(index.expiry, { score: expiresAt, value: id }, { condition: "NX" });
25517
+ }
25518
+ await indexTx.exec();
25519
+ }
25520
+ result.armed = armed.length;
25521
+ result.expired = expiredIds.length;
25522
+ result.deadIndexEntries = dead.length;
25523
+ if (dead.length > 0 || expiredIds.length > 0) {
25524
+ await this.runPruneScript(index, [...dead, ...expiredIds]);
25525
+ }
25526
+ return result;
25527
+ }
25528
+ /**
25529
+ * Up to `count` members of the time sorted set strictly after `cursor`,
25530
+ * oldest first by `(score, member)`. The members sharing the cursor's score
25531
+ * are read in full and filtered by member; the rest start just above that
25532
+ * score. Neither read uses an offset, so entries removed or added
25533
+ * meanwhile cannot shift the walk.
25534
+ */
25535
+ async readRetentionBatch(index, cursor, count2) {
25536
+ if (cursor === void 0) {
25537
+ return this.client.zRangeWithScores(index.time, "-inf", "+inf", {
25538
+ BY: "SCORE",
25539
+ LIMIT: { offset: 0, count: count2 }
25540
+ });
25541
+ }
25542
+ const ties = (await this.client.zRangeWithScores(index.time, cursor.time, cursor.time, { BY: "SCORE" })).filter((m) => compareMembers(m.value, cursor.id) > 0);
25543
+ if (ties.length >= count2) return ties.slice(0, count2);
25544
+ const rest = await this.client.zRangeWithScores(index.time, `(${cursor.time}`, "+inf", {
25545
+ BY: "SCORE",
25546
+ LIMIT: { offset: 0, count: count2 - ties.length }
25547
+ });
25548
+ return [...ties, ...rest];
24326
25549
  }
24327
25550
  // ── Sessions (Studio introspection) ────────────────────────────────────
24328
25551
  async listSessions() {
@@ -24366,10 +25589,31 @@ var RedisStore = class _RedisStore {
24366
25589
  * must not get a rejected promise for it. `MemoryStore.close()` and
24367
25590
  * better-sqlite3's `close()` are both no-ops the second time; node-redis
24368
25591
  * instead rejects with "The client is closed", so absorb that here.
25592
+ *
25593
+ * Bounded, so a shutdown can't hang on Redis. `QUIT` lets queued commands
25594
+ * finish on a healthy connection, but node-redis never settles it if the
25595
+ * socket closes first, and a stalled server may never answer it. While
25596
+ * disconnected it couldn't be sent at all.
24369
25597
  */
24370
25598
  async close() {
24371
- if (this.client.isOpen === false) return;
24372
- await this.client.quit();
25599
+ if (this.connection.isOpen === false) return;
25600
+ if (this.connection.isReady === false) {
25601
+ abandonConnection(this.connection);
25602
+ return;
25603
+ }
25604
+ let timer;
25605
+ try {
25606
+ await Promise.race([
25607
+ this.connection.quit(),
25608
+ new Promise((_resolve, reject) => {
25609
+ timer = setTimeout(() => reject(new Error("QUIT timed out")), CLOSE_TIMEOUT_MS);
25610
+ })
25611
+ ]);
25612
+ } catch {
25613
+ abandonConnection(this.connection);
25614
+ } finally {
25615
+ clearTimeout(timer);
25616
+ }
24373
25617
  }
24374
25618
  async deleteCheckpoints(executionId) {
24375
25619
  await this.client.del(this.checkpointKey(executionId));