@axlsdk/axl 0.26.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/index.cjs +1501 -257
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +564 -60
- package/dist/index.d.ts +564 -60
- package/dist/index.js +1501 -257
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -602,7 +602,7 @@ var sharedGuard = typeof installedGuard === "object" && installedGuard !== null
|
|
|
602
602
|
var buildPath = typeof __AXL_BUILD_PATH__ === "string" ? __AXL_BUILD_PATH__ : "<source>";
|
|
603
603
|
var thisCopy = {
|
|
604
604
|
path: buildPath.startsWith("file:") ? (0, import_node_url.fileURLToPath)(buildPath) : buildPath,
|
|
605
|
-
version: true ? "0.
|
|
605
|
+
version: true ? "0.27.1" : "unknown"
|
|
606
606
|
};
|
|
607
607
|
function warnCrossCopy(other, joined) {
|
|
608
608
|
if (other === thisCopy) return;
|
|
@@ -12869,8 +12869,17 @@ function detectDroppedRefinements(schema) {
|
|
|
12869
12869
|
var DOCS_URL = "docs/observability.md#schema-diagnostics";
|
|
12870
12870
|
var MAX_WARNED_KEYS = 2048;
|
|
12871
12871
|
var warnedDiagnosticKeys = /* @__PURE__ */ new Set();
|
|
12872
|
+
var silencedDepth = 0;
|
|
12873
|
+
function withDiagnosticsSilenced(fn) {
|
|
12874
|
+
silencedDepth++;
|
|
12875
|
+
try {
|
|
12876
|
+
return fn();
|
|
12877
|
+
} finally {
|
|
12878
|
+
silencedDepth--;
|
|
12879
|
+
}
|
|
12880
|
+
}
|
|
12872
12881
|
function warnDiagnosticOnce(key, message, silent, docs = DOCS_URL) {
|
|
12873
|
-
if (silent || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
|
|
12882
|
+
if (silent || silencedDepth > 0 || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
|
|
12874
12883
|
if (warnedDiagnosticKeys.has(key)) return;
|
|
12875
12884
|
if (warnedDiagnosticKeys.size >= MAX_WARNED_KEYS) warnedDiagnosticKeys.clear();
|
|
12876
12885
|
warnedDiagnosticKeys.add(key);
|
|
@@ -17271,7 +17280,7 @@ ${appendedSchemaText}`;
|
|
|
17271
17280
|
`ctx.checkpoint name "${name}" is reserved \u2014 names starting with "__auto/" (case-insensitive) are used by the runtime for ask/spawn/race/parallel/map auto-checkpointing.`
|
|
17272
17281
|
);
|
|
17273
17282
|
}
|
|
17274
|
-
return this._checkpoint(name, fn);
|
|
17283
|
+
return this._checkpoint(name, fn, { saveRequired: true });
|
|
17275
17284
|
}
|
|
17276
17285
|
/**
|
|
17277
17286
|
* Internal checkpoint implementation shared by both the public
|
|
@@ -17301,7 +17310,17 @@ ${appendedSchemaText}`;
|
|
|
17301
17310
|
return saved;
|
|
17302
17311
|
}
|
|
17303
17312
|
const result = await fn();
|
|
17304
|
-
|
|
17313
|
+
if (!options?.saveRequired && this.autoCheckpointCounters.saveFailed) return result;
|
|
17314
|
+
try {
|
|
17315
|
+
await this.stateStore.saveCheckpoint(this.executionId, name, result);
|
|
17316
|
+
} catch (err) {
|
|
17317
|
+
if (options?.saveRequired) throw err;
|
|
17318
|
+
this.autoCheckpointCounters.saveFailed = true;
|
|
17319
|
+
console.error(
|
|
17320
|
+
`[axl] Failed to save checkpoint "${name}" for executionId=${this.executionId}; a replay would run this step and later ones again: ${err instanceof Error ? err.message : String(err)}`
|
|
17321
|
+
);
|
|
17322
|
+
return result;
|
|
17323
|
+
}
|
|
17305
17324
|
this.emitEvent({
|
|
17306
17325
|
type: "checkpoint_save",
|
|
17307
17326
|
...agentStamp,
|
|
@@ -18934,10 +18953,135 @@ var import_node_async_hooks4 = require("async_hooks");
|
|
|
18934
18953
|
var import_node_fs3 = require("fs");
|
|
18935
18954
|
var import_node_path3 = require("path");
|
|
18936
18955
|
|
|
18956
|
+
// src/state/retention.ts
|
|
18957
|
+
var PRUNE_BATCH = 100;
|
|
18958
|
+
var DEFAULT_MEMORY_MAX_EXECUTIONS = 1e3;
|
|
18959
|
+
function warnExecutionRetentionUnset(storeName) {
|
|
18960
|
+
warnDiagnosticOnce(
|
|
18961
|
+
`execution_retention_unset\0${storeName}`,
|
|
18962
|
+
`${storeName} keeps execution history forever because no retention is set; set ttls.executionHistory${storeName === "RedisStore" ? " (or defaultTtl)" : ""} to a number of seconds, or ttls.executionHistory to null to keep it forever without this warning.`,
|
|
18963
|
+
void 0,
|
|
18964
|
+
"docs/migration/execution-history-retention.md#silencing-the-retention-warning"
|
|
18965
|
+
);
|
|
18966
|
+
}
|
|
18967
|
+
function resolveTtl(storeName, category, value) {
|
|
18968
|
+
if (value === void 0 || value === null) return null;
|
|
18969
|
+
if (typeof value !== "number") {
|
|
18970
|
+
throw new AxlError(
|
|
18971
|
+
"INVALID_CONFIG",
|
|
18972
|
+
`${storeName}: ttls.${category} must be a positive number of seconds or null; got ${typeof value}`
|
|
18973
|
+
);
|
|
18974
|
+
}
|
|
18975
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
18976
|
+
throw new AxlError(
|
|
18977
|
+
"INVALID_CONFIG",
|
|
18978
|
+
`${storeName}: ttls.${category} must be a positive number of seconds or null; got ${value}`
|
|
18979
|
+
);
|
|
18980
|
+
}
|
|
18981
|
+
return value;
|
|
18982
|
+
}
|
|
18983
|
+
function resolveHistoryTtls(storeName, ttls) {
|
|
18984
|
+
if (ttls !== void 0 && (ttls === null || typeof ttls !== "object")) {
|
|
18985
|
+
throw new AxlError(
|
|
18986
|
+
"INVALID_CONFIG",
|
|
18987
|
+
`${storeName}: ttls must be an object; got ${ttls === null ? "null" : typeof ttls}`
|
|
18988
|
+
);
|
|
18989
|
+
}
|
|
18990
|
+
return {
|
|
18991
|
+
executionHistory: resolveTtl(storeName, "executionHistory", ttls?.executionHistory),
|
|
18992
|
+
evalHistory: resolveTtl(storeName, "evalHistory", ttls?.evalHistory)
|
|
18993
|
+
};
|
|
18994
|
+
}
|
|
18995
|
+
function expiryFrom(now, ttlSeconds) {
|
|
18996
|
+
return ttlSeconds === null ? null : now + Math.round(ttlSeconds * 1e3);
|
|
18997
|
+
}
|
|
18998
|
+
function isExpired(expiresAt, now) {
|
|
18999
|
+
return expiresAt !== null && now >= expiresAt;
|
|
19000
|
+
}
|
|
19001
|
+
var DEFAULT_RETENTION_BATCH_SIZE = 500;
|
|
19002
|
+
function resolveHistoryRetentionOptions(storeName, options) {
|
|
19003
|
+
const method = `${storeName}.applyHistoryRetention()`;
|
|
19004
|
+
if (options === null || typeof options !== "object") {
|
|
19005
|
+
throw new AxlError("INVALID_ARGUMENT", `${method} options must be an object`);
|
|
19006
|
+
}
|
|
19007
|
+
const { kind, batchSize, cursor } = options;
|
|
19008
|
+
if (kind !== "executions" && kind !== "evals") {
|
|
19009
|
+
throw new AxlError(
|
|
19010
|
+
"INVALID_ARGUMENT",
|
|
19011
|
+
`${method} kind must be 'executions' or 'evals'; got ${String(kind)}`
|
|
19012
|
+
);
|
|
19013
|
+
}
|
|
19014
|
+
if (batchSize !== void 0 && !(typeof batchSize === "number" && Number.isSafeInteger(batchSize) && batchSize > 0)) {
|
|
19015
|
+
throw new AxlError(
|
|
19016
|
+
"INVALID_ARGUMENT",
|
|
19017
|
+
`${method} batchSize must be a positive integer; got ${String(batchSize)}`
|
|
19018
|
+
);
|
|
19019
|
+
}
|
|
19020
|
+
let parsedCursor;
|
|
19021
|
+
if (cursor !== void 0 && cursor !== null) {
|
|
19022
|
+
const candidate = cursor;
|
|
19023
|
+
if (typeof candidate !== "object" || typeof candidate.time !== "number" || !Number.isFinite(candidate.time) || typeof candidate.id !== "string") {
|
|
19024
|
+
throw new AxlError(
|
|
19025
|
+
"INVALID_ARGUMENT",
|
|
19026
|
+
`${method} cursor must be the { time, id } cursor returned by the previous call`
|
|
19027
|
+
);
|
|
19028
|
+
}
|
|
19029
|
+
parsedCursor = { time: candidate.time, id: candidate.id };
|
|
19030
|
+
}
|
|
19031
|
+
return {
|
|
19032
|
+
kind,
|
|
19033
|
+
batchSize: batchSize ?? DEFAULT_RETENTION_BATCH_SIZE,
|
|
19034
|
+
cursor: parsedCursor
|
|
19035
|
+
};
|
|
19036
|
+
}
|
|
19037
|
+
function retentionForUntimedRecord(lastSavedAt, ttlSeconds, now) {
|
|
19038
|
+
const expiresAt = lastSavedAt + Math.round(ttlSeconds * 1e3);
|
|
19039
|
+
return isExpired(expiresAt, now) ? { expired: true } : { expired: false, expiresAt };
|
|
19040
|
+
}
|
|
19041
|
+
function executionLastSavedAt(record) {
|
|
19042
|
+
return typeof record.completedAt === "number" && Number.isFinite(record.completedAt) ? record.completedAt : record.startedAt;
|
|
19043
|
+
}
|
|
19044
|
+
function compareExecutionsNewestFirst(a, b) {
|
|
19045
|
+
if (a.startedAt !== b.startedAt) return b.startedAt - a.startedAt;
|
|
19046
|
+
return compareCodePoints(b.executionId, a.executionId);
|
|
19047
|
+
}
|
|
19048
|
+
function compareRetentionPositions(a, b) {
|
|
19049
|
+
if (a.time !== b.time) return a.time - b.time;
|
|
19050
|
+
return compareCodePoints(a.id, b.id);
|
|
19051
|
+
}
|
|
19052
|
+
function compareCodePoints(a, b) {
|
|
19053
|
+
const len = Math.min(a.length, b.length);
|
|
19054
|
+
for (let i = 0; i < len; i++) {
|
|
19055
|
+
const x = a.charCodeAt(i);
|
|
19056
|
+
const y = b.charCodeAt(i);
|
|
19057
|
+
if (x !== y) return codePointRank(x) - codePointRank(y);
|
|
19058
|
+
}
|
|
19059
|
+
return a.length - b.length;
|
|
19060
|
+
}
|
|
19061
|
+
function codePointRank(unit) {
|
|
19062
|
+
if (unit >= 57344) return unit - 2048;
|
|
19063
|
+
if (unit >= 55296) return unit + 8192;
|
|
19064
|
+
return unit;
|
|
19065
|
+
}
|
|
19066
|
+
|
|
18937
19067
|
// src/state/memory.ts
|
|
18938
19068
|
var import_node_fs = require("fs");
|
|
18939
19069
|
var import_node_path = require("path");
|
|
18940
19070
|
var import_node_os = require("os");
|
|
19071
|
+
function compareOldestFirst(a, b) {
|
|
19072
|
+
return compareExecutionsNewestFirst(b, a);
|
|
19073
|
+
}
|
|
19074
|
+
function resolveMaxExecutions(value) {
|
|
19075
|
+
if (value === void 0) return DEFAULT_MEMORY_MAX_EXECUTIONS;
|
|
19076
|
+
if (value === Infinity) return Infinity;
|
|
19077
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
|
|
19078
|
+
throw new AxlError(
|
|
19079
|
+
"INVALID_CONFIG",
|
|
19080
|
+
`MemoryStore: maxExecutions must be a positive integer or Infinity; got ${typeof value === "number" ? value : typeof value}`
|
|
19081
|
+
);
|
|
19082
|
+
}
|
|
19083
|
+
return value;
|
|
19084
|
+
}
|
|
18941
19085
|
var AWAIT_HUMAN_TEMP_DIR = (0, import_node_path.join)((0, import_node_os.tmpdir)(), "axl-memory-store");
|
|
18942
19086
|
var AWAIT_HUMAN_TEMP_FILE = (0, import_node_path.join)(AWAIT_HUMAN_TEMP_DIR, "await-human-state.json");
|
|
18943
19087
|
var MemoryStore = class {
|
|
@@ -18947,9 +19091,32 @@ var MemoryStore = class {
|
|
|
18947
19091
|
decisions = /* @__PURE__ */ new Map();
|
|
18948
19092
|
executionStates = /* @__PURE__ */ new Map();
|
|
18949
19093
|
memories = /* @__PURE__ */ new Map();
|
|
19094
|
+
/**
|
|
19095
|
+
* Iteration order is last-save order: a save deletes and re-inserts its key.
|
|
19096
|
+
* With one retention per store and a clock that does not run backwards, that
|
|
19097
|
+
* is also expiry order, which lets the prune stop at the first live record.
|
|
19098
|
+
*/
|
|
18950
19099
|
executionHistory = /* @__PURE__ */ new Map();
|
|
19100
|
+
/** Every key in `executionHistory`, sorted oldest first in listing order. */
|
|
19101
|
+
executionOrder = [];
|
|
19102
|
+
/**
|
|
19103
|
+
* Iteration order is first-write order (a re-save keeps the key's position
|
|
19104
|
+
* and its expiry), which is also expiry order for the same reason as above.
|
|
19105
|
+
*/
|
|
18951
19106
|
evalHistory = /* @__PURE__ */ new Map();
|
|
18952
|
-
|
|
19107
|
+
ttls;
|
|
19108
|
+
maxExecutions;
|
|
19109
|
+
now;
|
|
19110
|
+
constructor(options) {
|
|
19111
|
+
this.ttls = resolveHistoryTtls("MemoryStore", options?.ttls);
|
|
19112
|
+
this.maxExecutions = resolveMaxExecutions(options?.maxExecutions);
|
|
19113
|
+
if (options?.now !== void 0 && typeof options.now !== "function") {
|
|
19114
|
+
throw new AxlError(
|
|
19115
|
+
"INVALID_CONFIG",
|
|
19116
|
+
"MemoryStore: now must be a function returning epoch milliseconds"
|
|
19117
|
+
);
|
|
19118
|
+
}
|
|
19119
|
+
this.now = options?.now ?? (() => Date.now());
|
|
18953
19120
|
this.loadPersistedState();
|
|
18954
19121
|
}
|
|
18955
19122
|
async saveCheckpoint(executionId, name, data) {
|
|
@@ -19046,23 +19213,43 @@ var MemoryStore = class {
|
|
|
19046
19213
|
}
|
|
19047
19214
|
// ── Execution History ──────────────────────────────────────────────
|
|
19048
19215
|
async saveExecution(execution) {
|
|
19049
|
-
const
|
|
19050
|
-
|
|
19051
|
-
|
|
19216
|
+
const now = this.now();
|
|
19217
|
+
const normalized = structuredClone(
|
|
19218
|
+
normalizeStoredExecution({
|
|
19219
|
+
...execution,
|
|
19220
|
+
eventSchemaVersion: getExecutionEventSchemaVersion(execution)
|
|
19221
|
+
})
|
|
19222
|
+
);
|
|
19223
|
+
this.removeExecutionRecord(normalized.executionId);
|
|
19224
|
+
this.executionHistory.set(normalized.executionId, {
|
|
19225
|
+
record: normalized,
|
|
19226
|
+
expiresAt: expiryFrom(now, this.ttls.executionHistory)
|
|
19052
19227
|
});
|
|
19053
|
-
this.
|
|
19228
|
+
this.insertExecutionOrder(normalized);
|
|
19229
|
+
this.pruneExpiredExecutions(now);
|
|
19230
|
+
this.evictOverflowExecutions();
|
|
19054
19231
|
}
|
|
19055
19232
|
async getExecution(executionId) {
|
|
19056
|
-
const
|
|
19057
|
-
|
|
19058
|
-
|
|
19059
|
-
|
|
19060
|
-
|
|
19061
|
-
|
|
19062
|
-
|
|
19233
|
+
const stored = this.executionHistory.get(executionId);
|
|
19234
|
+
if (!stored || isExpired(stored.expiresAt, this.now())) return null;
|
|
19235
|
+
return normalizeStoredExecution(structuredClone(stored.record));
|
|
19236
|
+
}
|
|
19237
|
+
/** Supports `listExecutions(limit, { cursor })`. */
|
|
19238
|
+
historyPaging = true;
|
|
19239
|
+
async listExecutions(limit, options) {
|
|
19240
|
+
const now = this.now();
|
|
19241
|
+
const max = limit ? limit : Infinity;
|
|
19242
|
+
const result = [];
|
|
19243
|
+
const start = options?.cursor !== void 0 ? this.executionOrderLowerBound(options.cursor) - 1 : this.executionOrder.length - 1;
|
|
19244
|
+
for (let i = start; i >= 0 && result.length < max; i--) {
|
|
19245
|
+
const stored = this.executionHistory.get(this.executionOrder[i].executionId);
|
|
19246
|
+
if (isExpired(stored.expiresAt, now)) continue;
|
|
19247
|
+
result.push(normalizeStoredExecution(structuredClone(stored.record)));
|
|
19248
|
+
}
|
|
19249
|
+
return result;
|
|
19063
19250
|
}
|
|
19064
19251
|
async deleteExecution(executionId) {
|
|
19065
|
-
const removedHistory = this.
|
|
19252
|
+
const removedHistory = this.removeExecutionRecord(executionId);
|
|
19066
19253
|
const removedCheckpoints = this.checkpoints.delete(executionId);
|
|
19067
19254
|
const removedState = this.executionStates.delete(executionId);
|
|
19068
19255
|
const removedStreaming = this.streamingEvents.delete(executionId);
|
|
@@ -19073,26 +19260,193 @@ var MemoryStore = class {
|
|
|
19073
19260
|
return removedHistory || removedCheckpoints || removedState || removedStreaming || removedDecision;
|
|
19074
19261
|
}
|
|
19075
19262
|
// ── Eval History ──────────────────────────────────────────────────
|
|
19263
|
+
/**
|
|
19264
|
+
* Write an eval history row. Retention counts from the row's first write:
|
|
19265
|
+
* re-saving a live row keeps its expiry (and its position in expiry order).
|
|
19266
|
+
* A row that has already expired is replaced as a new one.
|
|
19267
|
+
*/
|
|
19076
19268
|
async saveEvalResult(entry) {
|
|
19077
|
-
this.
|
|
19269
|
+
const now = this.now();
|
|
19270
|
+
const existing = this.evalHistory.get(entry.id);
|
|
19271
|
+
if (existing && !isExpired(existing.expiresAt, now)) {
|
|
19272
|
+
this.evalHistory.set(entry.id, {
|
|
19273
|
+
entry: structuredClone(entry),
|
|
19274
|
+
expiresAt: existing.expiresAt
|
|
19275
|
+
});
|
|
19276
|
+
} else {
|
|
19277
|
+
this.evalHistory.delete(entry.id);
|
|
19278
|
+
this.evalHistory.set(entry.id, {
|
|
19279
|
+
entry: structuredClone(entry),
|
|
19280
|
+
expiresAt: expiryFrom(now, this.ttls.evalHistory)
|
|
19281
|
+
});
|
|
19282
|
+
}
|
|
19283
|
+
this.pruneExpiredEvals(now);
|
|
19078
19284
|
}
|
|
19079
|
-
/** Update-only write: see `StateStore.updateEvalResult`. */
|
|
19285
|
+
/** Update-only write: see `StateStore.updateEvalResult`. Never changes retention. */
|
|
19080
19286
|
async updateEvalResult(entry) {
|
|
19081
|
-
|
|
19082
|
-
|
|
19287
|
+
const existing = this.evalHistory.get(entry.id);
|
|
19288
|
+
if (!existing || isExpired(existing.expiresAt, this.now())) return false;
|
|
19289
|
+
this.evalHistory.set(entry.id, {
|
|
19290
|
+
entry: structuredClone(entry),
|
|
19291
|
+
expiresAt: existing.expiresAt
|
|
19292
|
+
});
|
|
19083
19293
|
return true;
|
|
19084
19294
|
}
|
|
19085
19295
|
async listEvalResults(limit) {
|
|
19086
|
-
const
|
|
19296
|
+
const now = this.now();
|
|
19297
|
+
const sorted = [...this.evalHistory.values()].filter((stored) => !isExpired(stored.expiresAt, now)).map((stored) => stored.entry).sort((a, b) => b.timestamp - a.timestamp);
|
|
19087
19298
|
const result = limit ? sorted.slice(0, limit) : sorted;
|
|
19088
19299
|
return result.map((e) => structuredClone(e));
|
|
19089
19300
|
}
|
|
19090
19301
|
async deleteEvalResult(id) {
|
|
19091
19302
|
return this.evalHistory.delete(id);
|
|
19092
19303
|
}
|
|
19093
|
-
/**
|
|
19304
|
+
/** Existence plus the expiry stored at the row's first write, if any. */
|
|
19094
19305
|
async getEvalRetention(id) {
|
|
19095
|
-
|
|
19306
|
+
const stored = this.evalHistory.get(id);
|
|
19307
|
+
if (!stored || isExpired(stored.expiresAt, this.now())) return { exists: false };
|
|
19308
|
+
return stored.expiresAt === null ? { exists: true } : { exists: true, expiresAt: stored.expiresAt };
|
|
19309
|
+
}
|
|
19310
|
+
// ── History maintenance ───────────────────────────────────────────
|
|
19311
|
+
/**
|
|
19312
|
+
* See `StateStore.applyHistoryRetention`. Every record this store holds was
|
|
19313
|
+
* saved under the retention it was constructed with, so in practice a run
|
|
19314
|
+
* finds nothing to expire or arm; the method exists so the same maintenance
|
|
19315
|
+
* code works against every built-in store. There is no separate index, so
|
|
19316
|
+
* `indexed` and `deadIndexEntries` are always `0`.
|
|
19317
|
+
*
|
|
19318
|
+
* An armed record keeps its place in last-save order, which is also where
|
|
19319
|
+
* its expiry (last save + retention) belongs, so the prune-on-save order
|
|
19320
|
+
* still holds.
|
|
19321
|
+
*/
|
|
19322
|
+
async applyHistoryRetention(options) {
|
|
19323
|
+
const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("MemoryStore", options);
|
|
19324
|
+
const now = this.now();
|
|
19325
|
+
let expired = 0;
|
|
19326
|
+
let armed = 0;
|
|
19327
|
+
let positions;
|
|
19328
|
+
if (kind === "executions") {
|
|
19329
|
+
let start = 0;
|
|
19330
|
+
if (cursor !== void 0) {
|
|
19331
|
+
const key = { startedAt: cursor.time, executionId: cursor.id };
|
|
19332
|
+
start = this.executionOrderLowerBound(key);
|
|
19333
|
+
const at = this.executionOrder[start];
|
|
19334
|
+
if (at !== void 0 && compareOldestFirst(at, key) === 0) start++;
|
|
19335
|
+
}
|
|
19336
|
+
positions = this.executionOrder.slice(start, start + batchSize + 1).map((k) => ({ time: k.startedAt, id: k.executionId }));
|
|
19337
|
+
} else {
|
|
19338
|
+
positions = [...this.evalHistory.values()].map((stored) => ({ time: stored.entry.timestamp, id: stored.entry.id })).filter((p) => cursor === void 0 || compareRetentionPositions(cursor, p) < 0).sort(compareRetentionPositions).slice(0, batchSize + 1);
|
|
19339
|
+
}
|
|
19340
|
+
const batch = positions.slice(0, batchSize);
|
|
19341
|
+
const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
|
|
19342
|
+
if (ttl !== null) {
|
|
19343
|
+
for (const { id } of batch) {
|
|
19344
|
+
if (kind === "executions") {
|
|
19345
|
+
const stored = this.executionHistory.get(id);
|
|
19346
|
+
if (!stored || stored.expiresAt !== null) continue;
|
|
19347
|
+
const decision = retentionForUntimedRecord(executionLastSavedAt(stored.record), ttl, now);
|
|
19348
|
+
if (decision.expired) {
|
|
19349
|
+
this.expireExecutionRecord(id);
|
|
19350
|
+
expired++;
|
|
19351
|
+
} else {
|
|
19352
|
+
stored.expiresAt = decision.expiresAt;
|
|
19353
|
+
armed++;
|
|
19354
|
+
}
|
|
19355
|
+
} else {
|
|
19356
|
+
const stored = this.evalHistory.get(id);
|
|
19357
|
+
if (!stored || stored.expiresAt !== null) continue;
|
|
19358
|
+
const decision = retentionForUntimedRecord(stored.entry.timestamp, ttl, now);
|
|
19359
|
+
if (decision.expired) {
|
|
19360
|
+
this.evalHistory.delete(id);
|
|
19361
|
+
expired++;
|
|
19362
|
+
} else {
|
|
19363
|
+
stored.expiresAt = decision.expiresAt;
|
|
19364
|
+
armed++;
|
|
19365
|
+
}
|
|
19366
|
+
}
|
|
19367
|
+
}
|
|
19368
|
+
}
|
|
19369
|
+
return {
|
|
19370
|
+
expired,
|
|
19371
|
+
armed,
|
|
19372
|
+
indexed: 0,
|
|
19373
|
+
deadIndexEntries: 0,
|
|
19374
|
+
cursor: positions.length > batchSize ? batch[batch.length - 1] : null
|
|
19375
|
+
};
|
|
19376
|
+
}
|
|
19377
|
+
// ── History retention internals ───────────────────────────────────
|
|
19378
|
+
/** Index of the first order key not older than `key` (binary search). */
|
|
19379
|
+
executionOrderLowerBound(key) {
|
|
19380
|
+
let lo = 0;
|
|
19381
|
+
let hi = this.executionOrder.length;
|
|
19382
|
+
while (lo < hi) {
|
|
19383
|
+
const mid = lo + hi >>> 1;
|
|
19384
|
+
if (compareOldestFirst(this.executionOrder[mid], key) < 0) lo = mid + 1;
|
|
19385
|
+
else hi = mid;
|
|
19386
|
+
}
|
|
19387
|
+
return lo;
|
|
19388
|
+
}
|
|
19389
|
+
insertExecutionOrder(record) {
|
|
19390
|
+
const key = { startedAt: record.startedAt, executionId: record.executionId };
|
|
19391
|
+
this.executionOrder.splice(this.executionOrderLowerBound(key), 0, key);
|
|
19392
|
+
}
|
|
19393
|
+
/** Remove a history record and its listing-index key. Returns whether one existed. */
|
|
19394
|
+
removeExecutionRecord(executionId) {
|
|
19395
|
+
const stored = this.executionHistory.get(executionId);
|
|
19396
|
+
if (!stored) return false;
|
|
19397
|
+
this.executionHistory.delete(executionId);
|
|
19398
|
+
const index = this.executionOrderLowerBound(stored.record);
|
|
19399
|
+
if (this.executionOrder[index]?.executionId === executionId) {
|
|
19400
|
+
this.executionOrder.splice(index, 1);
|
|
19401
|
+
}
|
|
19402
|
+
return true;
|
|
19403
|
+
}
|
|
19404
|
+
/**
|
|
19405
|
+
* A run leaving history through retention (expiry or `maxExecutions`) takes
|
|
19406
|
+
* its leftover per-run data with it: checkpoints (kept today when a run
|
|
19407
|
+
* fails), saved execution state and any streaming buffer. Pending decisions
|
|
19408
|
+
* are not touched. The runtime never writes execution state, and a run is
|
|
19409
|
+
* only saved to history once it has finished, so nothing still running
|
|
19410
|
+
* depends on what is removed here.
|
|
19411
|
+
*/
|
|
19412
|
+
expireExecutionRecord(executionId) {
|
|
19413
|
+
this.removeExecutionRecord(executionId);
|
|
19414
|
+
this.checkpoints.delete(executionId);
|
|
19415
|
+
const state = this.executionStates.get(executionId);
|
|
19416
|
+
this.executionStates.delete(executionId);
|
|
19417
|
+
this.streamingEvents.delete(executionId);
|
|
19418
|
+
if (state?.status === "waiting") this.persistAwaitHumanState();
|
|
19419
|
+
}
|
|
19420
|
+
/**
|
|
19421
|
+
* Remove up to `PRUNE_BATCH` expired execution records, oldest save first.
|
|
19422
|
+
* Stops at the first live record: Map order is last-save order, which is
|
|
19423
|
+
* expiry order while the clock moves forward. If the clock steps back, a
|
|
19424
|
+
* few expired records wait for a later save; reads still filter them.
|
|
19425
|
+
*/
|
|
19426
|
+
pruneExpiredExecutions(now) {
|
|
19427
|
+
if (this.ttls.executionHistory === null) return;
|
|
19428
|
+
let removed = 0;
|
|
19429
|
+
for (const [id, stored] of this.executionHistory) {
|
|
19430
|
+
if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
|
|
19431
|
+
this.expireExecutionRecord(id);
|
|
19432
|
+
removed++;
|
|
19433
|
+
}
|
|
19434
|
+
}
|
|
19435
|
+
/** Evict the oldest records in listing order until the count bound holds. */
|
|
19436
|
+
evictOverflowExecutions() {
|
|
19437
|
+
while (this.executionHistory.size > this.maxExecutions) {
|
|
19438
|
+
this.expireExecutionRecord(this.executionOrder[0].executionId);
|
|
19439
|
+
}
|
|
19440
|
+
}
|
|
19441
|
+
/** Remove up to `PRUNE_BATCH` expired eval rows, oldest first write first. */
|
|
19442
|
+
pruneExpiredEvals(now) {
|
|
19443
|
+
if (this.ttls.evalHistory === null) return;
|
|
19444
|
+
let removed = 0;
|
|
19445
|
+
for (const [id, stored] of this.evalHistory) {
|
|
19446
|
+
if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
|
|
19447
|
+
this.evalHistory.delete(id);
|
|
19448
|
+
removed++;
|
|
19449
|
+
}
|
|
19096
19450
|
}
|
|
19097
19451
|
// ── Sessions (Studio introspection) ─────────────────────────────────
|
|
19098
19452
|
async listSessions() {
|
|
@@ -19170,6 +19524,7 @@ var MemoryStore = class {
|
|
|
19170
19524
|
};
|
|
19171
19525
|
|
|
19172
19526
|
// src/state/sqlite.ts
|
|
19527
|
+
var LIVE = "(expires_at IS NULL OR expires_at > ?)";
|
|
19173
19528
|
function safeJsonParse(data) {
|
|
19174
19529
|
try {
|
|
19175
19530
|
return JSON.parse(data);
|
|
@@ -19206,7 +19561,17 @@ function rowToExecutionInfo(row) {
|
|
|
19206
19561
|
}
|
|
19207
19562
|
var SQLiteStore = class {
|
|
19208
19563
|
db;
|
|
19209
|
-
|
|
19564
|
+
ttls;
|
|
19565
|
+
now;
|
|
19566
|
+
constructor(filePath, options) {
|
|
19567
|
+
this.ttls = resolveHistoryTtls("SQLiteStore", options?.ttls);
|
|
19568
|
+
if (options?.now !== void 0 && typeof options.now !== "function") {
|
|
19569
|
+
throw new AxlError(
|
|
19570
|
+
"INVALID_CONFIG",
|
|
19571
|
+
"SQLiteStore: now must be a function returning epoch milliseconds"
|
|
19572
|
+
);
|
|
19573
|
+
}
|
|
19574
|
+
this.now = options?.now ?? (() => Date.now());
|
|
19210
19575
|
let BetterSqlite3;
|
|
19211
19576
|
try {
|
|
19212
19577
|
BetterSqlite3 = require("better-sqlite3");
|
|
@@ -19219,6 +19584,7 @@ var SQLiteStore = class {
|
|
|
19219
19584
|
this.db.pragma("journal_mode = WAL");
|
|
19220
19585
|
this.migrate();
|
|
19221
19586
|
this.initTables();
|
|
19587
|
+
if (options?.ttls?.executionHistory === void 0) warnExecutionRetentionUnset("SQLiteStore");
|
|
19222
19588
|
}
|
|
19223
19589
|
/**
|
|
19224
19590
|
* Idempotent schema migration. Tracks version via `PRAGMA user_version`
|
|
@@ -19246,6 +19612,11 @@ var SQLiteStore = class {
|
|
|
19246
19612
|
* Existing rows remain `NULL`, the documented v1 sentinel.
|
|
19247
19613
|
* - v5 (observation completeness): add `execution_history.observation`.
|
|
19248
19614
|
* Existing rows remain `NULL`, meaning they predate the contract.
|
|
19615
|
+
* - v6 (history retention): add nullable `expires_at` (epoch ms) to
|
|
19616
|
+
* `execution_history` and `eval_history`. Existing rows remain `NULL`
|
|
19617
|
+
* (never expire), so an upgrade deletes nothing. The single-column
|
|
19618
|
+
* `started_at` index is replaced by `(started_at, execution_id)`, which
|
|
19619
|
+
* serves the tie-safe listing order.
|
|
19249
19620
|
*
|
|
19250
19621
|
* Applied BEFORE `initTables()` so the subsequent `CREATE TABLE
|
|
19251
19622
|
* IF NOT EXISTS` runs against the post-migration column name. Fresh
|
|
@@ -19263,7 +19634,7 @@ var SQLiteStore = class {
|
|
|
19263
19634
|
* transactional re-read costs nothing.
|
|
19264
19635
|
*/
|
|
19265
19636
|
migrate() {
|
|
19266
|
-
const TARGET_VERSION =
|
|
19637
|
+
const TARGET_VERSION = 6;
|
|
19267
19638
|
const current = this.db.pragma("user_version", { simple: true });
|
|
19268
19639
|
if (current >= TARGET_VERSION) return;
|
|
19269
19640
|
this.db.exec("BEGIN IMMEDIATE");
|
|
@@ -19303,6 +19674,15 @@ var SQLiteStore = class {
|
|
|
19303
19674
|
this.db.exec("ALTER TABLE execution_history ADD COLUMN observation TEXT");
|
|
19304
19675
|
}
|
|
19305
19676
|
}
|
|
19677
|
+
if (committed < 6) {
|
|
19678
|
+
for (const table of ["execution_history", "eval_history"]) {
|
|
19679
|
+
const cols = this.db.pragma(`table_info(${table})`);
|
|
19680
|
+
if (cols.length > 0 && !cols.some((c) => c.name === "expires_at")) {
|
|
19681
|
+
this.db.exec(`ALTER TABLE ${table} ADD COLUMN expires_at INTEGER`);
|
|
19682
|
+
}
|
|
19683
|
+
}
|
|
19684
|
+
this.db.exec("DROP INDEX IF EXISTS idx_exec_history_started");
|
|
19685
|
+
}
|
|
19306
19686
|
this.db.pragma(`user_version = ${TARGET_VERSION}`);
|
|
19307
19687
|
this.db.exec("COMMIT");
|
|
19308
19688
|
} catch (err) {
|
|
@@ -19368,18 +19748,28 @@ var SQLiteStore = class {
|
|
|
19368
19748
|
events TEXT NOT NULL,
|
|
19369
19749
|
metadata TEXT,
|
|
19370
19750
|
event_schema_version INTEGER,
|
|
19371
|
-
observation TEXT
|
|
19751
|
+
observation TEXT,
|
|
19752
|
+
expires_at INTEGER
|
|
19372
19753
|
);
|
|
19373
19754
|
|
|
19374
19755
|
CREATE TABLE IF NOT EXISTS eval_history (
|
|
19375
19756
|
id TEXT PRIMARY KEY,
|
|
19376
19757
|
eval_name TEXT NOT NULL,
|
|
19377
19758
|
timestamp INTEGER NOT NULL,
|
|
19378
|
-
data TEXT NOT NULL
|
|
19759
|
+
data TEXT NOT NULL,
|
|
19760
|
+
expires_at INTEGER
|
|
19379
19761
|
);
|
|
19380
19762
|
|
|
19381
|
-
CREATE INDEX IF NOT EXISTS
|
|
19763
|
+
CREATE INDEX IF NOT EXISTS idx_exec_history_started_id
|
|
19764
|
+
ON execution_history (started_at DESC, execution_id DESC);
|
|
19765
|
+
CREATE INDEX IF NOT EXISTS idx_exec_history_expires ON execution_history (expires_at);
|
|
19382
19766
|
CREATE INDEX IF NOT EXISTS idx_eval_history_timestamp ON eval_history (timestamp DESC);
|
|
19767
|
+
CREATE INDEX IF NOT EXISTS idx_eval_history_expires ON eval_history (expires_at);
|
|
19768
|
+
|
|
19769
|
+
-- Superseded by idx_exec_history_started_id in v6. Dropped here as well as in
|
|
19770
|
+
-- migrate(): a pre-v6 process reopening a v6 file re-creates it, and
|
|
19771
|
+
-- migrate() never runs again once user_version is 6.
|
|
19772
|
+
DROP INDEX IF EXISTS idx_exec_history_started;
|
|
19383
19773
|
`);
|
|
19384
19774
|
}
|
|
19385
19775
|
// ── Checkpoints ────────────────────────────────────────────────────────
|
|
@@ -19492,30 +19882,71 @@ var SQLiteStore = class {
|
|
|
19492
19882
|
...execution,
|
|
19493
19883
|
eventSchemaVersion: getExecutionEventSchemaVersion(execution)
|
|
19494
19884
|
});
|
|
19885
|
+
const now = this.now();
|
|
19886
|
+
this.db.transaction(() => {
|
|
19887
|
+
this.db.prepare(
|
|
19888
|
+
"INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, events, metadata, event_schema_version, observation, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
19889
|
+
).run(
|
|
19890
|
+
normalized.executionId,
|
|
19891
|
+
normalized.workflow,
|
|
19892
|
+
normalized.status,
|
|
19893
|
+
normalized.totalCost,
|
|
19894
|
+
normalized.startedAt,
|
|
19895
|
+
normalized.completedAt ?? null,
|
|
19896
|
+
normalized.duration,
|
|
19897
|
+
normalized.error ?? null,
|
|
19898
|
+
JSON.stringify(normalized.events),
|
|
19899
|
+
normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
|
|
19900
|
+
normalized.eventSchemaVersion,
|
|
19901
|
+
normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null,
|
|
19902
|
+
expiryFrom(now, this.ttls.executionHistory)
|
|
19903
|
+
);
|
|
19904
|
+
this.pruneExpiredExecutions(now);
|
|
19905
|
+
})();
|
|
19906
|
+
}
|
|
19907
|
+
/**
|
|
19908
|
+
* Remove up to `PRUNE_BATCH` expired execution-history rows, together with
|
|
19909
|
+
* each run's leftover checkpoints and saved execution state, inside the
|
|
19910
|
+
* caller's transaction. Pending decisions are not touched. The runtime
|
|
19911
|
+
* never writes execution state, and a run is only saved to history once it
|
|
19912
|
+
* has finished, so nothing still running depends on what is removed here.
|
|
19913
|
+
*/
|
|
19914
|
+
pruneExpiredExecutions(now) {
|
|
19915
|
+
const ids = this.db.prepare(
|
|
19916
|
+
"SELECT execution_id FROM execution_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?"
|
|
19917
|
+
).all(now, PRUNE_BATCH).map((row) => row.execution_id);
|
|
19918
|
+
if (ids.length === 0) return;
|
|
19919
|
+
const idList = JSON.stringify(ids);
|
|
19920
|
+
for (const table of ["execution_history", "checkpoints", "execution_state"]) {
|
|
19921
|
+
this.db.prepare(`DELETE FROM ${table} WHERE execution_id IN (SELECT value FROM json_each(?))`).run(idList);
|
|
19922
|
+
}
|
|
19923
|
+
}
|
|
19924
|
+
/** Remove up to `PRUNE_BATCH` expired eval-history rows (caller's transaction). */
|
|
19925
|
+
pruneExpiredEvals(now) {
|
|
19495
19926
|
this.db.prepare(
|
|
19496
|
-
"
|
|
19497
|
-
).run(
|
|
19498
|
-
normalized.executionId,
|
|
19499
|
-
normalized.workflow,
|
|
19500
|
-
normalized.status,
|
|
19501
|
-
normalized.totalCost,
|
|
19502
|
-
normalized.startedAt,
|
|
19503
|
-
normalized.completedAt ?? null,
|
|
19504
|
-
normalized.duration,
|
|
19505
|
-
normalized.error ?? null,
|
|
19506
|
-
JSON.stringify(normalized.events),
|
|
19507
|
-
normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
|
|
19508
|
-
normalized.eventSchemaVersion,
|
|
19509
|
-
normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null
|
|
19510
|
-
);
|
|
19927
|
+
"DELETE FROM eval_history WHERE rowid IN (SELECT rowid FROM eval_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?)"
|
|
19928
|
+
).run(now, PRUNE_BATCH);
|
|
19511
19929
|
}
|
|
19512
19930
|
async getExecution(executionId) {
|
|
19513
|
-
const row = this.db.prepare(
|
|
19931
|
+
const row = this.db.prepare(`SELECT * FROM execution_history WHERE execution_id = ? AND ${LIVE}`).get(executionId, this.now());
|
|
19514
19932
|
return row ? rowToExecutionInfo(row) : null;
|
|
19515
19933
|
}
|
|
19516
|
-
|
|
19517
|
-
|
|
19518
|
-
|
|
19934
|
+
/** Supports `listExecutions(limit, { cursor })`. */
|
|
19935
|
+
historyPaging = true;
|
|
19936
|
+
async listExecutions(limit, options) {
|
|
19937
|
+
const cursor = options?.cursor;
|
|
19938
|
+
const params = [this.now()];
|
|
19939
|
+
let sql = `SELECT * FROM execution_history WHERE ${LIVE}`;
|
|
19940
|
+
if (cursor !== void 0) {
|
|
19941
|
+
sql += " AND (started_at < ? OR (started_at = ? AND execution_id < ?))";
|
|
19942
|
+
params.push(cursor.startedAt, cursor.startedAt, cursor.executionId);
|
|
19943
|
+
}
|
|
19944
|
+
sql += " ORDER BY started_at DESC, execution_id DESC";
|
|
19945
|
+
if (limit) {
|
|
19946
|
+
sql += " LIMIT ?";
|
|
19947
|
+
params.push(limit);
|
|
19948
|
+
}
|
|
19949
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
19519
19950
|
return rows.map(rowToExecutionInfo);
|
|
19520
19951
|
}
|
|
19521
19952
|
async deleteExecution(executionId) {
|
|
@@ -19529,24 +19960,38 @@ var SQLiteStore = class {
|
|
|
19529
19960
|
return tx();
|
|
19530
19961
|
}
|
|
19531
19962
|
// ── Eval History ────────────────────────────────────────────────────────
|
|
19963
|
+
/**
|
|
19964
|
+
* Write an eval history row. Retention counts from the row's first write:
|
|
19965
|
+
* re-saving a live row keeps its `expires_at` (including none, for a row
|
|
19966
|
+
* written without retention). A row that has already expired is replaced
|
|
19967
|
+
* as a new one.
|
|
19968
|
+
*/
|
|
19532
19969
|
async saveEvalResult(entry) {
|
|
19533
|
-
this.
|
|
19534
|
-
|
|
19535
|
-
|
|
19970
|
+
const now = this.now();
|
|
19971
|
+
this.db.transaction(() => {
|
|
19972
|
+
const existing = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(entry.id, now);
|
|
19973
|
+
const expiresAt = existing ? existing.expires_at : expiryFrom(now, this.ttls.evalHistory);
|
|
19974
|
+
this.db.prepare(
|
|
19975
|
+
"INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data, expires_at) VALUES (?, ?, ?, ?, ?)"
|
|
19976
|
+
).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data), expiresAt);
|
|
19977
|
+
this.pruneExpiredEvals(now);
|
|
19978
|
+
})();
|
|
19536
19979
|
}
|
|
19537
19980
|
/**
|
|
19538
19981
|
* Update-only write: see `StateStore.updateEvalResult`. `UPDATE ... WHERE id`
|
|
19539
|
-
* touches nothing when the row is gone, where `INSERT OR REPLACE`
|
|
19540
|
-
* resurrect it
|
|
19541
|
-
* permanent.
|
|
19982
|
+
* touches nothing when the row is gone or expired, where `INSERT OR REPLACE`
|
|
19983
|
+
* would resurrect it. `expires_at` is left as it was.
|
|
19542
19984
|
*/
|
|
19543
19985
|
async updateEvalResult(entry) {
|
|
19544
|
-
const result = this.db.prepare(
|
|
19986
|
+
const result = this.db.prepare(
|
|
19987
|
+
`UPDATE eval_history SET eval_name = ?, timestamp = ?, data = ? WHERE id = ? AND ${LIVE}`
|
|
19988
|
+
).run(entry.eval, entry.timestamp, JSON.stringify(entry.data), entry.id, this.now());
|
|
19545
19989
|
return result.changes > 0;
|
|
19546
19990
|
}
|
|
19547
19991
|
async listEvalResults(limit) {
|
|
19548
|
-
const
|
|
19549
|
-
const
|
|
19992
|
+
const base = `SELECT * FROM eval_history WHERE ${LIVE} ORDER BY timestamp DESC`;
|
|
19993
|
+
const now = this.now();
|
|
19994
|
+
const rows = limit ? this.db.prepare(`${base} LIMIT ?`).all(now, limit) : this.db.prepare(base).all(now);
|
|
19550
19995
|
return rows.map((r) => ({
|
|
19551
19996
|
id: r.id,
|
|
19552
19997
|
eval: r.eval_name,
|
|
@@ -19558,10 +20003,75 @@ var SQLiteStore = class {
|
|
|
19558
20003
|
const result = this.db.prepare("DELETE FROM eval_history WHERE id = ?").run(id);
|
|
19559
20004
|
return result.changes > 0;
|
|
19560
20005
|
}
|
|
19561
|
-
/**
|
|
20006
|
+
/** Existence plus the expiry stored at the row's first write, if any. */
|
|
19562
20007
|
async getEvalRetention(id) {
|
|
19563
|
-
const row = this.db.prepare(
|
|
19564
|
-
return { exists:
|
|
20008
|
+
const row = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(id, this.now());
|
|
20009
|
+
if (!row) return { exists: false };
|
|
20010
|
+
return row.expires_at === null ? { exists: true } : { exists: true, expiresAt: row.expires_at };
|
|
20011
|
+
}
|
|
20012
|
+
// ── History maintenance ─────────────────────────────────────────────────
|
|
20013
|
+
/**
|
|
20014
|
+
* See `StateStore.applyHistoryRetention`. One batch runs in one immediate
|
|
20015
|
+
* transaction, so no other writer can change a row between this call
|
|
20016
|
+
* reading it and acting on it. Only rows whose `expires_at` is `NULL` are
|
|
20017
|
+
* touched. An expired execution takes its checkpoints and saved execution
|
|
20018
|
+
* state with it, as a normal expiry does. There is no separate index, so
|
|
20019
|
+
* `indexed` and `deadIndexEntries` are always `0`.
|
|
20020
|
+
*/
|
|
20021
|
+
async applyHistoryRetention(options) {
|
|
20022
|
+
const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("SQLiteStore", options);
|
|
20023
|
+
const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
|
|
20024
|
+
const table = kind === "executions" ? { name: "execution_history", id: "execution_id", time: "started_at" } : { name: "eval_history", id: "id", time: "timestamp" };
|
|
20025
|
+
const lastSaved = kind === "executions" ? "completed_at" : "NULL";
|
|
20026
|
+
const now = this.now();
|
|
20027
|
+
return this.db.transaction(() => {
|
|
20028
|
+
const params = [];
|
|
20029
|
+
let sql = `SELECT ${table.id} AS id, ${table.time} AS time, ${lastSaved} AS last_saved, expires_at FROM ${table.name}`;
|
|
20030
|
+
if (cursor !== void 0) {
|
|
20031
|
+
sql += ` WHERE ${table.time} > ? OR (${table.time} = ? AND ${table.id} > ?)`;
|
|
20032
|
+
params.push(cursor.time, cursor.time, cursor.id);
|
|
20033
|
+
}
|
|
20034
|
+
sql += ` ORDER BY ${table.time}, ${table.id} LIMIT ?`;
|
|
20035
|
+
params.push(batchSize + 1);
|
|
20036
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
20037
|
+
const batch = rows.slice(0, batchSize);
|
|
20038
|
+
const toExpire = [];
|
|
20039
|
+
let armed = 0;
|
|
20040
|
+
if (ttl !== null) {
|
|
20041
|
+
const arm = this.db.prepare(
|
|
20042
|
+
`UPDATE ${table.name} SET expires_at = ? WHERE ${table.id} = ? AND expires_at IS NULL`
|
|
20043
|
+
);
|
|
20044
|
+
for (const row of batch) {
|
|
20045
|
+
if (row.expires_at !== null) continue;
|
|
20046
|
+
const lastSavedAt = kind === "executions" ? executionLastSavedAt({ startedAt: row.time, completedAt: row.last_saved }) : row.time;
|
|
20047
|
+
const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
|
|
20048
|
+
if (decision.expired) toExpire.push(row.id);
|
|
20049
|
+
else armed += arm.run(decision.expiresAt, row.id).changes;
|
|
20050
|
+
}
|
|
20051
|
+
}
|
|
20052
|
+
let expired = 0;
|
|
20053
|
+
if (toExpire.length > 0) {
|
|
20054
|
+
const idList = JSON.stringify(toExpire);
|
|
20055
|
+
expired = this.db.prepare(
|
|
20056
|
+
`DELETE FROM ${table.name} WHERE expires_at IS NULL AND ${table.id} IN (SELECT value FROM json_each(?))`
|
|
20057
|
+
).run(idList).changes;
|
|
20058
|
+
if (kind === "executions") {
|
|
20059
|
+
for (const side of ["checkpoints", "execution_state"]) {
|
|
20060
|
+
this.db.prepare(
|
|
20061
|
+
`DELETE FROM ${side} WHERE execution_id IN (SELECT value FROM json_each(?))`
|
|
20062
|
+
).run(idList);
|
|
20063
|
+
}
|
|
20064
|
+
}
|
|
20065
|
+
}
|
|
20066
|
+
const last = batch[batch.length - 1];
|
|
20067
|
+
return {
|
|
20068
|
+
expired,
|
|
20069
|
+
armed,
|
|
20070
|
+
indexed: 0,
|
|
20071
|
+
deadIndexEntries: 0,
|
|
20072
|
+
cursor: rows.length > batchSize ? { time: last.time, id: last.id } : null
|
|
20073
|
+
};
|
|
20074
|
+
}).immediate();
|
|
19565
20075
|
}
|
|
19566
20076
|
// ── Sessions (Studio introspection) ────────────────────────────────────
|
|
19567
20077
|
async listSessions() {
|
|
@@ -19731,11 +20241,18 @@ var Session = class _Session {
|
|
|
19731
20241
|
try {
|
|
19732
20242
|
const completion = axlStream.promise.then(
|
|
19733
20243
|
(result) => this.commitHistory(history, result).catch((err) => {
|
|
19734
|
-
|
|
19735
|
-
|
|
19736
|
-
|
|
19737
|
-
|
|
19738
|
-
|
|
20244
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
20245
|
+
try {
|
|
20246
|
+
this.runtime.emit("error", {
|
|
20247
|
+
type: "session_history_save_failed",
|
|
20248
|
+
sessionId: this.sessionId,
|
|
20249
|
+
error
|
|
20250
|
+
});
|
|
20251
|
+
} catch {
|
|
20252
|
+
console.error(
|
|
20253
|
+
`[axl] Failed to save session history for sessionId=${this.sessionId}: ${error}`
|
|
20254
|
+
);
|
|
20255
|
+
}
|
|
19739
20256
|
}),
|
|
19740
20257
|
() => void 0
|
|
19741
20258
|
);
|
|
@@ -20901,6 +21418,9 @@ function hashInput(input) {
|
|
|
20901
21418
|
}
|
|
20902
21419
|
var DEFAULT_MAX_EVENTS_PER_EXECUTION = 5e4;
|
|
20903
21420
|
var DEFAULT_STREAMING_BATCH_SIZE = 100;
|
|
21421
|
+
var DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE = 1e3;
|
|
21422
|
+
var LARGE_EXECUTION_LISTING = 1e3;
|
|
21423
|
+
var EXECUTION_SCAN_PAGE_SIZE = 100;
|
|
20904
21424
|
var DEFAULT_STREAMING_BATCH_INTERVAL = 1e3;
|
|
20905
21425
|
var DEFAULT_ARTIFACT_SWEEP_MS = 6e4;
|
|
20906
21426
|
var DEFAULT_ARTIFACT_LEASE_MS = 3e5;
|
|
@@ -21113,6 +21633,78 @@ function assertNoRemovedObservationCallbacks(options) {
|
|
|
21113
21633
|
`runtime.createContext() no longer accepts ${removed.join(", ")}. Observe ctx.events or use runtime.stream() instead. See docs/migration/stream-first-observation.md.`
|
|
21114
21634
|
);
|
|
21115
21635
|
}
|
|
21636
|
+
var RecentRunsCache = class {
|
|
21637
|
+
constructor(capacity) {
|
|
21638
|
+
this.capacity = capacity;
|
|
21639
|
+
}
|
|
21640
|
+
entries = /* @__PURE__ */ new Map();
|
|
21641
|
+
get size() {
|
|
21642
|
+
return this.entries.size;
|
|
21643
|
+
}
|
|
21644
|
+
get(executionId) {
|
|
21645
|
+
const entry = this.entries.get(executionId);
|
|
21646
|
+
if (entry) {
|
|
21647
|
+
this.entries.delete(executionId);
|
|
21648
|
+
this.entries.set(executionId, entry);
|
|
21649
|
+
}
|
|
21650
|
+
return entry;
|
|
21651
|
+
}
|
|
21652
|
+
set(executionId, execution) {
|
|
21653
|
+
this.entries.delete(executionId);
|
|
21654
|
+
this.entries.set(executionId, execution);
|
|
21655
|
+
while (this.entries.size > this.capacity) {
|
|
21656
|
+
const oldest = this.entries.keys().next().value;
|
|
21657
|
+
this.entries.delete(oldest);
|
|
21658
|
+
}
|
|
21659
|
+
}
|
|
21660
|
+
delete(executionId) {
|
|
21661
|
+
return this.entries.delete(executionId);
|
|
21662
|
+
}
|
|
21663
|
+
values() {
|
|
21664
|
+
return [...this.entries.values()];
|
|
21665
|
+
}
|
|
21666
|
+
};
|
|
21667
|
+
function resolveRecentExecutionsCacheSize(requested) {
|
|
21668
|
+
if (requested === void 0) return DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE;
|
|
21669
|
+
if (typeof requested !== "number" || !Number.isInteger(requested) || requested < 0) {
|
|
21670
|
+
throw new AxlError(
|
|
21671
|
+
"INVALID_CONFIG",
|
|
21672
|
+
`config.state.recentExecutionsCacheSize must be a non-negative integer; got ${String(requested)}`
|
|
21673
|
+
);
|
|
21674
|
+
}
|
|
21675
|
+
return requested;
|
|
21676
|
+
}
|
|
21677
|
+
function parseGetExecutionsOptions(options) {
|
|
21678
|
+
if (options === void 0) return {};
|
|
21679
|
+
if (options === null || typeof options !== "object") {
|
|
21680
|
+
throw new AxlError(
|
|
21681
|
+
"INVALID_ARGUMENT",
|
|
21682
|
+
`getExecutions() options must be an object; got ${options === null ? "null" : typeof options}`
|
|
21683
|
+
);
|
|
21684
|
+
}
|
|
21685
|
+
const { limit, cursor } = options;
|
|
21686
|
+
if (limit !== void 0 && limit !== Infinity && !(typeof limit === "number" && Number.isInteger(limit) && limit > 0)) {
|
|
21687
|
+
throw new AxlError(
|
|
21688
|
+
"INVALID_ARGUMENT",
|
|
21689
|
+
`getExecutions() limit must be a positive integer; got ${String(limit)}`
|
|
21690
|
+
);
|
|
21691
|
+
}
|
|
21692
|
+
let parsedCursor;
|
|
21693
|
+
if (cursor !== void 0) {
|
|
21694
|
+
const candidate = cursor;
|
|
21695
|
+
if (candidate === null || typeof candidate !== "object" || typeof candidate.startedAt !== "number" || !Number.isFinite(candidate.startedAt) || typeof candidate.executionId !== "string") {
|
|
21696
|
+
throw new AxlError(
|
|
21697
|
+
"INVALID_ARGUMENT",
|
|
21698
|
+
"getExecutions() cursor must be { startedAt: number; executionId: string }, built from the last item of the previous page"
|
|
21699
|
+
);
|
|
21700
|
+
}
|
|
21701
|
+
parsedCursor = { startedAt: candidate.startedAt, executionId: candidate.executionId };
|
|
21702
|
+
}
|
|
21703
|
+
return {
|
|
21704
|
+
...limit === void 0 || limit === Infinity ? {} : { limit },
|
|
21705
|
+
...parsedCursor ? { cursor: parsedCursor } : {}
|
|
21706
|
+
};
|
|
21707
|
+
}
|
|
21116
21708
|
var costScopeStorage = new import_node_async_hooks4.AsyncLocalStorage();
|
|
21117
21709
|
function forwardAbortSignal(external, internal) {
|
|
21118
21710
|
if (!external) return () => {
|
|
@@ -21159,8 +21751,6 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21159
21751
|
mcpManager;
|
|
21160
21752
|
memoryManager;
|
|
21161
21753
|
spanManager = new NoopSpanManager();
|
|
21162
|
-
historicalExecutions = /* @__PURE__ */ new Map();
|
|
21163
|
-
historicalExecutionsLoadPromise = null;
|
|
21164
21754
|
/** Per-session in-process serialization. Each entry is a chain of pending
|
|
21165
21755
|
* work for that session id — `Session.send` / `Session.stream` await the
|
|
21166
21756
|
* prior entry before reading history, so concurrent calls produce a
|
|
@@ -21196,12 +21786,34 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21196
21786
|
* a delete from resurrecting the (intentionally-removed) row. Entries are
|
|
21197
21787
|
* removed once the would-be persist has been short-circuited. */
|
|
21198
21788
|
pendingDeletedExecutions = /* @__PURE__ */ new Set();
|
|
21199
|
-
/** In-flight `persistExecution` chains (save → finalize)
|
|
21200
|
-
* awaits these before closing the StateStore so a workflow whose abort
|
|
21789
|
+
/** In-flight `persistExecution` chains (save → finalize) and crash-
|
|
21790
|
+
* recovery saves. `shutdown()` awaits these before closing the StateStore so a workflow whose abort
|
|
21201
21791
|
* triggers a persist-during-shutdown isn't racing a closed connection.
|
|
21202
21792
|
* The Set is mutated by the chain's own `.finally` so entries clear as
|
|
21203
21793
|
* soon as the work settles. */
|
|
21204
21794
|
persistInflight = /* @__PURE__ */ new Set();
|
|
21795
|
+
/** Finished runs whose history save has not settled, keyed by execution
|
|
21796
|
+
* id. Reads serve the snapshot until the save lands, so a run is visible
|
|
21797
|
+
* the moment it finishes (read-your-writes). `deleteExecution()` awaits
|
|
21798
|
+
* the entry before deleting from the store, so a save that lands late
|
|
21799
|
+
* cannot re-create a deleted run. Each entry removes itself when its save
|
|
21800
|
+
* settles (success or failure), but only if it is still the current entry
|
|
21801
|
+
* for that id, so the map holds only in-flight saves. */
|
|
21802
|
+
pendingSaves = /* @__PURE__ */ new Map();
|
|
21803
|
+
/** Recently finished runs, kept only when the store cannot save, list, or
|
|
21804
|
+
* fetch execution history itself (and the cache size is not `0`).
|
|
21805
|
+
* `undefined` for every built-in store: execution history is read through
|
|
21806
|
+
* the store on every call and never cached here. */
|
|
21807
|
+
recentRuns;
|
|
21808
|
+
/** Execution ids `recoverIncompleteStreams()` is working on, each with a
|
|
21809
|
+
* promise that settles when recovery is done with that id.
|
|
21810
|
+
* `deleteExecution()` awaits it, so a recovered record cannot land after
|
|
21811
|
+
* the delete. */
|
|
21812
|
+
recoveringExecutions = /* @__PURE__ */ new Map();
|
|
21813
|
+
/** Execution ids with a `deleteExecution()` in progress, with a count of
|
|
21814
|
+
* concurrent deletes. Recovery skips them: their store sweep removes the
|
|
21815
|
+
* streaming buffer it would otherwise turn back into a record. */
|
|
21816
|
+
deletingExecutions = /* @__PURE__ */ new Map();
|
|
21205
21817
|
/** Resolved diagnostic artifact backend, when `diagnostics.artifacts` is set. */
|
|
21206
21818
|
artifactStore;
|
|
21207
21819
|
artifactLeaseMs = DEFAULT_ARTIFACT_LEASE_MS;
|
|
@@ -21235,6 +21847,13 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21235
21847
|
this.providerRegistry = new ProviderRegistry();
|
|
21236
21848
|
this.transcriptionProviderRegistry = new TranscriptionProviderRegistry();
|
|
21237
21849
|
this.stateStore = this.createStateStore();
|
|
21850
|
+
const recentRunsSize = resolveRecentExecutionsCacheSize(
|
|
21851
|
+
this.config.state?.recentExecutionsCacheSize
|
|
21852
|
+
);
|
|
21853
|
+
const storeKeepsHistory = Boolean(
|
|
21854
|
+
this.stateStore.saveExecution && this.stateStore.listExecutions
|
|
21855
|
+
);
|
|
21856
|
+
this.recentRuns = !storeKeepsHistory && recentRunsSize > 0 ? new RecentRunsCache(recentRunsSize) : void 0;
|
|
21238
21857
|
const requested = this.config.state?.maxEventsPerExecution;
|
|
21239
21858
|
if (requested === void 0) {
|
|
21240
21859
|
this.maxEventsPerExecution = DEFAULT_MAX_EVENTS_PER_EXECUTION;
|
|
@@ -21640,11 +22259,15 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21640
22259
|
const storeOption = this.config.state?.store ?? "memory";
|
|
21641
22260
|
if (typeof storeOption !== "string") return storeOption;
|
|
21642
22261
|
switch (storeOption) {
|
|
21643
|
-
case "sqlite":
|
|
21644
|
-
|
|
22262
|
+
case "sqlite": {
|
|
22263
|
+
const create = () => new SQLiteStore(this.config.state?.sqlite?.path ?? "./data/axl.db", {
|
|
22264
|
+
ttls: this.config.state?.sqlite?.ttls
|
|
22265
|
+
});
|
|
22266
|
+
return this.config.diagnostics?.silent ? withDiagnosticsSilenced(create) : create();
|
|
22267
|
+
}
|
|
21645
22268
|
case "memory":
|
|
21646
22269
|
default:
|
|
21647
|
-
return new MemoryStore();
|
|
22270
|
+
return new MemoryStore(this.config.state?.memory);
|
|
21648
22271
|
}
|
|
21649
22272
|
}
|
|
21650
22273
|
/**
|
|
@@ -21696,7 +22319,11 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21696
22319
|
observation
|
|
21697
22320
|
});
|
|
21698
22321
|
if (this.stateStore.deleteCheckpoints) {
|
|
21699
|
-
await this.stateStore.deleteCheckpoints(execInfo.executionId)
|
|
22322
|
+
await this.stateStore.deleteCheckpoints(execInfo.executionId).catch((err) => {
|
|
22323
|
+
console.error(
|
|
22324
|
+
`[axl] Failed to delete checkpoints for executionId=${execInfo.executionId}: ` + (err instanceof Error ? err.message : String(err))
|
|
22325
|
+
);
|
|
22326
|
+
});
|
|
21700
22327
|
}
|
|
21701
22328
|
span?.setAttribute("axl.workflow.cost", execInfo.totalCost);
|
|
21702
22329
|
span?.setAttribute("axl.workflow.duration", execInfo.duration);
|
|
@@ -21732,13 +22359,15 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21732
22359
|
}
|
|
21733
22360
|
/**
|
|
21734
22361
|
* Persist a completed/failed execution to the state store (fire-and-forget)
|
|
21735
|
-
* and
|
|
22362
|
+
* and remove it from the active executions map. Until the save settles the
|
|
22363
|
+
* run is served from `pendingSaves`; afterwards from the store (or, for a
|
|
22364
|
+
* store that cannot keep history, from `recentRuns`).
|
|
21736
22365
|
*
|
|
21737
22366
|
* Resurrection guard: if `runtime.deleteExecution(id)` was called while
|
|
21738
22367
|
* this execution was still running, the id will be in
|
|
21739
|
-
* `pendingDeletedExecutions`. In that case we skip the save AND
|
|
21740
|
-
*
|
|
21741
|
-
* the buffer doesn't outlive the delete.
|
|
22368
|
+
* `pendingDeletedExecutions`. In that case we skip the save AND every
|
|
22369
|
+
* in-process record of the run, and finalize the streaming buffer (if any)
|
|
22370
|
+
* so the buffer doesn't outlive the delete.
|
|
21742
22371
|
*/
|
|
21743
22372
|
persistExecution(execInfo) {
|
|
21744
22373
|
const id = execInfo.executionId;
|
|
@@ -21766,17 +22395,25 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21766
22395
|
const chain = this.stateStore.saveExecution(snapshot).then(() => {
|
|
21767
22396
|
return this.streamingFlusher?.finalize(id).catch(() => {
|
|
21768
22397
|
});
|
|
21769
|
-
}).catch(() => {
|
|
22398
|
+
}).catch((err) => {
|
|
22399
|
+
console.error(
|
|
22400
|
+
`[axl] Failed to save execution history for executionId=${id}: ` + (err instanceof Error ? err.message : String(err))
|
|
22401
|
+
);
|
|
21770
22402
|
}).then(() => void 0);
|
|
21771
22403
|
this.persistInflight.add(chain);
|
|
21772
|
-
|
|
22404
|
+
const pending = { snapshot, settled: chain };
|
|
22405
|
+
this.pendingSaves.set(id, pending);
|
|
22406
|
+
chain.finally(() => {
|
|
22407
|
+
this.persistInflight.delete(chain);
|
|
22408
|
+
if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
|
|
22409
|
+
});
|
|
21773
22410
|
} else if (this.streamingFlusher) {
|
|
21774
22411
|
const chain = this.streamingFlusher.finalize(id).catch(() => {
|
|
21775
22412
|
});
|
|
21776
22413
|
this.persistInflight.add(chain);
|
|
21777
22414
|
chain.finally(() => this.persistInflight.delete(chain));
|
|
21778
22415
|
}
|
|
21779
|
-
this.
|
|
22416
|
+
this.recentRuns?.set(id, snapshot);
|
|
21780
22417
|
this.executions.delete(id);
|
|
21781
22418
|
}
|
|
21782
22419
|
/** Register a workflow with the runtime. */
|
|
@@ -21902,7 +22539,6 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21902
22539
|
}
|
|
21903
22540
|
return result;
|
|
21904
22541
|
}
|
|
21905
|
-
/** Get all execution info (running + completed + historical). */
|
|
21906
22542
|
/** Normalize an execution loaded from a `StateStore` so the
|
|
21907
22543
|
* historical event union holds for downstream consumers
|
|
21908
22544
|
* (Studio aggregators, REST routes, redaction). Custom or legacy stores
|
|
@@ -21922,27 +22558,78 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
21922
22558
|
}
|
|
21923
22559
|
return normalizeStoredExecution(exec);
|
|
21924
22560
|
}
|
|
21925
|
-
|
|
21926
|
-
|
|
21927
|
-
|
|
21928
|
-
|
|
21929
|
-
|
|
21930
|
-
|
|
21931
|
-
|
|
21932
|
-
|
|
21933
|
-
|
|
21934
|
-
|
|
21935
|
-
|
|
21936
|
-
|
|
21937
|
-
|
|
21938
|
-
|
|
21939
|
-
|
|
21940
|
-
|
|
21941
|
-
|
|
21942
|
-
|
|
21943
|
-
|
|
22561
|
+
/**
|
|
22562
|
+
* List executions, newest first: running runs, runs whose save is still in
|
|
22563
|
+
* flight, and the history the state store holds. Ordered by `startedAt`,
|
|
22564
|
+
* then `executionId`, both descending.
|
|
22565
|
+
*
|
|
22566
|
+
* With no options, returns every retained run. Pass `limit` for one page,
|
|
22567
|
+
* and `cursor` (built from the last item of the previous page) for the
|
|
22568
|
+
* page after it. Each call reads through the store, so runs finished or
|
|
22569
|
+
* deleted by other runtimes sharing the store are reflected immediately.
|
|
22570
|
+
* A call without `limit` that returns more than 1 000 runs warns once per
|
|
22571
|
+
* process (silenced by `diagnostics.silent`); pass `limit: Infinity` to ask
|
|
22572
|
+
* for everything without the warning.
|
|
22573
|
+
*
|
|
22574
|
+
* @throws {AxlError} `INVALID_ARGUMENT` for a `limit` that is not a positive
|
|
22575
|
+
* integer (`Infinity` means no limit) or a malformed `cursor`.
|
|
22576
|
+
*/
|
|
22577
|
+
async getExecutions(options) {
|
|
22578
|
+
const { limit, cursor } = parseGetExecutionsOptions(options);
|
|
22579
|
+
const isAfterCursor = (exec) => cursor === void 0 || compareExecutionsNewestFirst(exec, cursor) > 0;
|
|
22580
|
+
const live = new Map(this.executions);
|
|
22581
|
+
for (const [id, { snapshot }] of this.pendingSaves) {
|
|
22582
|
+
if (!live.has(id)) live.set(id, snapshot);
|
|
22583
|
+
}
|
|
22584
|
+
const recent = this.recentRunsServeListing() ? this.recentRuns.values() : [];
|
|
22585
|
+
const stored = await this.listStoredExecutions({
|
|
22586
|
+
limit,
|
|
22587
|
+
cursor,
|
|
22588
|
+
overlap: live.size + recent.length
|
|
22589
|
+
});
|
|
22590
|
+
const merged = /* @__PURE__ */ new Map();
|
|
22591
|
+
for (const [id, exec] of live) {
|
|
22592
|
+
if (isAfterCursor(exec)) merged.set(id, exec);
|
|
22593
|
+
}
|
|
22594
|
+
for (const exec of stored) {
|
|
22595
|
+
const id = exec.executionId;
|
|
22596
|
+
if (!live.has(id) && !merged.has(id)) merged.set(id, exec);
|
|
22597
|
+
}
|
|
22598
|
+
for (const exec of recent) {
|
|
22599
|
+
const id = exec.executionId;
|
|
22600
|
+
if (!live.has(id) && !merged.has(id) && isAfterCursor(exec)) merged.set(id, exec);
|
|
22601
|
+
}
|
|
22602
|
+
const sorted = [...merged.values()].sort(compareExecutionsNewestFirst);
|
|
22603
|
+
if (limit !== void 0) return sorted.slice(0, limit);
|
|
22604
|
+
if (options?.limit === void 0 && sorted.length > LARGE_EXECUTION_LISTING) {
|
|
22605
|
+
warnDiagnosticOnce(
|
|
22606
|
+
"large_execution_listing",
|
|
22607
|
+
`runtime.getExecutions() without a limit returned ${sorted.length} runs and reads the whole history on every call; pass { limit, cursor } to read it a page at a time.`,
|
|
22608
|
+
this.config.diagnostics?.silent,
|
|
22609
|
+
"docs/migration/execution-history-retention.md#page-through-large-histories"
|
|
22610
|
+
);
|
|
21944
22611
|
}
|
|
21945
|
-
return
|
|
22612
|
+
return sorted;
|
|
22613
|
+
}
|
|
22614
|
+
/**
|
|
22615
|
+
* Read one page of execution history from the store: the records strictly
|
|
22616
|
+
* after `cursor`, newest first, at least `limit + overlap` of them when the
|
|
22617
|
+
* store holds that many (all of them when `limit` is unset).
|
|
22618
|
+
*
|
|
22619
|
+
* Only a store that declares `historyPaging` is handed the cursor. Any
|
|
22620
|
+
* other store is listed in full and paged here: filtering the output of a
|
|
22621
|
+
* store that ignored the cursor would return its first page again, and an
|
|
22622
|
+
* empty page after that.
|
|
22623
|
+
*/
|
|
22624
|
+
async listStoredExecutions(page) {
|
|
22625
|
+
const store = this.stateStore;
|
|
22626
|
+
if (!store.listExecutions) return [];
|
|
22627
|
+
const wanted = page.limit === void 0 ? void 0 : page.limit + page.overlap;
|
|
22628
|
+
const rows = store.historyPaging === true ? await store.listExecutions(wanted, { cursor: page.cursor }) : await store.listExecutions();
|
|
22629
|
+
const cursor = page.cursor;
|
|
22630
|
+
const after = cursor === void 0 ? rows : rows.filter((row) => compareExecutionsNewestFirst(row, cursor) > 0);
|
|
22631
|
+
const window = wanted === void 0 ? after : [...after].sort(compareExecutionsNewestFirst).slice(0, wanted);
|
|
22632
|
+
return window.map((row) => this.normalizeStoredExecution(row));
|
|
21946
22633
|
}
|
|
21947
22634
|
/**
|
|
21948
22635
|
* Create a WorkflowContext for ad-hoc use (evals, tool testing, prototyping).
|
|
@@ -22373,20 +23060,25 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
22373
23060
|
* Recover executions whose process died mid-run when `state.persist:
|
|
22374
23061
|
* 'streaming'` was configured. Synthesizes a partial `ExecutionInfo`
|
|
22375
23062
|
* from the streaming buffer for each — `status: 'failed'`, error
|
|
22376
|
-
* `'process terminated'` —
|
|
22377
|
-
* they show up in `getExecutions()
|
|
22378
|
-
*
|
|
23063
|
+
* `'process terminated'` — persists them via the store's
|
|
23064
|
+
* `saveExecution` (so they show up in `getExecutions()`), and finalizes
|
|
23065
|
+
* the streaming buffer.
|
|
22379
23066
|
*
|
|
22380
23067
|
* Idempotent — if the streaming buffer is already gone (or
|
|
22381
23068
|
* `listStreamingExecutions` is empty), this is a no-op. Returns the
|
|
22382
23069
|
* list of recovered `ExecutionInfo`s for caller introspection.
|
|
22383
23070
|
*
|
|
22384
|
-
*
|
|
22385
|
-
*
|
|
22386
|
-
*
|
|
22387
|
-
*
|
|
22388
|
-
*
|
|
22389
|
-
*
|
|
23071
|
+
* A run whose final record already exists is never recovered: for each
|
|
23072
|
+
* buffered id this asks the store's `getExecution(id)` and, when a record
|
|
23073
|
+
* is there, only drops the leftover buffer. It does not list history, so
|
|
23074
|
+
* its cost does not grow with the size of the store. A store without
|
|
23075
|
+
* `getExecution` cannot be checked, and every buffered run is recovered.
|
|
23076
|
+
*
|
|
23077
|
+
* Wire this into process startup, before the process accepts new work:
|
|
23078
|
+
* a run live in another process looks the same as a crashed one.
|
|
23079
|
+
* `deleteExecution(id)` called while recovery is working on that id waits
|
|
23080
|
+
* for recovery to finish with it, so the recovered record cannot outlive
|
|
23081
|
+
* the delete.
|
|
22390
23082
|
*/
|
|
22391
23083
|
async recoverIncompleteStreams() {
|
|
22392
23084
|
if (!this.stateStore.listStreamingExecutions || !this.stateStore.getStreamingEvents) {
|
|
@@ -22394,73 +23086,119 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
22394
23086
|
}
|
|
22395
23087
|
const ids = await this.stateStore.listStreamingExecutions();
|
|
22396
23088
|
if (ids.length === 0) return [];
|
|
22397
|
-
await this.getExecutions();
|
|
22398
23089
|
const recovered = [];
|
|
22399
23090
|
for (const id of ids) {
|
|
22400
23091
|
if (this.executions.has(id) || this.streamableExecutionIds.has(id)) {
|
|
22401
23092
|
continue;
|
|
22402
23093
|
}
|
|
22403
|
-
|
|
22404
|
-
|
|
22405
|
-
|
|
22406
|
-
|
|
22407
|
-
|
|
22408
|
-
|
|
22409
|
-
|
|
22410
|
-
|
|
22411
|
-
|
|
22412
|
-
|
|
22413
|
-
|
|
22414
|
-
|
|
22415
|
-
|
|
22416
|
-
|
|
22417
|
-
}
|
|
23094
|
+
if (this.deletingExecutions.has(id) || this.recoveringExecutions.has(id)) continue;
|
|
23095
|
+
let finishRecovery;
|
|
23096
|
+
this.recoveringExecutions.set(
|
|
23097
|
+
id,
|
|
23098
|
+
new Promise((resolve2) => {
|
|
23099
|
+
finishRecovery = resolve2;
|
|
23100
|
+
})
|
|
23101
|
+
);
|
|
23102
|
+
try {
|
|
23103
|
+
const result = await this.recoverStreamingExecution(id);
|
|
23104
|
+
if (result) recovered.push(result);
|
|
23105
|
+
} finally {
|
|
23106
|
+
this.recoveringExecutions.delete(id);
|
|
23107
|
+
finishRecovery();
|
|
22418
23108
|
}
|
|
22419
|
-
|
|
22420
|
-
|
|
22421
|
-
|
|
22422
|
-
|
|
22423
|
-
|
|
22424
|
-
|
|
22425
|
-
|
|
22426
|
-
|
|
22427
|
-
|
|
22428
|
-
|
|
22429
|
-
|
|
22430
|
-
|
|
22431
|
-
|
|
22432
|
-
|
|
22433
|
-
|
|
22434
|
-
|
|
22435
|
-
|
|
22436
|
-
|
|
22437
|
-
|
|
22438
|
-
|
|
22439
|
-
|
|
22440
|
-
|
|
22441
|
-
|
|
22442
|
-
|
|
22443
|
-
|
|
22444
|
-
|
|
22445
|
-
|
|
22446
|
-
|
|
22447
|
-
|
|
23109
|
+
}
|
|
23110
|
+
return recovered;
|
|
23111
|
+
}
|
|
23112
|
+
/** Recover one buffered execution. Returns the synthesized record, or
|
|
23113
|
+
* `undefined` when there was nothing to recover. */
|
|
23114
|
+
async recoverStreamingExecution(id) {
|
|
23115
|
+
if (!this.stateStore.getStreamingEvents) return void 0;
|
|
23116
|
+
const existing = await this.stateStore.getExecution?.(id);
|
|
23117
|
+
if (existing) {
|
|
23118
|
+
await this.stateStore.finalizeStreamingEvents?.(id);
|
|
23119
|
+
return void 0;
|
|
23120
|
+
}
|
|
23121
|
+
const events = await this.stateStore.getStreamingEvents(id);
|
|
23122
|
+
if (events.length === 0) {
|
|
23123
|
+
await this.stateStore.finalizeStreamingEvents?.(id);
|
|
23124
|
+
return void 0;
|
|
23125
|
+
}
|
|
23126
|
+
const eventSchemaVersion = getEventSchemaVersion(events[0]);
|
|
23127
|
+
for (const event of events) {
|
|
23128
|
+
if (getEventSchemaVersion(event) !== eventSchemaVersion) {
|
|
23129
|
+
throw new Error(`Streaming buffer ${id} contains mixed event schema versions`);
|
|
23130
|
+
}
|
|
23131
|
+
}
|
|
23132
|
+
const boundedEvents = boundRecoveredEvents(events, this.maxEventsPerExecution);
|
|
23133
|
+
const workflowStartEvent = boundedEvents.find((e) => e.type === "workflow_start");
|
|
23134
|
+
const lastEvent = boundedEvents[boundedEvents.length - 1];
|
|
23135
|
+
const startedAt = workflowStartEvent?.timestamp ?? boundedEvents[0]?.timestamp ?? Date.now();
|
|
23136
|
+
const completedAt = lastEvent.timestamp ?? startedAt;
|
|
23137
|
+
const workflowName = workflowStartEvent?.workflow ?? RECOVERED_UNKNOWN_WORKFLOW;
|
|
23138
|
+
const totalCost = boundedEvents.reduce((sum, e) => sum + eventCostContribution(e), 0);
|
|
23139
|
+
const unpriced = boundedEvents.some(isUnpricedLeaf);
|
|
23140
|
+
const executionBase = {
|
|
23141
|
+
executionId: id,
|
|
23142
|
+
workflow: workflowName,
|
|
23143
|
+
status: "failed",
|
|
23144
|
+
events: boundedEvents,
|
|
23145
|
+
totalCost,
|
|
23146
|
+
unpriced,
|
|
23147
|
+
startedAt,
|
|
23148
|
+
completedAt,
|
|
23149
|
+
duration: completedAt - startedAt,
|
|
23150
|
+
error: "process terminated (recovered from streaming buffer)"
|
|
23151
|
+
};
|
|
23152
|
+
const synthesized = eventSchemaVersion === 2 ? {
|
|
23153
|
+
...executionBase,
|
|
23154
|
+
eventSchemaVersion: 2,
|
|
23155
|
+
events: boundedEvents,
|
|
23156
|
+
observation: { complete: false, reason: "process_interrupted" }
|
|
23157
|
+
} : {
|
|
23158
|
+
...executionBase,
|
|
23159
|
+
eventSchemaVersion: 1,
|
|
23160
|
+
events: boundedEvents
|
|
23161
|
+
};
|
|
23162
|
+
const saveExecution = this.stateStore.saveExecution;
|
|
23163
|
+
if (saveExecution) {
|
|
23164
|
+
const save = (async () => saveExecution.call(this.stateStore, synthesized))();
|
|
23165
|
+
const pending = {
|
|
23166
|
+
snapshot: synthesized,
|
|
23167
|
+
settled: save.then(
|
|
23168
|
+
() => void 0,
|
|
23169
|
+
() => void 0
|
|
23170
|
+
)
|
|
22448
23171
|
};
|
|
22449
|
-
|
|
22450
|
-
|
|
22451
|
-
|
|
22452
|
-
|
|
22453
|
-
|
|
22454
|
-
|
|
22455
|
-
)
|
|
22456
|
-
|
|
22457
|
-
|
|
23172
|
+
this.pendingSaves.set(id, pending);
|
|
23173
|
+
this.persistInflight.add(pending.settled);
|
|
23174
|
+
try {
|
|
23175
|
+
await save;
|
|
23176
|
+
} catch (err) {
|
|
23177
|
+
console.error(
|
|
23178
|
+
`[axl] recoverIncompleteStreams: failed to save synthesized execution ${id}: ${err instanceof Error ? err.message : String(err)}. Streaming buffer left in place for next recovery attempt.`
|
|
23179
|
+
);
|
|
23180
|
+
return void 0;
|
|
23181
|
+
} finally {
|
|
23182
|
+
this.persistInflight.delete(pending.settled);
|
|
23183
|
+
if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
|
|
22458
23184
|
}
|
|
22459
|
-
this.historicalExecutions.set(id, synthesized);
|
|
22460
|
-
await this.stateStore.finalizeStreamingEvents?.(id);
|
|
22461
|
-
recovered.push(synthesized);
|
|
22462
23185
|
}
|
|
22463
|
-
|
|
23186
|
+
this.recentRuns?.set(id, synthesized);
|
|
23187
|
+
const finalize = Promise.resolve(this.stateStore.finalizeStreamingEvents?.(id)).then(
|
|
23188
|
+
() => void 0,
|
|
23189
|
+
(err) => {
|
|
23190
|
+
console.error(
|
|
23191
|
+
`[axl] recoverIncompleteStreams: saved execution ${id} but failed to drop its streaming buffer: ${err instanceof Error ? err.message : String(err)}. The next recovery removes it.`
|
|
23192
|
+
);
|
|
23193
|
+
}
|
|
23194
|
+
);
|
|
23195
|
+
this.persistInflight.add(finalize);
|
|
23196
|
+
try {
|
|
23197
|
+
await finalize;
|
|
23198
|
+
} finally {
|
|
23199
|
+
this.persistInflight.delete(finalize);
|
|
23200
|
+
}
|
|
23201
|
+
return synthesized;
|
|
22464
23202
|
}
|
|
22465
23203
|
/** Abort a running execution by its ID. */
|
|
22466
23204
|
abort(executionId) {
|
|
@@ -22470,21 +23208,57 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
22470
23208
|
this.abortControllers.delete(executionId);
|
|
22471
23209
|
}
|
|
22472
23210
|
}
|
|
22473
|
-
/**
|
|
23211
|
+
/**
|
|
23212
|
+
* Get execution details by ID. Looks in order at running runs, runs whose
|
|
23213
|
+
* save is still in flight, the state store (its `getExecution`, or a scan
|
|
23214
|
+
* of its listing when it has `listExecutions` but no `getExecution`), and
|
|
23215
|
+
* (only for a store that cannot list or save history) the recent-runs
|
|
23216
|
+
* cache. Nothing read from the store is cached, so a run deleted or
|
|
23217
|
+
* expired elsewhere is gone on the next call, and every run
|
|
23218
|
+
* `getExecutions()` lists can be fetched here.
|
|
23219
|
+
*/
|
|
22474
23220
|
async getExecution(executionId) {
|
|
22475
|
-
const
|
|
22476
|
-
if (
|
|
22477
|
-
const
|
|
22478
|
-
if (
|
|
22479
|
-
|
|
22480
|
-
|
|
22481
|
-
|
|
22482
|
-
|
|
22483
|
-
|
|
22484
|
-
|
|
22485
|
-
|
|
23221
|
+
const active = this.executions.get(executionId);
|
|
23222
|
+
if (active) return active;
|
|
23223
|
+
const pending = this.pendingSaves.get(executionId);
|
|
23224
|
+
if (pending) return pending.snapshot;
|
|
23225
|
+
const stored = this.stateStore.getExecution ? await this.stateStore.getExecution(executionId) : await this.findInStoreListing(executionId);
|
|
23226
|
+
if (stored) return this.normalizeStoredExecution(stored);
|
|
23227
|
+
return this.recentRunsServeReads() ? this.recentRuns.get(executionId) : void 0;
|
|
23228
|
+
}
|
|
23229
|
+
/**
|
|
23230
|
+
* By-id read for a store that lists but cannot fetch by id. A paging store
|
|
23231
|
+
* is scanned page by page with the cursor, stopping at the match; any
|
|
23232
|
+
* other store is listed once in full. Returns `null` when the store cannot
|
|
23233
|
+
* list or does not hold the run.
|
|
23234
|
+
*/
|
|
23235
|
+
async findInStoreListing(executionId) {
|
|
23236
|
+
const store = this.stateStore;
|
|
23237
|
+
if (!store.listExecutions) return null;
|
|
23238
|
+
if (store.historyPaging !== true) {
|
|
23239
|
+
return (await store.listExecutions()).find((e) => e.executionId === executionId) ?? null;
|
|
22486
23240
|
}
|
|
22487
|
-
|
|
23241
|
+
let cursor;
|
|
23242
|
+
for (; ; ) {
|
|
23243
|
+
const page = await store.listExecutions(EXECUTION_SCAN_PAGE_SIZE, { cursor });
|
|
23244
|
+
const found = page.find((e) => e.executionId === executionId);
|
|
23245
|
+
if (found) return found;
|
|
23246
|
+
if (page.length === 0) return null;
|
|
23247
|
+
const last = [...page].sort(compareExecutionsNewestFirst)[page.length - 1];
|
|
23248
|
+
if (cursor && compareExecutionsNewestFirst(last, cursor) <= 0) return null;
|
|
23249
|
+
cursor = { startedAt: last.startedAt, executionId: last.executionId };
|
|
23250
|
+
}
|
|
23251
|
+
}
|
|
23252
|
+
/** The recent-runs cache stands in for listing when the store cannot list
|
|
23253
|
+
* runs, or never receives them because it cannot save. */
|
|
23254
|
+
recentRunsServeListing() {
|
|
23255
|
+
return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.listExecutions);
|
|
23256
|
+
}
|
|
23257
|
+
/** The recent-runs cache stands in for by-id reads when the store cannot
|
|
23258
|
+
* fetch or list runs (it exists only then), or never receives them because
|
|
23259
|
+
* it cannot save. */
|
|
23260
|
+
recentRunsServeReads() {
|
|
23261
|
+
return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.getExecution);
|
|
22488
23262
|
}
|
|
22489
23263
|
/** Save an eval result to history. */
|
|
22490
23264
|
async saveEvalResult(entry) {
|
|
@@ -22517,11 +23291,16 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
22517
23291
|
}
|
|
22518
23292
|
/**
|
|
22519
23293
|
* Delete an execution from history by id. Removes from the active map (if
|
|
22520
|
-
* still running), the
|
|
23294
|
+
* still running), the recent-runs cache, the configured StateStore, AND
|
|
22521
23295
|
* any execution-scoped side state — pending awaitHuman decisions,
|
|
22522
23296
|
* resolver maps, streaming buffer registration, in-flight abort controller.
|
|
22523
|
-
* Returns true if
|
|
22524
|
-
*
|
|
23297
|
+
* Returns true if the run was running, had a save in flight, or was
|
|
23298
|
+
* removed from the recent-runs cache or the store.
|
|
23299
|
+
*
|
|
23300
|
+
* If the run has already finished but its history save has not landed
|
|
23301
|
+
* yet, this method waits for that save (whether it succeeds or fails)
|
|
23302
|
+
* before deleting, so the save cannot re-create the deleted record. It
|
|
23303
|
+
* likewise waits for `recoverIncompleteStreams()` to finish with the id.
|
|
22525
23304
|
*
|
|
22526
23305
|
* In-flight handling: if the execution is still running, this method
|
|
22527
23306
|
* **aborts** it (via the registered abort controller) AND marks the id for
|
|
@@ -22546,7 +23325,18 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
22546
23325
|
* (`defaultTtl` / `ttls.executionHistory`).
|
|
22547
23326
|
*/
|
|
22548
23327
|
async deleteExecution(id) {
|
|
23328
|
+
this.deletingExecutions.set(id, (this.deletingExecutions.get(id) ?? 0) + 1);
|
|
23329
|
+
try {
|
|
23330
|
+
return await this.deleteExecutionWhileMarked(id);
|
|
23331
|
+
} finally {
|
|
23332
|
+
const remaining = (this.deletingExecutions.get(id) ?? 1) - 1;
|
|
23333
|
+
if (remaining > 0) this.deletingExecutions.set(id, remaining);
|
|
23334
|
+
else this.deletingExecutions.delete(id);
|
|
23335
|
+
}
|
|
23336
|
+
}
|
|
23337
|
+
async deleteExecutionWhileMarked(id) {
|
|
22549
23338
|
const wasActive = this.executions.has(id);
|
|
23339
|
+
const hadPendingSave = this.pendingSaves.has(id);
|
|
22550
23340
|
const pendingDecisionResolver = this.pendingDecisionResolvers.get(id);
|
|
22551
23341
|
const hadPendingDecision = pendingDecisionResolver !== void 0;
|
|
22552
23342
|
if (wasActive) {
|
|
@@ -22564,22 +23354,25 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
22564
23354
|
this.pendingDecisionResolvers.delete(id);
|
|
22565
23355
|
}
|
|
22566
23356
|
}
|
|
23357
|
+
await this.pendingSaves.get(id)?.settled;
|
|
23358
|
+
await this.recoveringExecutions.get(id);
|
|
22567
23359
|
const existing = await this.getExecution(id);
|
|
22568
23360
|
const workflow2 = existing?.workflow;
|
|
22569
23361
|
const removedFromActive = this.executions.delete(id);
|
|
22570
|
-
const
|
|
23362
|
+
const removedFromRecent = this.recentRuns?.delete(id) ?? false;
|
|
22571
23363
|
let removedFromStore = false;
|
|
22572
23364
|
if (this.stateStore.deleteExecution) {
|
|
22573
23365
|
removedFromStore = await this.stateStore.deleteExecution(id);
|
|
22574
23366
|
}
|
|
23367
|
+
const removed = removedFromActive || hadPendingSave || removedFromRecent || removedFromStore;
|
|
22575
23368
|
this.emit("execution_deleted", {
|
|
22576
23369
|
executionId: id,
|
|
22577
23370
|
workflow: workflow2,
|
|
22578
23371
|
wasActive,
|
|
22579
23372
|
hadPendingDecision,
|
|
22580
|
-
removed
|
|
23373
|
+
removed
|
|
22581
23374
|
});
|
|
22582
|
-
return
|
|
23375
|
+
return removed;
|
|
22583
23376
|
}
|
|
22584
23377
|
/**
|
|
22585
23378
|
* Delete an eval history entry by id. Removes from in-memory cache and
|
|
@@ -23997,12 +24790,153 @@ async function* stringStreamFromEvents(source, opts) {
|
|
|
23997
24790
|
}
|
|
23998
24791
|
|
|
23999
24792
|
// src/state/redis.ts
|
|
24793
|
+
var PRUNE_HISTORY_INDEX_SCRIPT = `
|
|
24794
|
+
local now = tonumber(ARGV[1])
|
|
24795
|
+
local removed = 0
|
|
24796
|
+
for i = 2, #ARGV do
|
|
24797
|
+
local id = ARGV[i]
|
|
24798
|
+
local pttl = redis.call('PTTL', KEYS[i + 2])
|
|
24799
|
+
if pttl == -2 then
|
|
24800
|
+
redis.call('SREM', KEYS[1], id)
|
|
24801
|
+
redis.call('ZREM', KEYS[2], id)
|
|
24802
|
+
redis.call('ZREM', KEYS[3], id)
|
|
24803
|
+
removed = removed + 1
|
|
24804
|
+
elseif pttl == -1 then
|
|
24805
|
+
redis.call('ZREM', KEYS[3], id)
|
|
24806
|
+
else
|
|
24807
|
+
redis.call('ZADD', KEYS[3], string.format('%d', now + pttl), id)
|
|
24808
|
+
end
|
|
24809
|
+
end
|
|
24810
|
+
return removed
|
|
24811
|
+
`;
|
|
24812
|
+
function compareMembers(a, b) {
|
|
24813
|
+
return Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8"));
|
|
24814
|
+
}
|
|
24815
|
+
function compareNewestFirst(a, b) {
|
|
24816
|
+
if (a.score !== b.score) return b.score - a.score;
|
|
24817
|
+
return compareMembers(b.id, a.id);
|
|
24818
|
+
}
|
|
24819
|
+
function warnIfRedisExecutionRetentionUnset(opts) {
|
|
24820
|
+
if (opts.ttls?.executionHistory === void 0 && typeof opts.defaultTtl !== "number") {
|
|
24821
|
+
warnExecutionRetentionUnset("RedisStore");
|
|
24822
|
+
}
|
|
24823
|
+
}
|
|
24824
|
+
var CLOSE_TIMEOUT_MS = 2e3;
|
|
24825
|
+
function reconnectDelayMs(retries) {
|
|
24826
|
+
return Math.min(2 ** retries * 50, 2e3) + Math.floor(Math.random() * 200);
|
|
24827
|
+
}
|
|
24828
|
+
function abandonConnection(client) {
|
|
24829
|
+
try {
|
|
24830
|
+
client.destroy?.();
|
|
24831
|
+
} catch {
|
|
24832
|
+
}
|
|
24833
|
+
client.unref?.();
|
|
24834
|
+
}
|
|
24835
|
+
function errorMessage(error) {
|
|
24836
|
+
return error instanceof Error ? error.message : String(error);
|
|
24837
|
+
}
|
|
24838
|
+
function connectionEventReporter(client, onConnectionEvent) {
|
|
24839
|
+
if (onConnectionEvent) {
|
|
24840
|
+
const reportFailure = (error) => console.error(`[axl] RedisStore: onConnectionEvent failed: ${errorMessage(error)}`);
|
|
24841
|
+
return (event) => {
|
|
24842
|
+
try {
|
|
24843
|
+
Promise.resolve(onConnectionEvent(event)).catch(reportFailure);
|
|
24844
|
+
} catch (error) {
|
|
24845
|
+
reportFailure(error);
|
|
24846
|
+
}
|
|
24847
|
+
};
|
|
24848
|
+
}
|
|
24849
|
+
let outageLogged = false;
|
|
24850
|
+
return (event) => {
|
|
24851
|
+
if (event.type === "reconnected") {
|
|
24852
|
+
outageLogged = false;
|
|
24853
|
+
console.warn("[axl] RedisStore: reconnected to Redis.");
|
|
24854
|
+
} else if (!outageLogged) {
|
|
24855
|
+
outageLogged = client.isReady === false;
|
|
24856
|
+
console.warn(
|
|
24857
|
+
`[axl] RedisStore: Redis connection lost (${event.error.message}). Store calls fail until it reconnects, which happens automatically.`
|
|
24858
|
+
);
|
|
24859
|
+
}
|
|
24860
|
+
};
|
|
24861
|
+
}
|
|
24862
|
+
async function connectClient(createClient, opts) {
|
|
24863
|
+
let established = false;
|
|
24864
|
+
const client = createClient({
|
|
24865
|
+
...opts.url ? { url: opts.url } : {},
|
|
24866
|
+
// A command accepted just before a drop rejects instead of waiting for
|
|
24867
|
+
// the reconnect.
|
|
24868
|
+
disableOfflineQueue: true,
|
|
24869
|
+
socket: { reconnectStrategy: (retries) => established ? reconnectDelayMs(retries) : false }
|
|
24870
|
+
});
|
|
24871
|
+
const reportConnectionEvent = connectionEventReporter(client, opts.onConnectionEvent);
|
|
24872
|
+
client.on("error", (error) => {
|
|
24873
|
+
if (!established || client.isOpen === false) return;
|
|
24874
|
+
reportConnectionEvent({
|
|
24875
|
+
type: "error",
|
|
24876
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
24877
|
+
});
|
|
24878
|
+
});
|
|
24879
|
+
client.on("connect", () => {
|
|
24880
|
+
if (client.isOpen === false) client.unref?.();
|
|
24881
|
+
});
|
|
24882
|
+
client.on("ready", () => {
|
|
24883
|
+
if (!established) {
|
|
24884
|
+
established = true;
|
|
24885
|
+
} else if (client.isOpen !== false) {
|
|
24886
|
+
reportConnectionEvent({ type: "reconnected" });
|
|
24887
|
+
}
|
|
24888
|
+
});
|
|
24889
|
+
await client.connect();
|
|
24890
|
+
return client;
|
|
24891
|
+
}
|
|
24000
24892
|
var RedisStore = class _RedisStore {
|
|
24001
|
-
constructor(
|
|
24002
|
-
this.
|
|
24893
|
+
constructor(connection, keyPrefix, ttls) {
|
|
24894
|
+
this.connection = connection;
|
|
24003
24895
|
this.keyPrefix = keyPrefix;
|
|
24004
24896
|
this.ttls = ttls;
|
|
24005
24897
|
}
|
|
24898
|
+
/**
|
|
24899
|
+
* Whether the store can send commands right now, for a health check or for
|
|
24900
|
+
* telling an outage from other failures when a call rejects. `false` while
|
|
24901
|
+
* reconnecting and after `close()`.
|
|
24902
|
+
*/
|
|
24903
|
+
get isConnected() {
|
|
24904
|
+
return this.connection.isOpen !== false && this.connection.isReady !== false;
|
|
24905
|
+
}
|
|
24906
|
+
/**
|
|
24907
|
+
* The client for sending a command, failing immediately while disconnected.
|
|
24908
|
+
*
|
|
24909
|
+
* Every command goes through here because node-redis's `disableOfflineQueue`
|
|
24910
|
+
* rejects single commands but not `multi().exec()`, which would wait for a
|
|
24911
|
+
* reconnect attempt to fail or succeed. Nothing may `await` between
|
|
24912
|
+
* `client.multi()` and `.exec()`, or a transaction could get past this check.
|
|
24913
|
+
*
|
|
24914
|
+
* A closed store isn't "unavailable": it falls through to node-redis's own
|
|
24915
|
+
* "The client is closed" rejection.
|
|
24916
|
+
*/
|
|
24917
|
+
get client() {
|
|
24918
|
+
if (this.connection.isOpen !== false && this.connection.isReady === false) {
|
|
24919
|
+
throw new AxlError(
|
|
24920
|
+
"REDIS_UNAVAILABLE",
|
|
24921
|
+
"RedisStore: not connected to Redis, so the command was not sent. The store reconnects automatically; retry once Redis is back."
|
|
24922
|
+
);
|
|
24923
|
+
}
|
|
24924
|
+
return this.connection;
|
|
24925
|
+
}
|
|
24926
|
+
/**
|
|
24927
|
+
* History kinds with a prune running in this process. A save skips its
|
|
24928
|
+
* prune while another one of the same kind is in flight: the running one
|
|
24929
|
+
* already covers the oldest expired entries. Created lazily so the
|
|
24930
|
+
* instance needs no constructor state.
|
|
24931
|
+
*/
|
|
24932
|
+
prunesInFlight;
|
|
24933
|
+
/**
|
|
24934
|
+
* Supports `listExecutions(limit, { cursor })`. A getter rather than a
|
|
24935
|
+
* field so it lives on the prototype.
|
|
24936
|
+
*/
|
|
24937
|
+
get historyPaging() {
|
|
24938
|
+
return true;
|
|
24939
|
+
}
|
|
24006
24940
|
/**
|
|
24007
24941
|
* Resolve the TTL for a given storage category. Returns `undefined` when
|
|
24008
24942
|
* no TTL applies (category is `null` in resolved config, the value is
|
|
@@ -24058,8 +24992,7 @@ var RedisStore = class _RedisStore {
|
|
|
24058
24992
|
if (err instanceof Error && err.message.includes("createClient")) throw err;
|
|
24059
24993
|
throw new Error("redis is required for RedisStore. Install it with: npm install redis");
|
|
24060
24994
|
}
|
|
24061
|
-
const client =
|
|
24062
|
-
await client.connect();
|
|
24995
|
+
const client = await connectClient(createClient, opts);
|
|
24063
24996
|
const resolveCategory = (override) => {
|
|
24064
24997
|
if (override === null) return null;
|
|
24065
24998
|
if (typeof override === "number") return override;
|
|
@@ -24081,9 +25014,15 @@ var RedisStore = class _RedisStore {
|
|
|
24081
25014
|
streamingEvents: resolveStreamingEvents(opts.ttls?.streamingEvents)
|
|
24082
25015
|
};
|
|
24083
25016
|
const store = new _RedisStore(client, keyPrefix, effectiveTtls);
|
|
25017
|
+
warnIfRedisExecutionRetentionUnset(opts);
|
|
24084
25018
|
if (!opts.skipMigration) {
|
|
24085
|
-
|
|
24086
|
-
|
|
25019
|
+
try {
|
|
25020
|
+
await store.backfillExecutionIndex();
|
|
25021
|
+
await store.backfillEvalIndex();
|
|
25022
|
+
} catch (error) {
|
|
25023
|
+
abandonConnection(client);
|
|
25024
|
+
throw error;
|
|
25025
|
+
}
|
|
24087
25026
|
}
|
|
24088
25027
|
return store;
|
|
24089
25028
|
}
|
|
@@ -24192,13 +25131,9 @@ var RedisStore = class _RedisStore {
|
|
|
24192
25131
|
execHistoryKey(executionId) {
|
|
24193
25132
|
return `${this.keyPrefix}exec-history:${executionId}`;
|
|
24194
25133
|
}
|
|
24195
|
-
//
|
|
24196
|
-
//
|
|
24197
|
-
//
|
|
24198
|
-
// The slow-path fallback (`listHistoryByZset`) reads from the SET when
|
|
24199
|
-
// `zsetCard < setCard`, so SET-based eviction would race with the fallback
|
|
24200
|
-
// and surface dead IDs whose data has been TTL'd away. ZSET eviction +
|
|
24201
|
-
// dual-write `SREM` is the clean path; doc the assumption then.
|
|
25134
|
+
// The legacy id set and the sorted set below always change together (save,
|
|
25135
|
+
// delete, prune), so the slow-path guard `ZCARD >= SCARD` in
|
|
25136
|
+
// `listHistoryByZset` and backfill stays meaningful.
|
|
24202
25137
|
execHistorySetKey() {
|
|
24203
25138
|
return `${this.keyPrefix}exec-history-ids`;
|
|
24204
25139
|
}
|
|
@@ -24221,6 +25156,36 @@ var RedisStore = class _RedisStore {
|
|
|
24221
25156
|
evalHistoryZsetKey() {
|
|
24222
25157
|
return `${this.keyPrefix}eval-history-z`;
|
|
24223
25158
|
}
|
|
25159
|
+
/** Keys and blob parsing for one kind of history. See {@link HistoryIndex}. */
|
|
25160
|
+
historyIndex(kind) {
|
|
25161
|
+
if (kind === "execution") {
|
|
25162
|
+
return {
|
|
25163
|
+
kind,
|
|
25164
|
+
label: "execution-history",
|
|
25165
|
+
ids: this.execHistorySetKey(),
|
|
25166
|
+
time: this.execHistoryZsetKey(),
|
|
25167
|
+
expiry: `${this.keyPrefix}exec-history-exp`,
|
|
25168
|
+
dataKey: (id) => this.execHistoryKey(id),
|
|
25169
|
+
lastSavedAtOf: (raw) => {
|
|
25170
|
+
const record = JSON.parse(raw);
|
|
25171
|
+
return typeof record.startedAt === "number" ? executionLastSavedAt(record) : null;
|
|
25172
|
+
}
|
|
25173
|
+
};
|
|
25174
|
+
}
|
|
25175
|
+
return {
|
|
25176
|
+
kind,
|
|
25177
|
+
label: "eval-history",
|
|
25178
|
+
ids: this.evalHistorySetKey(),
|
|
25179
|
+
time: this.evalHistoryZsetKey(),
|
|
25180
|
+
expiry: `${this.keyPrefix}eval-history-exp`,
|
|
25181
|
+
dataKey: (id) => this.evalHistoryKey(id),
|
|
25182
|
+
// Eval retention counts from the first write, which is the row's timestamp.
|
|
25183
|
+
lastSavedAtOf: (raw) => {
|
|
25184
|
+
const timestamp = JSON.parse(raw).timestamp;
|
|
25185
|
+
return typeof timestamp === "number" ? timestamp : null;
|
|
25186
|
+
}
|
|
25187
|
+
};
|
|
25188
|
+
}
|
|
24224
25189
|
memoryKey(scope) {
|
|
24225
25190
|
return `${this.keyPrefix}memory:${scope}`;
|
|
24226
25191
|
}
|
|
@@ -24326,7 +25291,7 @@ var RedisStore = class _RedisStore {
|
|
|
24326
25291
|
}
|
|
24327
25292
|
}
|
|
24328
25293
|
if (dead.length > 0) {
|
|
24329
|
-
this.
|
|
25294
|
+
this.connection.sRem(this.pendingExecSetKey(), dead).catch((err) => {
|
|
24330
25295
|
console.error(
|
|
24331
25296
|
`[axl] RedisStore: failed to prune ${dead.length} stale pending-exec ids: ` + (err instanceof Error ? err.message : String(err))
|
|
24332
25297
|
);
|
|
@@ -24372,7 +25337,7 @@ var RedisStore = class _RedisStore {
|
|
|
24372
25337
|
}
|
|
24373
25338
|
const ttl = this.ttlFor("memory");
|
|
24374
25339
|
if (ttl !== void 0) {
|
|
24375
|
-
this.
|
|
25340
|
+
this.connection.expire(this.memoryKey(scope), ttl).catch((err) => {
|
|
24376
25341
|
console.error(
|
|
24377
25342
|
`[axl] RedisStore: failed to set TTL on migrated memory key ${this.memoryKey(scope)}: ` + (err instanceof Error ? err.message : String(err))
|
|
24378
25343
|
);
|
|
@@ -24392,31 +25357,35 @@ var RedisStore = class _RedisStore {
|
|
|
24392
25357
|
async saveExecution(execution) {
|
|
24393
25358
|
const ttl = this.ttlFor("executionHistory");
|
|
24394
25359
|
const setOptions = ttl !== void 0 ? { EX: ttl } : void 0;
|
|
25360
|
+
const index = this.historyIndex("execution");
|
|
25361
|
+
const id = execution.executionId;
|
|
24395
25362
|
const storedExecution = normalizeStoredExecution({
|
|
24396
25363
|
...execution,
|
|
24397
25364
|
eventSchemaVersion: getExecutionEventSchemaVersion(execution)
|
|
24398
25365
|
});
|
|
24399
|
-
|
|
24400
|
-
|
|
24401
|
-
|
|
24402
|
-
|
|
25366
|
+
const tx = this.client.multi().sAdd(index.ids, id).set(index.dataKey(id), JSON.stringify(storedExecution), setOptions).zAdd(index.time, { score: execution.startedAt, value: id });
|
|
25367
|
+
if (ttl !== void 0) tx.zAdd(index.expiry, { score: Date.now() + ttl * 1e3, value: id });
|
|
25368
|
+
else tx.zRem(index.expiry, id);
|
|
25369
|
+
await tx.exec();
|
|
25370
|
+
await this.pruneHistoryAfterSave(index);
|
|
24403
25371
|
}
|
|
24404
25372
|
async getExecution(executionId) {
|
|
24405
25373
|
const raw = await this.client.get(this.execHistoryKey(executionId));
|
|
24406
25374
|
return raw ? normalizeStoredExecution(JSON.parse(raw)) : null;
|
|
24407
25375
|
}
|
|
24408
|
-
async listExecutions(limit) {
|
|
25376
|
+
async listExecutions(limit, options) {
|
|
25377
|
+
const cursor = options?.cursor;
|
|
24409
25378
|
const executions = await this.listHistoryByZset(
|
|
24410
|
-
this.
|
|
24411
|
-
|
|
24412
|
-
|
|
24413
|
-
|
|
24414
|
-
limit
|
|
25379
|
+
this.historyIndex("execution"),
|
|
25380
|
+
(e) => ({ score: e.startedAt, id: e.executionId }),
|
|
25381
|
+
limit,
|
|
25382
|
+
cursor !== void 0 ? { score: cursor.startedAt, id: cursor.executionId } : void 0
|
|
24415
25383
|
);
|
|
24416
25384
|
return executions.map(normalizeStoredExecution);
|
|
24417
25385
|
}
|
|
24418
25386
|
async deleteExecution(executionId) {
|
|
24419
|
-
const
|
|
25387
|
+
const index = this.historyIndex("execution");
|
|
25388
|
+
const results = await this.client.multi().sRem(index.ids, executionId).del(index.dataKey(executionId)).zRem(index.time, executionId).zRem(index.expiry, executionId).del(this.checkpointKey(executionId)).del(this.executionStateKey(executionId)).sRem(this.pendingExecSetKey(), executionId).del(this.streamingEventsKey(executionId)).sRem(this.streamingIdsKey(), executionId).hDel(this.decisionsKey(), executionId).exec();
|
|
24420
25389
|
return results.some((result) => typeof result === "number" && result > 0);
|
|
24421
25390
|
}
|
|
24422
25391
|
// ── Eval History ────────────────────────────────────────────────────
|
|
@@ -24441,18 +25410,31 @@ var RedisStore = class _RedisStore {
|
|
|
24441
25410
|
* lifetime, so the race costs at most one window's length, never the window
|
|
24442
25411
|
* itself. It also carries no Redis version floor, where `KEEPTTL` needs 6.0.
|
|
24443
25412
|
*
|
|
24444
|
-
*
|
|
25413
|
+
* As with `saveExecution`, a timed row is also scored in the expiry sorted
|
|
25414
|
+
* set (by the expiry it carries, so a re-save keeps its score), an untimed
|
|
25415
|
+
* one is taken out of it, and a bounded prune follows the save.
|
|
24445
25416
|
*/
|
|
24446
25417
|
async saveEvalResult(entry) {
|
|
24447
25418
|
const ttl = this.ttlFor("evalHistory");
|
|
24448
|
-
const
|
|
25419
|
+
const index = this.historyIndex("eval");
|
|
25420
|
+
const key = index.dataKey(entry.id);
|
|
24449
25421
|
let setOptions;
|
|
25422
|
+
let expiresAt;
|
|
24450
25423
|
if (ttl !== void 0) {
|
|
24451
25424
|
const remaining = await this.client.pTTL(key);
|
|
24452
|
-
if (remaining === -2)
|
|
24453
|
-
|
|
25425
|
+
if (remaining === -2) {
|
|
25426
|
+
setOptions = { EX: ttl };
|
|
25427
|
+
expiresAt = Date.now() + ttl * 1e3;
|
|
25428
|
+
} else if (remaining > 0) {
|
|
25429
|
+
setOptions = { PX: remaining };
|
|
25430
|
+
expiresAt = Date.now() + remaining;
|
|
25431
|
+
}
|
|
24454
25432
|
}
|
|
24455
|
-
|
|
25433
|
+
const tx = this.client.multi().sAdd(index.ids, entry.id).set(key, JSON.stringify(entry), setOptions).zAdd(index.time, { score: entry.timestamp, value: entry.id });
|
|
25434
|
+
if (expiresAt !== void 0) tx.zAdd(index.expiry, { score: expiresAt, value: entry.id });
|
|
25435
|
+
else tx.zRem(index.expiry, entry.id);
|
|
25436
|
+
await tx.exec();
|
|
25437
|
+
await this.pruneHistoryAfterSave(index);
|
|
24456
25438
|
}
|
|
24457
25439
|
/**
|
|
24458
25440
|
* Update-only write: see `StateStore.updateEvalResult`.
|
|
@@ -24492,15 +25474,14 @@ var RedisStore = class _RedisStore {
|
|
|
24492
25474
|
}
|
|
24493
25475
|
async listEvalResults(limit) {
|
|
24494
25476
|
return this.listHistoryByZset(
|
|
24495
|
-
this.
|
|
24496
|
-
|
|
24497
|
-
(id) => this.evalHistoryKey(id),
|
|
24498
|
-
(a, b) => b.timestamp - a.timestamp,
|
|
25477
|
+
this.historyIndex("eval"),
|
|
25478
|
+
(e) => ({ score: e.timestamp, id: e.id }),
|
|
24499
25479
|
limit
|
|
24500
25480
|
);
|
|
24501
25481
|
}
|
|
24502
25482
|
async deleteEvalResult(id) {
|
|
24503
|
-
const
|
|
25483
|
+
const index = this.historyIndex("eval");
|
|
25484
|
+
const [, deletedCount] = await this.client.multi().sRem(index.ids, id).del(index.dataKey(id)).zRem(index.time, id).zRem(index.expiry, id).exec();
|
|
24504
25485
|
const deleted = typeof deletedCount === "number" ? deletedCount : 0;
|
|
24505
25486
|
return deleted > 0;
|
|
24506
25487
|
}
|
|
@@ -24524,50 +25505,292 @@ var RedisStore = class _RedisStore {
|
|
|
24524
25505
|
return { exists: true, expiresAt: Date.now() + ttl };
|
|
24525
25506
|
}
|
|
24526
25507
|
/**
|
|
24527
|
-
* Shared list-by-sorted-set implementation
|
|
24528
|
-
*
|
|
24529
|
-
*
|
|
24530
|
-
*
|
|
24531
|
-
*
|
|
25508
|
+
* Shared list-by-sorted-set implementation, newest first by
|
|
25509
|
+
* `(score DESC, member DESC)`, starting strictly after `cursor` when given.
|
|
25510
|
+
*
|
|
25511
|
+
* Fast path (the time sorted set is at least as large as the id set): walk
|
|
25512
|
+
* the sorted set in chunks and MGET each chunk's blobs. Index entries whose
|
|
25513
|
+
* blob is gone (expired, or deleted before the prune caught up) are skipped,
|
|
25514
|
+
* and the walk keeps reading until it has `limit` live records or the set
|
|
25515
|
+
* runs out, however many dead entries are in the way. Each chunk restarts
|
|
25516
|
+
* at the last member read, by score and member, rather than at an offset,
|
|
25517
|
+
* so entries added or removed meanwhile cannot shift the walk.
|
|
25518
|
+
*
|
|
25519
|
+
* Slow path (the time sorted set is behind the id set: backfill has not
|
|
25520
|
+
* run): read every id with GET, then sort and apply the cursor in process.
|
|
24532
25521
|
*/
|
|
24533
|
-
async listHistoryByZset(
|
|
24534
|
-
const setSize = await this.client.sCard(
|
|
25522
|
+
async listHistoryByZset(index, positionOf, limit, cursor) {
|
|
25523
|
+
const setSize = await this.client.sCard(index.ids);
|
|
24535
25524
|
if (setSize === 0) return [];
|
|
24536
|
-
const zsetSize = await this.client.zCard(
|
|
24537
|
-
|
|
24538
|
-
|
|
24539
|
-
const
|
|
25525
|
+
const zsetSize = await this.client.zCard(index.time);
|
|
25526
|
+
const max = limit ? limit : void 0;
|
|
25527
|
+
if (zsetSize < setSize) {
|
|
25528
|
+
const ids = await this.client.sMembers(index.ids);
|
|
25529
|
+
const entries = [];
|
|
25530
|
+
for (const id of ids) {
|
|
25531
|
+
const raw = await this.client.get(index.dataKey(id));
|
|
25532
|
+
if (raw == null) continue;
|
|
25533
|
+
try {
|
|
25534
|
+
entries.push(JSON.parse(raw));
|
|
25535
|
+
} catch {
|
|
25536
|
+
}
|
|
25537
|
+
}
|
|
25538
|
+
const ordered = entries.filter((e) => cursor === void 0 || compareNewestFirst(cursor, positionOf(e)) < 0).sort((a, b) => compareNewestFirst(positionOf(a), positionOf(b)));
|
|
25539
|
+
return max !== void 0 ? ordered.slice(0, max) : ordered;
|
|
25540
|
+
}
|
|
25541
|
+
const results = [];
|
|
25542
|
+
const seen = /* @__PURE__ */ new Set();
|
|
25543
|
+
let bound = cursor;
|
|
25544
|
+
let rangeMax = cursor !== void 0 ? cursor.score : "+inf";
|
|
25545
|
+
let anchor;
|
|
25546
|
+
let grow = 1;
|
|
25547
|
+
const read = (offset, count2) => (
|
|
25548
|
+
// ZRANGE BY SCORE REV: start is the highest score, stop the lowest.
|
|
25549
|
+
this.client.zRangeWithScores(index.time, rangeMax, "-inf", {
|
|
24540
25550
|
BY: "SCORE",
|
|
24541
25551
|
REV: true,
|
|
24542
|
-
...
|
|
25552
|
+
...count2 !== void 0 ? { LIMIT: { offset, count: count2 } } : {}
|
|
25553
|
+
})
|
|
25554
|
+
);
|
|
25555
|
+
for (; ; ) {
|
|
25556
|
+
const remaining = max !== void 0 ? max - results.length : void 0;
|
|
25557
|
+
const want = remaining !== void 0 ? Math.max(remaining * 2, remaining + 5) * grow : void 0;
|
|
25558
|
+
let offset = 0;
|
|
25559
|
+
let count2 = want;
|
|
25560
|
+
let members;
|
|
25561
|
+
if (anchor !== void 0 && want !== void 0) {
|
|
25562
|
+
offset = anchor.pos;
|
|
25563
|
+
count2 = want + 1;
|
|
25564
|
+
members = await read(offset, count2);
|
|
25565
|
+
if (members[0]?.value !== anchor.id) {
|
|
25566
|
+
offset = 0;
|
|
25567
|
+
count2 = anchor.pos + 1 + want;
|
|
25568
|
+
members = await read(offset, count2);
|
|
25569
|
+
}
|
|
25570
|
+
} else {
|
|
25571
|
+
members = await read(offset, count2);
|
|
25572
|
+
}
|
|
25573
|
+
const fresh = members.filter(
|
|
25574
|
+
(m) => !seen.has(m.value) && (bound === void 0 || compareNewestFirst(bound, { score: m.score, id: m.value }) < 0)
|
|
25575
|
+
);
|
|
25576
|
+
if (fresh.length > 0) {
|
|
25577
|
+
const raws = await this.client.mGet(fresh.map((m) => index.dataKey(m.value)));
|
|
25578
|
+
for (let i = 0; i < fresh.length; i++) {
|
|
25579
|
+
seen.add(fresh[i].value);
|
|
25580
|
+
const raw = raws[i];
|
|
25581
|
+
if (raw == null) continue;
|
|
25582
|
+
try {
|
|
25583
|
+
results.push(JSON.parse(raw));
|
|
25584
|
+
} catch {
|
|
25585
|
+
continue;
|
|
25586
|
+
}
|
|
25587
|
+
if (max !== void 0 && results.length >= max) return results;
|
|
25588
|
+
}
|
|
25589
|
+
const lastFresh = fresh[fresh.length - 1];
|
|
25590
|
+
bound = { score: lastFresh.score, id: lastFresh.value };
|
|
25591
|
+
grow = 1;
|
|
25592
|
+
} else {
|
|
25593
|
+
grow *= 2;
|
|
25594
|
+
}
|
|
25595
|
+
if (count2 === void 0 || members.length < count2) return results;
|
|
25596
|
+
const last = members[members.length - 1];
|
|
25597
|
+
if (last.score === rangeMax) {
|
|
25598
|
+
anchor = { id: last.value, score: last.score, pos: offset + members.length - 1 };
|
|
25599
|
+
} else {
|
|
25600
|
+
rangeMax = last.score;
|
|
25601
|
+
const before = members.filter((m) => m.score === last.score).length - 1;
|
|
25602
|
+
anchor = { id: last.value, score: last.score, pos: before };
|
|
25603
|
+
}
|
|
25604
|
+
}
|
|
25605
|
+
}
|
|
25606
|
+
// ── History index cleanup ──────────────────────────────────────────────
|
|
25607
|
+
//
|
|
25608
|
+
// Index members cannot carry a TTL, so when a history record expires its
|
|
25609
|
+
// entries in the id set and the time sorted set stay behind. Every history
|
|
25610
|
+
// save runs one bounded prune of its kind: it reads up to PRUNE_BATCH ids
|
|
25611
|
+
// whose expiry has passed from the expiry sorted set, then runs
|
|
25612
|
+
// PRUNE_HISTORY_INDEX_SCRIPT over them. The script checks each record's blob
|
|
25613
|
+
// and updates the indexes in one atomic step, so a save or delete landing
|
|
25614
|
+
// between the read and the script cannot be undone by it: a record saved
|
|
25615
|
+
// again is seen live and stays listed, and a deleted one is already gone.
|
|
25616
|
+
// The script only removes index entries and re-scores expiry entries; it
|
|
25617
|
+
// never adds a listing entry or writes a blob, so it cannot bring back
|
|
25618
|
+
// deleted data. Several instances pruning at once is safe.
|
|
25619
|
+
/**
|
|
25620
|
+
* Run one prune of `index`'s kind after a save. A prune failure does not
|
|
25621
|
+
* fail the save, whose MULTI already committed the record: it is reported
|
|
25622
|
+
* once per process and the next save of that kind tries again. That also
|
|
25623
|
+
* covers a Redis user whose ACL denies `EVAL`.
|
|
25624
|
+
*/
|
|
25625
|
+
async pruneHistoryAfterSave(index) {
|
|
25626
|
+
const inFlight = this.prunesInFlight ??= /* @__PURE__ */ new Set();
|
|
25627
|
+
if (inFlight.has(index.kind)) return;
|
|
25628
|
+
inFlight.add(index.kind);
|
|
25629
|
+
try {
|
|
25630
|
+
await this.pruneHistory(index);
|
|
25631
|
+
} catch (err) {
|
|
25632
|
+
if (this.connection.isReady === false) return;
|
|
25633
|
+
warnDiagnosticOnce(
|
|
25634
|
+
`redis_history_prune_failed\0${index.kind}`,
|
|
25635
|
+
`RedisStore: removing expired ${index.label} index entries failed (${err instanceof Error ? err.message : String(err)}). The record was saved; the next save retries the cleanup. Listings stay correct meanwhile. The cleanup runs a Lua script, so the Redis user needs permission to run EVAL.`,
|
|
25636
|
+
void 0,
|
|
25637
|
+
"docs/api-reference.md#redisstore-options"
|
|
25638
|
+
);
|
|
25639
|
+
} finally {
|
|
25640
|
+
inFlight.delete(index.kind);
|
|
25641
|
+
}
|
|
25642
|
+
}
|
|
25643
|
+
async pruneHistory(index) {
|
|
25644
|
+
const candidates = await this.client.zRange(index.expiry, "-inf", Date.now(), {
|
|
25645
|
+
BY: "SCORE",
|
|
25646
|
+
LIMIT: { offset: 0, count: PRUNE_BATCH }
|
|
25647
|
+
});
|
|
25648
|
+
if (candidates.length > 0) await this.runPruneScript(index, candidates);
|
|
25649
|
+
}
|
|
25650
|
+
/**
|
|
25651
|
+
* Run {@link PRUNE_HISTORY_INDEX_SCRIPT} over `ids`, at most PRUNE_BATCH
|
|
25652
|
+
* per call so one script never blocks the server for long.
|
|
25653
|
+
*/
|
|
25654
|
+
async runPruneScript(index, ids) {
|
|
25655
|
+
for (let start = 0; start < ids.length; start += PRUNE_BATCH) {
|
|
25656
|
+
const chunk = ids.slice(start, start + PRUNE_BATCH);
|
|
25657
|
+
await this.client.eval(PRUNE_HISTORY_INDEX_SCRIPT, {
|
|
25658
|
+
keys: [index.ids, index.time, index.expiry, ...chunk.map((id) => index.dataKey(id))],
|
|
25659
|
+
arguments: [String(Date.now()), ...chunk]
|
|
24543
25660
|
});
|
|
24544
|
-
|
|
24545
|
-
|
|
24546
|
-
|
|
24547
|
-
|
|
24548
|
-
|
|
24549
|
-
|
|
25661
|
+
}
|
|
25662
|
+
}
|
|
25663
|
+
// ── History maintenance ────────────────────────────────────────────────
|
|
25664
|
+
/**
|
|
25665
|
+
* See `StateStore.applyHistoryRetention`. One call:
|
|
25666
|
+
*
|
|
25667
|
+
* 1. On the first call of a walk (no cursor), backfills the time sorted
|
|
25668
|
+
* set from the id set if it is behind (see `backfillExecutionIndex`).
|
|
25669
|
+
* 2. Reads the next `batchSize` ids from the time sorted set, oldest first
|
|
25670
|
+
* by `(score, member)`, strictly after the cursor. The read restarts at
|
|
25671
|
+
* the cursor's position by value, never by offset.
|
|
25672
|
+
* 3. MGETs their blobs and reads each live blob's PTTL.
|
|
25673
|
+
* 4. In one MULTI, for each live blob:
|
|
25674
|
+
* - a TTL (`PTTL > 0`): `ZADD NX` to the expiry set at now + PTTL. A
|
|
25675
|
+
* record already in the expiry set keeps its entry (`indexed` counts
|
|
25676
|
+
* only new entries).
|
|
25677
|
+
* - no TTL (`-1`), with a retention configured: `PEXPIREAT … NX` at last
|
|
25678
|
+
* save + retention. `NX` applies only to a key that still has no TTL,
|
|
25679
|
+
* so a record re-saved meanwhile keeps the fresh window its save gave
|
|
25680
|
+
* it. A window that has already passed uses a timestamp in the past,
|
|
25681
|
+
* which makes Redis delete the key, again only if it is still untimed.
|
|
25682
|
+
* 5. Adds each newly armed record to the expiry set (`ZADD NX`).
|
|
25683
|
+
* 6. Runs the prune script (`PRUNE_HISTORY_INDEX_SCRIPT`) over ids whose
|
|
25684
|
+
* blob is gone and ids just deleted in step 4. It checks each blob
|
|
25685
|
+
* atomically, so a record re-saved at any point stays listed and a
|
|
25686
|
+
* deleted one is never re-indexed.
|
|
25687
|
+
*
|
|
25688
|
+
* Blobs are never written. A record deleted while the call runs can leave
|
|
25689
|
+
* an entry in the expiry set only; the next prune removes it.
|
|
25690
|
+
*/
|
|
25691
|
+
async applyHistoryRetention(options) {
|
|
25692
|
+
const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("RedisStore", options);
|
|
25693
|
+
const index = this.historyIndex(kind === "executions" ? "execution" : "eval");
|
|
25694
|
+
const ttl = this.ttlFor(kind === "executions" ? "executionHistory" : "evalHistory");
|
|
25695
|
+
if (cursor === void 0) {
|
|
25696
|
+
if (kind === "executions") await this.backfillExecutionIndex();
|
|
25697
|
+
else await this.backfillEvalIndex();
|
|
25698
|
+
}
|
|
25699
|
+
const positions = await this.readRetentionBatch(index, cursor, batchSize + 1);
|
|
25700
|
+
const batch = positions.slice(0, batchSize);
|
|
25701
|
+
const result = {
|
|
25702
|
+
expired: 0,
|
|
25703
|
+
armed: 0,
|
|
25704
|
+
indexed: 0,
|
|
25705
|
+
deadIndexEntries: 0,
|
|
25706
|
+
cursor: positions.length > batchSize ? { time: batch[batch.length - 1].score, id: batch[batch.length - 1].value } : null
|
|
25707
|
+
};
|
|
25708
|
+
if (batch.length === 0) return result;
|
|
25709
|
+
const ids = batch.map((m) => m.value);
|
|
25710
|
+
const raws = await this.client.mGet(ids.map((id) => index.dataKey(id)));
|
|
25711
|
+
const dead = ids.filter((_, i) => raws[i] == null);
|
|
25712
|
+
const present = ids.flatMap((id, i) => {
|
|
25713
|
+
const raw = raws[i];
|
|
25714
|
+
return raw == null ? [] : [{ id, raw }];
|
|
25715
|
+
});
|
|
25716
|
+
const pttls = await Promise.all(present.map((r) => this.client.pTTL(index.dataKey(r.id))));
|
|
25717
|
+
const now = Date.now();
|
|
25718
|
+
const tx = this.client.multi();
|
|
25719
|
+
const queued = [];
|
|
25720
|
+
for (let i = 0; i < present.length; i++) {
|
|
25721
|
+
const { id, raw } = present[i];
|
|
25722
|
+
const pttl = pttls[i];
|
|
25723
|
+
if (pttl === -2) {
|
|
25724
|
+
dead.push(id);
|
|
25725
|
+
} else if (pttl > 0) {
|
|
25726
|
+
tx.zAdd(index.expiry, { score: now + pttl, value: id }, { condition: "NX" });
|
|
25727
|
+
queued.push({ action: "index", id });
|
|
25728
|
+
} else if (pttl === -1 && ttl !== void 0) {
|
|
25729
|
+
let lastSavedAt;
|
|
24550
25730
|
try {
|
|
24551
|
-
|
|
25731
|
+
lastSavedAt = index.lastSavedAtOf(raw);
|
|
24552
25732
|
} catch {
|
|
25733
|
+
lastSavedAt = null;
|
|
25734
|
+
}
|
|
25735
|
+
if (lastSavedAt === null) continue;
|
|
25736
|
+
const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
|
|
25737
|
+
if (decision.expired) {
|
|
25738
|
+
tx.pExpireAt(index.dataKey(id), 1, "NX");
|
|
25739
|
+
queued.push({ action: "expire", id });
|
|
25740
|
+
} else {
|
|
25741
|
+
tx.pExpireAt(index.dataKey(id), decision.expiresAt, "NX");
|
|
25742
|
+
queued.push({ action: "arm", id, expiresAt: decision.expiresAt });
|
|
24553
25743
|
}
|
|
24554
|
-
if (limit !== void 0 && entries2.length >= limit) break;
|
|
24555
25744
|
}
|
|
24556
|
-
return entries2;
|
|
24557
25745
|
}
|
|
24558
|
-
const
|
|
24559
|
-
|
|
24560
|
-
|
|
24561
|
-
|
|
24562
|
-
|
|
24563
|
-
|
|
24564
|
-
|
|
24565
|
-
|
|
24566
|
-
|
|
25746
|
+
const expiredIds = [];
|
|
25747
|
+
const armed = [];
|
|
25748
|
+
if (queued.length > 0) {
|
|
25749
|
+
const replies = await tx.exec();
|
|
25750
|
+
for (let i = 0; i < queued.length; i++) {
|
|
25751
|
+
if (Number(replies[i]) !== 1) continue;
|
|
25752
|
+
const entry = queued[i];
|
|
25753
|
+
if (entry.action === "index") result.indexed++;
|
|
25754
|
+
else if (entry.action === "expire") expiredIds.push(entry.id);
|
|
25755
|
+
else armed.push(entry);
|
|
24567
25756
|
}
|
|
24568
25757
|
}
|
|
24569
|
-
|
|
24570
|
-
|
|
25758
|
+
if (armed.length > 0) {
|
|
25759
|
+
const indexTx = this.client.multi();
|
|
25760
|
+
for (const { id, expiresAt } of armed) {
|
|
25761
|
+
indexTx.zAdd(index.expiry, { score: expiresAt, value: id }, { condition: "NX" });
|
|
25762
|
+
}
|
|
25763
|
+
await indexTx.exec();
|
|
25764
|
+
}
|
|
25765
|
+
result.armed = armed.length;
|
|
25766
|
+
result.expired = expiredIds.length;
|
|
25767
|
+
result.deadIndexEntries = dead.length;
|
|
25768
|
+
if (dead.length > 0 || expiredIds.length > 0) {
|
|
25769
|
+
await this.runPruneScript(index, [...dead, ...expiredIds]);
|
|
25770
|
+
}
|
|
25771
|
+
return result;
|
|
25772
|
+
}
|
|
25773
|
+
/**
|
|
25774
|
+
* Up to `count` members of the time sorted set strictly after `cursor`,
|
|
25775
|
+
* oldest first by `(score, member)`. The members sharing the cursor's score
|
|
25776
|
+
* are read in full and filtered by member; the rest start just above that
|
|
25777
|
+
* score. Neither read uses an offset, so entries removed or added
|
|
25778
|
+
* meanwhile cannot shift the walk.
|
|
25779
|
+
*/
|
|
25780
|
+
async readRetentionBatch(index, cursor, count2) {
|
|
25781
|
+
if (cursor === void 0) {
|
|
25782
|
+
return this.client.zRangeWithScores(index.time, "-inf", "+inf", {
|
|
25783
|
+
BY: "SCORE",
|
|
25784
|
+
LIMIT: { offset: 0, count: count2 }
|
|
25785
|
+
});
|
|
25786
|
+
}
|
|
25787
|
+
const ties = (await this.client.zRangeWithScores(index.time, cursor.time, cursor.time, { BY: "SCORE" })).filter((m) => compareMembers(m.value, cursor.id) > 0);
|
|
25788
|
+
if (ties.length >= count2) return ties.slice(0, count2);
|
|
25789
|
+
const rest = await this.client.zRangeWithScores(index.time, `(${cursor.time}`, "+inf", {
|
|
25790
|
+
BY: "SCORE",
|
|
25791
|
+
LIMIT: { offset: 0, count: count2 - ties.length }
|
|
25792
|
+
});
|
|
25793
|
+
return [...ties, ...rest];
|
|
24571
25794
|
}
|
|
24572
25795
|
// ── Sessions (Studio introspection) ────────────────────────────────────
|
|
24573
25796
|
async listSessions() {
|
|
@@ -24611,10 +25834,31 @@ var RedisStore = class _RedisStore {
|
|
|
24611
25834
|
* must not get a rejected promise for it. `MemoryStore.close()` and
|
|
24612
25835
|
* better-sqlite3's `close()` are both no-ops the second time; node-redis
|
|
24613
25836
|
* instead rejects with "The client is closed", so absorb that here.
|
|
25837
|
+
*
|
|
25838
|
+
* Bounded, so a shutdown can't hang on Redis. `QUIT` lets queued commands
|
|
25839
|
+
* finish on a healthy connection, but node-redis never settles it if the
|
|
25840
|
+
* socket closes first, and a stalled server may never answer it. While
|
|
25841
|
+
* disconnected it couldn't be sent at all.
|
|
24614
25842
|
*/
|
|
24615
25843
|
async close() {
|
|
24616
|
-
if (this.
|
|
24617
|
-
|
|
25844
|
+
if (this.connection.isOpen === false) return;
|
|
25845
|
+
if (this.connection.isReady === false) {
|
|
25846
|
+
abandonConnection(this.connection);
|
|
25847
|
+
return;
|
|
25848
|
+
}
|
|
25849
|
+
let timer;
|
|
25850
|
+
try {
|
|
25851
|
+
await Promise.race([
|
|
25852
|
+
this.connection.quit(),
|
|
25853
|
+
new Promise((_resolve, reject) => {
|
|
25854
|
+
timer = setTimeout(() => reject(new Error("QUIT timed out")), CLOSE_TIMEOUT_MS);
|
|
25855
|
+
})
|
|
25856
|
+
]);
|
|
25857
|
+
} catch {
|
|
25858
|
+
abandonConnection(this.connection);
|
|
25859
|
+
} finally {
|
|
25860
|
+
clearTimeout(timer);
|
|
25861
|
+
}
|
|
24618
25862
|
}
|
|
24619
25863
|
async deleteCheckpoints(executionId) {
|
|
24620
25864
|
await this.client.del(this.checkpointKey(executionId));
|