@axlsdk/axl 0.26.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/index.cjs +1501 -257
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +564 -60
- package/dist/index.d.ts +564 -60
- package/dist/index.js +1501 -257
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -348,7 +348,7 @@ var sharedGuard = typeof installedGuard === "object" && installedGuard !== null
|
|
|
348
348
|
var buildPath = typeof __AXL_BUILD_PATH__ === "string" ? __AXL_BUILD_PATH__ : "<source>";
|
|
349
349
|
var thisCopy = {
|
|
350
350
|
path: buildPath.startsWith("file:") ? fileURLToPath(buildPath) : buildPath,
|
|
351
|
-
version: true ? "0.
|
|
351
|
+
version: true ? "0.27.1" : "unknown"
|
|
352
352
|
};
|
|
353
353
|
function warnCrossCopy(other, joined) {
|
|
354
354
|
if (other === thisCopy) return;
|
|
@@ -12615,8 +12615,17 @@ function detectDroppedRefinements(schema) {
|
|
|
12615
12615
|
var DOCS_URL = "docs/observability.md#schema-diagnostics";
|
|
12616
12616
|
var MAX_WARNED_KEYS = 2048;
|
|
12617
12617
|
var warnedDiagnosticKeys = /* @__PURE__ */ new Set();
|
|
12618
|
+
var silencedDepth = 0;
|
|
12619
|
+
function withDiagnosticsSilenced(fn) {
|
|
12620
|
+
silencedDepth++;
|
|
12621
|
+
try {
|
|
12622
|
+
return fn();
|
|
12623
|
+
} finally {
|
|
12624
|
+
silencedDepth--;
|
|
12625
|
+
}
|
|
12626
|
+
}
|
|
12618
12627
|
function warnDiagnosticOnce(key, message, silent, docs = DOCS_URL) {
|
|
12619
|
-
if (silent || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
|
|
12628
|
+
if (silent || silencedDepth > 0 || process.env.AXL_DIAGNOSTICS_SILENT === "true") return;
|
|
12620
12629
|
if (warnedDiagnosticKeys.has(key)) return;
|
|
12621
12630
|
if (warnedDiagnosticKeys.size >= MAX_WARNED_KEYS) warnedDiagnosticKeys.clear();
|
|
12622
12631
|
warnedDiagnosticKeys.add(key);
|
|
@@ -17017,7 +17026,7 @@ ${appendedSchemaText}`;
|
|
|
17017
17026
|
`ctx.checkpoint name "${name}" is reserved \u2014 names starting with "__auto/" (case-insensitive) are used by the runtime for ask/spawn/race/parallel/map auto-checkpointing.`
|
|
17018
17027
|
);
|
|
17019
17028
|
}
|
|
17020
|
-
return this._checkpoint(name, fn);
|
|
17029
|
+
return this._checkpoint(name, fn, { saveRequired: true });
|
|
17021
17030
|
}
|
|
17022
17031
|
/**
|
|
17023
17032
|
* Internal checkpoint implementation shared by both the public
|
|
@@ -17047,7 +17056,17 @@ ${appendedSchemaText}`;
|
|
|
17047
17056
|
return saved;
|
|
17048
17057
|
}
|
|
17049
17058
|
const result = await fn();
|
|
17050
|
-
|
|
17059
|
+
if (!options?.saveRequired && this.autoCheckpointCounters.saveFailed) return result;
|
|
17060
|
+
try {
|
|
17061
|
+
await this.stateStore.saveCheckpoint(this.executionId, name, result);
|
|
17062
|
+
} catch (err) {
|
|
17063
|
+
if (options?.saveRequired) throw err;
|
|
17064
|
+
this.autoCheckpointCounters.saveFailed = true;
|
|
17065
|
+
console.error(
|
|
17066
|
+
`[axl] Failed to save checkpoint "${name}" for executionId=${this.executionId}; a replay would run this step and later ones again: ${err instanceof Error ? err.message : String(err)}`
|
|
17067
|
+
);
|
|
17068
|
+
return result;
|
|
17069
|
+
}
|
|
17051
17070
|
this.emitEvent({
|
|
17052
17071
|
type: "checkpoint_save",
|
|
17053
17072
|
...agentStamp,
|
|
@@ -18680,10 +18699,135 @@ import { AsyncLocalStorage as AsyncLocalStorage4 } from "async_hooks";
|
|
|
18680
18699
|
import { appendFileSync, mkdirSync as mkdirSync2 } from "fs";
|
|
18681
18700
|
import { dirname } from "path";
|
|
18682
18701
|
|
|
18702
|
+
// src/state/retention.ts
|
|
18703
|
+
var PRUNE_BATCH = 100;
|
|
18704
|
+
var DEFAULT_MEMORY_MAX_EXECUTIONS = 1e3;
|
|
18705
|
+
function warnExecutionRetentionUnset(storeName) {
|
|
18706
|
+
warnDiagnosticOnce(
|
|
18707
|
+
`execution_retention_unset\0${storeName}`,
|
|
18708
|
+
`${storeName} keeps execution history forever because no retention is set; set ttls.executionHistory${storeName === "RedisStore" ? " (or defaultTtl)" : ""} to a number of seconds, or ttls.executionHistory to null to keep it forever without this warning.`,
|
|
18709
|
+
void 0,
|
|
18710
|
+
"docs/migration/execution-history-retention.md#silencing-the-retention-warning"
|
|
18711
|
+
);
|
|
18712
|
+
}
|
|
18713
|
+
function resolveTtl(storeName, category, value) {
|
|
18714
|
+
if (value === void 0 || value === null) return null;
|
|
18715
|
+
if (typeof value !== "number") {
|
|
18716
|
+
throw new AxlError(
|
|
18717
|
+
"INVALID_CONFIG",
|
|
18718
|
+
`${storeName}: ttls.${category} must be a positive number of seconds or null; got ${typeof value}`
|
|
18719
|
+
);
|
|
18720
|
+
}
|
|
18721
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
18722
|
+
throw new AxlError(
|
|
18723
|
+
"INVALID_CONFIG",
|
|
18724
|
+
`${storeName}: ttls.${category} must be a positive number of seconds or null; got ${value}`
|
|
18725
|
+
);
|
|
18726
|
+
}
|
|
18727
|
+
return value;
|
|
18728
|
+
}
|
|
18729
|
+
function resolveHistoryTtls(storeName, ttls) {
|
|
18730
|
+
if (ttls !== void 0 && (ttls === null || typeof ttls !== "object")) {
|
|
18731
|
+
throw new AxlError(
|
|
18732
|
+
"INVALID_CONFIG",
|
|
18733
|
+
`${storeName}: ttls must be an object; got ${ttls === null ? "null" : typeof ttls}`
|
|
18734
|
+
);
|
|
18735
|
+
}
|
|
18736
|
+
return {
|
|
18737
|
+
executionHistory: resolveTtl(storeName, "executionHistory", ttls?.executionHistory),
|
|
18738
|
+
evalHistory: resolveTtl(storeName, "evalHistory", ttls?.evalHistory)
|
|
18739
|
+
};
|
|
18740
|
+
}
|
|
18741
|
+
function expiryFrom(now, ttlSeconds) {
|
|
18742
|
+
return ttlSeconds === null ? null : now + Math.round(ttlSeconds * 1e3);
|
|
18743
|
+
}
|
|
18744
|
+
function isExpired(expiresAt, now) {
|
|
18745
|
+
return expiresAt !== null && now >= expiresAt;
|
|
18746
|
+
}
|
|
18747
|
+
var DEFAULT_RETENTION_BATCH_SIZE = 500;
|
|
18748
|
+
function resolveHistoryRetentionOptions(storeName, options) {
|
|
18749
|
+
const method = `${storeName}.applyHistoryRetention()`;
|
|
18750
|
+
if (options === null || typeof options !== "object") {
|
|
18751
|
+
throw new AxlError("INVALID_ARGUMENT", `${method} options must be an object`);
|
|
18752
|
+
}
|
|
18753
|
+
const { kind, batchSize, cursor } = options;
|
|
18754
|
+
if (kind !== "executions" && kind !== "evals") {
|
|
18755
|
+
throw new AxlError(
|
|
18756
|
+
"INVALID_ARGUMENT",
|
|
18757
|
+
`${method} kind must be 'executions' or 'evals'; got ${String(kind)}`
|
|
18758
|
+
);
|
|
18759
|
+
}
|
|
18760
|
+
if (batchSize !== void 0 && !(typeof batchSize === "number" && Number.isSafeInteger(batchSize) && batchSize > 0)) {
|
|
18761
|
+
throw new AxlError(
|
|
18762
|
+
"INVALID_ARGUMENT",
|
|
18763
|
+
`${method} batchSize must be a positive integer; got ${String(batchSize)}`
|
|
18764
|
+
);
|
|
18765
|
+
}
|
|
18766
|
+
let parsedCursor;
|
|
18767
|
+
if (cursor !== void 0 && cursor !== null) {
|
|
18768
|
+
const candidate = cursor;
|
|
18769
|
+
if (typeof candidate !== "object" || typeof candidate.time !== "number" || !Number.isFinite(candidate.time) || typeof candidate.id !== "string") {
|
|
18770
|
+
throw new AxlError(
|
|
18771
|
+
"INVALID_ARGUMENT",
|
|
18772
|
+
`${method} cursor must be the { time, id } cursor returned by the previous call`
|
|
18773
|
+
);
|
|
18774
|
+
}
|
|
18775
|
+
parsedCursor = { time: candidate.time, id: candidate.id };
|
|
18776
|
+
}
|
|
18777
|
+
return {
|
|
18778
|
+
kind,
|
|
18779
|
+
batchSize: batchSize ?? DEFAULT_RETENTION_BATCH_SIZE,
|
|
18780
|
+
cursor: parsedCursor
|
|
18781
|
+
};
|
|
18782
|
+
}
|
|
18783
|
+
function retentionForUntimedRecord(lastSavedAt, ttlSeconds, now) {
|
|
18784
|
+
const expiresAt = lastSavedAt + Math.round(ttlSeconds * 1e3);
|
|
18785
|
+
return isExpired(expiresAt, now) ? { expired: true } : { expired: false, expiresAt };
|
|
18786
|
+
}
|
|
18787
|
+
function executionLastSavedAt(record) {
|
|
18788
|
+
return typeof record.completedAt === "number" && Number.isFinite(record.completedAt) ? record.completedAt : record.startedAt;
|
|
18789
|
+
}
|
|
18790
|
+
function compareExecutionsNewestFirst(a, b) {
|
|
18791
|
+
if (a.startedAt !== b.startedAt) return b.startedAt - a.startedAt;
|
|
18792
|
+
return compareCodePoints(b.executionId, a.executionId);
|
|
18793
|
+
}
|
|
18794
|
+
function compareRetentionPositions(a, b) {
|
|
18795
|
+
if (a.time !== b.time) return a.time - b.time;
|
|
18796
|
+
return compareCodePoints(a.id, b.id);
|
|
18797
|
+
}
|
|
18798
|
+
function compareCodePoints(a, b) {
|
|
18799
|
+
const len = Math.min(a.length, b.length);
|
|
18800
|
+
for (let i = 0; i < len; i++) {
|
|
18801
|
+
const x = a.charCodeAt(i);
|
|
18802
|
+
const y = b.charCodeAt(i);
|
|
18803
|
+
if (x !== y) return codePointRank(x) - codePointRank(y);
|
|
18804
|
+
}
|
|
18805
|
+
return a.length - b.length;
|
|
18806
|
+
}
|
|
18807
|
+
function codePointRank(unit) {
|
|
18808
|
+
if (unit >= 57344) return unit - 2048;
|
|
18809
|
+
if (unit >= 55296) return unit + 8192;
|
|
18810
|
+
return unit;
|
|
18811
|
+
}
|
|
18812
|
+
|
|
18683
18813
|
// src/state/memory.ts
|
|
18684
18814
|
import { writeFileSync, readFileSync, existsSync, mkdirSync, unlinkSync } from "fs";
|
|
18685
18815
|
import { join } from "path";
|
|
18686
18816
|
import { tmpdir } from "os";
|
|
18817
|
+
function compareOldestFirst(a, b) {
|
|
18818
|
+
return compareExecutionsNewestFirst(b, a);
|
|
18819
|
+
}
|
|
18820
|
+
function resolveMaxExecutions(value) {
|
|
18821
|
+
if (value === void 0) return DEFAULT_MEMORY_MAX_EXECUTIONS;
|
|
18822
|
+
if (value === Infinity) return Infinity;
|
|
18823
|
+
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) {
|
|
18824
|
+
throw new AxlError(
|
|
18825
|
+
"INVALID_CONFIG",
|
|
18826
|
+
`MemoryStore: maxExecutions must be a positive integer or Infinity; got ${typeof value === "number" ? value : typeof value}`
|
|
18827
|
+
);
|
|
18828
|
+
}
|
|
18829
|
+
return value;
|
|
18830
|
+
}
|
|
18687
18831
|
var AWAIT_HUMAN_TEMP_DIR = join(tmpdir(), "axl-memory-store");
|
|
18688
18832
|
var AWAIT_HUMAN_TEMP_FILE = join(AWAIT_HUMAN_TEMP_DIR, "await-human-state.json");
|
|
18689
18833
|
var MemoryStore = class {
|
|
@@ -18693,9 +18837,32 @@ var MemoryStore = class {
|
|
|
18693
18837
|
decisions = /* @__PURE__ */ new Map();
|
|
18694
18838
|
executionStates = /* @__PURE__ */ new Map();
|
|
18695
18839
|
memories = /* @__PURE__ */ new Map();
|
|
18840
|
+
/**
|
|
18841
|
+
* Iteration order is last-save order: a save deletes and re-inserts its key.
|
|
18842
|
+
* With one retention per store and a clock that does not run backwards, that
|
|
18843
|
+
* is also expiry order, which lets the prune stop at the first live record.
|
|
18844
|
+
*/
|
|
18696
18845
|
executionHistory = /* @__PURE__ */ new Map();
|
|
18846
|
+
/** Every key in `executionHistory`, sorted oldest first in listing order. */
|
|
18847
|
+
executionOrder = [];
|
|
18848
|
+
/**
|
|
18849
|
+
* Iteration order is first-write order (a re-save keeps the key's position
|
|
18850
|
+
* and its expiry), which is also expiry order for the same reason as above.
|
|
18851
|
+
*/
|
|
18697
18852
|
evalHistory = /* @__PURE__ */ new Map();
|
|
18698
|
-
|
|
18853
|
+
ttls;
|
|
18854
|
+
maxExecutions;
|
|
18855
|
+
now;
|
|
18856
|
+
constructor(options) {
|
|
18857
|
+
this.ttls = resolveHistoryTtls("MemoryStore", options?.ttls);
|
|
18858
|
+
this.maxExecutions = resolveMaxExecutions(options?.maxExecutions);
|
|
18859
|
+
if (options?.now !== void 0 && typeof options.now !== "function") {
|
|
18860
|
+
throw new AxlError(
|
|
18861
|
+
"INVALID_CONFIG",
|
|
18862
|
+
"MemoryStore: now must be a function returning epoch milliseconds"
|
|
18863
|
+
);
|
|
18864
|
+
}
|
|
18865
|
+
this.now = options?.now ?? (() => Date.now());
|
|
18699
18866
|
this.loadPersistedState();
|
|
18700
18867
|
}
|
|
18701
18868
|
async saveCheckpoint(executionId, name, data) {
|
|
@@ -18792,23 +18959,43 @@ var MemoryStore = class {
|
|
|
18792
18959
|
}
|
|
18793
18960
|
// ── Execution History ──────────────────────────────────────────────
|
|
18794
18961
|
async saveExecution(execution) {
|
|
18795
|
-
const
|
|
18796
|
-
|
|
18797
|
-
|
|
18962
|
+
const now = this.now();
|
|
18963
|
+
const normalized = structuredClone(
|
|
18964
|
+
normalizeStoredExecution({
|
|
18965
|
+
...execution,
|
|
18966
|
+
eventSchemaVersion: getExecutionEventSchemaVersion(execution)
|
|
18967
|
+
})
|
|
18968
|
+
);
|
|
18969
|
+
this.removeExecutionRecord(normalized.executionId);
|
|
18970
|
+
this.executionHistory.set(normalized.executionId, {
|
|
18971
|
+
record: normalized,
|
|
18972
|
+
expiresAt: expiryFrom(now, this.ttls.executionHistory)
|
|
18798
18973
|
});
|
|
18799
|
-
this.
|
|
18974
|
+
this.insertExecutionOrder(normalized);
|
|
18975
|
+
this.pruneExpiredExecutions(now);
|
|
18976
|
+
this.evictOverflowExecutions();
|
|
18800
18977
|
}
|
|
18801
18978
|
async getExecution(executionId) {
|
|
18802
|
-
const
|
|
18803
|
-
|
|
18804
|
-
|
|
18805
|
-
|
|
18806
|
-
|
|
18807
|
-
|
|
18808
|
-
|
|
18979
|
+
const stored = this.executionHistory.get(executionId);
|
|
18980
|
+
if (!stored || isExpired(stored.expiresAt, this.now())) return null;
|
|
18981
|
+
return normalizeStoredExecution(structuredClone(stored.record));
|
|
18982
|
+
}
|
|
18983
|
+
/** Supports `listExecutions(limit, { cursor })`. */
|
|
18984
|
+
historyPaging = true;
|
|
18985
|
+
async listExecutions(limit, options) {
|
|
18986
|
+
const now = this.now();
|
|
18987
|
+
const max = limit ? limit : Infinity;
|
|
18988
|
+
const result = [];
|
|
18989
|
+
const start = options?.cursor !== void 0 ? this.executionOrderLowerBound(options.cursor) - 1 : this.executionOrder.length - 1;
|
|
18990
|
+
for (let i = start; i >= 0 && result.length < max; i--) {
|
|
18991
|
+
const stored = this.executionHistory.get(this.executionOrder[i].executionId);
|
|
18992
|
+
if (isExpired(stored.expiresAt, now)) continue;
|
|
18993
|
+
result.push(normalizeStoredExecution(structuredClone(stored.record)));
|
|
18994
|
+
}
|
|
18995
|
+
return result;
|
|
18809
18996
|
}
|
|
18810
18997
|
async deleteExecution(executionId) {
|
|
18811
|
-
const removedHistory = this.
|
|
18998
|
+
const removedHistory = this.removeExecutionRecord(executionId);
|
|
18812
18999
|
const removedCheckpoints = this.checkpoints.delete(executionId);
|
|
18813
19000
|
const removedState = this.executionStates.delete(executionId);
|
|
18814
19001
|
const removedStreaming = this.streamingEvents.delete(executionId);
|
|
@@ -18819,26 +19006,193 @@ var MemoryStore = class {
|
|
|
18819
19006
|
return removedHistory || removedCheckpoints || removedState || removedStreaming || removedDecision;
|
|
18820
19007
|
}
|
|
18821
19008
|
// ── Eval History ──────────────────────────────────────────────────
|
|
19009
|
+
/**
|
|
19010
|
+
* Write an eval history row. Retention counts from the row's first write:
|
|
19011
|
+
* re-saving a live row keeps its expiry (and its position in expiry order).
|
|
19012
|
+
* A row that has already expired is replaced as a new one.
|
|
19013
|
+
*/
|
|
18822
19014
|
async saveEvalResult(entry) {
|
|
18823
|
-
this.
|
|
19015
|
+
const now = this.now();
|
|
19016
|
+
const existing = this.evalHistory.get(entry.id);
|
|
19017
|
+
if (existing && !isExpired(existing.expiresAt, now)) {
|
|
19018
|
+
this.evalHistory.set(entry.id, {
|
|
19019
|
+
entry: structuredClone(entry),
|
|
19020
|
+
expiresAt: existing.expiresAt
|
|
19021
|
+
});
|
|
19022
|
+
} else {
|
|
19023
|
+
this.evalHistory.delete(entry.id);
|
|
19024
|
+
this.evalHistory.set(entry.id, {
|
|
19025
|
+
entry: structuredClone(entry),
|
|
19026
|
+
expiresAt: expiryFrom(now, this.ttls.evalHistory)
|
|
19027
|
+
});
|
|
19028
|
+
}
|
|
19029
|
+
this.pruneExpiredEvals(now);
|
|
18824
19030
|
}
|
|
18825
|
-
/** Update-only write: see `StateStore.updateEvalResult`. */
|
|
19031
|
+
/** Update-only write: see `StateStore.updateEvalResult`. Never changes retention. */
|
|
18826
19032
|
async updateEvalResult(entry) {
|
|
18827
|
-
|
|
18828
|
-
|
|
19033
|
+
const existing = this.evalHistory.get(entry.id);
|
|
19034
|
+
if (!existing || isExpired(existing.expiresAt, this.now())) return false;
|
|
19035
|
+
this.evalHistory.set(entry.id, {
|
|
19036
|
+
entry: structuredClone(entry),
|
|
19037
|
+
expiresAt: existing.expiresAt
|
|
19038
|
+
});
|
|
18829
19039
|
return true;
|
|
18830
19040
|
}
|
|
18831
19041
|
async listEvalResults(limit) {
|
|
18832
|
-
const
|
|
19042
|
+
const now = this.now();
|
|
19043
|
+
const sorted = [...this.evalHistory.values()].filter((stored) => !isExpired(stored.expiresAt, now)).map((stored) => stored.entry).sort((a, b) => b.timestamp - a.timestamp);
|
|
18833
19044
|
const result = limit ? sorted.slice(0, limit) : sorted;
|
|
18834
19045
|
return result.map((e) => structuredClone(e));
|
|
18835
19046
|
}
|
|
18836
19047
|
async deleteEvalResult(id) {
|
|
18837
19048
|
return this.evalHistory.delete(id);
|
|
18838
19049
|
}
|
|
18839
|
-
/**
|
|
19050
|
+
/** Existence plus the expiry stored at the row's first write, if any. */
|
|
18840
19051
|
async getEvalRetention(id) {
|
|
18841
|
-
|
|
19052
|
+
const stored = this.evalHistory.get(id);
|
|
19053
|
+
if (!stored || isExpired(stored.expiresAt, this.now())) return { exists: false };
|
|
19054
|
+
return stored.expiresAt === null ? { exists: true } : { exists: true, expiresAt: stored.expiresAt };
|
|
19055
|
+
}
|
|
19056
|
+
// ── History maintenance ───────────────────────────────────────────
|
|
19057
|
+
/**
|
|
19058
|
+
* See `StateStore.applyHistoryRetention`. Every record this store holds was
|
|
19059
|
+
* saved under the retention it was constructed with, so in practice a run
|
|
19060
|
+
* finds nothing to expire or arm; the method exists so the same maintenance
|
|
19061
|
+
* code works against every built-in store. There is no separate index, so
|
|
19062
|
+
* `indexed` and `deadIndexEntries` are always `0`.
|
|
19063
|
+
*
|
|
19064
|
+
* An armed record keeps its place in last-save order, which is also where
|
|
19065
|
+
* its expiry (last save + retention) belongs, so the prune-on-save order
|
|
19066
|
+
* still holds.
|
|
19067
|
+
*/
|
|
19068
|
+
async applyHistoryRetention(options) {
|
|
19069
|
+
const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("MemoryStore", options);
|
|
19070
|
+
const now = this.now();
|
|
19071
|
+
let expired = 0;
|
|
19072
|
+
let armed = 0;
|
|
19073
|
+
let positions;
|
|
19074
|
+
if (kind === "executions") {
|
|
19075
|
+
let start = 0;
|
|
19076
|
+
if (cursor !== void 0) {
|
|
19077
|
+
const key = { startedAt: cursor.time, executionId: cursor.id };
|
|
19078
|
+
start = this.executionOrderLowerBound(key);
|
|
19079
|
+
const at = this.executionOrder[start];
|
|
19080
|
+
if (at !== void 0 && compareOldestFirst(at, key) === 0) start++;
|
|
19081
|
+
}
|
|
19082
|
+
positions = this.executionOrder.slice(start, start + batchSize + 1).map((k) => ({ time: k.startedAt, id: k.executionId }));
|
|
19083
|
+
} else {
|
|
19084
|
+
positions = [...this.evalHistory.values()].map((stored) => ({ time: stored.entry.timestamp, id: stored.entry.id })).filter((p) => cursor === void 0 || compareRetentionPositions(cursor, p) < 0).sort(compareRetentionPositions).slice(0, batchSize + 1);
|
|
19085
|
+
}
|
|
19086
|
+
const batch = positions.slice(0, batchSize);
|
|
19087
|
+
const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
|
|
19088
|
+
if (ttl !== null) {
|
|
19089
|
+
for (const { id } of batch) {
|
|
19090
|
+
if (kind === "executions") {
|
|
19091
|
+
const stored = this.executionHistory.get(id);
|
|
19092
|
+
if (!stored || stored.expiresAt !== null) continue;
|
|
19093
|
+
const decision = retentionForUntimedRecord(executionLastSavedAt(stored.record), ttl, now);
|
|
19094
|
+
if (decision.expired) {
|
|
19095
|
+
this.expireExecutionRecord(id);
|
|
19096
|
+
expired++;
|
|
19097
|
+
} else {
|
|
19098
|
+
stored.expiresAt = decision.expiresAt;
|
|
19099
|
+
armed++;
|
|
19100
|
+
}
|
|
19101
|
+
} else {
|
|
19102
|
+
const stored = this.evalHistory.get(id);
|
|
19103
|
+
if (!stored || stored.expiresAt !== null) continue;
|
|
19104
|
+
const decision = retentionForUntimedRecord(stored.entry.timestamp, ttl, now);
|
|
19105
|
+
if (decision.expired) {
|
|
19106
|
+
this.evalHistory.delete(id);
|
|
19107
|
+
expired++;
|
|
19108
|
+
} else {
|
|
19109
|
+
stored.expiresAt = decision.expiresAt;
|
|
19110
|
+
armed++;
|
|
19111
|
+
}
|
|
19112
|
+
}
|
|
19113
|
+
}
|
|
19114
|
+
}
|
|
19115
|
+
return {
|
|
19116
|
+
expired,
|
|
19117
|
+
armed,
|
|
19118
|
+
indexed: 0,
|
|
19119
|
+
deadIndexEntries: 0,
|
|
19120
|
+
cursor: positions.length > batchSize ? batch[batch.length - 1] : null
|
|
19121
|
+
};
|
|
19122
|
+
}
|
|
19123
|
+
// ── History retention internals ───────────────────────────────────
|
|
19124
|
+
/** Index of the first order key not older than `key` (binary search). */
|
|
19125
|
+
executionOrderLowerBound(key) {
|
|
19126
|
+
let lo = 0;
|
|
19127
|
+
let hi = this.executionOrder.length;
|
|
19128
|
+
while (lo < hi) {
|
|
19129
|
+
const mid = lo + hi >>> 1;
|
|
19130
|
+
if (compareOldestFirst(this.executionOrder[mid], key) < 0) lo = mid + 1;
|
|
19131
|
+
else hi = mid;
|
|
19132
|
+
}
|
|
19133
|
+
return lo;
|
|
19134
|
+
}
|
|
19135
|
+
insertExecutionOrder(record) {
|
|
19136
|
+
const key = { startedAt: record.startedAt, executionId: record.executionId };
|
|
19137
|
+
this.executionOrder.splice(this.executionOrderLowerBound(key), 0, key);
|
|
19138
|
+
}
|
|
19139
|
+
/** Remove a history record and its listing-index key. Returns whether one existed. */
|
|
19140
|
+
removeExecutionRecord(executionId) {
|
|
19141
|
+
const stored = this.executionHistory.get(executionId);
|
|
19142
|
+
if (!stored) return false;
|
|
19143
|
+
this.executionHistory.delete(executionId);
|
|
19144
|
+
const index = this.executionOrderLowerBound(stored.record);
|
|
19145
|
+
if (this.executionOrder[index]?.executionId === executionId) {
|
|
19146
|
+
this.executionOrder.splice(index, 1);
|
|
19147
|
+
}
|
|
19148
|
+
return true;
|
|
19149
|
+
}
|
|
19150
|
+
/**
|
|
19151
|
+
* A run leaving history through retention (expiry or `maxExecutions`) takes
|
|
19152
|
+
* its leftover per-run data with it: checkpoints (kept today when a run
|
|
19153
|
+
* fails), saved execution state and any streaming buffer. Pending decisions
|
|
19154
|
+
* are not touched. The runtime never writes execution state, and a run is
|
|
19155
|
+
* only saved to history once it has finished, so nothing still running
|
|
19156
|
+
* depends on what is removed here.
|
|
19157
|
+
*/
|
|
19158
|
+
expireExecutionRecord(executionId) {
|
|
19159
|
+
this.removeExecutionRecord(executionId);
|
|
19160
|
+
this.checkpoints.delete(executionId);
|
|
19161
|
+
const state = this.executionStates.get(executionId);
|
|
19162
|
+
this.executionStates.delete(executionId);
|
|
19163
|
+
this.streamingEvents.delete(executionId);
|
|
19164
|
+
if (state?.status === "waiting") this.persistAwaitHumanState();
|
|
19165
|
+
}
|
|
19166
|
+
/**
|
|
19167
|
+
* Remove up to `PRUNE_BATCH` expired execution records, oldest save first.
|
|
19168
|
+
* Stops at the first live record: Map order is last-save order, which is
|
|
19169
|
+
* expiry order while the clock moves forward. If the clock steps back, a
|
|
19170
|
+
* few expired records wait for a later save; reads still filter them.
|
|
19171
|
+
*/
|
|
19172
|
+
pruneExpiredExecutions(now) {
|
|
19173
|
+
if (this.ttls.executionHistory === null) return;
|
|
19174
|
+
let removed = 0;
|
|
19175
|
+
for (const [id, stored] of this.executionHistory) {
|
|
19176
|
+
if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
|
|
19177
|
+
this.expireExecutionRecord(id);
|
|
19178
|
+
removed++;
|
|
19179
|
+
}
|
|
19180
|
+
}
|
|
19181
|
+
/** Evict the oldest records in listing order until the count bound holds. */
|
|
19182
|
+
evictOverflowExecutions() {
|
|
19183
|
+
while (this.executionHistory.size > this.maxExecutions) {
|
|
19184
|
+
this.expireExecutionRecord(this.executionOrder[0].executionId);
|
|
19185
|
+
}
|
|
19186
|
+
}
|
|
19187
|
+
/** Remove up to `PRUNE_BATCH` expired eval rows, oldest first write first. */
|
|
19188
|
+
pruneExpiredEvals(now) {
|
|
19189
|
+
if (this.ttls.evalHistory === null) return;
|
|
19190
|
+
let removed = 0;
|
|
19191
|
+
for (const [id, stored] of this.evalHistory) {
|
|
19192
|
+
if (removed >= PRUNE_BATCH || !isExpired(stored.expiresAt, now)) break;
|
|
19193
|
+
this.evalHistory.delete(id);
|
|
19194
|
+
removed++;
|
|
19195
|
+
}
|
|
18842
19196
|
}
|
|
18843
19197
|
// ── Sessions (Studio introspection) ─────────────────────────────────
|
|
18844
19198
|
async listSessions() {
|
|
@@ -18916,6 +19270,7 @@ var MemoryStore = class {
|
|
|
18916
19270
|
};
|
|
18917
19271
|
|
|
18918
19272
|
// src/state/sqlite.ts
|
|
19273
|
+
var LIVE = "(expires_at IS NULL OR expires_at > ?)";
|
|
18919
19274
|
function safeJsonParse(data) {
|
|
18920
19275
|
try {
|
|
18921
19276
|
return JSON.parse(data);
|
|
@@ -18952,7 +19307,17 @@ function rowToExecutionInfo(row) {
|
|
|
18952
19307
|
}
|
|
18953
19308
|
var SQLiteStore = class {
|
|
18954
19309
|
db;
|
|
18955
|
-
|
|
19310
|
+
ttls;
|
|
19311
|
+
now;
|
|
19312
|
+
constructor(filePath, options) {
|
|
19313
|
+
this.ttls = resolveHistoryTtls("SQLiteStore", options?.ttls);
|
|
19314
|
+
if (options?.now !== void 0 && typeof options.now !== "function") {
|
|
19315
|
+
throw new AxlError(
|
|
19316
|
+
"INVALID_CONFIG",
|
|
19317
|
+
"SQLiteStore: now must be a function returning epoch milliseconds"
|
|
19318
|
+
);
|
|
19319
|
+
}
|
|
19320
|
+
this.now = options?.now ?? (() => Date.now());
|
|
18956
19321
|
let BetterSqlite3;
|
|
18957
19322
|
try {
|
|
18958
19323
|
BetterSqlite3 = __require("better-sqlite3");
|
|
@@ -18965,6 +19330,7 @@ var SQLiteStore = class {
|
|
|
18965
19330
|
this.db.pragma("journal_mode = WAL");
|
|
18966
19331
|
this.migrate();
|
|
18967
19332
|
this.initTables();
|
|
19333
|
+
if (options?.ttls?.executionHistory === void 0) warnExecutionRetentionUnset("SQLiteStore");
|
|
18968
19334
|
}
|
|
18969
19335
|
/**
|
|
18970
19336
|
* Idempotent schema migration. Tracks version via `PRAGMA user_version`
|
|
@@ -18992,6 +19358,11 @@ var SQLiteStore = class {
|
|
|
18992
19358
|
* Existing rows remain `NULL`, the documented v1 sentinel.
|
|
18993
19359
|
* - v5 (observation completeness): add `execution_history.observation`.
|
|
18994
19360
|
* Existing rows remain `NULL`, meaning they predate the contract.
|
|
19361
|
+
* - v6 (history retention): add nullable `expires_at` (epoch ms) to
|
|
19362
|
+
* `execution_history` and `eval_history`. Existing rows remain `NULL`
|
|
19363
|
+
* (never expire), so an upgrade deletes nothing. The single-column
|
|
19364
|
+
* `started_at` index is replaced by `(started_at, execution_id)`, which
|
|
19365
|
+
* serves the tie-safe listing order.
|
|
18995
19366
|
*
|
|
18996
19367
|
* Applied BEFORE `initTables()` so the subsequent `CREATE TABLE
|
|
18997
19368
|
* IF NOT EXISTS` runs against the post-migration column name. Fresh
|
|
@@ -19009,7 +19380,7 @@ var SQLiteStore = class {
|
|
|
19009
19380
|
* transactional re-read costs nothing.
|
|
19010
19381
|
*/
|
|
19011
19382
|
migrate() {
|
|
19012
|
-
const TARGET_VERSION =
|
|
19383
|
+
const TARGET_VERSION = 6;
|
|
19013
19384
|
const current = this.db.pragma("user_version", { simple: true });
|
|
19014
19385
|
if (current >= TARGET_VERSION) return;
|
|
19015
19386
|
this.db.exec("BEGIN IMMEDIATE");
|
|
@@ -19049,6 +19420,15 @@ var SQLiteStore = class {
|
|
|
19049
19420
|
this.db.exec("ALTER TABLE execution_history ADD COLUMN observation TEXT");
|
|
19050
19421
|
}
|
|
19051
19422
|
}
|
|
19423
|
+
if (committed < 6) {
|
|
19424
|
+
for (const table of ["execution_history", "eval_history"]) {
|
|
19425
|
+
const cols = this.db.pragma(`table_info(${table})`);
|
|
19426
|
+
if (cols.length > 0 && !cols.some((c) => c.name === "expires_at")) {
|
|
19427
|
+
this.db.exec(`ALTER TABLE ${table} ADD COLUMN expires_at INTEGER`);
|
|
19428
|
+
}
|
|
19429
|
+
}
|
|
19430
|
+
this.db.exec("DROP INDEX IF EXISTS idx_exec_history_started");
|
|
19431
|
+
}
|
|
19052
19432
|
this.db.pragma(`user_version = ${TARGET_VERSION}`);
|
|
19053
19433
|
this.db.exec("COMMIT");
|
|
19054
19434
|
} catch (err) {
|
|
@@ -19114,18 +19494,28 @@ var SQLiteStore = class {
|
|
|
19114
19494
|
events TEXT NOT NULL,
|
|
19115
19495
|
metadata TEXT,
|
|
19116
19496
|
event_schema_version INTEGER,
|
|
19117
|
-
observation TEXT
|
|
19497
|
+
observation TEXT,
|
|
19498
|
+
expires_at INTEGER
|
|
19118
19499
|
);
|
|
19119
19500
|
|
|
19120
19501
|
CREATE TABLE IF NOT EXISTS eval_history (
|
|
19121
19502
|
id TEXT PRIMARY KEY,
|
|
19122
19503
|
eval_name TEXT NOT NULL,
|
|
19123
19504
|
timestamp INTEGER NOT NULL,
|
|
19124
|
-
data TEXT NOT NULL
|
|
19505
|
+
data TEXT NOT NULL,
|
|
19506
|
+
expires_at INTEGER
|
|
19125
19507
|
);
|
|
19126
19508
|
|
|
19127
|
-
CREATE INDEX IF NOT EXISTS
|
|
19509
|
+
CREATE INDEX IF NOT EXISTS idx_exec_history_started_id
|
|
19510
|
+
ON execution_history (started_at DESC, execution_id DESC);
|
|
19511
|
+
CREATE INDEX IF NOT EXISTS idx_exec_history_expires ON execution_history (expires_at);
|
|
19128
19512
|
CREATE INDEX IF NOT EXISTS idx_eval_history_timestamp ON eval_history (timestamp DESC);
|
|
19513
|
+
CREATE INDEX IF NOT EXISTS idx_eval_history_expires ON eval_history (expires_at);
|
|
19514
|
+
|
|
19515
|
+
-- Superseded by idx_exec_history_started_id in v6. Dropped here as well as in
|
|
19516
|
+
-- migrate(): a pre-v6 process reopening a v6 file re-creates it, and
|
|
19517
|
+
-- migrate() never runs again once user_version is 6.
|
|
19518
|
+
DROP INDEX IF EXISTS idx_exec_history_started;
|
|
19129
19519
|
`);
|
|
19130
19520
|
}
|
|
19131
19521
|
// ── Checkpoints ────────────────────────────────────────────────────────
|
|
@@ -19238,30 +19628,71 @@ var SQLiteStore = class {
|
|
|
19238
19628
|
...execution,
|
|
19239
19629
|
eventSchemaVersion: getExecutionEventSchemaVersion(execution)
|
|
19240
19630
|
});
|
|
19631
|
+
const now = this.now();
|
|
19632
|
+
this.db.transaction(() => {
|
|
19633
|
+
this.db.prepare(
|
|
19634
|
+
"INSERT OR REPLACE INTO execution_history (execution_id, workflow, status, total_cost, started_at, completed_at, duration, error, events, metadata, event_schema_version, observation, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
|
|
19635
|
+
).run(
|
|
19636
|
+
normalized.executionId,
|
|
19637
|
+
normalized.workflow,
|
|
19638
|
+
normalized.status,
|
|
19639
|
+
normalized.totalCost,
|
|
19640
|
+
normalized.startedAt,
|
|
19641
|
+
normalized.completedAt ?? null,
|
|
19642
|
+
normalized.duration,
|
|
19643
|
+
normalized.error ?? null,
|
|
19644
|
+
JSON.stringify(normalized.events),
|
|
19645
|
+
normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
|
|
19646
|
+
normalized.eventSchemaVersion,
|
|
19647
|
+
normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null,
|
|
19648
|
+
expiryFrom(now, this.ttls.executionHistory)
|
|
19649
|
+
);
|
|
19650
|
+
this.pruneExpiredExecutions(now);
|
|
19651
|
+
})();
|
|
19652
|
+
}
|
|
19653
|
+
/**
|
|
19654
|
+
* Remove up to `PRUNE_BATCH` expired execution-history rows, together with
|
|
19655
|
+
* each run's leftover checkpoints and saved execution state, inside the
|
|
19656
|
+
* caller's transaction. Pending decisions are not touched. The runtime
|
|
19657
|
+
* never writes execution state, and a run is only saved to history once it
|
|
19658
|
+
* has finished, so nothing still running depends on what is removed here.
|
|
19659
|
+
*/
|
|
19660
|
+
pruneExpiredExecutions(now) {
|
|
19661
|
+
const ids = this.db.prepare(
|
|
19662
|
+
"SELECT execution_id FROM execution_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?"
|
|
19663
|
+
).all(now, PRUNE_BATCH).map((row) => row.execution_id);
|
|
19664
|
+
if (ids.length === 0) return;
|
|
19665
|
+
const idList = JSON.stringify(ids);
|
|
19666
|
+
for (const table of ["execution_history", "checkpoints", "execution_state"]) {
|
|
19667
|
+
this.db.prepare(`DELETE FROM ${table} WHERE execution_id IN (SELECT value FROM json_each(?))`).run(idList);
|
|
19668
|
+
}
|
|
19669
|
+
}
|
|
19670
|
+
/** Remove up to `PRUNE_BATCH` expired eval-history rows (caller's transaction). */
|
|
19671
|
+
pruneExpiredEvals(now) {
|
|
19241
19672
|
this.db.prepare(
|
|
19242
|
-
"
|
|
19243
|
-
).run(
|
|
19244
|
-
normalized.executionId,
|
|
19245
|
-
normalized.workflow,
|
|
19246
|
-
normalized.status,
|
|
19247
|
-
normalized.totalCost,
|
|
19248
|
-
normalized.startedAt,
|
|
19249
|
-
normalized.completedAt ?? null,
|
|
19250
|
-
normalized.duration,
|
|
19251
|
-
normalized.error ?? null,
|
|
19252
|
-
JSON.stringify(normalized.events),
|
|
19253
|
-
normalized.metadata !== void 0 ? JSON.stringify(normalized.metadata) : null,
|
|
19254
|
-
normalized.eventSchemaVersion,
|
|
19255
|
-
normalized.observation !== void 0 ? JSON.stringify(normalized.observation) : null
|
|
19256
|
-
);
|
|
19673
|
+
"DELETE FROM eval_history WHERE rowid IN (SELECT rowid FROM eval_history WHERE expires_at <= ? ORDER BY expires_at LIMIT ?)"
|
|
19674
|
+
).run(now, PRUNE_BATCH);
|
|
19257
19675
|
}
|
|
19258
19676
|
async getExecution(executionId) {
|
|
19259
|
-
const row = this.db.prepare(
|
|
19677
|
+
const row = this.db.prepare(`SELECT * FROM execution_history WHERE execution_id = ? AND ${LIVE}`).get(executionId, this.now());
|
|
19260
19678
|
return row ? rowToExecutionInfo(row) : null;
|
|
19261
19679
|
}
|
|
19262
|
-
|
|
19263
|
-
|
|
19264
|
-
|
|
19680
|
+
/** Supports `listExecutions(limit, { cursor })`. */
|
|
19681
|
+
historyPaging = true;
|
|
19682
|
+
async listExecutions(limit, options) {
|
|
19683
|
+
const cursor = options?.cursor;
|
|
19684
|
+
const params = [this.now()];
|
|
19685
|
+
let sql = `SELECT * FROM execution_history WHERE ${LIVE}`;
|
|
19686
|
+
if (cursor !== void 0) {
|
|
19687
|
+
sql += " AND (started_at < ? OR (started_at = ? AND execution_id < ?))";
|
|
19688
|
+
params.push(cursor.startedAt, cursor.startedAt, cursor.executionId);
|
|
19689
|
+
}
|
|
19690
|
+
sql += " ORDER BY started_at DESC, execution_id DESC";
|
|
19691
|
+
if (limit) {
|
|
19692
|
+
sql += " LIMIT ?";
|
|
19693
|
+
params.push(limit);
|
|
19694
|
+
}
|
|
19695
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
19265
19696
|
return rows.map(rowToExecutionInfo);
|
|
19266
19697
|
}
|
|
19267
19698
|
async deleteExecution(executionId) {
|
|
@@ -19275,24 +19706,38 @@ var SQLiteStore = class {
|
|
|
19275
19706
|
return tx();
|
|
19276
19707
|
}
|
|
19277
19708
|
// ── Eval History ────────────────────────────────────────────────────────
|
|
19709
|
+
/**
|
|
19710
|
+
* Write an eval history row. Retention counts from the row's first write:
|
|
19711
|
+
* re-saving a live row keeps its `expires_at` (including none, for a row
|
|
19712
|
+
* written without retention). A row that has already expired is replaced
|
|
19713
|
+
* as a new one.
|
|
19714
|
+
*/
|
|
19278
19715
|
async saveEvalResult(entry) {
|
|
19279
|
-
this.
|
|
19280
|
-
|
|
19281
|
-
|
|
19716
|
+
const now = this.now();
|
|
19717
|
+
this.db.transaction(() => {
|
|
19718
|
+
const existing = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(entry.id, now);
|
|
19719
|
+
const expiresAt = existing ? existing.expires_at : expiryFrom(now, this.ttls.evalHistory);
|
|
19720
|
+
this.db.prepare(
|
|
19721
|
+
"INSERT OR REPLACE INTO eval_history (id, eval_name, timestamp, data, expires_at) VALUES (?, ?, ?, ?, ?)"
|
|
19722
|
+
).run(entry.id, entry.eval, entry.timestamp, JSON.stringify(entry.data), expiresAt);
|
|
19723
|
+
this.pruneExpiredEvals(now);
|
|
19724
|
+
})();
|
|
19282
19725
|
}
|
|
19283
19726
|
/**
|
|
19284
19727
|
* Update-only write: see `StateStore.updateEvalResult`. `UPDATE ... WHERE id`
|
|
19285
|
-
* touches nothing when the row is gone, where `INSERT OR REPLACE`
|
|
19286
|
-
* resurrect it
|
|
19287
|
-
* permanent.
|
|
19728
|
+
* touches nothing when the row is gone or expired, where `INSERT OR REPLACE`
|
|
19729
|
+
* would resurrect it. `expires_at` is left as it was.
|
|
19288
19730
|
*/
|
|
19289
19731
|
async updateEvalResult(entry) {
|
|
19290
|
-
const result = this.db.prepare(
|
|
19732
|
+
const result = this.db.prepare(
|
|
19733
|
+
`UPDATE eval_history SET eval_name = ?, timestamp = ?, data = ? WHERE id = ? AND ${LIVE}`
|
|
19734
|
+
).run(entry.eval, entry.timestamp, JSON.stringify(entry.data), entry.id, this.now());
|
|
19291
19735
|
return result.changes > 0;
|
|
19292
19736
|
}
|
|
19293
19737
|
async listEvalResults(limit) {
|
|
19294
|
-
const
|
|
19295
|
-
const
|
|
19738
|
+
const base = `SELECT * FROM eval_history WHERE ${LIVE} ORDER BY timestamp DESC`;
|
|
19739
|
+
const now = this.now();
|
|
19740
|
+
const rows = limit ? this.db.prepare(`${base} LIMIT ?`).all(now, limit) : this.db.prepare(base).all(now);
|
|
19296
19741
|
return rows.map((r) => ({
|
|
19297
19742
|
id: r.id,
|
|
19298
19743
|
eval: r.eval_name,
|
|
@@ -19304,10 +19749,75 @@ var SQLiteStore = class {
|
|
|
19304
19749
|
const result = this.db.prepare("DELETE FROM eval_history WHERE id = ?").run(id);
|
|
19305
19750
|
return result.changes > 0;
|
|
19306
19751
|
}
|
|
19307
|
-
/**
|
|
19752
|
+
/** Existence plus the expiry stored at the row's first write, if any. */
|
|
19308
19753
|
async getEvalRetention(id) {
|
|
19309
|
-
const row = this.db.prepare(
|
|
19310
|
-
return { exists:
|
|
19754
|
+
const row = this.db.prepare(`SELECT expires_at FROM eval_history WHERE id = ? AND ${LIVE}`).get(id, this.now());
|
|
19755
|
+
if (!row) return { exists: false };
|
|
19756
|
+
return row.expires_at === null ? { exists: true } : { exists: true, expiresAt: row.expires_at };
|
|
19757
|
+
}
|
|
19758
|
+
// ── History maintenance ─────────────────────────────────────────────────
|
|
19759
|
+
/**
|
|
19760
|
+
* See `StateStore.applyHistoryRetention`. One batch runs in one immediate
|
|
19761
|
+
* transaction, so no other writer can change a row between this call
|
|
19762
|
+
* reading it and acting on it. Only rows whose `expires_at` is `NULL` are
|
|
19763
|
+
* touched. An expired execution takes its checkpoints and saved execution
|
|
19764
|
+
* state with it, as a normal expiry does. There is no separate index, so
|
|
19765
|
+
* `indexed` and `deadIndexEntries` are always `0`.
|
|
19766
|
+
*/
|
|
19767
|
+
async applyHistoryRetention(options) {
|
|
19768
|
+
const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("SQLiteStore", options);
|
|
19769
|
+
const ttl = kind === "executions" ? this.ttls.executionHistory : this.ttls.evalHistory;
|
|
19770
|
+
const table = kind === "executions" ? { name: "execution_history", id: "execution_id", time: "started_at" } : { name: "eval_history", id: "id", time: "timestamp" };
|
|
19771
|
+
const lastSaved = kind === "executions" ? "completed_at" : "NULL";
|
|
19772
|
+
const now = this.now();
|
|
19773
|
+
return this.db.transaction(() => {
|
|
19774
|
+
const params = [];
|
|
19775
|
+
let sql = `SELECT ${table.id} AS id, ${table.time} AS time, ${lastSaved} AS last_saved, expires_at FROM ${table.name}`;
|
|
19776
|
+
if (cursor !== void 0) {
|
|
19777
|
+
sql += ` WHERE ${table.time} > ? OR (${table.time} = ? AND ${table.id} > ?)`;
|
|
19778
|
+
params.push(cursor.time, cursor.time, cursor.id);
|
|
19779
|
+
}
|
|
19780
|
+
sql += ` ORDER BY ${table.time}, ${table.id} LIMIT ?`;
|
|
19781
|
+
params.push(batchSize + 1);
|
|
19782
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
19783
|
+
const batch = rows.slice(0, batchSize);
|
|
19784
|
+
const toExpire = [];
|
|
19785
|
+
let armed = 0;
|
|
19786
|
+
if (ttl !== null) {
|
|
19787
|
+
const arm = this.db.prepare(
|
|
19788
|
+
`UPDATE ${table.name} SET expires_at = ? WHERE ${table.id} = ? AND expires_at IS NULL`
|
|
19789
|
+
);
|
|
19790
|
+
for (const row of batch) {
|
|
19791
|
+
if (row.expires_at !== null) continue;
|
|
19792
|
+
const lastSavedAt = kind === "executions" ? executionLastSavedAt({ startedAt: row.time, completedAt: row.last_saved }) : row.time;
|
|
19793
|
+
const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
|
|
19794
|
+
if (decision.expired) toExpire.push(row.id);
|
|
19795
|
+
else armed += arm.run(decision.expiresAt, row.id).changes;
|
|
19796
|
+
}
|
|
19797
|
+
}
|
|
19798
|
+
let expired = 0;
|
|
19799
|
+
if (toExpire.length > 0) {
|
|
19800
|
+
const idList = JSON.stringify(toExpire);
|
|
19801
|
+
expired = this.db.prepare(
|
|
19802
|
+
`DELETE FROM ${table.name} WHERE expires_at IS NULL AND ${table.id} IN (SELECT value FROM json_each(?))`
|
|
19803
|
+
).run(idList).changes;
|
|
19804
|
+
if (kind === "executions") {
|
|
19805
|
+
for (const side of ["checkpoints", "execution_state"]) {
|
|
19806
|
+
this.db.prepare(
|
|
19807
|
+
`DELETE FROM ${side} WHERE execution_id IN (SELECT value FROM json_each(?))`
|
|
19808
|
+
).run(idList);
|
|
19809
|
+
}
|
|
19810
|
+
}
|
|
19811
|
+
}
|
|
19812
|
+
const last = batch[batch.length - 1];
|
|
19813
|
+
return {
|
|
19814
|
+
expired,
|
|
19815
|
+
armed,
|
|
19816
|
+
indexed: 0,
|
|
19817
|
+
deadIndexEntries: 0,
|
|
19818
|
+
cursor: rows.length > batchSize ? { time: last.time, id: last.id } : null
|
|
19819
|
+
};
|
|
19820
|
+
}).immediate();
|
|
19311
19821
|
}
|
|
19312
19822
|
// ── Sessions (Studio introspection) ────────────────────────────────────
|
|
19313
19823
|
async listSessions() {
|
|
@@ -19477,11 +19987,18 @@ var Session = class _Session {
|
|
|
19477
19987
|
try {
|
|
19478
19988
|
const completion = axlStream.promise.then(
|
|
19479
19989
|
(result) => this.commitHistory(history, result).catch((err) => {
|
|
19480
|
-
|
|
19481
|
-
|
|
19482
|
-
|
|
19483
|
-
|
|
19484
|
-
|
|
19990
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
19991
|
+
try {
|
|
19992
|
+
this.runtime.emit("error", {
|
|
19993
|
+
type: "session_history_save_failed",
|
|
19994
|
+
sessionId: this.sessionId,
|
|
19995
|
+
error
|
|
19996
|
+
});
|
|
19997
|
+
} catch {
|
|
19998
|
+
console.error(
|
|
19999
|
+
`[axl] Failed to save session history for sessionId=${this.sessionId}: ${error}`
|
|
20000
|
+
);
|
|
20001
|
+
}
|
|
19485
20002
|
}),
|
|
19486
20003
|
() => void 0
|
|
19487
20004
|
);
|
|
@@ -20656,6 +21173,9 @@ function hashInput(input) {
|
|
|
20656
21173
|
}
|
|
20657
21174
|
var DEFAULT_MAX_EVENTS_PER_EXECUTION = 5e4;
|
|
20658
21175
|
var DEFAULT_STREAMING_BATCH_SIZE = 100;
|
|
21176
|
+
var DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE = 1e3;
|
|
21177
|
+
var LARGE_EXECUTION_LISTING = 1e3;
|
|
21178
|
+
var EXECUTION_SCAN_PAGE_SIZE = 100;
|
|
20659
21179
|
var DEFAULT_STREAMING_BATCH_INTERVAL = 1e3;
|
|
20660
21180
|
var DEFAULT_ARTIFACT_SWEEP_MS = 6e4;
|
|
20661
21181
|
var DEFAULT_ARTIFACT_LEASE_MS = 3e5;
|
|
@@ -20868,6 +21388,78 @@ function assertNoRemovedObservationCallbacks(options) {
|
|
|
20868
21388
|
`runtime.createContext() no longer accepts ${removed.join(", ")}. Observe ctx.events or use runtime.stream() instead. See docs/migration/stream-first-observation.md.`
|
|
20869
21389
|
);
|
|
20870
21390
|
}
|
|
21391
|
+
var RecentRunsCache = class {
|
|
21392
|
+
constructor(capacity) {
|
|
21393
|
+
this.capacity = capacity;
|
|
21394
|
+
}
|
|
21395
|
+
entries = /* @__PURE__ */ new Map();
|
|
21396
|
+
get size() {
|
|
21397
|
+
return this.entries.size;
|
|
21398
|
+
}
|
|
21399
|
+
get(executionId) {
|
|
21400
|
+
const entry = this.entries.get(executionId);
|
|
21401
|
+
if (entry) {
|
|
21402
|
+
this.entries.delete(executionId);
|
|
21403
|
+
this.entries.set(executionId, entry);
|
|
21404
|
+
}
|
|
21405
|
+
return entry;
|
|
21406
|
+
}
|
|
21407
|
+
set(executionId, execution) {
|
|
21408
|
+
this.entries.delete(executionId);
|
|
21409
|
+
this.entries.set(executionId, execution);
|
|
21410
|
+
while (this.entries.size > this.capacity) {
|
|
21411
|
+
const oldest = this.entries.keys().next().value;
|
|
21412
|
+
this.entries.delete(oldest);
|
|
21413
|
+
}
|
|
21414
|
+
}
|
|
21415
|
+
delete(executionId) {
|
|
21416
|
+
return this.entries.delete(executionId);
|
|
21417
|
+
}
|
|
21418
|
+
values() {
|
|
21419
|
+
return [...this.entries.values()];
|
|
21420
|
+
}
|
|
21421
|
+
};
|
|
21422
|
+
function resolveRecentExecutionsCacheSize(requested) {
|
|
21423
|
+
if (requested === void 0) return DEFAULT_RECENT_EXECUTIONS_CACHE_SIZE;
|
|
21424
|
+
if (typeof requested !== "number" || !Number.isInteger(requested) || requested < 0) {
|
|
21425
|
+
throw new AxlError(
|
|
21426
|
+
"INVALID_CONFIG",
|
|
21427
|
+
`config.state.recentExecutionsCacheSize must be a non-negative integer; got ${String(requested)}`
|
|
21428
|
+
);
|
|
21429
|
+
}
|
|
21430
|
+
return requested;
|
|
21431
|
+
}
|
|
21432
|
+
function parseGetExecutionsOptions(options) {
|
|
21433
|
+
if (options === void 0) return {};
|
|
21434
|
+
if (options === null || typeof options !== "object") {
|
|
21435
|
+
throw new AxlError(
|
|
21436
|
+
"INVALID_ARGUMENT",
|
|
21437
|
+
`getExecutions() options must be an object; got ${options === null ? "null" : typeof options}`
|
|
21438
|
+
);
|
|
21439
|
+
}
|
|
21440
|
+
const { limit, cursor } = options;
|
|
21441
|
+
if (limit !== void 0 && limit !== Infinity && !(typeof limit === "number" && Number.isInteger(limit) && limit > 0)) {
|
|
21442
|
+
throw new AxlError(
|
|
21443
|
+
"INVALID_ARGUMENT",
|
|
21444
|
+
`getExecutions() limit must be a positive integer; got ${String(limit)}`
|
|
21445
|
+
);
|
|
21446
|
+
}
|
|
21447
|
+
let parsedCursor;
|
|
21448
|
+
if (cursor !== void 0) {
|
|
21449
|
+
const candidate = cursor;
|
|
21450
|
+
if (candidate === null || typeof candidate !== "object" || typeof candidate.startedAt !== "number" || !Number.isFinite(candidate.startedAt) || typeof candidate.executionId !== "string") {
|
|
21451
|
+
throw new AxlError(
|
|
21452
|
+
"INVALID_ARGUMENT",
|
|
21453
|
+
"getExecutions() cursor must be { startedAt: number; executionId: string }, built from the last item of the previous page"
|
|
21454
|
+
);
|
|
21455
|
+
}
|
|
21456
|
+
parsedCursor = { startedAt: candidate.startedAt, executionId: candidate.executionId };
|
|
21457
|
+
}
|
|
21458
|
+
return {
|
|
21459
|
+
...limit === void 0 || limit === Infinity ? {} : { limit },
|
|
21460
|
+
...parsedCursor ? { cursor: parsedCursor } : {}
|
|
21461
|
+
};
|
|
21462
|
+
}
|
|
20871
21463
|
var costScopeStorage = new AsyncLocalStorage4();
|
|
20872
21464
|
function forwardAbortSignal(external, internal) {
|
|
20873
21465
|
if (!external) return () => {
|
|
@@ -20914,8 +21506,6 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
20914
21506
|
mcpManager;
|
|
20915
21507
|
memoryManager;
|
|
20916
21508
|
spanManager = new NoopSpanManager();
|
|
20917
|
-
historicalExecutions = /* @__PURE__ */ new Map();
|
|
20918
|
-
historicalExecutionsLoadPromise = null;
|
|
20919
21509
|
/** Per-session in-process serialization. Each entry is a chain of pending
|
|
20920
21510
|
* work for that session id — `Session.send` / `Session.stream` await the
|
|
20921
21511
|
* prior entry before reading history, so concurrent calls produce a
|
|
@@ -20951,12 +21541,34 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
20951
21541
|
* a delete from resurrecting the (intentionally-removed) row. Entries are
|
|
20952
21542
|
* removed once the would-be persist has been short-circuited. */
|
|
20953
21543
|
pendingDeletedExecutions = /* @__PURE__ */ new Set();
|
|
20954
|
-
/** In-flight `persistExecution` chains (save → finalize)
|
|
20955
|
-
* awaits these before closing the StateStore so a workflow whose abort
|
|
21544
|
+
/** In-flight `persistExecution` chains (save → finalize) and crash-
|
|
21545
|
+
* recovery saves. `shutdown()` awaits these before closing the StateStore so a workflow whose abort
|
|
20956
21546
|
* triggers a persist-during-shutdown isn't racing a closed connection.
|
|
20957
21547
|
* The Set is mutated by the chain's own `.finally` so entries clear as
|
|
20958
21548
|
* soon as the work settles. */
|
|
20959
21549
|
persistInflight = /* @__PURE__ */ new Set();
|
|
21550
|
+
/** Finished runs whose history save has not settled, keyed by execution
|
|
21551
|
+
* id. Reads serve the snapshot until the save lands, so a run is visible
|
|
21552
|
+
* the moment it finishes (read-your-writes). `deleteExecution()` awaits
|
|
21553
|
+
* the entry before deleting from the store, so a save that lands late
|
|
21554
|
+
* cannot re-create a deleted run. Each entry removes itself when its save
|
|
21555
|
+
* settles (success or failure), but only if it is still the current entry
|
|
21556
|
+
* for that id, so the map holds only in-flight saves. */
|
|
21557
|
+
pendingSaves = /* @__PURE__ */ new Map();
|
|
21558
|
+
/** Recently finished runs, kept only when the store cannot save, list, or
|
|
21559
|
+
* fetch execution history itself (and the cache size is not `0`).
|
|
21560
|
+
* `undefined` for every built-in store: execution history is read through
|
|
21561
|
+
* the store on every call and never cached here. */
|
|
21562
|
+
recentRuns;
|
|
21563
|
+
/** Execution ids `recoverIncompleteStreams()` is working on, each with a
|
|
21564
|
+
* promise that settles when recovery is done with that id.
|
|
21565
|
+
* `deleteExecution()` awaits it, so a recovered record cannot land after
|
|
21566
|
+
* the delete. */
|
|
21567
|
+
recoveringExecutions = /* @__PURE__ */ new Map();
|
|
21568
|
+
/** Execution ids with a `deleteExecution()` in progress, with a count of
|
|
21569
|
+
* concurrent deletes. Recovery skips them: their store sweep removes the
|
|
21570
|
+
* streaming buffer it would otherwise turn back into a record. */
|
|
21571
|
+
deletingExecutions = /* @__PURE__ */ new Map();
|
|
20960
21572
|
/** Resolved diagnostic artifact backend, when `diagnostics.artifacts` is set. */
|
|
20961
21573
|
artifactStore;
|
|
20962
21574
|
artifactLeaseMs = DEFAULT_ARTIFACT_LEASE_MS;
|
|
@@ -20990,6 +21602,13 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
20990
21602
|
this.providerRegistry = new ProviderRegistry();
|
|
20991
21603
|
this.transcriptionProviderRegistry = new TranscriptionProviderRegistry();
|
|
20992
21604
|
this.stateStore = this.createStateStore();
|
|
21605
|
+
const recentRunsSize = resolveRecentExecutionsCacheSize(
|
|
21606
|
+
this.config.state?.recentExecutionsCacheSize
|
|
21607
|
+
);
|
|
21608
|
+
const storeKeepsHistory = Boolean(
|
|
21609
|
+
this.stateStore.saveExecution && this.stateStore.listExecutions
|
|
21610
|
+
);
|
|
21611
|
+
this.recentRuns = !storeKeepsHistory && recentRunsSize > 0 ? new RecentRunsCache(recentRunsSize) : void 0;
|
|
20993
21612
|
const requested = this.config.state?.maxEventsPerExecution;
|
|
20994
21613
|
if (requested === void 0) {
|
|
20995
21614
|
this.maxEventsPerExecution = DEFAULT_MAX_EVENTS_PER_EXECUTION;
|
|
@@ -21395,11 +22014,15 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
21395
22014
|
const storeOption = this.config.state?.store ?? "memory";
|
|
21396
22015
|
if (typeof storeOption !== "string") return storeOption;
|
|
21397
22016
|
switch (storeOption) {
|
|
21398
|
-
case "sqlite":
|
|
21399
|
-
|
|
22017
|
+
case "sqlite": {
|
|
22018
|
+
const create = () => new SQLiteStore(this.config.state?.sqlite?.path ?? "./data/axl.db", {
|
|
22019
|
+
ttls: this.config.state?.sqlite?.ttls
|
|
22020
|
+
});
|
|
22021
|
+
return this.config.diagnostics?.silent ? withDiagnosticsSilenced(create) : create();
|
|
22022
|
+
}
|
|
21400
22023
|
case "memory":
|
|
21401
22024
|
default:
|
|
21402
|
-
return new MemoryStore();
|
|
22025
|
+
return new MemoryStore(this.config.state?.memory);
|
|
21403
22026
|
}
|
|
21404
22027
|
}
|
|
21405
22028
|
/**
|
|
@@ -21451,7 +22074,11 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
21451
22074
|
observation
|
|
21452
22075
|
});
|
|
21453
22076
|
if (this.stateStore.deleteCheckpoints) {
|
|
21454
|
-
await this.stateStore.deleteCheckpoints(execInfo.executionId)
|
|
22077
|
+
await this.stateStore.deleteCheckpoints(execInfo.executionId).catch((err) => {
|
|
22078
|
+
console.error(
|
|
22079
|
+
`[axl] Failed to delete checkpoints for executionId=${execInfo.executionId}: ` + (err instanceof Error ? err.message : String(err))
|
|
22080
|
+
);
|
|
22081
|
+
});
|
|
21455
22082
|
}
|
|
21456
22083
|
span?.setAttribute("axl.workflow.cost", execInfo.totalCost);
|
|
21457
22084
|
span?.setAttribute("axl.workflow.duration", execInfo.duration);
|
|
@@ -21487,13 +22114,15 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
21487
22114
|
}
|
|
21488
22115
|
/**
|
|
21489
22116
|
* Persist a completed/failed execution to the state store (fire-and-forget)
|
|
21490
|
-
* and
|
|
22117
|
+
* and remove it from the active executions map. Until the save settles the
|
|
22118
|
+
* run is served from `pendingSaves`; afterwards from the store (or, for a
|
|
22119
|
+
* store that cannot keep history, from `recentRuns`).
|
|
21491
22120
|
*
|
|
21492
22121
|
* Resurrection guard: if `runtime.deleteExecution(id)` was called while
|
|
21493
22122
|
* this execution was still running, the id will be in
|
|
21494
|
-
* `pendingDeletedExecutions`. In that case we skip the save AND
|
|
21495
|
-
*
|
|
21496
|
-
* the buffer doesn't outlive the delete.
|
|
22123
|
+
* `pendingDeletedExecutions`. In that case we skip the save AND every
|
|
22124
|
+
* in-process record of the run, and finalize the streaming buffer (if any)
|
|
22125
|
+
* so the buffer doesn't outlive the delete.
|
|
21497
22126
|
*/
|
|
21498
22127
|
persistExecution(execInfo) {
|
|
21499
22128
|
const id = execInfo.executionId;
|
|
@@ -21521,17 +22150,25 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
21521
22150
|
const chain = this.stateStore.saveExecution(snapshot).then(() => {
|
|
21522
22151
|
return this.streamingFlusher?.finalize(id).catch(() => {
|
|
21523
22152
|
});
|
|
21524
|
-
}).catch(() => {
|
|
22153
|
+
}).catch((err) => {
|
|
22154
|
+
console.error(
|
|
22155
|
+
`[axl] Failed to save execution history for executionId=${id}: ` + (err instanceof Error ? err.message : String(err))
|
|
22156
|
+
);
|
|
21525
22157
|
}).then(() => void 0);
|
|
21526
22158
|
this.persistInflight.add(chain);
|
|
21527
|
-
|
|
22159
|
+
const pending = { snapshot, settled: chain };
|
|
22160
|
+
this.pendingSaves.set(id, pending);
|
|
22161
|
+
chain.finally(() => {
|
|
22162
|
+
this.persistInflight.delete(chain);
|
|
22163
|
+
if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
|
|
22164
|
+
});
|
|
21528
22165
|
} else if (this.streamingFlusher) {
|
|
21529
22166
|
const chain = this.streamingFlusher.finalize(id).catch(() => {
|
|
21530
22167
|
});
|
|
21531
22168
|
this.persistInflight.add(chain);
|
|
21532
22169
|
chain.finally(() => this.persistInflight.delete(chain));
|
|
21533
22170
|
}
|
|
21534
|
-
this.
|
|
22171
|
+
this.recentRuns?.set(id, snapshot);
|
|
21535
22172
|
this.executions.delete(id);
|
|
21536
22173
|
}
|
|
21537
22174
|
/** Register a workflow with the runtime. */
|
|
@@ -21657,7 +22294,6 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
21657
22294
|
}
|
|
21658
22295
|
return result;
|
|
21659
22296
|
}
|
|
21660
|
-
/** Get all execution info (running + completed + historical). */
|
|
21661
22297
|
/** Normalize an execution loaded from a `StateStore` so the
|
|
21662
22298
|
* historical event union holds for downstream consumers
|
|
21663
22299
|
* (Studio aggregators, REST routes, redaction). Custom or legacy stores
|
|
@@ -21677,27 +22313,78 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
21677
22313
|
}
|
|
21678
22314
|
return normalizeStoredExecution(exec);
|
|
21679
22315
|
}
|
|
21680
|
-
|
|
21681
|
-
|
|
21682
|
-
|
|
21683
|
-
|
|
21684
|
-
|
|
21685
|
-
|
|
21686
|
-
|
|
21687
|
-
|
|
21688
|
-
|
|
21689
|
-
|
|
21690
|
-
|
|
21691
|
-
|
|
21692
|
-
|
|
21693
|
-
|
|
21694
|
-
|
|
21695
|
-
|
|
21696
|
-
|
|
21697
|
-
|
|
21698
|
-
|
|
22316
|
+
/**
|
|
22317
|
+
* List executions, newest first: running runs, runs whose save is still in
|
|
22318
|
+
* flight, and the history the state store holds. Ordered by `startedAt`,
|
|
22319
|
+
* then `executionId`, both descending.
|
|
22320
|
+
*
|
|
22321
|
+
* With no options, returns every retained run. Pass `limit` for one page,
|
|
22322
|
+
* and `cursor` (built from the last item of the previous page) for the
|
|
22323
|
+
* page after it. Each call reads through the store, so runs finished or
|
|
22324
|
+
* deleted by other runtimes sharing the store are reflected immediately.
|
|
22325
|
+
* A call without `limit` that returns more than 1 000 runs warns once per
|
|
22326
|
+
* process (silenced by `diagnostics.silent`); pass `limit: Infinity` to ask
|
|
22327
|
+
* for everything without the warning.
|
|
22328
|
+
*
|
|
22329
|
+
* @throws {AxlError} `INVALID_ARGUMENT` for a `limit` that is not a positive
|
|
22330
|
+
* integer (`Infinity` means no limit) or a malformed `cursor`.
|
|
22331
|
+
*/
|
|
22332
|
+
async getExecutions(options) {
|
|
22333
|
+
const { limit, cursor } = parseGetExecutionsOptions(options);
|
|
22334
|
+
const isAfterCursor = (exec) => cursor === void 0 || compareExecutionsNewestFirst(exec, cursor) > 0;
|
|
22335
|
+
const live = new Map(this.executions);
|
|
22336
|
+
for (const [id, { snapshot }] of this.pendingSaves) {
|
|
22337
|
+
if (!live.has(id)) live.set(id, snapshot);
|
|
22338
|
+
}
|
|
22339
|
+
const recent = this.recentRunsServeListing() ? this.recentRuns.values() : [];
|
|
22340
|
+
const stored = await this.listStoredExecutions({
|
|
22341
|
+
limit,
|
|
22342
|
+
cursor,
|
|
22343
|
+
overlap: live.size + recent.length
|
|
22344
|
+
});
|
|
22345
|
+
const merged = /* @__PURE__ */ new Map();
|
|
22346
|
+
for (const [id, exec] of live) {
|
|
22347
|
+
if (isAfterCursor(exec)) merged.set(id, exec);
|
|
22348
|
+
}
|
|
22349
|
+
for (const exec of stored) {
|
|
22350
|
+
const id = exec.executionId;
|
|
22351
|
+
if (!live.has(id) && !merged.has(id)) merged.set(id, exec);
|
|
22352
|
+
}
|
|
22353
|
+
for (const exec of recent) {
|
|
22354
|
+
const id = exec.executionId;
|
|
22355
|
+
if (!live.has(id) && !merged.has(id) && isAfterCursor(exec)) merged.set(id, exec);
|
|
22356
|
+
}
|
|
22357
|
+
const sorted = [...merged.values()].sort(compareExecutionsNewestFirst);
|
|
22358
|
+
if (limit !== void 0) return sorted.slice(0, limit);
|
|
22359
|
+
if (options?.limit === void 0 && sorted.length > LARGE_EXECUTION_LISTING) {
|
|
22360
|
+
warnDiagnosticOnce(
|
|
22361
|
+
"large_execution_listing",
|
|
22362
|
+
`runtime.getExecutions() without a limit returned ${sorted.length} runs and reads the whole history on every call; pass { limit, cursor } to read it a page at a time.`,
|
|
22363
|
+
this.config.diagnostics?.silent,
|
|
22364
|
+
"docs/migration/execution-history-retention.md#page-through-large-histories"
|
|
22365
|
+
);
|
|
21699
22366
|
}
|
|
21700
|
-
return
|
|
22367
|
+
return sorted;
|
|
22368
|
+
}
|
|
22369
|
+
/**
|
|
22370
|
+
* Read one page of execution history from the store: the records strictly
|
|
22371
|
+
* after `cursor`, newest first, at least `limit + overlap` of them when the
|
|
22372
|
+
* store holds that many (all of them when `limit` is unset).
|
|
22373
|
+
*
|
|
22374
|
+
* Only a store that declares `historyPaging` is handed the cursor. Any
|
|
22375
|
+
* other store is listed in full and paged here: filtering the output of a
|
|
22376
|
+
* store that ignored the cursor would return its first page again, and an
|
|
22377
|
+
* empty page after that.
|
|
22378
|
+
*/
|
|
22379
|
+
async listStoredExecutions(page) {
|
|
22380
|
+
const store = this.stateStore;
|
|
22381
|
+
if (!store.listExecutions) return [];
|
|
22382
|
+
const wanted = page.limit === void 0 ? void 0 : page.limit + page.overlap;
|
|
22383
|
+
const rows = store.historyPaging === true ? await store.listExecutions(wanted, { cursor: page.cursor }) : await store.listExecutions();
|
|
22384
|
+
const cursor = page.cursor;
|
|
22385
|
+
const after = cursor === void 0 ? rows : rows.filter((row) => compareExecutionsNewestFirst(row, cursor) > 0);
|
|
22386
|
+
const window = wanted === void 0 ? after : [...after].sort(compareExecutionsNewestFirst).slice(0, wanted);
|
|
22387
|
+
return window.map((row) => this.normalizeStoredExecution(row));
|
|
21701
22388
|
}
|
|
21702
22389
|
/**
|
|
21703
22390
|
* Create a WorkflowContext for ad-hoc use (evals, tool testing, prototyping).
|
|
@@ -22128,20 +22815,25 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
22128
22815
|
* Recover executions whose process died mid-run when `state.persist:
|
|
22129
22816
|
* 'streaming'` was configured. Synthesizes a partial `ExecutionInfo`
|
|
22130
22817
|
* from the streaming buffer for each — `status: 'failed'`, error
|
|
22131
|
-
* `'process terminated'` —
|
|
22132
|
-
* they show up in `getExecutions()
|
|
22133
|
-
*
|
|
22818
|
+
* `'process terminated'` — persists them via the store's
|
|
22819
|
+
* `saveExecution` (so they show up in `getExecutions()`), and finalizes
|
|
22820
|
+
* the streaming buffer.
|
|
22134
22821
|
*
|
|
22135
22822
|
* Idempotent — if the streaming buffer is already gone (or
|
|
22136
22823
|
* `listStreamingExecutions` is empty), this is a no-op. Returns the
|
|
22137
22824
|
* list of recovered `ExecutionInfo`s for caller introspection.
|
|
22138
22825
|
*
|
|
22139
|
-
*
|
|
22140
|
-
*
|
|
22141
|
-
*
|
|
22142
|
-
*
|
|
22143
|
-
*
|
|
22144
|
-
*
|
|
22826
|
+
* A run whose final record already exists is never recovered: for each
|
|
22827
|
+
* buffered id this asks the store's `getExecution(id)` and, when a record
|
|
22828
|
+
* is there, only drops the leftover buffer. It does not list history, so
|
|
22829
|
+
* its cost does not grow with the size of the store. A store without
|
|
22830
|
+
* `getExecution` cannot be checked, and every buffered run is recovered.
|
|
22831
|
+
*
|
|
22832
|
+
* Wire this into process startup, before the process accepts new work:
|
|
22833
|
+
* a run live in another process looks the same as a crashed one.
|
|
22834
|
+
* `deleteExecution(id)` called while recovery is working on that id waits
|
|
22835
|
+
* for recovery to finish with it, so the recovered record cannot outlive
|
|
22836
|
+
* the delete.
|
|
22145
22837
|
*/
|
|
22146
22838
|
async recoverIncompleteStreams() {
|
|
22147
22839
|
if (!this.stateStore.listStreamingExecutions || !this.stateStore.getStreamingEvents) {
|
|
@@ -22149,73 +22841,119 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
22149
22841
|
}
|
|
22150
22842
|
const ids = await this.stateStore.listStreamingExecutions();
|
|
22151
22843
|
if (ids.length === 0) return [];
|
|
22152
|
-
await this.getExecutions();
|
|
22153
22844
|
const recovered = [];
|
|
22154
22845
|
for (const id of ids) {
|
|
22155
22846
|
if (this.executions.has(id) || this.streamableExecutionIds.has(id)) {
|
|
22156
22847
|
continue;
|
|
22157
22848
|
}
|
|
22158
|
-
|
|
22159
|
-
|
|
22160
|
-
|
|
22161
|
-
|
|
22162
|
-
|
|
22163
|
-
|
|
22164
|
-
|
|
22165
|
-
|
|
22166
|
-
|
|
22167
|
-
|
|
22168
|
-
|
|
22169
|
-
|
|
22170
|
-
|
|
22171
|
-
|
|
22172
|
-
}
|
|
22849
|
+
if (this.deletingExecutions.has(id) || this.recoveringExecutions.has(id)) continue;
|
|
22850
|
+
let finishRecovery;
|
|
22851
|
+
this.recoveringExecutions.set(
|
|
22852
|
+
id,
|
|
22853
|
+
new Promise((resolve2) => {
|
|
22854
|
+
finishRecovery = resolve2;
|
|
22855
|
+
})
|
|
22856
|
+
);
|
|
22857
|
+
try {
|
|
22858
|
+
const result = await this.recoverStreamingExecution(id);
|
|
22859
|
+
if (result) recovered.push(result);
|
|
22860
|
+
} finally {
|
|
22861
|
+
this.recoveringExecutions.delete(id);
|
|
22862
|
+
finishRecovery();
|
|
22173
22863
|
}
|
|
22174
|
-
|
|
22175
|
-
|
|
22176
|
-
|
|
22177
|
-
|
|
22178
|
-
|
|
22179
|
-
|
|
22180
|
-
|
|
22181
|
-
|
|
22182
|
-
|
|
22183
|
-
|
|
22184
|
-
|
|
22185
|
-
|
|
22186
|
-
|
|
22187
|
-
|
|
22188
|
-
|
|
22189
|
-
|
|
22190
|
-
|
|
22191
|
-
|
|
22192
|
-
|
|
22193
|
-
|
|
22194
|
-
|
|
22195
|
-
|
|
22196
|
-
|
|
22197
|
-
|
|
22198
|
-
|
|
22199
|
-
|
|
22200
|
-
|
|
22201
|
-
|
|
22202
|
-
|
|
22864
|
+
}
|
|
22865
|
+
return recovered;
|
|
22866
|
+
}
|
|
22867
|
+
/** Recover one buffered execution. Returns the synthesized record, or
|
|
22868
|
+
* `undefined` when there was nothing to recover. */
|
|
22869
|
+
async recoverStreamingExecution(id) {
|
|
22870
|
+
if (!this.stateStore.getStreamingEvents) return void 0;
|
|
22871
|
+
const existing = await this.stateStore.getExecution?.(id);
|
|
22872
|
+
if (existing) {
|
|
22873
|
+
await this.stateStore.finalizeStreamingEvents?.(id);
|
|
22874
|
+
return void 0;
|
|
22875
|
+
}
|
|
22876
|
+
const events = await this.stateStore.getStreamingEvents(id);
|
|
22877
|
+
if (events.length === 0) {
|
|
22878
|
+
await this.stateStore.finalizeStreamingEvents?.(id);
|
|
22879
|
+
return void 0;
|
|
22880
|
+
}
|
|
22881
|
+
const eventSchemaVersion = getEventSchemaVersion(events[0]);
|
|
22882
|
+
for (const event of events) {
|
|
22883
|
+
if (getEventSchemaVersion(event) !== eventSchemaVersion) {
|
|
22884
|
+
throw new Error(`Streaming buffer ${id} contains mixed event schema versions`);
|
|
22885
|
+
}
|
|
22886
|
+
}
|
|
22887
|
+
const boundedEvents = boundRecoveredEvents(events, this.maxEventsPerExecution);
|
|
22888
|
+
const workflowStartEvent = boundedEvents.find((e) => e.type === "workflow_start");
|
|
22889
|
+
const lastEvent = boundedEvents[boundedEvents.length - 1];
|
|
22890
|
+
const startedAt = workflowStartEvent?.timestamp ?? boundedEvents[0]?.timestamp ?? Date.now();
|
|
22891
|
+
const completedAt = lastEvent.timestamp ?? startedAt;
|
|
22892
|
+
const workflowName = workflowStartEvent?.workflow ?? RECOVERED_UNKNOWN_WORKFLOW;
|
|
22893
|
+
const totalCost = boundedEvents.reduce((sum, e) => sum + eventCostContribution(e), 0);
|
|
22894
|
+
const unpriced = boundedEvents.some(isUnpricedLeaf);
|
|
22895
|
+
const executionBase = {
|
|
22896
|
+
executionId: id,
|
|
22897
|
+
workflow: workflowName,
|
|
22898
|
+
status: "failed",
|
|
22899
|
+
events: boundedEvents,
|
|
22900
|
+
totalCost,
|
|
22901
|
+
unpriced,
|
|
22902
|
+
startedAt,
|
|
22903
|
+
completedAt,
|
|
22904
|
+
duration: completedAt - startedAt,
|
|
22905
|
+
error: "process terminated (recovered from streaming buffer)"
|
|
22906
|
+
};
|
|
22907
|
+
const synthesized = eventSchemaVersion === 2 ? {
|
|
22908
|
+
...executionBase,
|
|
22909
|
+
eventSchemaVersion: 2,
|
|
22910
|
+
events: boundedEvents,
|
|
22911
|
+
observation: { complete: false, reason: "process_interrupted" }
|
|
22912
|
+
} : {
|
|
22913
|
+
...executionBase,
|
|
22914
|
+
eventSchemaVersion: 1,
|
|
22915
|
+
events: boundedEvents
|
|
22916
|
+
};
|
|
22917
|
+
const saveExecution = this.stateStore.saveExecution;
|
|
22918
|
+
if (saveExecution) {
|
|
22919
|
+
const save = (async () => saveExecution.call(this.stateStore, synthesized))();
|
|
22920
|
+
const pending = {
|
|
22921
|
+
snapshot: synthesized,
|
|
22922
|
+
settled: save.then(
|
|
22923
|
+
() => void 0,
|
|
22924
|
+
() => void 0
|
|
22925
|
+
)
|
|
22203
22926
|
};
|
|
22204
|
-
|
|
22205
|
-
|
|
22206
|
-
|
|
22207
|
-
|
|
22208
|
-
|
|
22209
|
-
|
|
22210
|
-
)
|
|
22211
|
-
|
|
22212
|
-
|
|
22927
|
+
this.pendingSaves.set(id, pending);
|
|
22928
|
+
this.persistInflight.add(pending.settled);
|
|
22929
|
+
try {
|
|
22930
|
+
await save;
|
|
22931
|
+
} catch (err) {
|
|
22932
|
+
console.error(
|
|
22933
|
+
`[axl] recoverIncompleteStreams: failed to save synthesized execution ${id}: ${err instanceof Error ? err.message : String(err)}. Streaming buffer left in place for next recovery attempt.`
|
|
22934
|
+
);
|
|
22935
|
+
return void 0;
|
|
22936
|
+
} finally {
|
|
22937
|
+
this.persistInflight.delete(pending.settled);
|
|
22938
|
+
if (this.pendingSaves.get(id) === pending) this.pendingSaves.delete(id);
|
|
22213
22939
|
}
|
|
22214
|
-
this.historicalExecutions.set(id, synthesized);
|
|
22215
|
-
await this.stateStore.finalizeStreamingEvents?.(id);
|
|
22216
|
-
recovered.push(synthesized);
|
|
22217
22940
|
}
|
|
22218
|
-
|
|
22941
|
+
this.recentRuns?.set(id, synthesized);
|
|
22942
|
+
const finalize = Promise.resolve(this.stateStore.finalizeStreamingEvents?.(id)).then(
|
|
22943
|
+
() => void 0,
|
|
22944
|
+
(err) => {
|
|
22945
|
+
console.error(
|
|
22946
|
+
`[axl] recoverIncompleteStreams: saved execution ${id} but failed to drop its streaming buffer: ${err instanceof Error ? err.message : String(err)}. The next recovery removes it.`
|
|
22947
|
+
);
|
|
22948
|
+
}
|
|
22949
|
+
);
|
|
22950
|
+
this.persistInflight.add(finalize);
|
|
22951
|
+
try {
|
|
22952
|
+
await finalize;
|
|
22953
|
+
} finally {
|
|
22954
|
+
this.persistInflight.delete(finalize);
|
|
22955
|
+
}
|
|
22956
|
+
return synthesized;
|
|
22219
22957
|
}
|
|
22220
22958
|
/** Abort a running execution by its ID. */
|
|
22221
22959
|
abort(executionId) {
|
|
@@ -22225,21 +22963,57 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
22225
22963
|
this.abortControllers.delete(executionId);
|
|
22226
22964
|
}
|
|
22227
22965
|
}
|
|
22228
|
-
/**
|
|
22966
|
+
/**
|
|
22967
|
+
* Get execution details by ID. Looks in order at running runs, runs whose
|
|
22968
|
+
* save is still in flight, the state store (its `getExecution`, or a scan
|
|
22969
|
+
* of its listing when it has `listExecutions` but no `getExecution`), and
|
|
22970
|
+
* (only for a store that cannot list or save history) the recent-runs
|
|
22971
|
+
* cache. Nothing read from the store is cached, so a run deleted or
|
|
22972
|
+
* expired elsewhere is gone on the next call, and every run
|
|
22973
|
+
* `getExecutions()` lists can be fetched here.
|
|
22974
|
+
*/
|
|
22229
22975
|
async getExecution(executionId) {
|
|
22230
|
-
const
|
|
22231
|
-
if (
|
|
22232
|
-
const
|
|
22233
|
-
if (
|
|
22234
|
-
|
|
22235
|
-
|
|
22236
|
-
|
|
22237
|
-
|
|
22238
|
-
|
|
22239
|
-
|
|
22240
|
-
|
|
22976
|
+
const active = this.executions.get(executionId);
|
|
22977
|
+
if (active) return active;
|
|
22978
|
+
const pending = this.pendingSaves.get(executionId);
|
|
22979
|
+
if (pending) return pending.snapshot;
|
|
22980
|
+
const stored = this.stateStore.getExecution ? await this.stateStore.getExecution(executionId) : await this.findInStoreListing(executionId);
|
|
22981
|
+
if (stored) return this.normalizeStoredExecution(stored);
|
|
22982
|
+
return this.recentRunsServeReads() ? this.recentRuns.get(executionId) : void 0;
|
|
22983
|
+
}
|
|
22984
|
+
/**
|
|
22985
|
+
* By-id read for a store that lists but cannot fetch by id. A paging store
|
|
22986
|
+
* is scanned page by page with the cursor, stopping at the match; any
|
|
22987
|
+
* other store is listed once in full. Returns `null` when the store cannot
|
|
22988
|
+
* list or does not hold the run.
|
|
22989
|
+
*/
|
|
22990
|
+
async findInStoreListing(executionId) {
|
|
22991
|
+
const store = this.stateStore;
|
|
22992
|
+
if (!store.listExecutions) return null;
|
|
22993
|
+
if (store.historyPaging !== true) {
|
|
22994
|
+
return (await store.listExecutions()).find((e) => e.executionId === executionId) ?? null;
|
|
22241
22995
|
}
|
|
22242
|
-
|
|
22996
|
+
let cursor;
|
|
22997
|
+
for (; ; ) {
|
|
22998
|
+
const page = await store.listExecutions(EXECUTION_SCAN_PAGE_SIZE, { cursor });
|
|
22999
|
+
const found = page.find((e) => e.executionId === executionId);
|
|
23000
|
+
if (found) return found;
|
|
23001
|
+
if (page.length === 0) return null;
|
|
23002
|
+
const last = [...page].sort(compareExecutionsNewestFirst)[page.length - 1];
|
|
23003
|
+
if (cursor && compareExecutionsNewestFirst(last, cursor) <= 0) return null;
|
|
23004
|
+
cursor = { startedAt: last.startedAt, executionId: last.executionId };
|
|
23005
|
+
}
|
|
23006
|
+
}
|
|
23007
|
+
/** The recent-runs cache stands in for listing when the store cannot list
|
|
23008
|
+
* runs, or never receives them because it cannot save. */
|
|
23009
|
+
recentRunsServeListing() {
|
|
23010
|
+
return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.listExecutions);
|
|
23011
|
+
}
|
|
23012
|
+
/** The recent-runs cache stands in for by-id reads when the store cannot
|
|
23013
|
+
* fetch or list runs (it exists only then), or never receives them because
|
|
23014
|
+
* it cannot save. */
|
|
23015
|
+
recentRunsServeReads() {
|
|
23016
|
+
return this.recentRuns !== void 0 && (!this.stateStore.saveExecution || !this.stateStore.getExecution);
|
|
22243
23017
|
}
|
|
22244
23018
|
/** Save an eval result to history. */
|
|
22245
23019
|
async saveEvalResult(entry) {
|
|
@@ -22272,11 +23046,16 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
22272
23046
|
}
|
|
22273
23047
|
/**
|
|
22274
23048
|
* Delete an execution from history by id. Removes from the active map (if
|
|
22275
|
-
* still running), the
|
|
23049
|
+
* still running), the recent-runs cache, the configured StateStore, AND
|
|
22276
23050
|
* any execution-scoped side state — pending awaitHuman decisions,
|
|
22277
23051
|
* resolver maps, streaming buffer registration, in-flight abort controller.
|
|
22278
|
-
* Returns true if
|
|
22279
|
-
*
|
|
23052
|
+
* Returns true if the run was running, had a save in flight, or was
|
|
23053
|
+
* removed from the recent-runs cache or the store.
|
|
23054
|
+
*
|
|
23055
|
+
* If the run has already finished but its history save has not landed
|
|
23056
|
+
* yet, this method waits for that save (whether it succeeds or fails)
|
|
23057
|
+
* before deleting, so the save cannot re-create the deleted record. It
|
|
23058
|
+
* likewise waits for `recoverIncompleteStreams()` to finish with the id.
|
|
22280
23059
|
*
|
|
22281
23060
|
* In-flight handling: if the execution is still running, this method
|
|
22282
23061
|
* **aborts** it (via the registered abort controller) AND marks the id for
|
|
@@ -22301,7 +23080,18 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
22301
23080
|
* (`defaultTtl` / `ttls.executionHistory`).
|
|
22302
23081
|
*/
|
|
22303
23082
|
async deleteExecution(id) {
|
|
23083
|
+
this.deletingExecutions.set(id, (this.deletingExecutions.get(id) ?? 0) + 1);
|
|
23084
|
+
try {
|
|
23085
|
+
return await this.deleteExecutionWhileMarked(id);
|
|
23086
|
+
} finally {
|
|
23087
|
+
const remaining = (this.deletingExecutions.get(id) ?? 1) - 1;
|
|
23088
|
+
if (remaining > 0) this.deletingExecutions.set(id, remaining);
|
|
23089
|
+
else this.deletingExecutions.delete(id);
|
|
23090
|
+
}
|
|
23091
|
+
}
|
|
23092
|
+
async deleteExecutionWhileMarked(id) {
|
|
22304
23093
|
const wasActive = this.executions.has(id);
|
|
23094
|
+
const hadPendingSave = this.pendingSaves.has(id);
|
|
22305
23095
|
const pendingDecisionResolver = this.pendingDecisionResolvers.get(id);
|
|
22306
23096
|
const hadPendingDecision = pendingDecisionResolver !== void 0;
|
|
22307
23097
|
if (wasActive) {
|
|
@@ -22319,22 +23109,25 @@ var AxlRuntime = class extends EventEmitter2 {
|
|
|
22319
23109
|
this.pendingDecisionResolvers.delete(id);
|
|
22320
23110
|
}
|
|
22321
23111
|
}
|
|
23112
|
+
await this.pendingSaves.get(id)?.settled;
|
|
23113
|
+
await this.recoveringExecutions.get(id);
|
|
22322
23114
|
const existing = await this.getExecution(id);
|
|
22323
23115
|
const workflow2 = existing?.workflow;
|
|
22324
23116
|
const removedFromActive = this.executions.delete(id);
|
|
22325
|
-
const
|
|
23117
|
+
const removedFromRecent = this.recentRuns?.delete(id) ?? false;
|
|
22326
23118
|
let removedFromStore = false;
|
|
22327
23119
|
if (this.stateStore.deleteExecution) {
|
|
22328
23120
|
removedFromStore = await this.stateStore.deleteExecution(id);
|
|
22329
23121
|
}
|
|
23122
|
+
const removed = removedFromActive || hadPendingSave || removedFromRecent || removedFromStore;
|
|
22330
23123
|
this.emit("execution_deleted", {
|
|
22331
23124
|
executionId: id,
|
|
22332
23125
|
workflow: workflow2,
|
|
22333
23126
|
wasActive,
|
|
22334
23127
|
hadPendingDecision,
|
|
22335
|
-
removed
|
|
23128
|
+
removed
|
|
22336
23129
|
});
|
|
22337
|
-
return
|
|
23130
|
+
return removed;
|
|
22338
23131
|
}
|
|
22339
23132
|
/**
|
|
22340
23133
|
* Delete an eval history entry by id. Removes from in-memory cache and
|
|
@@ -23752,12 +24545,153 @@ async function* stringStreamFromEvents(source, opts) {
|
|
|
23752
24545
|
}
|
|
23753
24546
|
|
|
23754
24547
|
// src/state/redis.ts
|
|
24548
|
+
var PRUNE_HISTORY_INDEX_SCRIPT = `
|
|
24549
|
+
local now = tonumber(ARGV[1])
|
|
24550
|
+
local removed = 0
|
|
24551
|
+
for i = 2, #ARGV do
|
|
24552
|
+
local id = ARGV[i]
|
|
24553
|
+
local pttl = redis.call('PTTL', KEYS[i + 2])
|
|
24554
|
+
if pttl == -2 then
|
|
24555
|
+
redis.call('SREM', KEYS[1], id)
|
|
24556
|
+
redis.call('ZREM', KEYS[2], id)
|
|
24557
|
+
redis.call('ZREM', KEYS[3], id)
|
|
24558
|
+
removed = removed + 1
|
|
24559
|
+
elseif pttl == -1 then
|
|
24560
|
+
redis.call('ZREM', KEYS[3], id)
|
|
24561
|
+
else
|
|
24562
|
+
redis.call('ZADD', KEYS[3], string.format('%d', now + pttl), id)
|
|
24563
|
+
end
|
|
24564
|
+
end
|
|
24565
|
+
return removed
|
|
24566
|
+
`;
|
|
24567
|
+
function compareMembers(a, b) {
|
|
24568
|
+
return Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8"));
|
|
24569
|
+
}
|
|
24570
|
+
function compareNewestFirst(a, b) {
|
|
24571
|
+
if (a.score !== b.score) return b.score - a.score;
|
|
24572
|
+
return compareMembers(b.id, a.id);
|
|
24573
|
+
}
|
|
24574
|
+
function warnIfRedisExecutionRetentionUnset(opts) {
|
|
24575
|
+
if (opts.ttls?.executionHistory === void 0 && typeof opts.defaultTtl !== "number") {
|
|
24576
|
+
warnExecutionRetentionUnset("RedisStore");
|
|
24577
|
+
}
|
|
24578
|
+
}
|
|
24579
|
+
var CLOSE_TIMEOUT_MS = 2e3;
|
|
24580
|
+
function reconnectDelayMs(retries) {
|
|
24581
|
+
return Math.min(2 ** retries * 50, 2e3) + Math.floor(Math.random() * 200);
|
|
24582
|
+
}
|
|
24583
|
+
function abandonConnection(client) {
|
|
24584
|
+
try {
|
|
24585
|
+
client.destroy?.();
|
|
24586
|
+
} catch {
|
|
24587
|
+
}
|
|
24588
|
+
client.unref?.();
|
|
24589
|
+
}
|
|
24590
|
+
function errorMessage(error) {
|
|
24591
|
+
return error instanceof Error ? error.message : String(error);
|
|
24592
|
+
}
|
|
24593
|
+
function connectionEventReporter(client, onConnectionEvent) {
|
|
24594
|
+
if (onConnectionEvent) {
|
|
24595
|
+
const reportFailure = (error) => console.error(`[axl] RedisStore: onConnectionEvent failed: ${errorMessage(error)}`);
|
|
24596
|
+
return (event) => {
|
|
24597
|
+
try {
|
|
24598
|
+
Promise.resolve(onConnectionEvent(event)).catch(reportFailure);
|
|
24599
|
+
} catch (error) {
|
|
24600
|
+
reportFailure(error);
|
|
24601
|
+
}
|
|
24602
|
+
};
|
|
24603
|
+
}
|
|
24604
|
+
let outageLogged = false;
|
|
24605
|
+
return (event) => {
|
|
24606
|
+
if (event.type === "reconnected") {
|
|
24607
|
+
outageLogged = false;
|
|
24608
|
+
console.warn("[axl] RedisStore: reconnected to Redis.");
|
|
24609
|
+
} else if (!outageLogged) {
|
|
24610
|
+
outageLogged = client.isReady === false;
|
|
24611
|
+
console.warn(
|
|
24612
|
+
`[axl] RedisStore: Redis connection lost (${event.error.message}). Store calls fail until it reconnects, which happens automatically.`
|
|
24613
|
+
);
|
|
24614
|
+
}
|
|
24615
|
+
};
|
|
24616
|
+
}
|
|
24617
|
+
async function connectClient(createClient, opts) {
|
|
24618
|
+
let established = false;
|
|
24619
|
+
const client = createClient({
|
|
24620
|
+
...opts.url ? { url: opts.url } : {},
|
|
24621
|
+
// A command accepted just before a drop rejects instead of waiting for
|
|
24622
|
+
// the reconnect.
|
|
24623
|
+
disableOfflineQueue: true,
|
|
24624
|
+
socket: { reconnectStrategy: (retries) => established ? reconnectDelayMs(retries) : false }
|
|
24625
|
+
});
|
|
24626
|
+
const reportConnectionEvent = connectionEventReporter(client, opts.onConnectionEvent);
|
|
24627
|
+
client.on("error", (error) => {
|
|
24628
|
+
if (!established || client.isOpen === false) return;
|
|
24629
|
+
reportConnectionEvent({
|
|
24630
|
+
type: "error",
|
|
24631
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
24632
|
+
});
|
|
24633
|
+
});
|
|
24634
|
+
client.on("connect", () => {
|
|
24635
|
+
if (client.isOpen === false) client.unref?.();
|
|
24636
|
+
});
|
|
24637
|
+
client.on("ready", () => {
|
|
24638
|
+
if (!established) {
|
|
24639
|
+
established = true;
|
|
24640
|
+
} else if (client.isOpen !== false) {
|
|
24641
|
+
reportConnectionEvent({ type: "reconnected" });
|
|
24642
|
+
}
|
|
24643
|
+
});
|
|
24644
|
+
await client.connect();
|
|
24645
|
+
return client;
|
|
24646
|
+
}
|
|
23755
24647
|
var RedisStore = class _RedisStore {
|
|
23756
|
-
constructor(
|
|
23757
|
-
this.
|
|
24648
|
+
constructor(connection, keyPrefix, ttls) {
|
|
24649
|
+
this.connection = connection;
|
|
23758
24650
|
this.keyPrefix = keyPrefix;
|
|
23759
24651
|
this.ttls = ttls;
|
|
23760
24652
|
}
|
|
24653
|
+
/**
|
|
24654
|
+
* Whether the store can send commands right now, for a health check or for
|
|
24655
|
+
* telling an outage from other failures when a call rejects. `false` while
|
|
24656
|
+
* reconnecting and after `close()`.
|
|
24657
|
+
*/
|
|
24658
|
+
get isConnected() {
|
|
24659
|
+
return this.connection.isOpen !== false && this.connection.isReady !== false;
|
|
24660
|
+
}
|
|
24661
|
+
/**
|
|
24662
|
+
* The client for sending a command, failing immediately while disconnected.
|
|
24663
|
+
*
|
|
24664
|
+
* Every command goes through here because node-redis's `disableOfflineQueue`
|
|
24665
|
+
* rejects single commands but not `multi().exec()`, which would wait for a
|
|
24666
|
+
* reconnect attempt to fail or succeed. Nothing may `await` between
|
|
24667
|
+
* `client.multi()` and `.exec()`, or a transaction could get past this check.
|
|
24668
|
+
*
|
|
24669
|
+
* A closed store isn't "unavailable": it falls through to node-redis's own
|
|
24670
|
+
* "The client is closed" rejection.
|
|
24671
|
+
*/
|
|
24672
|
+
get client() {
|
|
24673
|
+
if (this.connection.isOpen !== false && this.connection.isReady === false) {
|
|
24674
|
+
throw new AxlError(
|
|
24675
|
+
"REDIS_UNAVAILABLE",
|
|
24676
|
+
"RedisStore: not connected to Redis, so the command was not sent. The store reconnects automatically; retry once Redis is back."
|
|
24677
|
+
);
|
|
24678
|
+
}
|
|
24679
|
+
return this.connection;
|
|
24680
|
+
}
|
|
24681
|
+
/**
|
|
24682
|
+
* History kinds with a prune running in this process. A save skips its
|
|
24683
|
+
* prune while another one of the same kind is in flight: the running one
|
|
24684
|
+
* already covers the oldest expired entries. Created lazily so the
|
|
24685
|
+
* instance needs no constructor state.
|
|
24686
|
+
*/
|
|
24687
|
+
prunesInFlight;
|
|
24688
|
+
/**
|
|
24689
|
+
* Supports `listExecutions(limit, { cursor })`. A getter rather than a
|
|
24690
|
+
* field so it lives on the prototype.
|
|
24691
|
+
*/
|
|
24692
|
+
get historyPaging() {
|
|
24693
|
+
return true;
|
|
24694
|
+
}
|
|
23761
24695
|
/**
|
|
23762
24696
|
* Resolve the TTL for a given storage category. Returns `undefined` when
|
|
23763
24697
|
* no TTL applies (category is `null` in resolved config, the value is
|
|
@@ -23813,8 +24747,7 @@ var RedisStore = class _RedisStore {
|
|
|
23813
24747
|
if (err instanceof Error && err.message.includes("createClient")) throw err;
|
|
23814
24748
|
throw new Error("redis is required for RedisStore. Install it with: npm install redis");
|
|
23815
24749
|
}
|
|
23816
|
-
const client =
|
|
23817
|
-
await client.connect();
|
|
24750
|
+
const client = await connectClient(createClient, opts);
|
|
23818
24751
|
const resolveCategory = (override) => {
|
|
23819
24752
|
if (override === null) return null;
|
|
23820
24753
|
if (typeof override === "number") return override;
|
|
@@ -23836,9 +24769,15 @@ var RedisStore = class _RedisStore {
|
|
|
23836
24769
|
streamingEvents: resolveStreamingEvents(opts.ttls?.streamingEvents)
|
|
23837
24770
|
};
|
|
23838
24771
|
const store = new _RedisStore(client, keyPrefix, effectiveTtls);
|
|
24772
|
+
warnIfRedisExecutionRetentionUnset(opts);
|
|
23839
24773
|
if (!opts.skipMigration) {
|
|
23840
|
-
|
|
23841
|
-
|
|
24774
|
+
try {
|
|
24775
|
+
await store.backfillExecutionIndex();
|
|
24776
|
+
await store.backfillEvalIndex();
|
|
24777
|
+
} catch (error) {
|
|
24778
|
+
abandonConnection(client);
|
|
24779
|
+
throw error;
|
|
24780
|
+
}
|
|
23842
24781
|
}
|
|
23843
24782
|
return store;
|
|
23844
24783
|
}
|
|
@@ -23947,13 +24886,9 @@ var RedisStore = class _RedisStore {
|
|
|
23947
24886
|
execHistoryKey(executionId) {
|
|
23948
24887
|
return `${this.keyPrefix}exec-history:${executionId}`;
|
|
23949
24888
|
}
|
|
23950
|
-
//
|
|
23951
|
-
//
|
|
23952
|
-
//
|
|
23953
|
-
// The slow-path fallback (`listHistoryByZset`) reads from the SET when
|
|
23954
|
-
// `zsetCard < setCard`, so SET-based eviction would race with the fallback
|
|
23955
|
-
// and surface dead IDs whose data has been TTL'd away. ZSET eviction +
|
|
23956
|
-
// dual-write `SREM` is the clean path; doc the assumption then.
|
|
24889
|
+
// The legacy id set and the sorted set below always change together (save,
|
|
24890
|
+
// delete, prune), so the slow-path guard `ZCARD >= SCARD` in
|
|
24891
|
+
// `listHistoryByZset` and backfill stays meaningful.
|
|
23957
24892
|
execHistorySetKey() {
|
|
23958
24893
|
return `${this.keyPrefix}exec-history-ids`;
|
|
23959
24894
|
}
|
|
@@ -23976,6 +24911,36 @@ var RedisStore = class _RedisStore {
|
|
|
23976
24911
|
evalHistoryZsetKey() {
|
|
23977
24912
|
return `${this.keyPrefix}eval-history-z`;
|
|
23978
24913
|
}
|
|
24914
|
+
/** Keys and blob parsing for one kind of history. See {@link HistoryIndex}. */
|
|
24915
|
+
historyIndex(kind) {
|
|
24916
|
+
if (kind === "execution") {
|
|
24917
|
+
return {
|
|
24918
|
+
kind,
|
|
24919
|
+
label: "execution-history",
|
|
24920
|
+
ids: this.execHistorySetKey(),
|
|
24921
|
+
time: this.execHistoryZsetKey(),
|
|
24922
|
+
expiry: `${this.keyPrefix}exec-history-exp`,
|
|
24923
|
+
dataKey: (id) => this.execHistoryKey(id),
|
|
24924
|
+
lastSavedAtOf: (raw) => {
|
|
24925
|
+
const record = JSON.parse(raw);
|
|
24926
|
+
return typeof record.startedAt === "number" ? executionLastSavedAt(record) : null;
|
|
24927
|
+
}
|
|
24928
|
+
};
|
|
24929
|
+
}
|
|
24930
|
+
return {
|
|
24931
|
+
kind,
|
|
24932
|
+
label: "eval-history",
|
|
24933
|
+
ids: this.evalHistorySetKey(),
|
|
24934
|
+
time: this.evalHistoryZsetKey(),
|
|
24935
|
+
expiry: `${this.keyPrefix}eval-history-exp`,
|
|
24936
|
+
dataKey: (id) => this.evalHistoryKey(id),
|
|
24937
|
+
// Eval retention counts from the first write, which is the row's timestamp.
|
|
24938
|
+
lastSavedAtOf: (raw) => {
|
|
24939
|
+
const timestamp = JSON.parse(raw).timestamp;
|
|
24940
|
+
return typeof timestamp === "number" ? timestamp : null;
|
|
24941
|
+
}
|
|
24942
|
+
};
|
|
24943
|
+
}
|
|
23979
24944
|
memoryKey(scope) {
|
|
23980
24945
|
return `${this.keyPrefix}memory:${scope}`;
|
|
23981
24946
|
}
|
|
@@ -24081,7 +25046,7 @@ var RedisStore = class _RedisStore {
|
|
|
24081
25046
|
}
|
|
24082
25047
|
}
|
|
24083
25048
|
if (dead.length > 0) {
|
|
24084
|
-
this.
|
|
25049
|
+
this.connection.sRem(this.pendingExecSetKey(), dead).catch((err) => {
|
|
24085
25050
|
console.error(
|
|
24086
25051
|
`[axl] RedisStore: failed to prune ${dead.length} stale pending-exec ids: ` + (err instanceof Error ? err.message : String(err))
|
|
24087
25052
|
);
|
|
@@ -24127,7 +25092,7 @@ var RedisStore = class _RedisStore {
|
|
|
24127
25092
|
}
|
|
24128
25093
|
const ttl = this.ttlFor("memory");
|
|
24129
25094
|
if (ttl !== void 0) {
|
|
24130
|
-
this.
|
|
25095
|
+
this.connection.expire(this.memoryKey(scope), ttl).catch((err) => {
|
|
24131
25096
|
console.error(
|
|
24132
25097
|
`[axl] RedisStore: failed to set TTL on migrated memory key ${this.memoryKey(scope)}: ` + (err instanceof Error ? err.message : String(err))
|
|
24133
25098
|
);
|
|
@@ -24147,31 +25112,35 @@ var RedisStore = class _RedisStore {
|
|
|
24147
25112
|
async saveExecution(execution) {
|
|
24148
25113
|
const ttl = this.ttlFor("executionHistory");
|
|
24149
25114
|
const setOptions = ttl !== void 0 ? { EX: ttl } : void 0;
|
|
25115
|
+
const index = this.historyIndex("execution");
|
|
25116
|
+
const id = execution.executionId;
|
|
24150
25117
|
const storedExecution = normalizeStoredExecution({
|
|
24151
25118
|
...execution,
|
|
24152
25119
|
eventSchemaVersion: getExecutionEventSchemaVersion(execution)
|
|
24153
25120
|
});
|
|
24154
|
-
|
|
24155
|
-
|
|
24156
|
-
|
|
24157
|
-
|
|
25121
|
+
const tx = this.client.multi().sAdd(index.ids, id).set(index.dataKey(id), JSON.stringify(storedExecution), setOptions).zAdd(index.time, { score: execution.startedAt, value: id });
|
|
25122
|
+
if (ttl !== void 0) tx.zAdd(index.expiry, { score: Date.now() + ttl * 1e3, value: id });
|
|
25123
|
+
else tx.zRem(index.expiry, id);
|
|
25124
|
+
await tx.exec();
|
|
25125
|
+
await this.pruneHistoryAfterSave(index);
|
|
24158
25126
|
}
|
|
24159
25127
|
async getExecution(executionId) {
|
|
24160
25128
|
const raw = await this.client.get(this.execHistoryKey(executionId));
|
|
24161
25129
|
return raw ? normalizeStoredExecution(JSON.parse(raw)) : null;
|
|
24162
25130
|
}
|
|
24163
|
-
async listExecutions(limit) {
|
|
25131
|
+
async listExecutions(limit, options) {
|
|
25132
|
+
const cursor = options?.cursor;
|
|
24164
25133
|
const executions = await this.listHistoryByZset(
|
|
24165
|
-
this.
|
|
24166
|
-
|
|
24167
|
-
|
|
24168
|
-
|
|
24169
|
-
limit
|
|
25134
|
+
this.historyIndex("execution"),
|
|
25135
|
+
(e) => ({ score: e.startedAt, id: e.executionId }),
|
|
25136
|
+
limit,
|
|
25137
|
+
cursor !== void 0 ? { score: cursor.startedAt, id: cursor.executionId } : void 0
|
|
24170
25138
|
);
|
|
24171
25139
|
return executions.map(normalizeStoredExecution);
|
|
24172
25140
|
}
|
|
24173
25141
|
async deleteExecution(executionId) {
|
|
24174
|
-
const
|
|
25142
|
+
const index = this.historyIndex("execution");
|
|
25143
|
+
const results = await this.client.multi().sRem(index.ids, executionId).del(index.dataKey(executionId)).zRem(index.time, executionId).zRem(index.expiry, executionId).del(this.checkpointKey(executionId)).del(this.executionStateKey(executionId)).sRem(this.pendingExecSetKey(), executionId).del(this.streamingEventsKey(executionId)).sRem(this.streamingIdsKey(), executionId).hDel(this.decisionsKey(), executionId).exec();
|
|
24175
25144
|
return results.some((result) => typeof result === "number" && result > 0);
|
|
24176
25145
|
}
|
|
24177
25146
|
// ── Eval History ────────────────────────────────────────────────────
|
|
@@ -24196,18 +25165,31 @@ var RedisStore = class _RedisStore {
|
|
|
24196
25165
|
* lifetime, so the race costs at most one window's length, never the window
|
|
24197
25166
|
* itself. It also carries no Redis version floor, where `KEEPTTL` needs 6.0.
|
|
24198
25167
|
*
|
|
24199
|
-
*
|
|
25168
|
+
* As with `saveExecution`, a timed row is also scored in the expiry sorted
|
|
25169
|
+
* set (by the expiry it carries, so a re-save keeps its score), an untimed
|
|
25170
|
+
* one is taken out of it, and a bounded prune follows the save.
|
|
24200
25171
|
*/
|
|
24201
25172
|
async saveEvalResult(entry) {
|
|
24202
25173
|
const ttl = this.ttlFor("evalHistory");
|
|
24203
|
-
const
|
|
25174
|
+
const index = this.historyIndex("eval");
|
|
25175
|
+
const key = index.dataKey(entry.id);
|
|
24204
25176
|
let setOptions;
|
|
25177
|
+
let expiresAt;
|
|
24205
25178
|
if (ttl !== void 0) {
|
|
24206
25179
|
const remaining = await this.client.pTTL(key);
|
|
24207
|
-
if (remaining === -2)
|
|
24208
|
-
|
|
25180
|
+
if (remaining === -2) {
|
|
25181
|
+
setOptions = { EX: ttl };
|
|
25182
|
+
expiresAt = Date.now() + ttl * 1e3;
|
|
25183
|
+
} else if (remaining > 0) {
|
|
25184
|
+
setOptions = { PX: remaining };
|
|
25185
|
+
expiresAt = Date.now() + remaining;
|
|
25186
|
+
}
|
|
24209
25187
|
}
|
|
24210
|
-
|
|
25188
|
+
const tx = this.client.multi().sAdd(index.ids, entry.id).set(key, JSON.stringify(entry), setOptions).zAdd(index.time, { score: entry.timestamp, value: entry.id });
|
|
25189
|
+
if (expiresAt !== void 0) tx.zAdd(index.expiry, { score: expiresAt, value: entry.id });
|
|
25190
|
+
else tx.zRem(index.expiry, entry.id);
|
|
25191
|
+
await tx.exec();
|
|
25192
|
+
await this.pruneHistoryAfterSave(index);
|
|
24211
25193
|
}
|
|
24212
25194
|
/**
|
|
24213
25195
|
* Update-only write: see `StateStore.updateEvalResult`.
|
|
@@ -24247,15 +25229,14 @@ var RedisStore = class _RedisStore {
|
|
|
24247
25229
|
}
|
|
24248
25230
|
async listEvalResults(limit) {
|
|
24249
25231
|
return this.listHistoryByZset(
|
|
24250
|
-
this.
|
|
24251
|
-
|
|
24252
|
-
(id) => this.evalHistoryKey(id),
|
|
24253
|
-
(a, b) => b.timestamp - a.timestamp,
|
|
25232
|
+
this.historyIndex("eval"),
|
|
25233
|
+
(e) => ({ score: e.timestamp, id: e.id }),
|
|
24254
25234
|
limit
|
|
24255
25235
|
);
|
|
24256
25236
|
}
|
|
24257
25237
|
async deleteEvalResult(id) {
|
|
24258
|
-
const
|
|
25238
|
+
const index = this.historyIndex("eval");
|
|
25239
|
+
const [, deletedCount] = await this.client.multi().sRem(index.ids, id).del(index.dataKey(id)).zRem(index.time, id).zRem(index.expiry, id).exec();
|
|
24259
25240
|
const deleted = typeof deletedCount === "number" ? deletedCount : 0;
|
|
24260
25241
|
return deleted > 0;
|
|
24261
25242
|
}
|
|
@@ -24279,50 +25260,292 @@ var RedisStore = class _RedisStore {
|
|
|
24279
25260
|
return { exists: true, expiresAt: Date.now() + ttl };
|
|
24280
25261
|
}
|
|
24281
25262
|
/**
|
|
24282
|
-
* Shared list-by-sorted-set implementation
|
|
24283
|
-
*
|
|
24284
|
-
*
|
|
24285
|
-
*
|
|
24286
|
-
*
|
|
25263
|
+
* Shared list-by-sorted-set implementation, newest first by
|
|
25264
|
+
* `(score DESC, member DESC)`, starting strictly after `cursor` when given.
|
|
25265
|
+
*
|
|
25266
|
+
* Fast path (the time sorted set is at least as large as the id set): walk
|
|
25267
|
+
* the sorted set in chunks and MGET each chunk's blobs. Index entries whose
|
|
25268
|
+
* blob is gone (expired, or deleted before the prune caught up) are skipped,
|
|
25269
|
+
* and the walk keeps reading until it has `limit` live records or the set
|
|
25270
|
+
* runs out, however many dead entries are in the way. Each chunk restarts
|
|
25271
|
+
* at the last member read, by score and member, rather than at an offset,
|
|
25272
|
+
* so entries added or removed meanwhile cannot shift the walk.
|
|
25273
|
+
*
|
|
25274
|
+
* Slow path (the time sorted set is behind the id set: backfill has not
|
|
25275
|
+
* run): read every id with GET, then sort and apply the cursor in process.
|
|
24287
25276
|
*/
|
|
24288
|
-
async listHistoryByZset(
|
|
24289
|
-
const setSize = await this.client.sCard(
|
|
25277
|
+
async listHistoryByZset(index, positionOf, limit, cursor) {
|
|
25278
|
+
const setSize = await this.client.sCard(index.ids);
|
|
24290
25279
|
if (setSize === 0) return [];
|
|
24291
|
-
const zsetSize = await this.client.zCard(
|
|
24292
|
-
|
|
24293
|
-
|
|
24294
|
-
const
|
|
25280
|
+
const zsetSize = await this.client.zCard(index.time);
|
|
25281
|
+
const max = limit ? limit : void 0;
|
|
25282
|
+
if (zsetSize < setSize) {
|
|
25283
|
+
const ids = await this.client.sMembers(index.ids);
|
|
25284
|
+
const entries = [];
|
|
25285
|
+
for (const id of ids) {
|
|
25286
|
+
const raw = await this.client.get(index.dataKey(id));
|
|
25287
|
+
if (raw == null) continue;
|
|
25288
|
+
try {
|
|
25289
|
+
entries.push(JSON.parse(raw));
|
|
25290
|
+
} catch {
|
|
25291
|
+
}
|
|
25292
|
+
}
|
|
25293
|
+
const ordered = entries.filter((e) => cursor === void 0 || compareNewestFirst(cursor, positionOf(e)) < 0).sort((a, b) => compareNewestFirst(positionOf(a), positionOf(b)));
|
|
25294
|
+
return max !== void 0 ? ordered.slice(0, max) : ordered;
|
|
25295
|
+
}
|
|
25296
|
+
const results = [];
|
|
25297
|
+
const seen = /* @__PURE__ */ new Set();
|
|
25298
|
+
let bound = cursor;
|
|
25299
|
+
let rangeMax = cursor !== void 0 ? cursor.score : "+inf";
|
|
25300
|
+
let anchor;
|
|
25301
|
+
let grow = 1;
|
|
25302
|
+
const read = (offset, count2) => (
|
|
25303
|
+
// ZRANGE BY SCORE REV: start is the highest score, stop the lowest.
|
|
25304
|
+
this.client.zRangeWithScores(index.time, rangeMax, "-inf", {
|
|
24295
25305
|
BY: "SCORE",
|
|
24296
25306
|
REV: true,
|
|
24297
|
-
...
|
|
25307
|
+
...count2 !== void 0 ? { LIMIT: { offset, count: count2 } } : {}
|
|
25308
|
+
})
|
|
25309
|
+
);
|
|
25310
|
+
for (; ; ) {
|
|
25311
|
+
const remaining = max !== void 0 ? max - results.length : void 0;
|
|
25312
|
+
const want = remaining !== void 0 ? Math.max(remaining * 2, remaining + 5) * grow : void 0;
|
|
25313
|
+
let offset = 0;
|
|
25314
|
+
let count2 = want;
|
|
25315
|
+
let members;
|
|
25316
|
+
if (anchor !== void 0 && want !== void 0) {
|
|
25317
|
+
offset = anchor.pos;
|
|
25318
|
+
count2 = want + 1;
|
|
25319
|
+
members = await read(offset, count2);
|
|
25320
|
+
if (members[0]?.value !== anchor.id) {
|
|
25321
|
+
offset = 0;
|
|
25322
|
+
count2 = anchor.pos + 1 + want;
|
|
25323
|
+
members = await read(offset, count2);
|
|
25324
|
+
}
|
|
25325
|
+
} else {
|
|
25326
|
+
members = await read(offset, count2);
|
|
25327
|
+
}
|
|
25328
|
+
const fresh = members.filter(
|
|
25329
|
+
(m) => !seen.has(m.value) && (bound === void 0 || compareNewestFirst(bound, { score: m.score, id: m.value }) < 0)
|
|
25330
|
+
);
|
|
25331
|
+
if (fresh.length > 0) {
|
|
25332
|
+
const raws = await this.client.mGet(fresh.map((m) => index.dataKey(m.value)));
|
|
25333
|
+
for (let i = 0; i < fresh.length; i++) {
|
|
25334
|
+
seen.add(fresh[i].value);
|
|
25335
|
+
const raw = raws[i];
|
|
25336
|
+
if (raw == null) continue;
|
|
25337
|
+
try {
|
|
25338
|
+
results.push(JSON.parse(raw));
|
|
25339
|
+
} catch {
|
|
25340
|
+
continue;
|
|
25341
|
+
}
|
|
25342
|
+
if (max !== void 0 && results.length >= max) return results;
|
|
25343
|
+
}
|
|
25344
|
+
const lastFresh = fresh[fresh.length - 1];
|
|
25345
|
+
bound = { score: lastFresh.score, id: lastFresh.value };
|
|
25346
|
+
grow = 1;
|
|
25347
|
+
} else {
|
|
25348
|
+
grow *= 2;
|
|
25349
|
+
}
|
|
25350
|
+
if (count2 === void 0 || members.length < count2) return results;
|
|
25351
|
+
const last = members[members.length - 1];
|
|
25352
|
+
if (last.score === rangeMax) {
|
|
25353
|
+
anchor = { id: last.value, score: last.score, pos: offset + members.length - 1 };
|
|
25354
|
+
} else {
|
|
25355
|
+
rangeMax = last.score;
|
|
25356
|
+
const before = members.filter((m) => m.score === last.score).length - 1;
|
|
25357
|
+
anchor = { id: last.value, score: last.score, pos: before };
|
|
25358
|
+
}
|
|
25359
|
+
}
|
|
25360
|
+
}
|
|
25361
|
+
// ── History index cleanup ──────────────────────────────────────────────
|
|
25362
|
+
//
|
|
25363
|
+
// Index members cannot carry a TTL, so when a history record expires its
|
|
25364
|
+
// entries in the id set and the time sorted set stay behind. Every history
|
|
25365
|
+
// save runs one bounded prune of its kind: it reads up to PRUNE_BATCH ids
|
|
25366
|
+
// whose expiry has passed from the expiry sorted set, then runs
|
|
25367
|
+
// PRUNE_HISTORY_INDEX_SCRIPT over them. The script checks each record's blob
|
|
25368
|
+
// and updates the indexes in one atomic step, so a save or delete landing
|
|
25369
|
+
// between the read and the script cannot be undone by it: a record saved
|
|
25370
|
+
// again is seen live and stays listed, and a deleted one is already gone.
|
|
25371
|
+
// The script only removes index entries and re-scores expiry entries; it
|
|
25372
|
+
// never adds a listing entry or writes a blob, so it cannot bring back
|
|
25373
|
+
// deleted data. Several instances pruning at once is safe.
|
|
25374
|
+
/**
|
|
25375
|
+
* Run one prune of `index`'s kind after a save. A prune failure does not
|
|
25376
|
+
* fail the save, whose MULTI already committed the record: it is reported
|
|
25377
|
+
* once per process and the next save of that kind tries again. That also
|
|
25378
|
+
* covers a Redis user whose ACL denies `EVAL`.
|
|
25379
|
+
*/
|
|
25380
|
+
async pruneHistoryAfterSave(index) {
|
|
25381
|
+
const inFlight = this.prunesInFlight ??= /* @__PURE__ */ new Set();
|
|
25382
|
+
if (inFlight.has(index.kind)) return;
|
|
25383
|
+
inFlight.add(index.kind);
|
|
25384
|
+
try {
|
|
25385
|
+
await this.pruneHistory(index);
|
|
25386
|
+
} catch (err) {
|
|
25387
|
+
if (this.connection.isReady === false) return;
|
|
25388
|
+
warnDiagnosticOnce(
|
|
25389
|
+
`redis_history_prune_failed\0${index.kind}`,
|
|
25390
|
+
`RedisStore: removing expired ${index.label} index entries failed (${err instanceof Error ? err.message : String(err)}). The record was saved; the next save retries the cleanup. Listings stay correct meanwhile. The cleanup runs a Lua script, so the Redis user needs permission to run EVAL.`,
|
|
25391
|
+
void 0,
|
|
25392
|
+
"docs/api-reference.md#redisstore-options"
|
|
25393
|
+
);
|
|
25394
|
+
} finally {
|
|
25395
|
+
inFlight.delete(index.kind);
|
|
25396
|
+
}
|
|
25397
|
+
}
|
|
25398
|
+
async pruneHistory(index) {
|
|
25399
|
+
const candidates = await this.client.zRange(index.expiry, "-inf", Date.now(), {
|
|
25400
|
+
BY: "SCORE",
|
|
25401
|
+
LIMIT: { offset: 0, count: PRUNE_BATCH }
|
|
25402
|
+
});
|
|
25403
|
+
if (candidates.length > 0) await this.runPruneScript(index, candidates);
|
|
25404
|
+
}
|
|
25405
|
+
/**
|
|
25406
|
+
* Run {@link PRUNE_HISTORY_INDEX_SCRIPT} over `ids`, at most PRUNE_BATCH
|
|
25407
|
+
* per call so one script never blocks the server for long.
|
|
25408
|
+
*/
|
|
25409
|
+
async runPruneScript(index, ids) {
|
|
25410
|
+
for (let start = 0; start < ids.length; start += PRUNE_BATCH) {
|
|
25411
|
+
const chunk = ids.slice(start, start + PRUNE_BATCH);
|
|
25412
|
+
await this.client.eval(PRUNE_HISTORY_INDEX_SCRIPT, {
|
|
25413
|
+
keys: [index.ids, index.time, index.expiry, ...chunk.map((id) => index.dataKey(id))],
|
|
25414
|
+
arguments: [String(Date.now()), ...chunk]
|
|
24298
25415
|
});
|
|
24299
|
-
|
|
24300
|
-
|
|
24301
|
-
|
|
24302
|
-
|
|
24303
|
-
|
|
24304
|
-
|
|
25416
|
+
}
|
|
25417
|
+
}
|
|
25418
|
+
// ── History maintenance ────────────────────────────────────────────────
|
|
25419
|
+
/**
|
|
25420
|
+
* See `StateStore.applyHistoryRetention`. One call:
|
|
25421
|
+
*
|
|
25422
|
+
* 1. On the first call of a walk (no cursor), backfills the time sorted
|
|
25423
|
+
* set from the id set if it is behind (see `backfillExecutionIndex`).
|
|
25424
|
+
* 2. Reads the next `batchSize` ids from the time sorted set, oldest first
|
|
25425
|
+
* by `(score, member)`, strictly after the cursor. The read restarts at
|
|
25426
|
+
* the cursor's position by value, never by offset.
|
|
25427
|
+
* 3. MGETs their blobs and reads each live blob's PTTL.
|
|
25428
|
+
* 4. In one MULTI, for each live blob:
|
|
25429
|
+
* - a TTL (`PTTL > 0`): `ZADD NX` to the expiry set at now + PTTL. A
|
|
25430
|
+
* record already in the expiry set keeps its entry (`indexed` counts
|
|
25431
|
+
* only new entries).
|
|
25432
|
+
* - no TTL (`-1`), with a retention configured: `PEXPIREAT … NX` at last
|
|
25433
|
+
* save + retention. `NX` applies only to a key that still has no TTL,
|
|
25434
|
+
* so a record re-saved meanwhile keeps the fresh window its save gave
|
|
25435
|
+
* it. A window that has already passed uses a timestamp in the past,
|
|
25436
|
+
* which makes Redis delete the key, again only if it is still untimed.
|
|
25437
|
+
* 5. Adds each newly armed record to the expiry set (`ZADD NX`).
|
|
25438
|
+
* 6. Runs the prune script (`PRUNE_HISTORY_INDEX_SCRIPT`) over ids whose
|
|
25439
|
+
* blob is gone and ids just deleted in step 4. It checks each blob
|
|
25440
|
+
* atomically, so a record re-saved at any point stays listed and a
|
|
25441
|
+
* deleted one is never re-indexed.
|
|
25442
|
+
*
|
|
25443
|
+
* Blobs are never written. A record deleted while the call runs can leave
|
|
25444
|
+
* an entry in the expiry set only; the next prune removes it.
|
|
25445
|
+
*/
|
|
25446
|
+
async applyHistoryRetention(options) {
|
|
25447
|
+
const { kind, batchSize, cursor } = resolveHistoryRetentionOptions("RedisStore", options);
|
|
25448
|
+
const index = this.historyIndex(kind === "executions" ? "execution" : "eval");
|
|
25449
|
+
const ttl = this.ttlFor(kind === "executions" ? "executionHistory" : "evalHistory");
|
|
25450
|
+
if (cursor === void 0) {
|
|
25451
|
+
if (kind === "executions") await this.backfillExecutionIndex();
|
|
25452
|
+
else await this.backfillEvalIndex();
|
|
25453
|
+
}
|
|
25454
|
+
const positions = await this.readRetentionBatch(index, cursor, batchSize + 1);
|
|
25455
|
+
const batch = positions.slice(0, batchSize);
|
|
25456
|
+
const result = {
|
|
25457
|
+
expired: 0,
|
|
25458
|
+
armed: 0,
|
|
25459
|
+
indexed: 0,
|
|
25460
|
+
deadIndexEntries: 0,
|
|
25461
|
+
cursor: positions.length > batchSize ? { time: batch[batch.length - 1].score, id: batch[batch.length - 1].value } : null
|
|
25462
|
+
};
|
|
25463
|
+
if (batch.length === 0) return result;
|
|
25464
|
+
const ids = batch.map((m) => m.value);
|
|
25465
|
+
const raws = await this.client.mGet(ids.map((id) => index.dataKey(id)));
|
|
25466
|
+
const dead = ids.filter((_, i) => raws[i] == null);
|
|
25467
|
+
const present = ids.flatMap((id, i) => {
|
|
25468
|
+
const raw = raws[i];
|
|
25469
|
+
return raw == null ? [] : [{ id, raw }];
|
|
25470
|
+
});
|
|
25471
|
+
const pttls = await Promise.all(present.map((r) => this.client.pTTL(index.dataKey(r.id))));
|
|
25472
|
+
const now = Date.now();
|
|
25473
|
+
const tx = this.client.multi();
|
|
25474
|
+
const queued = [];
|
|
25475
|
+
for (let i = 0; i < present.length; i++) {
|
|
25476
|
+
const { id, raw } = present[i];
|
|
25477
|
+
const pttl = pttls[i];
|
|
25478
|
+
if (pttl === -2) {
|
|
25479
|
+
dead.push(id);
|
|
25480
|
+
} else if (pttl > 0) {
|
|
25481
|
+
tx.zAdd(index.expiry, { score: now + pttl, value: id }, { condition: "NX" });
|
|
25482
|
+
queued.push({ action: "index", id });
|
|
25483
|
+
} else if (pttl === -1 && ttl !== void 0) {
|
|
25484
|
+
let lastSavedAt;
|
|
24305
25485
|
try {
|
|
24306
|
-
|
|
25486
|
+
lastSavedAt = index.lastSavedAtOf(raw);
|
|
24307
25487
|
} catch {
|
|
25488
|
+
lastSavedAt = null;
|
|
25489
|
+
}
|
|
25490
|
+
if (lastSavedAt === null) continue;
|
|
25491
|
+
const decision = retentionForUntimedRecord(lastSavedAt, ttl, now);
|
|
25492
|
+
if (decision.expired) {
|
|
25493
|
+
tx.pExpireAt(index.dataKey(id), 1, "NX");
|
|
25494
|
+
queued.push({ action: "expire", id });
|
|
25495
|
+
} else {
|
|
25496
|
+
tx.pExpireAt(index.dataKey(id), decision.expiresAt, "NX");
|
|
25497
|
+
queued.push({ action: "arm", id, expiresAt: decision.expiresAt });
|
|
24308
25498
|
}
|
|
24309
|
-
if (limit !== void 0 && entries2.length >= limit) break;
|
|
24310
25499
|
}
|
|
24311
|
-
return entries2;
|
|
24312
25500
|
}
|
|
24313
|
-
const
|
|
24314
|
-
|
|
24315
|
-
|
|
24316
|
-
|
|
24317
|
-
|
|
24318
|
-
|
|
24319
|
-
|
|
24320
|
-
|
|
24321
|
-
|
|
25501
|
+
const expiredIds = [];
|
|
25502
|
+
const armed = [];
|
|
25503
|
+
if (queued.length > 0) {
|
|
25504
|
+
const replies = await tx.exec();
|
|
25505
|
+
for (let i = 0; i < queued.length; i++) {
|
|
25506
|
+
if (Number(replies[i]) !== 1) continue;
|
|
25507
|
+
const entry = queued[i];
|
|
25508
|
+
if (entry.action === "index") result.indexed++;
|
|
25509
|
+
else if (entry.action === "expire") expiredIds.push(entry.id);
|
|
25510
|
+
else armed.push(entry);
|
|
24322
25511
|
}
|
|
24323
25512
|
}
|
|
24324
|
-
|
|
24325
|
-
|
|
25513
|
+
if (armed.length > 0) {
|
|
25514
|
+
const indexTx = this.client.multi();
|
|
25515
|
+
for (const { id, expiresAt } of armed) {
|
|
25516
|
+
indexTx.zAdd(index.expiry, { score: expiresAt, value: id }, { condition: "NX" });
|
|
25517
|
+
}
|
|
25518
|
+
await indexTx.exec();
|
|
25519
|
+
}
|
|
25520
|
+
result.armed = armed.length;
|
|
25521
|
+
result.expired = expiredIds.length;
|
|
25522
|
+
result.deadIndexEntries = dead.length;
|
|
25523
|
+
if (dead.length > 0 || expiredIds.length > 0) {
|
|
25524
|
+
await this.runPruneScript(index, [...dead, ...expiredIds]);
|
|
25525
|
+
}
|
|
25526
|
+
return result;
|
|
25527
|
+
}
|
|
25528
|
+
/**
|
|
25529
|
+
* Up to `count` members of the time sorted set strictly after `cursor`,
|
|
25530
|
+
* oldest first by `(score, member)`. The members sharing the cursor's score
|
|
25531
|
+
* are read in full and filtered by member; the rest start just above that
|
|
25532
|
+
* score. Neither read uses an offset, so entries removed or added
|
|
25533
|
+
* meanwhile cannot shift the walk.
|
|
25534
|
+
*/
|
|
25535
|
+
async readRetentionBatch(index, cursor, count2) {
|
|
25536
|
+
if (cursor === void 0) {
|
|
25537
|
+
return this.client.zRangeWithScores(index.time, "-inf", "+inf", {
|
|
25538
|
+
BY: "SCORE",
|
|
25539
|
+
LIMIT: { offset: 0, count: count2 }
|
|
25540
|
+
});
|
|
25541
|
+
}
|
|
25542
|
+
const ties = (await this.client.zRangeWithScores(index.time, cursor.time, cursor.time, { BY: "SCORE" })).filter((m) => compareMembers(m.value, cursor.id) > 0);
|
|
25543
|
+
if (ties.length >= count2) return ties.slice(0, count2);
|
|
25544
|
+
const rest = await this.client.zRangeWithScores(index.time, `(${cursor.time}`, "+inf", {
|
|
25545
|
+
BY: "SCORE",
|
|
25546
|
+
LIMIT: { offset: 0, count: count2 - ties.length }
|
|
25547
|
+
});
|
|
25548
|
+
return [...ties, ...rest];
|
|
24326
25549
|
}
|
|
24327
25550
|
// ── Sessions (Studio introspection) ────────────────────────────────────
|
|
24328
25551
|
async listSessions() {
|
|
@@ -24366,10 +25589,31 @@ var RedisStore = class _RedisStore {
|
|
|
24366
25589
|
* must not get a rejected promise for it. `MemoryStore.close()` and
|
|
24367
25590
|
* better-sqlite3's `close()` are both no-ops the second time; node-redis
|
|
24368
25591
|
* instead rejects with "The client is closed", so absorb that here.
|
|
25592
|
+
*
|
|
25593
|
+
* Bounded, so a shutdown can't hang on Redis. `QUIT` lets queued commands
|
|
25594
|
+
* finish on a healthy connection, but node-redis never settles it if the
|
|
25595
|
+
* socket closes first, and a stalled server may never answer it. While
|
|
25596
|
+
* disconnected it couldn't be sent at all.
|
|
24369
25597
|
*/
|
|
24370
25598
|
async close() {
|
|
24371
|
-
if (this.
|
|
24372
|
-
|
|
25599
|
+
if (this.connection.isOpen === false) return;
|
|
25600
|
+
if (this.connection.isReady === false) {
|
|
25601
|
+
abandonConnection(this.connection);
|
|
25602
|
+
return;
|
|
25603
|
+
}
|
|
25604
|
+
let timer;
|
|
25605
|
+
try {
|
|
25606
|
+
await Promise.race([
|
|
25607
|
+
this.connection.quit(),
|
|
25608
|
+
new Promise((_resolve, reject) => {
|
|
25609
|
+
timer = setTimeout(() => reject(new Error("QUIT timed out")), CLOSE_TIMEOUT_MS);
|
|
25610
|
+
})
|
|
25611
|
+
]);
|
|
25612
|
+
} catch {
|
|
25613
|
+
abandonConnection(this.connection);
|
|
25614
|
+
} finally {
|
|
25615
|
+
clearTimeout(timer);
|
|
25616
|
+
}
|
|
24373
25617
|
}
|
|
24374
25618
|
async deleteCheckpoints(executionId) {
|
|
24375
25619
|
await this.client.del(this.checkpointKey(executionId));
|