@jini-ai/daemon 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -0
- package/dist/agent-executor.d.ts +714 -42
- package/dist/agent-executor.d.ts.map +1 -1
- package/dist/agent-executor.js +1416 -351
- package/dist/agent-executor.js.map +1 -1
- package/dist/continuation/index.d.ts +1 -0
- package/dist/continuation/index.d.ts.map +1 -1
- package/dist/continuation/index.js +1 -0
- package/dist/continuation/index.js.map +1 -1
- package/dist/continuation/run-scoped-context-store.d.ts +79 -0
- package/dist/continuation/run-scoped-context-store.d.ts.map +1 -0
- package/dist/continuation/run-scoped-context-store.js +56 -0
- package/dist/continuation/run-scoped-context-store.js.map +1 -0
- package/dist/continuation/run-start-handler.d.ts +39 -9
- package/dist/continuation/run-start-handler.d.ts.map +1 -1
- package/dist/continuation/run-start-handler.js +12 -2
- package/dist/continuation/run-start-handler.js.map +1 -1
- package/dist/delegated-tool-bridge.d.ts +8 -0
- package/dist/delegated-tool-bridge.d.ts.map +1 -1
- package/dist/delegated-tool-bridge.js +117 -1
- package/dist/delegated-tool-bridge.js.map +1 -1
- package/dist/event-log.d.ts +18 -105
- package/dist/event-log.d.ts.map +1 -1
- package/dist/event-log.js +0 -17
- package/dist/event-log.js.map +1 -1
- package/dist/frontend-capability-tools.d.ts +1 -1
- package/dist/frontend-capability-tools.js +1 -1
- package/dist/frontend-session-registry.d.ts.map +1 -1
- package/dist/frontend-session-registry.js +26 -10
- package/dist/frontend-session-registry.js.map +1 -1
- package/dist/image-prompt-delivery.d.ts +56 -0
- package/dist/image-prompt-delivery.d.ts.map +1 -0
- package/dist/image-prompt-delivery.js +104 -0
- package/dist/image-prompt-delivery.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/remote-tool-bridge.d.ts +45 -0
- package/dist/remote-tool-bridge.d.ts.map +1 -0
- package/dist/remote-tool-bridge.js +23 -0
- package/dist/remote-tool-bridge.js.map +1 -0
- package/dist/routines/routine-store.js +1 -1
- package/dist/routines/schedule.d.ts +0 -8
- package/dist/routines/schedule.d.ts.map +1 -1
- package/dist/routines/schedule.js +71 -44
- package/dist/routines/schedule.js.map +1 -1
- package/dist/routines/scheduler.d.ts +43 -0
- package/dist/routines/scheduler.d.ts.map +1 -1
- package/dist/routines/scheduler.js +160 -120
- package/dist/routines/scheduler.js.map +1 -1
- package/dist/run/core/retry.d.ts.map +1 -1
- package/dist/run/core/retry.js +47 -35
- package/dist/run/core/retry.js.map +1 -1
- package/dist/run/diagnostics/diagnostics.d.ts.map +1 -1
- package/dist/run/diagnostics/diagnostics.js +149 -91
- package/dist/run/diagnostics/diagnostics.js.map +1 -1
- package/dist/run-lifecycle.d.ts +84 -4
- package/dist/run-lifecycle.d.ts.map +1 -1
- package/dist/run-lifecycle.js +378 -116
- package/dist/run-lifecycle.js.map +1 -1
- package/dist/terminal-session.d.ts +1 -1
- package/dist/terminal-session.d.ts.map +1 -1
- package/dist/terminal-session.js +1 -1
- package/dist/tool-executor.d.ts +26 -6
- package/dist/tool-executor.d.ts.map +1 -1
- package/dist/tool-executor.js +220 -55
- package/dist/tool-executor.js.map +1 -1
- package/dist/tool-result-media.d.ts +79 -0
- package/dist/tool-result-media.d.ts.map +1 -0
- package/dist/tool-result-media.js +80 -0
- package/dist/tool-result-media.js.map +1 -0
- package/dist/tool-result-surfaces.d.ts +78 -0
- package/dist/tool-result-surfaces.d.ts.map +1 -0
- package/dist/tool-result-surfaces.js +92 -0
- package/dist/tool-result-surfaces.js.map +1 -0
- package/package.json +18 -9
package/dist/run-lifecycle.js
CHANGED
|
@@ -56,6 +56,59 @@ function toRunEvent(runId, entry) {
|
|
|
56
56
|
function isRecord(value) {
|
|
57
57
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Reads the two run-level fields the kernel persists on a run's `'start'` entry. Both are absent
|
|
61
|
+
* on a log whose start entry carried a non-record payload (or no start entry at all).
|
|
62
|
+
*/
|
|
63
|
+
function readStartMetadata(startEntry) {
|
|
64
|
+
const startData = startEntry && isRecord(startEntry.data) ? startEntry.data : null;
|
|
65
|
+
return {
|
|
66
|
+
contextRef: typeof startData?.contextRef === 'string' ? startData.contextRef : undefined,
|
|
67
|
+
idempotencyKey: typeof startData?.idempotencyKey === 'string' ? startData.idempotencyKey : undefined,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
/** Rebuilds the public status block for a rehydrated run from its first/last/start/end log entries. */
|
|
71
|
+
function rehydratedStatus(runId, bounds) {
|
|
72
|
+
return {
|
|
73
|
+
id: runId,
|
|
74
|
+
state: bounds.terminal?.state ?? 'running',
|
|
75
|
+
startedAt: bounds.startEntry?.recordedAt ?? bounds.firstEntry.recordedAt,
|
|
76
|
+
updatedAt: bounds.lastEntry.recordedAt,
|
|
77
|
+
endedAt: bounds.endEntry?.recordedAt,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Rebuilds one in-memory `RunRecord` from a run's replayed durable entries. Volatile fields
|
|
82
|
+
* (listeners, subscribers, watchdog, in-flight promises) cannot survive a restart and are reset.
|
|
83
|
+
*/
|
|
84
|
+
function rehydratedRunRecord(runId, entries) {
|
|
85
|
+
const startEntry = entries.find((entry) => entry.event === 'start');
|
|
86
|
+
const endEntry = [...entries].reverse().find((entry) => entry.event === 'end');
|
|
87
|
+
const firstEntry = entries[0];
|
|
88
|
+
const lastEntry = entries[entries.length - 1];
|
|
89
|
+
const terminal = endEntry === undefined ? null : terminalStateFromEndEntry(endEntry);
|
|
90
|
+
const { contextRef, idempotencyKey } = readStartMetadata(startEntry);
|
|
91
|
+
return {
|
|
92
|
+
record: {
|
|
93
|
+
contextRef,
|
|
94
|
+
idempotencyKey,
|
|
95
|
+
status: rehydratedStatus(runId, { startEntry, endEntry, firstEntry, lastEntry, terminal }),
|
|
96
|
+
resumable: terminal?.resumable ?? false,
|
|
97
|
+
cancelRequested: false,
|
|
98
|
+
lastCancelRequest: undefined,
|
|
99
|
+
cancelListeners: new Set(),
|
|
100
|
+
subscribers: new Set(),
|
|
101
|
+
terminalWaiters: [],
|
|
102
|
+
terminalEndEntry: endEntry,
|
|
103
|
+
watchdog: undefined,
|
|
104
|
+
startPromise: undefined,
|
|
105
|
+
finishPromise: undefined,
|
|
106
|
+
retentionTimer: undefined,
|
|
107
|
+
},
|
|
108
|
+
idempotencyKey,
|
|
109
|
+
isTerminal: terminal !== null,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
59
112
|
function terminalStateFromEndEntry(entry) {
|
|
60
113
|
if (!isRecord(entry.data))
|
|
61
114
|
return { state: 'failed', resumable: false };
|
|
@@ -63,18 +116,176 @@ function terminalStateFromEndEntry(entry) {
|
|
|
63
116
|
const state = status === 'succeeded' ? 'succeeded' : status === 'canceled' ? 'cancelled' : 'failed';
|
|
64
117
|
return { state, resumable: entry.data.resumable === true };
|
|
65
118
|
}
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
// start()/stream() phase helpers
|
|
121
|
+
//
|
|
122
|
+
// `start()` and `stream()` were each over the complexity gate (cyclomatic 12/13,
|
|
123
|
+
// cognitive 12/12). Both are broken up the same way: pure/near-pure pieces with
|
|
124
|
+
// real branching extracted to named, independently-testable top-level functions
|
|
125
|
+
// (typed against plain object shapes rather than the private `RunRecord`, so
|
|
126
|
+
// they can be exported without leaking that internal type), leaving each method
|
|
127
|
+
// itself a flat sequence with only its two or three irreducible guard clauses.
|
|
128
|
+
// ---------------------------------------------------------------------------
|
|
129
|
+
/**
|
|
130
|
+
* `start()`'s idempotency-replay lookup: the run id an existing `idempotencyKey` already maps to,
|
|
131
|
+
* or `undefined` when there is no key or no existing mapping (a fresh start proceeds normally).
|
|
132
|
+
* Pure.
|
|
133
|
+
*/
|
|
134
|
+
export function resolveIdempotentReplayRunId(idempotencyIndex, idempotencyKey) {
|
|
135
|
+
if (idempotencyKey === undefined)
|
|
136
|
+
return undefined;
|
|
137
|
+
return idempotencyIndex.get(idempotencyKey);
|
|
138
|
+
}
|
|
139
|
+
/** Registers `runId` under `idempotencyKey` in `idempotencyIndex` — a no-op when no key was supplied. */
|
|
140
|
+
export function registerIdempotencyKeyIfPresent(idempotencyIndex, idempotencyKey, runId) {
|
|
141
|
+
if (idempotencyKey === undefined)
|
|
142
|
+
return;
|
|
143
|
+
idempotencyIndex.set(idempotencyKey, runId);
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Removes `idempotencyKey`'s mapping from `idempotencyIndex`, but only when it still points at
|
|
147
|
+
* `runId` — used to roll back a failed durable `'start'` append without clobbering a different
|
|
148
|
+
* run that may have since claimed the same key (defensive; `start()`'s own locking makes that
|
|
149
|
+
* race unreachable today, but the check costs nothing and documents the invariant).
|
|
150
|
+
*/
|
|
151
|
+
export function clearIdempotencyIndexEntryIfMatching(idempotencyIndex, idempotencyKey, runId) {
|
|
152
|
+
if (idempotencyKey !== undefined && idempotencyIndex.get(idempotencyKey) === runId) {
|
|
153
|
+
idempotencyIndex.delete(idempotencyKey);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Default terminal-run retention window: how long a completed/failed/cancelled run's in-memory
|
|
158
|
+
* record stays readable via `get`/`list`/`stream`/`resume` before eviction. `runs.set()` never had
|
|
159
|
+
* a matching `runs.delete()` for a run that finishes normally — every terminal run's record was kept
|
|
160
|
+
* for the daemon process's entire lifetime, unbounded by run volume or uptime. Terminal runs are
|
|
161
|
+
* still read after they end (status polling, run-history listing, stream reconnects), so the fix
|
|
162
|
+
* cannot be "delete on completion" — this bounds *how long* a terminal record survives instead of
|
|
163
|
+
* deleting it immediately. 24h comfortably covers same-day polling/listing/reconnect without
|
|
164
|
+
* retaining every run a long-lived daemon ever processed.
|
|
165
|
+
*/
|
|
166
|
+
export const DEFAULT_TERMINAL_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
167
|
+
/**
|
|
168
|
+
* Default hard cap on concurrently-retained terminal run records, independent of
|
|
169
|
+
* {@link DEFAULT_TERMINAL_RETENTION_MS} — the oldest terminal record is evicted once this is
|
|
170
|
+
* exceeded. A TTL alone does not bound memory when runs complete faster than they age out (a burst
|
|
171
|
+
* arriving within the retention window keeps growing); this cap gives a true worst-case bound
|
|
172
|
+
* regardless of arrival rate.
|
|
173
|
+
*/
|
|
174
|
+
export const DEFAULT_MAX_TERMINAL_RUNS = 1000;
|
|
175
|
+
/**
|
|
176
|
+
* How long a just-terminal run should remain retained before its eviction timer fires: `retentionMs`
|
|
177
|
+
* minus however much of that window has already elapsed since `terminalAt`. Floored at `0` so a run
|
|
178
|
+
* that was already past its retention window when this is computed (relevant for `rehydrate()`,
|
|
179
|
+
* where a run may have gone terminal long before this process started) schedules an immediate
|
|
180
|
+
* eviction rather than a negative-delay timer. Pure.
|
|
181
|
+
*/
|
|
182
|
+
export function computeRetentionDelayMs(terminalAt, retentionMs, now) {
|
|
183
|
+
return Math.max(0, retentionMs - (now - terminalAt));
|
|
184
|
+
}
|
|
185
|
+
/** The durable `'start'` entry's payload — pure field mapping, split out of `start()` so its two optional-field spreads don't count toward that method's own complexity. */
|
|
186
|
+
export function buildStartPayload(runId, startInput) {
|
|
187
|
+
return {
|
|
188
|
+
runId,
|
|
189
|
+
contextRef: startInput.contextRef,
|
|
190
|
+
...(startInput.agentId !== undefined ? { agentId: startInput.agentId } : {}),
|
|
191
|
+
...(startInput.idempotencyKey !== undefined ? { idempotencyKey: startInput.idempotencyKey } : {}),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Arms `record.watchdog` when `timeoutMs` is configured — a no-op otherwise. `record` is typed as a
|
|
196
|
+
* plain `{watchdog}` shape (not `Pick<RunRecord, 'watchdog'>`) purely so this function can be
|
|
197
|
+
* exported without naming the private `RunRecord` type in a public signature.
|
|
198
|
+
*/
|
|
199
|
+
export function armWatchdogIfConfigured(record, timeoutMs, onTimeout) {
|
|
200
|
+
if (timeoutMs === undefined)
|
|
201
|
+
return;
|
|
202
|
+
record.watchdog = createInactivityWatchdog({ timeoutMs, onTimeout });
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* `stream()`'s replay-delivery step: sends every already-durable entry to `onEvent`, returning the
|
|
206
|
+
* delivered eventIds so a caller can avoid re-delivering the same event from a different source
|
|
207
|
+
* (buffered live events observed during the replay, a terminal catch-up event). Pure aside from
|
|
208
|
+
* calling the injected `onEvent`.
|
|
209
|
+
*/
|
|
210
|
+
export function deliverReplayedEvents(runId, entries, onEvent) {
|
|
211
|
+
const deliveredEventIds = new Set();
|
|
212
|
+
for (const entry of entries) {
|
|
213
|
+
const event = toRunEvent(runId, entry);
|
|
214
|
+
deliveredEventIds.add(event.eventId);
|
|
215
|
+
onEvent(event);
|
|
216
|
+
}
|
|
217
|
+
return deliveredEventIds;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Delivers every event in `events` not already present in `deliveredEventIds`, marking each as
|
|
221
|
+
* delivered as it goes. Reused in `stream()` for both the buffered-live-event catch-up and the
|
|
222
|
+
* single terminal-event catch-up, so the "don't double-deliver" bookkeeping lives in one place.
|
|
223
|
+
*/
|
|
224
|
+
export function deliverUndeliveredEvents(events, deliveredEventIds, onEvent) {
|
|
225
|
+
for (const event of events) {
|
|
226
|
+
if (!deliveredEventIds.has(event.eventId)) {
|
|
227
|
+
deliveredEventIds.add(event.eventId);
|
|
228
|
+
onEvent(event);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Closes out a `stream()` subscription once replay and catch-up delivery are done: a terminal run's
|
|
234
|
+
* subscriber is removed immediately (no further event will ever come), a still-live run's stays
|
|
235
|
+
* registered behind the returned `unsubscribe`. `record` is typed as a plain `{subscribers}` shape
|
|
236
|
+
* for the same exportability reason as {@link armWatchdogIfConfigured}.
|
|
237
|
+
*/
|
|
238
|
+
export function finishStreamSubscription(record, subscriber, terminal) {
|
|
239
|
+
if (terminal) {
|
|
240
|
+
record.subscribers.delete(subscriber);
|
|
241
|
+
return { kind: 'ok', unsubscribe: () => { } };
|
|
242
|
+
}
|
|
243
|
+
return { kind: 'ok', unsubscribe: () => record.subscribers.delete(subscriber) };
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Constructs a fresh in-memory `RunRecord` for a new `start()` call. Pure field mapping with no
|
|
247
|
+
* branches at all — not exported (would leak the private `RunRecord` type), kept as a named
|
|
248
|
+
* function purely so `start()` itself reads as "build the record, register it" rather than an
|
|
249
|
+
* 11-field literal inline.
|
|
250
|
+
*/
|
|
251
|
+
function buildNewRunRecord(startInput, runId, now) {
|
|
252
|
+
return {
|
|
253
|
+
contextRef: startInput.contextRef,
|
|
254
|
+
idempotencyKey: startInput.idempotencyKey,
|
|
255
|
+
status: { id: runId, state: 'running', startedAt: now, updatedAt: now, endedAt: undefined },
|
|
256
|
+
resumable: false,
|
|
257
|
+
cancelRequested: false,
|
|
258
|
+
lastCancelRequest: undefined,
|
|
259
|
+
cancelListeners: new Set(),
|
|
260
|
+
subscribers: new Set(),
|
|
261
|
+
terminalWaiters: [],
|
|
262
|
+
terminalEndEntry: undefined,
|
|
263
|
+
watchdog: undefined,
|
|
264
|
+
startPromise: undefined,
|
|
265
|
+
finishPromise: undefined,
|
|
266
|
+
retentionTimer: undefined,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
66
269
|
/**
|
|
67
270
|
* Creates the in-process `RunLifecycle` reference implementation.
|
|
68
271
|
*
|
|
69
272
|
* @param input.eventLog - The durable `EventLog` port this lifecycle appends to and replays from.
|
|
273
|
+
* @param input.terminalRetentionMs - See {@link CreateRunLifecycleInput.terminalRetentionMs}.
|
|
274
|
+
* @param input.maxTerminalRuns - See {@link CreateRunLifecycleInput.maxTerminalRuns}.
|
|
70
275
|
* @returns A `RunLifecycle` backed by an in-memory run registry plus the injected `EventLog`.
|
|
71
|
-
* @complexity Per-call complexities documented on each method; the registry itself is a `Map` keyed by `runId` (O(1) lookup).
|
|
276
|
+
* @complexity Per-call complexities documented on each method; the registry itself is a `Map` keyed by `runId` (O(1) lookup). Memory is bounded: non-terminal runs are O(n) in concurrently-live runs, and terminal runs are capped at `maxTerminalRuns` (each additionally bounded in retention time by `terminalRetentionMs`) rather than growing for the life of the process.
|
|
72
277
|
* @overallScore 100/100
|
|
73
278
|
*/
|
|
74
279
|
export function createRunLifecycle(input) {
|
|
75
280
|
const { eventLog } = input;
|
|
281
|
+
const terminalRetentionMs = input.terminalRetentionMs ?? DEFAULT_TERMINAL_RETENTION_MS;
|
|
282
|
+
const maxTerminalRuns = input.maxTerminalRuns ?? DEFAULT_MAX_TERMINAL_RUNS;
|
|
76
283
|
const runs = new Map();
|
|
77
284
|
const idempotencyIndex = new Map();
|
|
285
|
+
// Terminal runIds in the order they were tracked (oldest first) — the LRU order `enforceTerminalCap`
|
|
286
|
+
// trims from. A run leaves this list exactly once, either via its retention timer firing or via
|
|
287
|
+
// `resume()` reclaiming it; see `evictTerminalRun`/`untrackTerminalRun`.
|
|
288
|
+
const terminalRunOrder = [];
|
|
78
289
|
let hydration = null;
|
|
79
290
|
function requireRun(runId) {
|
|
80
291
|
const record = runs.get(runId);
|
|
@@ -106,6 +317,40 @@ export function createRunLifecycle(input) {
|
|
|
106
317
|
notifySubscribers(record, runEvent);
|
|
107
318
|
return runEvent;
|
|
108
319
|
}
|
|
320
|
+
/** `start()`'s phase 2: appends the durable `'start'` entry, rolling back the in-memory record and idempotency-index entry on failure. */
|
|
321
|
+
async function appendStartOrRollback(runId, record, startInput) {
|
|
322
|
+
const startPayload = buildStartPayload(runId, startInput);
|
|
323
|
+
const startPromise = appendEvent(runId, record, 'start', startPayload).then(() => undefined);
|
|
324
|
+
record.startPromise = startPromise;
|
|
325
|
+
try {
|
|
326
|
+
await startPromise;
|
|
327
|
+
record.startPromise = undefined;
|
|
328
|
+
}
|
|
329
|
+
catch (error) {
|
|
330
|
+
runs.delete(runId);
|
|
331
|
+
clearIdempotencyIndexEntryIfMatching(idempotencyIndex, startInput.idempotencyKey, runId);
|
|
332
|
+
throw error;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Waits out a record's in-flight durable `'start'` append, swallowing its failure.
|
|
337
|
+
*
|
|
338
|
+
* The read-only queries (`get`/`list`) use this so they never observe the window in which the
|
|
339
|
+
* in-memory record exists but its durable `'start'` entry does not: a rejecting append unwinds the
|
|
340
|
+
* record entirely, so anything reported from inside that window is a run that never existed. The
|
|
341
|
+
* rejection itself belongs to `start()`'s own caller, which is why it is dropped here rather than
|
|
342
|
+
* propagated out of a query that was only asked "what runs are there".
|
|
343
|
+
*/
|
|
344
|
+
async function settlePendingStart(record) {
|
|
345
|
+
if (record?.startPromise === undefined)
|
|
346
|
+
return;
|
|
347
|
+
try {
|
|
348
|
+
await record.startPromise;
|
|
349
|
+
}
|
|
350
|
+
catch {
|
|
351
|
+
// Reported to `start()`'s caller — see above.
|
|
352
|
+
}
|
|
353
|
+
}
|
|
109
354
|
function resolveTerminalWaiters(record) {
|
|
110
355
|
const waiters = record.terminalWaiters.splice(0, record.terminalWaiters.length);
|
|
111
356
|
const status = toPublicStatus(record);
|
|
@@ -113,14 +358,84 @@ export function createRunLifecycle(input) {
|
|
|
113
358
|
resolve(status);
|
|
114
359
|
}
|
|
115
360
|
}
|
|
361
|
+
function removeFromTerminalOrder(runId) {
|
|
362
|
+
const index = terminalRunOrder.indexOf(runId);
|
|
363
|
+
if (index !== -1)
|
|
364
|
+
terminalRunOrder.splice(index, 1);
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Evicts a terminal run's in-memory record: drops it from `runs`, cleans its `idempotencyIndex`
|
|
368
|
+
* entry (a stale entry would otherwise resolve `start()`'s idempotency replay to a run
|
|
369
|
+
* `requireRun` can no longer find, turning a harmless re-post into a thrown error), and removes it
|
|
370
|
+
* from `terminalRunOrder`. Called from a retention timer firing or from `enforceTerminalCap`. A
|
|
371
|
+
* no-op if the run was already reclaimed by `resume()` (its timer is cancelled there) or evicted
|
|
372
|
+
* by the other path (cap eviction can race a timer for the same run — only the first wins,
|
|
373
|
+
* `runs.get` returns `undefined` for the second).
|
|
374
|
+
*/
|
|
375
|
+
function evictTerminalRun(runId) {
|
|
376
|
+
removeFromTerminalOrder(runId);
|
|
377
|
+
const record = runs.get(runId);
|
|
378
|
+
if (!record)
|
|
379
|
+
return;
|
|
380
|
+
record.retentionTimer = undefined;
|
|
381
|
+
runs.delete(runId);
|
|
382
|
+
clearIdempotencyIndexEntryIfMatching(idempotencyIndex, record.idempotencyKey, runId);
|
|
383
|
+
}
|
|
384
|
+
/** Trims the oldest terminal records until `terminalRunOrder` is back at or under `maxTerminalRuns`. */
|
|
385
|
+
function enforceTerminalCap() {
|
|
386
|
+
while (terminalRunOrder.length > maxTerminalRuns) {
|
|
387
|
+
evictTerminalRun(terminalRunOrder[0]);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Arms `record`'s retention timer and enrolls it in `terminalRunOrder` — called once per run,
|
|
392
|
+
* right when it first becomes terminal (from `finish()`) or when an already-terminal run is
|
|
393
|
+
* rehydrated (from `rehydrateOne`). `terminalAt` is the real terminal timestamp (the durable
|
|
394
|
+
* `'end'` entry's `recordedAt`), not "now": a rehydrated run may have gone terminal long before
|
|
395
|
+
* this process started, and its remaining retention window must account for that instead of
|
|
396
|
+
* restarting the full window on every restart.
|
|
397
|
+
*/
|
|
398
|
+
function trackTerminalRun(runId, record, terminalAt) {
|
|
399
|
+
terminalRunOrder.push(runId);
|
|
400
|
+
const delay = computeRetentionDelayMs(terminalAt, terminalRetentionMs, Date.now());
|
|
401
|
+
const timer = setTimeout(() => evictTerminalRun(runId), delay);
|
|
402
|
+
if (typeof timer.unref === 'function')
|
|
403
|
+
timer.unref();
|
|
404
|
+
record.retentionTimer = timer;
|
|
405
|
+
enforceTerminalCap();
|
|
406
|
+
}
|
|
407
|
+
/** Reverses `trackTerminalRun`: cancels the pending eviction and un-enrolls `runId`. Called from `resume()` so a reclaimed run cannot be evicted out from under its new, non-terminal life. */
|
|
408
|
+
function untrackTerminalRun(runId, record) {
|
|
409
|
+
removeFromTerminalOrder(runId);
|
|
410
|
+
if (record.retentionTimer) {
|
|
411
|
+
clearTimeout(record.retentionTimer);
|
|
412
|
+
record.retentionTimer = undefined;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
116
415
|
/**
|
|
117
416
|
* Fires when a run's inactivity watchdog times out with no intervening
|
|
118
417
|
* `emit()`/`finish()`. Classified as a resumable failure (`code: null,
|
|
119
418
|
* signal: null`), mirroring OD's own timeout/inactivity classification
|
|
120
419
|
* (`isResumableFailure` in the researched `run-failure-classification.ts`
|
|
121
420
|
* treats timeout/inactivity as one of only two resumable categories).
|
|
122
|
-
*
|
|
123
|
-
*
|
|
421
|
+
*
|
|
422
|
+
* The `!record || isTerminalRunState(...)` guard below is defensive only and is currently
|
|
423
|
+
* unreachable through the public API — deliberately kept, not fake-tested, and the reason is
|
|
424
|
+
* recorded here so the next reader does not spend time trying to cover it:
|
|
425
|
+
*
|
|
426
|
+
* - `isTerminalRunState(...)`: every route to a terminal state runs through `finish()`, which
|
|
427
|
+
* calls `record.watchdog?.cancel()` and assigns `record.status.state` in the *same synchronous
|
|
428
|
+
* block* after its durable end append resolves. There is no interleaving point between the two,
|
|
429
|
+
* so "watchdog still armed" and "record already terminal" cannot both hold when this fires. (If
|
|
430
|
+
* the end append *throws*, the watchdog stays armed but the run also stays non-terminal.)
|
|
431
|
+
* - `!record`: the only `runs.delete()` is `start()`'s unwind on a failed durable start append,
|
|
432
|
+
* which happens strictly before the watchdog is armed a few lines later. It is also required
|
|
433
|
+
* for type-narrowing `record` before `record.status.state` is read, so it cannot simply be
|
|
434
|
+
* dropped.
|
|
435
|
+
*
|
|
436
|
+
* Note the "start/finish race" test below does *not* exercise this guard despite its name: by the
|
|
437
|
+
* time it advances the timers, `finish()` has already cancelled the watchdog, so the callback
|
|
438
|
+
* never runs at all. It still correctly asserts that no second `finish()` happens.
|
|
124
439
|
*/
|
|
125
440
|
async function handleInactivityTimeout(runId) {
|
|
126
441
|
const record = runs.get(runId);
|
|
@@ -149,6 +464,29 @@ export function createRunLifecycle(input) {
|
|
|
149
464
|
}
|
|
150
465
|
}
|
|
151
466
|
}
|
|
467
|
+
/**
|
|
468
|
+
* Restores one run into the in-memory index from its durable log. Returns what the caller must do
|
|
469
|
+
* about it: `'skipped'` for a run already resident or with no usable log, otherwise whether the
|
|
470
|
+
* restored run was already terminal.
|
|
471
|
+
*/
|
|
472
|
+
async function rehydrateOne(runId) {
|
|
473
|
+
if (runs.has(runId))
|
|
474
|
+
return 'skipped';
|
|
475
|
+
const replay = await eventLog.replay(runId, null);
|
|
476
|
+
if (replay.kind !== 'ok' || replay.entries.length === 0)
|
|
477
|
+
return 'skipped';
|
|
478
|
+
const { record, idempotencyKey, isTerminal } = rehydratedRunRecord(runId, replay.entries);
|
|
479
|
+
runs.set(runId, record);
|
|
480
|
+
if (idempotencyKey !== undefined)
|
|
481
|
+
idempotencyIndex.set(idempotencyKey, runId);
|
|
482
|
+
// `eventLog.listRunIds()` returns every run id the durable log has ever seen, with no bound of
|
|
483
|
+
// its own — without this, a restart would re-populate `runs` with the daemon's entire terminal
|
|
484
|
+
// run history on every boot, reproducing the same unbounded growth this module now guards
|
|
485
|
+
// against at runtime.
|
|
486
|
+
if (isTerminal)
|
|
487
|
+
trackTerminalRun(runId, record, record.status.endedAt ?? Date.now());
|
|
488
|
+
return isTerminal ? 'terminal' : 'non-terminal';
|
|
489
|
+
}
|
|
152
490
|
const lifecycle = {
|
|
153
491
|
async rehydrate() {
|
|
154
492
|
if (hydration)
|
|
@@ -156,45 +494,9 @@ export function createRunLifecycle(input) {
|
|
|
156
494
|
hydration = (async () => {
|
|
157
495
|
const rehydratedNonTerminalRunIds = [];
|
|
158
496
|
for (const runId of await eventLog.listRunIds()) {
|
|
159
|
-
if (
|
|
160
|
-
continue;
|
|
161
|
-
const replay = await eventLog.replay(runId, null);
|
|
162
|
-
if (replay.kind !== 'ok' || replay.entries.length === 0)
|
|
163
|
-
continue;
|
|
164
|
-
const entries = replay.entries;
|
|
165
|
-
const startEntry = entries.find((entry) => entry.event === 'start');
|
|
166
|
-
const endEntry = [...entries].reverse().find((entry) => entry.event === 'end');
|
|
167
|
-
const startData = startEntry && isRecord(startEntry.data) ? startEntry.data : null;
|
|
168
|
-
const firstEntry = entries[0];
|
|
169
|
-
const lastEntry = entries[entries.length - 1];
|
|
170
|
-
const terminal = endEntry === undefined ? null : terminalStateFromEndEntry(endEntry);
|
|
171
|
-
const contextRef = typeof startData?.contextRef === 'string' ? startData.contextRef : undefined;
|
|
172
|
-
const idempotencyKey = typeof startData?.idempotencyKey === 'string' ? startData.idempotencyKey : undefined;
|
|
173
|
-
const record = {
|
|
174
|
-
contextRef,
|
|
175
|
-
status: {
|
|
176
|
-
id: runId,
|
|
177
|
-
state: terminal?.state ?? 'running',
|
|
178
|
-
startedAt: startEntry?.recordedAt ?? firstEntry.recordedAt,
|
|
179
|
-
updatedAt: lastEntry.recordedAt,
|
|
180
|
-
endedAt: endEntry?.recordedAt,
|
|
181
|
-
},
|
|
182
|
-
resumable: terminal?.resumable ?? false,
|
|
183
|
-
cancelRequested: false,
|
|
184
|
-
lastCancelRequest: undefined,
|
|
185
|
-
cancelListeners: new Set(),
|
|
186
|
-
subscribers: new Set(),
|
|
187
|
-
terminalWaiters: [],
|
|
188
|
-
terminalEndEntry: endEntry,
|
|
189
|
-
watchdog: undefined,
|
|
190
|
-
startPromise: undefined,
|
|
191
|
-
finishPromise: undefined,
|
|
192
|
-
};
|
|
193
|
-
runs.set(runId, record);
|
|
194
|
-
if (idempotencyKey !== undefined)
|
|
195
|
-
idempotencyIndex.set(idempotencyKey, runId);
|
|
196
|
-
if (terminal === null)
|
|
497
|
+
if (await rehydrateOne(runId) === 'non-terminal') {
|
|
197
498
|
rehydratedNonTerminalRunIds.push(runId);
|
|
499
|
+
}
|
|
198
500
|
}
|
|
199
501
|
// A process restart cannot retain an in-memory child process or cancellation listener.
|
|
200
502
|
// Persist an honest terminal outcome instead of advertising an orphaned run as still live.
|
|
@@ -205,72 +507,42 @@ export function createRunLifecycle(input) {
|
|
|
205
507
|
return hydration;
|
|
206
508
|
},
|
|
207
509
|
async start(startInput) {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
510
|
+
// Synchronous on purpose: `resolveIdempotentReplayRunId` does no I/O, and the common case
|
|
511
|
+
// (no idempotencyKey, or a fresh one) must reach `runs.set()` below in the same synchronous
|
|
512
|
+
// turn as this call — not after even one microtask tick — so a caller that races `finish()`
|
|
513
|
+
// in immediately after `start()` (before ever awaiting it) still finds the record `finish()`
|
|
514
|
+
// needs. Wrapping this check in its own `await`ed async function previously broke exactly
|
|
515
|
+
// that: an `async function` always defers its continuation by a microtask even when its body
|
|
516
|
+
// never itself awaits, which shifted `runs.set()` behind the racing `finish()` call.
|
|
517
|
+
const existingRunId = resolveIdempotentReplayRunId(idempotencyIndex, startInput.idempotencyKey);
|
|
518
|
+
if (existingRunId !== undefined) {
|
|
519
|
+
const existing = requireRun(existingRunId);
|
|
520
|
+
await existing.startPromise;
|
|
521
|
+
return { run: toPublicStatus(existing), started: false };
|
|
215
522
|
}
|
|
216
523
|
const runId = startInput.runId ?? randomUUID();
|
|
217
524
|
if (runs.has(runId)) {
|
|
218
525
|
throw new Error(`RunLifecycle: run "${runId}" already exists`);
|
|
219
526
|
}
|
|
220
|
-
const
|
|
221
|
-
const record = {
|
|
222
|
-
contextRef: startInput.contextRef,
|
|
223
|
-
status: { id: runId, state: 'running', startedAt: now, updatedAt: now, endedAt: undefined },
|
|
224
|
-
resumable: false,
|
|
225
|
-
cancelRequested: false,
|
|
226
|
-
lastCancelRequest: undefined,
|
|
227
|
-
cancelListeners: new Set(),
|
|
228
|
-
subscribers: new Set(),
|
|
229
|
-
terminalWaiters: [],
|
|
230
|
-
terminalEndEntry: undefined,
|
|
231
|
-
watchdog: undefined,
|
|
232
|
-
startPromise: undefined,
|
|
233
|
-
finishPromise: undefined,
|
|
234
|
-
};
|
|
527
|
+
const record = buildNewRunRecord(startInput, runId, Date.now());
|
|
235
528
|
runs.set(runId, record);
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
contextRef: startInput.contextRef,
|
|
242
|
-
...(startInput.agentId !== undefined ? { agentId: startInput.agentId } : {}),
|
|
243
|
-
...(startInput.idempotencyKey !== undefined ? { idempotencyKey: startInput.idempotencyKey } : {}),
|
|
244
|
-
};
|
|
245
|
-
const startPromise = appendEvent(runId, record, 'start', startPayload).then(() => undefined);
|
|
246
|
-
record.startPromise = startPromise;
|
|
247
|
-
try {
|
|
248
|
-
await startPromise;
|
|
249
|
-
record.startPromise = undefined;
|
|
250
|
-
}
|
|
251
|
-
catch (error) {
|
|
252
|
-
runs.delete(runId);
|
|
253
|
-
if (startInput.idempotencyKey !== undefined &&
|
|
254
|
-
idempotencyIndex.get(startInput.idempotencyKey) === runId) {
|
|
255
|
-
idempotencyIndex.delete(startInput.idempotencyKey);
|
|
256
|
-
}
|
|
257
|
-
throw error;
|
|
258
|
-
}
|
|
259
|
-
if (startInput.inactivityTimeoutMs !== undefined) {
|
|
260
|
-
record.watchdog = createInactivityWatchdog({
|
|
261
|
-
timeoutMs: startInput.inactivityTimeoutMs,
|
|
262
|
-
onTimeout: () => {
|
|
263
|
-
void handleInactivityTimeout(runId);
|
|
264
|
-
},
|
|
265
|
-
});
|
|
266
|
-
}
|
|
529
|
+
registerIdempotencyKeyIfPresent(idempotencyIndex, startInput.idempotencyKey, runId);
|
|
530
|
+
await appendStartOrRollback(runId, record, startInput);
|
|
531
|
+
armWatchdogIfConfigured(record, startInput.inactivityTimeoutMs, () => {
|
|
532
|
+
void handleInactivityTimeout(runId);
|
|
533
|
+
});
|
|
267
534
|
return { run: toPublicStatus(record), started: true };
|
|
268
535
|
},
|
|
269
536
|
async get(runId) {
|
|
537
|
+
await settlePendingStart(runs.get(runId));
|
|
538
|
+
// Re-read rather than reusing the record above: a failed start deletes it (see `start()`'s
|
|
539
|
+
// unwind), and reporting a run whose durable `'start'` entry does not exist would advertise a
|
|
540
|
+
// run no restart could ever rehydrate.
|
|
270
541
|
const record = runs.get(runId);
|
|
271
542
|
return record ? toPublicStatus(record) : undefined;
|
|
272
543
|
},
|
|
273
544
|
async list(contextRef) {
|
|
545
|
+
await Promise.all(Array.from(runs.values(), settlePendingStart));
|
|
274
546
|
const all = Array.from(runs.values());
|
|
275
547
|
const filtered = contextRef === undefined ? all : all.filter((record) => record.contextRef === contextRef);
|
|
276
548
|
return filtered.map(toPublicStatus);
|
|
@@ -343,6 +615,7 @@ export function createRunLifecycle(input) {
|
|
|
343
615
|
record.status.endedAt = endEntry.recordedAt;
|
|
344
616
|
record.resumable = finishInput.resumable;
|
|
345
617
|
record.terminalEndEntry = endEntry;
|
|
618
|
+
trackTerminalRun(finishInput.runId, record, endEntry.recordedAt);
|
|
346
619
|
const endEvent = toRunEvent(finishInput.runId, endEntry);
|
|
347
620
|
notifySubscribers(record, endEvent);
|
|
348
621
|
resolveTerminalWaiters(record);
|
|
@@ -364,6 +637,7 @@ export function createRunLifecycle(input) {
|
|
|
364
637
|
if (!eligible) {
|
|
365
638
|
return { run: toPublicStatus(record), resumed: false };
|
|
366
639
|
}
|
|
640
|
+
untrackTerminalRun(runId, record);
|
|
367
641
|
const now = Date.now();
|
|
368
642
|
record.status.state = 'running';
|
|
369
643
|
record.status.updatedAt = now;
|
|
@@ -380,6 +654,12 @@ export function createRunLifecycle(input) {
|
|
|
380
654
|
},
|
|
381
655
|
async waitForTerminal(runId) {
|
|
382
656
|
const record = requireRun(runId);
|
|
657
|
+
// Awaited (and *propagated*, unlike in `get`/`list`) before any waiter is registered. A run
|
|
658
|
+
// whose durable start append rejects is unwound, and nothing ever calls `finish()` for it — so
|
|
659
|
+
// a waiter parked inside that window would never be resolved by anyone. Failing the wait with
|
|
660
|
+
// the same error that failed the start is the only honest terminal answer, and awaiting first
|
|
661
|
+
// means `terminalWaiters` can only ever hold waiters for a run that really does exist.
|
|
662
|
+
await record.startPromise;
|
|
383
663
|
if (isTerminalRunState(record.status.state)) {
|
|
384
664
|
return toPublicStatus(record);
|
|
385
665
|
}
|
|
@@ -411,32 +691,14 @@ export function createRunLifecycle(input) {
|
|
|
411
691
|
record.subscribers.delete(subscriber);
|
|
412
692
|
return replay;
|
|
413
693
|
}
|
|
414
|
-
const deliveredEventIds =
|
|
415
|
-
|
|
416
|
-
const event = toRunEvent(runId, entry);
|
|
417
|
-
deliveredEventIds.add(event.eventId);
|
|
418
|
-
onEvent(event);
|
|
419
|
-
}
|
|
420
|
-
for (const event of bufferedLiveEvents) {
|
|
421
|
-
if (!deliveredEventIds.has(event.eventId)) {
|
|
422
|
-
deliveredEventIds.add(event.eventId);
|
|
423
|
-
onEvent(event);
|
|
424
|
-
}
|
|
425
|
-
}
|
|
694
|
+
const deliveredEventIds = deliverReplayedEvents(runId, replay.entries, onEvent);
|
|
695
|
+
deliverUndeliveredEvents(bufferedLiveEvents, deliveredEventIds, onEvent);
|
|
426
696
|
replaying = false;
|
|
427
697
|
const terminal = isTerminalRunState(record.status.state);
|
|
428
698
|
if (terminal && record.terminalEndEntry) {
|
|
429
|
-
|
|
430
|
-
if (!deliveredEventIds.has(terminalEvent.eventId)) {
|
|
431
|
-
deliveredEventIds.add(terminalEvent.eventId);
|
|
432
|
-
onEvent(terminalEvent);
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
if (terminal) {
|
|
436
|
-
record.subscribers.delete(subscriber);
|
|
437
|
-
return { kind: 'ok', unsubscribe: () => { } };
|
|
699
|
+
deliverUndeliveredEvents([toRunEvent(runId, record.terminalEndEntry)], deliveredEventIds, onEvent);
|
|
438
700
|
}
|
|
439
|
-
return
|
|
701
|
+
return finishStreamSubscription(record, subscriber, terminal);
|
|
440
702
|
}
|
|
441
703
|
catch (error) {
|
|
442
704
|
// Subscription is installed before durable replay to close the replay→live race. Any
|