@womp/spark-agent 4.2.0 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -0
- package/dist/server/index.d.ts +2 -2
- package/dist/server/index.js +1 -1
- package/dist/server/service.d.ts +64 -4
- package/dist/server/service.js +269 -46
- package/dist/server/sqlite-store.d.ts +1 -0
- package/dist/server/sqlite-store.js +3 -0
- package/dist/server/store.d.ts +1 -0
- package/dist/server/store.js +7 -0
- package/dist/server/types.d.ts +7 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -372,6 +372,27 @@ Releases use a manual package publication; no separate release service is requir
|
|
|
372
372
|
- Server cancellation waits for active execution to settle before reporting its terminal state.
|
|
373
373
|
- Includes the latest geometry material, scalar-port, and primitive improvements from main.
|
|
374
374
|
|
|
375
|
+
### 4.2.2
|
|
376
|
+
|
|
377
|
+
The published 4.2.1 was packed from the 4.2.0 sources and differs from 4.2.0 only in its version
|
|
378
|
+
number, so none of the changes below are in it. Use 4.2.2.
|
|
379
|
+
|
|
380
|
+
Compatible: both additions are optional, and a host that sets neither behaves as before except that a
|
|
381
|
+
run whose progress cannot be stored now ends instead of staying `running`.
|
|
382
|
+
|
|
383
|
+
- A run whose stream chunk, `running` status or ending cannot be stored ends `failed` with a sentence
|
|
384
|
+
instead of staying `running` until a restart. The final status and the conversation release are
|
|
385
|
+
retried in the background for about five minutes; a message or a Stop for that conversation stores
|
|
386
|
+
the ending at once.
|
|
387
|
+
- `SparkAgentServiceOptions.endOrphanedRuns` (default `false`): a message or a Stop ends an unfinished
|
|
388
|
+
active run that no execution in this process holds, with "This run stopped unexpectedly and could
|
|
389
|
+
not be finished." Turn it on only when one process owns the store; on a shared store it would end
|
|
390
|
+
another process's live runs.
|
|
391
|
+
- `SparkAgentStore.releaseActiveRun(conversationId, runId, updatedAt)` (optional): clears the active
|
|
392
|
+
run only while it still names that run. The memory and SQLite stores implement it. A store without
|
|
393
|
+
it has the read-and-save release serialised against the service's own submits, so a late retry
|
|
394
|
+
cannot overwrite a newer message.
|
|
395
|
+
|
|
375
396
|
### 4.2.0
|
|
376
397
|
|
|
377
398
|
Incompatible: a color that can only be a 0-1 one is now refused where it used to be stored as
|
package/dist/server/index.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export { MemorySparkArtifactStore } from "./artifacts.js";
|
|
2
2
|
export type { SparkArtifactStore, StoredSparkArtifact } from "./artifacts.js";
|
|
3
3
|
export { createSparkRunExecutor, type ResolvedSparkModel, type SparkRunExecutorOptions, } from "./executor.js";
|
|
4
|
-
export { SparkAgentService, SparkServiceError } from "./service.js";
|
|
5
|
-
export type { SubmitSparkMessageInput } from "./service.js";
|
|
4
|
+
export { DEFAULT_ENDING_RETRY_DELAYS_MS, RUN_INTERRUPTED_ERROR, RUN_NOT_SAVED_ERROR, SparkAgentService, SparkServiceError, } from "./service.js";
|
|
5
|
+
export type { SparkAgentServiceOptions, SubmitSparkMessageInput } from "./service.js";
|
|
6
6
|
export { MemorySparkAgentStore } from "./store.js";
|
|
7
7
|
export { SqliteSparkStore } from "./sqlite-store.js";
|
|
8
8
|
export { createStandaloneSparkAgentApp, type StandaloneSparkAgentOptions, type StandaloneSparkAgentApp, } from "./standalone.js";
|
package/dist/server/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { MemorySparkArtifactStore } from "./artifacts.js";
|
|
2
2
|
export { createSparkRunExecutor, } from "./executor.js";
|
|
3
|
-
export { SparkAgentService, SparkServiceError } from "./service.js";
|
|
3
|
+
export { DEFAULT_ENDING_RETRY_DELAYS_MS, RUN_INTERRUPTED_ERROR, RUN_NOT_SAVED_ERROR, SparkAgentService, SparkServiceError, } from "./service.js";
|
|
4
4
|
export { MemorySparkAgentStore } from "./store.js";
|
|
5
5
|
export { SqliteSparkStore } from "./sqlite-store.js";
|
|
6
6
|
export { createStandaloneSparkAgentApp, } from "./standalone.js";
|
package/dist/server/service.d.ts
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
import { type SparkUIMessage, type StoredSparkChunk } from "../ui.js";
|
|
2
2
|
import type { SparkAgentStore, SparkConversation, SparkConversationSummary, SparkRun, SparkRunExecutor, SparkRunSettings } from "./types.js";
|
|
3
|
+
/** Why a run ended when its progress could not be stored. */
|
|
4
|
+
export declare const RUN_NOT_SAVED_ERROR = "This run was stopped because its progress could not be saved.";
|
|
5
|
+
/** Why a run ended that no execution in this process was running any more. */
|
|
6
|
+
export declare const RUN_INTERRUPTED_ERROR = "This run stopped unexpectedly and could not be finished.";
|
|
7
|
+
/** How long to wait before each further attempt to store a run's ending. About 5 minutes in all. */
|
|
8
|
+
export declare const DEFAULT_ENDING_RETRY_DELAYS_MS: readonly number[];
|
|
9
|
+
export type SparkAgentServiceOptions = {
|
|
10
|
+
store: SparkAgentStore;
|
|
11
|
+
executeRun: SparkRunExecutor;
|
|
12
|
+
/**
|
|
13
|
+
* Ends, with `RUN_INTERRUPTED_ERROR`, an unfinished active run that no
|
|
14
|
+
* execution in this process is running, when a message is submitted to its
|
|
15
|
+
* conversation or it is cancelled. Only for a host that is the only process
|
|
16
|
+
* running runs from its store: another process's live run would be ended.
|
|
17
|
+
*/
|
|
18
|
+
endOrphanedRuns?: boolean;
|
|
19
|
+
/** Delays between the attempts to store an ending that could not be stored. */
|
|
20
|
+
endingRetryDelaysMs?: readonly number[];
|
|
21
|
+
};
|
|
3
22
|
export declare class SparkServiceError extends Error {
|
|
4
23
|
readonly statusCode: number;
|
|
5
24
|
constructor(statusCode: number, message: string);
|
|
@@ -18,10 +37,15 @@ export declare class SparkAgentService {
|
|
|
18
37
|
private readonly controllers;
|
|
19
38
|
private readonly listeners;
|
|
20
39
|
private readonly executions;
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
40
|
+
/** Runs this process started and has not finished ending. */
|
|
41
|
+
private readonly live;
|
|
42
|
+
/** Runs whose execution is over but whose ending is still being stored. */
|
|
43
|
+
private readonly ending;
|
|
44
|
+
private readonly endOrphanedRuns;
|
|
45
|
+
/** Per conversation: the tail of its read-modify-write sections in this process. */
|
|
46
|
+
private readonly conversationTurns;
|
|
47
|
+
private readonly endingRetryDelaysMs;
|
|
48
|
+
constructor(options: SparkAgentServiceOptions);
|
|
25
49
|
createConversation(ownerId: string, input?: {
|
|
26
50
|
id?: string;
|
|
27
51
|
title?: string;
|
|
@@ -36,9 +60,45 @@ export declare class SparkAgentService {
|
|
|
36
60
|
recoverInterruptedRun(runId: string): Promise<void>;
|
|
37
61
|
close(): Promise<void>;
|
|
38
62
|
submitMessage(input: SubmitSparkMessageInput): Promise<SparkRun>;
|
|
63
|
+
private submitInTurn;
|
|
39
64
|
cancelRun(ownerId: string, runId: string): Promise<SparkRun>;
|
|
40
65
|
streamRun(ownerId: string, runId: string, afterSequence: number, signal: AbortSignal): AsyncGenerator<StoredSparkChunk>;
|
|
41
66
|
private runInBackground;
|
|
67
|
+
/** Ends a run whose execution, or whose own bookkeeping, failed. */
|
|
68
|
+
private endAfterError;
|
|
69
|
+
/**
|
|
70
|
+
* The last resort when a run's ending could not be written in full: the run
|
|
71
|
+
* row gets its terminal status and the conversation lets go of the run, so
|
|
72
|
+
* it is not left unfinished. Its stored chunks may lack the ending. A store
|
|
73
|
+
* that is still failing is retried in the background with backoff
|
|
74
|
+
* (`endingRetryDelaysMs`); until then, a message or a Stop for the
|
|
75
|
+
* conversation ends the run at once (`endIfNotRunning`).
|
|
76
|
+
*/
|
|
77
|
+
private endWithoutHistory;
|
|
78
|
+
/** `storeEnding`, answering whether it was stored rather than throwing. */
|
|
79
|
+
private tryStoreEnding;
|
|
80
|
+
/**
|
|
81
|
+
* Runs `action` after every earlier section for the conversation in this
|
|
82
|
+
* process has finished, so a read-modify-write of it (a submit, a released
|
|
83
|
+
* run) never saves back a copy another one has since replaced.
|
|
84
|
+
*/
|
|
85
|
+
private inConversationTurn;
|
|
86
|
+
/**
|
|
87
|
+
* Writes a run's terminal status unless the stored run has already ended,
|
|
88
|
+
* then clears the conversation's active run while it still names the run.
|
|
89
|
+
* Read back first, so a late write never replaces a newer ending or a
|
|
90
|
+
* newer active run.
|
|
91
|
+
*/
|
|
92
|
+
private storeEnding;
|
|
93
|
+
/** Clears the conversation's active run while it is still `runId`, and nothing else. */
|
|
94
|
+
private releaseRun;
|
|
95
|
+
/**
|
|
96
|
+
* Ends an unfinished run that is not being executed: one whose ending this
|
|
97
|
+
* process is still retrying, or, with `endOrphanedRuns`, one no execution in
|
|
98
|
+
* this process is running at all (its process stopped). Answers whether it
|
|
99
|
+
* was ended; throws when the store is still failing.
|
|
100
|
+
*/
|
|
101
|
+
private endIfNotRunning;
|
|
42
102
|
private finishRun;
|
|
43
103
|
private materializeMessage;
|
|
44
104
|
private appendStatus;
|
package/dist/server/service.js
CHANGED
|
@@ -7,6 +7,23 @@ const TERMINAL_STATUSES = new Set([
|
|
|
7
7
|
"cancelled",
|
|
8
8
|
"incomplete",
|
|
9
9
|
]);
|
|
10
|
+
/** Why a run ended when its progress could not be stored. */
|
|
11
|
+
export const RUN_NOT_SAVED_ERROR = "This run was stopped because its progress could not be saved.";
|
|
12
|
+
/** Why a run ended that no execution in this process was running any more. */
|
|
13
|
+
export const RUN_INTERRUPTED_ERROR = "This run stopped unexpectedly and could not be finished.";
|
|
14
|
+
/** How long to wait before each further attempt to store a run's ending. About 5 minutes in all. */
|
|
15
|
+
export const DEFAULT_ENDING_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 15_000, 30_000, 60_000, 60_000, 60_000, 60_000];
|
|
16
|
+
/** Waits without keeping the process alive. */
|
|
17
|
+
function sleep(ms) {
|
|
18
|
+
return new Promise((resolve) => {
|
|
19
|
+
const timer = setTimeout(resolve, ms);
|
|
20
|
+
if (typeof timer === "object" && timer !== null && "unref" in timer)
|
|
21
|
+
timer.unref();
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
function runNotSaved(cause) {
|
|
25
|
+
return new Error(RUN_NOT_SAVED_ERROR, { cause });
|
|
26
|
+
}
|
|
10
27
|
export class SparkServiceError extends Error {
|
|
11
28
|
statusCode;
|
|
12
29
|
constructor(statusCode, message) {
|
|
@@ -70,9 +87,19 @@ export class SparkAgentService {
|
|
|
70
87
|
controllers = new Map();
|
|
71
88
|
listeners = new Map();
|
|
72
89
|
executions = new Map();
|
|
90
|
+
/** Runs this process started and has not finished ending. */
|
|
91
|
+
live = new Set();
|
|
92
|
+
/** Runs whose execution is over but whose ending is still being stored. */
|
|
93
|
+
ending = new Map();
|
|
94
|
+
endOrphanedRuns;
|
|
95
|
+
/** Per conversation: the tail of its read-modify-write sections in this process. */
|
|
96
|
+
conversationTurns = new Map();
|
|
97
|
+
endingRetryDelaysMs;
|
|
73
98
|
constructor(options) {
|
|
74
99
|
this.store = options.store;
|
|
75
100
|
this.executeRun = options.executeRun;
|
|
101
|
+
this.endOrphanedRuns = options.endOrphanedRuns ?? false;
|
|
102
|
+
this.endingRetryDelaysMs = options.endingRetryDelaysMs ?? DEFAULT_ENDING_RETRY_DELAYS_MS;
|
|
76
103
|
}
|
|
77
104
|
async createConversation(ownerId, input) {
|
|
78
105
|
const createdAt = now();
|
|
@@ -142,12 +169,18 @@ export class SparkAgentService {
|
|
|
142
169
|
await Promise.all([...this.executions.values()]);
|
|
143
170
|
}
|
|
144
171
|
async submitMessage(input) {
|
|
172
|
+
return this.inConversationTurn(input.conversationId, () => this.submitInTurn(input));
|
|
173
|
+
}
|
|
174
|
+
async submitInTurn(input) {
|
|
145
175
|
const conversation = await this.getConversation(input.ownerId, input.conversationId);
|
|
146
176
|
const activeRun = conversation.activeRunId === undefined
|
|
147
177
|
? undefined
|
|
148
178
|
: await this.store.getRun(conversation.activeRunId);
|
|
149
179
|
if (activeRun !== undefined && TERMINAL_STATUSES.has(activeRun.status) === false) {
|
|
150
|
-
|
|
180
|
+
if (!(await this.endIfNotRunning(activeRun, true)))
|
|
181
|
+
throw new SparkServiceError(409, "This conversation already has an active run.");
|
|
182
|
+
if (conversation.activeRunId === activeRun.id)
|
|
183
|
+
conversation.activeRunId = undefined;
|
|
151
184
|
}
|
|
152
185
|
let inputMessage;
|
|
153
186
|
if (input.trigger === "regenerate-message") {
|
|
@@ -193,25 +226,47 @@ export class SparkAgentService {
|
|
|
193
226
|
};
|
|
194
227
|
conversation.activeRunId = run.id;
|
|
195
228
|
conversation.updatedAt = createdAt;
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
229
|
+
this.live.add(run.id);
|
|
230
|
+
try {
|
|
231
|
+
await this.store.saveConversation(conversation);
|
|
232
|
+
await this.store.createRun(run);
|
|
233
|
+
}
|
|
234
|
+
catch (error) {
|
|
235
|
+
this.live.delete(run.id);
|
|
236
|
+
throw error;
|
|
237
|
+
}
|
|
238
|
+
try {
|
|
239
|
+
await this.appendChunk(run, {
|
|
240
|
+
type: "start",
|
|
241
|
+
messageId: run.assistantMessageId,
|
|
242
|
+
messageMetadata: {
|
|
243
|
+
createdAt,
|
|
244
|
+
runId: run.id,
|
|
245
|
+
runStatus: "queued",
|
|
246
|
+
modelId: run.settings.modelId,
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
await this.appendStatus(run, "queued");
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
// The run exists and the conversation names it, so it is ended here
|
|
253
|
+
// rather than left `queued` with the conversation refusing every later
|
|
254
|
+
// message.
|
|
255
|
+
const failure = runNotSaved(error);
|
|
256
|
+
const diagnostics = sparkErrorDiagnostics(failure);
|
|
257
|
+
await this.endWithoutHistory(run, { status: "failed", error: diagnostics.message, errorDiagnostics: diagnostics }, error, true);
|
|
258
|
+
throw failure;
|
|
259
|
+
}
|
|
209
260
|
const controller = new AbortController();
|
|
210
261
|
this.controllers.set(run.id, controller);
|
|
211
262
|
const execution = this.runInBackground(run, conversation, controller);
|
|
212
263
|
this.executions.set(run.id, execution);
|
|
213
264
|
void execution
|
|
214
|
-
.finally(() =>
|
|
265
|
+
.finally(() => {
|
|
266
|
+
this.executions.delete(run.id);
|
|
267
|
+
if (!this.ending.has(run.id))
|
|
268
|
+
this.live.delete(run.id);
|
|
269
|
+
})
|
|
215
270
|
.catch((error) => {
|
|
216
271
|
console.error("Spark run persistence failed", error);
|
|
217
272
|
});
|
|
@@ -223,8 +278,11 @@ export class SparkAgentService {
|
|
|
223
278
|
return run;
|
|
224
279
|
const controller = this.controllers.get(run.id);
|
|
225
280
|
const execution = this.executions.get(run.id);
|
|
226
|
-
if (!controller || !execution)
|
|
281
|
+
if (!controller || !execution) {
|
|
282
|
+
if (await this.endIfNotRunning(run))
|
|
283
|
+
return this.getRun(ownerId, runId);
|
|
227
284
|
throw new Error("The run's executor is unavailable; cancellation could not be confirmed.");
|
|
285
|
+
}
|
|
228
286
|
controller.abort(new Error("Run cancelled by the user."));
|
|
229
287
|
// Drain receipts and pending host work before publishing a terminal state.
|
|
230
288
|
await execution;
|
|
@@ -287,21 +345,26 @@ export class SparkAgentService {
|
|
|
287
345
|
}
|
|
288
346
|
}
|
|
289
347
|
async runInBackground(run, conversation, controller) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
updatedAt: startedAt,
|
|
296
|
-
};
|
|
297
|
-
await this.store.saveRun(run);
|
|
298
|
-
const runningChunk = {
|
|
299
|
-
type: "data-run-status",
|
|
300
|
-
id: `run-status-${run.id}`,
|
|
301
|
-
data: { runId: run.id, status: "running" },
|
|
302
|
-
};
|
|
303
|
-
await this.appendChunk(run, runningChunk);
|
|
348
|
+
// Aborts the execution when its progress cannot be stored. Unlike
|
|
349
|
+
// `controller`, it does not mean the user cancelled.
|
|
350
|
+
const unsaved = new AbortController();
|
|
351
|
+
// A terminal status already in the run's chunks; a later failure keeps it.
|
|
352
|
+
let written;
|
|
304
353
|
try {
|
|
354
|
+
const startedAt = now();
|
|
355
|
+
run = {
|
|
356
|
+
...run,
|
|
357
|
+
status: "running",
|
|
358
|
+
startedAt,
|
|
359
|
+
updatedAt: startedAt,
|
|
360
|
+
};
|
|
361
|
+
await this.store.saveRun(run);
|
|
362
|
+
const runningChunk = {
|
|
363
|
+
type: "data-run-status",
|
|
364
|
+
id: `run-status-${run.id}`,
|
|
365
|
+
data: { runId: run.id, status: "running" },
|
|
366
|
+
};
|
|
367
|
+
await this.appendChunk(run, runningChunk);
|
|
305
368
|
const execution = await this.executeRun({
|
|
306
369
|
ownerId: conversation.ownerId,
|
|
307
370
|
conversationId: conversation.id,
|
|
@@ -309,13 +372,33 @@ export class SparkAgentService {
|
|
|
309
372
|
assistantMessageId: run.assistantMessageId,
|
|
310
373
|
uiMessages: conversation.messages,
|
|
311
374
|
settings: run.settings,
|
|
312
|
-
signal: controller.signal,
|
|
375
|
+
signal: AbortSignal.any([controller.signal, unsaved.signal]),
|
|
313
376
|
});
|
|
377
|
+
// A chunk that cannot be stored aborts the execution, and the stream is
|
|
378
|
+
// still read to its end: an execution settles its result only once its
|
|
379
|
+
// stream has been read, so a reader that stopped here would leave the
|
|
380
|
+
// run waiting on its result for good.
|
|
381
|
+
let persistError;
|
|
314
382
|
const consume = (async () => {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
383
|
+
const reader = execution.stream.getReader();
|
|
384
|
+
try {
|
|
385
|
+
for (;;) {
|
|
386
|
+
const next = await reader.read();
|
|
387
|
+
if (next.done)
|
|
388
|
+
return;
|
|
389
|
+
if (next.value.type === "start" || persistError !== undefined)
|
|
390
|
+
continue;
|
|
391
|
+
try {
|
|
392
|
+
await this.appendChunk(run, next.value);
|
|
393
|
+
}
|
|
394
|
+
catch (error) {
|
|
395
|
+
persistError = error;
|
|
396
|
+
unsaved.abort(runNotSaved(error));
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
finally {
|
|
401
|
+
reader.releaseLock();
|
|
319
402
|
}
|
|
320
403
|
})().then(() => undefined, (error) => error);
|
|
321
404
|
let executionError;
|
|
@@ -327,27 +410,42 @@ export class SparkAgentService {
|
|
|
327
410
|
executionError = error;
|
|
328
411
|
}
|
|
329
412
|
const streamError = await consume;
|
|
413
|
+
if (persistError !== undefined)
|
|
414
|
+
throw runNotSaved(persistError);
|
|
330
415
|
if (executionError !== undefined)
|
|
331
416
|
throw executionError;
|
|
332
417
|
if (streamError !== undefined)
|
|
333
418
|
throw streamError;
|
|
334
|
-
|
|
335
|
-
await this.appendStatus(run, "cancelled");
|
|
336
|
-
await this.finishRun(run, conversation, "cancelled");
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
const status = result?.status ?? "incomplete";
|
|
419
|
+
const status = controller.signal.aborted ? "cancelled" : (result?.status ?? "incomplete");
|
|
340
420
|
await this.appendStatus(run, status);
|
|
421
|
+
written = status;
|
|
341
422
|
await this.finishRun(run, conversation, status);
|
|
342
423
|
}
|
|
343
424
|
catch (error) {
|
|
344
|
-
if (
|
|
425
|
+
if (written !== undefined)
|
|
426
|
+
await this.endWithoutHistory(run, { status: written }, error);
|
|
427
|
+
else
|
|
428
|
+
await this.endAfterError(run, conversation, controller, error);
|
|
429
|
+
}
|
|
430
|
+
finally {
|
|
431
|
+
this.controllers.delete(run.id);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
/** Ends a run whose execution, or whose own bookkeeping, failed. */
|
|
435
|
+
async endAfterError(run, conversation, controller, error) {
|
|
436
|
+
if (controller.signal.aborted) {
|
|
437
|
+
try {
|
|
345
438
|
await this.appendStatus(run, "cancelled");
|
|
346
439
|
await this.finishRun(run, conversation, "cancelled");
|
|
347
|
-
return;
|
|
348
440
|
}
|
|
349
|
-
|
|
350
|
-
|
|
441
|
+
catch (endingError) {
|
|
442
|
+
await this.endWithoutHistory(run, { status: "cancelled" }, endingError);
|
|
443
|
+
}
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
const diagnostics = sparkErrorDiagnostics(error);
|
|
447
|
+
const message = diagnostics.message;
|
|
448
|
+
try {
|
|
351
449
|
const failedChunk = {
|
|
352
450
|
type: "data-run-status",
|
|
353
451
|
id: `run-status-${run.id}`,
|
|
@@ -357,9 +455,134 @@ export class SparkAgentService {
|
|
|
357
455
|
await this.appendChunk(run, failedChunk);
|
|
358
456
|
await this.finishRun(run, conversation, "failed", message, diagnostics);
|
|
359
457
|
}
|
|
458
|
+
catch (endingError) {
|
|
459
|
+
await this.endWithoutHistory(run, { status: "failed", error: message, errorDiagnostics: diagnostics }, endingError);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
/**
|
|
463
|
+
* The last resort when a run's ending could not be written in full: the run
|
|
464
|
+
* row gets its terminal status and the conversation lets go of the run, so
|
|
465
|
+
* it is not left unfinished. Its stored chunks may lack the ending. A store
|
|
466
|
+
* that is still failing is retried in the background with backoff
|
|
467
|
+
* (`endingRetryDelaysMs`); until then, a message or a Stop for the
|
|
468
|
+
* conversation ends the run at once (`endIfNotRunning`).
|
|
469
|
+
*/
|
|
470
|
+
async endWithoutHistory(run, ending, cause, inTurn = false) {
|
|
471
|
+
console.error("Spark run ending could not be stored in full", cause);
|
|
472
|
+
this.ending.set(run.id, ending);
|
|
473
|
+
if (await this.tryStoreEnding(run, ending, inTurn)) {
|
|
474
|
+
this.ending.delete(run.id);
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
477
|
+
void (async () => {
|
|
478
|
+
for (const delay of this.endingRetryDelaysMs) {
|
|
479
|
+
await sleep(delay);
|
|
480
|
+
// Ended meanwhile by a message or a Stop (`endIfNotRunning`).
|
|
481
|
+
if (!this.ending.has(run.id))
|
|
482
|
+
break;
|
|
483
|
+
if (await this.tryStoreEnding(run, ending))
|
|
484
|
+
break;
|
|
485
|
+
}
|
|
486
|
+
this.ending.delete(run.id);
|
|
487
|
+
if (!this.executions.has(run.id))
|
|
488
|
+
this.live.delete(run.id);
|
|
489
|
+
})();
|
|
490
|
+
}
|
|
491
|
+
/** `storeEnding`, answering whether it was stored rather than throwing. */
|
|
492
|
+
async tryStoreEnding(run, ending, inTurn = false) {
|
|
493
|
+
try {
|
|
494
|
+
await this.storeEnding(run.id, ending, inTurn);
|
|
495
|
+
return true;
|
|
496
|
+
}
|
|
497
|
+
catch (error) {
|
|
498
|
+
console.error("Spark run ending could not be stored", error);
|
|
499
|
+
return false;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Runs `action` after every earlier section for the conversation in this
|
|
504
|
+
* process has finished, so a read-modify-write of it (a submit, a released
|
|
505
|
+
* run) never saves back a copy another one has since replaced.
|
|
506
|
+
*/
|
|
507
|
+
async inConversationTurn(conversationId, action) {
|
|
508
|
+
const previous = this.conversationTurns.get(conversationId) ?? Promise.resolve();
|
|
509
|
+
let done = () => undefined;
|
|
510
|
+
const turn = new Promise((resolve) => {
|
|
511
|
+
done = resolve;
|
|
512
|
+
});
|
|
513
|
+
const tail = previous.then(() => turn);
|
|
514
|
+
this.conversationTurns.set(conversationId, tail);
|
|
515
|
+
await previous;
|
|
516
|
+
try {
|
|
517
|
+
return await action();
|
|
518
|
+
}
|
|
360
519
|
finally {
|
|
361
|
-
|
|
520
|
+
done();
|
|
521
|
+
if (this.conversationTurns.get(conversationId) === tail)
|
|
522
|
+
this.conversationTurns.delete(conversationId);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Writes a run's terminal status unless the stored run has already ended,
|
|
527
|
+
* then clears the conversation's active run while it still names the run.
|
|
528
|
+
* Read back first, so a late write never replaces a newer ending or a
|
|
529
|
+
* newer active run.
|
|
530
|
+
*/
|
|
531
|
+
async storeEnding(runId, ending, inTurn = false) {
|
|
532
|
+
const stored = await this.store.getRun(runId);
|
|
533
|
+
if (stored === undefined)
|
|
534
|
+
return;
|
|
535
|
+
const completedAt = now();
|
|
536
|
+
if (!TERMINAL_STATUSES.has(stored.status)) {
|
|
537
|
+
await this.store.saveRun({
|
|
538
|
+
...stored,
|
|
539
|
+
status: ending.status,
|
|
540
|
+
...(ending.error === undefined ? {} : { error: ending.error }),
|
|
541
|
+
...(ending.errorDiagnostics === undefined ? {} : { errorDiagnostics: ending.errorDiagnostics }),
|
|
542
|
+
updatedAt: completedAt,
|
|
543
|
+
completedAt,
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
await this.releaseRun(stored.conversationId, runId, completedAt, inTurn);
|
|
547
|
+
this.ending.delete(runId);
|
|
548
|
+
this.wakeRun(runId);
|
|
549
|
+
}
|
|
550
|
+
/** Clears the conversation's active run while it is still `runId`, and nothing else. */
|
|
551
|
+
async releaseRun(conversationId, runId, at, inTurn) {
|
|
552
|
+
if (this.store.releaseActiveRun !== undefined) {
|
|
553
|
+
await this.store.releaseActiveRun(conversationId, runId, at);
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
const release = async () => {
|
|
557
|
+
const conversation = await this.store.getConversation(conversationId);
|
|
558
|
+
if (conversation === undefined || conversation.activeRunId !== runId)
|
|
559
|
+
return;
|
|
560
|
+
conversation.activeRunId = undefined;
|
|
561
|
+
conversation.updatedAt = at;
|
|
562
|
+
await this.store.saveConversation(conversation);
|
|
563
|
+
};
|
|
564
|
+
if (inTurn)
|
|
565
|
+
await release();
|
|
566
|
+
else
|
|
567
|
+
await this.inConversationTurn(conversationId, release);
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* Ends an unfinished run that is not being executed: one whose ending this
|
|
571
|
+
* process is still retrying, or, with `endOrphanedRuns`, one no execution in
|
|
572
|
+
* this process is running at all (its process stopped). Answers whether it
|
|
573
|
+
* was ended; throws when the store is still failing.
|
|
574
|
+
*/
|
|
575
|
+
async endIfNotRunning(run, inTurn = false) {
|
|
576
|
+
const pending = this.ending.get(run.id);
|
|
577
|
+
if (pending !== undefined) {
|
|
578
|
+
await this.storeEnding(run.id, pending, inTurn);
|
|
579
|
+
return true;
|
|
362
580
|
}
|
|
581
|
+
if (!this.endOrphanedRuns || this.live.has(run.id))
|
|
582
|
+
return false;
|
|
583
|
+
const diagnostics = sparkErrorDiagnostics(RUN_INTERRUPTED_ERROR);
|
|
584
|
+
await this.storeEnding(run.id, { status: "failed", error: diagnostics.message, errorDiagnostics: diagnostics }, inTurn);
|
|
585
|
+
return true;
|
|
363
586
|
}
|
|
364
587
|
async finishRun(run, conversation, status, error, errorDiagnostics) {
|
|
365
588
|
const completedAt = now();
|
|
@@ -21,6 +21,7 @@ export declare class SqliteSparkStore implements SparkAgentStore, SparkArtifactS
|
|
|
21
21
|
getConversation(id: string): Promise<SparkConversation | undefined>;
|
|
22
22
|
listConversations(ownerId: string): Promise<SparkConversationSummary[]>;
|
|
23
23
|
saveConversation(value: SparkConversation): Promise<void>;
|
|
24
|
+
releaseActiveRun(conversationId: string, runId: string, updatedAt: string): Promise<void>;
|
|
24
25
|
saveConversationSettings(id: string, settings: SparkRunSettings): Promise<void>;
|
|
25
26
|
deleteConversation(id: string): Promise<boolean>;
|
|
26
27
|
createRun(value: SparkRun): Promise<void>;
|
|
@@ -163,6 +163,9 @@ export class SqliteSparkStore {
|
|
|
163
163
|
if (result.changes === 0)
|
|
164
164
|
throw new Error(`Conversation ${value.id} does not exist.`);
|
|
165
165
|
}
|
|
166
|
+
async releaseActiveRun(conversationId, runId, updatedAt) {
|
|
167
|
+
this.database.prepare("UPDATE conversations SET updated_at = ?, data = json_set(json_remove(data, '$.activeRunId'), '$.updatedAt', ?) WHERE id = ? AND json_extract(data, '$.activeRunId') = ?").run(updatedAt, updatedAt, conversationId, runId);
|
|
168
|
+
}
|
|
166
169
|
async saveConversationSettings(id, settings) {
|
|
167
170
|
const result = this.database.prepare("UPDATE conversations SET data = json_set(data, '$.settings', json(?)) WHERE id = ?").run(JSON.stringify(settings), id);
|
|
168
171
|
if (result.changes === 0)
|
package/dist/server/store.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export declare class MemorySparkAgentStore implements SparkAgentStore {
|
|
|
8
8
|
getConversation(id: string): Promise<SparkConversation | undefined>;
|
|
9
9
|
listConversations(ownerId: string): Promise<SparkConversationSummary[]>;
|
|
10
10
|
saveConversation(conversation: SparkConversation): Promise<void>;
|
|
11
|
+
releaseActiveRun(conversationId: string, runId: string, updatedAt: string): Promise<void>;
|
|
11
12
|
saveConversationSettings(id: string, settings: SparkRunSettings): Promise<void>;
|
|
12
13
|
deleteConversation(id: string): Promise<boolean>;
|
|
13
14
|
createRun(run: SparkRun): Promise<void>;
|
package/dist/server/store.js
CHANGED
|
@@ -32,6 +32,13 @@ export class MemorySparkAgentStore {
|
|
|
32
32
|
const settings = this.conversations.get(conversation.id).settings;
|
|
33
33
|
this.conversations.set(conversation.id, copy({ ...history, ...(settings ? { settings } : {}) }));
|
|
34
34
|
}
|
|
35
|
+
async releaseActiveRun(conversationId, runId, updatedAt) {
|
|
36
|
+
const conversation = this.conversations.get(conversationId);
|
|
37
|
+
if (conversation === undefined || conversation.activeRunId !== runId)
|
|
38
|
+
return;
|
|
39
|
+
conversation.activeRunId = undefined;
|
|
40
|
+
conversation.updatedAt = updatedAt;
|
|
41
|
+
}
|
|
35
42
|
async saveConversationSettings(id, settings) {
|
|
36
43
|
const conversation = this.conversations.get(id);
|
|
37
44
|
if (conversation === undefined)
|
package/dist/server/types.d.ts
CHANGED
|
@@ -57,4 +57,11 @@ export interface SparkAgentStore {
|
|
|
57
57
|
saveRun(run: SparkRun): Promise<void>;
|
|
58
58
|
appendRunChunk(runId: string, conversationId: string, chunk: StoredSparkChunk["chunk"]): Promise<StoredSparkChunk>;
|
|
59
59
|
listRunChunks(runId: string, afterSequence: number): Promise<StoredSparkChunk[]>;
|
|
60
|
+
/**
|
|
61
|
+
* Clears the conversation's active run only while it is still `runId`,
|
|
62
|
+
* touching nothing else on it. Optional: without it, the service releases
|
|
63
|
+
* a run by reading the conversation and saving it back, serialised against
|
|
64
|
+
* its own submits in this process.
|
|
65
|
+
*/
|
|
66
|
+
releaseActiveRun?(conversationId: string, runId: string, updatedAt: string): Promise<void>;
|
|
60
67
|
}
|