@jskit-ai/assistant-core 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/examples/conversation/package.json +5 -5
- package/package.json +5 -5
- package/src/server/conversation/claudeNativeTurn.js +9 -1
- package/src/server/conversation/claudeSessionCleanup.js +9 -1
- package/src/server/conversation/claudeTurn.js +1 -0
- package/src/server/conversation/codexConversations.js +65 -6
- package/src/server/conversation/codexRunJournal.js +33 -1
- package/src/server/conversation/codexRuntimeLifecycle.js +20 -1
- package/src/server/conversation/codexTurn.js +41 -4
- package/src/server/conversation/codexTurnEndings.js +9 -6
- package/src/server/conversation/nativeConversationLifecycle.js +22 -0
- package/src/server/conversation/openCodeProcessOwner.js +23 -6
- package/src/server/conversation/openCodeRuntime.js +1 -0
- package/src/server/conversation/openCodeSessionTurns.js +10 -3
- package/src/server/conversation/openCodeTemporaryConversations.js +10 -3
- package/src/server/conversation/openCodeTurn.js +15 -3
- package/src/server/conversation/openCodeTurnMonitor.js +11 -2
- package/src/server/conversation/providers/claudeDriver.js +5 -0
- package/src/server/conversation/providers/codexDriver.js +5 -0
- package/src/server/conversation/providers/opencodeDriver.js +5 -0
- package/src/server/conversation/runtime.js +1 -0
- package/test/claudeLifecycle.test.js +28 -0
- package/test/codexConversationsCheckpoint.test.js +2 -0
- package/test/codexLifecycleIsolation.test.js +2 -0
- package/test/codexShutdownStop.test.js +168 -0
- package/test/codexTemporaryProviderRelease.test.js +2 -0
- package/test/codexTurn.test.js +57 -0
- package/test/conversationSession.test.js +29 -0
- package/test/openCodeLifecycle.test.js +338 -0
|
@@ -333,7 +333,7 @@ export function createOpenCodeSessionTurns({ processes, monitors, turns, process
|
|
|
333
333
|
}
|
|
334
334
|
|
|
335
335
|
async function reconcileSessionTurn(key, target, activeRun, options, application) {
|
|
336
|
-
if (activeRun
|
|
336
|
+
if (endedOnPreviousServer(activeRun)) {
|
|
337
337
|
await processOwner.stopUnobservedSession(target, activeRun.threadId);
|
|
338
338
|
await application.writeRun({ ...activeRun, id: activeRun.turnId }, "interrupted", activeRun.observationError);
|
|
339
339
|
} else if (activeRun) {
|
|
@@ -344,7 +344,7 @@ export function createOpenCodeSessionTurns({ processes, monitors, turns, process
|
|
|
344
344
|
startedAt: normalizeNullableText(activeRun.startedAt)
|
|
345
345
|
}, options, application.monitor);
|
|
346
346
|
}
|
|
347
|
-
return Boolean(activeRun && !activeRun
|
|
347
|
+
return Boolean(activeRun && !endedOnPreviousServer(activeRun));
|
|
348
348
|
}
|
|
349
349
|
|
|
350
350
|
async function ensurePreparedSessionReadiness(prepared) {
|
|
@@ -443,8 +443,15 @@ export function createOpenCodeSessionTurns({ processes, monitors, turns, process
|
|
|
443
443
|
return prepare();
|
|
444
444
|
}
|
|
445
445
|
|
|
446
|
+
// A saved active run whose observation already ended: lost, or stopped by the
|
|
447
|
+
// previous server's shutdown. Recovery records it interrupted once native
|
|
448
|
+
// work is idle; neither is the turn's own failure.
|
|
449
|
+
function endedOnPreviousServer(run) {
|
|
450
|
+
return Boolean(run?.observationError || run?.stoppedByShutdown === true);
|
|
451
|
+
}
|
|
452
|
+
|
|
446
453
|
function needsObservationRecovery(key, run) {
|
|
447
|
-
return Boolean(run?.active && run
|
|
454
|
+
return Boolean(run?.active && endedOnPreviousServer(run) && !monitors.has(key));
|
|
448
455
|
}
|
|
449
456
|
|
|
450
457
|
async function recordStoppedObservation(key, run, threadId, writeRun) {
|
|
@@ -172,18 +172,25 @@ export function createOpenCodeTemporaryConversations({ processes, temporaryConve
|
|
|
172
172
|
const rows = openCodeMessageRows(await target.server.client.messages(conversationId));
|
|
173
173
|
const result = openCodeLastAssistantResult(rows, { readError });
|
|
174
174
|
const messages = openCodePersistentConversationMessages(rows, projection.messageId);
|
|
175
|
+
const runId = tracked?.inputMessageId || [...rows].reverse().find((row) => row.type === "user")?.id || "";
|
|
176
|
+
// The application marks the run a previous server's shutdown stopped: that
|
|
177
|
+
// run's native abort is the shutdown's interruption, not its failure, and
|
|
178
|
+
// recording it lets the application make the checkpoint it still owes.
|
|
179
|
+
const shutdownStop = projection.stoppedByShutdown;
|
|
180
|
+
const stoppedByShutdown = !tracked && Boolean(runId) && shutdownStop?.runId === runId;
|
|
175
181
|
let conversationStatus = "completed";
|
|
176
182
|
if (status.type !== "idle" || tracked?.active) conversationStatus = "inProgress";
|
|
177
|
-
else if (tracked?.interrupted) conversationStatus = "interrupted";
|
|
183
|
+
else if (tracked?.interrupted || stoppedByShutdown) conversationStatus = "interrupted";
|
|
178
184
|
else if (tracked?.error || result.error) conversationStatus = "failed";
|
|
185
|
+
if (stoppedByShutdown) await shutdownStop.ended();
|
|
179
186
|
return {
|
|
180
187
|
conversationId,
|
|
181
188
|
ok: true,
|
|
182
189
|
messages,
|
|
183
190
|
text: result.text,
|
|
184
191
|
admitted: Boolean(projection.inputMessageId && rows.some((row) => row.id === projection.inputMessageId)),
|
|
185
|
-
error: normalizeNullableText(tracked?.error?.message) || result.error,
|
|
186
|
-
runId
|
|
192
|
+
error: stoppedByShutdown ? "" : normalizeNullableText(tracked?.error?.message) || result.error,
|
|
193
|
+
runId,
|
|
187
194
|
status: conversationStatus
|
|
188
195
|
};
|
|
189
196
|
}
|
|
@@ -478,6 +478,10 @@ export async function waitForOpenCodeFinalResponse(client, conversationId, turn,
|
|
|
478
478
|
/**
|
|
479
479
|
* Production detached-turn lifetime. Applications project the completed native
|
|
480
480
|
* result before observation is closed, then publish/checkpoint their own state.
|
|
481
|
+
* A turn the server's shutdown stopped (`turn.stoppedByShutdown`), or that
|
|
482
|
+
* ends on its own once shutdown has begun, ends interrupted however its wait
|
|
483
|
+
* ended, and `afterClose` is told so: the application leaves its checkpoint to
|
|
484
|
+
* the next server.
|
|
481
485
|
* @param {import("./openCodeProcessOwner.js").OpenCodeProcessOwner} processOwner
|
|
482
486
|
*/
|
|
483
487
|
export async function observeOpenCodeTurnCompletion(processOwner, target, conversationId, turn, {
|
|
@@ -486,6 +490,10 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
|
|
|
486
490
|
const turnAbort = turn.abortController;
|
|
487
491
|
let stopped = true;
|
|
488
492
|
let outcome = "completed";
|
|
493
|
+
const leftToNextServer = () => {
|
|
494
|
+
if (processOwner.closed) turn.stoppedByShutdown = true;
|
|
495
|
+
return turn.stoppedByShutdown === true;
|
|
496
|
+
};
|
|
489
497
|
try {
|
|
490
498
|
const options = { ...waitOptions, signal: AbortSignal.any([
|
|
491
499
|
signal, ...(timeoutMs ? [AbortSignal.timeout(timeoutMs)] : [])
|
|
@@ -493,11 +501,15 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
|
|
|
493
501
|
const completion = finalResponse
|
|
494
502
|
? await waitForOpenCodeFinalResponse(target.server.client, conversationId, turn, { ...finalResponse, ...options })
|
|
495
503
|
: await waitForOpenCodeMessages(target.server.client, conversationId, () => turn.inputMessageId, options);
|
|
504
|
+
if (leftToNextServer()) {
|
|
505
|
+
outcome = "interrupted";
|
|
506
|
+
return { interrupted: true };
|
|
507
|
+
}
|
|
496
508
|
return await projectResult(completion);
|
|
497
509
|
} catch (error) {
|
|
498
510
|
const failure = signal.aborted ? signal.reason : error;
|
|
499
|
-
outcome = turn.interrupted ? "interrupted" : "failed";
|
|
500
|
-
if (
|
|
511
|
+
outcome = leftToNextServer() || turn.interrupted ? "interrupted" : "failed";
|
|
512
|
+
if (outcome === "interrupted") return { interrupted: true };
|
|
501
513
|
if (!target.abortController.signal.aborted) {
|
|
502
514
|
try {
|
|
503
515
|
await processOwner.stopUnobservedSession(target, conversationId);
|
|
@@ -513,7 +525,7 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
|
|
|
513
525
|
turn.active = !stopped;
|
|
514
526
|
turn.mayBeRunning = !stopped;
|
|
515
527
|
}
|
|
516
|
-
await afterClose({ stopped, outcome });
|
|
528
|
+
await afterClose({ stopped, outcome, stoppedByShutdown: turn.stoppedByShutdown === true });
|
|
517
529
|
}
|
|
518
530
|
}
|
|
519
531
|
|
|
@@ -351,11 +351,20 @@ export function createOpenCodeTurnMonitor({ processes, monitors, turns, processO
|
|
|
351
351
|
}
|
|
352
352
|
}
|
|
353
353
|
} finally {
|
|
354
|
+
// The end of a turn the server's shutdown stopped belongs to the next
|
|
355
|
+
// server: its saved run stays active and carries the shutdown's mark,
|
|
356
|
+
// so that server's recovery records the turn interrupted and runs the
|
|
357
|
+
// host's checkpoint. Run here, that checkpoint (a Git snapshot, say)
|
|
358
|
+
// could be refused or hold the host's bounded shutdown for as long as
|
|
359
|
+
// it takes. A turn that ends on its own once shutdown has begun is left
|
|
360
|
+
// the same way. A user's Stop outside shutdown ends its turn here.
|
|
361
|
+
if (processOwner.closed && finalState !== "active") turn.stoppedByShutdown = true;
|
|
362
|
+
const nextServerEnds = turn.stoppedByShutdown === true && finalState !== "active";
|
|
354
363
|
// Claim retirement before awaited cleanup/checkpointing. A new Send
|
|
355
364
|
// joins this monitor's cleanup instead of steering an unobserved turn.
|
|
356
365
|
turn.active = finalState === "active";
|
|
357
366
|
await events.close();
|
|
358
|
-
if (credentialFailure || finalState === "failed") {
|
|
367
|
+
if (!nextServerEnds && (credentialFailure || finalState === "failed")) {
|
|
359
368
|
failure = await completeResult(turn, { credentialFailure, providerApiFailure, finalState, failure });
|
|
360
369
|
}
|
|
361
370
|
turn.error = failure;
|
|
@@ -364,7 +373,7 @@ export function createOpenCodeTurnMonitor({ processes, monitors, turns, processO
|
|
|
364
373
|
turn.finalAssistantResultTarget = turn.finalAssistantResult ? finalAssistantResultTarget : null;
|
|
365
374
|
turn.finalAssistantResultProcess = turn.finalAssistantResult ? finalAssistantResultProcess : null;
|
|
366
375
|
turn.updatedAt = new Date().toISOString();
|
|
367
|
-
await writeRun(turn, finalState, failure).catch(() => null);
|
|
376
|
+
await (nextServerEnds ? writeRun(turn, "active", "") : writeRun(turn, finalState, failure)).catch(() => null);
|
|
368
377
|
}
|
|
369
378
|
return openCodeTurnSnapshot(turn, target.upstreamSessionId);
|
|
370
379
|
}).finally(() => {
|
|
@@ -346,6 +346,10 @@ export function createClaudeConversationDriver({
|
|
|
346
346
|
return native.owner.closeProject(input, native.application);
|
|
347
347
|
}
|
|
348
348
|
|
|
349
|
+
function beginShutdown({ native }) {
|
|
350
|
+
return native.owner.beginShutdown();
|
|
351
|
+
}
|
|
352
|
+
|
|
349
353
|
function invalidateRuntimes({ native, input }) {
|
|
350
354
|
return native.owner.invalidateRuntimes(input, native.application, native.account);
|
|
351
355
|
}
|
|
@@ -568,6 +572,7 @@ export function createClaudeConversationDriver({
|
|
|
568
572
|
listConversationStorage,
|
|
569
573
|
retireConversationHistory,
|
|
570
574
|
closeProject,
|
|
575
|
+
beginShutdown,
|
|
571
576
|
invalidateRuntimes,
|
|
572
577
|
reconcileSessions,
|
|
573
578
|
createConversation,
|
|
@@ -63,6 +63,10 @@ function nativeGoalChanged() {
|
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
function beginShutdown({ native }) {
|
|
67
|
+
return native.runOwner.beginShutdown();
|
|
68
|
+
}
|
|
69
|
+
|
|
66
70
|
/**
|
|
67
71
|
* Conversations own threads and observers on the account's shared native runtime.
|
|
68
72
|
* @param {object} dependencies
|
|
@@ -653,6 +657,7 @@ export function createCodexConversationDriver({ connections, host = {}, limits =
|
|
|
653
657
|
deleteDetachedConversation,
|
|
654
658
|
listConversationStorage,
|
|
655
659
|
retireConversationHistory,
|
|
660
|
+
beginShutdown,
|
|
656
661
|
invalidateRuntimes,
|
|
657
662
|
reconcileSessions,
|
|
658
663
|
unsubscribeSessions,
|
|
@@ -126,6 +126,10 @@ export function createOpenCodeConversationDriver({
|
|
|
126
126
|
return native.owner.closeProject(input, native.preparation.projectCleanup());
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
+
function beginShutdown({ native }) {
|
|
130
|
+
return native.owner.beginShutdown();
|
|
131
|
+
}
|
|
132
|
+
|
|
129
133
|
function invalidateRuntimes({ native, input }) {
|
|
130
134
|
return native.owner.invalidateRuntimes(input, native.preparation);
|
|
131
135
|
}
|
|
@@ -375,6 +379,7 @@ export function createOpenCodeConversationDriver({
|
|
|
375
379
|
listConversationStorage,
|
|
376
380
|
retireConversationHistory,
|
|
377
381
|
closeProject,
|
|
382
|
+
beginShutdown,
|
|
378
383
|
invalidateRuntimes,
|
|
379
384
|
reconcileSessions,
|
|
380
385
|
createConversation,
|
|
@@ -227,6 +227,7 @@ export function createConversationRuntime({ engine: defaultEngine = "api", defau
|
|
|
227
227
|
runNativeDetachedConversation: lifecycle.runNativeDetachedConversation,
|
|
228
228
|
open: handles.open,
|
|
229
229
|
closeNativeProject: lifecycle.closeNativeProject,
|
|
230
|
+
beginNativeShutdown: lifecycle.beginNativeShutdown,
|
|
230
231
|
invalidateNativeRuntimes: lifecycle.invalidateNativeRuntimes,
|
|
231
232
|
reconcileNativeSessions: lifecycle.reconcileNativeSessions,
|
|
232
233
|
unsubscribeNativeSessions: lifecycle.unsubscribeNativeSessions,
|
|
@@ -242,6 +242,34 @@ test("Claude shutdown leaves a running turn's end and its checkpoint to the next
|
|
|
242
242
|
assert.deepEqual(custody(JSON.parse(f.session("a").records.get(MAIN_IDS.a))), { executionId: "", state: "interrupted" });
|
|
243
243
|
});
|
|
244
244
|
|
|
245
|
+
test("once Claude's shutdown has begun, a turn that ends on its own is left to the next server, which runs its one checkpoint", { timeout: 5_000 }, async t => {
|
|
246
|
+
const f = await lifecycleFixture(t);
|
|
247
|
+
const a = await f.running("a");
|
|
248
|
+
const saved = f.session("a").records.get(MAIN_IDS.a);
|
|
249
|
+
f.owner.beginShutdown();
|
|
250
|
+
assert.deepEqual(f.calls.stops, [], "beginning the shutdown stopped nothing");
|
|
251
|
+
// The turn ends on its own.
|
|
252
|
+
await f.processes[0].options.onEvent({ type: "result", subtype: "success", result: "Done." });
|
|
253
|
+
assert.deepEqual(f.calls.checkpoints, [], "the turn's end ran no checkpoint");
|
|
254
|
+
assert.equal(f.session("a").records.get(MAIN_IDS.a), saved, "the saved record keeps the active turn");
|
|
255
|
+
assert.equal((await f.owner.invalidateRuntimes({ reason: "server-shutdown" }, f.application, f.account)).ok, true);
|
|
256
|
+
assert.equal(a.process, null, "the full stop still stopped the process");
|
|
257
|
+
assert.deepEqual(f.calls.checkpoints, []);
|
|
258
|
+
// The next server's close proves that exit, then interrupts the turn through its checkpoint, once.
|
|
259
|
+
f.restart();
|
|
260
|
+
assert.equal((await f.owner.closeSession({ sessionId: "a" }, f.application)).ok, true);
|
|
261
|
+
assert.equal((await f.owner.closeSession({ sessionId: "a" }, f.application)).ok, true);
|
|
262
|
+
assert.deepEqual(f.calls.checkpoints, ["a:interrupted"]);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test("outside shutdown a Claude turn that ends on its own still runs its checkpoint at once", { timeout: 5_000 }, async t => {
|
|
266
|
+
const f = await lifecycleFixture(t);
|
|
267
|
+
await f.running("a");
|
|
268
|
+
await f.processes[0].options.onEvent({ type: "result", subtype: "success", result: "Done." });
|
|
269
|
+
assert.deepEqual(f.calls.checkpoints, ["a:completed"]);
|
|
270
|
+
assert.equal(JSON.parse(f.session("a").records.get(MAIN_IDS.a)).state, "completed");
|
|
271
|
+
});
|
|
272
|
+
|
|
245
273
|
test("Claude project close releases its sessions from memory without reopening saved conversations", async t => {
|
|
246
274
|
const f = await lifecycleFixture(t);
|
|
247
275
|
await f.idle("a");
|
|
@@ -20,6 +20,8 @@ function persistentConversationFixture() {
|
|
|
20
20
|
};
|
|
21
21
|
const commands = createCodexConversationCommands({
|
|
22
22
|
async checkpoint(_sessionId, input) { checkpoints.push(input); },
|
|
23
|
+
// Outside shutdown, with no request contexts.
|
|
24
|
+
isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
|
|
23
25
|
conversationPreparation: { async context() { return { result: { provider } }; } },
|
|
24
26
|
namespace: (sessionId) => sessionId,
|
|
25
27
|
// The provider owner still holds the provider observing the conversation;
|
|
@@ -231,6 +231,8 @@ function conversationCommands({ context = async () => ({ result: { ok: false, co
|
|
|
231
231
|
providerSessions = {} } = {}) {
|
|
232
232
|
return createCodexConversationCommands({
|
|
233
233
|
checkpoint: async () => {},
|
|
234
|
+
// Outside shutdown, with no request contexts.
|
|
235
|
+
isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
|
|
234
236
|
conversationPreparation: {
|
|
235
237
|
context,
|
|
236
238
|
scope,
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { createCodexConversationCommands } from "../src/server/conversation/codexConversations.js";
|
|
4
|
+
import { createCodexRuntimeLifecycle } from "../src/server/conversation/codexRuntimeLifecycle.js";
|
|
5
|
+
|
|
6
|
+
// A Temporary chat's Codex turn that the server's shutdown stops, or that ends
|
|
7
|
+
// while the server is stopping, leaves its checkpoint to the next server: the
|
|
8
|
+
// application records it as owed, and that server's next read makes it once.
|
|
9
|
+
|
|
10
|
+
const turnOutcomes = { USER_CANCELLED: "user_cancelled", SERVICE_RESTART: "service_restart" };
|
|
11
|
+
|
|
12
|
+
// One server's observed conversation. `latest` is the turn Codex's history
|
|
13
|
+
// reports to a read; turn notifications reach the conversation's observer.
|
|
14
|
+
function server({ closing = false, latest = { id: "turn-1", status: "inProgress", items: [] }, threadError = null } = {}) {
|
|
15
|
+
const checkpoints = [];
|
|
16
|
+
const deferred = [];
|
|
17
|
+
const listeners = new Set();
|
|
18
|
+
const emit = notification => listeners.forEach(listener => listener(notification));
|
|
19
|
+
const provider = {
|
|
20
|
+
subscribe(listener) {
|
|
21
|
+
listeners.add(listener);
|
|
22
|
+
return () => listeners.delete(listener);
|
|
23
|
+
},
|
|
24
|
+
async readThreadStatus(threadId) {
|
|
25
|
+
return { raw: { id: threadId, historyMode: "paginated", status: "idle", ...(threadError ? { error: threadError } : {}) } };
|
|
26
|
+
},
|
|
27
|
+
async listThreadTurns() { return { data: [latest] }; },
|
|
28
|
+
async readGoal() { return { goal: null }; },
|
|
29
|
+
async stopThreadForObservationLoss() {}
|
|
30
|
+
};
|
|
31
|
+
const lifecycle = { closing };
|
|
32
|
+
const commands = createCodexConversationCommands({
|
|
33
|
+
async checkpoint(sessionId, input) { checkpoints.push({ sessionId, ...input }); },
|
|
34
|
+
async deferCheckpoint(sessionId, input) { deferred.push({ sessionId, ...input }); },
|
|
35
|
+
isClosing: () => lifecycle.closing,
|
|
36
|
+
captureContext: () => null,
|
|
37
|
+
runInContext: (_context, operation) => operation(),
|
|
38
|
+
codexAppServerConversationMessageId: (...parts) => parts.join(":"),
|
|
39
|
+
conversationPreparation: { async context() { return { result: { provider } }; } },
|
|
40
|
+
namespace: sessionId => sessionId,
|
|
41
|
+
providerSessions: { owner: { providers: new Map([["provider", provider]]), holdProvider: () => () => {} } },
|
|
42
|
+
turnOutcomes
|
|
43
|
+
});
|
|
44
|
+
const startTurn = () => {
|
|
45
|
+
commands.observeCodexConversation("session", "thread", provider);
|
|
46
|
+
emit({ method: "turn/started", params: { threadId: "thread", turn: { id: "turn-1", status: "inProgress" } } });
|
|
47
|
+
};
|
|
48
|
+
const complete = status => emit({ method: "turn/completed", params: { threadId: "thread", turn: { id: "turn-1", status } } });
|
|
49
|
+
const read = stoppedByShutdown => commands.readPersistentCodexConversation("session", { conversationId: "thread", stoppedByShutdown },
|
|
50
|
+
{ provider }).then(({ status, error, runId }) => ({ status, error, runId }));
|
|
51
|
+
return { checkpoints, deferred, commands, lifecycle, startTurn, complete, read };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const OWED = { sessionId: "session", outerTurnId: "codex:thread:turn-1", threadId: "thread", turnId: "turn-1" };
|
|
55
|
+
|
|
56
|
+
test("shutdown leaves a running Temporary turn's interrupted checkpoint to the next server, which makes it once on its read", async () => {
|
|
57
|
+
const first = server();
|
|
58
|
+
first.startTurn();
|
|
59
|
+
first.lifecycle.closing = true;
|
|
60
|
+
await first.commands.leaveRunningTurnsToNextServer();
|
|
61
|
+
assert.deepEqual(first.checkpoints, [], "shutdown made no checkpoint");
|
|
62
|
+
assert.deepEqual(first.deferred, [{ ...OWED, status: "interrupted", turnOutcome: "service_restart" }]);
|
|
63
|
+
|
|
64
|
+
// The stopped process ends the turn natively as interrupted.
|
|
65
|
+
const next = server({ latest: { id: "turn-1", status: "interrupted", items: [] }, threadError: { message: "Turn aborted." } });
|
|
66
|
+
const ended = [];
|
|
67
|
+
const mark = runId => ({ runId, async ended() { ended.push(runId); } });
|
|
68
|
+
assert.deepEqual(await next.read(mark("turn-1")), { status: "interrupted", error: "", runId: "turn-1" });
|
|
69
|
+
assert.deepEqual(ended, ["turn-1"], "recording the stopped run lets the application make its checkpoint");
|
|
70
|
+
assert.deepEqual(await next.read(undefined), { status: "interrupted", error: "Turn aborted.", runId: "turn-1" },
|
|
71
|
+
"an unmarked interruption keeps Codex's own report");
|
|
72
|
+
await next.read(mark("turn-0"));
|
|
73
|
+
assert.deepEqual(ended, ["turn-1"], "only the marked run is recorded as ended");
|
|
74
|
+
assert.deepEqual(next.checkpoints, []);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("a Temporary turn that ends on its own while the server stops leaves its checkpoint, with its own outcome, to the next server", async () => {
|
|
78
|
+
const f = server();
|
|
79
|
+
f.startTurn();
|
|
80
|
+
f.lifecycle.closing = true;
|
|
81
|
+
f.complete("completed");
|
|
82
|
+
await Promise.all(f.commands.deferredCheckpoints);
|
|
83
|
+
assert.deepEqual(f.checkpoints, []);
|
|
84
|
+
assert.deepEqual(f.deferred, [{ ...OWED, status: "completed", turnOutcome: "" }]);
|
|
85
|
+
assert.equal(f.commands.deferredCheckpoints.size, 0, "shutdown awaits each deferred record");
|
|
86
|
+
|
|
87
|
+
// The next server reads the completed turn as completed, and makes its checkpoint.
|
|
88
|
+
const next = server({ latest: { id: "turn-1", status: "completed", items: [] } });
|
|
89
|
+
const ended = [];
|
|
90
|
+
assert.equal((await next.read({ runId: "turn-1", async ended() { ended.push("turn-1"); } })).status, "completed");
|
|
91
|
+
assert.deepEqual(ended, ["turn-1"]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("outside shutdown a Temporary turn's end still makes its checkpoint at once", async () => {
|
|
95
|
+
const f = server();
|
|
96
|
+
f.startTurn();
|
|
97
|
+
f.complete("interrupted");
|
|
98
|
+
assert.deepEqual(f.checkpoints, [{ ...OWED, status: "interrupted", turnOutcome: "" }]);
|
|
99
|
+
assert.deepEqual(f.deferred, []);
|
|
100
|
+
await f.commands.leaveRunningTurnsToNextServer();
|
|
101
|
+
assert.deepEqual(f.deferred, [], "a finished turn owes nothing at shutdown");
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// The run owner's runtime lifecycle, with the owner calls it makes recorded.
|
|
105
|
+
function runtimeLifecycle(calls, leaveTurnsToNextServer) {
|
|
106
|
+
return createCodexRuntimeLifecycle({
|
|
107
|
+
pendingWork: { conversationTurnStarts: new Map(), messageDeliveries: new Map(), reasoningPersistQueues: new Map(),
|
|
108
|
+
resultFinalizations: new Map(), threadReconciliations: new Map(), notificationTasks: new Map(), deferredCheckpoints: new Set() },
|
|
109
|
+
observer: { eventSubscriptions: new Map() },
|
|
110
|
+
checkManagedConnection: async () => {},
|
|
111
|
+
providerSessions: { managed: new Map(), assertOpen() {}, owner: {
|
|
112
|
+
lifecycleTasks: new Set(), runtimeAcquisitions: new Set(),
|
|
113
|
+
beginShutdown() { calls.push("owner-shutdown"); },
|
|
114
|
+
async invalidateRuntimes() { calls.push("stop-runtimes"); return { failed: [], providerCount: 0, results: [], stopped: 0 }; }
|
|
115
|
+
} },
|
|
116
|
+
helperLifecycle: { projectOperations: new Map(), cleanups: new Map(), mutations: new Map(), restores: new Map(), turnStarts: new Map() },
|
|
117
|
+
captureContext: () => null,
|
|
118
|
+
runInContext: (_context, operation) => operation(),
|
|
119
|
+
debugLog() {},
|
|
120
|
+
debugError: error => error?.message,
|
|
121
|
+
leaveTurnsToNextServer
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
test("a Codex shutdown leaves the running turns to the next server before any runtime stops, and waits for their records", async () => {
|
|
126
|
+
const calls = [];
|
|
127
|
+
const recorded = Promise.withResolvers();
|
|
128
|
+
const lifecycle = runtimeLifecycle(calls, async () => {
|
|
129
|
+
calls.push("leave-turns");
|
|
130
|
+
await recorded.promise;
|
|
131
|
+
calls.push("turns-recorded");
|
|
132
|
+
});
|
|
133
|
+
const shutdown = lifecycle.invalidateRuntimes({ reason: "server-shutdown" }, { enabled: true });
|
|
134
|
+
assert.equal(lifecycle.closing, true);
|
|
135
|
+
await Promise.resolve();
|
|
136
|
+
recorded.resolve();
|
|
137
|
+
await shutdown;
|
|
138
|
+
assert.equal(calls[0], "leave-turns");
|
|
139
|
+
assert.ok(calls.indexOf("turns-recorded") < calls.lastIndexOf("stop-runtimes"), "shutdown waited for the records");
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test("a begun Codex shutdown stops nothing, and a Temporary turn that then ends on its own leaves its checkpoint to the next server", async () => {
|
|
143
|
+
const f = server();
|
|
144
|
+
f.startTurn();
|
|
145
|
+
const calls = [];
|
|
146
|
+
const lifecycle = runtimeLifecycle(calls, () => f.commands.leaveRunningTurnsToNextServer());
|
|
147
|
+
Object.defineProperty(f.lifecycle, "closing", { get: () => lifecycle.closing });
|
|
148
|
+
lifecycle.beginShutdown();
|
|
149
|
+
assert.equal(lifecycle.closing, true, "the closing fence is set before the call returns");
|
|
150
|
+
assert.deepEqual(calls, ["owner-shutdown"], "beginning the shutdown stopped nothing");
|
|
151
|
+
f.complete("completed");
|
|
152
|
+
await Promise.all(f.commands.deferredCheckpoints);
|
|
153
|
+
assert.deepEqual(f.checkpoints, [], "the turn's end made no checkpoint");
|
|
154
|
+
assert.deepEqual(f.deferred.at(-1), { ...OWED, status: "completed", turnOutcome: "" }, "its own outcome is owed");
|
|
155
|
+
|
|
156
|
+
// The full stop that follows still stops the runtimes, and leaves the turns only once.
|
|
157
|
+
const deferred = f.deferred.length;
|
|
158
|
+
await lifecycle.invalidateRuntimes({ reason: "server-shutdown" }, { enabled: true });
|
|
159
|
+
assert.ok(calls.includes("stop-runtimes"));
|
|
160
|
+
assert.equal(f.deferred.length, deferred);
|
|
161
|
+
assert.deepEqual(f.checkpoints, []);
|
|
162
|
+
|
|
163
|
+
// The next server's read makes the owed checkpoint once.
|
|
164
|
+
const next = server({ latest: { id: "turn-1", status: "completed", items: [] } });
|
|
165
|
+
const ended = [];
|
|
166
|
+
assert.equal((await next.read({ runId: "turn-1", async ended() { ended.push("turn-1"); } })).status, "completed");
|
|
167
|
+
assert.deepEqual(ended, ["turn-1"]);
|
|
168
|
+
});
|
|
@@ -53,6 +53,8 @@ function codexFixture(t, { idleMs = 20 } = {}) {
|
|
|
53
53
|
});
|
|
54
54
|
const commands = createCodexConversationCommands({
|
|
55
55
|
async checkpoint() {},
|
|
56
|
+
// Outside shutdown, with no request contexts.
|
|
57
|
+
isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
|
|
56
58
|
codexAppServerConversationMessageId: (...parts) => parts.join(":"),
|
|
57
59
|
codexAppServerStreamMessage: () => ({}),
|
|
58
60
|
conversationPreparation: {
|
package/test/codexTurn.test.js
CHANGED
|
@@ -3856,3 +3856,60 @@ test("native history budget rejects unavailable dynamic limits before pending de
|
|
|
3856
3856
|
assert.equal(sends, 0);
|
|
3857
3857
|
}
|
|
3858
3858
|
});
|
|
3859
|
+
|
|
3860
|
+
test("a Main turn the previous server's shutdown stopped reads as interrupted by a restart, with one interrupted checkpoint", async () => {
|
|
3861
|
+
const notices = [];
|
|
3862
|
+
const checkpoints = [];
|
|
3863
|
+
const f = await receiptFixture({
|
|
3864
|
+
outcomeNotice(_runtime, _sessionId, _threadId, _turnId, outcome) { notices.push(outcome); },
|
|
3865
|
+
checkpoint(_sessionId, input) { checkpoints.push(input); }
|
|
3866
|
+
});
|
|
3867
|
+
f.provider.interruptionOutcome = () => "";
|
|
3868
|
+
// The shutdown marks the running turn and leaves the rest of its saved run as it was.
|
|
3869
|
+
assert.deepEqual(await f.owner.markTurnStoppedByShutdown(f.sessionId, f.threadId), { marked: true });
|
|
3870
|
+
assert.deepEqual({ state: f.run.state, stoppedByShutdownTurnId: f.run.stoppedByShutdownTurnId },
|
|
3871
|
+
{ state: "active", stoppedByShutdownTurnId: f.turnId });
|
|
3872
|
+
assert.deepEqual(await f.owner.markTurnStoppedByShutdown(f.sessionId, "another-thread"), { marked: false });
|
|
3873
|
+
// The next server reads the native turn as interrupted.
|
|
3874
|
+
const result = await f.owner.stopTurnWithProviderFailure(f.sessionId, f.threadId, f.turnId, {
|
|
3875
|
+
status: "interrupted", provider: f.provider, verifyInactive: false
|
|
3876
|
+
});
|
|
3877
|
+
assert.equal(result.status, "interrupted");
|
|
3878
|
+
assert.match(result.error, /interrupted by a restart/u);
|
|
3879
|
+
assert.deepEqual(notices, ["service_restart"]);
|
|
3880
|
+
assert.deepEqual(checkpoints.map(({ status, turnOutcome }) => ({ status, turnOutcome })),
|
|
3881
|
+
[{ status: "interrupted", turnOutcome: "service_restart" }]);
|
|
3882
|
+
assert.equal(f.run.state, "interrupted");
|
|
3883
|
+
});
|
|
3884
|
+
|
|
3885
|
+
test("an interrupted Main turn without the shutdown's mark keeps the plain interrupted outcome", async () => {
|
|
3886
|
+
const notices = [];
|
|
3887
|
+
const checkpoints = [];
|
|
3888
|
+
const f = await receiptFixture({
|
|
3889
|
+
outcomeNotice(_runtime, _sessionId, _threadId, _turnId, outcome) { notices.push(outcome); },
|
|
3890
|
+
checkpoint(_sessionId, input) { checkpoints.push(input); }
|
|
3891
|
+
});
|
|
3892
|
+
f.provider.interruptionOutcome = () => "";
|
|
3893
|
+
// A mark left by an earlier turn does not apply to this one.
|
|
3894
|
+
f.run = { ...f.run, stoppedByShutdownTurnId: "an-earlier-turn" };
|
|
3895
|
+
await f.owner.stopTurnWithProviderFailure(f.sessionId, f.threadId, f.turnId, {
|
|
3896
|
+
status: "interrupted", provider: f.provider, verifyInactive: false
|
|
3897
|
+
});
|
|
3898
|
+
assert.deepEqual(notices, ["interrupted"]);
|
|
3899
|
+
assert.deepEqual(checkpoints.map(({ turnOutcome }) => turnOutcome), ["interrupted"]);
|
|
3900
|
+
});
|
|
3901
|
+
|
|
3902
|
+
test("a Main turn that ends once shutdown has begun is left to the next server, with no checkpoint now", async () => {
|
|
3903
|
+
const f = await settlementFixture();
|
|
3904
|
+
f.owner.runtimeLifecycle = { closing: true };
|
|
3905
|
+
const before = structuredClone(f.run);
|
|
3906
|
+
assert.deepEqual(await f.owner.markTurnIdle(f.sessionId, { status: "completed", threadId: f.threadId, turnId: f.turnId }),
|
|
3907
|
+
{ ok: true, processed: false, reason: "left_to_next_server" });
|
|
3908
|
+
assert.deepEqual(f.run, before, "the saved run is unchanged");
|
|
3909
|
+
assert.deepEqual(f.checkpoints, []);
|
|
3910
|
+
// Outside shutdown the same end is recorded and checkpointed at once.
|
|
3911
|
+
f.owner.runtimeLifecycle = { closing: false };
|
|
3912
|
+
await f.owner.markTurnIdle(f.sessionId, { status: "completed", threadId: f.threadId, turnId: f.turnId });
|
|
3913
|
+
assert.equal(f.run.state, "completed");
|
|
3914
|
+
assert.deepEqual(f.checkpoints.map(({ status }) => status), ["completed"]);
|
|
3915
|
+
});
|
|
@@ -706,6 +706,35 @@ test("native invalidation preserves synchronous fences and failures without open
|
|
|
706
706
|
await result;
|
|
707
707
|
});
|
|
708
708
|
|
|
709
|
+
test("beginning native shutdown reaches the configured owner synchronously and opens no conversation", async t => {
|
|
710
|
+
const input = { reason: "server-shutdown" };
|
|
711
|
+
const calls = [];
|
|
712
|
+
let asynchronousHost = true;
|
|
713
|
+
const native = { runOwner: {
|
|
714
|
+
beginShutdown() { calls.push("begin"); return "begun"; },
|
|
715
|
+
invalidateRuntimes() { assert.fail("Beginning shutdown stops nothing."); }
|
|
716
|
+
} };
|
|
717
|
+
const runtime = createConversationRuntime({
|
|
718
|
+
authorize() { assert.fail("Trusted shutdown must not acquire session authority."); },
|
|
719
|
+
storage: {
|
|
720
|
+
read() { assert.fail("Shutdown must not read a conversation."); },
|
|
721
|
+
write() { assert.fail("Shutdown must not allocate a conversation."); }
|
|
722
|
+
},
|
|
723
|
+
host: { conversation(request) {
|
|
724
|
+
calls.push("host");
|
|
725
|
+
assert.deepEqual(request, { context, input, operation: "beginShutdown" });
|
|
726
|
+
const prepared = { engine: "codex", native, context, input };
|
|
727
|
+
return asynchronousHost ? Promise.resolve(prepared) : prepared;
|
|
728
|
+
} }
|
|
729
|
+
});
|
|
730
|
+
t.after(() => runtime.close());
|
|
731
|
+
assert.throws(() => runtime.beginNativeShutdown({ context, input }), /synchronous configured host/);
|
|
732
|
+
assert.deepEqual(calls, ["host"]);
|
|
733
|
+
asynchronousHost = false;
|
|
734
|
+
assert.equal(runtime.beginNativeShutdown({ context, input }), "begun");
|
|
735
|
+
assert.deepEqual(calls, ["host", "host", "begin"], "the owner's fence is set before the call returns");
|
|
736
|
+
});
|
|
737
|
+
|
|
709
738
|
test("renewal proof cleanup preserves OpenCode failure, consumption and retry through the configured owner", async t => {
|
|
710
739
|
const owner = createOpenCodeSharedRuntime({});
|
|
711
740
|
const calls = [];
|