@jskit-ai/assistant-core 0.2.16 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/examples/conversation/package.json +5 -5
- package/package.json +5 -5
- package/src/server/conversation/claudeNativeTurn.js +9 -1
- package/src/server/conversation/claudeSessionCleanup.js +9 -1
- package/src/server/conversation/claudeTurn.js +1 -0
- package/src/server/conversation/codexConversations.js +65 -6
- package/src/server/conversation/codexRunJournal.js +33 -1
- package/src/server/conversation/codexRuntimeLifecycle.js +20 -1
- package/src/server/conversation/codexTurn.js +41 -4
- package/src/server/conversation/codexTurnEndings.js +9 -6
- package/src/server/conversation/nativeConversationLifecycle.js +22 -0
- package/src/server/conversation/openCodeProcessOwner.js +11 -5
- package/src/server/conversation/openCodeRuntime.js +1 -0
- package/src/server/conversation/openCodeTurn.js +10 -5
- package/src/server/conversation/openCodeTurnMonitor.js +4 -2
- package/src/server/conversation/providers/claudeDriver.js +5 -0
- package/src/server/conversation/providers/codexDriver.js +5 -0
- package/src/server/conversation/providers/opencodeDriver.js +5 -0
- package/src/server/conversation/runtime.js +1 -0
- package/test/claudeLifecycle.test.js +28 -0
- package/test/codexConversationsCheckpoint.test.js +2 -0
- package/test/codexLifecycleIsolation.test.js +2 -0
- package/test/codexShutdownStop.test.js +168 -0
- package/test/codexTemporaryProviderRelease.test.js +2 -0
- package/test/codexTurn.test.js +57 -0
- package/test/conversationSession.test.js +29 -0
- package/test/openCodeLifecycle.test.js +55 -3
|
@@ -8,14 +8,14 @@
|
|
|
8
8
|
"build": "vite build"
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"@jskit-ai/assistant-core": "0.2.
|
|
12
|
-
"@jskit-ai/http-runtime": "0.2.
|
|
13
|
-
"@jskit-ai/kernel": "0.2.
|
|
11
|
+
"@jskit-ai/assistant-core": "0.2.17",
|
|
12
|
+
"@jskit-ai/http-runtime": "0.2.7",
|
|
13
|
+
"@jskit-ai/kernel": "0.2.7",
|
|
14
14
|
"@mdi/js": "^7.4.47",
|
|
15
15
|
"vue": "^3.5.13",
|
|
16
16
|
"vuetify": "^4.0.0",
|
|
17
|
-
"@jskit-ai/connectors-core": "0.2.
|
|
18
|
-
"@jskit-ai/connectors-catalog": "0.2.
|
|
17
|
+
"@jskit-ai/connectors-core": "0.2.8",
|
|
18
|
+
"@jskit-ai/connectors-catalog": "0.2.8"
|
|
19
19
|
},
|
|
20
20
|
"devDependencies": {
|
|
21
21
|
"@vitejs/plugin-vue": "^6.0.7",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jskit-ai/assistant-core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.17",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "node --test --test-concurrency=1"
|
|
@@ -51,8 +51,8 @@
|
|
|
51
51
|
"@ai-sdk/togetherai": "^3.0.49",
|
|
52
52
|
"@ai-sdk/xai": "^4.0.58",
|
|
53
53
|
"@jskit-ai/agent": "0.1.0",
|
|
54
|
-
"@jskit-ai/resource-core": "0.1.
|
|
55
|
-
"@jskit-ai/resource-crud-core": "0.1.
|
|
54
|
+
"@jskit-ai/resource-core": "0.1.198",
|
|
55
|
+
"@jskit-ai/resource-crud-core": "0.1.198",
|
|
56
56
|
"@mdi/js": "^7.4.47",
|
|
57
57
|
"@openrouter/ai-sdk-provider": "^3.0.0",
|
|
58
58
|
"ai": "^7.0.152",
|
|
@@ -61,8 +61,8 @@
|
|
|
61
61
|
"ws": "^8.20.1"
|
|
62
62
|
},
|
|
63
63
|
"peerDependencies": {
|
|
64
|
-
"@jskit-ai/http-runtime": "0.2.
|
|
65
|
-
"@jskit-ai/kernel": "0.2.
|
|
64
|
+
"@jskit-ai/http-runtime": "0.2.7",
|
|
65
|
+
"@jskit-ai/kernel": "0.2.7",
|
|
66
66
|
"vue": "^3.5.13",
|
|
67
67
|
"vuetify": "^4.0.0"
|
|
68
68
|
},
|
|
@@ -355,7 +355,15 @@ export function createClaudeConversationTurn({ conversationId, onEvent, process,
|
|
|
355
355
|
} else if (event.type === "settled") {
|
|
356
356
|
const state = event.status === "interrupted" ? "interrupted" : event.status === "failed" ? "failed" : "completed";
|
|
357
357
|
if (bound) setClaudeTurnState(entry, state, event.error);
|
|
358
|
-
else if (
|
|
358
|
+
else if (entry.turn?.active && process.ownerClosing()) {
|
|
359
|
+
// A turn that ends on its own once shutdown has begun is left to the
|
|
360
|
+
// next server like one the shutdown stops: its saved record keeps the
|
|
361
|
+
// execution and the active turn, so that server's recovery interrupts
|
|
362
|
+
// the turn and runs the host's checkpoint. Run here, that checkpoint
|
|
363
|
+
// (a Git snapshot, say) could be refused or hold up shutdown.
|
|
364
|
+
resolveCompletion({ ...readResult(), status: "interrupted", error: "" });
|
|
365
|
+
return;
|
|
366
|
+
} else if (!await updateState(state, event.error, event.generation)) return;
|
|
359
367
|
}
|
|
360
368
|
const published = await onEvent(event);
|
|
361
369
|
const change = event.type === "provider-event" ? claudeProviderChange(event.event) : "";
|
|
@@ -121,7 +121,7 @@ export function createClaudeSessionCleanup({
|
|
|
121
121
|
account.queries.invalidate().catch(() => {});
|
|
122
122
|
return { ok: true, closed: 0 };
|
|
123
123
|
}
|
|
124
|
-
|
|
124
|
+
beginShutdown();
|
|
125
125
|
// Neither waits for the other: a failed account query cleanup must not
|
|
126
126
|
// leave the sessions' processes running.
|
|
127
127
|
const [queries, retained] = await Promise.allSettled([account.queries.invalidate(), stopRetained(application)]);
|
|
@@ -130,6 +130,13 @@ export function createClaudeSessionCleanup({
|
|
|
130
130
|
return retained.value;
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
// The server's shutdown begins: the owner closes, so each running turn's
|
|
134
|
+
// end, and its checkpoint, is left to the next server, but nothing stops.
|
|
135
|
+
// The later server-shutdown invalidation stops the processes.
|
|
136
|
+
function beginShutdown() {
|
|
137
|
+
closing = true;
|
|
138
|
+
}
|
|
139
|
+
|
|
133
140
|
// Shutdown stops only what this server runs: a Claude process cannot outlive
|
|
134
141
|
// the server that started it, so retained entries hold every one. Saved
|
|
135
142
|
// conversations are not reopened just to be stopped.
|
|
@@ -233,6 +240,7 @@ export function createClaudeSessionCleanup({
|
|
|
233
240
|
closeSession,
|
|
234
241
|
releaseSession,
|
|
235
242
|
closeProject,
|
|
243
|
+
beginShutdown,
|
|
236
244
|
invalidateRuntimes,
|
|
237
245
|
drain,
|
|
238
246
|
forget,
|
|
@@ -701,6 +701,7 @@ export function createClaudeConversationOwner({
|
|
|
701
701
|
closeSession: cleanup.closeSession,
|
|
702
702
|
releaseSession: cleanup.releaseSession,
|
|
703
703
|
closeProject: cleanup.closeProject,
|
|
704
|
+
beginShutdown: cleanup.beginShutdown,
|
|
704
705
|
invalidateRuntimes: cleanup.invalidateRuntimes,
|
|
705
706
|
drain: cleanup.drain,
|
|
706
707
|
forget: cleanup.forget,
|
|
@@ -57,6 +57,11 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
|
57
57
|
* @param {import("./codexProviderSelection.js").CodexProviderSelection["borrowSessionProvider"]} dependencies.borrowSessionProvider
|
|
58
58
|
* @param {string} dependencies.steerFailedCode
|
|
59
59
|
* @param {import("./codexTurn.js").CodexTurnOutcomes} dependencies.turnOutcomes
|
|
60
|
+
* @param {import("./codexTurn.js").CodexDeferCheckpoint} dependencies.deferCheckpoint Absent when the host predates
|
|
61
|
+
* it, which then makes every checkpoint at once.
|
|
62
|
+
* @param {() => boolean} dependencies.isClosing Whether the server's shutdown has begun.
|
|
63
|
+
* @param {import("./codexTurn.js").CodexCaptureContext} dependencies.captureContext
|
|
64
|
+
* @param {import("./codexTurn.js").CodexRunInContext} dependencies.runInContext
|
|
60
65
|
*/
|
|
61
66
|
export function createCodexConversationCommands({
|
|
62
67
|
checkpoint,
|
|
@@ -73,10 +78,17 @@ export function createCodexConversationCommands({
|
|
|
73
78
|
ensureSessionProvider,
|
|
74
79
|
borrowSessionProvider,
|
|
75
80
|
steerFailedCode,
|
|
76
|
-
turnOutcomes
|
|
81
|
+
turnOutcomes,
|
|
82
|
+
deferCheckpoint,
|
|
83
|
+
isClosing,
|
|
84
|
+
captureContext,
|
|
85
|
+
runInContext
|
|
77
86
|
}) {
|
|
78
87
|
const codexAppServerConversations = new Map();
|
|
79
88
|
const codexAppServerConversationTurnStarts = new Map();
|
|
89
|
+
// The application's records of checkpoints left to the next server, which
|
|
90
|
+
// shutdown awaits before the process exits.
|
|
91
|
+
const codexAppServerDeferredCheckpoints = new Set();
|
|
80
92
|
// Turns, detached runs and stops live in their own modules. Each receives the
|
|
81
93
|
// registry above and the registry and context functions declared below.
|
|
82
94
|
const {
|
|
@@ -157,6 +169,42 @@ export function createCodexConversationCommands({
|
|
|
157
169
|
holdWhileActive(state);
|
|
158
170
|
}
|
|
159
171
|
|
|
172
|
+
// Once the server's shutdown has begun a turn's checkpoint belongs to the
|
|
173
|
+
// next server: the application records it as owed, and that server's next
|
|
174
|
+
// read of the conversation makes it. Made now, a checkpoint (a Git snapshot,
|
|
175
|
+
// say) could be refused or hold up the host's bounded shutdown.
|
|
176
|
+
function leaveCheckpointToNextServer(state, ending) {
|
|
177
|
+
const deferred = Promise.resolve()
|
|
178
|
+
.then(() => runInContext(state.context, () => deferCheckpoint(state.sessionId, ending)))
|
|
179
|
+
.catch(() => null);
|
|
180
|
+
codexAppServerDeferredCheckpoints.add(deferred);
|
|
181
|
+
void deferred.finally(() => codexAppServerDeferredCheckpoints.delete(deferred));
|
|
182
|
+
return deferred;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* At shutdown, each observed conversation's running turn: the stopped process
|
|
187
|
+
* ends it without a completion this server observes, so its interrupted
|
|
188
|
+
* checkpoint is left to the next server.
|
|
189
|
+
*/
|
|
190
|
+
function leaveRunningTurnsToNextServer() {
|
|
191
|
+
if (!deferCheckpoint) return Promise.resolve([]);
|
|
192
|
+
const deferred = [];
|
|
193
|
+
for (const conversations of codexAppServerConversations.values()) {
|
|
194
|
+
for (const state of conversations.values()) {
|
|
195
|
+
if (!state.persistent || !state.unsubscribe || !state.runId || !codexAppServerConversationTurnIsActive(state.status)) continue;
|
|
196
|
+
deferred.push(leaveCheckpointToNextServer(state, {
|
|
197
|
+
outerTurnId: codexAppServerNativeOuterTurnId(state.conversationId, state.runId),
|
|
198
|
+
threadId: state.conversationId,
|
|
199
|
+
turnId: state.runId,
|
|
200
|
+
status: "interrupted",
|
|
201
|
+
turnOutcome: turnOutcomes.SERVICE_RESTART
|
|
202
|
+
}));
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return Promise.all(deferred);
|
|
206
|
+
}
|
|
207
|
+
|
|
160
208
|
function codexAppServerConversation(sessionId = "", conversationId = "") {
|
|
161
209
|
return codexAppServerConversations.get(
|
|
162
210
|
namespace(sessionId)
|
|
@@ -201,8 +249,8 @@ export function createCodexConversationCommands({
|
|
|
201
249
|
let state = conversations.get(conversationId);
|
|
202
250
|
if (state?.provider === provider && state.unsubscribe) return state;
|
|
203
251
|
if (state) forgetObservation(state);
|
|
204
|
-
state = { conversationId, provider, persistent: true, status: "ready", runId: "", error: "", turnOutcome: "",
|
|
205
|
-
onEvent: ignoreCodexConversationEvent };
|
|
252
|
+
state = { sessionId, conversationId, provider, persistent: true, status: "ready", runId: "", error: "", turnOutcome: "",
|
|
253
|
+
context: captureContext(), onEvent: ignoreCodexConversationEvent };
|
|
206
254
|
state.unsubscribe = provider.subscribe((notification) => {
|
|
207
255
|
if (codexAppServerNotificationThreadId(notification) !== conversationId) return;
|
|
208
256
|
const turnId = codexAppServerNotificationTurnId(notification);
|
|
@@ -228,13 +276,15 @@ export function createCodexConversationCommands({
|
|
|
228
276
|
// A turn the person stopped reaches the checkpoint as cancelled, as Main turns do.
|
|
229
277
|
const completedOutcome = state.turnOutcome;
|
|
230
278
|
state.turnOutcome = "";
|
|
231
|
-
|
|
279
|
+
const ending = {
|
|
232
280
|
outerTurnId: codexAppServerNativeOuterTurnId(conversationId, completedTurnId),
|
|
233
281
|
threadId: conversationId,
|
|
234
282
|
turnId: completedTurnId,
|
|
235
283
|
status: completedStatus,
|
|
236
284
|
turnOutcome: completedOutcome
|
|
237
|
-
}
|
|
285
|
+
};
|
|
286
|
+
if (isClosing() && deferCheckpoint) leaveCheckpointToNextServer(state, ending);
|
|
287
|
+
else void checkpoint(sessionId, ending).catch(() => {});
|
|
238
288
|
}
|
|
239
289
|
holdWhileActive(state);
|
|
240
290
|
});
|
|
@@ -284,6 +334,13 @@ export function createCodexConversationCommands({
|
|
|
284
334
|
if (goal?.status === "active") status = "inProgress";
|
|
285
335
|
Object.assign(state, { runId, status, goal });
|
|
286
336
|
holdWhileActive(state);
|
|
337
|
+
// The application marks the run whose checkpoint a previous server's
|
|
338
|
+
// shutdown left owed (`stoppedByShutdown: { runId, ended }`). Recording
|
|
339
|
+
// that it ended lets the application make that checkpoint; a run the
|
|
340
|
+
// shutdown stopped reads interrupted with no error.
|
|
341
|
+
const shutdownStop = input.stoppedByShutdown;
|
|
342
|
+
const endedAtShutdown = Boolean(runId) && shutdownStop?.runId === runId && !codexAppServerConversationTurnIsActive(status);
|
|
343
|
+
if (endedAtShutdown) await shutdownStop.ended();
|
|
287
344
|
const messages = [];
|
|
288
345
|
for (const turn of turns) {
|
|
289
346
|
const turnId = codexAppServerProviderTurnId(turn);
|
|
@@ -321,7 +378,7 @@ export function createCodexConversationCommands({
|
|
|
321
378
|
goal,
|
|
322
379
|
admitted: Boolean(input.messageId && turns.some((turn) => codexAppServerProviderTurnClientIds(turn).includes(input.messageId))),
|
|
323
380
|
rawText: latestText.trim(),
|
|
324
|
-
error: state.error || codexAppServerThreadError(thread)
|
|
381
|
+
error: endedAtShutdown && status === "interrupted" ? "" : state.error || codexAppServerThreadError(thread)
|
|
325
382
|
};
|
|
326
383
|
}
|
|
327
384
|
|
|
@@ -555,6 +612,8 @@ export function createCodexConversationCommands({
|
|
|
555
612
|
return Object.freeze({
|
|
556
613
|
codexAppServerConversations,
|
|
557
614
|
codexAppServerConversationTurnStarts,
|
|
615
|
+
deferredCheckpoints: codexAppServerDeferredCheckpoints,
|
|
616
|
+
leaveRunningTurnsToNextServer,
|
|
558
617
|
codexAppServerConversation,
|
|
559
618
|
hasActiveConversation,
|
|
560
619
|
observeCodexConversation,
|
|
@@ -3,6 +3,32 @@ import crypto from "node:crypto";
|
|
|
3
3
|
import { isRecord, normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
4
4
|
|
|
5
5
|
const CODEX_APP_SERVER_PROMPT_CLAIM_GRACE_MS = 15000;
|
|
6
|
+
const CODEX_APP_SERVER_STOPPED_BY_SHUTDOWN_EVENT = "codex-app-server-stopped-by-shutdown";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Marks the session's running turn as stopped by the server's shutdown, so the
|
|
10
|
+
* next server records its interruption as a restart. Nothing else of the saved
|
|
11
|
+
* run changes.
|
|
12
|
+
* @param {{ createRuntime: import("./codexTurn.js").CodexCreateRuntime, turnState: import("./codexTurn.js").CodexTurnStateReader }} dependencies
|
|
13
|
+
* @param {string} sessionId
|
|
14
|
+
* @param {string} [threadId] the session's Main thread, when known
|
|
15
|
+
*/
|
|
16
|
+
export async function markCodexAppServerTurnStoppedByShutdown({ createRuntime, turnState }, sessionId = "", threadId = "") {
|
|
17
|
+
const normalizedSessionId = normalizeText(sessionId);
|
|
18
|
+
if (!normalizedSessionId) return { marked: false };
|
|
19
|
+
const runtime = await createRuntime();
|
|
20
|
+
let marked = false;
|
|
21
|
+
await runtime.store.mutateSession(normalizedSessionId, async () => {
|
|
22
|
+
const turn = turnState(await runtime.getSession(normalizedSessionId).catch(() => null) || {});
|
|
23
|
+
if (!turn.active || !turn.turnId || (normalizeText(threadId) && turn.threadId !== normalizeText(threadId))) return;
|
|
24
|
+
await runtime.store.writeAgentRunEvent(normalizedSessionId, CODEX_APP_SERVER_AGENT_RUN_ID, {
|
|
25
|
+
event: { kind: CODEX_APP_SERVER_STOPPED_BY_SHUTDOWN_EVENT },
|
|
26
|
+
patch: { stoppedByShutdownTurnId: turn.turnId }
|
|
27
|
+
});
|
|
28
|
+
marked = true;
|
|
29
|
+
});
|
|
30
|
+
return { marked };
|
|
31
|
+
}
|
|
6
32
|
|
|
7
33
|
function terminalCodexAppServerAgentRunState(status = "") {
|
|
8
34
|
const normalizedStatus = normalizeText(status);
|
|
@@ -40,6 +66,7 @@ function codexAppServerDateValueMs(value = "") {
|
|
|
40
66
|
* @param {number} dependencies.finalizingGraceMs
|
|
41
67
|
* @param {(session: object, run: object) => boolean} dependencies.admissionTaskFinished
|
|
42
68
|
* @param {string} dependencies.orphanedPromptMessage
|
|
69
|
+
* @param {() => boolean} dependencies.isClosing Whether the server's shutdown has begun.
|
|
43
70
|
*/
|
|
44
71
|
export function createCodexRunJournal({
|
|
45
72
|
namespace,
|
|
@@ -57,7 +84,8 @@ export function createCodexRunJournal({
|
|
|
57
84
|
idlePublishPayload,
|
|
58
85
|
finalizingGraceMs,
|
|
59
86
|
admissionTaskFinished,
|
|
60
|
-
orphanedPromptMessage
|
|
87
|
+
orphanedPromptMessage,
|
|
88
|
+
isClosing
|
|
61
89
|
}) {
|
|
62
90
|
const codexAppServerCompletedTurns = new Set();
|
|
63
91
|
const codexAppServerProcessedTurns = new Set();
|
|
@@ -394,6 +422,10 @@ export function createCodexRunJournal({
|
|
|
394
422
|
}
|
|
395
423
|
|
|
396
424
|
async function markCodexAppServerTurnIdle(sessionId = "", input = {}) {
|
|
425
|
+
// Once shutdown has begun the turn's end, and its checkpoint, are left to the next server's recovery, as after a crash.
|
|
426
|
+
if (isClosing()) {
|
|
427
|
+
return { ok: true, processed: false, reason: "left_to_next_server" };
|
|
428
|
+
}
|
|
397
429
|
const status = normalizeText(input.status) || "completed";
|
|
398
430
|
const result = await publishCodexAppServerAgentRun(sessionId, {
|
|
399
431
|
error: normalizeText(input.error),
|
|
@@ -6,6 +6,9 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
|
6
6
|
/** @typedef {ReturnType<typeof import("./codexTurn.js").createCodexAppServerRunOwner>} CodexRunOwner */
|
|
7
7
|
/** @typedef {(event: string, details: object) => void} CodexDebugLog */
|
|
8
8
|
|
|
9
|
+
/** A lifecycle without running turns to hand over leaves none. */
|
|
10
|
+
async function leaveNoTurns() {}
|
|
11
|
+
|
|
9
12
|
/**
|
|
10
13
|
* Shutdown, wellbeing timers and managed-session records for shared Codex
|
|
11
14
|
* runtimes. It works on the run/provider owners' collections; it owns no servers.
|
|
@@ -23,6 +26,8 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
|
23
26
|
* @param {CodexDebugLog} dependencies.debugLog
|
|
24
27
|
* @param {(error: unknown) => unknown} dependencies.debugError
|
|
25
28
|
* @param {{ message: string, code: string, retryable: boolean } | null} [dependencies.closingError]
|
|
29
|
+
* @param {() => Promise<unknown>} [dependencies.leaveTurnsToNextServer] When shutdown begins, leaves each running
|
|
30
|
+
* turn's end, and its checkpoint, to the next server.
|
|
26
31
|
*/
|
|
27
32
|
export function createCodexRuntimeLifecycle({
|
|
28
33
|
pendingWork,
|
|
@@ -35,7 +40,8 @@ export function createCodexRuntimeLifecycle({
|
|
|
35
40
|
continueGoalAfterAccountChange,
|
|
36
41
|
debugLog,
|
|
37
42
|
debugError,
|
|
38
|
-
closingError = null
|
|
43
|
+
closingError = null,
|
|
44
|
+
leaveTurnsToNextServer = leaveNoTurns
|
|
39
45
|
}) {
|
|
40
46
|
const codexAppServerSessionClosures = new Map();
|
|
41
47
|
const codexAppServerWellbeingTimers = new Map();
|
|
@@ -44,6 +50,7 @@ export function createCodexRuntimeLifecycle({
|
|
|
44
50
|
let codexAppServerThreadReconcileGeneration = 0;
|
|
45
51
|
let codexAppServerServerClosing = false;
|
|
46
52
|
let codexAppServerShutdownPromise = null;
|
|
53
|
+
let codexAppServerTurnsLeft = null;
|
|
47
54
|
|
|
48
55
|
function assertOpen() {
|
|
49
56
|
if (!closingError) return providerSessions.assertOpen();
|
|
@@ -216,11 +223,20 @@ export function createCodexRuntimeLifecycle({
|
|
|
216
223
|
}
|
|
217
224
|
}
|
|
218
225
|
|
|
226
|
+
// The server's shutdown begins: the closing fence is set and each running
|
|
227
|
+
// turn's end, and its checkpoint, is left to the next server, but nothing
|
|
228
|
+
// stops. The later server-shutdown invalidation stops the runtimes.
|
|
219
229
|
function beginCodexAppServerShutdown() {
|
|
220
230
|
if (codexAppServerServerClosing) {
|
|
221
231
|
return;
|
|
222
232
|
}
|
|
223
233
|
codexAppServerServerClosing = true;
|
|
234
|
+
// Taken now, before any runtime stops, while each running turn is known.
|
|
235
|
+
codexAppServerTurnsLeft = (async () => leaveTurnsToNextServer())().catch(error => {
|
|
236
|
+
debugLog("appServerShutdown.leaveTurns.error", { error: debugError(error) });
|
|
237
|
+
}).finally(() => {
|
|
238
|
+
codexAppServerTurnsLeft = null;
|
|
239
|
+
});
|
|
224
240
|
providerSessions.owner.beginShutdown();
|
|
225
241
|
codexAppServerThreadReconcileGeneration += 1;
|
|
226
242
|
for (const providerKey of [...codexAppServerWellbeingTimers.keys()]) {
|
|
@@ -231,6 +247,8 @@ export function createCodexRuntimeLifecycle({
|
|
|
231
247
|
async function drainCodexAppServerControllerTasks() {
|
|
232
248
|
while (true) {
|
|
233
249
|
const pending = new Set([
|
|
250
|
+
...(codexAppServerTurnsLeft ? [codexAppServerTurnsLeft] : []),
|
|
251
|
+
...pendingWork.deferredCheckpoints.values(),
|
|
234
252
|
...pendingWork.conversationTurnStarts.values(),
|
|
235
253
|
...helperLifecycle.projectOperations.values(),
|
|
236
254
|
...helperLifecycle.cleanups.values(),
|
|
@@ -356,6 +374,7 @@ export function createCodexRuntimeLifecycle({
|
|
|
356
374
|
|
|
357
375
|
return Object.freeze({
|
|
358
376
|
get closing() { return codexAppServerServerClosing; },
|
|
377
|
+
beginShutdown: beginCodexAppServerShutdown,
|
|
359
378
|
assertOpen,
|
|
360
379
|
get reconcileGeneration() { return codexAppServerThreadReconcileGeneration; },
|
|
361
380
|
nextReconcileGeneration,
|
|
@@ -10,7 +10,7 @@ import { createCodexNotificationObserver } from "./codexNotificationObserver.js"
|
|
|
10
10
|
import { createCodexThreadStatus } from "./codexThreadStatus.js";
|
|
11
11
|
import { createCodexTurnRecovery } from "./codexTurnRecovery.js";
|
|
12
12
|
import { createCodexResultSettlement } from "./codexResultSettlement.js";
|
|
13
|
-
import { createCodexRunJournal } from "./codexRunJournal.js";
|
|
13
|
+
import { createCodexRunJournal, markCodexAppServerTurnStoppedByShutdown } from "./codexRunJournal.js";
|
|
14
14
|
import { createCodexRenewalCommands } from "./codexRenewalCommands.js";
|
|
15
15
|
import { createCodexGoalCommands } from "./codexGoalCommands.js";
|
|
16
16
|
import { createCodexProviderSelection } from "./codexProviderSelection.js";
|
|
@@ -80,6 +80,13 @@ export { codexAppServerThreadIsMissing } from "./codexProviderThreads.js";
|
|
|
80
80
|
/** @typedef {typeof CODEX_TURN_OUTCOMES} CodexTurnOutcomes */
|
|
81
81
|
/** @typedef {(runtime: any, session: object, options: object) => Promise<any>} CodexAdmissionRecovery */
|
|
82
82
|
/** @typedef {(sessionId: string, input: object) => Promise<unknown>} CodexCheckpoint */
|
|
83
|
+
/**
|
|
84
|
+
* Records a scoped conversation turn's checkpoint as owed to the next server,
|
|
85
|
+
* once the server's shutdown has begun. The application passes the run back
|
|
86
|
+
* on that server's next read as `stoppedByShutdown: { runId, ended }`, and
|
|
87
|
+
* makes the checkpoint when `ended` is called.
|
|
88
|
+
* @typedef {(sessionId: string, input: { outerTurnId: string, threadId: string, turnId: string, status: string, turnOutcome: string }) => Promise<unknown>} CodexDeferCheckpoint
|
|
89
|
+
*/
|
|
83
90
|
|
|
84
91
|
/**
|
|
85
92
|
* The application hooks that add metadata to the turns a run writes.
|
|
@@ -161,6 +168,8 @@ function runInNoContext(_context, operation) {
|
|
|
161
168
|
* @param {string} [options.messageIdPrefix]
|
|
162
169
|
* @param {object} [options.idlePublishPayload]
|
|
163
170
|
* @param {CodexCheckpoint} [options.checkpoint]
|
|
171
|
+
* @param {CodexDeferCheckpoint} options.deferCheckpoint A host that predates it leaves it out: a scoped
|
|
172
|
+
* conversation turn that ends during shutdown then makes its checkpoint at once, and one the shutdown stops makes none.
|
|
164
173
|
* @param {Partial<CodexMessageMetadataHooks>} [options.messageMetadata]
|
|
165
174
|
* @param {boolean} [options.persistCommentary]
|
|
166
175
|
* @param {string} [options.deliveryStateMetadataKey]
|
|
@@ -204,6 +213,7 @@ export function createCodexAppServerRunOwner({
|
|
|
204
213
|
messageIdPrefix = "codex:",
|
|
205
214
|
idlePublishPayload,
|
|
206
215
|
checkpoint = answerNothing,
|
|
216
|
+
deferCheckpoint,
|
|
207
217
|
messageMetadata = {},
|
|
208
218
|
persistCommentary = true,
|
|
209
219
|
deliveryStateMetadataKey = "assistant_delivery",
|
|
@@ -317,7 +327,8 @@ export function createCodexAppServerRunOwner({
|
|
|
317
327
|
idlePublishPayload,
|
|
318
328
|
finalizingGraceMs,
|
|
319
329
|
admissionTaskFinished,
|
|
320
|
-
orphanedPromptMessage
|
|
330
|
+
orphanedPromptMessage,
|
|
331
|
+
isClosing: () => runOwner.runtimeLifecycle?.closing === true
|
|
321
332
|
});
|
|
322
333
|
const admissionRecovery = recoverAdmission || codexRunJournal.recoverAbandonedPromptClaim;
|
|
323
334
|
|
|
@@ -424,7 +435,11 @@ export function createCodexAppServerRunOwner({
|
|
|
424
435
|
ensureSessionProvider: codexProviderSelection.ensureSessionProvider,
|
|
425
436
|
borrowSessionProvider: codexProviderSelection.borrowSessionProvider,
|
|
426
437
|
steerFailedCode,
|
|
427
|
-
turnOutcomes
|
|
438
|
+
turnOutcomes,
|
|
439
|
+
deferCheckpoint,
|
|
440
|
+
isClosing: () => runOwner.runtimeLifecycle?.closing === true,
|
|
441
|
+
captureContext,
|
|
442
|
+
runInContext
|
|
428
443
|
});
|
|
429
444
|
|
|
430
445
|
const codexObservationControl = createCodexObservationControl({
|
|
@@ -636,6 +651,7 @@ export function createCodexAppServerRunOwner({
|
|
|
636
651
|
clearSessionRecoveryTimers: codexRunJournal.clearSessionRecoveryTimers,
|
|
637
652
|
publishAgentRun: codexRunJournal.publishAgentRun,
|
|
638
653
|
markTurnIdle: codexRunJournal.markTurnIdle,
|
|
654
|
+
markTurnStoppedByShutdown,
|
|
639
655
|
finalizeAssistantResult: codexResultSettlement.finalizeAssistantResult,
|
|
640
656
|
finalizedTurns: codexAppServerOutput.finalizedTurns,
|
|
641
657
|
liveProgressItems: codexAppServerOutput.liveProgressItems,
|
|
@@ -661,8 +677,10 @@ export function createCodexAppServerRunOwner({
|
|
|
661
677
|
reasoningPersistQueues: codexAppServerOutput.reasoningPersistQueues,
|
|
662
678
|
resultFinalizations: codexResultSettlement.resultFinalizations,
|
|
663
679
|
threadReconciliations: codexThreadReadiness.threadReconciliations,
|
|
664
|
-
notificationTasks: codexAppServerNotificationQueue.tasks
|
|
680
|
+
notificationTasks: codexAppServerNotificationQueue.tasks,
|
|
681
|
+
deferredCheckpoints: codexConversations.deferredCheckpoints
|
|
665
682
|
},
|
|
683
|
+
leaveTurnsToNextServer: leaveRunningTurnsToNextServer,
|
|
666
684
|
observer: codexNotificationObserver,
|
|
667
685
|
checkManagedConnection: codexThreadReadiness.checkManagedConnection,
|
|
668
686
|
providerSessions,
|
|
@@ -678,6 +696,7 @@ export function createCodexAppServerRunOwner({
|
|
|
678
696
|
providerSessions.owner.attachRuntimeLifecycle(runOwner.runtimeLifecycle);
|
|
679
697
|
providerSessions.assertOpen = runOwner.runtimeLifecycle.assertOpen;
|
|
680
698
|
}
|
|
699
|
+
runOwner.beginShutdown = runOwner.runtimeLifecycle.beginShutdown;
|
|
681
700
|
runOwner.invalidateRuntimes = runOwner.runtimeLifecycle.invalidateRuntimes;
|
|
682
701
|
Object.assign(runOwner, createCodexSessionReconciliation({
|
|
683
702
|
readiness: codexThreadReadiness, recoverSessionObservationLoss,
|
|
@@ -685,6 +704,24 @@ export function createCodexAppServerRunOwner({
|
|
|
685
704
|
createRuntime, namespace, turnState, sessionIdRequiredError, debugLog, debugError
|
|
686
705
|
}));
|
|
687
706
|
}
|
|
707
|
+
function markTurnStoppedByShutdown(sessionId = "", threadId = "") {
|
|
708
|
+
return markCodexAppServerTurnStoppedByShutdown({ createRuntime, turnState }, sessionId, threadId);
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
// At shutdown each running turn's end, and its checkpoint, belong to the
|
|
712
|
+
// next server: Main turns are marked so it records their interruption as a
|
|
713
|
+
// restart, and scoped conversations leave their checkpoints owed.
|
|
714
|
+
async function leaveRunningTurnsToNextServer() {
|
|
715
|
+
const results = await Promise.allSettled([
|
|
716
|
+
codexConversations.leaveRunningTurnsToNextServer(),
|
|
717
|
+
...[...providerSessions.managed.values()].map(managed => Promise.resolve().then(() => runInContext(
|
|
718
|
+
managed.projectContext, () => markTurnStoppedByShutdown(managed.sessionId, managed.threadId))))
|
|
719
|
+
]);
|
|
720
|
+
for (const result of results) {
|
|
721
|
+
if (result.status === "rejected") debugLog("appServerShutdown.leaveTurn.error", { error: debugError(result.reason) });
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
688
725
|
const cleanup = createCodexSessionCleanup({
|
|
689
726
|
namespace,
|
|
690
727
|
runOwner,
|
|
@@ -253,9 +253,9 @@ export function createCodexTurnEndings({
|
|
|
253
253
|
const normalizedStatus = normalizeText(status);
|
|
254
254
|
let base = "Codex app-server failed before completing this turn.";
|
|
255
255
|
if (normalizedStatus === "interrupted") {
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
256
|
+
if (outcome === turnOutcomes.CONTROL_RECONFIGURATION) base = "Codex app-server was interrupted while its controls were restored.";
|
|
257
|
+
else if (outcome === turnOutcomes.SERVICE_RESTART) base = "Codex app-server was interrupted by a restart before completing this turn.";
|
|
258
|
+
else base = "Codex app-server was interrupted before completing this turn.";
|
|
259
259
|
}
|
|
260
260
|
const normalizedError = normalizeText(error);
|
|
261
261
|
return normalizedError ? `${base} ${normalizedError}` : base;
|
|
@@ -277,17 +277,20 @@ export function createCodexTurnEndings({
|
|
|
277
277
|
const normalizedThreadId = normalizeText(threadId);
|
|
278
278
|
const normalizedTurnId = normalizeText(turnId);
|
|
279
279
|
const normalizedStatus = normalizeText(status) || "failed";
|
|
280
|
+
const runtime = await createRuntime();
|
|
280
281
|
let resolvedOutcome = normalizeText(outcome);
|
|
281
282
|
if (!resolvedOutcome) {
|
|
282
283
|
if (normalizedStatus === "interrupted") {
|
|
283
284
|
const recorded = provider ? provider.interruptionOutcome(normalizedThreadId, normalizedTurnId) : "";
|
|
284
|
-
|
|
285
|
-
|
|
285
|
+
if ([turnOutcomes.USER_CANCELLED, turnOutcomes.CONTROL_RECONFIGURATION].includes(recorded)) resolvedOutcome = recorded;
|
|
286
|
+
// The previous server's shutdown stopped this turn: a restart, not a failure.
|
|
287
|
+
else if (codexAppServerAgentRun(await runtime.getSession(normalizedSessionId))?.stoppedByShutdownTurnId === normalizedTurnId) {
|
|
288
|
+
resolvedOutcome = turnOutcomes.SERVICE_RESTART;
|
|
289
|
+
} else resolvedOutcome = turnOutcomes.INTERRUPTED;
|
|
286
290
|
} else {
|
|
287
291
|
resolvedOutcome = turnOutcomes.PROVIDER_FAILURE;
|
|
288
292
|
}
|
|
289
293
|
}
|
|
290
|
-
const runtime = await createRuntime();
|
|
291
294
|
const staleFailure = async () => {
|
|
292
295
|
const turn = turnState(await runtime.getSession(normalizedSessionId));
|
|
293
296
|
if (codexAppServerTurnCanReceiveProviderCompletion(turn, normalizedThreadId, normalizedTurnId)) {
|
|
@@ -218,6 +218,27 @@ function createNativeConversationLifecycle({ authorize, defaultHost, hostPrepare
|
|
|
218
218
|
return driver.invalidateRuntimes(prepared);
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
+
// The server's shutdown begins before anything stops, on the same trusted
|
|
222
|
+
// boundary: the configured owner sets its closing fence and leaves each
|
|
223
|
+
// running turn's end, and its checkpoint, to the next server, synchronously.
|
|
224
|
+
// A later invalidation with reason "server-shutdown" stops its processes.
|
|
225
|
+
function beginNativeShutdown({ context, input = {} }) {
|
|
226
|
+
if (!hostPrepares) {
|
|
227
|
+
throw new TypeError("Native shutdown requires a configured conversation host.");
|
|
228
|
+
}
|
|
229
|
+
const prepared = defaultHost.conversation({ context, input, operation: "beginShutdown" });
|
|
230
|
+
if (!prepared || isThenable(prepared) ||
|
|
231
|
+
!["codex", "claude", "opencode"].includes(prepared.engine) || !prepared.native ||
|
|
232
|
+
!Object.hasOwn(prepared, "input") || !Object.hasOwn(prepared, "context")) {
|
|
233
|
+
throw new TypeError("Native shutdown requires synchronous configured host preparation.");
|
|
234
|
+
}
|
|
235
|
+
const driver = providers.createDriver(prepared.engine, defaultHost);
|
|
236
|
+
if (!implementsOperation(driver, "beginShutdown")) {
|
|
237
|
+
throw new TypeError("The selected conversation driver does not implement native shutdown.");
|
|
238
|
+
}
|
|
239
|
+
return driver.beginShutdown(prepared);
|
|
240
|
+
}
|
|
241
|
+
|
|
221
242
|
// Startup reconciliation and subscription reset use the existing trusted
|
|
222
243
|
// lifecycle caller. Native owners retain their own shutdown fences and tasks.
|
|
223
244
|
function reconcileNativeSessions({ context, sessions, options = {} }) {
|
|
@@ -363,6 +384,7 @@ function createNativeConversationLifecycle({ authorize, defaultHost, hostPrepare
|
|
|
363
384
|
createNativeConversation,
|
|
364
385
|
runNativeDetachedConversation,
|
|
365
386
|
closeNativeProject,
|
|
387
|
+
beginNativeShutdown,
|
|
366
388
|
invalidateNativeRuntimes,
|
|
367
389
|
reconcileNativeSessions,
|
|
368
390
|
unsubscribeNativeSessions,
|
|
@@ -441,12 +441,17 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
|
|
|
441
441
|
};
|
|
442
442
|
}
|
|
443
443
|
|
|
444
|
+
// The server's shutdown begins: the owner closes, so each running turn's
|
|
445
|
+
// end, and its checkpoint, is left to the next server, but nothing stops.
|
|
446
|
+
// The later server-shutdown invalidation stops the processes.
|
|
447
|
+
function beginShutdown() {
|
|
448
|
+
closed = true;
|
|
449
|
+
clearTimeout(storageIdle);
|
|
450
|
+
storageIdle = null;
|
|
451
|
+
}
|
|
452
|
+
|
|
444
453
|
async function invalidateRuntimes(input = {}, preparation) {
|
|
445
|
-
if (normalizeNullableText(input.reason) === "server-shutdown")
|
|
446
|
-
closed = true;
|
|
447
|
-
clearTimeout(storageIdle);
|
|
448
|
-
storageIdle = null;
|
|
449
|
-
}
|
|
454
|
+
if (normalizeNullableText(input.reason) === "server-shutdown") beginShutdown();
|
|
450
455
|
const application = preparation.release;
|
|
451
456
|
// readable-check: allow method-probe - input validation: a connection change carries the publication of the application's saved connections
|
|
452
457
|
if (typeof input.publish === "function") {
|
|
@@ -567,6 +572,7 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
|
|
|
567
572
|
releaseProcessTarget,
|
|
568
573
|
closeSession,
|
|
569
574
|
closeProject,
|
|
575
|
+
beginShutdown,
|
|
570
576
|
invalidateRuntimes,
|
|
571
577
|
releaseRenewalPredecessorProcessExitProof,
|
|
572
578
|
releaseProcessExitProof
|
|
@@ -226,6 +226,7 @@ export function createOpenCodeSharedRuntime({ onStop = ignoreRuntimeStop, scope,
|
|
|
226
226
|
sendPreparedMessage: sessionTurns.sendPreparedMessage,
|
|
227
227
|
inspectPreparedMessage: sessionTurns.inspectPreparedMessage,
|
|
228
228
|
interruptPreparedTurn: sessionTurns.interruptPreparedTurn,
|
|
229
|
+
beginShutdown: processOwner.beginShutdown,
|
|
229
230
|
invalidateRuntimes: processOwner.invalidateRuntimes,
|
|
230
231
|
releaseRenewalPredecessorProcessExitProof: processOwner.releaseRenewalPredecessorProcessExitProof,
|
|
231
232
|
releaseProcessExitProof: processOwner.releaseProcessExitProof,
|
|
@@ -478,9 +478,10 @@ export async function waitForOpenCodeFinalResponse(client, conversationId, turn,
|
|
|
478
478
|
/**
|
|
479
479
|
* Production detached-turn lifetime. Applications project the completed native
|
|
480
480
|
* result before observation is closed, then publish/checkpoint their own state.
|
|
481
|
-
* A turn the server's shutdown stopped (`turn.stoppedByShutdown`)
|
|
482
|
-
*
|
|
483
|
-
* application leaves its checkpoint to
|
|
481
|
+
* A turn the server's shutdown stopped (`turn.stoppedByShutdown`), or that
|
|
482
|
+
* ends on its own once shutdown has begun, ends interrupted however its wait
|
|
483
|
+
* ended, and `afterClose` is told so: the application leaves its checkpoint to
|
|
484
|
+
* the next server.
|
|
484
485
|
* @param {import("./openCodeProcessOwner.js").OpenCodeProcessOwner} processOwner
|
|
485
486
|
*/
|
|
486
487
|
export async function observeOpenCodeTurnCompletion(processOwner, target, conversationId, turn, {
|
|
@@ -489,6 +490,10 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
|
|
|
489
490
|
const turnAbort = turn.abortController;
|
|
490
491
|
let stopped = true;
|
|
491
492
|
let outcome = "completed";
|
|
493
|
+
const leftToNextServer = () => {
|
|
494
|
+
if (processOwner.closed) turn.stoppedByShutdown = true;
|
|
495
|
+
return turn.stoppedByShutdown === true;
|
|
496
|
+
};
|
|
492
497
|
try {
|
|
493
498
|
const options = { ...waitOptions, signal: AbortSignal.any([
|
|
494
499
|
signal, ...(timeoutMs ? [AbortSignal.timeout(timeoutMs)] : [])
|
|
@@ -496,14 +501,14 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
|
|
|
496
501
|
const completion = finalResponse
|
|
497
502
|
? await waitForOpenCodeFinalResponse(target.server.client, conversationId, turn, { ...finalResponse, ...options })
|
|
498
503
|
: await waitForOpenCodeMessages(target.server.client, conversationId, () => turn.inputMessageId, options);
|
|
499
|
-
if (
|
|
504
|
+
if (leftToNextServer()) {
|
|
500
505
|
outcome = "interrupted";
|
|
501
506
|
return { interrupted: true };
|
|
502
507
|
}
|
|
503
508
|
return await projectResult(completion);
|
|
504
509
|
} catch (error) {
|
|
505
510
|
const failure = signal.aborted ? signal.reason : error;
|
|
506
|
-
outcome =
|
|
511
|
+
outcome = leftToNextServer() || turn.interrupted ? "interrupted" : "failed";
|
|
507
512
|
if (outcome === "interrupted") return { interrupted: true };
|
|
508
513
|
if (!target.abortController.signal.aborted) {
|
|
509
514
|
try {
|
|
@@ -355,8 +355,10 @@ export function createOpenCodeTurnMonitor({ processes, monitors, turns, processO
|
|
|
355
355
|
// server: its saved run stays active and carries the shutdown's mark,
|
|
356
356
|
// so that server's recovery records the turn interrupted and runs the
|
|
357
357
|
// host's checkpoint. Run here, that checkpoint (a Git snapshot, say)
|
|
358
|
-
//
|
|
359
|
-
//
|
|
358
|
+
// could be refused or hold the host's bounded shutdown for as long as
|
|
359
|
+
// it takes. A turn that ends on its own once shutdown has begun is left
|
|
360
|
+
// the same way. A user's Stop outside shutdown ends its turn here.
|
|
361
|
+
if (processOwner.closed && finalState !== "active") turn.stoppedByShutdown = true;
|
|
360
362
|
const nextServerEnds = turn.stoppedByShutdown === true && finalState !== "active";
|
|
361
363
|
// Claim retirement before awaited cleanup/checkpointing. A new Send
|
|
362
364
|
// joins this monitor's cleanup instead of steering an unobserved turn.
|
|
@@ -346,6 +346,10 @@ export function createClaudeConversationDriver({
|
|
|
346
346
|
return native.owner.closeProject(input, native.application);
|
|
347
347
|
}
|
|
348
348
|
|
|
349
|
+
function beginShutdown({ native }) {
|
|
350
|
+
return native.owner.beginShutdown();
|
|
351
|
+
}
|
|
352
|
+
|
|
349
353
|
function invalidateRuntimes({ native, input }) {
|
|
350
354
|
return native.owner.invalidateRuntimes(input, native.application, native.account);
|
|
351
355
|
}
|
|
@@ -568,6 +572,7 @@ export function createClaudeConversationDriver({
|
|
|
568
572
|
listConversationStorage,
|
|
569
573
|
retireConversationHistory,
|
|
570
574
|
closeProject,
|
|
575
|
+
beginShutdown,
|
|
571
576
|
invalidateRuntimes,
|
|
572
577
|
reconcileSessions,
|
|
573
578
|
createConversation,
|
|
@@ -63,6 +63,10 @@ function nativeGoalChanged() {
|
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
function beginShutdown({ native }) {
|
|
67
|
+
return native.runOwner.beginShutdown();
|
|
68
|
+
}
|
|
69
|
+
|
|
66
70
|
/**
|
|
67
71
|
* Conversations own threads and observers on the account's shared native runtime.
|
|
68
72
|
* @param {object} dependencies
|
|
@@ -653,6 +657,7 @@ export function createCodexConversationDriver({ connections, host = {}, limits =
|
|
|
653
657
|
deleteDetachedConversation,
|
|
654
658
|
listConversationStorage,
|
|
655
659
|
retireConversationHistory,
|
|
660
|
+
beginShutdown,
|
|
656
661
|
invalidateRuntimes,
|
|
657
662
|
reconcileSessions,
|
|
658
663
|
unsubscribeSessions,
|
|
@@ -126,6 +126,10 @@ export function createOpenCodeConversationDriver({
|
|
|
126
126
|
return native.owner.closeProject(input, native.preparation.projectCleanup());
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
+
function beginShutdown({ native }) {
|
|
130
|
+
return native.owner.beginShutdown();
|
|
131
|
+
}
|
|
132
|
+
|
|
129
133
|
function invalidateRuntimes({ native, input }) {
|
|
130
134
|
return native.owner.invalidateRuntimes(input, native.preparation);
|
|
131
135
|
}
|
|
@@ -375,6 +379,7 @@ export function createOpenCodeConversationDriver({
|
|
|
375
379
|
listConversationStorage,
|
|
376
380
|
retireConversationHistory,
|
|
377
381
|
closeProject,
|
|
382
|
+
beginShutdown,
|
|
378
383
|
invalidateRuntimes,
|
|
379
384
|
reconcileSessions,
|
|
380
385
|
createConversation,
|
|
@@ -227,6 +227,7 @@ export function createConversationRuntime({ engine: defaultEngine = "api", defau
|
|
|
227
227
|
runNativeDetachedConversation: lifecycle.runNativeDetachedConversation,
|
|
228
228
|
open: handles.open,
|
|
229
229
|
closeNativeProject: lifecycle.closeNativeProject,
|
|
230
|
+
beginNativeShutdown: lifecycle.beginNativeShutdown,
|
|
230
231
|
invalidateNativeRuntimes: lifecycle.invalidateNativeRuntimes,
|
|
231
232
|
reconcileNativeSessions: lifecycle.reconcileNativeSessions,
|
|
232
233
|
unsubscribeNativeSessions: lifecycle.unsubscribeNativeSessions,
|
|
@@ -242,6 +242,34 @@ test("Claude shutdown leaves a running turn's end and its checkpoint to the next
|
|
|
242
242
|
assert.deepEqual(custody(JSON.parse(f.session("a").records.get(MAIN_IDS.a))), { executionId: "", state: "interrupted" });
|
|
243
243
|
});
|
|
244
244
|
|
|
245
|
+
test("once Claude's shutdown has begun, a turn that ends on its own is left to the next server, which runs its one checkpoint", { timeout: 5_000 }, async t => {
|
|
246
|
+
const f = await lifecycleFixture(t);
|
|
247
|
+
const a = await f.running("a");
|
|
248
|
+
const saved = f.session("a").records.get(MAIN_IDS.a);
|
|
249
|
+
f.owner.beginShutdown();
|
|
250
|
+
assert.deepEqual(f.calls.stops, [], "beginning the shutdown stopped nothing");
|
|
251
|
+
// The turn ends on its own.
|
|
252
|
+
await f.processes[0].options.onEvent({ type: "result", subtype: "success", result: "Done." });
|
|
253
|
+
assert.deepEqual(f.calls.checkpoints, [], "the turn's end ran no checkpoint");
|
|
254
|
+
assert.equal(f.session("a").records.get(MAIN_IDS.a), saved, "the saved record keeps the active turn");
|
|
255
|
+
assert.equal((await f.owner.invalidateRuntimes({ reason: "server-shutdown" }, f.application, f.account)).ok, true);
|
|
256
|
+
assert.equal(a.process, null, "the full stop still stopped the process");
|
|
257
|
+
assert.deepEqual(f.calls.checkpoints, []);
|
|
258
|
+
// The next server's close proves that exit, then interrupts the turn through its checkpoint, once.
|
|
259
|
+
f.restart();
|
|
260
|
+
assert.equal((await f.owner.closeSession({ sessionId: "a" }, f.application)).ok, true);
|
|
261
|
+
assert.equal((await f.owner.closeSession({ sessionId: "a" }, f.application)).ok, true);
|
|
262
|
+
assert.deepEqual(f.calls.checkpoints, ["a:interrupted"]);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test("outside shutdown a Claude turn that ends on its own still runs its checkpoint at once", { timeout: 5_000 }, async t => {
|
|
266
|
+
const f = await lifecycleFixture(t);
|
|
267
|
+
await f.running("a");
|
|
268
|
+
await f.processes[0].options.onEvent({ type: "result", subtype: "success", result: "Done." });
|
|
269
|
+
assert.deepEqual(f.calls.checkpoints, ["a:completed"]);
|
|
270
|
+
assert.equal(JSON.parse(f.session("a").records.get(MAIN_IDS.a)).state, "completed");
|
|
271
|
+
});
|
|
272
|
+
|
|
245
273
|
test("Claude project close releases its sessions from memory without reopening saved conversations", async t => {
|
|
246
274
|
const f = await lifecycleFixture(t);
|
|
247
275
|
await f.idle("a");
|
|
@@ -20,6 +20,8 @@ function persistentConversationFixture() {
|
|
|
20
20
|
};
|
|
21
21
|
const commands = createCodexConversationCommands({
|
|
22
22
|
async checkpoint(_sessionId, input) { checkpoints.push(input); },
|
|
23
|
+
// Outside shutdown, with no request contexts.
|
|
24
|
+
isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
|
|
23
25
|
conversationPreparation: { async context() { return { result: { provider } }; } },
|
|
24
26
|
namespace: (sessionId) => sessionId,
|
|
25
27
|
// The provider owner still holds the provider observing the conversation;
|
|
@@ -231,6 +231,8 @@ function conversationCommands({ context = async () => ({ result: { ok: false, co
|
|
|
231
231
|
providerSessions = {} } = {}) {
|
|
232
232
|
return createCodexConversationCommands({
|
|
233
233
|
checkpoint: async () => {},
|
|
234
|
+
// Outside shutdown, with no request contexts.
|
|
235
|
+
isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
|
|
234
236
|
conversationPreparation: {
|
|
235
237
|
context,
|
|
236
238
|
scope,
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import { createCodexConversationCommands } from "../src/server/conversation/codexConversations.js";
|
|
4
|
+
import { createCodexRuntimeLifecycle } from "../src/server/conversation/codexRuntimeLifecycle.js";
|
|
5
|
+
|
|
6
|
+
// A Temporary chat's Codex turn that the server's shutdown stops, or that ends
|
|
7
|
+
// while the server is stopping, leaves its checkpoint to the next server: the
|
|
8
|
+
// application records it as owed, and that server's next read makes it once.
|
|
9
|
+
|
|
10
|
+
const turnOutcomes = { USER_CANCELLED: "user_cancelled", SERVICE_RESTART: "service_restart" };
|
|
11
|
+
|
|
12
|
+
// One server's observed conversation. `latest` is the turn Codex's history
|
|
13
|
+
// reports to a read; turn notifications reach the conversation's observer.
|
|
14
|
+
function server({ closing = false, latest = { id: "turn-1", status: "inProgress", items: [] }, threadError = null } = {}) {
|
|
15
|
+
const checkpoints = [];
|
|
16
|
+
const deferred = [];
|
|
17
|
+
const listeners = new Set();
|
|
18
|
+
const emit = notification => listeners.forEach(listener => listener(notification));
|
|
19
|
+
const provider = {
|
|
20
|
+
subscribe(listener) {
|
|
21
|
+
listeners.add(listener);
|
|
22
|
+
return () => listeners.delete(listener);
|
|
23
|
+
},
|
|
24
|
+
async readThreadStatus(threadId) {
|
|
25
|
+
return { raw: { id: threadId, historyMode: "paginated", status: "idle", ...(threadError ? { error: threadError } : {}) } };
|
|
26
|
+
},
|
|
27
|
+
async listThreadTurns() { return { data: [latest] }; },
|
|
28
|
+
async readGoal() { return { goal: null }; },
|
|
29
|
+
async stopThreadForObservationLoss() {}
|
|
30
|
+
};
|
|
31
|
+
const lifecycle = { closing };
|
|
32
|
+
const commands = createCodexConversationCommands({
|
|
33
|
+
async checkpoint(sessionId, input) { checkpoints.push({ sessionId, ...input }); },
|
|
34
|
+
async deferCheckpoint(sessionId, input) { deferred.push({ sessionId, ...input }); },
|
|
35
|
+
isClosing: () => lifecycle.closing,
|
|
36
|
+
captureContext: () => null,
|
|
37
|
+
runInContext: (_context, operation) => operation(),
|
|
38
|
+
codexAppServerConversationMessageId: (...parts) => parts.join(":"),
|
|
39
|
+
conversationPreparation: { async context() { return { result: { provider } }; } },
|
|
40
|
+
namespace: sessionId => sessionId,
|
|
41
|
+
providerSessions: { owner: { providers: new Map([["provider", provider]]), holdProvider: () => () => {} } },
|
|
42
|
+
turnOutcomes
|
|
43
|
+
});
|
|
44
|
+
const startTurn = () => {
|
|
45
|
+
commands.observeCodexConversation("session", "thread", provider);
|
|
46
|
+
emit({ method: "turn/started", params: { threadId: "thread", turn: { id: "turn-1", status: "inProgress" } } });
|
|
47
|
+
};
|
|
48
|
+
const complete = status => emit({ method: "turn/completed", params: { threadId: "thread", turn: { id: "turn-1", status } } });
|
|
49
|
+
const read = stoppedByShutdown => commands.readPersistentCodexConversation("session", { conversationId: "thread", stoppedByShutdown },
|
|
50
|
+
{ provider }).then(({ status, error, runId }) => ({ status, error, runId }));
|
|
51
|
+
return { checkpoints, deferred, commands, lifecycle, startTurn, complete, read };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const OWED = { sessionId: "session", outerTurnId: "codex:thread:turn-1", threadId: "thread", turnId: "turn-1" };
|
|
55
|
+
|
|
56
|
+
test("shutdown leaves a running Temporary turn's interrupted checkpoint to the next server, which makes it once on its read", async () => {
|
|
57
|
+
const first = server();
|
|
58
|
+
first.startTurn();
|
|
59
|
+
first.lifecycle.closing = true;
|
|
60
|
+
await first.commands.leaveRunningTurnsToNextServer();
|
|
61
|
+
assert.deepEqual(first.checkpoints, [], "shutdown made no checkpoint");
|
|
62
|
+
assert.deepEqual(first.deferred, [{ ...OWED, status: "interrupted", turnOutcome: "service_restart" }]);
|
|
63
|
+
|
|
64
|
+
// The stopped process ends the turn natively as interrupted.
|
|
65
|
+
const next = server({ latest: { id: "turn-1", status: "interrupted", items: [] }, threadError: { message: "Turn aborted." } });
|
|
66
|
+
const ended = [];
|
|
67
|
+
const mark = runId => ({ runId, async ended() { ended.push(runId); } });
|
|
68
|
+
assert.deepEqual(await next.read(mark("turn-1")), { status: "interrupted", error: "", runId: "turn-1" });
|
|
69
|
+
assert.deepEqual(ended, ["turn-1"], "recording the stopped run lets the application make its checkpoint");
|
|
70
|
+
assert.deepEqual(await next.read(undefined), { status: "interrupted", error: "Turn aborted.", runId: "turn-1" },
|
|
71
|
+
"an unmarked interruption keeps Codex's own report");
|
|
72
|
+
await next.read(mark("turn-0"));
|
|
73
|
+
assert.deepEqual(ended, ["turn-1"], "only the marked run is recorded as ended");
|
|
74
|
+
assert.deepEqual(next.checkpoints, []);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("a Temporary turn that ends on its own while the server stops leaves its checkpoint, with its own outcome, to the next server", async () => {
|
|
78
|
+
const f = server();
|
|
79
|
+
f.startTurn();
|
|
80
|
+
f.lifecycle.closing = true;
|
|
81
|
+
f.complete("completed");
|
|
82
|
+
await Promise.all(f.commands.deferredCheckpoints);
|
|
83
|
+
assert.deepEqual(f.checkpoints, []);
|
|
84
|
+
assert.deepEqual(f.deferred, [{ ...OWED, status: "completed", turnOutcome: "" }]);
|
|
85
|
+
assert.equal(f.commands.deferredCheckpoints.size, 0, "shutdown awaits each deferred record");
|
|
86
|
+
|
|
87
|
+
// The next server reads the completed turn as completed, and makes its checkpoint.
|
|
88
|
+
const next = server({ latest: { id: "turn-1", status: "completed", items: [] } });
|
|
89
|
+
const ended = [];
|
|
90
|
+
assert.equal((await next.read({ runId: "turn-1", async ended() { ended.push("turn-1"); } })).status, "completed");
|
|
91
|
+
assert.deepEqual(ended, ["turn-1"]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("outside shutdown a Temporary turn's end still makes its checkpoint at once", async () => {
|
|
95
|
+
const f = server();
|
|
96
|
+
f.startTurn();
|
|
97
|
+
f.complete("interrupted");
|
|
98
|
+
assert.deepEqual(f.checkpoints, [{ ...OWED, status: "interrupted", turnOutcome: "" }]);
|
|
99
|
+
assert.deepEqual(f.deferred, []);
|
|
100
|
+
await f.commands.leaveRunningTurnsToNextServer();
|
|
101
|
+
assert.deepEqual(f.deferred, [], "a finished turn owes nothing at shutdown");
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// The run owner's runtime lifecycle, with the owner calls it makes recorded.
|
|
105
|
+
function runtimeLifecycle(calls, leaveTurnsToNextServer) {
|
|
106
|
+
return createCodexRuntimeLifecycle({
|
|
107
|
+
pendingWork: { conversationTurnStarts: new Map(), messageDeliveries: new Map(), reasoningPersistQueues: new Map(),
|
|
108
|
+
resultFinalizations: new Map(), threadReconciliations: new Map(), notificationTasks: new Map(), deferredCheckpoints: new Set() },
|
|
109
|
+
observer: { eventSubscriptions: new Map() },
|
|
110
|
+
checkManagedConnection: async () => {},
|
|
111
|
+
providerSessions: { managed: new Map(), assertOpen() {}, owner: {
|
|
112
|
+
lifecycleTasks: new Set(), runtimeAcquisitions: new Set(),
|
|
113
|
+
beginShutdown() { calls.push("owner-shutdown"); },
|
|
114
|
+
async invalidateRuntimes() { calls.push("stop-runtimes"); return { failed: [], providerCount: 0, results: [], stopped: 0 }; }
|
|
115
|
+
} },
|
|
116
|
+
helperLifecycle: { projectOperations: new Map(), cleanups: new Map(), mutations: new Map(), restores: new Map(), turnStarts: new Map() },
|
|
117
|
+
captureContext: () => null,
|
|
118
|
+
runInContext: (_context, operation) => operation(),
|
|
119
|
+
debugLog() {},
|
|
120
|
+
debugError: error => error?.message,
|
|
121
|
+
leaveTurnsToNextServer
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
test("a Codex shutdown leaves the running turns to the next server before any runtime stops, and waits for their records", async () => {
|
|
126
|
+
const calls = [];
|
|
127
|
+
const recorded = Promise.withResolvers();
|
|
128
|
+
const lifecycle = runtimeLifecycle(calls, async () => {
|
|
129
|
+
calls.push("leave-turns");
|
|
130
|
+
await recorded.promise;
|
|
131
|
+
calls.push("turns-recorded");
|
|
132
|
+
});
|
|
133
|
+
const shutdown = lifecycle.invalidateRuntimes({ reason: "server-shutdown" }, { enabled: true });
|
|
134
|
+
assert.equal(lifecycle.closing, true);
|
|
135
|
+
await Promise.resolve();
|
|
136
|
+
recorded.resolve();
|
|
137
|
+
await shutdown;
|
|
138
|
+
assert.equal(calls[0], "leave-turns");
|
|
139
|
+
assert.ok(calls.indexOf("turns-recorded") < calls.lastIndexOf("stop-runtimes"), "shutdown waited for the records");
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test("a begun Codex shutdown stops nothing, and a Temporary turn that then ends on its own leaves its checkpoint to the next server", async () => {
|
|
143
|
+
const f = server();
|
|
144
|
+
f.startTurn();
|
|
145
|
+
const calls = [];
|
|
146
|
+
const lifecycle = runtimeLifecycle(calls, () => f.commands.leaveRunningTurnsToNextServer());
|
|
147
|
+
Object.defineProperty(f.lifecycle, "closing", { get: () => lifecycle.closing });
|
|
148
|
+
lifecycle.beginShutdown();
|
|
149
|
+
assert.equal(lifecycle.closing, true, "the closing fence is set before the call returns");
|
|
150
|
+
assert.deepEqual(calls, ["owner-shutdown"], "beginning the shutdown stopped nothing");
|
|
151
|
+
f.complete("completed");
|
|
152
|
+
await Promise.all(f.commands.deferredCheckpoints);
|
|
153
|
+
assert.deepEqual(f.checkpoints, [], "the turn's end made no checkpoint");
|
|
154
|
+
assert.deepEqual(f.deferred.at(-1), { ...OWED, status: "completed", turnOutcome: "" }, "its own outcome is owed");
|
|
155
|
+
|
|
156
|
+
// The full stop that follows still stops the runtimes, and leaves the turns only once.
|
|
157
|
+
const deferred = f.deferred.length;
|
|
158
|
+
await lifecycle.invalidateRuntimes({ reason: "server-shutdown" }, { enabled: true });
|
|
159
|
+
assert.ok(calls.includes("stop-runtimes"));
|
|
160
|
+
assert.equal(f.deferred.length, deferred);
|
|
161
|
+
assert.deepEqual(f.checkpoints, []);
|
|
162
|
+
|
|
163
|
+
// The next server's read makes the owed checkpoint once.
|
|
164
|
+
const next = server({ latest: { id: "turn-1", status: "completed", items: [] } });
|
|
165
|
+
const ended = [];
|
|
166
|
+
assert.equal((await next.read({ runId: "turn-1", async ended() { ended.push("turn-1"); } })).status, "completed");
|
|
167
|
+
assert.deepEqual(ended, ["turn-1"]);
|
|
168
|
+
});
|
|
@@ -53,6 +53,8 @@ function codexFixture(t, { idleMs = 20 } = {}) {
|
|
|
53
53
|
});
|
|
54
54
|
const commands = createCodexConversationCommands({
|
|
55
55
|
async checkpoint() {},
|
|
56
|
+
// Outside shutdown, with no request contexts.
|
|
57
|
+
isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
|
|
56
58
|
codexAppServerConversationMessageId: (...parts) => parts.join(":"),
|
|
57
59
|
codexAppServerStreamMessage: () => ({}),
|
|
58
60
|
conversationPreparation: {
|
package/test/codexTurn.test.js
CHANGED
|
@@ -3856,3 +3856,60 @@ test("native history budget rejects unavailable dynamic limits before pending de
|
|
|
3856
3856
|
assert.equal(sends, 0);
|
|
3857
3857
|
}
|
|
3858
3858
|
});
|
|
3859
|
+
|
|
3860
|
+
test("a Main turn the previous server's shutdown stopped reads as interrupted by a restart, with one interrupted checkpoint", async () => {
|
|
3861
|
+
const notices = [];
|
|
3862
|
+
const checkpoints = [];
|
|
3863
|
+
const f = await receiptFixture({
|
|
3864
|
+
outcomeNotice(_runtime, _sessionId, _threadId, _turnId, outcome) { notices.push(outcome); },
|
|
3865
|
+
checkpoint(_sessionId, input) { checkpoints.push(input); }
|
|
3866
|
+
});
|
|
3867
|
+
f.provider.interruptionOutcome = () => "";
|
|
3868
|
+
// The shutdown marks the running turn and leaves the rest of its saved run as it was.
|
|
3869
|
+
assert.deepEqual(await f.owner.markTurnStoppedByShutdown(f.sessionId, f.threadId), { marked: true });
|
|
3870
|
+
assert.deepEqual({ state: f.run.state, stoppedByShutdownTurnId: f.run.stoppedByShutdownTurnId },
|
|
3871
|
+
{ state: "active", stoppedByShutdownTurnId: f.turnId });
|
|
3872
|
+
assert.deepEqual(await f.owner.markTurnStoppedByShutdown(f.sessionId, "another-thread"), { marked: false });
|
|
3873
|
+
// The next server reads the native turn as interrupted.
|
|
3874
|
+
const result = await f.owner.stopTurnWithProviderFailure(f.sessionId, f.threadId, f.turnId, {
|
|
3875
|
+
status: "interrupted", provider: f.provider, verifyInactive: false
|
|
3876
|
+
});
|
|
3877
|
+
assert.equal(result.status, "interrupted");
|
|
3878
|
+
assert.match(result.error, /interrupted by a restart/u);
|
|
3879
|
+
assert.deepEqual(notices, ["service_restart"]);
|
|
3880
|
+
assert.deepEqual(checkpoints.map(({ status, turnOutcome }) => ({ status, turnOutcome })),
|
|
3881
|
+
[{ status: "interrupted", turnOutcome: "service_restart" }]);
|
|
3882
|
+
assert.equal(f.run.state, "interrupted");
|
|
3883
|
+
});
|
|
3884
|
+
|
|
3885
|
+
test("an interrupted Main turn without the shutdown's mark keeps the plain interrupted outcome", async () => {
|
|
3886
|
+
const notices = [];
|
|
3887
|
+
const checkpoints = [];
|
|
3888
|
+
const f = await receiptFixture({
|
|
3889
|
+
outcomeNotice(_runtime, _sessionId, _threadId, _turnId, outcome) { notices.push(outcome); },
|
|
3890
|
+
checkpoint(_sessionId, input) { checkpoints.push(input); }
|
|
3891
|
+
});
|
|
3892
|
+
f.provider.interruptionOutcome = () => "";
|
|
3893
|
+
// A mark left by an earlier turn does not apply to this one.
|
|
3894
|
+
f.run = { ...f.run, stoppedByShutdownTurnId: "an-earlier-turn" };
|
|
3895
|
+
await f.owner.stopTurnWithProviderFailure(f.sessionId, f.threadId, f.turnId, {
|
|
3896
|
+
status: "interrupted", provider: f.provider, verifyInactive: false
|
|
3897
|
+
});
|
|
3898
|
+
assert.deepEqual(notices, ["interrupted"]);
|
|
3899
|
+
assert.deepEqual(checkpoints.map(({ turnOutcome }) => turnOutcome), ["interrupted"]);
|
|
3900
|
+
});
|
|
3901
|
+
|
|
3902
|
+
test("a Main turn that ends once shutdown has begun is left to the next server, with no checkpoint now", async () => {
|
|
3903
|
+
const f = await settlementFixture();
|
|
3904
|
+
f.owner.runtimeLifecycle = { closing: true };
|
|
3905
|
+
const before = structuredClone(f.run);
|
|
3906
|
+
assert.deepEqual(await f.owner.markTurnIdle(f.sessionId, { status: "completed", threadId: f.threadId, turnId: f.turnId }),
|
|
3907
|
+
{ ok: true, processed: false, reason: "left_to_next_server" });
|
|
3908
|
+
assert.deepEqual(f.run, before, "the saved run is unchanged");
|
|
3909
|
+
assert.deepEqual(f.checkpoints, []);
|
|
3910
|
+
// Outside shutdown the same end is recorded and checkpointed at once.
|
|
3911
|
+
f.owner.runtimeLifecycle = { closing: false };
|
|
3912
|
+
await f.owner.markTurnIdle(f.sessionId, { status: "completed", threadId: f.threadId, turnId: f.turnId });
|
|
3913
|
+
assert.equal(f.run.state, "completed");
|
|
3914
|
+
assert.deepEqual(f.checkpoints.map(({ status }) => status), ["completed"]);
|
|
3915
|
+
});
|
|
@@ -706,6 +706,35 @@ test("native invalidation preserves synchronous fences and failures without open
|
|
|
706
706
|
await result;
|
|
707
707
|
});
|
|
708
708
|
|
|
709
|
+
test("beginning native shutdown reaches the configured owner synchronously and opens no conversation", async t => {
|
|
710
|
+
const input = { reason: "server-shutdown" };
|
|
711
|
+
const calls = [];
|
|
712
|
+
let asynchronousHost = true;
|
|
713
|
+
const native = { runOwner: {
|
|
714
|
+
beginShutdown() { calls.push("begin"); return "begun"; },
|
|
715
|
+
invalidateRuntimes() { assert.fail("Beginning shutdown stops nothing."); }
|
|
716
|
+
} };
|
|
717
|
+
const runtime = createConversationRuntime({
|
|
718
|
+
authorize() { assert.fail("Trusted shutdown must not acquire session authority."); },
|
|
719
|
+
storage: {
|
|
720
|
+
read() { assert.fail("Shutdown must not read a conversation."); },
|
|
721
|
+
write() { assert.fail("Shutdown must not allocate a conversation."); }
|
|
722
|
+
},
|
|
723
|
+
host: { conversation(request) {
|
|
724
|
+
calls.push("host");
|
|
725
|
+
assert.deepEqual(request, { context, input, operation: "beginShutdown" });
|
|
726
|
+
const prepared = { engine: "codex", native, context, input };
|
|
727
|
+
return asynchronousHost ? Promise.resolve(prepared) : prepared;
|
|
728
|
+
} }
|
|
729
|
+
});
|
|
730
|
+
t.after(() => runtime.close());
|
|
731
|
+
assert.throws(() => runtime.beginNativeShutdown({ context, input }), /synchronous configured host/);
|
|
732
|
+
assert.deepEqual(calls, ["host"]);
|
|
733
|
+
asynchronousHost = false;
|
|
734
|
+
assert.equal(runtime.beginNativeShutdown({ context, input }), "begun");
|
|
735
|
+
assert.deepEqual(calls, ["host", "host", "begin"], "the owner's fence is set before the call returns");
|
|
736
|
+
});
|
|
737
|
+
|
|
709
738
|
test("renewal proof cleanup preserves OpenCode failure, consumption and retry through the configured owner", async t => {
|
|
710
739
|
const owner = createOpenCodeSharedRuntime({});
|
|
711
740
|
const calls = [];
|
|
@@ -13,8 +13,8 @@ const ABORTED = "The operation was aborted.";
|
|
|
13
13
|
const PROVIDER_FAILURE = "The provider refused this request.";
|
|
14
14
|
|
|
15
15
|
// OpenCode's native history outlives each server that runs it. The turn's reply
|
|
16
|
-
// is still being written until an interrupt ends it as aborted,
|
|
17
|
-
// fails it.
|
|
16
|
+
// is still being written until an interrupt ends it as aborted, its provider
|
|
17
|
+
// fails it, or it completes on its own.
|
|
18
18
|
function nativeOpenCode() {
|
|
19
19
|
const rows = [
|
|
20
20
|
{ id: INPUT_ID, type: "user", time: { created: 1 } },
|
|
@@ -46,7 +46,11 @@ function nativeOpenCode() {
|
|
|
46
46
|
rows[1] = { ...rows[1], error: { name: "APIError", data: { message: PROVIDER_FAILURE } }, time: { created: 2, completed: 3 } };
|
|
47
47
|
busy = false;
|
|
48
48
|
}
|
|
49
|
-
|
|
49
|
+
function complete() {
|
|
50
|
+
rows[1] = { ...rows[1], text: "Done.", finish: "stop", time: { created: 2, completed: 3 } };
|
|
51
|
+
busy = false;
|
|
52
|
+
}
|
|
53
|
+
return { calls, create, fail, complete };
|
|
50
54
|
}
|
|
51
55
|
|
|
52
56
|
// One server with Vibe64's run rules: the session record keeps the OpenCode run
|
|
@@ -108,6 +112,8 @@ function server(native, saved, { turnEndCheckpoint = async () => {} } = {}) {
|
|
|
108
112
|
return {
|
|
109
113
|
runtime, calls, release,
|
|
110
114
|
shutdown: () => runtime.invalidateRuntimes({ reason: "server-shutdown" }, { release }),
|
|
115
|
+
// The server's shutdown begins: nothing stops yet.
|
|
116
|
+
begin: () => runtime.beginShutdown(),
|
|
111
117
|
async running() {
|
|
112
118
|
const target = await runtime.acquirePrepared(processPreparation);
|
|
113
119
|
void runtime.beginMessageMonitor(KEY, target, { id: INPUT_ID, eventStartedAt: 1, startedAt: "2026-11-05T00:00:00.000Z" }, {}, monitor);
|
|
@@ -284,3 +290,49 @@ test("OpenCode shutdown ends a temporary conversation's turn as interrupted, and
|
|
|
284
290
|
await closed.entry.completion.catch(() => null);
|
|
285
291
|
assert.deepEqual(closed.closes, [{ stopped: true, outcome: "failed", stoppedByShutdown: false }]);
|
|
286
292
|
});
|
|
293
|
+
|
|
294
|
+
test("once OpenCode's shutdown has begun, a main turn that ends on its own is left to the next server, which records it with one checkpoint", { timeout: 5_000 }, async t => {
|
|
295
|
+
const native = nativeOpenCode();
|
|
296
|
+
const saved = { run: null };
|
|
297
|
+
const first = server(native, saved);
|
|
298
|
+
t.after(() => first.shutdown());
|
|
299
|
+
await first.running();
|
|
300
|
+
first.begin();
|
|
301
|
+
assert.deepEqual({ interrupts: native.calls.interrupts, stops: native.calls.stops }, { interrupts: [], stops: 0 },
|
|
302
|
+
"beginning the shutdown stopped nothing");
|
|
303
|
+
native.complete();
|
|
304
|
+
await first.runtime.monitors.get(KEY);
|
|
305
|
+
assert.deepEqual(first.calls.checkpoints, [], "the turn's end made no checkpoint");
|
|
306
|
+
assert.deepEqual(runOf(saved), { active: true, state: "active", error: "", stoppedByShutdown: true, turnId: INPUT_ID });
|
|
307
|
+
assert.equal((await first.shutdown()).ok, true);
|
|
308
|
+
assert.equal(native.calls.stops, 1, "the full stop still proved the OpenCode process exit");
|
|
309
|
+
assert.deepEqual(first.calls.checkpoints, []);
|
|
310
|
+
|
|
311
|
+
const next = server(native, saved);
|
|
312
|
+
t.after(() => next.shutdown());
|
|
313
|
+
assert.equal((await next.ready()).ok, true);
|
|
314
|
+
assert.equal((await next.ready()).ok, true);
|
|
315
|
+
assert.deepEqual(next.calls.checkpoints, [`${INPUT_ID}:interrupted`], "the next server made the checkpoint once");
|
|
316
|
+
assert.deepEqual(runOf(saved), { active: false, state: "interrupted", error: "", stoppedByShutdown: true, turnId: INPUT_ID });
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
test("once OpenCode's shutdown has begun, a temporary conversation's turn that ends on its own leaves its checkpoint to the next server", { timeout: 5_000 }, async t => {
|
|
320
|
+
const native = nativeOpenCode();
|
|
321
|
+
const first = server(native, { run: null });
|
|
322
|
+
t.after(() => first.shutdown());
|
|
323
|
+
const { entry, closes } = await first.temporary("ses_temp");
|
|
324
|
+
first.begin();
|
|
325
|
+
native.complete();
|
|
326
|
+
assert.deepEqual(await entry.completion, { interrupted: true });
|
|
327
|
+
assert.deepEqual(native.calls.interrupts, [], "nothing interrupted the native turn");
|
|
328
|
+
assert.deepEqual(closes, [{ stopped: true, outcome: "interrupted", stoppedByShutdown: true }],
|
|
329
|
+
"the application is told to leave the turn's checkpoint to the next server");
|
|
330
|
+
assert.equal((await first.shutdown()).ok, true);
|
|
331
|
+
|
|
332
|
+
const next = server(native, { run: null });
|
|
333
|
+
t.after(() => next.shutdown());
|
|
334
|
+
const ended = [];
|
|
335
|
+
const read = await next.readTemporary("ses_temp", { runId: INPUT_ID, async ended() { ended.push(INPUT_ID); } });
|
|
336
|
+
assert.deepEqual({ status: read.status, error: read.error }, { status: "interrupted", error: "" });
|
|
337
|
+
assert.deepEqual(ended, [INPUT_ID], "recording the run lets the application make its checkpoint");
|
|
338
|
+
});
|