@jskit-ai/assistant-core 0.2.16 → 0.2.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/examples/conversation/package.json +5 -5
  2. package/package.json +5 -5
  3. package/src/server/conversation/claudeNativeTurn.js +9 -1
  4. package/src/server/conversation/claudeSessionCleanup.js +9 -1
  5. package/src/server/conversation/claudeTurn.js +1 -0
  6. package/src/server/conversation/codexConversations.js +65 -6
  7. package/src/server/conversation/codexRunJournal.js +33 -1
  8. package/src/server/conversation/codexRuntimeLifecycle.js +20 -1
  9. package/src/server/conversation/codexTurn.js +41 -4
  10. package/src/server/conversation/codexTurnEndings.js +9 -6
  11. package/src/server/conversation/nativeConversationLifecycle.js +22 -0
  12. package/src/server/conversation/openCodeProcessOwner.js +11 -5
  13. package/src/server/conversation/openCodeRuntime.js +1 -0
  14. package/src/server/conversation/openCodeTurn.js +10 -5
  15. package/src/server/conversation/openCodeTurnMonitor.js +4 -2
  16. package/src/server/conversation/providers/claudeDriver.js +5 -0
  17. package/src/server/conversation/providers/codexDriver.js +5 -0
  18. package/src/server/conversation/providers/opencodeDriver.js +5 -0
  19. package/src/server/conversation/runtime.js +1 -0
  20. package/test/claudeLifecycle.test.js +28 -0
  21. package/test/codexConversationsCheckpoint.test.js +2 -0
  22. package/test/codexLifecycleIsolation.test.js +2 -0
  23. package/test/codexShutdownStop.test.js +168 -0
  24. package/test/codexTemporaryProviderRelease.test.js +2 -0
  25. package/test/codexTurn.test.js +57 -0
  26. package/test/conversationSession.test.js +29 -0
  27. package/test/openCodeLifecycle.test.js +55 -3
@@ -8,14 +8,14 @@
8
8
  "build": "vite build"
9
9
  },
10
10
  "dependencies": {
11
- "@jskit-ai/assistant-core": "0.2.16",
12
- "@jskit-ai/http-runtime": "0.2.6",
13
- "@jskit-ai/kernel": "0.2.6",
11
+ "@jskit-ai/assistant-core": "0.2.17",
12
+ "@jskit-ai/http-runtime": "0.2.7",
13
+ "@jskit-ai/kernel": "0.2.7",
14
14
  "@mdi/js": "^7.4.47",
15
15
  "vue": "^3.5.13",
16
16
  "vuetify": "^4.0.0",
17
- "@jskit-ai/connectors-core": "0.2.7",
18
- "@jskit-ai/connectors-catalog": "0.2.7"
17
+ "@jskit-ai/connectors-core": "0.2.8",
18
+ "@jskit-ai/connectors-catalog": "0.2.8"
19
19
  },
20
20
  "devDependencies": {
21
21
  "@vitejs/plugin-vue": "^6.0.7",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jskit-ai/assistant-core",
3
- "version": "0.2.16",
3
+ "version": "0.2.17",
4
4
  "type": "module",
5
5
  "scripts": {
6
6
  "test": "node --test --test-concurrency=1"
@@ -51,8 +51,8 @@
51
51
  "@ai-sdk/togetherai": "^3.0.49",
52
52
  "@ai-sdk/xai": "^4.0.58",
53
53
  "@jskit-ai/agent": "0.1.0",
54
- "@jskit-ai/resource-core": "0.1.197",
55
- "@jskit-ai/resource-crud-core": "0.1.197",
54
+ "@jskit-ai/resource-core": "0.1.198",
55
+ "@jskit-ai/resource-crud-core": "0.1.198",
56
56
  "@mdi/js": "^7.4.47",
57
57
  "@openrouter/ai-sdk-provider": "^3.0.0",
58
58
  "ai": "^7.0.152",
@@ -61,8 +61,8 @@
61
61
  "ws": "^8.20.1"
62
62
  },
63
63
  "peerDependencies": {
64
- "@jskit-ai/http-runtime": "0.2.6",
65
- "@jskit-ai/kernel": "0.2.6",
64
+ "@jskit-ai/http-runtime": "0.2.7",
65
+ "@jskit-ai/kernel": "0.2.7",
66
66
  "vue": "^3.5.13",
67
67
  "vuetify": "^4.0.0"
68
68
  },
@@ -355,7 +355,15 @@ export function createClaudeConversationTurn({ conversationId, onEvent, process,
355
355
  } else if (event.type === "settled") {
356
356
  const state = event.status === "interrupted" ? "interrupted" : event.status === "failed" ? "failed" : "completed";
357
357
  if (bound) setClaudeTurnState(entry, state, event.error);
358
- else if (!await updateState(state, event.error, event.generation)) return;
358
+ else if (entry.turn?.active && process.ownerClosing()) {
359
+ // A turn that ends on its own once shutdown has begun is left to the
360
+ // next server like one the shutdown stops: its saved record keeps the
361
+ // execution and the active turn, so that server's recovery interrupts
362
+ // the turn and runs the host's checkpoint. Run here, that checkpoint
363
+ // (a Git snapshot, say) could be refused or hold up shutdown.
364
+ resolveCompletion({ ...readResult(), status: "interrupted", error: "" });
365
+ return;
366
+ } else if (!await updateState(state, event.error, event.generation)) return;
359
367
  }
360
368
  const published = await onEvent(event);
361
369
  const change = event.type === "provider-event" ? claudeProviderChange(event.event) : "";
@@ -121,7 +121,7 @@ export function createClaudeSessionCleanup({
121
121
  account.queries.invalidate().catch(() => {});
122
122
  return { ok: true, closed: 0 };
123
123
  }
124
- closing = true;
124
+ beginShutdown();
125
125
  // Neither waits for the other: a failed account query cleanup must not
126
126
  // leave the sessions' processes running.
127
127
  const [queries, retained] = await Promise.allSettled([account.queries.invalidate(), stopRetained(application)]);
@@ -130,6 +130,13 @@ export function createClaudeSessionCleanup({
130
130
  return retained.value;
131
131
  }
132
132
 
133
+ // The server's shutdown begins: the owner closes, so each running turn's
134
+ // end, and its checkpoint, is left to the next server, but nothing stops.
135
+ // The later server-shutdown invalidation stops the processes.
136
+ function beginShutdown() {
137
+ closing = true;
138
+ }
139
+
133
140
  // Shutdown stops only what this server runs: a Claude process cannot outlive
134
141
  // the server that started it, so retained entries hold every one. Saved
135
142
  // conversations are not reopened just to be stopped.
@@ -233,6 +240,7 @@ export function createClaudeSessionCleanup({
233
240
  closeSession,
234
241
  releaseSession,
235
242
  closeProject,
243
+ beginShutdown,
236
244
  invalidateRuntimes,
237
245
  drain,
238
246
  forget,
@@ -701,6 +701,7 @@ export function createClaudeConversationOwner({
701
701
  closeSession: cleanup.closeSession,
702
702
  releaseSession: cleanup.releaseSession,
703
703
  closeProject: cleanup.closeProject,
704
+ beginShutdown: cleanup.beginShutdown,
704
705
  invalidateRuntimes: cleanup.invalidateRuntimes,
705
706
  drain: cleanup.drain,
706
707
  forget: cleanup.forget,
@@ -57,6 +57,11 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
57
57
  * @param {import("./codexProviderSelection.js").CodexProviderSelection["borrowSessionProvider"]} dependencies.borrowSessionProvider
58
58
  * @param {string} dependencies.steerFailedCode
59
59
  * @param {import("./codexTurn.js").CodexTurnOutcomes} dependencies.turnOutcomes
60
+ * @param {import("./codexTurn.js").CodexDeferCheckpoint} dependencies.deferCheckpoint Absent when the host predates
61
+ * it, which then makes every checkpoint at once.
62
+ * @param {() => boolean} dependencies.isClosing Whether the server's shutdown has begun.
63
+ * @param {import("./codexTurn.js").CodexCaptureContext} dependencies.captureContext
64
+ * @param {import("./codexTurn.js").CodexRunInContext} dependencies.runInContext
60
65
  */
61
66
  export function createCodexConversationCommands({
62
67
  checkpoint,
@@ -73,10 +78,17 @@ export function createCodexConversationCommands({
73
78
  ensureSessionProvider,
74
79
  borrowSessionProvider,
75
80
  steerFailedCode,
76
- turnOutcomes
81
+ turnOutcomes,
82
+ deferCheckpoint,
83
+ isClosing,
84
+ captureContext,
85
+ runInContext
77
86
  }) {
78
87
  const codexAppServerConversations = new Map();
79
88
  const codexAppServerConversationTurnStarts = new Map();
89
+ // The application's records of checkpoints left to the next server, which
90
+ // shutdown awaits before the process exits.
91
+ const codexAppServerDeferredCheckpoints = new Set();
80
92
  // Turns, detached runs and stops live in their own modules. Each receives the
81
93
  // registry above and the registry and context functions declared below.
82
94
  const {
@@ -157,6 +169,42 @@ export function createCodexConversationCommands({
157
169
  holdWhileActive(state);
158
170
  }
159
171
 
172
+ // Once the server's shutdown has begun a turn's checkpoint belongs to the
173
+ // next server: the application records it as owed, and that server's next
174
+ // read of the conversation makes it. Made now, a checkpoint (a Git snapshot,
175
+ // say) could be refused or hold up the host's bounded shutdown.
176
+ function leaveCheckpointToNextServer(state, ending) {
177
+ const deferred = Promise.resolve()
178
+ .then(() => runInContext(state.context, () => deferCheckpoint(state.sessionId, ending)))
179
+ .catch(() => null);
180
+ codexAppServerDeferredCheckpoints.add(deferred);
181
+ void deferred.finally(() => codexAppServerDeferredCheckpoints.delete(deferred));
182
+ return deferred;
183
+ }
184
+
185
+ /**
186
+ * At shutdown, each observed conversation's running turn: the stopped process
187
+ * ends it without a completion this server observes, so its interrupted
188
+ * checkpoint is left to the next server.
189
+ */
190
+ function leaveRunningTurnsToNextServer() {
191
+ if (!deferCheckpoint) return Promise.resolve([]);
192
+ const deferred = [];
193
+ for (const conversations of codexAppServerConversations.values()) {
194
+ for (const state of conversations.values()) {
195
+ if (!state.persistent || !state.unsubscribe || !state.runId || !codexAppServerConversationTurnIsActive(state.status)) continue;
196
+ deferred.push(leaveCheckpointToNextServer(state, {
197
+ outerTurnId: codexAppServerNativeOuterTurnId(state.conversationId, state.runId),
198
+ threadId: state.conversationId,
199
+ turnId: state.runId,
200
+ status: "interrupted",
201
+ turnOutcome: turnOutcomes.SERVICE_RESTART
202
+ }));
203
+ }
204
+ }
205
+ return Promise.all(deferred);
206
+ }
207
+
160
208
  function codexAppServerConversation(sessionId = "", conversationId = "") {
161
209
  return codexAppServerConversations.get(
162
210
  namespace(sessionId)
@@ -201,8 +249,8 @@ export function createCodexConversationCommands({
201
249
  let state = conversations.get(conversationId);
202
250
  if (state?.provider === provider && state.unsubscribe) return state;
203
251
  if (state) forgetObservation(state);
204
- state = { conversationId, provider, persistent: true, status: "ready", runId: "", error: "", turnOutcome: "",
205
- onEvent: ignoreCodexConversationEvent };
252
+ state = { sessionId, conversationId, provider, persistent: true, status: "ready", runId: "", error: "", turnOutcome: "",
253
+ context: captureContext(), onEvent: ignoreCodexConversationEvent };
206
254
  state.unsubscribe = provider.subscribe((notification) => {
207
255
  if (codexAppServerNotificationThreadId(notification) !== conversationId) return;
208
256
  const turnId = codexAppServerNotificationTurnId(notification);
@@ -228,13 +276,15 @@ export function createCodexConversationCommands({
228
276
  // A turn the person stopped reaches the checkpoint as cancelled, as Main turns do.
229
277
  const completedOutcome = state.turnOutcome;
230
278
  state.turnOutcome = "";
231
- void checkpoint(sessionId, {
279
+ const ending = {
232
280
  outerTurnId: codexAppServerNativeOuterTurnId(conversationId, completedTurnId),
233
281
  threadId: conversationId,
234
282
  turnId: completedTurnId,
235
283
  status: completedStatus,
236
284
  turnOutcome: completedOutcome
237
- }).catch(() => {});
285
+ };
286
+ if (isClosing() && deferCheckpoint) leaveCheckpointToNextServer(state, ending);
287
+ else void checkpoint(sessionId, ending).catch(() => {});
238
288
  }
239
289
  holdWhileActive(state);
240
290
  });
@@ -284,6 +334,13 @@ export function createCodexConversationCommands({
284
334
  if (goal?.status === "active") status = "inProgress";
285
335
  Object.assign(state, { runId, status, goal });
286
336
  holdWhileActive(state);
337
+ // The application marks the run whose checkpoint a previous server's
338
+ // shutdown left owed (`stoppedByShutdown: { runId, ended }`). Recording
339
+ // that it ended lets the application make that checkpoint; a run the
340
+ // shutdown stopped reads interrupted with no error.
341
+ const shutdownStop = input.stoppedByShutdown;
342
+ const endedAtShutdown = Boolean(runId) && shutdownStop?.runId === runId && !codexAppServerConversationTurnIsActive(status);
343
+ if (endedAtShutdown) await shutdownStop.ended();
287
344
  const messages = [];
288
345
  for (const turn of turns) {
289
346
  const turnId = codexAppServerProviderTurnId(turn);
@@ -321,7 +378,7 @@ export function createCodexConversationCommands({
321
378
  goal,
322
379
  admitted: Boolean(input.messageId && turns.some((turn) => codexAppServerProviderTurnClientIds(turn).includes(input.messageId))),
323
380
  rawText: latestText.trim(),
324
- error: state.error || codexAppServerThreadError(thread)
381
+ error: endedAtShutdown && status === "interrupted" ? "" : state.error || codexAppServerThreadError(thread)
325
382
  };
326
383
  }
327
384
 
@@ -555,6 +612,8 @@ export function createCodexConversationCommands({
555
612
  return Object.freeze({
556
613
  codexAppServerConversations,
557
614
  codexAppServerConversationTurnStarts,
615
+ deferredCheckpoints: codexAppServerDeferredCheckpoints,
616
+ leaveRunningTurnsToNextServer,
558
617
  codexAppServerConversation,
559
618
  hasActiveConversation,
560
619
  observeCodexConversation,
@@ -3,6 +3,32 @@ import crypto from "node:crypto";
3
3
  import { isRecord, normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
4
4
 
5
5
  const CODEX_APP_SERVER_PROMPT_CLAIM_GRACE_MS = 15000;
6
+ const CODEX_APP_SERVER_STOPPED_BY_SHUTDOWN_EVENT = "codex-app-server-stopped-by-shutdown";
7
+
8
+ /**
9
+ * Marks the session's running turn as stopped by the server's shutdown, so the
10
+ * next server records its interruption as a restart. Nothing else of the saved
11
+ * run changes.
12
+ * @param {{ createRuntime: import("./codexTurn.js").CodexCreateRuntime, turnState: import("./codexTurn.js").CodexTurnStateReader }} dependencies
13
+ * @param {string} sessionId
14
+ * @param {string} [threadId] the session's Main thread, when known
15
+ */
16
+ export async function markCodexAppServerTurnStoppedByShutdown({ createRuntime, turnState }, sessionId = "", threadId = "") {
17
+ const normalizedSessionId = normalizeText(sessionId);
18
+ if (!normalizedSessionId) return { marked: false };
19
+ const runtime = await createRuntime();
20
+ let marked = false;
21
+ await runtime.store.mutateSession(normalizedSessionId, async () => {
22
+ const turn = turnState(await runtime.getSession(normalizedSessionId).catch(() => null) || {});
23
+ if (!turn.active || !turn.turnId || (normalizeText(threadId) && turn.threadId !== normalizeText(threadId))) return;
24
+ await runtime.store.writeAgentRunEvent(normalizedSessionId, CODEX_APP_SERVER_AGENT_RUN_ID, {
25
+ event: { kind: CODEX_APP_SERVER_STOPPED_BY_SHUTDOWN_EVENT },
26
+ patch: { stoppedByShutdownTurnId: turn.turnId }
27
+ });
28
+ marked = true;
29
+ });
30
+ return { marked };
31
+ }
6
32
 
7
33
  function terminalCodexAppServerAgentRunState(status = "") {
8
34
  const normalizedStatus = normalizeText(status);
@@ -40,6 +66,7 @@ function codexAppServerDateValueMs(value = "") {
40
66
  * @param {number} dependencies.finalizingGraceMs
41
67
  * @param {(session: object, run: object) => boolean} dependencies.admissionTaskFinished
42
68
  * @param {string} dependencies.orphanedPromptMessage
69
+ * @param {() => boolean} dependencies.isClosing Whether the server's shutdown has begun.
43
70
  */
44
71
  export function createCodexRunJournal({
45
72
  namespace,
@@ -57,7 +84,8 @@ export function createCodexRunJournal({
57
84
  idlePublishPayload,
58
85
  finalizingGraceMs,
59
86
  admissionTaskFinished,
60
- orphanedPromptMessage
87
+ orphanedPromptMessage,
88
+ isClosing
61
89
  }) {
62
90
  const codexAppServerCompletedTurns = new Set();
63
91
  const codexAppServerProcessedTurns = new Set();
@@ -394,6 +422,10 @@ export function createCodexRunJournal({
394
422
  }
395
423
 
396
424
  async function markCodexAppServerTurnIdle(sessionId = "", input = {}) {
425
+ // Once shutdown has begun the turn's end, and its checkpoint, are left to the next server's recovery, as after a crash.
426
+ if (isClosing()) {
427
+ return { ok: true, processed: false, reason: "left_to_next_server" };
428
+ }
397
429
  const status = normalizeText(input.status) || "completed";
398
430
  const result = await publishCodexAppServerAgentRun(sessionId, {
399
431
  error: normalizeText(input.error),
@@ -6,6 +6,9 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
6
6
  /** @typedef {ReturnType<typeof import("./codexTurn.js").createCodexAppServerRunOwner>} CodexRunOwner */
7
7
  /** @typedef {(event: string, details: object) => void} CodexDebugLog */
8
8
 
9
+ /** A lifecycle without running turns to hand over leaves none. */
10
+ async function leaveNoTurns() {}
11
+
9
12
  /**
10
13
  * Shutdown, wellbeing timers and managed-session records for shared Codex
11
14
  * runtimes. It works on the run/provider owners' collections; it owns no servers.
@@ -23,6 +26,8 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
23
26
  * @param {CodexDebugLog} dependencies.debugLog
24
27
  * @param {(error: unknown) => unknown} dependencies.debugError
25
28
  * @param {{ message: string, code: string, retryable: boolean } | null} [dependencies.closingError]
29
+ * @param {() => Promise<unknown>} [dependencies.leaveTurnsToNextServer] When shutdown begins, leaves each running
30
+ * turn's end, and its checkpoint, to the next server.
26
31
  */
27
32
  export function createCodexRuntimeLifecycle({
28
33
  pendingWork,
@@ -35,7 +40,8 @@ export function createCodexRuntimeLifecycle({
35
40
  continueGoalAfterAccountChange,
36
41
  debugLog,
37
42
  debugError,
38
- closingError = null
43
+ closingError = null,
44
+ leaveTurnsToNextServer = leaveNoTurns
39
45
  }) {
40
46
  const codexAppServerSessionClosures = new Map();
41
47
  const codexAppServerWellbeingTimers = new Map();
@@ -44,6 +50,7 @@ export function createCodexRuntimeLifecycle({
44
50
  let codexAppServerThreadReconcileGeneration = 0;
45
51
  let codexAppServerServerClosing = false;
46
52
  let codexAppServerShutdownPromise = null;
53
+ let codexAppServerTurnsLeft = null;
47
54
 
48
55
  function assertOpen() {
49
56
  if (!closingError) return providerSessions.assertOpen();
@@ -216,11 +223,20 @@ export function createCodexRuntimeLifecycle({
216
223
  }
217
224
  }
218
225
 
226
+ // The server's shutdown begins: the closing fence is set and each running
227
+ // turn's end, and its checkpoint, is left to the next server, but nothing
228
+ // stops. The later server-shutdown invalidation stops the runtimes.
219
229
  function beginCodexAppServerShutdown() {
220
230
  if (codexAppServerServerClosing) {
221
231
  return;
222
232
  }
223
233
  codexAppServerServerClosing = true;
234
+ // Taken now, before any runtime stops, while each running turn is known.
235
+ codexAppServerTurnsLeft = (async () => leaveTurnsToNextServer())().catch(error => {
236
+ debugLog("appServerShutdown.leaveTurns.error", { error: debugError(error) });
237
+ }).finally(() => {
238
+ codexAppServerTurnsLeft = null;
239
+ });
224
240
  providerSessions.owner.beginShutdown();
225
241
  codexAppServerThreadReconcileGeneration += 1;
226
242
  for (const providerKey of [...codexAppServerWellbeingTimers.keys()]) {
@@ -231,6 +247,8 @@ export function createCodexRuntimeLifecycle({
231
247
  async function drainCodexAppServerControllerTasks() {
232
248
  while (true) {
233
249
  const pending = new Set([
250
+ ...(codexAppServerTurnsLeft ? [codexAppServerTurnsLeft] : []),
251
+ ...pendingWork.deferredCheckpoints.values(),
234
252
  ...pendingWork.conversationTurnStarts.values(),
235
253
  ...helperLifecycle.projectOperations.values(),
236
254
  ...helperLifecycle.cleanups.values(),
@@ -356,6 +374,7 @@ export function createCodexRuntimeLifecycle({
356
374
 
357
375
  return Object.freeze({
358
376
  get closing() { return codexAppServerServerClosing; },
377
+ beginShutdown: beginCodexAppServerShutdown,
359
378
  assertOpen,
360
379
  get reconcileGeneration() { return codexAppServerThreadReconcileGeneration; },
361
380
  nextReconcileGeneration,
@@ -10,7 +10,7 @@ import { createCodexNotificationObserver } from "./codexNotificationObserver.js"
10
10
  import { createCodexThreadStatus } from "./codexThreadStatus.js";
11
11
  import { createCodexTurnRecovery } from "./codexTurnRecovery.js";
12
12
  import { createCodexResultSettlement } from "./codexResultSettlement.js";
13
- import { createCodexRunJournal } from "./codexRunJournal.js";
13
+ import { createCodexRunJournal, markCodexAppServerTurnStoppedByShutdown } from "./codexRunJournal.js";
14
14
  import { createCodexRenewalCommands } from "./codexRenewalCommands.js";
15
15
  import { createCodexGoalCommands } from "./codexGoalCommands.js";
16
16
  import { createCodexProviderSelection } from "./codexProviderSelection.js";
@@ -80,6 +80,13 @@ export { codexAppServerThreadIsMissing } from "./codexProviderThreads.js";
80
80
  /** @typedef {typeof CODEX_TURN_OUTCOMES} CodexTurnOutcomes */
81
81
  /** @typedef {(runtime: any, session: object, options: object) => Promise<any>} CodexAdmissionRecovery */
82
82
  /** @typedef {(sessionId: string, input: object) => Promise<unknown>} CodexCheckpoint */
83
+ /**
84
+ * Records a scoped conversation turn's checkpoint as owed to the next server,
85
+ * once the server's shutdown has begun. The application passes the run back
86
+ * on that server's next read as `stoppedByShutdown: { runId, ended }`, and
87
+ * makes the checkpoint when `ended` is called.
88
+ * @typedef {(sessionId: string, input: { outerTurnId: string, threadId: string, turnId: string, status: string, turnOutcome: string }) => Promise<unknown>} CodexDeferCheckpoint
89
+ */
83
90
 
84
91
  /**
85
92
  * The application hooks that add metadata to the turns a run writes.
@@ -161,6 +168,8 @@ function runInNoContext(_context, operation) {
161
168
  * @param {string} [options.messageIdPrefix]
162
169
  * @param {object} [options.idlePublishPayload]
163
170
  * @param {CodexCheckpoint} [options.checkpoint]
171
+ * @param {CodexDeferCheckpoint} options.deferCheckpoint A host that predates it leaves it out: a scoped
172
+ * conversation turn that ends during shutdown then makes its checkpoint at once, and one the shutdown stops makes none.
164
173
  * @param {Partial<CodexMessageMetadataHooks>} [options.messageMetadata]
165
174
  * @param {boolean} [options.persistCommentary]
166
175
  * @param {string} [options.deliveryStateMetadataKey]
@@ -204,6 +213,7 @@ export function createCodexAppServerRunOwner({
204
213
  messageIdPrefix = "codex:",
205
214
  idlePublishPayload,
206
215
  checkpoint = answerNothing,
216
+ deferCheckpoint,
207
217
  messageMetadata = {},
208
218
  persistCommentary = true,
209
219
  deliveryStateMetadataKey = "assistant_delivery",
@@ -317,7 +327,8 @@ export function createCodexAppServerRunOwner({
317
327
  idlePublishPayload,
318
328
  finalizingGraceMs,
319
329
  admissionTaskFinished,
320
- orphanedPromptMessage
330
+ orphanedPromptMessage,
331
+ isClosing: () => runOwner.runtimeLifecycle?.closing === true
321
332
  });
322
333
  const admissionRecovery = recoverAdmission || codexRunJournal.recoverAbandonedPromptClaim;
323
334
 
@@ -424,7 +435,11 @@ export function createCodexAppServerRunOwner({
424
435
  ensureSessionProvider: codexProviderSelection.ensureSessionProvider,
425
436
  borrowSessionProvider: codexProviderSelection.borrowSessionProvider,
426
437
  steerFailedCode,
427
- turnOutcomes
438
+ turnOutcomes,
439
+ deferCheckpoint,
440
+ isClosing: () => runOwner.runtimeLifecycle?.closing === true,
441
+ captureContext,
442
+ runInContext
428
443
  });
429
444
 
430
445
  const codexObservationControl = createCodexObservationControl({
@@ -636,6 +651,7 @@ export function createCodexAppServerRunOwner({
636
651
  clearSessionRecoveryTimers: codexRunJournal.clearSessionRecoveryTimers,
637
652
  publishAgentRun: codexRunJournal.publishAgentRun,
638
653
  markTurnIdle: codexRunJournal.markTurnIdle,
654
+ markTurnStoppedByShutdown,
639
655
  finalizeAssistantResult: codexResultSettlement.finalizeAssistantResult,
640
656
  finalizedTurns: codexAppServerOutput.finalizedTurns,
641
657
  liveProgressItems: codexAppServerOutput.liveProgressItems,
@@ -661,8 +677,10 @@ export function createCodexAppServerRunOwner({
661
677
  reasoningPersistQueues: codexAppServerOutput.reasoningPersistQueues,
662
678
  resultFinalizations: codexResultSettlement.resultFinalizations,
663
679
  threadReconciliations: codexThreadReadiness.threadReconciliations,
664
- notificationTasks: codexAppServerNotificationQueue.tasks
680
+ notificationTasks: codexAppServerNotificationQueue.tasks,
681
+ deferredCheckpoints: codexConversations.deferredCheckpoints
665
682
  },
683
+ leaveTurnsToNextServer: leaveRunningTurnsToNextServer,
666
684
  observer: codexNotificationObserver,
667
685
  checkManagedConnection: codexThreadReadiness.checkManagedConnection,
668
686
  providerSessions,
@@ -678,6 +696,7 @@ export function createCodexAppServerRunOwner({
678
696
  providerSessions.owner.attachRuntimeLifecycle(runOwner.runtimeLifecycle);
679
697
  providerSessions.assertOpen = runOwner.runtimeLifecycle.assertOpen;
680
698
  }
699
+ runOwner.beginShutdown = runOwner.runtimeLifecycle.beginShutdown;
681
700
  runOwner.invalidateRuntimes = runOwner.runtimeLifecycle.invalidateRuntimes;
682
701
  Object.assign(runOwner, createCodexSessionReconciliation({
683
702
  readiness: codexThreadReadiness, recoverSessionObservationLoss,
@@ -685,6 +704,24 @@ export function createCodexAppServerRunOwner({
685
704
  createRuntime, namespace, turnState, sessionIdRequiredError, debugLog, debugError
686
705
  }));
687
706
  }
707
+ function markTurnStoppedByShutdown(sessionId = "", threadId = "") {
708
+ return markCodexAppServerTurnStoppedByShutdown({ createRuntime, turnState }, sessionId, threadId);
709
+ }
710
+
711
+ // At shutdown each running turn's end, and its checkpoint, belong to the
712
+ // next server: Main turns are marked so it records their interruption as a
713
+ // restart, and scoped conversations leave their checkpoints owed.
714
+ async function leaveRunningTurnsToNextServer() {
715
+ const results = await Promise.allSettled([
716
+ codexConversations.leaveRunningTurnsToNextServer(),
717
+ ...[...providerSessions.managed.values()].map(managed => Promise.resolve().then(() => runInContext(
718
+ managed.projectContext, () => markTurnStoppedByShutdown(managed.sessionId, managed.threadId))))
719
+ ]);
720
+ for (const result of results) {
721
+ if (result.status === "rejected") debugLog("appServerShutdown.leaveTurn.error", { error: debugError(result.reason) });
722
+ }
723
+ }
724
+
688
725
  const cleanup = createCodexSessionCleanup({
689
726
  namespace,
690
727
  runOwner,
@@ -253,9 +253,9 @@ export function createCodexTurnEndings({
253
253
  const normalizedStatus = normalizeText(status);
254
254
  let base = "Codex app-server failed before completing this turn.";
255
255
  if (normalizedStatus === "interrupted") {
256
- base = outcome === turnOutcomes.CONTROL_RECONFIGURATION
257
- ? "Codex app-server was interrupted while its controls were restored."
258
- : "Codex app-server was interrupted before completing this turn.";
256
+ if (outcome === turnOutcomes.CONTROL_RECONFIGURATION) base = "Codex app-server was interrupted while its controls were restored.";
257
+ else if (outcome === turnOutcomes.SERVICE_RESTART) base = "Codex app-server was interrupted by a restart before completing this turn.";
258
+ else base = "Codex app-server was interrupted before completing this turn.";
259
259
  }
260
260
  const normalizedError = normalizeText(error);
261
261
  return normalizedError ? `${base} ${normalizedError}` : base;
@@ -277,17 +277,20 @@ export function createCodexTurnEndings({
277
277
  const normalizedThreadId = normalizeText(threadId);
278
278
  const normalizedTurnId = normalizeText(turnId);
279
279
  const normalizedStatus = normalizeText(status) || "failed";
280
+ const runtime = await createRuntime();
280
281
  let resolvedOutcome = normalizeText(outcome);
281
282
  if (!resolvedOutcome) {
282
283
  if (normalizedStatus === "interrupted") {
283
284
  const recorded = provider ? provider.interruptionOutcome(normalizedThreadId, normalizedTurnId) : "";
284
- resolvedOutcome = [turnOutcomes.USER_CANCELLED, turnOutcomes.CONTROL_RECONFIGURATION].includes(recorded)
285
- ? recorded : turnOutcomes.INTERRUPTED;
285
+ if ([turnOutcomes.USER_CANCELLED, turnOutcomes.CONTROL_RECONFIGURATION].includes(recorded)) resolvedOutcome = recorded;
286
+ // The previous server's shutdown stopped this turn: a restart, not a failure.
287
+ else if (codexAppServerAgentRun(await runtime.getSession(normalizedSessionId))?.stoppedByShutdownTurnId === normalizedTurnId) {
288
+ resolvedOutcome = turnOutcomes.SERVICE_RESTART;
289
+ } else resolvedOutcome = turnOutcomes.INTERRUPTED;
286
290
  } else {
287
291
  resolvedOutcome = turnOutcomes.PROVIDER_FAILURE;
288
292
  }
289
293
  }
290
- const runtime = await createRuntime();
291
294
  const staleFailure = async () => {
292
295
  const turn = turnState(await runtime.getSession(normalizedSessionId));
293
296
  if (codexAppServerTurnCanReceiveProviderCompletion(turn, normalizedThreadId, normalizedTurnId)) {
@@ -218,6 +218,27 @@ function createNativeConversationLifecycle({ authorize, defaultHost, hostPrepare
218
218
  return driver.invalidateRuntimes(prepared);
219
219
  }
220
220
 
221
+ // The server's shutdown begins before anything stops, on the same trusted
222
+ // boundary: the configured owner sets its closing fence and leaves each
223
+ // running turn's end, and its checkpoint, to the next server, synchronously.
224
+ // A later invalidation with reason "server-shutdown" stops its processes.
225
+ function beginNativeShutdown({ context, input = {} }) {
226
+ if (!hostPrepares) {
227
+ throw new TypeError("Native shutdown requires a configured conversation host.");
228
+ }
229
+ const prepared = defaultHost.conversation({ context, input, operation: "beginShutdown" });
230
+ if (!prepared || isThenable(prepared) ||
231
+ !["codex", "claude", "opencode"].includes(prepared.engine) || !prepared.native ||
232
+ !Object.hasOwn(prepared, "input") || !Object.hasOwn(prepared, "context")) {
233
+ throw new TypeError("Native shutdown requires synchronous configured host preparation.");
234
+ }
235
+ const driver = providers.createDriver(prepared.engine, defaultHost);
236
+ if (!implementsOperation(driver, "beginShutdown")) {
237
+ throw new TypeError("The selected conversation driver does not implement native shutdown.");
238
+ }
239
+ return driver.beginShutdown(prepared);
240
+ }
241
+
221
242
  // Startup reconciliation and subscription reset use the existing trusted
222
243
  // lifecycle caller. Native owners retain their own shutdown fences and tasks.
223
244
  function reconcileNativeSessions({ context, sessions, options = {} }) {
@@ -363,6 +384,7 @@ function createNativeConversationLifecycle({ authorize, defaultHost, hostPrepare
363
384
  createNativeConversation,
364
385
  runNativeDetachedConversation,
365
386
  closeNativeProject,
387
+ beginNativeShutdown,
366
388
  invalidateNativeRuntimes,
367
389
  reconcileNativeSessions,
368
390
  unsubscribeNativeSessions,
@@ -441,12 +441,17 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
441
441
  };
442
442
  }
443
443
 
444
+ // The server's shutdown begins: the owner closes, so each running turn's
445
+ // end, and its checkpoint, is left to the next server, but nothing stops.
446
+ // The later server-shutdown invalidation stops the processes.
447
+ function beginShutdown() {
448
+ closed = true;
449
+ clearTimeout(storageIdle);
450
+ storageIdle = null;
451
+ }
452
+
444
453
  async function invalidateRuntimes(input = {}, preparation) {
445
- if (normalizeNullableText(input.reason) === "server-shutdown") {
446
- closed = true;
447
- clearTimeout(storageIdle);
448
- storageIdle = null;
449
- }
454
+ if (normalizeNullableText(input.reason) === "server-shutdown") beginShutdown();
450
455
  const application = preparation.release;
451
456
  // readable-check: allow method-probe - input validation: a connection change carries the publication of the application's saved connections
452
457
  if (typeof input.publish === "function") {
@@ -567,6 +572,7 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
567
572
  releaseProcessTarget,
568
573
  closeSession,
569
574
  closeProject,
575
+ beginShutdown,
570
576
  invalidateRuntimes,
571
577
  releaseRenewalPredecessorProcessExitProof,
572
578
  releaseProcessExitProof
@@ -226,6 +226,7 @@ export function createOpenCodeSharedRuntime({ onStop = ignoreRuntimeStop, scope,
226
226
  sendPreparedMessage: sessionTurns.sendPreparedMessage,
227
227
  inspectPreparedMessage: sessionTurns.inspectPreparedMessage,
228
228
  interruptPreparedTurn: sessionTurns.interruptPreparedTurn,
229
+ beginShutdown: processOwner.beginShutdown,
229
230
  invalidateRuntimes: processOwner.invalidateRuntimes,
230
231
  releaseRenewalPredecessorProcessExitProof: processOwner.releaseRenewalPredecessorProcessExitProof,
231
232
  releaseProcessExitProof: processOwner.releaseProcessExitProof,
@@ -478,9 +478,10 @@ export async function waitForOpenCodeFinalResponse(client, conversationId, turn,
478
478
  /**
479
479
  * Production detached-turn lifetime. Applications project the completed native
480
480
  * result before observation is closed, then publish/checkpoint their own state.
481
- * A turn the server's shutdown stopped (`turn.stoppedByShutdown`) ends
482
- * interrupted however its wait ended, and `afterClose` is told so: the
483
- * application leaves its checkpoint to the next server.
481
+ * A turn the server's shutdown stopped (`turn.stoppedByShutdown`), or that
482
+ * ends on its own once shutdown has begun, ends interrupted however its wait
483
+ * ended, and `afterClose` is told so: the application leaves its checkpoint to
484
+ * the next server.
484
485
  * @param {import("./openCodeProcessOwner.js").OpenCodeProcessOwner} processOwner
485
486
  */
486
487
  export async function observeOpenCodeTurnCompletion(processOwner, target, conversationId, turn, {
@@ -489,6 +490,10 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
489
490
  const turnAbort = turn.abortController;
490
491
  let stopped = true;
491
492
  let outcome = "completed";
493
+ const leftToNextServer = () => {
494
+ if (processOwner.closed) turn.stoppedByShutdown = true;
495
+ return turn.stoppedByShutdown === true;
496
+ };
492
497
  try {
493
498
  const options = { ...waitOptions, signal: AbortSignal.any([
494
499
  signal, ...(timeoutMs ? [AbortSignal.timeout(timeoutMs)] : [])
@@ -496,14 +501,14 @@ export async function observeOpenCodeTurnCompletion(processOwner, target, conver
496
501
  const completion = finalResponse
497
502
  ? await waitForOpenCodeFinalResponse(target.server.client, conversationId, turn, { ...finalResponse, ...options })
498
503
  : await waitForOpenCodeMessages(target.server.client, conversationId, () => turn.inputMessageId, options);
499
- if (turn.stoppedByShutdown) {
504
+ if (leftToNextServer()) {
500
505
  outcome = "interrupted";
501
506
  return { interrupted: true };
502
507
  }
503
508
  return await projectResult(completion);
504
509
  } catch (error) {
505
510
  const failure = signal.aborted ? signal.reason : error;
506
- outcome = turn.interrupted || turn.stoppedByShutdown ? "interrupted" : "failed";
511
+ outcome = leftToNextServer() || turn.interrupted ? "interrupted" : "failed";
507
512
  if (outcome === "interrupted") return { interrupted: true };
508
513
  if (!target.abortController.signal.aborted) {
509
514
  try {
@@ -355,8 +355,10 @@ export function createOpenCodeTurnMonitor({ processes, monitors, turns, processO
355
355
  // server: its saved run stays active and carries the shutdown's mark,
356
356
  // so that server's recovery records the turn interrupted and runs the
357
357
  // host's checkpoint. Run here, that checkpoint (a Git snapshot, say)
358
- // would hold the host's bounded shutdown for as long as it takes. A
359
- // user's Stop ends its turn as interrupted here.
358
+ // could be refused or hold the host's bounded shutdown for as long as
359
+ // it takes. A turn that ends on its own once shutdown has begun is left
360
+ // the same way. A user's Stop outside shutdown ends its turn here.
361
+ if (processOwner.closed && finalState !== "active") turn.stoppedByShutdown = true;
360
362
  const nextServerEnds = turn.stoppedByShutdown === true && finalState !== "active";
361
363
  // Claim retirement before awaited cleanup/checkpointing. A new Send
362
364
  // joins this monitor's cleanup instead of steering an unobserved turn.
@@ -346,6 +346,10 @@ export function createClaudeConversationDriver({
346
346
  return native.owner.closeProject(input, native.application);
347
347
  }
348
348
 
349
+ function beginShutdown({ native }) {
350
+ return native.owner.beginShutdown();
351
+ }
352
+
349
353
  function invalidateRuntimes({ native, input }) {
350
354
  return native.owner.invalidateRuntimes(input, native.application, native.account);
351
355
  }
@@ -568,6 +572,7 @@ export function createClaudeConversationDriver({
568
572
  listConversationStorage,
569
573
  retireConversationHistory,
570
574
  closeProject,
575
+ beginShutdown,
571
576
  invalidateRuntimes,
572
577
  reconcileSessions,
573
578
  createConversation,
@@ -63,6 +63,10 @@ function nativeGoalChanged() {
63
63
  });
64
64
  }
65
65
 
66
+ function beginShutdown({ native }) {
67
+ return native.runOwner.beginShutdown();
68
+ }
69
+
66
70
  /**
67
71
  * Conversations own threads and observers on the account's shared native runtime.
68
72
  * @param {object} dependencies
@@ -653,6 +657,7 @@ export function createCodexConversationDriver({ connections, host = {}, limits =
653
657
  deleteDetachedConversation,
654
658
  listConversationStorage,
655
659
  retireConversationHistory,
660
+ beginShutdown,
656
661
  invalidateRuntimes,
657
662
  reconcileSessions,
658
663
  unsubscribeSessions,
@@ -126,6 +126,10 @@ export function createOpenCodeConversationDriver({
126
126
  return native.owner.closeProject(input, native.preparation.projectCleanup());
127
127
  }
128
128
 
129
+ function beginShutdown({ native }) {
130
+ return native.owner.beginShutdown();
131
+ }
132
+
129
133
  function invalidateRuntimes({ native, input }) {
130
134
  return native.owner.invalidateRuntimes(input, native.preparation);
131
135
  }
@@ -375,6 +379,7 @@ export function createOpenCodeConversationDriver({
375
379
  listConversationStorage,
376
380
  retireConversationHistory,
377
381
  closeProject,
382
+ beginShutdown,
378
383
  invalidateRuntimes,
379
384
  reconcileSessions,
380
385
  createConversation,
@@ -227,6 +227,7 @@ export function createConversationRuntime({ engine: defaultEngine = "api", defau
227
227
  runNativeDetachedConversation: lifecycle.runNativeDetachedConversation,
228
228
  open: handles.open,
229
229
  closeNativeProject: lifecycle.closeNativeProject,
230
+ beginNativeShutdown: lifecycle.beginNativeShutdown,
230
231
  invalidateNativeRuntimes: lifecycle.invalidateNativeRuntimes,
231
232
  reconcileNativeSessions: lifecycle.reconcileNativeSessions,
232
233
  unsubscribeNativeSessions: lifecycle.unsubscribeNativeSessions,
@@ -242,6 +242,34 @@ test("Claude shutdown leaves a running turn's end and its checkpoint to the next
242
242
  assert.deepEqual(custody(JSON.parse(f.session("a").records.get(MAIN_IDS.a))), { executionId: "", state: "interrupted" });
243
243
  });
244
244
 
245
+ test("once Claude's shutdown has begun, a turn that ends on its own is left to the next server, which runs its one checkpoint", { timeout: 5_000 }, async t => {
246
+ const f = await lifecycleFixture(t);
247
+ const a = await f.running("a");
248
+ const saved = f.session("a").records.get(MAIN_IDS.a);
249
+ f.owner.beginShutdown();
250
+ assert.deepEqual(f.calls.stops, [], "beginning the shutdown stopped nothing");
251
+ // The turn ends on its own.
252
+ await f.processes[0].options.onEvent({ type: "result", subtype: "success", result: "Done." });
253
+ assert.deepEqual(f.calls.checkpoints, [], "the turn's end ran no checkpoint");
254
+ assert.equal(f.session("a").records.get(MAIN_IDS.a), saved, "the saved record keeps the active turn");
255
+ assert.equal((await f.owner.invalidateRuntimes({ reason: "server-shutdown" }, f.application, f.account)).ok, true);
256
+ assert.equal(a.process, null, "the full stop still stopped the process");
257
+ assert.deepEqual(f.calls.checkpoints, []);
258
+ // The next server's close proves that exit, then interrupts the turn through its checkpoint, once.
259
+ f.restart();
260
+ assert.equal((await f.owner.closeSession({ sessionId: "a" }, f.application)).ok, true);
261
+ assert.equal((await f.owner.closeSession({ sessionId: "a" }, f.application)).ok, true);
262
+ assert.deepEqual(f.calls.checkpoints, ["a:interrupted"]);
263
+ });
264
+
265
+ test("outside shutdown a Claude turn that ends on its own still runs its checkpoint at once", { timeout: 5_000 }, async t => {
266
+ const f = await lifecycleFixture(t);
267
+ await f.running("a");
268
+ await f.processes[0].options.onEvent({ type: "result", subtype: "success", result: "Done." });
269
+ assert.deepEqual(f.calls.checkpoints, ["a:completed"]);
270
+ assert.equal(JSON.parse(f.session("a").records.get(MAIN_IDS.a)).state, "completed");
271
+ });
272
+
245
273
  test("Claude project close releases its sessions from memory without reopening saved conversations", async t => {
246
274
  const f = await lifecycleFixture(t);
247
275
  await f.idle("a");
@@ -20,6 +20,8 @@ function persistentConversationFixture() {
20
20
  };
21
21
  const commands = createCodexConversationCommands({
22
22
  async checkpoint(_sessionId, input) { checkpoints.push(input); },
23
+ // Outside shutdown, with no request contexts.
24
+ isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
23
25
  conversationPreparation: { async context() { return { result: { provider } }; } },
24
26
  namespace: (sessionId) => sessionId,
25
27
  // The provider owner still holds the provider observing the conversation;
@@ -231,6 +231,8 @@ function conversationCommands({ context = async () => ({ result: { ok: false, co
231
231
  providerSessions = {} } = {}) {
232
232
  return createCodexConversationCommands({
233
233
  checkpoint: async () => {},
234
+ // Outside shutdown, with no request contexts.
235
+ isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
234
236
  conversationPreparation: {
235
237
  context,
236
238
  scope,
@@ -0,0 +1,168 @@
1
+ import assert from "node:assert/strict";
2
+ import test from "node:test";
3
+ import { createCodexConversationCommands } from "../src/server/conversation/codexConversations.js";
4
+ import { createCodexRuntimeLifecycle } from "../src/server/conversation/codexRuntimeLifecycle.js";
5
+
6
+ // A Temporary chat's Codex turn that the server's shutdown stops, or that ends
7
+ // while the server is stopping, leaves its checkpoint to the next server: the
8
+ // application records it as owed, and that server's next read makes it once.
9
+
10
+ const turnOutcomes = { USER_CANCELLED: "user_cancelled", SERVICE_RESTART: "service_restart" };
11
+
12
+ // One server's observed conversation. `latest` is the turn Codex's history
13
+ // reports to a read; turn notifications reach the conversation's observer.
14
+ function server({ closing = false, latest = { id: "turn-1", status: "inProgress", items: [] }, threadError = null } = {}) {
15
+ const checkpoints = [];
16
+ const deferred = [];
17
+ const listeners = new Set();
18
+ const emit = notification => listeners.forEach(listener => listener(notification));
19
+ const provider = {
20
+ subscribe(listener) {
21
+ listeners.add(listener);
22
+ return () => listeners.delete(listener);
23
+ },
24
+ async readThreadStatus(threadId) {
25
+ return { raw: { id: threadId, historyMode: "paginated", status: "idle", ...(threadError ? { error: threadError } : {}) } };
26
+ },
27
+ async listThreadTurns() { return { data: [latest] }; },
28
+ async readGoal() { return { goal: null }; },
29
+ async stopThreadForObservationLoss() {}
30
+ };
31
+ const lifecycle = { closing };
32
+ const commands = createCodexConversationCommands({
33
+ async checkpoint(sessionId, input) { checkpoints.push({ sessionId, ...input }); },
34
+ async deferCheckpoint(sessionId, input) { deferred.push({ sessionId, ...input }); },
35
+ isClosing: () => lifecycle.closing,
36
+ captureContext: () => null,
37
+ runInContext: (_context, operation) => operation(),
38
+ codexAppServerConversationMessageId: (...parts) => parts.join(":"),
39
+ conversationPreparation: { async context() { return { result: { provider } }; } },
40
+ namespace: sessionId => sessionId,
41
+ providerSessions: { owner: { providers: new Map([["provider", provider]]), holdProvider: () => () => {} } },
42
+ turnOutcomes
43
+ });
44
+ const startTurn = () => {
45
+ commands.observeCodexConversation("session", "thread", provider);
46
+ emit({ method: "turn/started", params: { threadId: "thread", turn: { id: "turn-1", status: "inProgress" } } });
47
+ };
48
+ const complete = status => emit({ method: "turn/completed", params: { threadId: "thread", turn: { id: "turn-1", status } } });
49
+ const read = stoppedByShutdown => commands.readPersistentCodexConversation("session", { conversationId: "thread", stoppedByShutdown },
50
+ { provider }).then(({ status, error, runId }) => ({ status, error, runId }));
51
+ return { checkpoints, deferred, commands, lifecycle, startTurn, complete, read };
52
+ }
53
+
54
+ const OWED = { sessionId: "session", outerTurnId: "codex:thread:turn-1", threadId: "thread", turnId: "turn-1" };
55
+
56
+ test("shutdown leaves a running Temporary turn's interrupted checkpoint to the next server, which makes it once on its read", async () => {
57
+ const first = server();
58
+ first.startTurn();
59
+ first.lifecycle.closing = true;
60
+ await first.commands.leaveRunningTurnsToNextServer();
61
+ assert.deepEqual(first.checkpoints, [], "shutdown made no checkpoint");
62
+ assert.deepEqual(first.deferred, [{ ...OWED, status: "interrupted", turnOutcome: "service_restart" }]);
63
+
64
+ // The stopped process ends the turn natively as interrupted.
65
+ const next = server({ latest: { id: "turn-1", status: "interrupted", items: [] }, threadError: { message: "Turn aborted." } });
66
+ const ended = [];
67
+ const mark = runId => ({ runId, async ended() { ended.push(runId); } });
68
+ assert.deepEqual(await next.read(mark("turn-1")), { status: "interrupted", error: "", runId: "turn-1" });
69
+ assert.deepEqual(ended, ["turn-1"], "recording the stopped run lets the application make its checkpoint");
70
+ assert.deepEqual(await next.read(undefined), { status: "interrupted", error: "Turn aborted.", runId: "turn-1" },
71
+ "an unmarked interruption keeps Codex's own report");
72
+ await next.read(mark("turn-0"));
73
+ assert.deepEqual(ended, ["turn-1"], "only the marked run is recorded as ended");
74
+ assert.deepEqual(next.checkpoints, []);
75
+ });
76
+
77
+ test("a Temporary turn that ends on its own while the server stops leaves its checkpoint, with its own outcome, to the next server", async () => {
78
+ const f = server();
79
+ f.startTurn();
80
+ f.lifecycle.closing = true;
81
+ f.complete("completed");
82
+ await Promise.all(f.commands.deferredCheckpoints);
83
+ assert.deepEqual(f.checkpoints, []);
84
+ assert.deepEqual(f.deferred, [{ ...OWED, status: "completed", turnOutcome: "" }]);
85
+ assert.equal(f.commands.deferredCheckpoints.size, 0, "shutdown awaits each deferred record");
86
+
87
+ // The next server reads the completed turn as completed, and makes its checkpoint.
88
+ const next = server({ latest: { id: "turn-1", status: "completed", items: [] } });
89
+ const ended = [];
90
+ assert.equal((await next.read({ runId: "turn-1", async ended() { ended.push("turn-1"); } })).status, "completed");
91
+ assert.deepEqual(ended, ["turn-1"]);
92
+ });
93
+
94
+ test("outside shutdown a Temporary turn's end still makes its checkpoint at once", async () => {
95
+ const f = server();
96
+ f.startTurn();
97
+ f.complete("interrupted");
98
+ assert.deepEqual(f.checkpoints, [{ ...OWED, status: "interrupted", turnOutcome: "" }]);
99
+ assert.deepEqual(f.deferred, []);
100
+ await f.commands.leaveRunningTurnsToNextServer();
101
+ assert.deepEqual(f.deferred, [], "a finished turn owes nothing at shutdown");
102
+ });
103
+
104
+ // The run owner's runtime lifecycle, with the owner calls it makes recorded.
105
+ function runtimeLifecycle(calls, leaveTurnsToNextServer) {
106
+ return createCodexRuntimeLifecycle({
107
+ pendingWork: { conversationTurnStarts: new Map(), messageDeliveries: new Map(), reasoningPersistQueues: new Map(),
108
+ resultFinalizations: new Map(), threadReconciliations: new Map(), notificationTasks: new Map(), deferredCheckpoints: new Set() },
109
+ observer: { eventSubscriptions: new Map() },
110
+ checkManagedConnection: async () => {},
111
+ providerSessions: { managed: new Map(), assertOpen() {}, owner: {
112
+ lifecycleTasks: new Set(), runtimeAcquisitions: new Set(),
113
+ beginShutdown() { calls.push("owner-shutdown"); },
114
+ async invalidateRuntimes() { calls.push("stop-runtimes"); return { failed: [], providerCount: 0, results: [], stopped: 0 }; }
115
+ } },
116
+ helperLifecycle: { projectOperations: new Map(), cleanups: new Map(), mutations: new Map(), restores: new Map(), turnStarts: new Map() },
117
+ captureContext: () => null,
118
+ runInContext: (_context, operation) => operation(),
119
+ debugLog() {},
120
+ debugError: error => error?.message,
121
+ leaveTurnsToNextServer
122
+ });
123
+ }
124
+
125
+ test("a Codex shutdown leaves the running turns to the next server before any runtime stops, and waits for their records", async () => {
126
+ const calls = [];
127
+ const recorded = Promise.withResolvers();
128
+ const lifecycle = runtimeLifecycle(calls, async () => {
129
+ calls.push("leave-turns");
130
+ await recorded.promise;
131
+ calls.push("turns-recorded");
132
+ });
133
+ const shutdown = lifecycle.invalidateRuntimes({ reason: "server-shutdown" }, { enabled: true });
134
+ assert.equal(lifecycle.closing, true);
135
+ await Promise.resolve();
136
+ recorded.resolve();
137
+ await shutdown;
138
+ assert.equal(calls[0], "leave-turns");
139
+ assert.ok(calls.indexOf("turns-recorded") < calls.lastIndexOf("stop-runtimes"), "shutdown waited for the records");
140
+ });
141
+
142
+ test("a begun Codex shutdown stops nothing, and a Temporary turn that then ends on its own leaves its checkpoint to the next server", async () => {
143
+ const f = server();
144
+ f.startTurn();
145
+ const calls = [];
146
+ const lifecycle = runtimeLifecycle(calls, () => f.commands.leaveRunningTurnsToNextServer());
147
+ Object.defineProperty(f.lifecycle, "closing", { get: () => lifecycle.closing });
148
+ lifecycle.beginShutdown();
149
+ assert.equal(lifecycle.closing, true, "the closing fence is set before the call returns");
150
+ assert.deepEqual(calls, ["owner-shutdown"], "beginning the shutdown stopped nothing");
151
+ f.complete("completed");
152
+ await Promise.all(f.commands.deferredCheckpoints);
153
+ assert.deepEqual(f.checkpoints, [], "the turn's end made no checkpoint");
154
+ assert.deepEqual(f.deferred.at(-1), { ...OWED, status: "completed", turnOutcome: "" }, "its own outcome is owed");
155
+
156
+ // The full stop that follows still stops the runtimes, and leaves the turns only once.
157
+ const deferred = f.deferred.length;
158
+ await lifecycle.invalidateRuntimes({ reason: "server-shutdown" }, { enabled: true });
159
+ assert.ok(calls.includes("stop-runtimes"));
160
+ assert.equal(f.deferred.length, deferred);
161
+ assert.deepEqual(f.checkpoints, []);
162
+
163
+ // The next server's read makes the owed checkpoint once.
164
+ const next = server({ latest: { id: "turn-1", status: "completed", items: [] } });
165
+ const ended = [];
166
+ assert.equal((await next.read({ runId: "turn-1", async ended() { ended.push("turn-1"); } })).status, "completed");
167
+ assert.deepEqual(ended, ["turn-1"]);
168
+ });
@@ -53,6 +53,8 @@ function codexFixture(t, { idleMs = 20 } = {}) {
53
53
  });
54
54
  const commands = createCodexConversationCommands({
55
55
  async checkpoint() {},
56
+ // Outside shutdown, with no request contexts.
57
+ isClosing: () => false, captureContext: () => null, runInContext: (_context, operation) => operation(),
56
58
  codexAppServerConversationMessageId: (...parts) => parts.join(":"),
57
59
  codexAppServerStreamMessage: () => ({}),
58
60
  conversationPreparation: {
@@ -3856,3 +3856,60 @@ test("native history budget rejects unavailable dynamic limits before pending de
3856
3856
  assert.equal(sends, 0);
3857
3857
  }
3858
3858
  });
3859
+
3860
+ test("a Main turn the previous server's shutdown stopped reads as interrupted by a restart, with one interrupted checkpoint", async () => {
3861
+ const notices = [];
3862
+ const checkpoints = [];
3863
+ const f = await receiptFixture({
3864
+ outcomeNotice(_runtime, _sessionId, _threadId, _turnId, outcome) { notices.push(outcome); },
3865
+ checkpoint(_sessionId, input) { checkpoints.push(input); }
3866
+ });
3867
+ f.provider.interruptionOutcome = () => "";
3868
+ // The shutdown marks the running turn and leaves the rest of its saved run as it was.
3869
+ assert.deepEqual(await f.owner.markTurnStoppedByShutdown(f.sessionId, f.threadId), { marked: true });
3870
+ assert.deepEqual({ state: f.run.state, stoppedByShutdownTurnId: f.run.stoppedByShutdownTurnId },
3871
+ { state: "active", stoppedByShutdownTurnId: f.turnId });
3872
+ assert.deepEqual(await f.owner.markTurnStoppedByShutdown(f.sessionId, "another-thread"), { marked: false });
3873
+ // The next server reads the native turn as interrupted.
3874
+ const result = await f.owner.stopTurnWithProviderFailure(f.sessionId, f.threadId, f.turnId, {
3875
+ status: "interrupted", provider: f.provider, verifyInactive: false
3876
+ });
3877
+ assert.equal(result.status, "interrupted");
3878
+ assert.match(result.error, /interrupted by a restart/u);
3879
+ assert.deepEqual(notices, ["service_restart"]);
3880
+ assert.deepEqual(checkpoints.map(({ status, turnOutcome }) => ({ status, turnOutcome })),
3881
+ [{ status: "interrupted", turnOutcome: "service_restart" }]);
3882
+ assert.equal(f.run.state, "interrupted");
3883
+ });
3884
+
3885
+ test("an interrupted Main turn without the shutdown's mark keeps the plain interrupted outcome", async () => {
3886
+ const notices = [];
3887
+ const checkpoints = [];
3888
+ const f = await receiptFixture({
3889
+ outcomeNotice(_runtime, _sessionId, _threadId, _turnId, outcome) { notices.push(outcome); },
3890
+ checkpoint(_sessionId, input) { checkpoints.push(input); }
3891
+ });
3892
+ f.provider.interruptionOutcome = () => "";
3893
+ // A mark left by an earlier turn does not apply to this one.
3894
+ f.run = { ...f.run, stoppedByShutdownTurnId: "an-earlier-turn" };
3895
+ await f.owner.stopTurnWithProviderFailure(f.sessionId, f.threadId, f.turnId, {
3896
+ status: "interrupted", provider: f.provider, verifyInactive: false
3897
+ });
3898
+ assert.deepEqual(notices, ["interrupted"]);
3899
+ assert.deepEqual(checkpoints.map(({ turnOutcome }) => turnOutcome), ["interrupted"]);
3900
+ });
3901
+
3902
+ test("a Main turn that ends once shutdown has begun is left to the next server, with no checkpoint now", async () => {
3903
+ const f = await settlementFixture();
3904
+ f.owner.runtimeLifecycle = { closing: true };
3905
+ const before = structuredClone(f.run);
3906
+ assert.deepEqual(await f.owner.markTurnIdle(f.sessionId, { status: "completed", threadId: f.threadId, turnId: f.turnId }),
3907
+ { ok: true, processed: false, reason: "left_to_next_server" });
3908
+ assert.deepEqual(f.run, before, "the saved run is unchanged");
3909
+ assert.deepEqual(f.checkpoints, []);
3910
+ // Outside shutdown the same end is recorded and checkpointed at once.
3911
+ f.owner.runtimeLifecycle = { closing: false };
3912
+ await f.owner.markTurnIdle(f.sessionId, { status: "completed", threadId: f.threadId, turnId: f.turnId });
3913
+ assert.equal(f.run.state, "completed");
3914
+ assert.deepEqual(f.checkpoints.map(({ status }) => status), ["completed"]);
3915
+ });
@@ -706,6 +706,35 @@ test("native invalidation preserves synchronous fences and failures without open
706
706
  await result;
707
707
  });
708
708
 
709
+ test("beginning native shutdown reaches the configured owner synchronously and opens no conversation", async t => {
710
+ const input = { reason: "server-shutdown" };
711
+ const calls = [];
712
+ let asynchronousHost = true;
713
+ const native = { runOwner: {
714
+ beginShutdown() { calls.push("begin"); return "begun"; },
715
+ invalidateRuntimes() { assert.fail("Beginning shutdown stops nothing."); }
716
+ } };
717
+ const runtime = createConversationRuntime({
718
+ authorize() { assert.fail("Trusted shutdown must not acquire session authority."); },
719
+ storage: {
720
+ read() { assert.fail("Shutdown must not read a conversation."); },
721
+ write() { assert.fail("Shutdown must not allocate a conversation."); }
722
+ },
723
+ host: { conversation(request) {
724
+ calls.push("host");
725
+ assert.deepEqual(request, { context, input, operation: "beginShutdown" });
726
+ const prepared = { engine: "codex", native, context, input };
727
+ return asynchronousHost ? Promise.resolve(prepared) : prepared;
728
+ } }
729
+ });
730
+ t.after(() => runtime.close());
731
+ assert.throws(() => runtime.beginNativeShutdown({ context, input }), /synchronous configured host/);
732
+ assert.deepEqual(calls, ["host"]);
733
+ asynchronousHost = false;
734
+ assert.equal(runtime.beginNativeShutdown({ context, input }), "begun");
735
+ assert.deepEqual(calls, ["host", "host", "begin"], "the owner's fence is set before the call returns");
736
+ });
737
+
709
738
  test("renewal proof cleanup preserves OpenCode failure, consumption and retry through the configured owner", async t => {
710
739
  const owner = createOpenCodeSharedRuntime({});
711
740
  const calls = [];
@@ -13,8 +13,8 @@ const ABORTED = "The operation was aborted.";
13
13
  const PROVIDER_FAILURE = "The provider refused this request.";
14
14
 
15
15
  // OpenCode's native history outlives each server that runs it. The turn's reply
16
- // is still being written until an interrupt ends it as aborted, or its provider
17
- // fails it.
16
+ // is still being written until an interrupt ends it as aborted, its provider
17
+ // fails it, or it completes on its own.
18
18
  function nativeOpenCode() {
19
19
  const rows = [
20
20
  { id: INPUT_ID, type: "user", time: { created: 1 } },
@@ -46,7 +46,11 @@ function nativeOpenCode() {
46
46
  rows[1] = { ...rows[1], error: { name: "APIError", data: { message: PROVIDER_FAILURE } }, time: { created: 2, completed: 3 } };
47
47
  busy = false;
48
48
  }
49
- return { calls, create, fail };
49
+ function complete() {
50
+ rows[1] = { ...rows[1], text: "Done.", finish: "stop", time: { created: 2, completed: 3 } };
51
+ busy = false;
52
+ }
53
+ return { calls, create, fail, complete };
50
54
  }
51
55
 
52
56
  // One server with Vibe64's run rules: the session record keeps the OpenCode run
@@ -108,6 +112,8 @@ function server(native, saved, { turnEndCheckpoint = async () => {} } = {}) {
108
112
  return {
109
113
  runtime, calls, release,
110
114
  shutdown: () => runtime.invalidateRuntimes({ reason: "server-shutdown" }, { release }),
115
+ // The server's shutdown begins: nothing stops yet.
116
+ begin: () => runtime.beginShutdown(),
111
117
  async running() {
112
118
  const target = await runtime.acquirePrepared(processPreparation);
113
119
  void runtime.beginMessageMonitor(KEY, target, { id: INPUT_ID, eventStartedAt: 1, startedAt: "2026-11-05T00:00:00.000Z" }, {}, monitor);
@@ -284,3 +290,49 @@ test("OpenCode shutdown ends a temporary conversation's turn as interrupted, and
284
290
  await closed.entry.completion.catch(() => null);
285
291
  assert.deepEqual(closed.closes, [{ stopped: true, outcome: "failed", stoppedByShutdown: false }]);
286
292
  });
293
+
294
+ test("once OpenCode's shutdown has begun, a main turn that ends on its own is left to the next server, which records it with one checkpoint", { timeout: 5_000 }, async t => {
295
+ const native = nativeOpenCode();
296
+ const saved = { run: null };
297
+ const first = server(native, saved);
298
+ t.after(() => first.shutdown());
299
+ await first.running();
300
+ first.begin();
301
+ assert.deepEqual({ interrupts: native.calls.interrupts, stops: native.calls.stops }, { interrupts: [], stops: 0 },
302
+ "beginning the shutdown stopped nothing");
303
+ native.complete();
304
+ await first.runtime.monitors.get(KEY);
305
+ assert.deepEqual(first.calls.checkpoints, [], "the turn's end made no checkpoint");
306
+ assert.deepEqual(runOf(saved), { active: true, state: "active", error: "", stoppedByShutdown: true, turnId: INPUT_ID });
307
+ assert.equal((await first.shutdown()).ok, true);
308
+ assert.equal(native.calls.stops, 1, "the full stop still proved the OpenCode process exit");
309
+ assert.deepEqual(first.calls.checkpoints, []);
310
+
311
+ const next = server(native, saved);
312
+ t.after(() => next.shutdown());
313
+ assert.equal((await next.ready()).ok, true);
314
+ assert.equal((await next.ready()).ok, true);
315
+ assert.deepEqual(next.calls.checkpoints, [`${INPUT_ID}:interrupted`], "the next server made the checkpoint once");
316
+ assert.deepEqual(runOf(saved), { active: false, state: "interrupted", error: "", stoppedByShutdown: true, turnId: INPUT_ID });
317
+ });
318
+
319
+ test("once OpenCode's shutdown has begun, a temporary conversation's turn that ends on its own leaves its checkpoint to the next server", { timeout: 5_000 }, async t => {
320
+ const native = nativeOpenCode();
321
+ const first = server(native, { run: null });
322
+ t.after(() => first.shutdown());
323
+ const { entry, closes } = await first.temporary("ses_temp");
324
+ first.begin();
325
+ native.complete();
326
+ assert.deepEqual(await entry.completion, { interrupted: true });
327
+ assert.deepEqual(native.calls.interrupts, [], "nothing interrupted the native turn");
328
+ assert.deepEqual(closes, [{ stopped: true, outcome: "interrupted", stoppedByShutdown: true }],
329
+ "the application is told to leave the turn's checkpoint to the next server");
330
+ assert.equal((await first.shutdown()).ok, true);
331
+
332
+ const next = server(native, { run: null });
333
+ t.after(() => next.shutdown());
334
+ const ended = [];
335
+ const read = await next.readTemporary("ses_temp", { runId: INPUT_ID, async ended() { ended.push(INPUT_ID); } });
336
+ assert.deepEqual({ status: read.status, error: read.error }, { status: "interrupted", error: "" });
337
+ assert.deepEqual(ended, [INPUT_ID], "recording the run lets the application make its checkpoint");
338
+ });