@jskit-ai/assistant-core 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/examples/conversation/package.json +5 -5
- package/package.json +5 -5
- package/src/server/conversation/claudeNativeTurn.js +9 -1
- package/src/server/conversation/claudeSessionCleanup.js +9 -1
- package/src/server/conversation/claudeTurn.js +1 -0
- package/src/server/conversation/codexConversations.js +65 -6
- package/src/server/conversation/codexRunJournal.js +33 -1
- package/src/server/conversation/codexRuntimeLifecycle.js +20 -1
- package/src/server/conversation/codexTurn.js +41 -4
- package/src/server/conversation/codexTurnEndings.js +9 -6
- package/src/server/conversation/nativeConversationLifecycle.js +22 -0
- package/src/server/conversation/openCodeProcessOwner.js +23 -6
- package/src/server/conversation/openCodeRuntime.js +1 -0
- package/src/server/conversation/openCodeSessionTurns.js +10 -3
- package/src/server/conversation/openCodeTemporaryConversations.js +10 -3
- package/src/server/conversation/openCodeTurn.js +15 -3
- package/src/server/conversation/openCodeTurnMonitor.js +11 -2
- package/src/server/conversation/providers/claudeDriver.js +5 -0
- package/src/server/conversation/providers/codexDriver.js +5 -0
- package/src/server/conversation/providers/opencodeDriver.js +5 -0
- package/src/server/conversation/runtime.js +1 -0
- package/test/claudeLifecycle.test.js +28 -0
- package/test/codexConversationsCheckpoint.test.js +2 -0
- package/test/codexLifecycleIsolation.test.js +2 -0
- package/test/codexShutdownStop.test.js +168 -0
- package/test/codexTemporaryProviderRelease.test.js +2 -0
- package/test/codexTurn.test.js +57 -0
- package/test/conversationSession.test.js +29 -0
- package/test/openCodeLifecycle.test.js +338 -0
|
@@ -8,14 +8,14 @@
|
|
|
8
8
|
"build": "vite build"
|
|
9
9
|
},
|
|
10
10
|
"dependencies": {
|
|
11
|
-
"@jskit-ai/assistant-core": "0.2.
|
|
12
|
-
"@jskit-ai/http-runtime": "0.2.
|
|
13
|
-
"@jskit-ai/kernel": "0.2.
|
|
11
|
+
"@jskit-ai/assistant-core": "0.2.17",
|
|
12
|
+
"@jskit-ai/http-runtime": "0.2.7",
|
|
13
|
+
"@jskit-ai/kernel": "0.2.7",
|
|
14
14
|
"@mdi/js": "^7.4.47",
|
|
15
15
|
"vue": "^3.5.13",
|
|
16
16
|
"vuetify": "^4.0.0",
|
|
17
|
-
"@jskit-ai/connectors-core": "0.2.
|
|
18
|
-
"@jskit-ai/connectors-catalog": "0.2.
|
|
17
|
+
"@jskit-ai/connectors-core": "0.2.8",
|
|
18
|
+
"@jskit-ai/connectors-catalog": "0.2.8"
|
|
19
19
|
},
|
|
20
20
|
"devDependencies": {
|
|
21
21
|
"@vitejs/plugin-vue": "^6.0.7",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jskit-ai/assistant-core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.17",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"test": "node --test --test-concurrency=1"
|
|
@@ -51,8 +51,8 @@
|
|
|
51
51
|
"@ai-sdk/togetherai": "^3.0.49",
|
|
52
52
|
"@ai-sdk/xai": "^4.0.58",
|
|
53
53
|
"@jskit-ai/agent": "0.1.0",
|
|
54
|
-
"@jskit-ai/resource-core": "0.1.
|
|
55
|
-
"@jskit-ai/resource-crud-core": "0.1.
|
|
54
|
+
"@jskit-ai/resource-core": "0.1.198",
|
|
55
|
+
"@jskit-ai/resource-crud-core": "0.1.198",
|
|
56
56
|
"@mdi/js": "^7.4.47",
|
|
57
57
|
"@openrouter/ai-sdk-provider": "^3.0.0",
|
|
58
58
|
"ai": "^7.0.152",
|
|
@@ -61,8 +61,8 @@
|
|
|
61
61
|
"ws": "^8.20.1"
|
|
62
62
|
},
|
|
63
63
|
"peerDependencies": {
|
|
64
|
-
"@jskit-ai/http-runtime": "0.2.
|
|
65
|
-
"@jskit-ai/kernel": "0.2.
|
|
64
|
+
"@jskit-ai/http-runtime": "0.2.7",
|
|
65
|
+
"@jskit-ai/kernel": "0.2.7",
|
|
66
66
|
"vue": "^3.5.13",
|
|
67
67
|
"vuetify": "^4.0.0"
|
|
68
68
|
},
|
|
@@ -355,7 +355,15 @@ export function createClaudeConversationTurn({ conversationId, onEvent, process,
|
|
|
355
355
|
} else if (event.type === "settled") {
|
|
356
356
|
const state = event.status === "interrupted" ? "interrupted" : event.status === "failed" ? "failed" : "completed";
|
|
357
357
|
if (bound) setClaudeTurnState(entry, state, event.error);
|
|
358
|
-
else if (
|
|
358
|
+
else if (entry.turn?.active && process.ownerClosing()) {
|
|
359
|
+
// A turn that ends on its own once shutdown has begun is left to the
|
|
360
|
+
// next server like one the shutdown stops: its saved record keeps the
|
|
361
|
+
// execution and the active turn, so that server's recovery interrupts
|
|
362
|
+
// the turn and runs the host's checkpoint. Run here, that checkpoint
|
|
363
|
+
// (a Git snapshot, say) could be refused or hold up shutdown.
|
|
364
|
+
resolveCompletion({ ...readResult(), status: "interrupted", error: "" });
|
|
365
|
+
return;
|
|
366
|
+
} else if (!await updateState(state, event.error, event.generation)) return;
|
|
359
367
|
}
|
|
360
368
|
const published = await onEvent(event);
|
|
361
369
|
const change = event.type === "provider-event" ? claudeProviderChange(event.event) : "";
|
|
@@ -121,7 +121,7 @@ export function createClaudeSessionCleanup({
|
|
|
121
121
|
account.queries.invalidate().catch(() => {});
|
|
122
122
|
return { ok: true, closed: 0 };
|
|
123
123
|
}
|
|
124
|
-
|
|
124
|
+
beginShutdown();
|
|
125
125
|
// Neither waits for the other: a failed account query cleanup must not
|
|
126
126
|
// leave the sessions' processes running.
|
|
127
127
|
const [queries, retained] = await Promise.allSettled([account.queries.invalidate(), stopRetained(application)]);
|
|
@@ -130,6 +130,13 @@ export function createClaudeSessionCleanup({
|
|
|
130
130
|
return retained.value;
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
// The server's shutdown begins: the owner closes, so each running turn's
|
|
134
|
+
// end, and its checkpoint, is left to the next server, but nothing stops.
|
|
135
|
+
// The later server-shutdown invalidation stops the processes.
|
|
136
|
+
function beginShutdown() {
|
|
137
|
+
closing = true;
|
|
138
|
+
}
|
|
139
|
+
|
|
133
140
|
// Shutdown stops only what this server runs: a Claude process cannot outlive
|
|
134
141
|
// the server that started it, so retained entries hold every one. Saved
|
|
135
142
|
// conversations are not reopened just to be stopped.
|
|
@@ -233,6 +240,7 @@ export function createClaudeSessionCleanup({
|
|
|
233
240
|
closeSession,
|
|
234
241
|
releaseSession,
|
|
235
242
|
closeProject,
|
|
243
|
+
beginShutdown,
|
|
236
244
|
invalidateRuntimes,
|
|
237
245
|
drain,
|
|
238
246
|
forget,
|
|
@@ -701,6 +701,7 @@ export function createClaudeConversationOwner({
|
|
|
701
701
|
closeSession: cleanup.closeSession,
|
|
702
702
|
releaseSession: cleanup.releaseSession,
|
|
703
703
|
closeProject: cleanup.closeProject,
|
|
704
|
+
beginShutdown: cleanup.beginShutdown,
|
|
704
705
|
invalidateRuntimes: cleanup.invalidateRuntimes,
|
|
705
706
|
drain: cleanup.drain,
|
|
706
707
|
forget: cleanup.forget,
|
|
@@ -57,6 +57,11 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
|
57
57
|
* @param {import("./codexProviderSelection.js").CodexProviderSelection["borrowSessionProvider"]} dependencies.borrowSessionProvider
|
|
58
58
|
* @param {string} dependencies.steerFailedCode
|
|
59
59
|
* @param {import("./codexTurn.js").CodexTurnOutcomes} dependencies.turnOutcomes
|
|
60
|
+
* @param {import("./codexTurn.js").CodexDeferCheckpoint} dependencies.deferCheckpoint Absent when the host predates
|
|
61
|
+
* it, which then makes every checkpoint at once.
|
|
62
|
+
* @param {() => boolean} dependencies.isClosing Whether the server's shutdown has begun.
|
|
63
|
+
* @param {import("./codexTurn.js").CodexCaptureContext} dependencies.captureContext
|
|
64
|
+
* @param {import("./codexTurn.js").CodexRunInContext} dependencies.runInContext
|
|
60
65
|
*/
|
|
61
66
|
export function createCodexConversationCommands({
|
|
62
67
|
checkpoint,
|
|
@@ -73,10 +78,17 @@ export function createCodexConversationCommands({
|
|
|
73
78
|
ensureSessionProvider,
|
|
74
79
|
borrowSessionProvider,
|
|
75
80
|
steerFailedCode,
|
|
76
|
-
turnOutcomes
|
|
81
|
+
turnOutcomes,
|
|
82
|
+
deferCheckpoint,
|
|
83
|
+
isClosing,
|
|
84
|
+
captureContext,
|
|
85
|
+
runInContext
|
|
77
86
|
}) {
|
|
78
87
|
const codexAppServerConversations = new Map();
|
|
79
88
|
const codexAppServerConversationTurnStarts = new Map();
|
|
89
|
+
// The application's records of checkpoints left to the next server, which
|
|
90
|
+
// shutdown awaits before the process exits.
|
|
91
|
+
const codexAppServerDeferredCheckpoints = new Set();
|
|
80
92
|
// Turns, detached runs and stops live in their own modules. Each receives the
|
|
81
93
|
// registry above and the registry and context functions declared below.
|
|
82
94
|
const {
|
|
@@ -157,6 +169,42 @@ export function createCodexConversationCommands({
|
|
|
157
169
|
holdWhileActive(state);
|
|
158
170
|
}
|
|
159
171
|
|
|
172
|
+
// Once the server's shutdown has begun a turn's checkpoint belongs to the
|
|
173
|
+
// next server: the application records it as owed, and that server's next
|
|
174
|
+
// read of the conversation makes it. Made now, a checkpoint (a Git snapshot,
|
|
175
|
+
// say) could be refused or hold up the host's bounded shutdown.
|
|
176
|
+
function leaveCheckpointToNextServer(state, ending) {
|
|
177
|
+
const deferred = Promise.resolve()
|
|
178
|
+
.then(() => runInContext(state.context, () => deferCheckpoint(state.sessionId, ending)))
|
|
179
|
+
.catch(() => null);
|
|
180
|
+
codexAppServerDeferredCheckpoints.add(deferred);
|
|
181
|
+
void deferred.finally(() => codexAppServerDeferredCheckpoints.delete(deferred));
|
|
182
|
+
return deferred;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* At shutdown, each observed conversation's running turn: the stopped process
|
|
187
|
+
* ends it without a completion this server observes, so its interrupted
|
|
188
|
+
* checkpoint is left to the next server.
|
|
189
|
+
*/
|
|
190
|
+
function leaveRunningTurnsToNextServer() {
|
|
191
|
+
if (!deferCheckpoint) return Promise.resolve([]);
|
|
192
|
+
const deferred = [];
|
|
193
|
+
for (const conversations of codexAppServerConversations.values()) {
|
|
194
|
+
for (const state of conversations.values()) {
|
|
195
|
+
if (!state.persistent || !state.unsubscribe || !state.runId || !codexAppServerConversationTurnIsActive(state.status)) continue;
|
|
196
|
+
deferred.push(leaveCheckpointToNextServer(state, {
|
|
197
|
+
outerTurnId: codexAppServerNativeOuterTurnId(state.conversationId, state.runId),
|
|
198
|
+
threadId: state.conversationId,
|
|
199
|
+
turnId: state.runId,
|
|
200
|
+
status: "interrupted",
|
|
201
|
+
turnOutcome: turnOutcomes.SERVICE_RESTART
|
|
202
|
+
}));
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return Promise.all(deferred);
|
|
206
|
+
}
|
|
207
|
+
|
|
160
208
|
function codexAppServerConversation(sessionId = "", conversationId = "") {
|
|
161
209
|
return codexAppServerConversations.get(
|
|
162
210
|
namespace(sessionId)
|
|
@@ -201,8 +249,8 @@ export function createCodexConversationCommands({
|
|
|
201
249
|
let state = conversations.get(conversationId);
|
|
202
250
|
if (state?.provider === provider && state.unsubscribe) return state;
|
|
203
251
|
if (state) forgetObservation(state);
|
|
204
|
-
state = { conversationId, provider, persistent: true, status: "ready", runId: "", error: "", turnOutcome: "",
|
|
205
|
-
onEvent: ignoreCodexConversationEvent };
|
|
252
|
+
state = { sessionId, conversationId, provider, persistent: true, status: "ready", runId: "", error: "", turnOutcome: "",
|
|
253
|
+
context: captureContext(), onEvent: ignoreCodexConversationEvent };
|
|
206
254
|
state.unsubscribe = provider.subscribe((notification) => {
|
|
207
255
|
if (codexAppServerNotificationThreadId(notification) !== conversationId) return;
|
|
208
256
|
const turnId = codexAppServerNotificationTurnId(notification);
|
|
@@ -228,13 +276,15 @@ export function createCodexConversationCommands({
|
|
|
228
276
|
// A turn the person stopped reaches the checkpoint as cancelled, as Main turns do.
|
|
229
277
|
const completedOutcome = state.turnOutcome;
|
|
230
278
|
state.turnOutcome = "";
|
|
231
|
-
|
|
279
|
+
const ending = {
|
|
232
280
|
outerTurnId: codexAppServerNativeOuterTurnId(conversationId, completedTurnId),
|
|
233
281
|
threadId: conversationId,
|
|
234
282
|
turnId: completedTurnId,
|
|
235
283
|
status: completedStatus,
|
|
236
284
|
turnOutcome: completedOutcome
|
|
237
|
-
}
|
|
285
|
+
};
|
|
286
|
+
if (isClosing() && deferCheckpoint) leaveCheckpointToNextServer(state, ending);
|
|
287
|
+
else void checkpoint(sessionId, ending).catch(() => {});
|
|
238
288
|
}
|
|
239
289
|
holdWhileActive(state);
|
|
240
290
|
});
|
|
@@ -284,6 +334,13 @@ export function createCodexConversationCommands({
|
|
|
284
334
|
if (goal?.status === "active") status = "inProgress";
|
|
285
335
|
Object.assign(state, { runId, status, goal });
|
|
286
336
|
holdWhileActive(state);
|
|
337
|
+
// The application marks the run whose checkpoint a previous server's
|
|
338
|
+
// shutdown left owed (`stoppedByShutdown: { runId, ended }`). Recording
|
|
339
|
+
// that it ended lets the application make that checkpoint; a run the
|
|
340
|
+
// shutdown stopped reads interrupted with no error.
|
|
341
|
+
const shutdownStop = input.stoppedByShutdown;
|
|
342
|
+
const endedAtShutdown = Boolean(runId) && shutdownStop?.runId === runId && !codexAppServerConversationTurnIsActive(status);
|
|
343
|
+
if (endedAtShutdown) await shutdownStop.ended();
|
|
287
344
|
const messages = [];
|
|
288
345
|
for (const turn of turns) {
|
|
289
346
|
const turnId = codexAppServerProviderTurnId(turn);
|
|
@@ -321,7 +378,7 @@ export function createCodexConversationCommands({
|
|
|
321
378
|
goal,
|
|
322
379
|
admitted: Boolean(input.messageId && turns.some((turn) => codexAppServerProviderTurnClientIds(turn).includes(input.messageId))),
|
|
323
380
|
rawText: latestText.trim(),
|
|
324
|
-
error: state.error || codexAppServerThreadError(thread)
|
|
381
|
+
error: endedAtShutdown && status === "interrupted" ? "" : state.error || codexAppServerThreadError(thread)
|
|
325
382
|
};
|
|
326
383
|
}
|
|
327
384
|
|
|
@@ -555,6 +612,8 @@ export function createCodexConversationCommands({
|
|
|
555
612
|
return Object.freeze({
|
|
556
613
|
codexAppServerConversations,
|
|
557
614
|
codexAppServerConversationTurnStarts,
|
|
615
|
+
deferredCheckpoints: codexAppServerDeferredCheckpoints,
|
|
616
|
+
leaveRunningTurnsToNextServer,
|
|
558
617
|
codexAppServerConversation,
|
|
559
618
|
hasActiveConversation,
|
|
560
619
|
observeCodexConversation,
|
|
@@ -3,6 +3,32 @@ import crypto from "node:crypto";
|
|
|
3
3
|
import { isRecord, normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
4
4
|
|
|
5
5
|
const CODEX_APP_SERVER_PROMPT_CLAIM_GRACE_MS = 15000;
|
|
6
|
+
const CODEX_APP_SERVER_STOPPED_BY_SHUTDOWN_EVENT = "codex-app-server-stopped-by-shutdown";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Marks the session's running turn as stopped by the server's shutdown, so the
|
|
10
|
+
* next server records its interruption as a restart. Nothing else of the saved
|
|
11
|
+
* run changes.
|
|
12
|
+
* @param {{ createRuntime: import("./codexTurn.js").CodexCreateRuntime, turnState: import("./codexTurn.js").CodexTurnStateReader }} dependencies
|
|
13
|
+
* @param {string} sessionId
|
|
14
|
+
* @param {string} [threadId] the session's Main thread, when known
|
|
15
|
+
*/
|
|
16
|
+
export async function markCodexAppServerTurnStoppedByShutdown({ createRuntime, turnState }, sessionId = "", threadId = "") {
|
|
17
|
+
const normalizedSessionId = normalizeText(sessionId);
|
|
18
|
+
if (!normalizedSessionId) return { marked: false };
|
|
19
|
+
const runtime = await createRuntime();
|
|
20
|
+
let marked = false;
|
|
21
|
+
await runtime.store.mutateSession(normalizedSessionId, async () => {
|
|
22
|
+
const turn = turnState(await runtime.getSession(normalizedSessionId).catch(() => null) || {});
|
|
23
|
+
if (!turn.active || !turn.turnId || (normalizeText(threadId) && turn.threadId !== normalizeText(threadId))) return;
|
|
24
|
+
await runtime.store.writeAgentRunEvent(normalizedSessionId, CODEX_APP_SERVER_AGENT_RUN_ID, {
|
|
25
|
+
event: { kind: CODEX_APP_SERVER_STOPPED_BY_SHUTDOWN_EVENT },
|
|
26
|
+
patch: { stoppedByShutdownTurnId: turn.turnId }
|
|
27
|
+
});
|
|
28
|
+
marked = true;
|
|
29
|
+
});
|
|
30
|
+
return { marked };
|
|
31
|
+
}
|
|
6
32
|
|
|
7
33
|
function terminalCodexAppServerAgentRunState(status = "") {
|
|
8
34
|
const normalizedStatus = normalizeText(status);
|
|
@@ -40,6 +66,7 @@ function codexAppServerDateValueMs(value = "") {
|
|
|
40
66
|
* @param {number} dependencies.finalizingGraceMs
|
|
41
67
|
* @param {(session: object, run: object) => boolean} dependencies.admissionTaskFinished
|
|
42
68
|
* @param {string} dependencies.orphanedPromptMessage
|
|
69
|
+
* @param {() => boolean} dependencies.isClosing Whether the server's shutdown has begun.
|
|
43
70
|
*/
|
|
44
71
|
export function createCodexRunJournal({
|
|
45
72
|
namespace,
|
|
@@ -57,7 +84,8 @@ export function createCodexRunJournal({
|
|
|
57
84
|
idlePublishPayload,
|
|
58
85
|
finalizingGraceMs,
|
|
59
86
|
admissionTaskFinished,
|
|
60
|
-
orphanedPromptMessage
|
|
87
|
+
orphanedPromptMessage,
|
|
88
|
+
isClosing
|
|
61
89
|
}) {
|
|
62
90
|
const codexAppServerCompletedTurns = new Set();
|
|
63
91
|
const codexAppServerProcessedTurns = new Set();
|
|
@@ -394,6 +422,10 @@ export function createCodexRunJournal({
|
|
|
394
422
|
}
|
|
395
423
|
|
|
396
424
|
async function markCodexAppServerTurnIdle(sessionId = "", input = {}) {
|
|
425
|
+
// Once shutdown has begun the turn's end, and its checkpoint, are left to the next server's recovery, as after a crash.
|
|
426
|
+
if (isClosing()) {
|
|
427
|
+
return { ok: true, processed: false, reason: "left_to_next_server" };
|
|
428
|
+
}
|
|
397
429
|
const status = normalizeText(input.status) || "completed";
|
|
398
430
|
const result = await publishCodexAppServerAgentRun(sessionId, {
|
|
399
431
|
error: normalizeText(input.error),
|
|
@@ -6,6 +6,9 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
|
6
6
|
/** @typedef {ReturnType<typeof import("./codexTurn.js").createCodexAppServerRunOwner>} CodexRunOwner */
|
|
7
7
|
/** @typedef {(event: string, details: object) => void} CodexDebugLog */
|
|
8
8
|
|
|
9
|
+
/** A lifecycle without running turns to hand over leaves none. */
|
|
10
|
+
async function leaveNoTurns() {}
|
|
11
|
+
|
|
9
12
|
/**
|
|
10
13
|
* Shutdown, wellbeing timers and managed-session records for shared Codex
|
|
11
14
|
* runtimes. It works on the run/provider owners' collections; it owns no servers.
|
|
@@ -23,6 +26,8 @@ import { normalizeText } from "@jskit-ai/kernel/shared/support/normalize";
|
|
|
23
26
|
* @param {CodexDebugLog} dependencies.debugLog
|
|
24
27
|
* @param {(error: unknown) => unknown} dependencies.debugError
|
|
25
28
|
* @param {{ message: string, code: string, retryable: boolean } | null} [dependencies.closingError]
|
|
29
|
+
* @param {() => Promise<unknown>} [dependencies.leaveTurnsToNextServer] When shutdown begins, leaves each running
|
|
30
|
+
* turn's end, and its checkpoint, to the next server.
|
|
26
31
|
*/
|
|
27
32
|
export function createCodexRuntimeLifecycle({
|
|
28
33
|
pendingWork,
|
|
@@ -35,7 +40,8 @@ export function createCodexRuntimeLifecycle({
|
|
|
35
40
|
continueGoalAfterAccountChange,
|
|
36
41
|
debugLog,
|
|
37
42
|
debugError,
|
|
38
|
-
closingError = null
|
|
43
|
+
closingError = null,
|
|
44
|
+
leaveTurnsToNextServer = leaveNoTurns
|
|
39
45
|
}) {
|
|
40
46
|
const codexAppServerSessionClosures = new Map();
|
|
41
47
|
const codexAppServerWellbeingTimers = new Map();
|
|
@@ -44,6 +50,7 @@ export function createCodexRuntimeLifecycle({
|
|
|
44
50
|
let codexAppServerThreadReconcileGeneration = 0;
|
|
45
51
|
let codexAppServerServerClosing = false;
|
|
46
52
|
let codexAppServerShutdownPromise = null;
|
|
53
|
+
let codexAppServerTurnsLeft = null;
|
|
47
54
|
|
|
48
55
|
function assertOpen() {
|
|
49
56
|
if (!closingError) return providerSessions.assertOpen();
|
|
@@ -216,11 +223,20 @@ export function createCodexRuntimeLifecycle({
|
|
|
216
223
|
}
|
|
217
224
|
}
|
|
218
225
|
|
|
226
|
+
// The server's shutdown begins: the closing fence is set and each running
|
|
227
|
+
// turn's end, and its checkpoint, is left to the next server, but nothing
|
|
228
|
+
// stops. The later server-shutdown invalidation stops the runtimes.
|
|
219
229
|
function beginCodexAppServerShutdown() {
|
|
220
230
|
if (codexAppServerServerClosing) {
|
|
221
231
|
return;
|
|
222
232
|
}
|
|
223
233
|
codexAppServerServerClosing = true;
|
|
234
|
+
// Taken now, before any runtime stops, while each running turn is known.
|
|
235
|
+
codexAppServerTurnsLeft = (async () => leaveTurnsToNextServer())().catch(error => {
|
|
236
|
+
debugLog("appServerShutdown.leaveTurns.error", { error: debugError(error) });
|
|
237
|
+
}).finally(() => {
|
|
238
|
+
codexAppServerTurnsLeft = null;
|
|
239
|
+
});
|
|
224
240
|
providerSessions.owner.beginShutdown();
|
|
225
241
|
codexAppServerThreadReconcileGeneration += 1;
|
|
226
242
|
for (const providerKey of [...codexAppServerWellbeingTimers.keys()]) {
|
|
@@ -231,6 +247,8 @@ export function createCodexRuntimeLifecycle({
|
|
|
231
247
|
async function drainCodexAppServerControllerTasks() {
|
|
232
248
|
while (true) {
|
|
233
249
|
const pending = new Set([
|
|
250
|
+
...(codexAppServerTurnsLeft ? [codexAppServerTurnsLeft] : []),
|
|
251
|
+
...pendingWork.deferredCheckpoints.values(),
|
|
234
252
|
...pendingWork.conversationTurnStarts.values(),
|
|
235
253
|
...helperLifecycle.projectOperations.values(),
|
|
236
254
|
...helperLifecycle.cleanups.values(),
|
|
@@ -356,6 +374,7 @@ export function createCodexRuntimeLifecycle({
|
|
|
356
374
|
|
|
357
375
|
return Object.freeze({
|
|
358
376
|
get closing() { return codexAppServerServerClosing; },
|
|
377
|
+
beginShutdown: beginCodexAppServerShutdown,
|
|
359
378
|
assertOpen,
|
|
360
379
|
get reconcileGeneration() { return codexAppServerThreadReconcileGeneration; },
|
|
361
380
|
nextReconcileGeneration,
|
|
@@ -10,7 +10,7 @@ import { createCodexNotificationObserver } from "./codexNotificationObserver.js"
|
|
|
10
10
|
import { createCodexThreadStatus } from "./codexThreadStatus.js";
|
|
11
11
|
import { createCodexTurnRecovery } from "./codexTurnRecovery.js";
|
|
12
12
|
import { createCodexResultSettlement } from "./codexResultSettlement.js";
|
|
13
|
-
import { createCodexRunJournal } from "./codexRunJournal.js";
|
|
13
|
+
import { createCodexRunJournal, markCodexAppServerTurnStoppedByShutdown } from "./codexRunJournal.js";
|
|
14
14
|
import { createCodexRenewalCommands } from "./codexRenewalCommands.js";
|
|
15
15
|
import { createCodexGoalCommands } from "./codexGoalCommands.js";
|
|
16
16
|
import { createCodexProviderSelection } from "./codexProviderSelection.js";
|
|
@@ -80,6 +80,13 @@ export { codexAppServerThreadIsMissing } from "./codexProviderThreads.js";
|
|
|
80
80
|
/** @typedef {typeof CODEX_TURN_OUTCOMES} CodexTurnOutcomes */
|
|
81
81
|
/** @typedef {(runtime: any, session: object, options: object) => Promise<any>} CodexAdmissionRecovery */
|
|
82
82
|
/** @typedef {(sessionId: string, input: object) => Promise<unknown>} CodexCheckpoint */
|
|
83
|
+
/**
|
|
84
|
+
* Records a scoped conversation turn's checkpoint as owed to the next server,
|
|
85
|
+
* once the server's shutdown has begun. The application passes the run back
|
|
86
|
+
* on that server's next read as `stoppedByShutdown: { runId, ended }`, and
|
|
87
|
+
* makes the checkpoint when `ended` is called.
|
|
88
|
+
* @typedef {(sessionId: string, input: { outerTurnId: string, threadId: string, turnId: string, status: string, turnOutcome: string }) => Promise<unknown>} CodexDeferCheckpoint
|
|
89
|
+
*/
|
|
83
90
|
|
|
84
91
|
/**
|
|
85
92
|
* The application hooks that add metadata to the turns a run writes.
|
|
@@ -161,6 +168,8 @@ function runInNoContext(_context, operation) {
|
|
|
161
168
|
* @param {string} [options.messageIdPrefix]
|
|
162
169
|
* @param {object} [options.idlePublishPayload]
|
|
163
170
|
* @param {CodexCheckpoint} [options.checkpoint]
|
|
171
|
+
* @param {CodexDeferCheckpoint} options.deferCheckpoint A host that predates it leaves it out: a scoped
|
|
172
|
+
* conversation turn that ends during shutdown then makes its checkpoint at once, and one the shutdown stops makes none.
|
|
164
173
|
* @param {Partial<CodexMessageMetadataHooks>} [options.messageMetadata]
|
|
165
174
|
* @param {boolean} [options.persistCommentary]
|
|
166
175
|
* @param {string} [options.deliveryStateMetadataKey]
|
|
@@ -204,6 +213,7 @@ export function createCodexAppServerRunOwner({
|
|
|
204
213
|
messageIdPrefix = "codex:",
|
|
205
214
|
idlePublishPayload,
|
|
206
215
|
checkpoint = answerNothing,
|
|
216
|
+
deferCheckpoint,
|
|
207
217
|
messageMetadata = {},
|
|
208
218
|
persistCommentary = true,
|
|
209
219
|
deliveryStateMetadataKey = "assistant_delivery",
|
|
@@ -317,7 +327,8 @@ export function createCodexAppServerRunOwner({
|
|
|
317
327
|
idlePublishPayload,
|
|
318
328
|
finalizingGraceMs,
|
|
319
329
|
admissionTaskFinished,
|
|
320
|
-
orphanedPromptMessage
|
|
330
|
+
orphanedPromptMessage,
|
|
331
|
+
isClosing: () => runOwner.runtimeLifecycle?.closing === true
|
|
321
332
|
});
|
|
322
333
|
const admissionRecovery = recoverAdmission || codexRunJournal.recoverAbandonedPromptClaim;
|
|
323
334
|
|
|
@@ -424,7 +435,11 @@ export function createCodexAppServerRunOwner({
|
|
|
424
435
|
ensureSessionProvider: codexProviderSelection.ensureSessionProvider,
|
|
425
436
|
borrowSessionProvider: codexProviderSelection.borrowSessionProvider,
|
|
426
437
|
steerFailedCode,
|
|
427
|
-
turnOutcomes
|
|
438
|
+
turnOutcomes,
|
|
439
|
+
deferCheckpoint,
|
|
440
|
+
isClosing: () => runOwner.runtimeLifecycle?.closing === true,
|
|
441
|
+
captureContext,
|
|
442
|
+
runInContext
|
|
428
443
|
});
|
|
429
444
|
|
|
430
445
|
const codexObservationControl = createCodexObservationControl({
|
|
@@ -636,6 +651,7 @@ export function createCodexAppServerRunOwner({
|
|
|
636
651
|
clearSessionRecoveryTimers: codexRunJournal.clearSessionRecoveryTimers,
|
|
637
652
|
publishAgentRun: codexRunJournal.publishAgentRun,
|
|
638
653
|
markTurnIdle: codexRunJournal.markTurnIdle,
|
|
654
|
+
markTurnStoppedByShutdown,
|
|
639
655
|
finalizeAssistantResult: codexResultSettlement.finalizeAssistantResult,
|
|
640
656
|
finalizedTurns: codexAppServerOutput.finalizedTurns,
|
|
641
657
|
liveProgressItems: codexAppServerOutput.liveProgressItems,
|
|
@@ -661,8 +677,10 @@ export function createCodexAppServerRunOwner({
|
|
|
661
677
|
reasoningPersistQueues: codexAppServerOutput.reasoningPersistQueues,
|
|
662
678
|
resultFinalizations: codexResultSettlement.resultFinalizations,
|
|
663
679
|
threadReconciliations: codexThreadReadiness.threadReconciliations,
|
|
664
|
-
notificationTasks: codexAppServerNotificationQueue.tasks
|
|
680
|
+
notificationTasks: codexAppServerNotificationQueue.tasks,
|
|
681
|
+
deferredCheckpoints: codexConversations.deferredCheckpoints
|
|
665
682
|
},
|
|
683
|
+
leaveTurnsToNextServer: leaveRunningTurnsToNextServer,
|
|
666
684
|
observer: codexNotificationObserver,
|
|
667
685
|
checkManagedConnection: codexThreadReadiness.checkManagedConnection,
|
|
668
686
|
providerSessions,
|
|
@@ -678,6 +696,7 @@ export function createCodexAppServerRunOwner({
|
|
|
678
696
|
providerSessions.owner.attachRuntimeLifecycle(runOwner.runtimeLifecycle);
|
|
679
697
|
providerSessions.assertOpen = runOwner.runtimeLifecycle.assertOpen;
|
|
680
698
|
}
|
|
699
|
+
runOwner.beginShutdown = runOwner.runtimeLifecycle.beginShutdown;
|
|
681
700
|
runOwner.invalidateRuntimes = runOwner.runtimeLifecycle.invalidateRuntimes;
|
|
682
701
|
Object.assign(runOwner, createCodexSessionReconciliation({
|
|
683
702
|
readiness: codexThreadReadiness, recoverSessionObservationLoss,
|
|
@@ -685,6 +704,24 @@ export function createCodexAppServerRunOwner({
|
|
|
685
704
|
createRuntime, namespace, turnState, sessionIdRequiredError, debugLog, debugError
|
|
686
705
|
}));
|
|
687
706
|
}
|
|
707
|
+
function markTurnStoppedByShutdown(sessionId = "", threadId = "") {
|
|
708
|
+
return markCodexAppServerTurnStoppedByShutdown({ createRuntime, turnState }, sessionId, threadId);
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
// At shutdown each running turn's end, and its checkpoint, belong to the
|
|
712
|
+
// next server: Main turns are marked so it records their interruption as a
|
|
713
|
+
// restart, and scoped conversations leave their checkpoints owed.
|
|
714
|
+
async function leaveRunningTurnsToNextServer() {
|
|
715
|
+
const results = await Promise.allSettled([
|
|
716
|
+
codexConversations.leaveRunningTurnsToNextServer(),
|
|
717
|
+
...[...providerSessions.managed.values()].map(managed => Promise.resolve().then(() => runInContext(
|
|
718
|
+
managed.projectContext, () => markTurnStoppedByShutdown(managed.sessionId, managed.threadId))))
|
|
719
|
+
]);
|
|
720
|
+
for (const result of results) {
|
|
721
|
+
if (result.status === "rejected") debugLog("appServerShutdown.leaveTurn.error", { error: debugError(result.reason) });
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
688
725
|
const cleanup = createCodexSessionCleanup({
|
|
689
726
|
namespace,
|
|
690
727
|
runOwner,
|
|
@@ -253,9 +253,9 @@ export function createCodexTurnEndings({
|
|
|
253
253
|
const normalizedStatus = normalizeText(status);
|
|
254
254
|
let base = "Codex app-server failed before completing this turn.";
|
|
255
255
|
if (normalizedStatus === "interrupted") {
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
256
|
+
if (outcome === turnOutcomes.CONTROL_RECONFIGURATION) base = "Codex app-server was interrupted while its controls were restored.";
|
|
257
|
+
else if (outcome === turnOutcomes.SERVICE_RESTART) base = "Codex app-server was interrupted by a restart before completing this turn.";
|
|
258
|
+
else base = "Codex app-server was interrupted before completing this turn.";
|
|
259
259
|
}
|
|
260
260
|
const normalizedError = normalizeText(error);
|
|
261
261
|
return normalizedError ? `${base} ${normalizedError}` : base;
|
|
@@ -277,17 +277,20 @@ export function createCodexTurnEndings({
|
|
|
277
277
|
const normalizedThreadId = normalizeText(threadId);
|
|
278
278
|
const normalizedTurnId = normalizeText(turnId);
|
|
279
279
|
const normalizedStatus = normalizeText(status) || "failed";
|
|
280
|
+
const runtime = await createRuntime();
|
|
280
281
|
let resolvedOutcome = normalizeText(outcome);
|
|
281
282
|
if (!resolvedOutcome) {
|
|
282
283
|
if (normalizedStatus === "interrupted") {
|
|
283
284
|
const recorded = provider ? provider.interruptionOutcome(normalizedThreadId, normalizedTurnId) : "";
|
|
284
|
-
|
|
285
|
-
|
|
285
|
+
if ([turnOutcomes.USER_CANCELLED, turnOutcomes.CONTROL_RECONFIGURATION].includes(recorded)) resolvedOutcome = recorded;
|
|
286
|
+
// The previous server's shutdown stopped this turn: a restart, not a failure.
|
|
287
|
+
else if (codexAppServerAgentRun(await runtime.getSession(normalizedSessionId))?.stoppedByShutdownTurnId === normalizedTurnId) {
|
|
288
|
+
resolvedOutcome = turnOutcomes.SERVICE_RESTART;
|
|
289
|
+
} else resolvedOutcome = turnOutcomes.INTERRUPTED;
|
|
286
290
|
} else {
|
|
287
291
|
resolvedOutcome = turnOutcomes.PROVIDER_FAILURE;
|
|
288
292
|
}
|
|
289
293
|
}
|
|
290
|
-
const runtime = await createRuntime();
|
|
291
294
|
const staleFailure = async () => {
|
|
292
295
|
const turn = turnState(await runtime.getSession(normalizedSessionId));
|
|
293
296
|
if (codexAppServerTurnCanReceiveProviderCompletion(turn, normalizedThreadId, normalizedTurnId)) {
|
|
@@ -218,6 +218,27 @@ function createNativeConversationLifecycle({ authorize, defaultHost, hostPrepare
|
|
|
218
218
|
return driver.invalidateRuntimes(prepared);
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
+
// The server's shutdown begins before anything stops, on the same trusted
|
|
222
|
+
// boundary: the configured owner sets its closing fence and leaves each
|
|
223
|
+
// running turn's end, and its checkpoint, to the next server, synchronously.
|
|
224
|
+
// A later invalidation with reason "server-shutdown" stops its processes.
|
|
225
|
+
function beginNativeShutdown({ context, input = {} }) {
|
|
226
|
+
if (!hostPrepares) {
|
|
227
|
+
throw new TypeError("Native shutdown requires a configured conversation host.");
|
|
228
|
+
}
|
|
229
|
+
const prepared = defaultHost.conversation({ context, input, operation: "beginShutdown" });
|
|
230
|
+
if (!prepared || isThenable(prepared) ||
|
|
231
|
+
!["codex", "claude", "opencode"].includes(prepared.engine) || !prepared.native ||
|
|
232
|
+
!Object.hasOwn(prepared, "input") || !Object.hasOwn(prepared, "context")) {
|
|
233
|
+
throw new TypeError("Native shutdown requires synchronous configured host preparation.");
|
|
234
|
+
}
|
|
235
|
+
const driver = providers.createDriver(prepared.engine, defaultHost);
|
|
236
|
+
if (!implementsOperation(driver, "beginShutdown")) {
|
|
237
|
+
throw new TypeError("The selected conversation driver does not implement native shutdown.");
|
|
238
|
+
}
|
|
239
|
+
return driver.beginShutdown(prepared);
|
|
240
|
+
}
|
|
241
|
+
|
|
221
242
|
// Startup reconciliation and subscription reset use the existing trusted
|
|
222
243
|
// lifecycle caller. Native owners retain their own shutdown fences and tasks.
|
|
223
244
|
function reconcileNativeSessions({ context, sessions, options = {} }) {
|
|
@@ -363,6 +384,7 @@ function createNativeConversationLifecycle({ authorize, defaultHost, hostPrepare
|
|
|
363
384
|
createNativeConversation,
|
|
364
385
|
runNativeDetachedConversation,
|
|
365
386
|
closeNativeProject,
|
|
387
|
+
beginNativeShutdown,
|
|
366
388
|
invalidateNativeRuntimes,
|
|
367
389
|
reconcileNativeSessions,
|
|
368
390
|
unsubscribeNativeSessions,
|
|
@@ -349,9 +349,20 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
|
|
|
349
349
|
|
|
350
350
|
async function releaseTarget(target, { retainSharedProcess = false, onRemoved } = {}) {
|
|
351
351
|
const activeThreads = new Set();
|
|
352
|
-
|
|
352
|
+
// A turn the server's shutdown stops carries that mark: its observer
|
|
353
|
+
// leaves the turn's end to the next server, which records it interrupted,
|
|
354
|
+
// since the native abort this stop causes is not the turn's failure.
|
|
355
|
+
const stopping = turn => {
|
|
356
|
+
if (closed) turn.stoppedByShutdown = true;
|
|
357
|
+
};
|
|
358
|
+
const mainTurn = turns.get(target.key);
|
|
359
|
+
if (mainTurn?.active) {
|
|
360
|
+
stopping(mainTurn);
|
|
361
|
+
activeThreads.add(target.upstreamSessionId);
|
|
362
|
+
}
|
|
353
363
|
for (const entry of temporaryConversations.values()) {
|
|
354
364
|
if (entry.active && entry.target?.abortController === target.abortController) {
|
|
365
|
+
stopping(entry);
|
|
355
366
|
activeThreads.add(entry.conversationId);
|
|
356
367
|
}
|
|
357
368
|
}
|
|
@@ -430,12 +441,17 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
|
|
|
430
441
|
};
|
|
431
442
|
}
|
|
432
443
|
|
|
444
|
+
// The server's shutdown begins: the owner closes, so each running turn's
|
|
445
|
+
// end, and its checkpoint, is left to the next server, but nothing stops.
|
|
446
|
+
// The later server-shutdown invalidation stops the processes.
|
|
447
|
+
function beginShutdown() {
|
|
448
|
+
closed = true;
|
|
449
|
+
clearTimeout(storageIdle);
|
|
450
|
+
storageIdle = null;
|
|
451
|
+
}
|
|
452
|
+
|
|
433
453
|
async function invalidateRuntimes(input = {}, preparation) {
|
|
434
|
-
if (normalizeNullableText(input.reason) === "server-shutdown")
|
|
435
|
-
closed = true;
|
|
436
|
-
clearTimeout(storageIdle);
|
|
437
|
-
storageIdle = null;
|
|
438
|
-
}
|
|
454
|
+
if (normalizeNullableText(input.reason) === "server-shutdown") beginShutdown();
|
|
439
455
|
const application = preparation.release;
|
|
440
456
|
// readable-check: allow method-probe - input validation: a connection change carries the publication of the application's saved connections
|
|
441
457
|
if (typeof input.publish === "function") {
|
|
@@ -556,6 +572,7 @@ export function createOpenCodeProcessOwner({ processes, processStarts, monitors,
|
|
|
556
572
|
releaseProcessTarget,
|
|
557
573
|
closeSession,
|
|
558
574
|
closeProject,
|
|
575
|
+
beginShutdown,
|
|
559
576
|
invalidateRuntimes,
|
|
560
577
|
releaseRenewalPredecessorProcessExitProof,
|
|
561
578
|
releaseProcessExitProof
|
|
@@ -226,6 +226,7 @@ export function createOpenCodeSharedRuntime({ onStop = ignoreRuntimeStop, scope,
|
|
|
226
226
|
sendPreparedMessage: sessionTurns.sendPreparedMessage,
|
|
227
227
|
inspectPreparedMessage: sessionTurns.inspectPreparedMessage,
|
|
228
228
|
interruptPreparedTurn: sessionTurns.interruptPreparedTurn,
|
|
229
|
+
beginShutdown: processOwner.beginShutdown,
|
|
229
230
|
invalidateRuntimes: processOwner.invalidateRuntimes,
|
|
230
231
|
releaseRenewalPredecessorProcessExitProof: processOwner.releaseRenewalPredecessorProcessExitProof,
|
|
231
232
|
releaseProcessExitProof: processOwner.releaseProcessExitProof,
|