@oar-sh/oar 0.9.12 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/extension.mjs +9 -0
- package/.github/extensions/web-relay/server-lifecycle/managed-server.mjs +15 -1
- package/.github/extensions/web-relay/skills/question-bridge.mjs +4 -1
- package/CHANGELOG.md +508 -0
- package/README.md +12 -12
- package/bin/oar.js +155 -94
- package/package.json +5 -5
- package/server/README.md +479 -157
- package/server/claude-cloud-worker/claude-cloud-session-process.mjs +10 -4
- package/server/claude-worker/claude-session-process.mjs +92 -15
- package/server/claude-worker/claude-turn-publisher.mjs +20 -1
- package/server/claude-worker/subagent-timeline-recorder.mjs +75 -8
- package/server/claude-worker/task-output-discovery.mjs +28 -2
- package/server/copilot-worker/copilot-ask-user-tool.mjs +1 -1
- package/server/copilot-worker/copilot-continuation-signals.mjs +11 -5
- package/server/copilot-worker/copilot-prompt-context.mjs +3 -1
- package/server/copilot-worker/copilot-question-bridge.mjs +11 -6
- package/server/copilot-worker/copilot-sdk-adapter.mjs +11 -0
- package/server/copilot-worker/copilot-sdk-session-process.mjs +24 -1
- package/server/cursor-worker/cursor-ask-user-tool.mjs +1 -1
- package/server/cursor-worker/cursor-sdk-adapter.mjs +22 -7
- package/server/cursor-worker/cursor-session-worker.mjs +9 -2
- package/server/cursor-worker/cursor-turn-runner.mjs +18 -3
- package/server/db-schema.mjs +10 -0
- package/server/grok-worker/acp-host-services.mjs +6 -3
- package/server/grok-worker/grok-background-tasks.mjs +7 -0
- package/server/grok-worker/grok-sdk-adapter.mjs +14 -0
- package/server/grok-worker/grok-turn-runner.mjs +13 -3
- package/server/migrations/0018-background-task-rows.mjs +101 -0
- package/server/public/app/action-confirmations.js +12 -15
- package/server/public/app/api-client.js +60 -4
- package/server/public/app/ask-user-view.js +29 -4
- package/server/public/app/background-tasks-view.mjs +56 -7
- package/server/public/app/bootstrap.js +7 -0
- package/server/public/app/conversation-preferences.mjs +10 -4
- package/server/public/app/conversation-provider-indicator.mjs +18 -8
- package/server/public/app/conversation-view.js +33 -9
- package/server/public/app/host-suspend-ui.js +45 -44
- package/server/public/app/journal-view.js +9 -3
- package/server/public/app/push-settings.js +1 -0
- package/server/public/app/question-answer-rule.mjs +15 -0
- package/server/public/app/question-multi-select.mjs +9 -2
- package/server/public/app/relay-event-view.mjs +5 -1
- package/server/public/app/relay-prompt-sanitizer.mjs +9 -2
- package/server/public/app/relay-restart-note-view.mjs +36 -0
- package/server/public/app/relay-restart-ui.js +568 -0
- package/server/public/app/settings-modal.js +52 -0
- package/server/public/app/socket-handlers.js +1 -1
- package/server/public/app/task-console-render.mjs +171 -7
- package/server/public/app/task-console.mjs +111 -11
- package/server/public/app/task-output-text.mjs +15 -6
- package/server/public/app/transcript-jump-bottom-view.mjs +5 -1
- package/server/public/app/transcript-jump-bottom.mjs +20 -0
- package/server/public/index.html +76 -5
- package/server/relay-exit-codes.mjs +6 -0
- package/server/relay-self-restart.mjs +6 -1
- package/server/relay-tools.md +17 -9
- package/server/repositories/message-repository.mjs +32 -28
- package/server/repositories/remote-relay-repository.mjs +4 -2
- package/server/repositories/subagent-timeline-statements.mjs +2 -0
- package/server/routes/ask-user-routes.mjs +41 -7
- package/server/routes/messages-routes.mjs +220 -37
- package/server/routes/preview-routes.mjs +73 -7
- package/server/routes/relay-board-routes.mjs +19 -2
- package/server/routes/relay-restart-routes.mjs +73 -0
- package/server/routes/sessions-routes.mjs +79 -5
- package/server/server-runtime.mjs +335 -218
- package/server/services/about-service.mjs +13 -12
- package/server/services/background-task-persistence.mjs +133 -0
- package/server/services/background-task-store.mjs +240 -13
- package/server/services/conversation-model-fallback.mjs +57 -0
- package/server/services/cursor-session-root-service.mjs +70 -6
- package/server/services/data-dir-resolution.mjs +208 -19
- package/server/services/earlier-database-service.mjs +631 -0
- package/server/services/host-suspend-command.mjs +2 -1
- package/server/services/oar-cli-helpers.mjs +6 -1
- package/server/services/oar-state-layout.mjs +107 -0
- package/server/services/oar-state-migration-service.mjs +387 -96
- package/server/services/package-data-rescue-preinstall.mjs +22 -0
- package/server/services/package-data-rescue.mjs +137 -0
- package/server/services/preview-proxy-server.mjs +14 -1
- package/server/services/preview-registry-service.mjs +3 -2
- package/server/services/process-list-service.mjs +2 -1
- package/server/services/provider-model-selection.mjs +195 -0
- package/server/services/push-dispatch-service.mjs +42 -8
- package/server/services/relay-event-service.mjs +210 -14
- package/server/services/relay-prompt-sanitizer.mjs +7 -0
- package/server/services/relay-restart-picture.mjs +209 -0
- package/server/services/relay-restart-queue.mjs +387 -0
- package/server/services/relay-shutdown-gate.mjs +4 -1
- package/server/services/relay-singleton-guard.mjs +300 -36
- package/server/services/remote-relay-caller-context.mjs +7 -2
- package/server/services/remote-relay-dispatcher.mjs +217 -28
- package/server/services/restart-resume-service.mjs +303 -0
- package/server/services/session-activity-tracker.mjs +52 -15
- package/server/services/session-worker-launch-service.mjs +31 -1
- package/server/services/session-worker-process-service.mjs +6 -4
- package/server/services/stale-turn-recovery.mjs +99 -0
- package/server/services/subagent-timeline-service.mjs +4 -1
- package/server/services/task-output-service.mjs +217 -18
- package/server/services/windows-autostart-service.mjs +329 -31
- package/server/services/windows-boot-autostart-service.mjs +188 -60
- package/shared/preview-instructions.mjs +17 -12
- package/shared/preview-tool-core.mjs +32 -14
- package/shared/question-answer-rule.mjs +63 -0
- package/shared/relay-events.mjs +106 -3
- package/shared/remote-relay-contract.mjs +32 -15
- package/shared/subagent-timeline.mjs +4 -0
- package/shared/task-output-text.mjs +196 -83
- package/shared/windows-system-tools.mjs +25 -0
- package/shared/worker-runtime/config-loader.mjs +54 -8
- package/shared/worker-runtime/prompt-context.mjs +3 -1
- package/shared/worker-runtime/worker-websocket-link.mjs +96 -20
- package/shared/worker-runtime/working-directory.mjs +65 -0
- package/server/uploads/image_input.md +0 -540
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* "After a relay restart, tell interrupted sessions to carry on" (opt-in,
|
|
5
|
+
* per relay, off by default).
|
|
6
|
+
*
|
|
7
|
+
* Session workers outlive the relay's runtime, and a turn whose worker lived
|
|
8
|
+
* through the restart simply goes on. A turn whose worker did not (it died
|
|
9
|
+
* with the relay, or came back without the turn) is dealt with by the
|
|
10
|
+
* recovery paths, and most of them requeue it: the row runs again by itself,
|
|
11
|
+
* and nothing more is needed. A few cannot run again, because the prompt
|
|
12
|
+
* already reached the agent and a second delivery would run it twice:
|
|
13
|
+
*
|
|
14
|
+
* - a background continuation (a turn the agent opened itself), torn down;
|
|
15
|
+
* - a steer the CLI had already taken (`consumed_at`), failed;
|
|
16
|
+
* - a turn that had started (activity, a stream) when its live worker no
|
|
17
|
+
* longer claimed it, failed instead of replayed (`turn-replay-blocked`).
|
|
18
|
+
*
|
|
19
|
+
* The agent never hears that its turn stopped. With the setting on, such a
|
|
20
|
+
* session gets ONE note of the relay's in the conversation once its rows are
|
|
21
|
+
* settled: the relay restarted, the turn was interrupted, check what you
|
|
22
|
+
* were doing and carry on. The user's message is never sent again.
|
|
23
|
+
*
|
|
24
|
+
* Which turns count: the rows that were `processing` when this runtime
|
|
25
|
+
* started (`captureBoot`), followed through the recovery paths
|
|
26
|
+
* (`noteOutcome`). A row its worker claims after the start (a heartbeat
|
|
27
|
+
* that lists it: the worker lived through the restart and has the turn) is
|
|
28
|
+
* the restart's no more: whatever ends it later is not put down to it.
|
|
29
|
+
* A conversation gets a note when one of its rows was cut off and none was
|
|
30
|
+
* requeued (that one runs again: a note would double the work), claimed by
|
|
31
|
+
* a worker that lived through the restart, or failed for another reason.
|
|
32
|
+
* Nothing for idle sessions, nothing when
|
|
33
|
+
* the setting is off when the note would be sent, nothing for an archived or
|
|
34
|
+
* deleted conversation, nothing while a usage-limit follow-up is held (it
|
|
35
|
+
* carries the turn on itself), at most one note per conversation and
|
|
36
|
+
* runtime. The note is an ordinary pending row, queued behind whatever the
|
|
37
|
+
* user has queued already, so it never jumps ahead of a user message; its
|
|
38
|
+
* delivery starts the session's worker as for any message.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { randomUUID } from 'node:crypto';
|
|
42
|
+
|
|
43
|
+
export const RESTART_RESUME_SETTING_KEY = 'restart_resume_notes';
|
|
44
|
+
export const RESTART_RESUME_KIND = 'relay-restart-resume';
|
|
45
|
+
export const RESTART_RESUME_LEAD = 'Automatic message from OAR, not from the user:';
|
|
46
|
+
|
|
47
|
+
// How long after the start a cut-off turn is still put down to the restart.
|
|
48
|
+
const DEFAULT_WINDOW_MS = 30 * 60_000;
|
|
49
|
+
|
|
50
|
+
/** The stored value as on or off; off unless it was switched on. */
|
|
51
|
+
export function readRestartResumeSetting(raw) {
|
|
52
|
+
return ['1', 'true', 'on'].includes(String(raw ?? '').trim().toLowerCase());
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function parseRestartResumeUpdate(value) {
|
|
56
|
+
if (typeof value !== 'boolean') return { ok: false, error: 'enabled must be true or false' };
|
|
57
|
+
return { ok: true, enabled: value };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function defaultFormatTime(ms) {
|
|
61
|
+
const at = new Date(Number(ms));
|
|
62
|
+
if (!Number.isFinite(at.getTime())) return 'just now';
|
|
63
|
+
return `${at.toISOString().slice(0, 16).replace('T', ' ')} UTC`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The note's text. */
|
|
67
|
+
export function buildRestartResumeNoteText({ restartedAt, formatTime = defaultFormatTime } = {}) {
|
|
68
|
+
return `${RESTART_RESUME_LEAD} the relay restarted at ${formatTime(restartedAt)}, and the turn you were working on was interrupted before it finished. `
|
|
69
|
+
+ 'Check where you were (the files you changed, the commands and background tasks you started) and carry on from there. '
|
|
70
|
+
+ 'Do not repeat steps that already completed; if the work was already done, say so briefly.';
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const OUTCOMES = new Set(['cut', 'requeued', 'failed', 'claimed']);
|
|
74
|
+
|
|
75
|
+
function text(value) {
|
|
76
|
+
return String(value ?? '').trim();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function createRestartResumeService({
|
|
80
|
+
db,
|
|
81
|
+
stmts,
|
|
82
|
+
isEnabled = () => false,
|
|
83
|
+
emit = () => {},
|
|
84
|
+
wake = () => {},
|
|
85
|
+
now = () => Date.now(),
|
|
86
|
+
bootAt = now(),
|
|
87
|
+
windowMs = DEFAULT_WINDOW_MS,
|
|
88
|
+
formatTime = defaultFormatTime,
|
|
89
|
+
logger = console,
|
|
90
|
+
} = {}) {
|
|
91
|
+
if (!db || !stmts) throw new Error('createRestartResumeService requires db and stmts');
|
|
92
|
+
const listProcessing = db.prepare(`
|
|
93
|
+
SELECT id, conversation_id, model, model_variant_id, reasoning_effort, context_tier, relay_mode,
|
|
94
|
+
owner_sdk_session_id, runtime_session_id
|
|
95
|
+
FROM queue WHERE status = 'processing'
|
|
96
|
+
`);
|
|
97
|
+
const statusOf = db.prepare(`SELECT status FROM queue WHERE id = ?`);
|
|
98
|
+
let heldPause = null;
|
|
99
|
+
try {
|
|
100
|
+
// A held usage-limit follow-up (usage-limit-pause-service.mjs).
|
|
101
|
+
heldPause = db.prepare(`
|
|
102
|
+
SELECT 1 FROM queue
|
|
103
|
+
WHERE conversation_id = ? AND status = 'pending' AND usage_limit_pause IS NOT NULL
|
|
104
|
+
LIMIT 1
|
|
105
|
+
`);
|
|
106
|
+
} catch {}
|
|
107
|
+
|
|
108
|
+
// queue id → conversation id, for the rows processing at the start.
|
|
109
|
+
const watched = new Map();
|
|
110
|
+
// conversation id → { rows: Map(queue id → { row, outcome }) }
|
|
111
|
+
const conversations = new Map();
|
|
112
|
+
// Conversations that had their note in this runtime.
|
|
113
|
+
const noted = new Set();
|
|
114
|
+
|
|
115
|
+
function log(line) {
|
|
116
|
+
try { logger?.log?.(`[restart-resume] ${line}`); } catch {}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function expired() {
|
|
120
|
+
return now() - bootAt > windowMs;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The rows processing when this runtime started; returns how many. */
|
|
124
|
+
function captureBoot() {
|
|
125
|
+
let rows = [];
|
|
126
|
+
try { rows = listProcessing.all(); } catch {}
|
|
127
|
+
for (const row of rows) {
|
|
128
|
+
const conversationId = text(row.conversation_id);
|
|
129
|
+
const id = text(row.id);
|
|
130
|
+
if (!conversationId || !id) continue;
|
|
131
|
+
watched.set(id, conversationId);
|
|
132
|
+
let entry = conversations.get(conversationId);
|
|
133
|
+
if (!entry) conversations.set(conversationId, entry = { rows: new Map() });
|
|
134
|
+
entry.rows.set(id, { row, outcome: null });
|
|
135
|
+
}
|
|
136
|
+
if (rows.length) log(`${rows.length} turn(s) were running when the relay started; following them`);
|
|
137
|
+
return rows.length;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* What a recovery path did with a row: `cut` (it cannot run again),
|
|
142
|
+
* `requeued` (it runs again by itself) or `failed` (ended for another
|
|
143
|
+
* reason); or `claimed`: its worker, alive after the restart, holds it.
|
|
144
|
+
* Only the rows of the start count, and only their first outcome.
|
|
145
|
+
*/
|
|
146
|
+
function noteOutcome(row, outcome) {
|
|
147
|
+
const id = text(row?.id ?? row);
|
|
148
|
+
const conversationId = watched.get(id);
|
|
149
|
+
if (!conversationId || !OUTCOMES.has(outcome)) return false;
|
|
150
|
+
watched.delete(id);
|
|
151
|
+
const entry = conversations.get(conversationId)?.rows.get(id);
|
|
152
|
+
if (entry && !entry.outcome) entry.outcome = outcome;
|
|
153
|
+
settle(conversationId);
|
|
154
|
+
return true;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Decides a conversation once none of its rows of the start is processing.
|
|
158
|
+
function settle(conversationId) {
|
|
159
|
+
const entry = conversations.get(conversationId);
|
|
160
|
+
if (!entry) return;
|
|
161
|
+
for (const [id, item] of entry.rows) {
|
|
162
|
+
if (item.outcome) continue;
|
|
163
|
+
let status = '';
|
|
164
|
+
try { status = text(statusOf.get(id)?.status).toLowerCase(); } catch {}
|
|
165
|
+
if (status === 'processing') return;
|
|
166
|
+
// Back in the queue by a path that did not say so: it runs again.
|
|
167
|
+
item.outcome = status === 'pending' || status === 'parked' ? 'requeued' : 'settled';
|
|
168
|
+
watched.delete(id);
|
|
169
|
+
}
|
|
170
|
+
conversations.delete(conversationId);
|
|
171
|
+
const outcomes = [...entry.rows.values()].map((item) => item.outcome);
|
|
172
|
+
if (!outcomes.includes('cut')) return;
|
|
173
|
+
if (outcomes.includes('requeued')) {
|
|
174
|
+
log(`conv=${conversationId.slice(0, 8)} no note: the interrupted turn runs again by itself`);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
if (outcomes.includes('claimed')) {
|
|
178
|
+
log(`conv=${conversationId.slice(0, 8)} no note: its worker lived through the restart and has its turn`);
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (outcomes.includes('failed')) {
|
|
182
|
+
log(`conv=${conversationId.slice(0, 8)} no note: the turn failed, the failure says what to do`);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
const cut = [...entry.rows.values()].find((item) => item.outcome === 'cut')?.row || null;
|
|
186
|
+
sendNote(conversationId, cut);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function skip(conversationId, why) {
|
|
190
|
+
log(`conv=${conversationId.slice(0, 8)} no note: ${why}`);
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function sendNote(conversationId, cutRow) {
|
|
195
|
+
if (noted.has(conversationId)) return skip(conversationId, 'it had one after this restart');
|
|
196
|
+
let enabled = false;
|
|
197
|
+
try { enabled = isEnabled() === true; } catch {}
|
|
198
|
+
if (!enabled) return skip(conversationId, 'the setting is off');
|
|
199
|
+
const conversation = stmts.getConvAnyStatus?.get?.(conversationId) || null;
|
|
200
|
+
if (!conversation || text(conversation.status) === 'deleted') return skip(conversationId, 'the conversation is gone');
|
|
201
|
+
if (Number(conversation.archived) === 1) return skip(conversationId, 'the conversation is archived');
|
|
202
|
+
try {
|
|
203
|
+
if (heldPause?.get(conversationId)) return skip(conversationId, 'a usage-limit follow-up carries the turn on');
|
|
204
|
+
} catch {}
|
|
205
|
+
noted.add(conversationId);
|
|
206
|
+
|
|
207
|
+
const at = new Date(now()).toISOString();
|
|
208
|
+
const messageId = randomUUID();
|
|
209
|
+
const noteText = buildRestartResumeNoteText({ restartedAt: bootAt, formatTime });
|
|
210
|
+
const runtimeSession = stmts.getRuntimeSessionByConversation?.get?.(conversationId) || null;
|
|
211
|
+
const latestModel = stmts.getLatestConversationModel?.get?.(conversationId)?.model || null;
|
|
212
|
+
// The model and mode the interrupted turn ran with.
|
|
213
|
+
const model = text(cutRow?.model) || text(latestModel) || text(runtimeSession?.provider_model) || text(runtimeSession?.model) || null;
|
|
214
|
+
const modelVariant = text(cutRow?.model_variant_id) || model;
|
|
215
|
+
const relayMode = text(cutRow?.relay_mode) || 'agent';
|
|
216
|
+
const owner = text(cutRow?.owner_sdk_session_id) || text(runtimeSession?.sdk_session_id) || null;
|
|
217
|
+
const write = db.transaction(() => {
|
|
218
|
+
stmts.insertMsg.run(
|
|
219
|
+
messageId,
|
|
220
|
+
conversationId,
|
|
221
|
+
'user',
|
|
222
|
+
noteText,
|
|
223
|
+
modelVariant || null,
|
|
224
|
+
relayMode,
|
|
225
|
+
null,
|
|
226
|
+
at,
|
|
227
|
+
modelVariant || null,
|
|
228
|
+
null,
|
|
229
|
+
text(modelVariant).toLowerCase() === 'auto' ? 'auto' : 'manual',
|
|
230
|
+
);
|
|
231
|
+
stmts.setMessageKind?.run?.(RESTART_RESUME_KIND, messageId);
|
|
232
|
+
stmts.updateConvTime?.run?.(at, conversationId);
|
|
233
|
+
const queueArgs = [
|
|
234
|
+
messageId,
|
|
235
|
+
conversationId,
|
|
236
|
+
text(cutRow?.runtime_session_id) || runtimeSession?.id || null,
|
|
237
|
+
0,
|
|
238
|
+
model,
|
|
239
|
+
modelVariant || null,
|
|
240
|
+
text(cutRow?.reasoning_effort) || null,
|
|
241
|
+
text(cutRow?.context_tier) || null,
|
|
242
|
+
relayMode,
|
|
243
|
+
noteText,
|
|
244
|
+
null,
|
|
245
|
+
at,
|
|
246
|
+
owner,
|
|
247
|
+
owner ? at : null,
|
|
248
|
+
null,
|
|
249
|
+
null,
|
|
250
|
+
];
|
|
251
|
+
if (stmts.queueHasImageOperationId || stmts.insertQ?.source?.includes?.('image_operation_id')) queueArgs.push(null);
|
|
252
|
+
stmts.insertQ.run(...queueArgs);
|
|
253
|
+
});
|
|
254
|
+
try {
|
|
255
|
+
write();
|
|
256
|
+
} catch (error) {
|
|
257
|
+
noted.delete(conversationId);
|
|
258
|
+
try { logger?.warn?.(`[restart-resume] conv=${conversationId.slice(0, 8)} note not queued: ${error?.message || error}`); } catch {}
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
try {
|
|
262
|
+
emit('user_message', {
|
|
263
|
+
conversationId,
|
|
264
|
+
messageId,
|
|
265
|
+
senderClientId: null,
|
|
266
|
+
message: {
|
|
267
|
+
role: 'user',
|
|
268
|
+
text: noteText,
|
|
269
|
+
model: modelVariant || null,
|
|
270
|
+
mode: relayMode,
|
|
271
|
+
attachments: [],
|
|
272
|
+
timestamp: at,
|
|
273
|
+
kind: RESTART_RESUME_KIND,
|
|
274
|
+
},
|
|
275
|
+
});
|
|
276
|
+
} catch {}
|
|
277
|
+
try { emit('message_status', { messageId, conversationId, status: 'pending' }); } catch {}
|
|
278
|
+
try { wake({ conversationId, ownerSessionId: owner, messageId }); } catch {}
|
|
279
|
+
log(`conv=${conversationId.slice(0, 8)} note queued: its turn was cut off by the restart`);
|
|
280
|
+
return messageId;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Decides the conversations whose rows settled on their own; ends the watch after the window. */
|
|
284
|
+
function tick() {
|
|
285
|
+
if (!conversations.size) return;
|
|
286
|
+
if (expired()) {
|
|
287
|
+
watched.clear();
|
|
288
|
+
conversations.clear();
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
for (const conversationId of [...conversations.keys()]) settle(conversationId);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
return {
|
|
295
|
+
captureBoot,
|
|
296
|
+
noteOutcome,
|
|
297
|
+
tick,
|
|
298
|
+
/** What is still followed, for tests and diagnostics. */
|
|
299
|
+
pending() {
|
|
300
|
+
return [...conversations.keys()];
|
|
301
|
+
},
|
|
302
|
+
};
|
|
303
|
+
}
|
|
@@ -25,7 +25,9 @@
|
|
|
25
25
|
* - `noteTaskSet` — every POST /api/background-tasks (REPLACE semantics):
|
|
26
26
|
* a new task id is a start, a task gone from the set has ended, a changed
|
|
27
27
|
* task (tool call, summary, tokens) is activity. A task that started
|
|
28
|
-
* before this runtime did was announced by the runtime before it.
|
|
28
|
+
* before this runtime did was announced by the runtime before it. A task
|
|
29
|
+
* that ended and is listed under another run (`taskRunId`: a resumed
|
|
30
|
+
* Claude sub-agent keeps its task id) is a start again.
|
|
29
31
|
* - `noteTaskSettled` — how a task ended, where the worker says so (the
|
|
30
32
|
* Claude worker's task_notification). A task this runtime never knew
|
|
31
33
|
* still gets its end event when the worker describes it.
|
|
@@ -168,6 +170,11 @@ export function createSessionActivityTracker({
|
|
|
168
170
|
const sessions = new Map();
|
|
169
171
|
// Edges already reported for an end, by `${conversationId}:${key}`.
|
|
170
172
|
const endedKeys = new Map();
|
|
173
|
+
// `${conversationId}:${taskId}` → the run of the task that ended (the
|
|
174
|
+
// SDK's run id, `taskRunId`), where the worker names one. A resumed Claude
|
|
175
|
+
// sub-agent keeps its task id and runs again under a new run id: a set
|
|
176
|
+
// that lists it with another run is a new start, not a late set.
|
|
177
|
+
const endedRuns = new Map();
|
|
171
178
|
// `${conversationId}:${taskId}` of the ends nobody reported: the task was
|
|
172
179
|
// only missing from a set for the grace. A later set that lists it again
|
|
173
180
|
// says it never ended.
|
|
@@ -216,16 +223,25 @@ export function createSessionActivityTracker({
|
|
|
216
223
|
for (const key of [task.taskId, task.toolUseId ? `run:${task.toolUseId}` : '']) {
|
|
217
224
|
if (key) endedKeys.set(`${conversationId}:${key}`, at);
|
|
218
225
|
}
|
|
226
|
+
if (task.taskRunId) endedRuns.set(`${conversationId}:${task.taskId}`, task.taskRunId);
|
|
227
|
+
else endedRuns.delete(`${conversationId}:${task.taskId}`);
|
|
219
228
|
if (endedKeys.size > 2000) {
|
|
220
229
|
for (const [key, when] of endedKeys) {
|
|
221
230
|
if (at - when > RECENT_TASK_TTL_MS) {
|
|
222
231
|
endedKeys.delete(key);
|
|
223
232
|
endedByAbsence.delete(key);
|
|
233
|
+
endedRuns.delete(key);
|
|
224
234
|
}
|
|
225
235
|
}
|
|
226
236
|
}
|
|
227
237
|
}
|
|
228
238
|
|
|
239
|
+
/** The set lists an ended task under another run: it was resumed. */
|
|
240
|
+
function isNewRun(conversationId, taskId, taskRunId) {
|
|
241
|
+
const ended = endedRuns.get(`${conversationId}:${taskId}`);
|
|
242
|
+
return !!taskRunId && !!ended && ended !== taskRunId;
|
|
243
|
+
}
|
|
244
|
+
|
|
229
245
|
function wasEnded(conversationId, task) {
|
|
230
246
|
return [task.taskId, task.toolUseId ? `run:${task.toolUseId}` : '']
|
|
231
247
|
.some((key) => key && endedKeys.has(`${conversationId}:${key}`));
|
|
@@ -308,6 +324,7 @@ export function createSessionActivityTracker({
|
|
|
308
324
|
/** Takes back an end that was only an absence; the task's old row goes from the recent ones. */
|
|
309
325
|
function unEnd(conversationId, entry, taskId, toolUseId) {
|
|
310
326
|
endedByAbsence.delete(`${conversationId}:${taskId}`);
|
|
327
|
+
endedRuns.delete(`${conversationId}:${taskId}`);
|
|
311
328
|
const index = entry.recent.findIndex((task) => task.taskId === taskId);
|
|
312
329
|
const [old] = index >= 0 ? entry.recent.splice(index, 1) : [null];
|
|
313
330
|
for (const key of [taskId, toolUseId ? `run:${toolUseId}` : '', old?.toolUseId ? `run:${old.toolUseId}` : '']) {
|
|
@@ -332,6 +349,7 @@ export function createSessionActivityTracker({
|
|
|
332
349
|
seen.add(taskId);
|
|
333
350
|
const signature = taskSignature(raw);
|
|
334
351
|
const known = entry.live.get(taskId);
|
|
352
|
+
const taskRunId = toText(raw.taskRunId);
|
|
335
353
|
if (known && !known.leftAtMs) {
|
|
336
354
|
if (known.signature !== signature) {
|
|
337
355
|
known.signature = signature;
|
|
@@ -340,32 +358,45 @@ export function createSessionActivityTracker({
|
|
|
340
358
|
}
|
|
341
359
|
known.description = toText(raw.description) || known.description;
|
|
342
360
|
known.toolUseId = toText(raw.toolUseId) || known.toolUseId;
|
|
361
|
+
known.taskRunId = taskRunId || known.taskRunId;
|
|
343
362
|
known.summary = toText(raw.summary) || known.summary;
|
|
344
363
|
known.model = toText(raw.model) || known.model;
|
|
345
364
|
continue;
|
|
346
365
|
}
|
|
347
366
|
if (known?.leftAtMs) {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
367
|
+
if (!taskRunId || !known.taskRunId || known.taskRunId === taskRunId) {
|
|
368
|
+
// Back in the set after it left: it did not end after all.
|
|
369
|
+
if (known.graceTimer) clearTimer(known.graceTimer);
|
|
370
|
+
known.graceTimer = null;
|
|
371
|
+
known.leftAtMs = 0;
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
// Back under another run (resumed within the grace): the run that
|
|
375
|
+
// left ended, how is not known; the new one starts below.
|
|
376
|
+
finishTask(id, known, { status: 'completed', reported: false });
|
|
353
377
|
}
|
|
354
378
|
let revived = null;
|
|
355
379
|
if (wasEnded(id, { taskId, toolUseId: toText(raw.toolUseId) })) {
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
380
|
+
if (isNewRun(id, taskId, taskRunId)) {
|
|
381
|
+
// Resumed: the same task, a new run. Its first run's end stays
|
|
382
|
+
// reported; this run starts (and will end) on its own.
|
|
383
|
+
unEnd(id, entry, taskId, toText(raw.toolUseId));
|
|
384
|
+
} else {
|
|
385
|
+
// Settled already (its notice came before the set dropped it).
|
|
386
|
+
if (!endedByAbsence.has(`${id}:${taskId}`)) continue;
|
|
387
|
+
// Ended only because a set did not list it (a set that arrived out
|
|
388
|
+
// of turn, a worker that left it out for a moment) and listed
|
|
389
|
+
// again: it never ended. It comes back as the task it was, with no
|
|
390
|
+
// second start. The end that was reported for it stays reported;
|
|
391
|
+
// its real end will be another.
|
|
392
|
+
revived = unEnd(id, entry, taskId, toText(raw.toolUseId)) || {};
|
|
393
|
+
}
|
|
364
394
|
}
|
|
365
395
|
const taskStartedAtMs = msOf(raw.startedAt);
|
|
366
396
|
const task = {
|
|
367
397
|
taskId,
|
|
368
398
|
toolUseId: toText(raw.toolUseId) || null,
|
|
399
|
+
taskRunId: taskRunId || null,
|
|
369
400
|
taskType: toText(raw.taskType),
|
|
370
401
|
kind: taskKindOf(raw.taskType),
|
|
371
402
|
description: toText(raw.description).slice(0, 500),
|
|
@@ -395,16 +426,21 @@ export function createSessionActivityTracker({
|
|
|
395
426
|
}
|
|
396
427
|
|
|
397
428
|
function noteTaskSettled(conversationId, {
|
|
398
|
-
taskId, toolUseId, status, summary, exitCode, description, taskType, startedAt,
|
|
429
|
+
taskId, toolUseId, taskRunId, status, summary, exitCode, description, taskType, startedAt,
|
|
399
430
|
} = {}) {
|
|
400
431
|
const id = toText(conversationId);
|
|
401
432
|
if (!id) return;
|
|
402
433
|
const entry = session(id);
|
|
403
434
|
const key = toText(taskId);
|
|
404
435
|
const run = toText(toolUseId);
|
|
436
|
+
const settledRun = toText(taskRunId);
|
|
405
437
|
const task = (key && entry.live.get(key))
|
|
406
438
|
|| [...entry.live.values()].find((candidate) => run && candidate.toolUseId === run)
|
|
407
439
|
|| null;
|
|
440
|
+
// The end of an earlier run of a task that was resumed since (a notice
|
|
441
|
+
// the worker sent again late): the run that is live now goes on.
|
|
442
|
+
if (task && settledRun && task.taskRunId && task.taskRunId !== settledRun) return;
|
|
443
|
+
if (task && settledRun) task.taskRunId = settledRun;
|
|
408
444
|
const settled = {
|
|
409
445
|
status: normalizeTaskStatus(status || 'completed'),
|
|
410
446
|
summary: toText(summary) || null,
|
|
@@ -439,6 +475,7 @@ export function createSessionActivityTracker({
|
|
|
439
475
|
const unknown = {
|
|
440
476
|
taskId: key,
|
|
441
477
|
toolUseId: runId || null,
|
|
478
|
+
taskRunId: settledRun || null,
|
|
442
479
|
taskType: toText(taskType),
|
|
443
480
|
kind: taskKindOf(taskType),
|
|
444
481
|
description: toText(description).slice(0, 500),
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
isOarMcpServerDisabled,
|
|
15
15
|
} from '../mcp/oar-mcp-server.mjs';
|
|
16
16
|
import { WORKER_LOG_FILE_ENV } from '../../shared/worker-runtime/worker-log-file.mjs';
|
|
17
|
+
import { missingWorkingDirectoryError } from '../../shared/worker-runtime/working-directory.mjs';
|
|
17
18
|
import {
|
|
18
19
|
describeSpawnEnvironment,
|
|
19
20
|
envPathKey,
|
|
@@ -213,6 +214,9 @@ export function applyCursorProviderEnvironment(env = {}, {
|
|
|
213
214
|
enabled = false,
|
|
214
215
|
model = '',
|
|
215
216
|
apiKey = '',
|
|
217
|
+
// The relay's resolved, absolute data dir: the worker keeps its agent stores
|
|
218
|
+
// there, and must not resolve a relative one or the config's own again.
|
|
219
|
+
dataDir = '',
|
|
216
220
|
} = {}) {
|
|
217
221
|
const next = { ...env };
|
|
218
222
|
if (resolveWorkerKind(next) === 'cursor') delete next.COPILOT_WEB_RELAY_WORKER_KIND;
|
|
@@ -225,6 +229,8 @@ export function applyCursorProviderEnvironment(env = {}, {
|
|
|
225
229
|
next.CURSOR_API_KEY = normalizedApiKey;
|
|
226
230
|
const normalizedModel = normalizeText(model);
|
|
227
231
|
if (normalizedModel) next.CURSOR_RELAY_MODEL = normalizedModel;
|
|
232
|
+
const normalizedDataDir = normalizeText(dataDir);
|
|
233
|
+
if (normalizedDataDir) next.COPILOT_WEB_RELAY_DATA_DIR = path.resolve(normalizedDataDir);
|
|
228
234
|
return next;
|
|
229
235
|
}
|
|
230
236
|
|
|
@@ -717,6 +723,9 @@ export function buildTmuxWorkerShellCommand(targetSessionId, env = {}, {
|
|
|
717
723
|
'COPILOT_WEB_RELAY_CONFIG',
|
|
718
724
|
'COPILOT_WEB_RELAY_TOOLS',
|
|
719
725
|
'COPILOT_WEB_RELAY_LOG_DIR',
|
|
726
|
+
// The Cursor worker keeps its agent stores in the relay's data dir
|
|
727
|
+
// (applyCursorProviderEnvironment).
|
|
728
|
+
'COPILOT_WEB_RELAY_DATA_DIR',
|
|
720
729
|
'COPILOT_WEB_RELAY_CLI_EXECUTABLE',
|
|
721
730
|
'COPILOT_WEB_RELAY_EXTENSION_BOOTSTRAP_PATH',
|
|
722
731
|
'COPILOT_SDK_PATH',
|
|
@@ -847,6 +856,19 @@ export function describeWindowsStartFailure(state, { spawnCommand = 'cmd.exe', p
|
|
|
847
856
|
return null;
|
|
848
857
|
}
|
|
849
858
|
|
|
859
|
+
/**
|
|
860
|
+
* A spawn whose folder is gone fails with ENOENT (or ENOTDIR), which reads as
|
|
861
|
+
* a missing program (`node`, `cmd.exe`). Asked only after a spawn failed: the folder
|
|
862
|
+
* is named instead, or null when it is there. Exported for the tests.
|
|
863
|
+
* (A tmux launch never gets here: `new-session -c` falls back to the home
|
|
864
|
+
* folder, and the worker reports the folder on the turn.)
|
|
865
|
+
*/
|
|
866
|
+
export function describeLaunchFolderFailure(state, folder, { check = missingWorkingDirectoryError } = {}) {
|
|
867
|
+
if (state?.error?.code !== 'ENOENT' && state?.error?.code !== 'ENOTDIR') return null;
|
|
868
|
+
const missing = check(folder);
|
|
869
|
+
return missing ? `worker-spawn-failed: ${missing.message}` : null;
|
|
870
|
+
}
|
|
871
|
+
|
|
850
872
|
function buildWorkerLaunchEnv({ processCwd, workspaceRoot, env = process.env } = {}) {
|
|
851
873
|
const launchProcessCwd = String(processCwd || '').trim();
|
|
852
874
|
const launchWorkspaceRoot = String(workspaceRoot || '').trim() || launchProcessCwd;
|
|
@@ -900,6 +922,8 @@ export async function launchSessionCli({
|
|
|
900
922
|
// Where a launch that went wrong explains itself (one line per call).
|
|
901
923
|
log = null,
|
|
902
924
|
locateExecutableImpl = locateExecutable,
|
|
925
|
+
// Asked after a failed spawn whether its folder is gone (the error, or null).
|
|
926
|
+
checkWorkingDirectoryImpl = missingWorkingDirectoryError,
|
|
903
927
|
} = {}) {
|
|
904
928
|
const target = String(targetSessionId || '').trim();
|
|
905
929
|
if (!target) throw new Error('missing-target-session-id');
|
|
@@ -1154,7 +1178,8 @@ export async function launchSessionCli({
|
|
|
1154
1178
|
// `cmd /c start` returns at once when the program started. Still running,
|
|
1155
1179
|
// or ended with an error, means it did not: name the program, and log the
|
|
1156
1180
|
// command and the PATH it was looked up on.
|
|
1157
|
-
const failure =
|
|
1181
|
+
const failure = describeLaunchFolderFailure(childState, launchProcessCwd, { check: checkWorkingDirectoryImpl })
|
|
1182
|
+
|| describeWindowsStartFailure(childState, { spawnCommand, program: startedProgram });
|
|
1158
1183
|
if (failure) {
|
|
1159
1184
|
if (childState.tracked && !childState.exited) {
|
|
1160
1185
|
try { child.kill?.(); } catch {}
|
|
@@ -1176,6 +1201,11 @@ export async function launchSessionCli({
|
|
|
1176
1201
|
// A failed spawn reports its error on the next tick.
|
|
1177
1202
|
await new Promise((resolve) => setImmediate(resolve));
|
|
1178
1203
|
const reason = childState.error?.code || childState.error?.message || '';
|
|
1204
|
+
const folderFailure = describeLaunchFolderFailure(childState, launchProcessCwd, { check: checkWorkingDirectoryImpl });
|
|
1205
|
+
if (folderFailure) {
|
|
1206
|
+
explain(`could not start ${spawnCommand}: ${folderFailure}`);
|
|
1207
|
+
throw new Error(folderFailure);
|
|
1208
|
+
}
|
|
1179
1209
|
explain(`could not start ${spawnCommand}${reason ? ` (${reason})` : ''}; ${spawnEnvironmentText()}`);
|
|
1180
1210
|
throw new Error(`worker-spawn-unhealthy:missing-pid${reason ? ` (${spawnCommand}: ${reason})` : ''}`);
|
|
1181
1211
|
}
|
|
@@ -296,6 +296,8 @@ export function createSessionWorkerProcessInspector({
|
|
|
296
296
|
platform = process.platform,
|
|
297
297
|
execFileSyncImpl = execFileSync,
|
|
298
298
|
execFileImpl = execFile,
|
|
299
|
+
// Where Windows keeps PowerShell (the bare name when it is not there).
|
|
300
|
+
systemToolPath = windowsSystemToolPath,
|
|
299
301
|
now = () => Date.now(),
|
|
300
302
|
snapshotCacheMs = SNAPSHOT_CACHE_MS,
|
|
301
303
|
slowReadMs = SLOW_READ_MS,
|
|
@@ -320,7 +322,7 @@ export function createSessionWorkerProcessInspector({
|
|
|
320
322
|
|
|
321
323
|
function getWindowsProcessSnapshot() {
|
|
322
324
|
if (platform !== 'win32') return [];
|
|
323
|
-
const output = execFileSyncImpl(
|
|
325
|
+
const output = execFileSyncImpl(systemToolPath('powershell.exe'), ['-NoProfile', '-Command', WINDOWS_PROCESS_SNAPSHOT_SCRIPT], {
|
|
324
326
|
// stderr is kept (not shown): a failed read reports it.
|
|
325
327
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
326
328
|
windowsHide: true,
|
|
@@ -376,7 +378,7 @@ export function createSessionWorkerProcessInspector({
|
|
|
376
378
|
if (platform !== 'win32') return Promise.resolve([]);
|
|
377
379
|
return new Promise((resolve, reject) => {
|
|
378
380
|
execFileImpl(
|
|
379
|
-
|
|
381
|
+
systemToolPath('powershell.exe'),
|
|
380
382
|
['-NoProfile', '-Command', WINDOWS_PROCESS_SNAPSHOT_SCRIPT],
|
|
381
383
|
{ windowsHide: true, timeout: WINDOWS_PROCESS_SNAPSHOT_TIMEOUT_MS, maxBuffer: WINDOWS_PROCESS_SNAPSHOT_MAX_BUFFER },
|
|
382
384
|
(error, stdout, stderr) => {
|
|
@@ -545,7 +547,7 @@ export function createSessionWorkerProcessInspector({
|
|
|
545
547
|
function stopWindowsPids(pids) {
|
|
546
548
|
const ids = uniquePids(pids);
|
|
547
549
|
if (!ids.length) return [];
|
|
548
|
-
execFileSyncImpl(
|
|
550
|
+
execFileSyncImpl(systemToolPath('powershell.exe'), ['-NoProfile', '-Command', buildStopWindowsPidsScript(ids)], {
|
|
549
551
|
stdio: ['ignore', 'ignore', 'ignore'],
|
|
550
552
|
});
|
|
551
553
|
return ids;
|
|
@@ -557,7 +559,7 @@ export function createSessionWorkerProcessInspector({
|
|
|
557
559
|
if (!ids.length) return Promise.resolve([]);
|
|
558
560
|
return new Promise((resolve, reject) => {
|
|
559
561
|
execFileImpl(
|
|
560
|
-
|
|
562
|
+
systemToolPath('powershell.exe'),
|
|
561
563
|
['-NoProfile', '-Command', buildStopWindowsPidsScript(ids)],
|
|
562
564
|
{ windowsHide: true, timeout: WINDOWS_STOP_TIMEOUT_MS, maxBuffer: WINDOWS_PROCESS_SNAPSHOT_MAX_BUFFER },
|
|
563
565
|
(error) => {
|