@oar-sh/oar 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +269 -0
- package/README.md +407 -424
- package/bin/oar.js +25 -10
- package/package.json +7 -4
- package/server/README.md +1 -1
- package/server/claude-worker/claude-session-process.mjs +981 -44
- package/server/claude-worker/claude-session-worker.mjs +45 -4
- package/server/claude-worker/claude-turn-publisher.mjs +81 -20
- package/server/claude-worker/claude-worker-link-wiring.mjs +9 -0
- package/server/copilot-worker/copilot-ask-user-tool.mjs +89 -0
- package/server/copilot-worker/copilot-continuation-signals.mjs +55 -0
- package/server/copilot-worker/copilot-question-bridge.mjs +34 -12
- package/server/copilot-worker/copilot-sdk-event-normalizer.mjs +101 -12
- package/server/copilot-worker/copilot-sdk-session-process.mjs +2269 -492
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +82 -20
- package/server/db-schema.mjs +20 -0
- package/server/migrations/0004-steer-settle-markers.mjs +119 -0
- package/server/public/app/api-client.js +22 -2
- package/server/public/app/ask-user-view.js +102 -13
- package/server/public/app/attachments-view.js +37 -2
- package/server/public/app/background-task-timeout-options.mjs +9 -0
- package/server/public/app/background-tasks-view.mjs +33 -18
- package/server/public/app/bootstrap.js +34 -8
- package/server/public/app/composer-control-state.mjs +52 -34
- package/server/public/app/conversation-draft-sync.mjs +187 -0
- package/server/public/app/conversation-list-filter.mjs +86 -0
- package/server/public/app/conversation-provider-indicator.mjs +8 -0
- package/server/public/app/conversation-view.js +919 -194
- package/server/public/app/emoji-view.js +2 -0
- package/server/public/app/font-scaling.js +4 -37
- package/server/public/app/journal-view.js +145 -5
- package/server/public/app/model-selector-options.mjs +1 -1
- package/server/public/app/question-multi-select.mjs +52 -0
- package/server/public/app/relay-prompt-sanitizer.mjs +17 -2
- package/server/public/app/settings-modal.js +8 -6
- package/server/public/app/socket-handlers.js +19 -0
- package/server/public/app/steered-turn-merge.mjs +100 -0
- package/server/public/app/store.js +126 -15
- package/server/public/app/sync-outbox.mjs +16 -8
- package/server/public/app/tool-activity-emoji.mjs +36 -0
- package/server/public/app/transcript-viewport-anchor.mjs +210 -0
- package/server/public/index.html +206 -9
- package/server/public/sw.js +7 -0
- package/server/relay-tools.md +2 -0
- package/server/repositories/message-repository.mjs +61 -18
- package/server/repositories/question-repository.mjs +15 -0
- package/server/repositories/session-repository.mjs +7 -31
- package/server/routes/messages-routes.mjs +488 -46
- package/server/routes/sessions-routes.mjs +298 -55
- package/server/server-runtime.mjs +65 -25
- package/server/services/live-turn-picker.mjs +47 -0
- package/server/services/relay-prompt-sanitizer.mjs +17 -2
- package/server/services/relay-question-context.mjs +46 -0
- package/server/services/sdk-session-import-service.mjs +49 -7
- package/server/services/session-worker-registry-service.mjs +75 -0
- package/server/services/session-worker-supervisor-service.mjs +7 -1
- package/server/services/session-worker-websocket-service.mjs +23 -1
- package/server/services/steer-resend.mjs +45 -0
- package/shared/background-task-timeout.mjs +12 -4
- package/shared/control-poller.mjs +4 -1
- package/shared/question-multi-select.mjs +16 -0
- package/shared/relay-stop-failure.mjs +15 -0
- package/shared/slash-command-guard.mjs +76 -0
- package/shared/steer-note.mjs +66 -0
- package/shared/steer-settle-failure.mjs +89 -0
- package/shared/steer-settle-markers.mjs +18 -0
- package/shared/worker-crash-guard.mjs +40 -25
- package/shared/worker-runtime/heartbeat.mjs +55 -1
- package/shared/worker-runtime/runner-link-wiring.mjs +44 -0
- package/shared/worker-runtime/worker-websocket-link.mjs +119 -8
- package/server/claude-worker/claude-session-test-harness.mjs +0 -250
- package/server/claude-worker/fixtures/workflow-run-record.json +0 -1
- package/server/copilot-worker/copilot-sdk-test-harness.mjs +0 -382
- package/server/copilot-worker/fixtures/abort-turn.json +0 -588
- package/server/copilot-worker/fixtures/ask-user-turn.json +0 -1154
- package/server/copilot-worker/fixtures/background-timer-continuation.json +0 -445
- package/server/copilot-worker/fixtures/background-timer-turn.json +0 -497
- package/server/copilot-worker/fixtures/happy-turn.json +0 -566
- package/server/copilot-worker/fixtures/quota-turn.json +0 -647
- package/server/copilot-worker/fixtures/reasoning-turn.json +0 -625
- package/server/copilot-worker/fixtures/resume-turn.json +0 -700
- package/server/copilot-worker/fixtures/subagent-turn.json +0 -1378
- package/server/copilot-worker/fixtures/tool-permission-turn.json +0 -1293
- package/server/routes/messages-routes-test-harness.mjs +0 -111
- package/server/services/fixtures/claude-auth-stub.cmd +0 -1
- package/server/services/fixtures/claude-auth-stub.mjs +0 -76
- package/server/services/fixtures/claude-auth-stub.sh +0 -4
- package/server/services/fixtures/cli-install-stub.cmd +0 -1
- package/server/services/fixtures/cli-install-stub.mjs +0 -98
- package/server/services/fixtures/cli-install-stub.sh +0 -4
- package/server/services/fixtures/grok-stub.cmd +0 -1
- package/server/services/fixtures/grok-stub.mjs +0 -89
- package/server/services/fixtures/grok-stub.sh +0 -4
- package/shared/fixtures/copilot-catalog-raw-2026-09-07.json +0 -2363
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// The hidden note a steered message carries to the model.
|
|
2
|
+
//
|
|
3
|
+
// A message the user sends while a turn is running is pushed into that turn
|
|
4
|
+
// (steering). When it lands before the model has produced anything, the model
|
|
5
|
+
// sees two user messages at once — and in the 2026-09-25 burn-in gpt-5.4-mini
|
|
6
|
+
// answered only the newest and silently dropped the original request (the
|
|
7
|
+
// runtime's own event log confirmed nothing was lost by the relay). The note
|
|
8
|
+
// tells the model the new message is an addition, not a replacement, unless
|
|
9
|
+
// the user says otherwise. Only the model sees it: the relay stores and shows
|
|
10
|
+
// the user's own text.
|
|
11
|
+
//
|
|
12
|
+
// Shared by the Claude and Copilot SDK workers (maintainer decision, 2026-09-25: both).
|
|
13
|
+
|
|
14
|
+
// Worded to stay true whether the runtime folds the message into the running
|
|
15
|
+
// turn or runs it as a turn of its own after that one finished: "if not
|
|
16
|
+
// finished yet" never asks the model to redo work it already completed. The
|
|
17
|
+
// opening words are the stable handle the prompt sanitizers match on
|
|
18
|
+
// (services/relay-prompt-sanitizer.mjs and its browser copy).
|
|
19
|
+
export const STEERED_MESSAGE_NOTE = '[Sent while you were still working on my previous message. '
|
|
20
|
+
+ 'If that request is not finished yet, finish it too; do not drop it unless this message says so.]';
|
|
21
|
+
|
|
22
|
+
// The note LEADS the user's text, inside the same text block, and that
|
|
23
|
+
// position is load-bearing beyond the replay's shape. The Claude CLI runs a
|
|
24
|
+
// prompt as a slash command when the string, or its LAST text block trimmed,
|
|
25
|
+
// starts with "/" — and one dequeued mid-turn is deferred to a turn of its
|
|
26
|
+
// own, whose output the relay would misattribute. The relay itself runs only
|
|
27
|
+
// /compact and /preview as commands (any other "/x" is confirmed as text
|
|
28
|
+
// first), so a steered "/cmd …" must reach the model as plain text folded
|
|
29
|
+
// into the running turn, which the leading note guarantees. Appended, the
|
|
30
|
+
// prompt would still start with "/" and run as a command whose arguments
|
|
31
|
+
// include the note; as a separate leading block, the last block would still
|
|
32
|
+
// start with "/". A "/x" that is not steered (or is pushed into a continuation,
|
|
33
|
+
// which gets no note) is kept text by the Claude worker's slash-command guard
|
|
34
|
+
// instead (shared/slash-command-guard.mjs).
|
|
35
|
+
|
|
36
|
+
/** Prefix a plain-text prompt with the steer note. */
|
|
37
|
+
export function withSteerNote(text) {
|
|
38
|
+
const body = String(text ?? '');
|
|
39
|
+
return body.trim() ? `${STEERED_MESSAGE_NOTE}\n\n${body}` : STEERED_MESSAGE_NOTE;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Prefix a content-block array (Claude's user message content) with the note:
|
|
44
|
+
* into the first text block when there is one, so the block structure — and
|
|
45
|
+
* with it the replay the CLI echoes back — keeps its shape; otherwise as a new
|
|
46
|
+
* leading text block. Returns a new array; the input is not mutated.
|
|
47
|
+
*/
|
|
48
|
+
export function withSteerNoteContent(content) {
|
|
49
|
+
if (typeof content === 'string') return withSteerNote(content);
|
|
50
|
+
if (!Array.isArray(content)) return content;
|
|
51
|
+
const index = content.findIndex((block) => block?.type === 'text');
|
|
52
|
+
if (index < 0) return [{ type: 'text', text: STEERED_MESSAGE_NOTE }, ...content];
|
|
53
|
+
return content.map((block, i) => (i === index ? { ...block, text: withSteerNote(block.text) } : block));
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The user's own text from a prompt that may carry the steer note — what a
|
|
58
|
+
* worker compares a replayed message against. Note-agnostic matching: the
|
|
59
|
+
* replay the CLI echoes back carries the note, the relay row does not.
|
|
60
|
+
*/
|
|
61
|
+
export function stripSteerNote(text) {
|
|
62
|
+
const value = String(text ?? '');
|
|
63
|
+
if (value === STEERED_MESSAGE_NOTE) return '';
|
|
64
|
+
const prefix = `${STEERED_MESSAGE_NOTE}\n\n`;
|
|
65
|
+
return value.startsWith(prefix) ? value.slice(prefix.length) : value;
|
|
66
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// The terminal failure for a message whose prompt the agent already consumed
|
|
2
|
+
// but whose outcome could not be recorded. At-most-once: such a row is never
|
|
3
|
+
// requeued (a re-delivery would run the prompt a second time), so it fails for
|
|
4
|
+
// good with wording that tells the user where the answer went. Shared by the
|
|
5
|
+
// session workers (their own settle paths) and the relay (recovery paths that
|
|
6
|
+
// meet a row marked consumed).
|
|
7
|
+
//
|
|
8
|
+
// Provider-neutral: the wording names `agentLabel` ("the agent" unless the
|
|
9
|
+
// caller knows better — the Claude worker passes 'Claude').
|
|
10
|
+
|
|
11
|
+
export const STEER_SETTLE_FAILED_CODE = 'steer-settle-failed';
|
|
12
|
+
export const STEER_SETTLE_FAILED_STABLE_CODE = `relay.${STEER_SETTLE_FAILED_CODE}`;
|
|
13
|
+
export const DEFAULT_STEER_AGENT_LABEL = 'the agent';
|
|
14
|
+
|
|
15
|
+
function normalizeAgentLabel(value) {
|
|
16
|
+
const text = String(value || '').trim();
|
|
17
|
+
return text || DEFAULT_STEER_AGENT_LABEL;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// "the agent" → "The agent" at the start of a sentence; a proper name is
|
|
21
|
+
// left as it is.
|
|
22
|
+
function sentenceCase(label) {
|
|
23
|
+
return label.charAt(0).toUpperCase() + label.slice(1);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const VARIANTS = Object.freeze({
|
|
27
|
+
// Folded into a turn whose reply answered it (or the path is unknown — the
|
|
28
|
+
// relay's recovery paths cannot tell which).
|
|
29
|
+
folded: (label) => ({
|
|
30
|
+
message: `This message was already sent to ${label} — check the reply above.`,
|
|
31
|
+
guidance: 'Resend it only if it went unanswered.',
|
|
32
|
+
}),
|
|
33
|
+
// Handed off mid-reply: its own partial reply stands, and the rest of the
|
|
34
|
+
// answer continued on the next message.
|
|
35
|
+
handoff: (label) => ({
|
|
36
|
+
message: `${sentenceCase(label)} started answering this message, then carried on with your next one — check both replies.`,
|
|
37
|
+
guidance: 'Resend it only if something went unanswered.',
|
|
38
|
+
}),
|
|
39
|
+
// Steered into a turn the user stopped.
|
|
40
|
+
stopped: (label) => ({
|
|
41
|
+
message: `This message was sent to ${label} just before you stopped the turn, so it was not answered.`,
|
|
42
|
+
guidance: 'Resend it if you still want an answer.',
|
|
43
|
+
}),
|
|
44
|
+
// Pulled back out of the runtime's queue by the user (un-steer) — the
|
|
45
|
+
// prompt never ran — but the relay could not be told in time.
|
|
46
|
+
cancelled: (label) => ({
|
|
47
|
+
message: `This message was cancelled before ${label} started on it, so it was not answered.`,
|
|
48
|
+
guidance: 'Resend it if you still want an answer.',
|
|
49
|
+
}),
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The name a provider's runtime goes by in the settle wording, for callers
|
|
54
|
+
* that only know the conversation's provider type (the relay's recovery
|
|
55
|
+
* paths). Workers that know exactly which runtime took the prompt pass the
|
|
56
|
+
* label themselves.
|
|
57
|
+
*/
|
|
58
|
+
export function steerAgentLabelForProvider(providerType) {
|
|
59
|
+
switch (String(providerType || '').trim().toLowerCase()) {
|
|
60
|
+
case 'claude': return 'Claude';
|
|
61
|
+
case 'github':
|
|
62
|
+
case 'openai': return 'Copilot';
|
|
63
|
+
case 'cursor': return 'Cursor';
|
|
64
|
+
case 'grok': return 'Grok';
|
|
65
|
+
default: return DEFAULT_STEER_AGENT_LABEL;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function wordingFor(variant, agentLabel) {
|
|
70
|
+
const build = VARIANTS[variant] || VARIANTS.folded;
|
|
71
|
+
return build(normalizeAgentLabel(agentLabel));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function steerSettleFailureText(variant = 'folded', agentLabel = DEFAULT_STEER_AGENT_LABEL) {
|
|
75
|
+
return wordingFor(variant, agentLabel).message;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function buildSteerSettleFailure(message, { variant = 'folded', agentLabel = DEFAULT_STEER_AGENT_LABEL } = {}) {
|
|
79
|
+
const wording = wordingFor(variant, agentLabel);
|
|
80
|
+
return {
|
|
81
|
+
kind: STEER_SETTLE_FAILED_CODE,
|
|
82
|
+
code: STEER_SETTLE_FAILED_CODE,
|
|
83
|
+
stableCode: STEER_SETTLE_FAILED_STABLE_CODE,
|
|
84
|
+
message: wording.message,
|
|
85
|
+
guidance: wording.guidance,
|
|
86
|
+
failedAt: new Date().toISOString(),
|
|
87
|
+
queueMessageId: String(message?.id || '') || null,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// The settle stubs of steered messages the Claude CLI consumed without opening
|
|
2
|
+
// a turn of their own. Each is stamped with its own message kind, which is
|
|
3
|
+
// what the client renders from; the texts are shared so the relay's migration
|
|
4
|
+
// can recognise stubs saved before the kind existed.
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A steer folded into the running turn and answered by that turn's reply.
|
|
8
|
+
* Stamped kind='folded' (before 0.9.3: kind='absorbed', which the client read
|
|
9
|
+
* as "the reply continues through the next message" and mis-merged the next
|
|
10
|
+
* ordinary message on reload).
|
|
11
|
+
*/
|
|
12
|
+
export const STEER_FOLDED_TEXT = '_(Handled together with the previous reply — this message was steered into that turn.)_';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* A steer pushed into a turn the user then stopped: never answered. Stamped
|
|
16
|
+
* kind='stopped' — the stable handle the client keys its Resend on.
|
|
17
|
+
*/
|
|
18
|
+
export const STEER_STOPPED_TEXT = '_(Stopped with the turn — not answered.)_';
|
|
@@ -11,6 +11,45 @@
|
|
|
11
11
|
* The requeue is strictly bounded: a crash handler that hangs is worse than
|
|
12
12
|
* no handler at all.
|
|
13
13
|
*/
|
|
14
|
+
/**
|
|
15
|
+
* Post a bounded, fenced requeue for each owed row. Entries are plain ids or
|
|
16
|
+
* { id, attemptId, terminalError } objects: the attempt id fences the requeue
|
|
17
|
+
* (a row that has moved on to another attempt must not be yanked back by a
|
|
18
|
+
* dying predecessor), and a `terminalError` makes the requeue route fail the
|
|
19
|
+
* row outright — for a row whose prompt was already consumed, which must not
|
|
20
|
+
* be requeued and run a second time. Never waits longer than `timeoutMs`.
|
|
21
|
+
*/
|
|
22
|
+
export async function requeueOwedRows({ api, entries = [], timeoutMs = 2_000 } = {}) {
|
|
23
|
+
const seen = new Set();
|
|
24
|
+
const owed = (Array.isArray(entries) ? entries : [])
|
|
25
|
+
.map((entry) => (entry && typeof entry === 'object'
|
|
26
|
+
? {
|
|
27
|
+
id: String(entry.id || '').trim(),
|
|
28
|
+
attemptId: String(entry.attemptId || '').trim() || null,
|
|
29
|
+
terminalError: entry.terminalError && typeof entry.terminalError === 'object' ? entry.terminalError : null,
|
|
30
|
+
}
|
|
31
|
+
: { id: String(entry || '').trim(), attemptId: null, terminalError: null }))
|
|
32
|
+
.filter((entry) => {
|
|
33
|
+
if (!entry.id || seen.has(entry.id)) return false;
|
|
34
|
+
seen.add(entry.id);
|
|
35
|
+
return true;
|
|
36
|
+
});
|
|
37
|
+
if (!owed.length || typeof api !== 'function') return;
|
|
38
|
+
let timer = null;
|
|
39
|
+
await Promise.race([
|
|
40
|
+
Promise.allSettled(owed.map((entry) => api('POST', '/api/requeue', {
|
|
41
|
+
messageId: entry.id,
|
|
42
|
+
...(entry.attemptId ? { attemptId: entry.attemptId } : {}),
|
|
43
|
+
...(entry.terminalError ? { terminalError: entry.terminalError } : {}),
|
|
44
|
+
}))),
|
|
45
|
+
new Promise((resolve) => {
|
|
46
|
+
timer = setTimeout(resolve, Math.max(0, Number(timeoutMs) || 0));
|
|
47
|
+
timer.unref?.();
|
|
48
|
+
}),
|
|
49
|
+
]);
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
}
|
|
52
|
+
|
|
14
53
|
export function installWorkerCrashGuard({
|
|
15
54
|
api,
|
|
16
55
|
workerName = 'session-worker',
|
|
@@ -28,31 +67,7 @@ export function installWorkerCrashGuard({
|
|
|
28
67
|
logError(`${workerName} ${kind}:`, error?.stack || error?.message || error);
|
|
29
68
|
(async () => {
|
|
30
69
|
try {
|
|
31
|
-
|
|
32
|
-
// when known, fences the requeue: a row that has already moved on to
|
|
33
|
-
// another attempt must not be yanked back by a dying predecessor.
|
|
34
|
-
const seen = new Set();
|
|
35
|
-
const entries = (getActiveQueueMessageIds() || [])
|
|
36
|
-
.map((entry) => (entry && typeof entry === 'object'
|
|
37
|
-
? { id: String(entry.id || '').trim(), attemptId: String(entry.attemptId || '').trim() || null }
|
|
38
|
-
: { id: String(entry || '').trim(), attemptId: null }))
|
|
39
|
-
.filter((entry) => {
|
|
40
|
-
if (!entry.id || seen.has(entry.id)) return false;
|
|
41
|
-
seen.add(entry.id);
|
|
42
|
-
return true;
|
|
43
|
-
});
|
|
44
|
-
if (entries.length && typeof api === 'function') {
|
|
45
|
-
await Promise.race([
|
|
46
|
-
Promise.allSettled(entries.map((entry) => api('POST', '/api/requeue', {
|
|
47
|
-
messageId: entry.id,
|
|
48
|
-
...(entry.attemptId ? { attemptId: entry.attemptId } : {}),
|
|
49
|
-
}))),
|
|
50
|
-
new Promise((resolve) => {
|
|
51
|
-
const timer = setTimeout(resolve, Math.max(0, Number(requeueTimeoutMs) || 0));
|
|
52
|
-
timer.unref?.();
|
|
53
|
-
}),
|
|
54
|
-
]);
|
|
55
|
-
}
|
|
70
|
+
await requeueOwedRows({ api, entries: getActiveQueueMessageIds() || [], timeoutMs: requeueTimeoutMs });
|
|
56
71
|
} catch {}
|
|
57
72
|
try { onBeforeExit(); } catch {}
|
|
58
73
|
exit(1);
|
|
@@ -10,7 +10,24 @@ export function createHeartbeatController({
|
|
|
10
10
|
// queued behind it, and a background continuation at once). Each reported
|
|
11
11
|
// id gets its owner lease refreshed; unreported owned rows are recovered.
|
|
12
12
|
getActiveQueueMessageIds,
|
|
13
|
+
// Optional: the worker's steering snapshot ({turnActive, canSteer,
|
|
14
|
+
// holdReason}) for the composer. Workers that do not steer leave it unset
|
|
15
|
+
// and the heartbeat body is unchanged.
|
|
16
|
+
getSteeringState,
|
|
17
|
+
// Optional: rows the worker gave up settling ([{id, attemptId,
|
|
18
|
+
// terminalError}]); the relay fails them and names them back in
|
|
19
|
+
// `settleFailedHandled`, which `onSettleFailedHandled` receives.
|
|
20
|
+
getSettleFailed,
|
|
21
|
+
onSettleFailedHandled = () => {},
|
|
22
|
+
// requestPulse's window: one turn transition can flip the steering
|
|
23
|
+
// snapshot several times in quick succession, and each flip only needs to
|
|
24
|
+
// reach the relay once.
|
|
25
|
+
pulseCoalesceMs = 300,
|
|
26
|
+
setTimeoutImpl = setTimeout,
|
|
27
|
+
clearTimeoutImpl = clearTimeout,
|
|
13
28
|
}) {
|
|
29
|
+
let requestedPulseTimer = null;
|
|
30
|
+
|
|
14
31
|
async function pulseHeartbeat() {
|
|
15
32
|
if (!getSessionReady()) return false;
|
|
16
33
|
try {
|
|
@@ -20,17 +37,49 @@ export function createHeartbeatController({
|
|
|
20
37
|
const activeQueueMessageIds = typeof getActiveQueueMessageIds === "function"
|
|
21
38
|
? (getActiveQueueMessageIds() || []).map((id) => String(id || "").trim()).filter(Boolean)
|
|
22
39
|
: [];
|
|
40
|
+
const steering = typeof getSteeringState === "function"
|
|
41
|
+
? getSteeringState()
|
|
42
|
+
: null;
|
|
43
|
+
const settleFailed = typeof getSettleFailed === "function"
|
|
44
|
+
? (getSettleFailed() || []).filter((entry) => entry && String(entry.id || "").trim())
|
|
45
|
+
: [];
|
|
23
46
|
const body = {
|
|
24
47
|
...(activeQueueMessageId ? { activeQueueMessageId } : {}),
|
|
25
48
|
...(activeQueueMessageIds.length ? { activeQueueMessageIds } : {}),
|
|
49
|
+
...(steering && typeof steering === "object" ? { steering } : {}),
|
|
50
|
+
...(settleFailed.length ? { settleFailed } : {}),
|
|
26
51
|
};
|
|
27
|
-
await api("POST", "/api/heartbeat", body);
|
|
52
|
+
const response = await api("POST", "/api/heartbeat", body);
|
|
53
|
+
// Failed by the relay, or skipped because this worker has no claim on
|
|
54
|
+
// the row: either way the worker releases it.
|
|
55
|
+
const handled = [
|
|
56
|
+
...(Array.isArray(response?.settleFailedHandled) ? response.settleFailedHandled : []),
|
|
57
|
+
...(Array.isArray(response?.settleFailedSkipped) ? response.settleFailedSkipped : []),
|
|
58
|
+
];
|
|
59
|
+
if (handled.length) {
|
|
60
|
+
try { onSettleFailedHandled(handled); } catch {}
|
|
61
|
+
}
|
|
28
62
|
return true;
|
|
29
63
|
} catch {
|
|
30
64
|
return false;
|
|
31
65
|
}
|
|
32
66
|
}
|
|
33
67
|
|
|
68
|
+
/**
|
|
69
|
+
* An out-of-cycle pulse for state the relay should see before the next
|
|
70
|
+
* periodic one (the composer's steering snapshot, a relay that just came
|
|
71
|
+
* up). Requests inside one window share a single pulse, sent at its end so
|
|
72
|
+
* it carries the settled state.
|
|
73
|
+
*/
|
|
74
|
+
function requestPulse() {
|
|
75
|
+
if (requestedPulseTimer) return;
|
|
76
|
+
requestedPulseTimer = setTimeoutImpl(() => {
|
|
77
|
+
requestedPulseTimer = null;
|
|
78
|
+
void pulseHeartbeat();
|
|
79
|
+
}, pulseCoalesceMs);
|
|
80
|
+
requestedPulseTimer?.unref?.();
|
|
81
|
+
}
|
|
82
|
+
|
|
34
83
|
function startHeartbeat() {
|
|
35
84
|
if (getHeartbeatTimer()) return;
|
|
36
85
|
void pulseHeartbeat();
|
|
@@ -41,6 +90,10 @@ export function createHeartbeatController({
|
|
|
41
90
|
}
|
|
42
91
|
|
|
43
92
|
function stopHeartbeat() {
|
|
93
|
+
if (requestedPulseTimer) {
|
|
94
|
+
clearTimeoutImpl(requestedPulseTimer);
|
|
95
|
+
requestedPulseTimer = null;
|
|
96
|
+
}
|
|
44
97
|
const timer = getHeartbeatTimer();
|
|
45
98
|
if (!timer) return;
|
|
46
99
|
clearInterval(timer);
|
|
@@ -49,6 +102,7 @@ export function createHeartbeatController({
|
|
|
49
102
|
|
|
50
103
|
return {
|
|
51
104
|
pulseHeartbeat,
|
|
105
|
+
requestPulse,
|
|
52
106
|
startHeartbeat,
|
|
53
107
|
stopHeartbeat,
|
|
54
108
|
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Glue between a steering-capable session runner and the relay socket link —
|
|
2
|
+
// shared by every worker whose runner exposes a delivery gate (the Claude
|
|
3
|
+
// session runner, the Copilot SDK runner). Kept out of the worker entry files
|
|
4
|
+
// (which run main() on import) so the wiring itself is testable.
|
|
5
|
+
|
|
6
|
+
import { requeueOwedRows } from '../worker-crash-guard.mjs';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The runner reports every flip of its delivery gate; the link turns them
|
|
10
|
+
* into relay readiness frames. `attach` is late-bound because the runner is
|
|
11
|
+
* built before the link (the link's probes call into the runner).
|
|
12
|
+
*/
|
|
13
|
+
export function createRunnerLinkBridge() {
|
|
14
|
+
let link = null;
|
|
15
|
+
return {
|
|
16
|
+
attach(nextLink) {
|
|
17
|
+
link = nextLink || null;
|
|
18
|
+
},
|
|
19
|
+
onDeliveryReadinessChange(ready) {
|
|
20
|
+
if (!link) return;
|
|
21
|
+
if (ready) void link.notifyReady('steering-resumed');
|
|
22
|
+
else link.notifyUnready('steering-held');
|
|
23
|
+
},
|
|
24
|
+
// Only a relay that advertised the capability in server.hello understands
|
|
25
|
+
// the penalty-free hand-back; before any hello, assume an older relay.
|
|
26
|
+
canHandBackHeldDelivery() {
|
|
27
|
+
return Boolean(link?.serverSupports?.('steering-held'));
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Worker shutdown (SIGTERM/SIGINT): rows whose prompt the engine already
|
|
34
|
+
* consumed and whose settle has not landed are failed terminally before the
|
|
35
|
+
* process goes — left behind, the relay's dead-worker recovery would requeue
|
|
36
|
+
* them and run them a second time. Bounded like the crash guard: a shutdown
|
|
37
|
+
* that hangs is worse than none. Other owed rows are left to the relay's
|
|
38
|
+
* recovery, as before.
|
|
39
|
+
*/
|
|
40
|
+
export async function failSettlingRowsOnShutdown({ api, runner, timeoutMs = 2_000 } = {}) {
|
|
41
|
+
const settling = (runner?.getActiveQueueMessageIds?.() || []).filter((entry) => entry?.terminalError);
|
|
42
|
+
await requeueOwedRows({ api, entries: settling, timeoutMs });
|
|
43
|
+
return settling.length;
|
|
44
|
+
}
|
|
@@ -45,6 +45,25 @@ export function createWorkerWebSocketLink({
|
|
|
45
45
|
// task). Never blocks delivery; failures are the handler's to log.
|
|
46
46
|
onControl = () => {},
|
|
47
47
|
getSessionReady = () => false,
|
|
48
|
+
// Mid-turn steering opt-in. A worker whose runner can absorb another
|
|
49
|
+
// delivered message into the live turn (the Claude session runner; the
|
|
50
|
+
// Copilot runner's steerIntoActiveTurn is compatible but not yet opted in)
|
|
51
|
+
// reports so here, and the link then keeps signalling readiness while a
|
|
52
|
+
// delivery is still in flight and runs the resulting concurrent delivery
|
|
53
|
+
// instead of coalescing it. Workers that leave this unset keep the strict
|
|
54
|
+
// single-flight contract.
|
|
55
|
+
getSteeringReady = null,
|
|
56
|
+
// Steering-capable workers only: true while the runner is holding deliveries
|
|
57
|
+
// (a question card or plan approval is open, a compaction or adoption is in
|
|
58
|
+
// progress, a turn-opening delivery is mid-flight). A held worker never
|
|
59
|
+
// signals readiness — idle or not — and tells the server so with
|
|
60
|
+
// worker.unready, because a delivery landing in a hold is handed back.
|
|
61
|
+
getDeliveryHeld = null,
|
|
62
|
+
// Called with the hello's reason once per connection, on the relay's first
|
|
63
|
+
// server.hello: from then on the relay (possibly a restarted one that knows
|
|
64
|
+
// nothing of this worker) has the socket. Later hellos only ack the
|
|
65
|
+
// worker.hello every idle readiness refresh re-sends, and are not reported.
|
|
66
|
+
onServerHello = () => {},
|
|
48
67
|
getSessionId = () => null,
|
|
49
68
|
getPid = () => null,
|
|
50
69
|
minBackoffMs = 1000,
|
|
@@ -65,11 +84,20 @@ export function createWorkerWebSocketLink({
|
|
|
65
84
|
let readyRefreshTimer = null;
|
|
66
85
|
let reconnectAttempt = 0;
|
|
67
86
|
let reconnectDelayMs = 0;
|
|
68
|
-
|
|
87
|
+
// Insertion-ordered: the first entry is the oldest in-flight delivery, which
|
|
88
|
+
// is what a legacy worker's coalesced delivery waits on.
|
|
89
|
+
const deliveriesInFlight = new Set();
|
|
90
|
+
// message id → in-flight delivery. Only a redelivery of the SAME message is
|
|
91
|
+
// coalesced; a different message is never folded onto another's promise.
|
|
92
|
+
const deliveriesById = new Map();
|
|
69
93
|
let lastOpenAt = null;
|
|
70
94
|
let lastMessageAt = null;
|
|
71
95
|
let lastHelloSentAt = null;
|
|
72
96
|
let lastReadySentAt = null;
|
|
97
|
+
let lastUnreadySentAt = null;
|
|
98
|
+
// What the connected relay advertised in server.hello (see
|
|
99
|
+
// WORKER_PROTOCOL_CAPABILITIES in the relay's worker websocket service).
|
|
100
|
+
let serverCapabilities = new Set();
|
|
73
101
|
let lastPingSentAt = null;
|
|
74
102
|
let lastPongAt = null;
|
|
75
103
|
let lastQueueChangedAt = null;
|
|
@@ -94,20 +122,61 @@ export function createWorkerWebSocketLink({
|
|
|
94
122
|
}
|
|
95
123
|
}
|
|
96
124
|
|
|
125
|
+
function steeringCapable() {
|
|
126
|
+
return typeof getSteeringReady === "function";
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function steeringReady() {
|
|
130
|
+
if (!steeringCapable()) return false;
|
|
131
|
+
try {
|
|
132
|
+
return getSteeringReady() === true;
|
|
133
|
+
} catch {
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function deliveryHeld() {
|
|
139
|
+
if (typeof getDeliveryHeld !== "function") return false;
|
|
140
|
+
try {
|
|
141
|
+
return getDeliveryHeld() === true;
|
|
142
|
+
} catch {
|
|
143
|
+
// Fail closed: a held worker that signals ready bounces every delivery.
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
97
148
|
function notifyHello(reason = "worker-hello") {
|
|
98
149
|
if (!getSessionReady()) return false;
|
|
150
|
+
// The server treats hello as readiness; a held worker still has to bind
|
|
151
|
+
// its identity, so it says hello without claiming to be ready.
|
|
152
|
+
const held = deliveryHeld();
|
|
99
153
|
const sent = send({
|
|
100
154
|
type: "worker.hello",
|
|
101
155
|
reason,
|
|
102
156
|
sessionId: normalizeText(getSessionId()),
|
|
103
157
|
pid: normalizePositiveInt(getPid()),
|
|
158
|
+
...(held ? { ready: false } : {}),
|
|
104
159
|
});
|
|
105
160
|
if (sent) lastHelloSentAt = getNowMs();
|
|
106
161
|
return sent;
|
|
107
162
|
}
|
|
108
163
|
|
|
164
|
+
function notifyUnready(reason = "worker-unready") {
|
|
165
|
+
if (!getSessionReady()) return false;
|
|
166
|
+
const sent = send({
|
|
167
|
+
type: "worker.unready",
|
|
168
|
+
reason,
|
|
169
|
+
sessionId: normalizeText(getSessionId()),
|
|
170
|
+
pid: normalizePositiveInt(getPid()),
|
|
171
|
+
});
|
|
172
|
+
if (sent) lastUnreadySentAt = getNowMs();
|
|
173
|
+
return sent;
|
|
174
|
+
}
|
|
175
|
+
|
|
109
176
|
async function notifyReady(reason = "worker-ready") {
|
|
110
|
-
if (!getSessionReady()
|
|
177
|
+
if (!getSessionReady()) return false;
|
|
178
|
+
if (deliveryHeld()) return false;
|
|
179
|
+
if (deliveriesInFlight.size && !steeringReady()) return false;
|
|
111
180
|
const sent = send({
|
|
112
181
|
type: "worker.ready",
|
|
113
182
|
reason,
|
|
@@ -132,8 +201,21 @@ export function createWorkerWebSocketLink({
|
|
|
132
201
|
|
|
133
202
|
async function deliverPending(pending, reason = "queue-deliver") {
|
|
134
203
|
if (!getSessionReady()) return false;
|
|
135
|
-
|
|
136
|
-
|
|
204
|
+
// A redelivery of a message that is already running (socket reconnect)
|
|
205
|
+
// coalesces onto it rather than double-running the turn.
|
|
206
|
+
const messageId = normalizeText(pending?.message?.id);
|
|
207
|
+
if (messageId && deliveriesById.has(messageId)) return deliveriesById.get(messageId);
|
|
208
|
+
// A DIFFERENT message while one is in flight: a steering-capable worker's
|
|
209
|
+
// runner decides — it steers the message into the live turn, or hands it
|
|
210
|
+
// back to the queue when the turn is holding (the server's readiness can
|
|
211
|
+
// lag a hold by a round-trip). Coalescing it here used to drop its payload
|
|
212
|
+
// and strand the row `processing` until heartbeat recovery. Workers
|
|
213
|
+
// without the steering opt-in keep strict single-flight: their runners
|
|
214
|
+
// have no hand-back path, and the heartbeat recovers the row.
|
|
215
|
+
if (deliveriesInFlight.size && !steeringCapable()) {
|
|
216
|
+
return deliveriesInFlight.values().next().value;
|
|
217
|
+
}
|
|
218
|
+
const delivery = Promise.resolve()
|
|
137
219
|
.then(() => onDeliver(pending, reason))
|
|
138
220
|
.catch((error) => {
|
|
139
221
|
dbg("worker ws delivery failed", reason, error?.message || String(error));
|
|
@@ -141,9 +223,12 @@ export function createWorkerWebSocketLink({
|
|
|
141
223
|
})
|
|
142
224
|
.finally(() => {
|
|
143
225
|
lastDeliveryAt = getNowMs();
|
|
144
|
-
|
|
226
|
+
deliveriesInFlight.delete(delivery);
|
|
227
|
+
if (messageId && deliveriesById.get(messageId) === delivery) deliveriesById.delete(messageId);
|
|
145
228
|
});
|
|
146
|
-
|
|
229
|
+
deliveriesInFlight.add(delivery);
|
|
230
|
+
if (messageId) deliveriesById.set(messageId, delivery);
|
|
231
|
+
return delivery;
|
|
147
232
|
}
|
|
148
233
|
|
|
149
234
|
function clearReconnectTimer() {
|
|
@@ -173,7 +258,18 @@ export function createWorkerWebSocketLink({
|
|
|
173
258
|
return closeStaleSocket(`no-server-message:${reason}`);
|
|
174
259
|
}
|
|
175
260
|
notifyPing(reason);
|
|
176
|
-
if (
|
|
261
|
+
if (deliveryHeld()) {
|
|
262
|
+
// Re-asserted on every refresh: a ready frame that crossed the hold on
|
|
263
|
+
// the wire would otherwise leave the server's readiness stuck on.
|
|
264
|
+
notifyUnready(reason);
|
|
265
|
+
return true;
|
|
266
|
+
}
|
|
267
|
+
if (deliveriesInFlight.size) {
|
|
268
|
+
// notifyReady gates itself: it only goes through mid-delivery when the
|
|
269
|
+
// worker is steering-ready. Hello stays an idle-only affair.
|
|
270
|
+
void notifyReady(reason);
|
|
271
|
+
return true;
|
|
272
|
+
}
|
|
177
273
|
notifyHello(reason);
|
|
178
274
|
void notifyReady(reason);
|
|
179
275
|
return true;
|
|
@@ -215,6 +311,7 @@ export function createWorkerWebSocketLink({
|
|
|
215
311
|
}
|
|
216
312
|
const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
|
|
217
313
|
ws = new WebSocketImpl(url);
|
|
314
|
+
let serverHelloSeen = false;
|
|
218
315
|
ws.addEventListener("open", () => {
|
|
219
316
|
const openedAt = getNowMs();
|
|
220
317
|
reconnectAttempt = 0;
|
|
@@ -240,6 +337,15 @@ export function createWorkerWebSocketLink({
|
|
|
240
337
|
return;
|
|
241
338
|
}
|
|
242
339
|
if (payload?.type === "server.hello") {
|
|
340
|
+
serverCapabilities = new Set(Array.isArray(payload.capabilities) ? payload.capabilities.map(String) : []);
|
|
341
|
+
if (!serverHelloSeen) {
|
|
342
|
+
serverHelloSeen = true;
|
|
343
|
+
try {
|
|
344
|
+
onServerHello(String(payload.reason || "server-hello"));
|
|
345
|
+
} catch (error) {
|
|
346
|
+
dbg("worker ws hello handler failed", error?.message || String(error));
|
|
347
|
+
}
|
|
348
|
+
}
|
|
243
349
|
void notifyReady(String(payload.reason || "server-hello"));
|
|
244
350
|
return;
|
|
245
351
|
}
|
|
@@ -271,6 +377,8 @@ export function createWorkerWebSocketLink({
|
|
|
271
377
|
});
|
|
272
378
|
ws.addEventListener("close", () => {
|
|
273
379
|
clearReadyRefreshTimer();
|
|
380
|
+
// The next connection may reach a different (restarted) relay.
|
|
381
|
+
serverCapabilities = new Set();
|
|
274
382
|
if (stopped) return;
|
|
275
383
|
dbg("worker ws disconnected");
|
|
276
384
|
scheduleReconnect();
|
|
@@ -302,13 +410,14 @@ export function createWorkerWebSocketLink({
|
|
|
302
410
|
const connected = !!ws && ws.readyState === resolveSocketStateValue(ws, "OPEN", 1);
|
|
303
411
|
return {
|
|
304
412
|
connected,
|
|
305
|
-
delivering:
|
|
413
|
+
delivering: deliveriesInFlight.size > 0,
|
|
306
414
|
reconnectAttempt,
|
|
307
415
|
reconnectDelayMs,
|
|
308
416
|
lastOpenAt,
|
|
309
417
|
lastMessageAt,
|
|
310
418
|
lastHelloSentAt,
|
|
311
419
|
lastReadySentAt,
|
|
420
|
+
lastUnreadySentAt,
|
|
312
421
|
lastPingSentAt,
|
|
313
422
|
lastPongAt,
|
|
314
423
|
lastQueueChangedAt,
|
|
@@ -318,6 +427,8 @@ export function createWorkerWebSocketLink({
|
|
|
318
427
|
|
|
319
428
|
return {
|
|
320
429
|
notifyReady,
|
|
430
|
+
notifyUnready,
|
|
431
|
+
serverSupports: (capability) => serverCapabilities.has(String(capability || "")),
|
|
321
432
|
start,
|
|
322
433
|
stop,
|
|
323
434
|
status,
|