@oar-sh/oar 0.9.4 → 0.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/polling/polling-loop.mjs +12 -32
- package/.github/extensions/web-relay/skills/question-bridge.mjs +14 -39
- package/CHANGELOG.md +199 -0
- package/README.md +35 -6
- package/package.json +2 -1
- package/server/claude-worker/claude-remote-relay-tool.mjs +1 -0
- package/server/claude-worker/claude-session-process.mjs +411 -19
- package/server/claude-worker/claude-session-worker.mjs +5 -0
- package/server/claude-worker/claude-turn-failure.mjs +92 -0
- package/server/claude-worker/claude-turn-publisher.mjs +52 -3
- package/server/claude-worker/sdk-message-normalizer.mjs +13 -1
- package/server/copilot-worker/copilot-model-switch.mjs +29 -3
- package/server/copilot-worker/copilot-question-bridge.mjs +2 -4
- package/server/copilot-worker/copilot-runtime-tree.mjs +91 -0
- package/server/copilot-worker/copilot-sdk-adapter.mjs +48 -8
- package/server/copilot-worker/copilot-sdk-session-process.mjs +897 -166
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +5 -0
- package/server/cursor-worker/cursor-sdk-adapter.mjs +80 -13
- package/server/cursor-worker/cursor-session-worker.mjs +5 -0
- package/server/cursor-worker/cursor-turn-runner.mjs +5 -2
- package/server/db-schema.mjs +7 -0
- package/server/grok-worker/acp-client.mjs +68 -12
- package/server/grok-worker/grok-sdk-adapter.mjs +4 -0
- package/server/grok-worker/grok-session-worker.mjs +5 -0
- package/server/grok-worker/grok-turn-runner.mjs +6 -2
- package/server/public/app/api-client.js +17 -2
- package/server/public/app/bootstrap.js +3 -0
- package/server/public/app/composer-control-state.mjs +4 -3
- package/server/public/app/conversation-view.js +97 -4
- package/server/public/app/journal-view.js +6 -0
- package/server/public/app/markdown-strikethrough.mjs +31 -0
- package/server/public/app/relay-board-inline.mjs +70 -0
- package/server/public/app/relay-board-view.js +157 -61
- package/server/public/app/relay-error-ctas.mjs +12 -3
- package/server/public/app/socket-handlers.js +9 -0
- package/server/public/app/steered-turn-merge.mjs +10 -1
- package/server/public/app/store.js +4 -1
- package/server/public/app/subagent-fold.mjs +61 -0
- package/server/public/app/usage-limit-ui.js +225 -0
- package/server/public/index.html +97 -4
- package/server/repositories/message-repository.mjs +10 -1
- package/server/repositories/question-repository.mjs +1 -0
- package/server/repositories/remote-relay-repository.mjs +30 -0
- package/server/routes/messages-routes.mjs +93 -17
- package/server/routes/relay-board-routes.mjs +56 -2
- package/server/routes/sessions-routes.mjs +40 -2
- package/server/server-runtime.mjs +33 -2
- package/server/services/preview-proxy-server.mjs +64 -36
- package/server/services/process-list-service.mjs +122 -0
- package/server/services/relay-console-log.mjs +199 -0
- package/server/services/remote-relay-caller-context.mjs +45 -20
- package/server/services/remote-relay-dispatcher.mjs +328 -21
- package/server/services/remote-relay-inbound.mjs +35 -0
- package/server/services/session-worker-launch-service.mjs +17 -2
- package/server/services/session-worker-process-service.mjs +85 -32
- package/server/services/session-worker-registry-service.mjs +1 -1
- package/server/services/session-worker-supervisor-service.mjs +33 -1
- package/server/services/usage-limit-pause-service.mjs +441 -0
- package/shared/ask-user-bridge.mjs +14 -39
- package/shared/claude-usage-limit.mjs +257 -0
- package/shared/control-poller.mjs +69 -20
- package/shared/failure-note-text.mjs +38 -0
- package/shared/question-wait.mjs +108 -0
- package/shared/remote-relay-contract.mjs +33 -2
- package/shared/steer-settle-failure.mjs +22 -0
- package/shared/steer-settle-markers.mjs +24 -0
- package/shared/worker-runtime/process-tree.mjs +483 -0
- package/shared/worker-runtime/relay-errors.mjs +31 -0
- package/shared/worker-runtime/send-and-wait-errors.mjs +4 -4
- package/shared/worker-runtime/turn-liveness.mjs +249 -0
- package/shared/worker-runtime/worker-log-file.mjs +97 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
// The Claude subscription's usage limit, as the CLI reports it.
|
|
2
|
+
//
|
|
3
|
+
// The CLI sends a `rate_limit_event` at the start of every turn of a
|
|
4
|
+
// claude.ai subscription session. At the limit it refuses the request without
|
|
5
|
+
// waiting or retrying (captured live, CLI 2.1.283 / SDK 0.3.283):
|
|
6
|
+
//
|
|
7
|
+
// rate_limit_event { status: 'rejected', rateLimitType: 'five_hour',
|
|
8
|
+
// resetsAt: <unix seconds>, unifiedWindows: {...} }
|
|
9
|
+
// assistant { model: '<synthetic>', error: 'rate_limit',
|
|
10
|
+
// is_api_error_message: true, text: "You've hit your
|
|
11
|
+
// session limit · resets 10pm (UTC)" }
|
|
12
|
+
// result { is_error: true, subtype: 'success',
|
|
13
|
+
// api_error_status: 429, terminal_reason: 'api_error' }
|
|
14
|
+
//
|
|
15
|
+
// Shared by the Claude worker (which sees the stream) and the relay (which
|
|
16
|
+
// pauses the turn and resumes it at the reset).
|
|
17
|
+
|
|
18
|
+
export const USAGE_LIMIT_CODE = 'usage-limit';
|
|
19
|
+
export const USAGE_LIMIT_STABLE_CODE = `claude.${USAGE_LIMIT_CODE}`;
|
|
20
|
+
export const USAGE_LIMIT_KIND = 'claude-usage-limit';
|
|
21
|
+
|
|
22
|
+
// A wait up to this long resumes by itself. A longer one (a weekly window can
|
|
23
|
+
// be days away) stays paused until the user resumes it.
|
|
24
|
+
export const USAGE_LIMIT_AUTO_RESUME_MAX_MS = 6 * 60 * 60 * 1000;
|
|
25
|
+
|
|
26
|
+
// The reset time is the server's; the first request after it must not race
|
|
27
|
+
// the window's own bookkeeping.
|
|
28
|
+
export const USAGE_LIMIT_RESUME_MARGIN_MS = 60 * 1000;
|
|
29
|
+
|
|
30
|
+
// A rejected report that names no reset still ahead counts for this long
|
|
31
|
+
// after it arrived. Right after a reset the CLI can refuse once more and name
|
|
32
|
+
// the reset that has just passed.
|
|
33
|
+
export const USAGE_LIMIT_REPORT_FRESH_MS = 10 * 60 * 1000;
|
|
34
|
+
|
|
35
|
+
const STATUSES = new Set(['allowed', 'allowed_warning', 'rejected']);
|
|
36
|
+
|
|
37
|
+
const WINDOW_LABELS = Object.freeze({
|
|
38
|
+
five_hour: '5-hour limit',
|
|
39
|
+
seven_day: 'weekly limit',
|
|
40
|
+
seven_day_opus: 'weekly Opus limit',
|
|
41
|
+
seven_day_sonnet: 'weekly Sonnet limit',
|
|
42
|
+
seven_day_overage_included: 'weekly limit',
|
|
43
|
+
overage: 'extra usage limit',
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const LIMIT_TEXT = /\b(?:hit|reached)\s+your\s+(?:\w+\s+){0,3}limit\b|\busage limit\b/i;
|
|
47
|
+
|
|
48
|
+
function toFraction(value) {
|
|
49
|
+
const n = Number(value);
|
|
50
|
+
if (!Number.isFinite(n) || n < 0) return null;
|
|
51
|
+
return Math.min(n, 1);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Unix seconds (what the CLI sends), milliseconds or an ISO string.
|
|
55
|
+
function toIso(value) {
|
|
56
|
+
if (value === null || value === undefined || value === '') return null;
|
|
57
|
+
let ms = NaN;
|
|
58
|
+
if (typeof value === 'number' || /^\d+(\.\d+)?$/.test(String(value).trim())) {
|
|
59
|
+
const n = Number(value);
|
|
60
|
+
ms = n < 1e11 ? n * 1000 : n;
|
|
61
|
+
} else {
|
|
62
|
+
ms = Date.parse(String(value));
|
|
63
|
+
}
|
|
64
|
+
return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : null;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function normalizeWindows(raw) {
|
|
68
|
+
if (!raw || typeof raw !== 'object') return [];
|
|
69
|
+
return Object.entries(raw)
|
|
70
|
+
.map(([id, window]) => ({
|
|
71
|
+
id: String(id),
|
|
72
|
+
utilization: toFraction(window?.utilization),
|
|
73
|
+
resetsAt: toIso(window?.resetsAt ?? window?.resets_at),
|
|
74
|
+
}))
|
|
75
|
+
.filter((window) => window.utilization !== null || window.resetsAt);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function usageLimitWindowLabel(rateLimitType) {
|
|
79
|
+
return WINDOW_LABELS[String(rateLimitType || '').trim()] || 'usage limit';
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The `rate_limit_info` of a `rate_limit_event`, in the shape the relay keeps.
|
|
84
|
+
* Returns null for anything that is not one.
|
|
85
|
+
*/
|
|
86
|
+
export function normalizeRateLimitInfo(raw, { now = Date.now() } = {}) {
|
|
87
|
+
if (!raw || typeof raw !== 'object') return null;
|
|
88
|
+
const status = String(raw.status || '').trim();
|
|
89
|
+
if (!STATUSES.has(status)) return null;
|
|
90
|
+
const rateLimitType = String(raw.rateLimitType || '').trim() || null;
|
|
91
|
+
const windows = normalizeWindows(raw.unifiedWindows);
|
|
92
|
+
const own = windows.find((window) => window.id === rateLimitType) || null;
|
|
93
|
+
return {
|
|
94
|
+
status,
|
|
95
|
+
rateLimitType,
|
|
96
|
+
resetsAt: toIso(raw.resetsAt) || own?.resetsAt || null,
|
|
97
|
+
// A rejected event carries no utilization of its own: the window is full.
|
|
98
|
+
utilization: toFraction(raw.utilization) ?? own?.utilization ?? (status === 'rejected' ? 1 : null),
|
|
99
|
+
surpassedThreshold: toFraction(raw.surpassedThreshold),
|
|
100
|
+
overageStatus: STATUSES.has(String(raw.overageStatus || '')) ? String(raw.overageStatus) : null,
|
|
101
|
+
isUsingOverage: raw.isUsingOverage === true || raw.overageInUse === true,
|
|
102
|
+
windows,
|
|
103
|
+
observedAt: new Date(now).toISOString(),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function isRateLimitEvent(sdkMessage) {
|
|
108
|
+
return String(sdkMessage?.type || '') === 'rate_limit_event';
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** What makes two reports the same to a reader: a repeat is not published. */
|
|
112
|
+
export function rateLimitInfoKey(info) {
|
|
113
|
+
if (!info) return '';
|
|
114
|
+
const percent = info.utilization === null ? '' : Math.round(info.utilization * 100);
|
|
115
|
+
return [info.status, info.rateLimitType || '', info.resetsAt || '', percent, info.isUsingOverage ? 'overage' : ''].join('|');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Did the CLI refuse this turn because the usage limit is reached?
|
|
120
|
+
*
|
|
121
|
+
* The ONE place that decides it. Returns null, or what the relay needs to
|
|
122
|
+
* pause the turn: `{ rateLimitType, resetsAt, text }`. `resetsAt` is null
|
|
123
|
+
* when the report names no reset still ahead; the relay then tries again
|
|
124
|
+
* shortly.
|
|
125
|
+
*
|
|
126
|
+
* - `rateLimit`: the latest normalized `rate_limit_info` of the process.
|
|
127
|
+
* - `result`: the raw SDK `result` message the turn ended with, if any.
|
|
128
|
+
* - `assistantError`: the `error` field of the turn's last assistant message.
|
|
129
|
+
* - `errorText`: the text of an exception, when the stream threw instead.
|
|
130
|
+
*
|
|
131
|
+
* A refusal needs both halves: the turn ended in an error that names the
|
|
132
|
+
* limit, and the CLI reported a rejected window, with its reset still ahead
|
|
133
|
+
* or just now. Without a rejected report (the limit reached in the middle of
|
|
134
|
+
* a long turn, under the report the turn began with) the error has to name
|
|
135
|
+
* the limit twice over: by its kind and in its words. A plain API 429 (an
|
|
136
|
+
* API key's requests per minute) comes with no report at all and stays the
|
|
137
|
+
* failure it is; a rejected window the account's extra usage covers ends no
|
|
138
|
+
* turn.
|
|
139
|
+
*/
|
|
140
|
+
export function classifyUsageLimitRefusal({
|
|
141
|
+
rateLimit = null,
|
|
142
|
+
result = null,
|
|
143
|
+
assistantError = '',
|
|
144
|
+
errorText = '',
|
|
145
|
+
now = Date.now(),
|
|
146
|
+
} = {}) {
|
|
147
|
+
if (!rateLimit) return null;
|
|
148
|
+
const resultText = String(result?.result || '').trim();
|
|
149
|
+
const thrownText = String(errorText || '').trim();
|
|
150
|
+
const endedInError = result ? result.is_error === true || String(result.subtype || '') !== 'success' : Boolean(thrownText);
|
|
151
|
+
if (!endedInError) return null;
|
|
152
|
+
|
|
153
|
+
const byKind = String(assistantError || '') === 'rate_limit' || Number(result?.api_error_status) === 429;
|
|
154
|
+
const inWords = LIMIT_TEXT.test(resultText) || LIMIT_TEXT.test(thrownText);
|
|
155
|
+
const rejected = rateLimit.status === 'rejected';
|
|
156
|
+
const resetsAtMs = Date.parse(rateLimit.resetsAt || '');
|
|
157
|
+
const resetAhead = Number.isFinite(resetsAtMs) && resetsAtMs > now;
|
|
158
|
+
|
|
159
|
+
if (rejected) {
|
|
160
|
+
if (!byKind && !inWords) return null;
|
|
161
|
+
if (!resetAhead) {
|
|
162
|
+
const observedAtMs = Date.parse(rateLimit.observedAt || '');
|
|
163
|
+
if (!Number.isFinite(observedAtMs) || now - observedAtMs > USAGE_LIMIT_REPORT_FRESH_MS) return null;
|
|
164
|
+
}
|
|
165
|
+
} else if (!byKind || !inWords) {
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// A warning names the window that was about to run out; a report that
|
|
170
|
+
// allowed everything says nothing about which reset to wait for.
|
|
171
|
+
const namesTheReset = resetAhead && (rejected || rateLimit.status === 'allowed_warning');
|
|
172
|
+
return {
|
|
173
|
+
rateLimitType: rejected || namesTheReset ? rateLimit.rateLimitType : null,
|
|
174
|
+
resetsAt: namesTheReset ? rateLimit.resetsAt : null,
|
|
175
|
+
text: resultText || thrownText,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** When a paused turn is sent again: the reset plus a margin. */
|
|
180
|
+
export function usageLimitResumeAt(resetsAt, { marginMs = USAGE_LIMIT_RESUME_MARGIN_MS } = {}) {
|
|
181
|
+
const ms = Date.parse(resetsAt || '');
|
|
182
|
+
return Number.isFinite(ms) ? new Date(ms + marginMs).toISOString() : null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Whether a pause ends by itself, or waits for the user (a reset days away). */
|
|
186
|
+
export function usageLimitResumesByItself(resetsAt, { now = Date.now(), maxWaitMs = USAGE_LIMIT_AUTO_RESUME_MAX_MS } = {}) {
|
|
187
|
+
const ms = Date.parse(resetsAt || '');
|
|
188
|
+
return Number.isFinite(ms) && ms - now <= maxWaitMs;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* What the worker reports for a refused turn, inside the response's
|
|
193
|
+
* `terminalError`. A relay that knows the kind pauses the row; an older one
|
|
194
|
+
* fails the turn with this message, as before.
|
|
195
|
+
*/
|
|
196
|
+
export function buildUsageLimitFailure(message, refusal) {
|
|
197
|
+
const label = usageLimitWindowLabel(refusal?.rateLimitType);
|
|
198
|
+
return {
|
|
199
|
+
kind: USAGE_LIMIT_KIND,
|
|
200
|
+
code: USAGE_LIMIT_CODE,
|
|
201
|
+
stableCode: USAGE_LIMIT_STABLE_CODE,
|
|
202
|
+
message: String(refusal?.text || '').trim() || `The Claude ${label} is reached.`,
|
|
203
|
+
guidance: 'The turn carries on by itself after the reset.',
|
|
204
|
+
rateLimitType: refusal?.rateLimitType || null,
|
|
205
|
+
resetsAt: refusal?.resetsAt || null,
|
|
206
|
+
failedAt: new Date().toISOString(),
|
|
207
|
+
queueMessageId: String(message?.id || '') || null,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Per-process bookkeeping for the worker: the latest report, what the turn's
|
|
213
|
+
* last assistant message said, and which reports are worth publishing.
|
|
214
|
+
*/
|
|
215
|
+
export function createUsageLimitTracker({ now = () => Date.now() } = {}) {
|
|
216
|
+
let latest = null;
|
|
217
|
+
let publishedKey = '';
|
|
218
|
+
let assistantError = '';
|
|
219
|
+
|
|
220
|
+
/** Returns the normalized report when it is news to the relay, else null. */
|
|
221
|
+
function observe(sdkMessage) {
|
|
222
|
+
const type = String(sdkMessage?.type || '');
|
|
223
|
+
if (type === 'rate_limit_event') {
|
|
224
|
+
const info = normalizeRateLimitInfo(sdkMessage.rate_limit_info, { now: now() });
|
|
225
|
+
if (!info) return null;
|
|
226
|
+
latest = info;
|
|
227
|
+
const key = rateLimitInfoKey(info);
|
|
228
|
+
if (key === publishedKey) return null;
|
|
229
|
+
publishedKey = key;
|
|
230
|
+
return info;
|
|
231
|
+
}
|
|
232
|
+
if (type === 'assistant' && !sdkMessage.parent_tool_use_id) {
|
|
233
|
+
assistantError = String(sdkMessage.error || '').trim();
|
|
234
|
+
}
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function classifyResult(sdkResult) {
|
|
239
|
+
return classifyUsageLimitRefusal({ rateLimit: latest, result: sdkResult, assistantError, now: now() });
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function classifyException(error) {
|
|
243
|
+
return classifyUsageLimitRefusal({
|
|
244
|
+
rateLimit: latest,
|
|
245
|
+
assistantError,
|
|
246
|
+
errorText: String(error?.message || error || ''),
|
|
247
|
+
now: now(),
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return {
|
|
252
|
+
observe,
|
|
253
|
+
classifyResult,
|
|
254
|
+
classifyException,
|
|
255
|
+
get latest() { return latest; },
|
|
256
|
+
};
|
|
257
|
+
}
|
|
@@ -23,7 +23,10 @@ export function createControlPoller({
|
|
|
23
23
|
onAbortSubagent = null,
|
|
24
24
|
dbg = () => {},
|
|
25
25
|
} = {}) {
|
|
26
|
-
|
|
26
|
+
// The turns a Stop is polled for, oldest first, and whether the one loop
|
|
27
|
+
// that serves them all is running.
|
|
28
|
+
const registrations = new Set();
|
|
29
|
+
let polling = false;
|
|
27
30
|
|
|
28
31
|
async function checkOnce({ queueMessageId, onAbortTurn }) {
|
|
29
32
|
const ownerSessionId = String(sdkSessionId || '').trim();
|
|
@@ -81,37 +84,83 @@ export function createControlPoller({
|
|
|
81
84
|
}
|
|
82
85
|
}
|
|
83
86
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
87
|
+
/**
|
|
88
|
+
* Hand a Stop to the turns that polled for it, newest first, until one of
|
|
89
|
+
* them takes it. The candidates were chosen before the request went out and
|
|
90
|
+
* are not re-checked here: the relay has claimed the control for this worker
|
|
91
|
+
* by now, so a turn that finished meanwhile still has to answer it.
|
|
92
|
+
*/
|
|
93
|
+
async function offerAbort(candidates, control) {
|
|
94
|
+
let failure = null;
|
|
95
|
+
for (const registration of candidates) {
|
|
96
|
+
try {
|
|
97
|
+
await registration.onAbortTurn(control);
|
|
98
|
+
return;
|
|
99
|
+
} catch (error) {
|
|
100
|
+
failure = error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
throw failure || new Error('abort failed');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function pollLoop() {
|
|
107
|
+
while (registrations.size) {
|
|
108
|
+
await sleep(pollMs);
|
|
109
|
+
// One request per row filter, however many turns share it, asked in the
|
|
110
|
+
// order of the newest turn that uses it.
|
|
111
|
+
const byFilter = new Map();
|
|
112
|
+
for (const registration of [...registrations].reverse()) {
|
|
113
|
+
const candidates = byFilter.get(registration.queueMessageId) || [];
|
|
114
|
+
candidates.push(registration);
|
|
115
|
+
byFilter.set(registration.queueMessageId, candidates);
|
|
116
|
+
}
|
|
117
|
+
for (const [queueMessageId, candidates] of byFilter) {
|
|
118
|
+
// Every turn behind this filter ended while an earlier one was asked.
|
|
119
|
+
if (candidates.every((registration) => registration.stopped)) continue;
|
|
92
120
|
try {
|
|
93
|
-
|
|
94
|
-
if (aborted) return;
|
|
121
|
+
await checkOnce({ queueMessageId, onAbortTurn: (control) => offerAbort(candidates, control) });
|
|
95
122
|
} catch (error) {
|
|
96
123
|
dbg('control poll failed', error?.message || String(error));
|
|
97
124
|
}
|
|
98
125
|
}
|
|
99
|
-
}
|
|
100
|
-
|
|
126
|
+
}
|
|
127
|
+
polling = false;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Poll for a Stop on behalf of one turn, until `stop(handle)`. Turns overlap
|
|
132
|
+
* (a settled turn still waiting on a question card or publishing, beside the
|
|
133
|
+
* one that runs), so a new registration never displaces an older one, and a
|
|
134
|
+
* handled Stop does not end the polling: the Stop may have been for another
|
|
135
|
+
* turn's row, and the turns that remain still need theirs.
|
|
136
|
+
*/
|
|
137
|
+
function start({ queueMessageId, onAbortTurn }) {
|
|
138
|
+
const registration = {
|
|
139
|
+
stopped: false,
|
|
140
|
+
queueMessageId: String(queueMessageId || ''),
|
|
141
|
+
onAbortTurn,
|
|
142
|
+
};
|
|
143
|
+
registrations.add(registration);
|
|
144
|
+
if (!polling) {
|
|
145
|
+
polling = true;
|
|
146
|
+
void pollLoop();
|
|
147
|
+
}
|
|
148
|
+
return registration;
|
|
101
149
|
}
|
|
102
150
|
|
|
103
151
|
function stop(handle) {
|
|
104
|
-
// Handle-scoped stop: a caller finishing
|
|
105
|
-
//
|
|
106
|
-
// context would
|
|
107
|
-
// its Stop button dead. A bare stop()
|
|
152
|
+
// Handle-scoped stop: a caller finishing a turn ends that turn's
|
|
153
|
+
// registration and no other. Without the scoping, a late finalize for a
|
|
154
|
+
// previous context would end the polling for the turn currently running,
|
|
155
|
+
// leaving its Stop button dead. A bare stop() ends every registration (the
|
|
156
|
+
// worker is shutting down).
|
|
108
157
|
if (handle) {
|
|
109
158
|
handle.stopped = true;
|
|
110
|
-
|
|
159
|
+
registrations.delete(handle);
|
|
111
160
|
return;
|
|
112
161
|
}
|
|
113
|
-
|
|
114
|
-
|
|
162
|
+
for (const registration of registrations) registration.stopped = true;
|
|
163
|
+
registrations.clear();
|
|
115
164
|
}
|
|
116
165
|
|
|
117
166
|
return { start, stop, checkOnce };
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// What makes a text a failure note of the relay.
|
|
2
|
+
//
|
|
3
|
+
// A failed turn is reported as a record. Where only text is there (a sender
|
|
4
|
+
// that reports a failure as text, a reply read from another relay), the note
|
|
5
|
+
// is known by its shape: one paragraph that leads up to "Error code:
|
|
6
|
+
// relay.<code>" within a few sentences. A reply that quotes or describes such
|
|
7
|
+
// a note, in a report on a failed test for example, is a reply.
|
|
8
|
+
//
|
|
9
|
+
// One rule for every reader: the relay's response route and the remote-relay
|
|
10
|
+
// dispatcher must not disagree about what a failure is.
|
|
11
|
+
|
|
12
|
+
export const FAILURE_NOTE_LEAD_MAX = 600;
|
|
13
|
+
|
|
14
|
+
const CODE_PATTERN = /error code:\s*(relay\.[a-z0-9-]+)/i;
|
|
15
|
+
// Somebody is talking about a note: the code stands in quotation marks, in a
|
|
16
|
+
// code span or a table cell, or the paragraph is a list item, a heading or a
|
|
17
|
+
// quotation.
|
|
18
|
+
const QUOTED_LEAD = /["“”'`|>]\s*$/;
|
|
19
|
+
const MARKUP_LEAD = /^\s*(?:#+|[>|*-]|\d+\.)\s/;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* `{ stableCode, index, lead }` when `text` is a failure note, else null.
|
|
23
|
+
* `index` is where "Error code" begins, `lead` the text before it.
|
|
24
|
+
*/
|
|
25
|
+
export function matchFailureNote(text) {
|
|
26
|
+
const raw = String(text ?? '').trim();
|
|
27
|
+
if (!raw) return null;
|
|
28
|
+
const match = CODE_PATTERN.exec(raw);
|
|
29
|
+
if (!match) return null;
|
|
30
|
+
const lead = raw.slice(0, match.index);
|
|
31
|
+
if (lead.length > FAILURE_NOTE_LEAD_MAX || /[\r\n]/.test(lead)) return null;
|
|
32
|
+
if (QUOTED_LEAD.test(lead) || MARKUP_LEAD.test(lead)) return null;
|
|
33
|
+
return { stableCode: match[1].toLowerCase(), index: match.index, lead };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function isFailureNoteText(text) {
|
|
37
|
+
return matchFailureNote(text) !== null;
|
|
38
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// Waiting for the answer to a question card, and creating the card, through
|
|
2
|
+
// a relay that is briefly not there.
|
|
3
|
+
//
|
|
4
|
+
// A card may be open for hours and is polled every 1.5 s. One poll that failed
|
|
5
|
+
// — the relay restarting, a reset connection, a 502 from a proxy in front of
|
|
6
|
+
// it — used to end the wait with an error, and the callers turn an error into
|
|
7
|
+
// an answer: "continue with your best judgement" for a question, a rejection
|
|
8
|
+
// for an approval. The card itself stayed open, so the user later answered a
|
|
9
|
+
// question nobody was waiting on any more. The relay keeps its cards across a
|
|
10
|
+
// restart; the wait only has to outlast it.
|
|
11
|
+
|
|
12
|
+
import { QUESTION_TIMEOUT_CONTINUATION_TEXT } from './question-timeout.mjs';
|
|
13
|
+
import { isRelayUnreachableError, isTransientRelayError } from './worker-runtime/relay-errors.mjs';
|
|
14
|
+
|
|
15
|
+
const DEFAULT_POLL_MS = 1500;
|
|
16
|
+
/** The spacing between polls grows to this while the relay does not answer. */
|
|
17
|
+
const MAX_OUTAGE_POLL_MS = 10_000;
|
|
18
|
+
/** Attempts to create a card while the relay refuses connections (~16 s). */
|
|
19
|
+
const DEFAULT_CREATE_RETRY_DELAYS_MS = Object.freeze([500, 1_000, 2_000, 4_000, 8_000]);
|
|
20
|
+
|
|
21
|
+
function sleepDefault(ms) {
|
|
22
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Create a question card. Retried only while the relay refuses the
|
|
27
|
+
* connection: any other failure may have created the card, and a second
|
|
28
|
+
* request would put a second one in front of the user.
|
|
29
|
+
*/
|
|
30
|
+
export async function createRelayQuestion({
|
|
31
|
+
api, payload, sleep = sleepDefault, retryDelaysMs = DEFAULT_CREATE_RETRY_DELAYS_MS, signal = null, dbg = () => {},
|
|
32
|
+
} = {}) {
|
|
33
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
34
|
+
try {
|
|
35
|
+
const created = await api('POST', '/api/relay-question', payload);
|
|
36
|
+
const questionId = created?.question?.id;
|
|
37
|
+
if (!questionId) throw new Error('Relay question could not be created');
|
|
38
|
+
return questionId;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (!isRelayUnreachableError(error) || attempt >= retryDelaysMs.length || signal?.aborted) throw error;
|
|
41
|
+
dbg('relay unreachable while creating a question; trying again', error?.message || String(error));
|
|
42
|
+
await sleep(retryDelaysMs[attempt]);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Poll a card until it is answered, times out, is cancelled, or `signal`
|
|
49
|
+
* aborts the wait. Resolves `{ answer, structuredAnswer, timedOut, aborted? }`.
|
|
50
|
+
* Rejects only on the relay's own refusal (a 4xx, a card that does not exist).
|
|
51
|
+
*/
|
|
52
|
+
export async function waitForRelayQuestion({
|
|
53
|
+
api,
|
|
54
|
+
questionId,
|
|
55
|
+
deadlineMs,
|
|
56
|
+
pollMs = DEFAULT_POLL_MS,
|
|
57
|
+
sleep = sleepDefault,
|
|
58
|
+
signal = null,
|
|
59
|
+
trimAnswer = true,
|
|
60
|
+
now = Date.now,
|
|
61
|
+
dbg = () => {},
|
|
62
|
+
} = {}) {
|
|
63
|
+
const started = now();
|
|
64
|
+
const expired = { answer: QUESTION_TIMEOUT_CONTINUATION_TEXT, structuredAnswer: null, timedOut: true };
|
|
65
|
+
let failedPolls = 0;
|
|
66
|
+
while (true) {
|
|
67
|
+
if (signal?.aborted) {
|
|
68
|
+
await api('POST', `/api/relay-question/${questionId}/timeout`, {}).catch(() => {});
|
|
69
|
+
return { ...expired, aborted: true };
|
|
70
|
+
}
|
|
71
|
+
let question = null;
|
|
72
|
+
let outage = null;
|
|
73
|
+
try {
|
|
74
|
+
({ question } = await api('GET', `/api/relay-question/${questionId}`));
|
|
75
|
+
if (!question) throw Object.assign(new Error('Relay question missing'), { status: 404 });
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (!isTransientRelayError(error)) throw error;
|
|
78
|
+
outage = error;
|
|
79
|
+
}
|
|
80
|
+
if (outage) {
|
|
81
|
+
failedPolls += 1;
|
|
82
|
+
if (failedPolls === 1) dbg('relay did not answer the question poll; the card stays open', questionId, outage?.message || String(outage));
|
|
83
|
+
} else {
|
|
84
|
+
if (failedPolls) dbg('relay answers the question poll again', questionId, `after ${failedPolls} failed poll(s)`);
|
|
85
|
+
failedPolls = 0;
|
|
86
|
+
if (question.status === 'answered') {
|
|
87
|
+
const answer = String(question.answer || '');
|
|
88
|
+
return {
|
|
89
|
+
answer: trimAnswer ? answer.trim() : answer,
|
|
90
|
+
// The validated structured submission, when the card carried a
|
|
91
|
+
// `requestedSchema` and the relay stored one. Null otherwise — flat
|
|
92
|
+
// consumers read `answer` and never see a shape change.
|
|
93
|
+
structuredAnswer: question.structuredAnswer && typeof question.structuredAnswer === 'object'
|
|
94
|
+
&& !Array.isArray(question.structuredAnswer)
|
|
95
|
+
? question.structuredAnswer
|
|
96
|
+
: null,
|
|
97
|
+
timedOut: false,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
if (question.status === 'timed_out' || question.status === 'cancelled') return { ...expired };
|
|
101
|
+
}
|
|
102
|
+
if (now() - started >= deadlineMs) {
|
|
103
|
+
await api('POST', `/api/relay-question/${questionId}/timeout`, {}).catch(() => {});
|
|
104
|
+
return { ...expired };
|
|
105
|
+
}
|
|
106
|
+
await sleep(failedPolls ? Math.min(MAX_OUTAGE_POLL_MS, pollMs * 2 ** Math.min(failedPolls, 6)) : pollMs);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -54,12 +54,19 @@ export const REMOTE_RELAY_ACTION_PERMISSION = Object.freeze({
|
|
|
54
54
|
// the calling session runs in ask or plan mode.
|
|
55
55
|
export const REMOTE_RELAY_WRITE_ACTIONS = Object.freeze(['send', 'create_session', 'answer_question', 'stop', 'archive']);
|
|
56
56
|
|
|
57
|
+
// The write actions that carry an agent's words to another relay's agent.
|
|
58
|
+
export const REMOTE_RELAY_PROMPT_ACTIONS = Object.freeze(['send', 'create_session', 'answer_question']);
|
|
59
|
+
|
|
57
60
|
export const REMOTE_RELAY_APPROVAL_MODES = Object.freeze(['ask', 'plan']);
|
|
58
61
|
|
|
59
62
|
export const REMOTE_RELAY_SESSION_SCOPES = Object.freeze(['active', 'recent', 'all']);
|
|
60
63
|
export const REMOTE_RELAY_PROVIDERS = Object.freeze(['github', 'openai', 'claude', 'cursor', 'grok']);
|
|
61
64
|
export const REMOTE_RELAY_MODES = Object.freeze(['plan', 'ask', 'agent', 'autopilot']);
|
|
62
65
|
export const REMOTE_RELAY_IF_BUSY = Object.freeze(['queue', 'fail']);
|
|
66
|
+
// Reasoning efforts differ per provider and model (none, low, medium, high,
|
|
67
|
+
// xhigh, max, ultracode, ...), so the contract only checks the shape of the
|
|
68
|
+
// value: the remote relay decides whether its model takes it.
|
|
69
|
+
export const REMOTE_RELAY_EFFORT_PATTERN = /^[a-z0-9][a-z0-9-]{0,31}$/;
|
|
63
70
|
|
|
64
71
|
export const REMOTE_RELAY_RESULT_STATUSES = Object.freeze([
|
|
65
72
|
'queued',
|
|
@@ -83,7 +90,12 @@ export const REMOTE_RELAY_LIMITS = Object.freeze({
|
|
|
83
90
|
maxCharsMax: 50000,
|
|
84
91
|
perMessageChars: 4000,
|
|
85
92
|
textMax: 100000,
|
|
93
|
+
// Only toward a relay the user did not unlock and that is not the one the
|
|
94
|
+
// prompt came from: a third relay.
|
|
86
95
|
hopLimit: 2,
|
|
96
|
+
// Prompts a conversation's agent may send to other relays' agents without a
|
|
97
|
+
// message from the user in between, before the user is asked.
|
|
98
|
+
agentPromptsBeforeAsking: 30,
|
|
87
99
|
callsPerMinute: 60,
|
|
88
100
|
writesPerMinute: 10,
|
|
89
101
|
healthIntervalMs: 60_000,
|
|
@@ -135,8 +147,10 @@ export const REMOTE_RELAY_TOOL_DESCRIPTION =
|
|
|
135
147
|
'Work with sessions on other OAR relays this relay is paired with ("remote relays"): '
|
|
136
148
|
+ 'list the relays, list or read their sessions, prompt a session, start a new one, '
|
|
137
149
|
+ 'wait for its reply, and answer or stop it. Use it only when the user asks for work '
|
|
138
|
-
+ 'on another relay or mentions one
|
|
139
|
-
+ '
|
|
150
|
+
+ 'on another relay or mentions one, or to answer an agent on another relay that wrote '
|
|
151
|
+
+ 'to this conversation. Start with {action:"list_relays"}. A relay stays locked until '
|
|
152
|
+
+ 'the user mentions it in this conversation (for example @name) or an agent on that '
|
|
153
|
+
+ 'relay writes to it; from then on it stays open for the whole conversation. A '
|
|
140
154
|
+ 'locked relay refuses everything except list_relays, so ask the user to mention it. '
|
|
141
155
|
+ 'send and create_session queue the prompt and wait up to wait_seconds (default 120, '
|
|
142
156
|
+ 'max 600) for the reply. If the remote turn is still running you get status '
|
|
@@ -203,6 +217,12 @@ export const REMOTE_RELAY_TOOL_INPUT_SCHEMA = Object.freeze({
|
|
|
203
217
|
enum: [...REMOTE_RELAY_MODES],
|
|
204
218
|
description: 'send / create_session: relay mode for the remote turn (default: this session\'s mode).',
|
|
205
219
|
},
|
|
220
|
+
effort: {
|
|
221
|
+
type: 'string',
|
|
222
|
+
description: 'send / create_session: reasoning effort for the remote turn, e.g. low, medium, high (relay_info lists '
|
|
223
|
+
+ 'what each model takes). create_session default: this session\'s effort when the remote model supports it, '
|
|
224
|
+
+ 'else the remote\'s default. send default: the remote session keeps its own.',
|
|
225
|
+
},
|
|
206
226
|
title: { type: 'string', description: 'create_session: session title (default: from the prompt).' },
|
|
207
227
|
question_id: { type: 'string', description: 'answer_question: id from pendingQuestions.' },
|
|
208
228
|
answer: { type: 'string', description: 'answer_question: free-text answer.' },
|
|
@@ -312,6 +332,13 @@ export function validateRemoteRelayToolInput(input = {}) {
|
|
|
312
332
|
optionalText('model', 200);
|
|
313
333
|
const modeCheck = optionalEnum('mode', REMOTE_RELAY_MODES);
|
|
314
334
|
if (!modeCheck.ok) return modeCheck;
|
|
335
|
+
const effort = toText(input?.effort).toLowerCase();
|
|
336
|
+
if (effort) {
|
|
337
|
+
if (!REMOTE_RELAY_EFFORT_PATTERN.test(effort)) {
|
|
338
|
+
return invalid('effort must be a short word such as low, medium or high (letters, digits and dashes, at most 32 characters)');
|
|
339
|
+
}
|
|
340
|
+
args.effort = effort;
|
|
341
|
+
}
|
|
315
342
|
if (action === 'send') {
|
|
316
343
|
const busyCheck = optionalEnum('if_busy', REMOTE_RELAY_IF_BUSY);
|
|
317
344
|
if (!busyCheck.ok) return busyCheck;
|
|
@@ -356,6 +383,10 @@ export function isRemoteRelayWriteAction(action) {
|
|
|
356
383
|
return REMOTE_RELAY_WRITE_ACTIONS.includes(action);
|
|
357
384
|
}
|
|
358
385
|
|
|
386
|
+
export function isRemoteRelayPromptAction(action) {
|
|
387
|
+
return REMOTE_RELAY_PROMPT_ACTIONS.includes(action);
|
|
388
|
+
}
|
|
389
|
+
|
|
359
390
|
// ─── Approval cards ──────────────────────────────────────────────────────────
|
|
360
391
|
|
|
361
392
|
// The Allow / Deny card a relay puts on its own turn before a write action on
|
|
@@ -41,6 +41,28 @@ const VARIANTS = Object.freeze({
|
|
|
41
41
|
message: `This message was sent to ${label} just before you stopped the turn, so it was not answered.`,
|
|
42
42
|
guidance: 'Resend it if you still want an answer.',
|
|
43
43
|
}),
|
|
44
|
+
// Steered into a turn that then failed.
|
|
45
|
+
turnFailed: (label) => ({
|
|
46
|
+
message: `This message was sent to ${label} in a turn that then failed, so it was not answered.`,
|
|
47
|
+
guidance: 'Resend it if you still want an answer.',
|
|
48
|
+
}),
|
|
49
|
+
// Not a steer: the message was answered, and the answer reached the relay
|
|
50
|
+
// neither as a reply nor as a requeue.
|
|
51
|
+
replyLost: (label) => ({
|
|
52
|
+
message: `${sentenceCase(label)} answered this message, but the answer could not be saved to the relay.`,
|
|
53
|
+
guidance: 'Ask for the answer again: the session still has it, and nothing is run a second time.',
|
|
54
|
+
}),
|
|
55
|
+
// Not a steer: the turn failed, and its failure note could not be saved.
|
|
56
|
+
failureLost: (label) => ({
|
|
57
|
+
message: `The turn ${label} ran for this message failed, and what went wrong could not be saved to the relay.`,
|
|
58
|
+
guidance: 'Send the message again to retry.',
|
|
59
|
+
}),
|
|
60
|
+
// Not a steer: delivered between turns and answered on a background
|
|
61
|
+
// continuation row (ANSWERED_ELSEWHERE_KIND in steer-settle-markers.mjs).
|
|
62
|
+
answeredElsewhere: (label) => ({
|
|
63
|
+
message: `${sentenceCase(label)} answered this message in a reply marked “background continuation” — check the replies next to it.`,
|
|
64
|
+
guidance: 'Resend it only if it went unanswered.',
|
|
65
|
+
}),
|
|
44
66
|
// Pulled back out of the runtime's queue by the user (un-steer) — the
|
|
45
67
|
// prompt never ran — but the relay could not be told in time.
|
|
46
68
|
cancelled: (label) => ({
|
|
@@ -16,3 +16,27 @@ export const STEER_FOLDED_TEXT = '_(Handled together with the previous reply —
|
|
|
16
16
|
* kind='stopped' — the stable handle the client keys its Resend on.
|
|
17
17
|
*/
|
|
18
18
|
export const STEER_STOPPED_TEXT = '_(Stopped with the turn — not answered.)_';
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A steer folded into a turn that then failed: never answered, and the
|
|
22
|
+
* failure itself is told once, on the reply of the turn. Stamped
|
|
23
|
+
* kind='stopped' like the steer of a stopped turn, so the client offers the
|
|
24
|
+
* same Resend.
|
|
25
|
+
*/
|
|
26
|
+
export const STEER_TURN_FAILED_TEXT = '_(Steered into the turn above, which failed — not answered.)_';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Not a steer: a message delivered between turns whose answer the worker
|
|
30
|
+
* published on a background-continuation row, because it took the turn for one
|
|
31
|
+
* the CLI had opened by itself. Stamped with a kind of its own: it is a
|
|
32
|
+
* marker, not a reply (no completion push, muted in the transcript, never
|
|
33
|
+
* handed to a calling relay as the answer), and not a steer marker either,
|
|
34
|
+
* which would label the message "steered mid-turn".
|
|
35
|
+
*
|
|
36
|
+
* The wording names the reply by its badge, not by its position: a live
|
|
37
|
+
* append puts the marker below that reply, a reload anchors it under its own
|
|
38
|
+
* message and so above it. The worker infers all this from silence, hence the
|
|
39
|
+
* way out at the end.
|
|
40
|
+
*/
|
|
41
|
+
export const ANSWERED_ELSEWHERE_KIND = 'answered-elsewhere';
|
|
42
|
+
export const DELIVERY_ANSWERED_ELSEWHERE_TEXT = '_(Answered in the reply marked “background continuation” next to this message. Resend the message if that reply does not answer it.)_';
|