@oar-sh/oar 0.9.23 → 0.9.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/skills/reasoning-stream.mjs +3 -1
- package/CHANGELOG.md +307 -0
- package/README.md +12 -11
- package/bin/oar.js +54 -4
- package/package.json +11 -7
- package/server/agent-sdk-loader.mjs +116 -0
- package/server/agent-sdks/claude/entry.mjs +4 -0
- package/server/agent-sdks/claude/package-lock.json +1438 -0
- package/server/agent-sdks/claude/package.json +13 -0
- package/server/agent-sdks/cursor/entry-sqlite.mjs +2 -0
- package/server/agent-sdks/cursor/entry.mjs +3 -0
- package/server/agent-sdks/cursor/package-lock.json +171 -0
- package/server/agent-sdks/cursor/package.json +11 -0
- package/server/agent-sdks/koffi/entry.cjs +2 -0
- package/server/agent-sdks/koffi/package-lock.json +368 -0
- package/server/agent-sdks/koffi/package.json +10 -0
- package/server/claude-worker/claude-preview-tool.mjs +5 -2
- package/server/claude-worker/claude-remote-relay-tool.mjs +20 -2
- package/server/claude-worker/claude-sdk-adapter.mjs +4 -1
- package/server/claude-worker/sdk-message-normalizer.mjs +29 -1
- package/server/copilot-worker/copilot-sdk-session-process.mjs +13 -8
- package/server/cursor-worker/cursor-sdk-adapter.mjs +5 -3
- package/server/cursor-worker/sdk-message-normalizer.mjs +13 -0
- package/server/db-schema.mjs +19 -0
- package/server/grok-worker/sdk-message-normalizer.mjs +3 -0
- package/server/migrations/0039-media-index.mjs +167 -0
- package/server/migrations/0040-image-session-defaults.mjs +93 -0
- package/server/public/app/agent-sdk-ui.js +349 -0
- package/server/public/app/api-client.js +149 -8
- package/server/public/app/ask-user-view.js +2 -1
- package/server/public/app/attachments-view.js +66 -9
- package/server/public/app/bootstrap.js +289 -94
- package/server/public/app/chat-actions-menu-state.mjs +1 -0
- package/server/public/app/context-media-tab.js +143 -0
- package/server/public/app/conversation-view.js +90 -5
- package/server/public/app/image-composer-ui.js +518 -0
- package/server/public/app/image-turn-placeholder.mjs +233 -0
- package/server/public/app/journal-view.js +12 -2
- package/server/public/app/media-gallery-model.mjs +202 -0
- package/server/public/app/media-gallery-ui.js +524 -0
- package/server/public/app/media-view.js +79 -0
- package/server/public/app/model-refresh-ui.mjs +413 -0
- package/server/public/app/no-conversation-state.mjs +5 -2
- package/server/public/app/openai-image-composer.mjs +492 -0
- package/server/public/app/openai-image-cost-ui.js +426 -0
- package/server/public/app/openai-image-spend.mjs +443 -0
- package/server/public/app/plan-usage-view.mjs +38 -6
- package/server/public/app/provider-readiness.mjs +67 -4
- package/server/public/app/relay-error-ctas.mjs +5 -0
- package/server/public/app/session-history-ui.js +2 -0
- package/server/public/app/settings-modal.js +171 -0
- package/server/public/app/socket-handlers.js +47 -0
- package/server/public/app/tab-strip-fade.mjs +1 -1
- package/server/public/app/viewer-gallery.mjs +23 -1
- package/server/public/index.html +478 -9
- package/server/relay-tools.md +1 -0
- package/server/repositories/image-conversation-repository.mjs +69 -0
- package/server/repositories/media-repository.mjs +305 -0
- package/server/routes/agent-sdk-routes.mjs +73 -0
- package/server/routes/media-routes.mjs +361 -0
- package/server/routes/messages-routes.mjs +1467 -343
- package/server/routes/sessions-routes.mjs +596 -123
- package/server/server-runtime.mjs +414 -81
- package/server/services/about-service.mjs +26 -6
- package/server/services/agent-sdk-launch-gate.mjs +124 -0
- package/server/services/agent-sdk-paths.mjs +307 -0
- package/server/services/agent-sdk-store.mjs +1211 -0
- package/server/services/claude-api-settings-service.mjs +54 -17
- package/server/services/cli-install-service.mjs +3 -2
- package/server/services/copilot-model-discovery-service.mjs +7 -1
- package/server/services/database-file.mjs +23 -5
- package/server/services/deepseek-settings-service.mjs +58 -20
- package/server/services/earlier-database-service.mjs +4 -2
- package/server/services/generated-image-cleanup-service.mjs +2 -1
- package/server/services/generated-image-files.mjs +110 -0
- package/server/services/image-input-paths.mjs +389 -0
- package/server/services/image-operation-service.mjs +6 -2
- package/server/services/media-extract.mjs +653 -0
- package/server/services/media-index-service.mjs +221 -0
- package/server/services/model-refresh-scheduler.mjs +153 -0
- package/server/services/model-variant-catalog-service.mjs +141 -9
- package/server/services/oar-cli-helpers.mjs +72 -9
- package/server/services/oar-state-migration-service.mjs +2 -12
- package/server/services/oar-state-root.mjs +17 -0
- package/server/services/openai-image-preview-turns.mjs +330 -0
- package/server/services/openai-image-spend-service.mjs +516 -0
- package/server/services/openai-image-stream.mjs +170 -0
- package/server/services/plan-usage-openai-image.mjs +148 -0
- package/server/services/provider-model-refresh.mjs +326 -0
- package/server/services/provider-model-state.mjs +395 -0
- package/server/services/relay-singleton-guard.mjs +72 -10
- package/server/services/remote-relay-caller-context.mjs +6 -0
- package/server/services/remote-relay-dispatcher.mjs +386 -22
- package/server/services/remote-relay-registry-service.mjs +3 -1
- package/server/services/session-worker-launch-service.mjs +13 -0
- package/server/services/session-worker-supervisor-service.mjs +19 -1
- package/server/services/session-worker-websocket-service.mjs +15 -0
- package/server/services/token-usage-service.mjs +15 -0
- package/server/services/update-install-service.mjs +2 -2
- package/server/services/worker-launch-note-service.mjs +14 -2
- package/server/services/worker-recycle-service.mjs +145 -11
- package/shared/image-header.mjs +90 -0
- package/shared/model-refresh-schedule.mjs +120 -0
- package/shared/openai-image-options.mjs +299 -0
- package/shared/openai-image-presets.mjs +175 -0
- package/shared/openai-image-previews.mjs +40 -0
- package/shared/openai-image-prices.mjs +291 -0
- package/shared/openai-image-spend.mjs +90 -0
- package/shared/remote-relay-contract.mjs +126 -4
- package/shared/worker-runtime/direct-openai-image.mjs +98 -48
|
@@ -226,7 +226,9 @@ export function createRemoteRelayRegistry({
|
|
|
226
226
|
version: toText(version) || null,
|
|
227
227
|
platform: toText(platform) || null,
|
|
228
228
|
publicUrl: settings.publicUrl || null,
|
|
229
|
-
|
|
229
|
+
// imageInputs: send and create_session may carry images and image
|
|
230
|
+
// options here; an older relay would drop them and run the turn without.
|
|
231
|
+
remoteRelays: { protocol: REMOTE_RELAY_PROTOCOL, inbound: settings.inboundEnabled, imageInputs: true },
|
|
230
232
|
};
|
|
231
233
|
}
|
|
232
234
|
|
|
@@ -197,6 +197,16 @@ export function resolveWorkerKind(env = {}) {
|
|
|
197
197
|
return Object.hasOwn(NODE_WORKER_DESCRIPTORS, kind) ? kind : 'copilot';
|
|
198
198
|
}
|
|
199
199
|
|
|
200
|
+
// The agent SDK a worker kind runs on (agent-sdk-store.mjs), or null: the
|
|
201
|
+
// Claude workers (subscription, API key, DeepSeek) on the Claude SDK, Cursor
|
|
202
|
+
// on its own; every other kind needs none.
|
|
203
|
+
const WORKER_KIND_AGENT_SDK = Object.freeze({ claude: 'claude', 'claude-api': 'claude', deepseek: 'claude', cursor: 'cursor' });
|
|
204
|
+
|
|
205
|
+
export function agentSdkForWorkerKind(kind) {
|
|
206
|
+
const key = String(kind || '').trim().toLowerCase();
|
|
207
|
+
return Object.hasOwn(WORKER_KIND_AGENT_SDK, key) ? WORKER_KIND_AGENT_SDK[key] : null;
|
|
208
|
+
}
|
|
209
|
+
|
|
200
210
|
export function applyClaudeProviderEnvironment(env = {}, {
|
|
201
211
|
enabled = false,
|
|
202
212
|
model = '',
|
|
@@ -933,6 +943,9 @@ export const TMUX_WORKER_EXPORTED_ENV_VARS = Object.freeze([
|
|
|
933
943
|
// extension engine's "it is attached" marker.
|
|
934
944
|
'OAR_MCP_SERVER',
|
|
935
945
|
'OAR_MCP_SERVER_ATTACHED',
|
|
946
|
+
// The agent SDK folder a worker runs on for its whole life (agent-sdk-loader.mjs).
|
|
947
|
+
'OAR_AGENT_SDK_DIR_CLAUDE',
|
|
948
|
+
'OAR_AGENT_SDK_DIR_CURSOR',
|
|
936
949
|
'SESSION_ID',
|
|
937
950
|
'COPILOT_WORKSPACE_ROOT',
|
|
938
951
|
'INIT_CWD',
|
|
@@ -556,7 +556,8 @@ export function createSessionWorkerSupervisor({
|
|
|
556
556
|
if (!to || from === to) return true;
|
|
557
557
|
const allowed = {
|
|
558
558
|
new: new Set(['starting', 'ready', 'error', 'processing']),
|
|
559
|
-
|
|
559
|
+
// 'new': a start the agent SDK gate deferred, so no worker came up.
|
|
560
|
+
starting: new Set(['ready', 'error', 'new']),
|
|
560
561
|
ready: new Set(['processing', 'starting', 'error']),
|
|
561
562
|
processing: new Set(['ready', 'error']),
|
|
562
563
|
error: new Set(['starting', 'ready']),
|
|
@@ -683,6 +684,23 @@ export function createSessionWorkerSupervisor({
|
|
|
683
684
|
if (!isStartActive() || isKillBlocked(sessionId)) {
|
|
684
685
|
return buildStartBlockedResult();
|
|
685
686
|
}
|
|
687
|
+
// Not launched, and not a failure either (the agent SDK it needs is
|
|
688
|
+
// still installing, agent-sdk-launch-gate.mjs): no worker, no
|
|
689
|
+
// restart backoff, no launch note; the turn stays queued until the
|
|
690
|
+
// gate hands the session its launch back.
|
|
691
|
+
if (spawned?.deferred) {
|
|
692
|
+
const waitingState = setWorkerState(sessionId, {
|
|
693
|
+
sdkSessionId: sessionId,
|
|
694
|
+
// No worker yet ('new'), so nothing waits for its heartbeat.
|
|
695
|
+
status: 'new',
|
|
696
|
+
workerId: null,
|
|
697
|
+
pid: null,
|
|
698
|
+
queueDepth: startingState.queueDepth || 0,
|
|
699
|
+
retryCount: startingState.retryCount || 0,
|
|
700
|
+
lastError: null,
|
|
701
|
+
});
|
|
702
|
+
return { ok: true, deferred: String(spawned.deferred), reused: false, worker: waitingState, lifecycle: toLifecycleSnapshot(sessionId) };
|
|
703
|
+
}
|
|
686
704
|
const readyState = setWorkerState(sessionId, {
|
|
687
705
|
...startingState,
|
|
688
706
|
workerId: String(spawned?.workerId || startingState.workerId || `worker-${sessionId.slice(0, 8)}`).trim(),
|
|
@@ -132,6 +132,11 @@ export function createSessionWorkerWebSocketService({
|
|
|
132
132
|
// ready: (sessionId, buildId, pid). The registry keeps it, so the relay can
|
|
133
133
|
// tell a worker on older code from one that runs what the relay runs.
|
|
134
134
|
onWorkerBuildId = () => {},
|
|
135
|
+
// A worker said hello: ({ sessionId, pid, kind, buildId }), with what the
|
|
136
|
+
// socket knows of it. Called before the hello's own request for work, so
|
|
137
|
+
// the relay can register the process and fence it first (a stale worker
|
|
138
|
+
// reconnecting to a new relay: the worker recycle).
|
|
139
|
+
onWorkerHello = () => {},
|
|
135
140
|
// The relay's own build id, sent in every server.hello so a worker can log
|
|
136
141
|
// that it runs other code; null leaves the key out.
|
|
137
142
|
buildId = null,
|
|
@@ -572,6 +577,16 @@ export function createSessionWorkerWebSocketService({
|
|
|
572
577
|
meta.ready = payload.ready !== false;
|
|
573
578
|
meta.lastHelloAt = nowIso();
|
|
574
579
|
noteHeartbeat(meta, 'worker-hello');
|
|
580
|
+
if (meta.sessionId) {
|
|
581
|
+
try {
|
|
582
|
+
onWorkerHello({
|
|
583
|
+
sessionId: meta.sessionId,
|
|
584
|
+
pid: meta.pid || null,
|
|
585
|
+
kind: meta.workerKind || null,
|
|
586
|
+
buildId: meta.workerBuildId || null,
|
|
587
|
+
});
|
|
588
|
+
} catch {}
|
|
589
|
+
}
|
|
575
590
|
emitEvent(ws, serverHello('ack', meta.sessionId));
|
|
576
591
|
void maybeDeliverToSocket(ws, 'worker-hello');
|
|
577
592
|
return;
|
|
@@ -219,6 +219,8 @@ export function createTokenUsageService({
|
|
|
219
219
|
listAllDayRows: db.prepare(`SELECT * FROM ${TOKEN_USAGE_DAYS_TABLE}`),
|
|
220
220
|
listTurnRows: db.prepare(`SELECT * FROM ${TURN_TOKEN_USAGE_TABLE} WHERE turn_id = ? ORDER BY at, run_id, model`),
|
|
221
221
|
listConversationRows: db.prepare(`SELECT * FROM ${TURN_TOKEN_USAGE_TABLE} WHERE conversation_id = ? ORDER BY at, turn_id, run_id, model`),
|
|
222
|
+
listBillingDayRows: db.prepare(`SELECT * FROM ${TURN_TOKEN_USAGE_TABLE} WHERE day = ? AND provider = ? AND billing = ? ORDER BY at, turn_id, run_id, model`),
|
|
223
|
+
listBillingConversationRows: db.prepare(`SELECT * FROM ${TURN_TOKEN_USAGE_TABLE} WHERE conversation_id = ? AND provider = ? AND billing = ? ORDER BY at, turn_id, run_id, model`),
|
|
222
224
|
deleteConversationRows: db.prepare(`DELETE FROM ${TURN_TOKEN_USAGE_TABLE} WHERE conversation_id = ?`),
|
|
223
225
|
};
|
|
224
226
|
}
|
|
@@ -410,6 +412,18 @@ export function createTokenUsageService({
|
|
|
410
412
|
};
|
|
411
413
|
}
|
|
412
414
|
|
|
415
|
+
/**
|
|
416
|
+
* The turn rows of one provider and billing, of one day or of one chat
|
|
417
|
+
* (`day` or `conversationId`), for the views that need what the day rows
|
|
418
|
+
* do not keep (cost_source); null when the tables are not there.
|
|
419
|
+
*/
|
|
420
|
+
function readTurnRows({ provider, billing = '', day = null, conversationId = null } = {}) {
|
|
421
|
+
const s = ready();
|
|
422
|
+
if (!s) return null;
|
|
423
|
+
if (conversationId) return s.listBillingConversationRows.all(String(conversationId), provider, billing).map(turnRowOut);
|
|
424
|
+
return s.listBillingDayRows.all(String(day || localDay(now())), provider, billing).map(turnRowOut);
|
|
425
|
+
}
|
|
426
|
+
|
|
413
427
|
/** The chat is deleted: its turn rows go, the day rows stay. */
|
|
414
428
|
function forgetConversation(conversationId) {
|
|
415
429
|
const s = ready();
|
|
@@ -423,6 +437,7 @@ export function createTokenUsageService({
|
|
|
423
437
|
readPeriods,
|
|
424
438
|
readTurn,
|
|
425
439
|
readConversation,
|
|
440
|
+
readTurnRows,
|
|
426
441
|
forgetConversation,
|
|
427
442
|
localDay: (ms = now()) => localDay(ms),
|
|
428
443
|
timeZone: formatDay.resolvedOptions().timeZone,
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
tailOf,
|
|
13
13
|
} from './cli-process-runner.mjs';
|
|
14
14
|
import { NATIVE_BUILD_COMMAND, buildBetterSqlite3, needsSourceBuild } from './native-addon-build-service.mjs';
|
|
15
|
-
import {
|
|
15
|
+
import { resolveNpmInvocation } from './oar-cli-helpers.mjs';
|
|
16
16
|
import { compareSemverIsh, parseSemverIsh } from '../../shared/update-semver.mjs';
|
|
17
17
|
|
|
18
18
|
// Applies an OAR self-update: `npm i -g @oar-sh/oar@<version>` with the log
|
|
@@ -227,7 +227,7 @@ export function createUpdateInstallService({
|
|
|
227
227
|
// (args are fixed strings + a validated semver, so no injection surface).
|
|
228
228
|
const npm = resolveNpmInvocation({ execPath, packageRoot, platform, env, existsImpl });
|
|
229
229
|
const command = npm.command;
|
|
230
|
-
const args = ['install', '-g', ...npm.prefixArgs,
|
|
230
|
+
const args = ['install', '-g', ...npm.prefixArgs, `${UPDATE_PACKAGE_NAME}@${target}`];
|
|
231
231
|
const result = await runToCompletion(
|
|
232
232
|
() => spawnImpl(command, args, {
|
|
233
233
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
import { randomUUID } from 'node:crypto';
|
|
19
19
|
|
|
20
20
|
export const WORKER_LAUNCH_NOTE_KINDS = Object.freeze({
|
|
21
|
+
installing: 'worker-launch-installing',
|
|
21
22
|
failed: 'worker-launch-failed',
|
|
22
23
|
stopped: 'worker-launch-stopped',
|
|
23
24
|
started: 'worker-launch-started',
|
|
@@ -47,6 +48,11 @@ export function buildWorkerLaunchNoteText({ type, lifecycle = {}, retryEveryMs =
|
|
|
47
48
|
const tries = Math.max(0, Number(lifecycle?.retryCount || 0));
|
|
48
49
|
const triesText = `${tries} ${tries === 1 ? 'try' : 'tries'}`;
|
|
49
50
|
const cause = clipCause(lifecycle?.lastError);
|
|
51
|
+
if (type === 'installing') {
|
|
52
|
+
// The first chat of a provider whose agent SDK is not installed yet
|
|
53
|
+
// (agent-sdk-store.mjs): the relay installs it before the worker starts.
|
|
54
|
+
return `⏳ Installing the ${lifecycle?.sdkLabel || 'agent SDK'} on the relay host, once for this version. Your message is delivered when it is ready.`;
|
|
55
|
+
}
|
|
50
56
|
if (type === 'launched') {
|
|
51
57
|
return `✅ The worker for this conversation started at ${formatTime(lifecycle?.launchAt || new Date().toISOString())}. The queued message is being delivered.`;
|
|
52
58
|
}
|
|
@@ -77,6 +83,7 @@ export function createWorkerLaunchNoteService({
|
|
|
77
83
|
const episodes = new Map();
|
|
78
84
|
|
|
79
85
|
function kindFor(type) {
|
|
86
|
+
if (type === 'installing') return WORKER_LAUNCH_NOTE_KINDS.installing;
|
|
80
87
|
if (type === 'launched') return WORKER_LAUNCH_NOTE_KINDS.started;
|
|
81
88
|
if (type === 'stopped') return WORKER_LAUNCH_NOTE_KINDS.stopped;
|
|
82
89
|
return WORKER_LAUNCH_NOTE_KINDS.failed;
|
|
@@ -105,7 +112,7 @@ export function createWorkerLaunchNoteService({
|
|
|
105
112
|
stmts.updateConvTime?.run?.(at, conversationId);
|
|
106
113
|
});
|
|
107
114
|
write();
|
|
108
|
-
episodes.set(sdkSessionId, { messageId, conversationId, timestamp: at });
|
|
115
|
+
episodes.set(sdkSessionId, { messageId, conversationId, timestamp: at, kind, text });
|
|
109
116
|
try { emit('assistant_message', { conversationId, messageId, message: messagePayload({ messageId, text, kind, timestamp: at }) }); } catch {}
|
|
110
117
|
try { logger.log?.(`[worker-launch] conv=${conversationId.slice(0, 8)} note opened (${type}): ${clipCause(lifecycle?.lastError)}`); } catch {}
|
|
111
118
|
}
|
|
@@ -113,6 +120,11 @@ export function createWorkerLaunchNoteService({
|
|
|
113
120
|
function update(episode, sdkSessionId, { type, lifecycle }) {
|
|
114
121
|
const kind = kindFor(type);
|
|
115
122
|
const text = buildWorkerLaunchNoteText({ type, lifecycle, retryEveryMs, retryWindowMs, formatTime });
|
|
123
|
+
// The same note again (a launch deferred on every prime while an agent
|
|
124
|
+
// SDK installs): no write, no broadcast.
|
|
125
|
+
if (episode.kind === kind && episode.text === text) return;
|
|
126
|
+
episode.kind = kind;
|
|
127
|
+
episode.text = text;
|
|
116
128
|
updateNote.run(text, kind, episode.messageId);
|
|
117
129
|
try {
|
|
118
130
|
emit('message_updated', {
|
|
@@ -130,7 +142,7 @@ export function createWorkerLaunchNoteService({
|
|
|
130
142
|
/** The supervisor's onLaunchState listener. */
|
|
131
143
|
function handle({ type, sdkSessionId, lifecycle = null } = {}) {
|
|
132
144
|
const sid = String(sdkSessionId || '').trim();
|
|
133
|
-
if (!sid || !['exhausted', 'retry-failed', 'stopped', 'launched'].includes(type)) return false;
|
|
145
|
+
if (!sid || !['installing', 'exhausted', 'retry-failed', 'stopped', 'launched'].includes(type)) return false;
|
|
134
146
|
const episode = episodes.get(sid) || null;
|
|
135
147
|
if (episode) {
|
|
136
148
|
update(episode, sid, { type, lifecycle });
|
|
@@ -27,6 +27,31 @@
|
|
|
27
27
|
* `quietMs` without a delivery or a turn end for the session, the runners'
|
|
28
28
|
* own idle time.
|
|
29
29
|
*
|
|
30
|
+
* A stale worker on an agent SDK the store installs (`isAgentSdkWorker`: the
|
|
31
|
+
* claude, claude-api, deepseek and cursor kinds, unless a git checkout runs
|
|
32
|
+
* that SDK from its own node_modules) is the exception. An update moves the
|
|
33
|
+
* SDKs out of the package, so the files such a worker would start its next
|
|
34
|
+
* CLI child from may be gone, and a message it gets once its child has idled
|
|
35
|
+
* out fails. So:
|
|
36
|
+
*
|
|
37
|
+
* - it skips the 'cli' and 'quiet' holds and is recycled as soon as its turn,
|
|
38
|
+
* question and compaction holds (and the processing row) allow;
|
|
39
|
+
* - its background hold stays a hold, and is the one hold it is not fenced
|
|
40
|
+
* for: background tasks run in the CLI child, so while they do the child is
|
|
41
|
+
* up and a delivery goes to it, never to a new child from the gone files;
|
|
42
|
+
* - from the first sweep that sees it stale until its recycle, the
|
|
43
|
+
* replacement fence (below) keeps every delivery away from it, so what is
|
|
44
|
+
* queued waits for the new worker's cold start rather than failing on the
|
|
45
|
+
* old one;
|
|
46
|
+
* - a relay that has just started knows its holds only from its first
|
|
47
|
+
* heartbeat's steering snapshot: until that comes it waits ('steering'),
|
|
48
|
+
* fenced, for at most `steeringWaitMs`; one that still has sent none by
|
|
49
|
+
* then is judged as any other worker, unfenced.
|
|
50
|
+
*
|
|
51
|
+
* Besides the periodic sweep, a sweep runs at start and on every worker hello
|
|
52
|
+
* (`sweepNow`), so a stale worker that reconnects to a new relay is fenced
|
|
53
|
+
* before the hello's own request for work is answered.
|
|
54
|
+
*
|
|
30
55
|
* From the moment a recycle passes its holds until its launch is done, the
|
|
31
56
|
* replacement fence (worker-replacement.mjs) keeps every delivery away from
|
|
32
57
|
* the process being replaced, and the stop and the launch run under the
|
|
@@ -61,9 +86,10 @@ const LOG_TAG = '[worker-recycle]';
|
|
|
61
86
|
* the worker's own release holds (workerReleaseHold; 'turn' also for a
|
|
62
87
|
* processing row the session owns), 'cli' while its CLI child runs, and
|
|
63
88
|
* 'quiet' while a worker that does not report its CLI child has had a
|
|
64
|
-
* delivery or a turn end within `quietMs
|
|
89
|
+
* delivery or a turn end within `quietMs`, and 'steering' while a worker on
|
|
90
|
+
* an agent SDK has not yet sent the snapshot that names its holds.
|
|
65
91
|
*/
|
|
66
|
-
export const RECYCLE_HOLDS = Object.freeze(['turn', 'question', 'compaction', 'background', 'cli', 'quiet']);
|
|
92
|
+
export const RECYCLE_HOLDS = Object.freeze(['turn', 'question', 'compaction', 'background', 'cli', 'quiet', 'steering']);
|
|
67
93
|
const RECYCLABLE_STATUSES = new Set(['ready', 'processing']);
|
|
68
94
|
|
|
69
95
|
function sid8(sid) {
|
|
@@ -155,6 +181,15 @@ export function createWorkerRecycleService({
|
|
|
155
181
|
// without a delivery or a turn end: the Claude and Copilot runners'
|
|
156
182
|
// idleShutdownMs, after which they stop the child themselves.
|
|
157
183
|
quietMs = 10 * 60_000,
|
|
184
|
+
// worker → whether its kind runs on an agent SDK (agentSdkForWorkerKind):
|
|
185
|
+
// such a worker, once stale, skips the cli and quiet holds and is fenced
|
|
186
|
+
// until its recycle (see the header).
|
|
187
|
+
isAgentSdkWorker = () => false,
|
|
188
|
+
// How long a stale worker on an agent SDK may go without a steering
|
|
189
|
+
// snapshot (from when this service first saw its process) before it is
|
|
190
|
+
// judged as any other worker. Its heartbeat sends one within a second of
|
|
191
|
+
// a relay's hello.
|
|
192
|
+
steeringWaitMs = 30_000,
|
|
158
193
|
logger = console,
|
|
159
194
|
} = {}) {
|
|
160
195
|
if (typeof stopWorker !== 'function' || typeof launchWorker !== 'function') {
|
|
@@ -214,8 +249,13 @@ export function createWorkerRecycleService({
|
|
|
214
249
|
pendingLaunch: null,
|
|
215
250
|
lastWaitLogAt: -Infinity,
|
|
216
251
|
// When this service first saw the session's current pid: the quiet
|
|
217
|
-
// window's start for a worker with no start time and no queue rows
|
|
252
|
+
// window's start for a worker with no start time and no queue rows,
|
|
253
|
+
// and the steering wait's start.
|
|
218
254
|
firstSeen: null,
|
|
255
|
+
// The pid this service fenced as a stale worker on an agent SDK
|
|
256
|
+
// waiting for its recycle (sdkFenceUp), or null. A recycle admitted
|
|
257
|
+
// takes the fence over and clears this.
|
|
258
|
+
sdkFence: null,
|
|
219
259
|
};
|
|
220
260
|
bySession.set(sid, state);
|
|
221
261
|
}
|
|
@@ -394,8 +434,18 @@ export function createWorkerRecycleService({
|
|
|
394
434
|
});
|
|
395
435
|
} finally {
|
|
396
436
|
// A stop that did not go through (or never ran) leaves the old worker
|
|
397
|
-
// serving its session: it takes work again at once.
|
|
398
|
-
|
|
437
|
+
// serving its session: it takes work again at once. Not one on an
|
|
438
|
+
// agent SDK the update took away: its fence stays up through the
|
|
439
|
+
// stop's backoff, until the worker goes or the tries reach stuckAfter
|
|
440
|
+
// (the sweep keeps it, sdkFenced), so no queued row goes to it.
|
|
441
|
+
if (!stopped) {
|
|
442
|
+
if (isAgentSdkWorker(worker) === true && state.failures > 0 && state.failures < stuckAfter && stillSameProcess(sid, pid)) {
|
|
443
|
+
state.sdkFence = pid;
|
|
444
|
+
log(`fence kept ${sid8(sid)} pid ${pid}: its stop failed, deliveries wait for the retry`);
|
|
445
|
+
} else {
|
|
446
|
+
fence.release(sid, pid);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
399
449
|
}
|
|
400
450
|
}
|
|
401
451
|
|
|
@@ -454,17 +504,68 @@ export function createWorkerRecycleService({
|
|
|
454
504
|
* worker's start, else when this service first saw the process. Throws
|
|
455
505
|
* when a probe does; the caller counts that as a probe failure.
|
|
456
506
|
*/
|
|
457
|
-
function holdFor(sid, worker, state) {
|
|
507
|
+
function holdFor(sid, worker, state, sdkMode = null) {
|
|
458
508
|
if (hasRunningTurn(sid) === true) return 'turn';
|
|
509
|
+
if (sdkMode === 'steering') return 'steering';
|
|
459
510
|
const releaseHold = workerReleaseHold(worker);
|
|
460
511
|
if (releaseHold) return releaseHold;
|
|
512
|
+
// A stale worker on an agent SDK goes as soon as its own holds allow: its
|
|
513
|
+
// next CLI child may have no files left to start from.
|
|
514
|
+
if (sdkMode === 'fast') return null;
|
|
461
515
|
const cliAlive = worker?.steering?.cliAlive;
|
|
462
516
|
if (cliAlive === true) return 'cli';
|
|
463
517
|
if (cliAlive === false) return null;
|
|
518
|
+
const since = toTimeMs(quietSince(sid)) ?? toTimeMs(worker.startedAt) ?? firstSeenAt(state, worker);
|
|
519
|
+
return now() - since < quietMs ? 'quiet' : null;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// When this service first saw the worker's current process.
|
|
523
|
+
function firstSeenAt(state, worker) {
|
|
464
524
|
const pid = toPid(worker.pid);
|
|
465
525
|
if (state.firstSeen?.pid !== pid) state.firstSeen = { pid, at: now() };
|
|
466
|
-
|
|
467
|
-
|
|
526
|
+
return state.firstSeen.at;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/**
|
|
530
|
+
* How a stale worker on an agent SDK is judged: 'fast' (its steering
|
|
531
|
+
* snapshot names its holds: no cli or quiet hold, fenced until the
|
|
532
|
+
* recycle), 'steering' (no snapshot yet, within `steeringWaitMs` of when
|
|
533
|
+
* this service first saw the process: wait for it, fenced), or null (not on
|
|
534
|
+
* an agent SDK, or a snapshot never came: as any other worker). Throws
|
|
535
|
+
* when the predicate does; the caller counts that as a probe failure.
|
|
536
|
+
*/
|
|
537
|
+
function sdkModeFor(worker, state) {
|
|
538
|
+
if (isAgentSdkWorker(worker) !== true) return null;
|
|
539
|
+
if (worker?.steering && typeof worker.steering === 'object') return 'fast';
|
|
540
|
+
return now() - firstSeenAt(state, worker) < steeringWaitMs ? 'steering' : null;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// Raises the fence for a stale worker on an agent SDK until its recycle:
|
|
544
|
+
// whatever is queued for the session waits for the new worker.
|
|
545
|
+
function sdkFenceUp(sid, pid, state, confirmed) {
|
|
546
|
+
confirmed.add(sid);
|
|
547
|
+
if (state.sdkFence === pid && fence.isHeld(sid, pid)) return;
|
|
548
|
+
fence.hold(sid, pid);
|
|
549
|
+
if (state.sdkFence !== pid) log(`fenced ${sid8(sid)} pid ${pid}: stale on an agent SDK, deliveries wait for the recycle`);
|
|
550
|
+
state.sdkFence = pid;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// A recycle (or an archived stop) admitted for the session owns the fence
|
|
554
|
+
// from here: it lowers it when it is done.
|
|
555
|
+
function sdkFenceHandOver(state) {
|
|
556
|
+
state.sdkFence = null;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// Lowers the fences this service raised for workers no longer judged
|
|
560
|
+
// fenced by this sweep: gone, recycled, in a background hold, a probe that
|
|
561
|
+
// failed, a snapshot that never came.
|
|
562
|
+
function sdkFencesDown(confirmed) {
|
|
563
|
+
for (const [sid, state] of bySession) {
|
|
564
|
+
if (state.sdkFence === null || confirmed.has(sid)) continue;
|
|
565
|
+
const pid = state.sdkFence;
|
|
566
|
+
state.sdkFence = null;
|
|
567
|
+
if (fence.release(sid, pid)) log(`unfenced ${sid8(sid)} pid ${pid}`);
|
|
568
|
+
}
|
|
468
569
|
}
|
|
469
570
|
|
|
470
571
|
function noteWaiting(sid, state, hold, nextWaiting) {
|
|
@@ -509,6 +610,8 @@ export function createWorkerRecycleService({
|
|
|
509
610
|
return;
|
|
510
611
|
}
|
|
511
612
|
const running = new Set();
|
|
613
|
+
// The sessions whose agent-SDK fence this sweep keeps up (sdkFenceUp).
|
|
614
|
+
const sdkFenced = new Set();
|
|
512
615
|
for (const worker of workers) {
|
|
513
616
|
const sid = String(worker?.sdkSessionId || '').trim();
|
|
514
617
|
if (!sid) continue;
|
|
@@ -527,36 +630,51 @@ export function createWorkerRecycleService({
|
|
|
527
630
|
continue;
|
|
528
631
|
}
|
|
529
632
|
const state = stateFor(sid);
|
|
530
|
-
if (state.inFlight
|
|
633
|
+
if (state.inFlight) continue;
|
|
634
|
+
if (now() < state.nextAttemptAt) {
|
|
635
|
+
// A failed stop's agent-SDK fence stays up through the backoff, up to
|
|
636
|
+
// stuckAfter tries; then the old worker serves again.
|
|
637
|
+
if (state.sdkFence !== null && state.sdkFence === pid && state.failures < stuckAfter) sdkFenced.add(sid);
|
|
638
|
+
continue;
|
|
639
|
+
}
|
|
531
640
|
if (state.recycledPid && state.recycledPid === toPid(worker.pid)) continue;
|
|
532
641
|
let hold = null;
|
|
533
642
|
let archived = false;
|
|
643
|
+
let sdkMode = null;
|
|
534
644
|
try {
|
|
535
|
-
|
|
645
|
+
sdkMode = sdkModeFor(worker, state);
|
|
646
|
+
hold = holdFor(sid, worker, state, sdkMode);
|
|
536
647
|
if (!hold) archived = isArchived(sid) === true;
|
|
537
648
|
} catch (error) {
|
|
538
649
|
// The relay cannot tell whether the session is busy or archived: the
|
|
539
650
|
// worker is not stopped on a guess, and the probe is retried later.
|
|
651
|
+
// Nor is it fenced on a guess: an agent-SDK fence comes down.
|
|
540
652
|
noteFailure(sid, state, 'probe', errorText(error, 'unknown'));
|
|
541
653
|
continue;
|
|
542
654
|
}
|
|
543
655
|
if (hold) {
|
|
656
|
+
// Background tasks keep the CLI child up, so a delivery goes to it:
|
|
657
|
+
// the one hold such a worker waits out unfenced.
|
|
658
|
+
if (sdkMode && hold !== 'background') sdkFenceUp(sid, pid, state, sdkFenced);
|
|
544
659
|
noteWaiting(sid, state, hold, nextWaiting);
|
|
545
660
|
continue;
|
|
546
661
|
}
|
|
547
662
|
if (inFlight >= maxInFlight) {
|
|
548
663
|
deferred = true;
|
|
664
|
+
if (sdkMode) sdkFenceUp(sid, pid, state, sdkFenced);
|
|
549
665
|
continue;
|
|
550
666
|
}
|
|
551
667
|
if (archived) {
|
|
552
668
|
// Fenced from here: the holds just passed, and a delivery to the
|
|
553
669
|
// process being stopped would start a turn on it.
|
|
554
670
|
fence.hold(sid, pid);
|
|
671
|
+
sdkFenceHandOver(state);
|
|
555
672
|
startedSteps.push(begin(state, () => stopArchived(sid, worker, state)));
|
|
556
673
|
continue;
|
|
557
674
|
}
|
|
558
675
|
if (!canLaunchNow()) {
|
|
559
676
|
deferred = true;
|
|
677
|
+
if (sdkMode) sdkFenceUp(sid, pid, state, sdkFenced);
|
|
560
678
|
continue;
|
|
561
679
|
}
|
|
562
680
|
lastAdmitAt = now();
|
|
@@ -564,8 +682,10 @@ export function createWorkerRecycleService({
|
|
|
564
682
|
// await: no delivery can reach the old process between this check and
|
|
565
683
|
// its stop, nor between the stop and the launch.
|
|
566
684
|
fence.hold(sid, pid);
|
|
685
|
+
sdkFenceHandOver(state);
|
|
567
686
|
startedSteps.push(begin(state, () => recycle(sid, worker, state)));
|
|
568
687
|
}
|
|
688
|
+
sdkFencesDown(sdkFenced);
|
|
569
689
|
// Launches owed after a stop went through: the registry lists no new
|
|
570
690
|
// process for those sessions.
|
|
571
691
|
for (const [sid, state] of bySession) {
|
|
@@ -639,6 +759,20 @@ export function createWorkerRecycleService({
|
|
|
639
759
|
}
|
|
640
760
|
started = true;
|
|
641
761
|
scheduleNext();
|
|
762
|
+
// The start's own sweep: workers left from before the restart need not
|
|
763
|
+
// wait a whole interval.
|
|
764
|
+
runSweep();
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
/**
|
|
768
|
+
* A sweep now, outside the interval, while the service runs (a worker's
|
|
769
|
+
* hello). Its judging, the fence included, is done synchronously before
|
|
770
|
+
* this returns; the promise resolves once the recycles it started are done.
|
|
771
|
+
* Never rejects.
|
|
772
|
+
*/
|
|
773
|
+
function sweepNow() {
|
|
774
|
+
if (!started) return Promise.resolve();
|
|
775
|
+
return sweep().catch(() => {});
|
|
642
776
|
}
|
|
643
777
|
|
|
644
778
|
function stop() {
|
|
@@ -667,7 +801,7 @@ export function createWorkerRecycleService({
|
|
|
667
801
|
};
|
|
668
802
|
}
|
|
669
803
|
|
|
670
|
-
return { start, stop, sweep, isStale, status };
|
|
804
|
+
return { start, stop, sweep, sweepNow, isStale, status };
|
|
671
805
|
}
|
|
672
806
|
|
|
673
807
|
/**
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// The type and pixel size of a PNG, JPEG or WebP image, read from its first
|
|
2
|
+
// bytes alone (no decoding). Anything else, or a header that does not hold
|
|
3
|
+
// together, gives null.
|
|
4
|
+
//
|
|
5
|
+
// SDK-free and side-effect free: imported by the server and the browser.
|
|
6
|
+
|
|
7
|
+
// Enough for every PNG and WebP header and for the JPEG frame header after
|
|
8
|
+
// usual EXIF/ICC blocks; a JPEG whose frame header lies further in gives null.
|
|
9
|
+
export const IMAGE_HEADER_BYTES = 256 * 1024;
|
|
10
|
+
|
|
11
|
+
function asBytes(input) {
|
|
12
|
+
if (input instanceof Uint8Array) return input;
|
|
13
|
+
if (input instanceof ArrayBuffer) return new Uint8Array(input);
|
|
14
|
+
return null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function u16be(bytes, at) {
|
|
18
|
+
return (bytes[at] << 8) | bytes[at + 1];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function u32be(bytes, at) {
|
|
22
|
+
return ((bytes[at] << 24) >>> 0) + (bytes[at + 1] << 16) + (bytes[at + 2] << 8) + bytes[at + 3];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function u24le(bytes, at) {
|
|
26
|
+
return bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function ascii(bytes, at, length) {
|
|
30
|
+
let text = '';
|
|
31
|
+
for (let index = at; index < at + length && index < bytes.length; index += 1) text += String.fromCharCode(bytes[index]);
|
|
32
|
+
return text;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function sized(mime, width, height) {
|
|
36
|
+
return width > 0 && height > 0 ? { mime, width, height } : null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function readPng(bytes) {
|
|
40
|
+
// Signature, then IHDR as the first chunk: length 13, width, height.
|
|
41
|
+
if (bytes.length < 24 || ascii(bytes, 12, 4) !== 'IHDR') return null;
|
|
42
|
+
return sized('image/png', u32be(bytes, 16), u32be(bytes, 20));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function readJpeg(bytes) {
|
|
46
|
+
let at = 2;
|
|
47
|
+
while (at + 9 < bytes.length) {
|
|
48
|
+
if (bytes[at] !== 0xff) return null;
|
|
49
|
+
const marker = bytes[at + 1];
|
|
50
|
+
// Fill bytes, and markers without a length.
|
|
51
|
+
if (marker === 0xff) { at += 1; continue; }
|
|
52
|
+
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd8)) { at += 2; continue; }
|
|
53
|
+
if (marker === 0xd9 || marker === 0xda) return null;
|
|
54
|
+
const length = u16be(bytes, at + 2);
|
|
55
|
+
if (length < 2) return null;
|
|
56
|
+
const isFrame = marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc;
|
|
57
|
+
if (isFrame) return sized('image/jpeg', u16be(bytes, at + 7), u16be(bytes, at + 5));
|
|
58
|
+
at += 2 + length;
|
|
59
|
+
}
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function readWebp(bytes) {
|
|
64
|
+
if (bytes.length < 30) return null;
|
|
65
|
+
const chunk = ascii(bytes, 12, 4);
|
|
66
|
+
if (chunk === 'VP8X') return sized('image/webp', u24le(bytes, 24) + 1, u24le(bytes, 27) + 1);
|
|
67
|
+
if (chunk === 'VP8 ') {
|
|
68
|
+
// Key frame start code, then 14-bit width and height.
|
|
69
|
+
if (bytes[23] !== 0x9d || bytes[24] !== 0x01 || bytes[25] !== 0x2a) return null;
|
|
70
|
+
return sized('image/webp', (bytes[26] | (bytes[27] << 8)) & 0x3fff, (bytes[28] | (bytes[29] << 8)) & 0x3fff);
|
|
71
|
+
}
|
|
72
|
+
if (chunk === 'VP8L') {
|
|
73
|
+
if (bytes[20] !== 0x2f) return null;
|
|
74
|
+
const bits = bytes[21] | (bytes[22] << 8) | (bytes[23] << 16) | (bytes[24] << 24);
|
|
75
|
+
return sized('image/webp', (bits & 0x3fff) + 1, ((bits >>> 14) & 0x3fff) + 1);
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** `{ mime, width, height }` of a PNG, JPEG or WebP image's first bytes, or null. */
|
|
81
|
+
export function readImageHeader(input) {
|
|
82
|
+
const bytes = asBytes(input);
|
|
83
|
+
if (!bytes || bytes.length < 12) return null;
|
|
84
|
+
if (bytes[0] === 0x89 && ascii(bytes, 1, 3) === 'PNG' && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) {
|
|
85
|
+
return readPng(bytes);
|
|
86
|
+
}
|
|
87
|
+
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return readJpeg(bytes);
|
|
88
|
+
if (ascii(bytes, 0, 4) === 'RIFF' && ascii(bytes, 8, 4) === 'WEBP') return readWebp(bytes);
|
|
89
|
+
return null;
|
|
90
|
+
}
|