@oar-sh/oar 0.9.2 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/extensions/web-relay/extension.mjs +6 -1
- package/.github/extensions/web-relay/skills/tool-activity.mjs +7 -0
- package/CHANGELOG.md +336 -0
- package/README.md +455 -423
- package/bin/oar.js +25 -10
- package/package.json +11 -5
- package/server/README.md +44 -4
- package/server/claude-worker/claude-remote-relay-tool.mjs +99 -0
- package/server/claude-worker/claude-sdk-adapter.mjs +26 -4
- package/server/claude-worker/claude-session-process.mjs +1073 -69
- package/server/claude-worker/claude-session-worker.mjs +45 -4
- package/server/claude-worker/claude-turn-publisher.mjs +81 -20
- package/server/claude-worker/claude-worker-link-wiring.mjs +9 -0
- package/server/claude-worker/sdk-message-normalizer.mjs +7 -0
- package/server/copilot-worker/copilot-ask-user-tool.mjs +89 -0
- package/server/copilot-worker/copilot-continuation-signals.mjs +55 -0
- package/server/copilot-worker/copilot-question-bridge.mjs +34 -12
- package/server/copilot-worker/copilot-remote-relay-tool.mjs +59 -0
- package/server/copilot-worker/copilot-sdk-event-normalizer.mjs +113 -14
- package/server/copilot-worker/copilot-sdk-session-process.mjs +2349 -498
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +82 -20
- package/server/cursor-worker/cursor-remote-relay-tool.mjs +65 -0
- package/server/cursor-worker/cursor-turn-runner.mjs +45 -2
- package/server/cursor-worker/sdk-message-normalizer.mjs +25 -0
- package/server/db-schema.mjs +28 -0
- package/server/grok-worker/acp-client.mjs +92 -6
- package/server/grok-worker/grok-turn-runner.mjs +59 -8
- package/server/grok-worker/sdk-message-normalizer.mjs +13 -0
- package/server/mcp/oar-mcp-server.mjs +473 -0
- package/server/migrations/0004-steer-settle-markers.mjs +119 -0
- package/server/migrations/0005-remote-relays.mjs +90 -0
- package/server/public/app/action-confirmations.js +141 -13
- package/server/public/app/api-client.js +125 -2
- package/server/public/app/ask-user-view.js +102 -13
- package/server/public/app/attachments-view.js +37 -2
- package/server/public/app/background-task-timeout-options.mjs +9 -0
- package/server/public/app/background-tasks-view.mjs +33 -18
- package/server/public/app/bootstrap.js +55 -14
- package/server/public/app/composer-control-state.mjs +52 -34
- package/server/public/app/conversation-deep-link.mjs +42 -0
- package/server/public/app/conversation-draft-sync.mjs +187 -0
- package/server/public/app/conversation-list-filter.mjs +86 -0
- package/server/public/app/conversation-provider-indicator.mjs +8 -0
- package/server/public/app/conversation-view.js +1016 -200
- package/server/public/app/emoji-view.js +2 -0
- package/server/public/app/font-scaling.js +4 -37
- package/server/public/app/host-suspend-ui.js +272 -0
- package/server/public/app/journal-view.js +149 -6
- package/server/public/app/mention-autocomplete.mjs +298 -0
- package/server/public/app/model-selector-options.mjs +1 -1
- package/server/public/app/push-settings.js +1 -0
- package/server/public/app/pwa-install.js +8 -3
- package/server/public/app/question-multi-select.mjs +52 -0
- package/server/public/app/relay-prompt-sanitizer.mjs +61 -3
- package/server/public/app/remote-relay-origin-view.mjs +76 -0
- package/server/public/app/remote-relay-shared.mjs +85 -0
- package/server/public/app/remote-relays-settings.js +618 -0
- package/server/public/app/remote-relays-store.mjs +126 -0
- package/server/public/app/settings-modal.js +12 -8
- package/server/public/app/settings-tabs.js +1 -1
- package/server/public/app/socket-handlers.js +37 -1
- package/server/public/app/steered-turn-merge.mjs +100 -0
- package/server/public/app/store.js +126 -15
- package/server/public/app/sync-outbox.mjs +16 -8
- package/server/public/app/thinking-follow.js +260 -0
- package/server/public/app/tool-activity-emoji.mjs +40 -0
- package/server/public/app/transcript-viewport-anchor.mjs +210 -0
- package/server/public/index.html +463 -17
- package/server/public/sw.js +7 -0
- package/server/relay-tools.md +7 -1
- package/server/repositories/message-repository.mjs +68 -18
- package/server/repositories/question-repository.mjs +18 -0
- package/server/repositories/remote-relay-repository.mjs +75 -0
- package/server/repositories/session-repository.mjs +13 -32
- package/server/routes/ask-user-routes.mjs +42 -2
- package/server/routes/messages-routes.mjs +572 -56
- package/server/routes/remote-relay-routes.mjs +133 -0
- package/server/routes/sessions-routes.mjs +378 -84
- package/server/server-runtime.mjs +303 -25
- package/server/services/github-ci-activity-service.mjs +199 -0
- package/server/services/host-suspend-command.mjs +38 -0
- package/server/services/host-suspend-service.mjs +254 -0
- package/server/services/live-turn-picker.mjs +47 -0
- package/server/services/push-dispatch-service.mjs +37 -4
- package/server/services/relay-prompt-sanitizer.mjs +17 -2
- package/server/services/relay-question-context.mjs +46 -0
- package/server/services/remote-relay-caller-context.mjs +341 -0
- package/server/services/remote-relay-client.mjs +332 -0
- package/server/services/remote-relay-dispatcher.mjs +1500 -0
- package/server/services/remote-relay-inbound.mjs +187 -0
- package/server/services/remote-relay-pairing.mjs +189 -0
- package/server/services/remote-relay-registry-service.mjs +581 -0
- package/server/services/sdk-session-import-service.mjs +49 -7
- package/server/services/session-worker-launch-service.mjs +131 -12
- package/server/services/session-worker-registry-service.mjs +75 -0
- package/server/services/session-worker-supervisor-service.mjs +7 -1
- package/server/services/session-worker-websocket-service.mjs +23 -1
- package/server/services/steer-resend.mjs +45 -0
- package/shared/background-task-timeout.mjs +12 -4
- package/shared/control-poller.mjs +4 -1
- package/shared/question-multi-select.mjs +16 -0
- package/shared/relay-stop-failure.mjs +15 -0
- package/shared/remote-relay-contract.mjs +539 -0
- package/shared/remote-relay-mentions.mjs +129 -0
- package/shared/remote-relay-tool-core.mjs +314 -0
- package/shared/slash-command-guard.mjs +76 -0
- package/shared/steer-note.mjs +66 -0
- package/shared/steer-settle-failure.mjs +89 -0
- package/shared/steer-settle-markers.mjs +18 -0
- package/shared/worker-crash-guard.mjs +40 -25
- package/shared/worker-runtime/api-client.mjs +112 -22
- package/shared/worker-runtime/heartbeat.mjs +55 -1
- package/shared/worker-runtime/runner-link-wiring.mjs +44 -0
- package/shared/worker-runtime/worker-websocket-link.mjs +119 -8
- package/server/claude-worker/claude-session-test-harness.mjs +0 -250
- package/server/claude-worker/fixtures/workflow-run-record.json +0 -1
- package/server/copilot-worker/copilot-sdk-test-harness.mjs +0 -382
- package/server/copilot-worker/fixtures/abort-turn.json +0 -588
- package/server/copilot-worker/fixtures/ask-user-turn.json +0 -1154
- package/server/copilot-worker/fixtures/background-timer-continuation.json +0 -445
- package/server/copilot-worker/fixtures/background-timer-turn.json +0 -497
- package/server/copilot-worker/fixtures/happy-turn.json +0 -566
- package/server/copilot-worker/fixtures/quota-turn.json +0 -647
- package/server/copilot-worker/fixtures/reasoning-turn.json +0 -625
- package/server/copilot-worker/fixtures/resume-turn.json +0 -700
- package/server/copilot-worker/fixtures/subagent-turn.json +0 -1378
- package/server/copilot-worker/fixtures/tool-permission-turn.json +0 -1293
- package/server/routes/messages-routes-test-harness.mjs +0 -111
- package/server/services/fixtures/claude-auth-stub.cmd +0 -1
- package/server/services/fixtures/claude-auth-stub.mjs +0 -76
- package/server/services/fixtures/claude-auth-stub.sh +0 -4
- package/server/services/fixtures/cli-install-stub.cmd +0 -1
- package/server/services/fixtures/cli-install-stub.mjs +0 -98
- package/server/services/fixtures/cli-install-stub.sh +0 -4
- package/server/services/fixtures/grok-stub.cmd +0 -1
- package/server/services/fixtures/grok-stub.mjs +0 -89
- package/server/services/fixtures/grok-stub.sh +0 -4
- package/shared/fixtures/copilot-catalog-raw-2026-09-07.json +0 -2363
|
@@ -24,6 +24,9 @@ import {
|
|
|
24
24
|
normalizeWorkspaceRootKey,
|
|
25
25
|
} from './services/workspace-root-path-policy.mjs';
|
|
26
26
|
import { stopSessionWorkerProcesses } from './services/session-worker-stop-service.mjs';
|
|
27
|
+
import { pickLiveTurnRowId } from './services/live-turn-picker.mjs';
|
|
28
|
+
import { buildSteerSettleFailure, steerAgentLabelForProvider } from '../shared/steer-settle-failure.mjs';
|
|
29
|
+
import { sanitizeRelayQuestionContext as sanitizeRelayQuestionContextWith } from './services/relay-question-context.mjs';
|
|
27
30
|
import { isRealPathWithinRoot } from './services/workspace-symlink-guard.mjs';
|
|
28
31
|
import { applySchema } from './db-schema.mjs';
|
|
29
32
|
import { createSessionRepository } from './repositories/session-repository.mjs';
|
|
@@ -37,6 +40,13 @@ import { registerRelayBoardRoutes } from './routes/relay-board-routes.mjs';
|
|
|
37
40
|
import { registerCacheRoutes } from './routes/cache-routes.mjs';
|
|
38
41
|
import { registerGitRoutes } from './routes/git-routes.mjs';
|
|
39
42
|
import { registerPreviewRoutes } from './routes/preview-routes.mjs';
|
|
43
|
+
import { registerRemoteRelayRoutes } from './routes/remote-relay-routes.mjs';
|
|
44
|
+
import { createRemoteRelayRepository } from './repositories/remote-relay-repository.mjs';
|
|
45
|
+
import { createRemoteRelayClient } from './services/remote-relay-client.mjs';
|
|
46
|
+
import { createRemoteRelayRegistry } from './services/remote-relay-registry-service.mjs';
|
|
47
|
+
import { createRemoteRelayPairing } from './services/remote-relay-pairing.mjs';
|
|
48
|
+
import { createRemoteRelayCallerContext } from './services/remote-relay-caller-context.mjs';
|
|
49
|
+
import { createRemoteRelayDispatcher } from './services/remote-relay-dispatcher.mjs';
|
|
40
50
|
import { createDeleteArchiveService } from './services/delete-archive-service.mjs';
|
|
41
51
|
import { createStatusEventService } from './services/status-event-service.mjs';
|
|
42
52
|
import { createClaudeAuthService } from './services/claude-auth-service.mjs';
|
|
@@ -45,6 +55,9 @@ import { createGrokAuthService } from './services/grok-auth-service.mjs';
|
|
|
45
55
|
import { sweepUnreferencedUploads, UNREFERENCED_UPLOADS_QUERY } from './services/upload-sweep.mjs';
|
|
46
56
|
import webpush from 'web-push';
|
|
47
57
|
import { createPushDispatchService, ensurePushVapidKeys } from './services/push-dispatch-service.mjs';
|
|
58
|
+
import { createHostSuspendService } from './services/host-suspend-service.mjs';
|
|
59
|
+
import { createGitHubCiActivityService } from './services/github-ci-activity-service.mjs';
|
|
60
|
+
import { runHostSuspendToRam } from './services/host-suspend-command.mjs';
|
|
48
61
|
import { createActiveDeviceTracker } from './services/push-active-device-service.mjs';
|
|
49
62
|
import { registerPushRoutes } from './routes/push-routes.mjs';
|
|
50
63
|
import { createImageOperationService } from './services/image-operation-service.mjs';
|
|
@@ -3133,6 +3146,36 @@ function getRelayShutdownState() {
|
|
|
3133
3146
|
};
|
|
3134
3147
|
}
|
|
3135
3148
|
|
|
3149
|
+
// Every client shows a banner while a shutdown/restart is queued, so the
|
|
3150
|
+
// state goes out on each transition (request, cancel, drain) and on connect.
|
|
3151
|
+
function relayShutdownStatePayload() {
|
|
3152
|
+
return { ...getRelayShutdownState(), queue: queueCounts() };
|
|
3153
|
+
}
|
|
3154
|
+
function emitRelayShutdownState() {
|
|
3155
|
+
try { io.emit('relay_shutdown_state', relayShutdownStatePayload()); } catch {}
|
|
3156
|
+
}
|
|
3157
|
+
|
|
3158
|
+
function cancelRelayShutdown({ requestedBy = 'localhost-api' } = {}) {
|
|
3159
|
+
if (runtimeShutdownStarted) {
|
|
3160
|
+
return { cancelled: false, status: 'shutting_down', ...relayShutdownStatePayload() };
|
|
3161
|
+
}
|
|
3162
|
+
if (!pendingRelayShutdownRequest) {
|
|
3163
|
+
return { cancelled: false, ...relayShutdownStatePayload() };
|
|
3164
|
+
}
|
|
3165
|
+
const dropped = pendingRelayShutdownRequest;
|
|
3166
|
+
pendingRelayShutdownRequest = null;
|
|
3167
|
+
if (runtimeTimers.shutdownDrain) {
|
|
3168
|
+
try { clearInterval(runtimeTimers.shutdownDrain); } catch {}
|
|
3169
|
+
runtimeTimers.shutdownDrain = null;
|
|
3170
|
+
}
|
|
3171
|
+
console.log(
|
|
3172
|
+
`${runtimeLogPrefix()}Relay ${dropped.action || 'shutdown'} request cancelled by `
|
|
3173
|
+
+ `${String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api'}`,
|
|
3174
|
+
);
|
|
3175
|
+
emitRelayShutdownState();
|
|
3176
|
+
return { cancelled: true, dropped, ...relayShutdownStatePayload() };
|
|
3177
|
+
}
|
|
3178
|
+
|
|
3136
3179
|
function requestRelayShutdown({ reason = 'manual-request', requestedBy = 'localhost-api', restart = false } = {}) {
|
|
3137
3180
|
const safeReason = String(reason || 'manual-request').trim().slice(0, 140) || 'manual-request';
|
|
3138
3181
|
const safeRequestedBy = String(requestedBy || 'localhost-api').trim().slice(0, 80) || 'localhost-api';
|
|
@@ -3185,12 +3228,14 @@ function requestRelayShutdown({ reason = 'manual-request', requestedBy = 'localh
|
|
|
3185
3228
|
);
|
|
3186
3229
|
sessionWorkerWebSocketService.emitDraining(`api-${action}:${requestInfo?.reason || 'manual-request'}`);
|
|
3187
3230
|
void shutdownRuntime(`api-${action}:${requestInfo?.reason || 'manual-request'}`, { exitCode });
|
|
3231
|
+
emitRelayShutdownState();
|
|
3188
3232
|
};
|
|
3189
3233
|
|
|
3190
3234
|
if (!runtimeTimers.shutdownDrain) {
|
|
3191
3235
|
runtimeTimers.shutdownDrain = setInterval(tryShutdownWhenIdle, 1000);
|
|
3192
3236
|
}
|
|
3193
3237
|
tryShutdownWhenIdle();
|
|
3238
|
+
if (pendingRelayShutdownRequest) emitRelayShutdownState();
|
|
3194
3239
|
const counts = queueCounts();
|
|
3195
3240
|
return {
|
|
3196
3241
|
accepted: true,
|
|
@@ -4880,8 +4925,127 @@ const backgroundTaskStore = {
|
|
|
4880
4925
|
},
|
|
4881
4926
|
};
|
|
4882
4927
|
|
|
4928
|
+
// ─── Deferred host suspend ───────────────────────────────────────────────────
|
|
4929
|
+
// "💤 Suspend host" waits until nothing runs any more: no queued/processing
|
|
4930
|
+
// turn, no live background task of a live worker, no open CI run in a
|
|
4931
|
+
// workspace that was busy since the request. The relay-side signals are read
|
|
4932
|
+
// synchronously each tick; CI is refreshed in the background and read from
|
|
4933
|
+
// the last snapshot (a never-checked workspace counts as busy).
|
|
4934
|
+
const hostActivityWorkerStatuses = new Set(['starting', 'ready', 'processing']);
|
|
4935
|
+
function hostActivityPidAlive(pidValue) {
|
|
4936
|
+
const pid = Number(pidValue);
|
|
4937
|
+
if (!Number.isInteger(pid) || pid <= 0) return true;
|
|
4938
|
+
try {
|
|
4939
|
+
process.kill(pid, 0);
|
|
4940
|
+
return true;
|
|
4941
|
+
} catch (error) {
|
|
4942
|
+
return error?.code === 'EPERM';
|
|
4943
|
+
}
|
|
4944
|
+
}
|
|
4945
|
+
function hostActivityWorkerAlive(key) {
|
|
4946
|
+
const worker = sessionWorkerRegistry?.getWorker?.(key)
|
|
4947
|
+
|| sessionWorkerRegistry?.getWorkerByConversationId?.(key)
|
|
4948
|
+
|| null;
|
|
4949
|
+
if (!worker) return false;
|
|
4950
|
+
return hostActivityWorkerStatuses.has(String(worker.status || '').trim().toLowerCase())
|
|
4951
|
+
&& hostActivityPidAlive(worker.pid);
|
|
4952
|
+
}
|
|
4953
|
+
const hostSuspendWatchedConversationIds = new Set();
|
|
4954
|
+
const githubCiActivityService = createGitHubCiActivityService({ logger: console });
|
|
4955
|
+
function hostActivityTitle(conversationId, fallbackTitle = null) {
|
|
4956
|
+
const title = String(fallbackTitle || stmts.getConv?.get?.(conversationId)?.title || '').trim();
|
|
4957
|
+
return title || String(conversationId || '').slice(0, 8);
|
|
4958
|
+
}
|
|
4959
|
+
function collectHostActivity({ request = null } = {}) {
|
|
4960
|
+
const blockers = [];
|
|
4961
|
+
const turnsByConversation = stmts.listActiveQueueCountsByConversation?.all?.() || [];
|
|
4962
|
+
for (const row of turnsByConversation) {
|
|
4963
|
+
const conversationId = String(row.conversation_id || '');
|
|
4964
|
+
if (!conversationId) continue;
|
|
4965
|
+
hostSuspendWatchedConversationIds.add(conversationId);
|
|
4966
|
+
const count = Number(row.cnt || 0);
|
|
4967
|
+
blockers.push({
|
|
4968
|
+
kind: 'turn',
|
|
4969
|
+
conversationId,
|
|
4970
|
+
title: hostActivityTitle(conversationId, row.title),
|
|
4971
|
+
count,
|
|
4972
|
+
detail: count > 1 ? `${count} turns running` : 'turn running',
|
|
4973
|
+
});
|
|
4974
|
+
}
|
|
4975
|
+
for (const [key, entry] of backgroundTaskStore.sets) {
|
|
4976
|
+
const tasks = Array.isArray(entry?.tasks) ? entry.tasks : [];
|
|
4977
|
+
if (!tasks.length || !hostActivityWorkerAlive(key)) continue;
|
|
4978
|
+
const worker = sessionWorkerRegistry?.getWorker?.(key) || sessionWorkerRegistry?.getWorkerByConversationId?.(key) || null;
|
|
4979
|
+
const conversationId = String(worker?.conversationId || key);
|
|
4980
|
+
hostSuspendWatchedConversationIds.add(conversationId);
|
|
4981
|
+
const what = tasks.slice(0, 3).map((task) => String(task?.description || task?.taskType || 'task').slice(0, 80)).join('; ');
|
|
4982
|
+
blockers.push({
|
|
4983
|
+
kind: 'background',
|
|
4984
|
+
conversationId,
|
|
4985
|
+
title: hostActivityTitle(conversationId),
|
|
4986
|
+
count: tasks.length,
|
|
4987
|
+
detail: `${tasks.length} background agent${tasks.length === 1 ? '' : 's'}: ${what}`,
|
|
4988
|
+
});
|
|
4989
|
+
}
|
|
4990
|
+
const runningSubagents = Number(stmts.countRunningSubagentRuns?.get?.()?.cnt || 0);
|
|
4991
|
+
// CI is only worth a network round-trip for workspaces that were busy since
|
|
4992
|
+
// the request (or, for the modal's peek, that are busy right now).
|
|
4993
|
+
const roots = [...hostSuspendWatchedConversationIds]
|
|
4994
|
+
.map((conversationId) => {
|
|
4995
|
+
try { return resolveConversationWorkspaceState({ conversationId })?.currentWorkspaceRootPath || ''; } catch { return ''; }
|
|
4996
|
+
})
|
|
4997
|
+
.filter(Boolean);
|
|
4998
|
+
if (roots.length) void githubCiActivityService.describeRuns(roots).catch(() => {});
|
|
4999
|
+
const ci = roots.length ? githubCiActivityService.snapshot(roots) : { repos: [], unknown: [], blockers: [] };
|
|
5000
|
+
blockers.push(...ci.blockers);
|
|
5001
|
+
if (!request && !hostSuspendService.getState().pending) hostSuspendWatchedConversationIds.clear();
|
|
5002
|
+
return { blockers, runningSubagents, ci };
|
|
5003
|
+
}
|
|
5004
|
+
const hostSuspendService = createHostSuspendService({
|
|
5005
|
+
collectActivity: collectHostActivity,
|
|
5006
|
+
runSuspend: () => runHostSuspendToRam({ logger: console }),
|
|
5007
|
+
logger: console,
|
|
5008
|
+
onStateChange: (state) => {
|
|
5009
|
+
if (state.status === 'idle') hostSuspendWatchedConversationIds.clear();
|
|
5010
|
+
try { io.emit('host_suspend_state', state); } catch {}
|
|
5011
|
+
},
|
|
5012
|
+
onSuspended: () => {
|
|
5013
|
+
void pushDispatchService.notifyHostSuspend({ body: 'All agents finished; the host is suspending now.' });
|
|
5014
|
+
},
|
|
5015
|
+
onDropped: (state, reason) => {
|
|
5016
|
+
void pushDispatchService.notifyHostSuspend({ body: `Queued host suspend dropped: relay ${reason || 'restarting'}.` });
|
|
5017
|
+
},
|
|
5018
|
+
});
|
|
5019
|
+
|
|
5020
|
+
// The live turn among a conversation's processing rows. Mid-turn steering can
|
|
5021
|
+
// leave several processing at once — the running turn, steered messages the
|
|
5022
|
+
// CLI folded into it (no output of their own), a handed-off row still settling
|
|
5023
|
+
// — so the live turn is the row that produced output most recently; with
|
|
5024
|
+
// nothing to separate them yet, the oldest processing row (the turn a steered
|
|
5025
|
+
// message joins) wins. See live-turn-picker.mjs for why recency, not per-row
|
|
5026
|
+
// stream seq. Returns the whole row so callers get relay_mode/timestamps too.
|
|
5027
|
+
function resolveLiveTurnQueueRow(conversationId) {
|
|
5028
|
+
const rows = stmts.listProcessingQueueByConversation?.all(conversationId)
|
|
5029
|
+
|| (stmts.getLatestProcessingQueueByConversation.get(conversationId)
|
|
5030
|
+
? [stmts.getLatestProcessingQueueByConversation.get(conversationId)]
|
|
5031
|
+
: []);
|
|
5032
|
+
if (!rows.length) return null;
|
|
5033
|
+
if (rows.length === 1) return rows[0];
|
|
5034
|
+
const byId = new Map(rows.map((row) => [String(row.id), row]));
|
|
5035
|
+
const liveId = pickLiveTurnRowId(rows.map((row) => {
|
|
5036
|
+
const output = stmts.getLastOutputAtByQueueMessage?.get({ id: row.id }) || null;
|
|
5037
|
+
return {
|
|
5038
|
+
id: String(row.id),
|
|
5039
|
+
lastOutputAtMs: Date.parse(String(output?.last_output_at || '')) || 0,
|
|
5040
|
+
streamDone: Number(output?.stream_done || 0) === 1,
|
|
5041
|
+
processingAtMs: Date.parse(String(row.processing_at || row.timestamp || '')) || 0,
|
|
5042
|
+
};
|
|
5043
|
+
}));
|
|
5044
|
+
return byId.get(liveId) || rows[0];
|
|
5045
|
+
}
|
|
5046
|
+
|
|
4883
5047
|
function inFlightStateForConversation(conversationId) {
|
|
4884
|
-
const row =
|
|
5048
|
+
const row = resolveLiveTurnQueueRow(conversationId);
|
|
4885
5049
|
if (!row) return null;
|
|
4886
5050
|
const streamEvents = relayStreamEventsForQueueMessage(row.id);
|
|
4887
5051
|
const lastStreamEvent = streamEvents.length ? streamEvents[streamEvents.length - 1] : null;
|
|
@@ -4906,17 +5070,17 @@ function inFlightStateForConversation(conversationId) {
|
|
|
4906
5070
|
// retry_count like /api/requeue does, but bypasses that route's budget check,
|
|
4907
5071
|
// so the sweep enforces it here). Minimal mirror of the route-side
|
|
4908
5072
|
// failQueueMessage: mark failed, insert the assistant failure message, emit.
|
|
4909
|
-
function failRecoveredRowTerminally(row, reason) {
|
|
5073
|
+
function failRecoveredRowTerminally(row, reason, { failureRecord = null, failureText: presetText = null } = {}) {
|
|
4910
5074
|
const now = new Date().toISOString();
|
|
4911
5075
|
const responseId = uuidv4();
|
|
4912
5076
|
const retryCount = Number(row?.retry_count || 0) + 1;
|
|
4913
|
-
const failureText = `Relay recovery limit reached after ${retryCount} attempts (${reason}). The worker kept dying before completing this turn, so the message was failed to keep the queue moving. Send it again to retry.`;
|
|
5077
|
+
const failureText = presetText || `Relay recovery limit reached after ${retryCount} attempts (${reason}). The worker kept dying before completing this turn, so the message was failed to keep the queue moving. Send it again to retry.`;
|
|
4914
5078
|
const requestedModel = String(row?.model || '').trim() || null;
|
|
4915
5079
|
const tx = db.transaction(() => {
|
|
4916
5080
|
// setFailed is compare-and-set on status (processing/pending/parked): a
|
|
4917
5081
|
// row that settled between the sweep's listing and this write must not
|
|
4918
5082
|
// grow an orphan failure bubble on top of its real outcome.
|
|
4919
|
-
const result = stmts.setFailed.run(JSON.stringify({
|
|
5083
|
+
const result = stmts.setFailed.run(JSON.stringify(failureRecord || {
|
|
4920
5084
|
kind: 'recovery-limit',
|
|
4921
5085
|
error: 'recovery-limit',
|
|
4922
5086
|
code: 'recovery-limit',
|
|
@@ -4940,6 +5104,7 @@ function failRecoveredRowTerminally(row, reason) {
|
|
|
4940
5104
|
null,
|
|
4941
5105
|
requestedModel ? 'user' : 'auto',
|
|
4942
5106
|
);
|
|
5107
|
+
stmts.setMessageSourceId?.run(row.id, responseId);
|
|
4943
5108
|
stmts.linkActivityToResponse?.run(responseId, row.id);
|
|
4944
5109
|
stmts.linkStreamEventsToResponse?.run(responseId, row.id);
|
|
4945
5110
|
stmts.linkThoughtsToResponse?.run(responseId, row.id);
|
|
@@ -4954,7 +5119,20 @@ function failRecoveredRowTerminally(row, reason) {
|
|
|
4954
5119
|
message: { role: 'assistant', text: failureText, model: requestedModel || 'unknown', mode: String(row?.relay_mode || '').trim() || null, timestamp: now },
|
|
4955
5120
|
});
|
|
4956
5121
|
io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
|
|
4957
|
-
console.warn(`${runtimeLogPrefix()}RECOVERY LIMIT ${String(row.id).slice(0, 8)} conv=${String(row.conversation_id).slice(0, 8)} retries=${retryCount} reason=${reason}`);
|
|
5122
|
+
console.warn(`${runtimeLogPrefix()}${failureRecord ? 'CONSUMED FAILED' : 'RECOVERY LIMIT'} ${String(row.id).slice(0, 8)} conv=${String(row.conversation_id).slice(0, 8)} retries=${retryCount} reason=${reason}`);
|
|
5123
|
+
}
|
|
5124
|
+
|
|
5125
|
+
// A row whose prompt the worker's runtime already consumed (queue.consumed_at)
|
|
5126
|
+
// is at-most-once: recovery must never requeue it — a re-delivery would run the
|
|
5127
|
+
// prompt twice — so it fails terminally with the steer-settle wording, naming
|
|
5128
|
+
// the provider's runtime the way the workers' own settle wording does.
|
|
5129
|
+
function failConsumedRowTerminally(row, reason) {
|
|
5130
|
+
const providerType = stmts.getRuntimeSessionByConversation?.get?.(row?.conversation_id)?.provider_type;
|
|
5131
|
+
const failureRecord = buildSteerSettleFailure(row, { agentLabel: steerAgentLabelForProvider(providerType) });
|
|
5132
|
+
failRecoveredRowTerminally(row, reason, {
|
|
5133
|
+
failureRecord: { ...failureRecord, error: 'terminal-error', reason: String(reason || 'recovery') },
|
|
5134
|
+
failureText: `${failureRecord.message} Error code: ${failureRecord.stableCode}. ${failureRecord.guidance}`,
|
|
5135
|
+
});
|
|
4958
5136
|
}
|
|
4959
5137
|
|
|
4960
5138
|
function recoverProcessingOlderThan(cutoffIso, requeueAtIso, { ceilingBeforeIso = null } = {}) {
|
|
@@ -4966,7 +5144,9 @@ function recoverProcessingOlderThan(cutoffIso, requeueAtIso, { ceilingBeforeIso
|
|
|
4966
5144
|
stmts.dropStaleContinuation?.run?.(row.id);
|
|
4967
5145
|
io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
|
|
4968
5146
|
}
|
|
4969
|
-
const
|
|
5147
|
+
const listedRows = stmts.listRecoverableProcessing.all(params);
|
|
5148
|
+
for (const row of listedRows.filter((entry) => entry?.consumed_at)) failConsumedRowTerminally(row, 'stale-recovery');
|
|
5149
|
+
const allRows = listedRows.filter((entry) => !entry?.consumed_at);
|
|
4970
5150
|
if (!allRows.length) return [];
|
|
4971
5151
|
// Enforce the retry budget before requeueing: failing these rows first
|
|
4972
5152
|
// flips them out of 'processing' so the recovery UPDATE below skips them.
|
|
@@ -5345,6 +5525,13 @@ const app = express();
|
|
|
5345
5525
|
// via config.trustProxy (e.g. a hop count or subnet).
|
|
5346
5526
|
app.set('trust proxy', config.trustProxy ?? 'loopback');
|
|
5347
5527
|
const httpServer = http.createServer(app);
|
|
5528
|
+
// Node closes an idle keep-alive connection after 5 s. A client that pools
|
|
5529
|
+
// connections (a fronting proxy such as Caddy or cloudflared, an API client)
|
|
5530
|
+
// can pick such a socket in the very moment the relay closes it; its request
|
|
5531
|
+
// is then reset before the relay reads a byte, and a POST is not replayed.
|
|
5532
|
+
// Holding idle connections longer than those pools do closes that window.
|
|
5533
|
+
httpServer.keepAliveTimeout = 65_000;
|
|
5534
|
+
httpServer.headersTimeout = 66_000;
|
|
5348
5535
|
httpServer.prependListener('request', (req, _res) => {
|
|
5349
5536
|
rewriteSocketIoRequestPath(req, remotePath);
|
|
5350
5537
|
});
|
|
@@ -5772,6 +5959,10 @@ function recoverDeadWorkerProcessingRows(sessionId, reason) {
|
|
|
5772
5959
|
io.emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
|
|
5773
5960
|
continue;
|
|
5774
5961
|
}
|
|
5962
|
+
if (row.consumed_at) {
|
|
5963
|
+
failConsumedRowTerminally(row, `worker-died-${reason}`);
|
|
5964
|
+
continue;
|
|
5965
|
+
}
|
|
5775
5966
|
if (Number(row.retry_count || 0) + 1 >= MAX_REQUEUE_RETRIES) {
|
|
5776
5967
|
failRecoveredRowTerminally(row, `worker-died-${reason}`);
|
|
5777
5968
|
continue;
|
|
@@ -5814,11 +6005,15 @@ function handleWorkerSocketClosed({ sessionId, pid = null } = {}) {
|
|
|
5814
6005
|
const sid = String(sessionId || '').trim();
|
|
5815
6006
|
if (!sid) return;
|
|
5816
6007
|
const owned = stmts.listProcessingRowsForOwner?.all?.(sid) || [];
|
|
5817
|
-
|
|
6008
|
+
// A dead worker's steering snapshot must not keep holding the composer even
|
|
6009
|
+
// when it owned no rows (the hold can outlive them, e.g. a continuation's).
|
|
6010
|
+
const heldSteering = Boolean(sessionWorkerRegistry?.getWorker?.(sid)?.steering);
|
|
6011
|
+
if (!owned.length && !heldSteering) return;
|
|
5818
6012
|
const timer = setTimeout(() => {
|
|
5819
6013
|
try {
|
|
5820
6014
|
if (runtimeShutdownStarted) return;
|
|
5821
6015
|
if (isWorkerSessionProcessAlive(sid, pid)) return; // reconnect or socket flap
|
|
6016
|
+
sessionWorkerRegistry?.clearSteering?.(sid);
|
|
5822
6017
|
const stillOwned = stmts.listProcessingRowsForOwner?.all?.(sid) || [];
|
|
5823
6018
|
if (!stillOwned.length) return;
|
|
5824
6019
|
console.warn(`${runtimeLogPrefix()}WORKER DIED session=${sid.slice(0, 8)} rows=${stillOwned.length} trigger=socket-close — recovering owned turns`);
|
|
@@ -5986,6 +6181,7 @@ const runtimeState = {
|
|
|
5986
6181
|
set relayPaused(value) { relayPaused = value; },
|
|
5987
6182
|
get ttyConsoleActive() { return Boolean(ttyConsoleRuntime?.tty); },
|
|
5988
6183
|
get relayShutdown() { return getRelayShutdownState(); },
|
|
6184
|
+
get hostSuspend() { return hostSuspendService.getState(); },
|
|
5989
6185
|
get activeBridgeOwner() { return relayBridgeOwnerService.getOwner(); },
|
|
5990
6186
|
get workerWebSocketStatus() { return sessionWorkerWebSocketService.status(); },
|
|
5991
6187
|
get tmuxInspectorStatus() { return tmuxInspectorSocketService.status(); },
|
|
@@ -6002,6 +6198,82 @@ const windowsBootAutostartService = createWindowsBootAutostartService({
|
|
|
6002
6198
|
});
|
|
6003
6199
|
|
|
6004
6200
|
const gitChangesService = createGitChangesService();
|
|
6201
|
+
|
|
6202
|
+
// ─── Remote Relays ────────────────────────────────────────────────────────────
|
|
6203
|
+
// Other OAR relays this one is paired with, so its agents can work there. Built
|
|
6204
|
+
// before the shared route deps: the message, session and question routes read
|
|
6205
|
+
// the registry for mentions, unlocks and the inbound switch.
|
|
6206
|
+
// Null when migration 0005 failed this boot (db-schema retries it next boot):
|
|
6207
|
+
// remotes still list and pair, nothing unlocks until then.
|
|
6208
|
+
const remoteRelayRepository = (() => {
|
|
6209
|
+
try {
|
|
6210
|
+
return createRemoteRelayRepository(db);
|
|
6211
|
+
} catch (error) {
|
|
6212
|
+
console.warn(`[remote-relays] unlock storage unavailable: ${error?.message || error}`);
|
|
6213
|
+
return null;
|
|
6214
|
+
}
|
|
6215
|
+
})();
|
|
6216
|
+
const remoteRelayClient = createRemoteRelayClient({
|
|
6217
|
+
// A closure: the registry owning the instance id is declared just below.
|
|
6218
|
+
getOwnRelayId: () => remoteRelayRegistry.instanceId(),
|
|
6219
|
+
getOwnToken: () => config.authToken,
|
|
6220
|
+
});
|
|
6221
|
+
const remoteRelayRegistry = createRemoteRelayRegistry({
|
|
6222
|
+
readSetting: readAppSettingValue,
|
|
6223
|
+
writeSetting: (key, value) => stmts.upsertAppSetting.run(key, value, new Date().toISOString()),
|
|
6224
|
+
client: remoteRelayClient,
|
|
6225
|
+
repository: remoteRelayRepository,
|
|
6226
|
+
getOwnToken: () => config.authToken,
|
|
6227
|
+
getSelfName: getPwaAppName,
|
|
6228
|
+
hostname: os.hostname(),
|
|
6229
|
+
platform: process.platform,
|
|
6230
|
+
version: RUNNING_VERSION,
|
|
6231
|
+
emit: (event, payload) => io.emit(event, payload),
|
|
6232
|
+
logger: console,
|
|
6233
|
+
});
|
|
6234
|
+
const remoteRelayPairing = createRemoteRelayPairing({
|
|
6235
|
+
registry: remoteRelayRegistry,
|
|
6236
|
+
client: remoteRelayClient,
|
|
6237
|
+
getOwnToken: () => config.authToken,
|
|
6238
|
+
});
|
|
6239
|
+
// What the inbound routes need: no tokens, only ids, names and state.
|
|
6240
|
+
const remoteRelayInbound = {
|
|
6241
|
+
listRelays: () => remoteRelayRegistry.list().map(({ id, name, url, lastStatus, version }) => ({ id, name, url, lastStatus, version })),
|
|
6242
|
+
selfNames: () => [remoteRelayRegistry.selfName()],
|
|
6243
|
+
inboundEnabled: () => remoteRelayRegistry.getSelfSettings().inboundEnabled,
|
|
6244
|
+
recordUnlock: (conversationId, relayId, messageId) => remoteRelayRepository?.recordUnlock(conversationId, relayId, messageId) ?? false,
|
|
6245
|
+
describeRelay: (id) => {
|
|
6246
|
+
const relay = remoteRelayRegistry.get(id);
|
|
6247
|
+
return relay ? { id: relay.id, name: relay.name, online: relay.lastStatus === 'online', version: relay.version } : null;
|
|
6248
|
+
},
|
|
6249
|
+
repository: remoteRelayRepository,
|
|
6250
|
+
};
|
|
6251
|
+
// The remote_relay tool: who is calling (live turn, provider, model, mode,
|
|
6252
|
+
// hops) and the Allow / Deny card write actions need in ask/plan mode.
|
|
6253
|
+
const remoteRelayCallerContext = createRemoteRelayCallerContext({
|
|
6254
|
+
db,
|
|
6255
|
+
repository: remoteRelayRepository,
|
|
6256
|
+
questions: stmts,
|
|
6257
|
+
resolveLiveTurnQueueRow,
|
|
6258
|
+
formatQuestionRow,
|
|
6259
|
+
normalizeRelayMode,
|
|
6260
|
+
defaultRelayMode: DEFAULT_RELAY_MODE,
|
|
6261
|
+
emit: (event, payload) => io.emit(event, payload),
|
|
6262
|
+
notifyQuestion: (question) => pushDispatchService.notifyQuestion?.(question),
|
|
6263
|
+
uuid: uuidv4,
|
|
6264
|
+
logger: console,
|
|
6265
|
+
});
|
|
6266
|
+
const remoteRelayDispatcher = createRemoteRelayDispatcher({
|
|
6267
|
+
registry: remoteRelayRegistry,
|
|
6268
|
+
client: remoteRelayClient,
|
|
6269
|
+
repository: remoteRelayRepository,
|
|
6270
|
+
getCallerContext: (conversationId) => remoteRelayCallerContext.getCallerContext(conversationId),
|
|
6271
|
+
requestApproval: (request) => remoteRelayCallerContext.requestApproval(request),
|
|
6272
|
+
emitStatusEvent: ({ type, ...details }) => statusEventService.recordEvent(type, details),
|
|
6273
|
+
resolveConversationId: (id) => resolveConversationRecord({ conversationId: id, sdkSessionId: id })?.id || id,
|
|
6274
|
+
logger: console,
|
|
6275
|
+
});
|
|
6276
|
+
|
|
6005
6277
|
const sharedRouteDeps = {
|
|
6006
6278
|
auth,
|
|
6007
6279
|
gitChangesService,
|
|
@@ -6113,6 +6385,7 @@ const sharedRouteDeps = {
|
|
|
6113
6385
|
previewCardsForResponse,
|
|
6114
6386
|
sanitizeActivityText,
|
|
6115
6387
|
inFlightStateForConversation,
|
|
6388
|
+
resolveLiveTurnQueueRow,
|
|
6116
6389
|
backgroundTaskStore,
|
|
6117
6390
|
// A closure, not the registry itself: this deps object is built before the
|
|
6118
6391
|
// preview lane is constructed further down the file.
|
|
@@ -6184,6 +6457,10 @@ const sharedRouteDeps = {
|
|
|
6184
6457
|
getPwaAppName,
|
|
6185
6458
|
setPwaAppName,
|
|
6186
6459
|
requestRelayShutdown,
|
|
6460
|
+
cancelRelayShutdown,
|
|
6461
|
+
relayShutdownStatePayload,
|
|
6462
|
+
hostSuspendService,
|
|
6463
|
+
collectHostActivity,
|
|
6187
6464
|
markSharedViewerPresence,
|
|
6188
6465
|
getSharedWatcherCount,
|
|
6189
6466
|
statusEventService,
|
|
@@ -6198,6 +6475,7 @@ const sharedRouteDeps = {
|
|
|
6198
6475
|
windowsAutostartService,
|
|
6199
6476
|
windowsBootAutostartService,
|
|
6200
6477
|
pushDispatchService,
|
|
6478
|
+
remoteRelayInbound,
|
|
6201
6479
|
};
|
|
6202
6480
|
registerMessagesRoutes(app, sharedRouteDeps);
|
|
6203
6481
|
registerSessionsRoutes(app, sharedRouteDeps);
|
|
@@ -6210,6 +6488,12 @@ registerPushRoutes(app, {
|
|
|
6210
6488
|
pushDispatchService,
|
|
6211
6489
|
getPushVapidPublicKey: () => pushVapidKeys.publicKey,
|
|
6212
6490
|
});
|
|
6491
|
+
registerRemoteRelayRoutes(app, {
|
|
6492
|
+
auth,
|
|
6493
|
+
registry: remoteRelayRegistry,
|
|
6494
|
+
pairing: remoteRelayPairing,
|
|
6495
|
+
dispatcher: remoteRelayDispatcher,
|
|
6496
|
+
});
|
|
6213
6497
|
function markCliOffline(reason = 'offline', { clearOwner = true } = {}) {
|
|
6214
6498
|
cliLastSeen = null;
|
|
6215
6499
|
const wasOnline = cliOnline;
|
|
@@ -6523,25 +6807,10 @@ function sanitizeRelayQuestionRequest(requestBody) {
|
|
|
6523
6807
|
}
|
|
6524
6808
|
}
|
|
6525
6809
|
|
|
6810
|
+
// The allow-list lives in services/relay-question-context.mjs (unit-tested);
|
|
6811
|
+
// this binds it to the relay's own mode normalizer.
|
|
6526
6812
|
function sanitizeRelayQuestionContext(rawContext) {
|
|
6527
|
-
|
|
6528
|
-
const context = {};
|
|
6529
|
-
if (typeof rawContext.source === 'string' && rawContext.source.trim()) {
|
|
6530
|
-
context.source = rawContext.source.trim().slice(0, 64);
|
|
6531
|
-
}
|
|
6532
|
-
if (typeof rawContext.rationale === 'string' && rawContext.rationale.trim()) {
|
|
6533
|
-
context.rationale = rawContext.rationale.trim().slice(0, 240);
|
|
6534
|
-
}
|
|
6535
|
-
if (typeof rawContext.queueMessageId === 'string' && rawContext.queueMessageId.trim()) {
|
|
6536
|
-
context.queueMessageId = rawContext.queueMessageId.trim();
|
|
6537
|
-
}
|
|
6538
|
-
if (typeof rawContext.conversationId === 'string' && rawContext.conversationId.trim()) {
|
|
6539
|
-
context.conversationId = rawContext.conversationId.trim();
|
|
6540
|
-
}
|
|
6541
|
-
if (typeof rawContext.relayMode === 'string' && rawContext.relayMode.trim()) {
|
|
6542
|
-
context.relayMode = normalizeRelayMode(rawContext.relayMode) || DEFAULT_RELAY_MODE;
|
|
6543
|
-
}
|
|
6544
|
-
return Object.keys(context).length ? context : null;
|
|
6813
|
+
return sanitizeRelayQuestionContextWith(rawContext, { normalizeRelayMode, defaultRelayMode: DEFAULT_RELAY_MODE });
|
|
6545
6814
|
}
|
|
6546
6815
|
|
|
6547
6816
|
|
|
@@ -6579,6 +6848,9 @@ io.on('connection', (socket) => {
|
|
|
6579
6848
|
});
|
|
6580
6849
|
// Send current CLI status immediately on connect
|
|
6581
6850
|
socket.emit('cli_status', { online: cliOnline });
|
|
6851
|
+
// Pending host suspend / relay restart banners must survive a reconnect.
|
|
6852
|
+
try { socket.emit('host_suspend_state', hostSuspendService.getState()); } catch {}
|
|
6853
|
+
try { socket.emit('relay_shutdown_state', relayShutdownStatePayload()); } catch {}
|
|
6582
6854
|
// Visibility heartbeats for push suppression; also clears a stale
|
|
6583
6855
|
// deviceVisible flag restored by connectionStateRecovery.
|
|
6584
6856
|
activeDeviceTracker.registerSocket(socket);
|
|
@@ -6713,6 +6985,9 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
|
|
|
6713
6985
|
runtimeShutdownStarted = true;
|
|
6714
6986
|
console.log(`${runtimeLogPrefix()}Runtime shutdown started (${reason}, exitCode=${runtimeShutdownExitCode})`);
|
|
6715
6987
|
clearRuntimeTimers();
|
|
6988
|
+
// A queued host suspend is in-memory only; drop it loudly rather than let
|
|
6989
|
+
// the next runtime forget about it in silence.
|
|
6990
|
+
try { hostSuspendService.dispose({ reason }); } catch {}
|
|
6716
6991
|
try { updateCheckService?.stop(); } catch {}
|
|
6717
6992
|
// Started here (flips the importer's closing flag synchronously) but awaited
|
|
6718
6993
|
// below before the transports close: fire-and-forget disposal let a
|
|
@@ -6741,6 +7016,7 @@ function shutdownRuntime(reason = 'unknown', { exitCode = 0 } = {}) {
|
|
|
6741
7016
|
sshTunnelManager.stop();
|
|
6742
7017
|
cloudflaredTunnelManager.stop();
|
|
6743
7018
|
previewHealthProbe.stop();
|
|
7019
|
+
try { remoteRelayRegistry.stopHealthLoop(); } catch {}
|
|
6744
7020
|
// Previews are in-memory by design: a restart is the only implicit cleanup,
|
|
6745
7021
|
// so every public link dies with the process.
|
|
6746
7022
|
previewRegistry.clear();
|
|
@@ -6873,6 +7149,8 @@ httpServer.listen(config.port, listenHost, () => {
|
|
|
6873
7149
|
// Start SSH tunnel after server is listening
|
|
6874
7150
|
sshTunnelManager.start();
|
|
6875
7151
|
cloudflaredTunnelManager.start();
|
|
7152
|
+
// Status, version and renames of paired relays; a first pass right away.
|
|
7153
|
+
remoteRelayRegistry.startHealthLoop();
|
|
6876
7154
|
// A preview-lane failure must never take the relay with it: the listener is
|
|
6877
7155
|
// optional infrastructure, so a bind error is logged and the relay carries on
|
|
6878
7156
|
// without previews.
|