@oar-sh/oar 0.9.20 → 0.9.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +212 -0
- package/README.md +13 -13
- package/bin/oar.js +2 -2
- package/package.json +1 -1
- package/server/claude-cloud-worker/claude-cloud-session-process.mjs +231 -1
- package/server/claude-cloud-worker/claude-cloud-session-worker.mjs +22 -1
- package/server/claude-cloud-worker/claude-cloud-worker-steering.mjs +34 -0
- package/server/claude-worker/claude-session-process.mjs +7 -0
- package/server/claude-worker/claude-session-worker-main.mjs +8 -1
- package/server/copilot-worker/copilot-sdk-session-process.mjs +74 -1
- package/server/copilot-worker/copilot-sdk-session-worker.mjs +8 -1
- package/server/cursor-worker/cursor-session-worker.mjs +12 -1
- package/server/cursor-worker/cursor-worker-steering.mjs +20 -0
- package/server/grok-worker/grok-background-tasks.mjs +9 -0
- package/server/grok-worker/grok-session-worker.mjs +12 -1
- package/server/grok-worker/grok-turn-runner.mjs +3 -0
- package/server/grok-worker/grok-worker-steering.mjs +23 -0
- package/server/public/app/about-ui.js +1 -1
- package/server/public/app/about-view.mjs +19 -0
- package/server/public/app/action-confirmations.js +1 -1
- package/server/public/app/api-client.js +4 -2
- package/server/public/app/ask-user-view.js +20 -6
- package/server/public/app/attachments-view.js +1 -1
- package/server/public/app/back-guard.mjs +31 -5
- package/server/public/app/bootstrap.js +556 -68
- package/server/public/app/chat-actions-menu-state.mjs +139 -4
- package/server/public/app/chat-header-meta.mjs +27 -0
- package/server/public/app/claude-billing-toggle-ui.js +2 -1
- package/server/public/app/claude-cloud-conversation-ui.js +2 -2
- package/server/public/app/composer-settings-label.mjs +90 -0
- package/server/public/app/composer-settings-ui.js +442 -0
- package/server/public/app/context-menu.mjs +36 -5
- package/server/public/app/conversation-deep-link.mjs +15 -11
- package/server/public/app/conversation-group-drag.mjs +12 -2
- package/server/public/app/conversation-group-order.mjs +9 -2
- package/server/public/app/conversation-group-state.mjs +47 -10
- package/server/public/app/conversation-origin-ui.js +17 -1
- package/server/public/app/conversation-view.js +3 -0
- package/server/public/app/cwd-menu-placement.mjs +18 -6
- package/server/public/app/cwd-picker.js +36 -6
- package/server/public/app/emoji-view.js +1 -1
- package/server/public/app/font-scaling.js +26 -0
- package/server/public/app/header-priority-actions.mjs +234 -0
- package/server/public/app/hidden-messages-view.mjs +4 -0
- package/server/public/app/inbox-ui.js +18 -2
- package/server/public/app/journal-view.js +311 -58
- package/server/public/app/phone-landscape-ui.js +225 -0
- package/server/public/app/phone-landscape.mjs +99 -0
- package/server/public/app/phone-typing-state.mjs +104 -0
- package/server/public/app/pinned-messages-view.mjs +5 -0
- package/server/public/app/push-settings.js +2 -2
- package/server/public/app/pwa-install.js +24 -3
- package/server/public/app/relay-event-subscriptions-ui.js +71 -22
- package/server/public/app/relay-event-view.mjs +38 -0
- package/server/public/app/relay-restart-ui.js +2 -2
- package/server/public/app/relay-schedule-view.mjs +32 -6
- package/server/public/app/relay-schedules-ui.js +69 -20
- package/server/public/app/remote-relay-origin-view.mjs +41 -23
- package/server/public/app/session-history-ui.js +250 -0
- package/server/public/app/session-history.mjs +599 -0
- package/server/public/app/settings-modal.js +61 -6
- package/server/public/app/settings-tabs.js +12 -0
- package/server/public/app/share-state.js +1 -1
- package/server/public/app/shared-conversations-settings.js +2 -2
- package/server/public/app/socket-handlers.js +5 -2
- package/server/public/app/status-view.mjs +18 -2
- package/server/public/app/store.js +87 -16
- package/server/public/app/subagent-viewer.mjs +27 -2
- package/server/public/app/tab-strip-fade.mjs +101 -0
- package/server/public/app/tap-actions.js +66 -0
- package/server/public/app/usage-limit-ui.js +63 -2
- package/server/public/app/viewer-history.mjs +13 -2
- package/server/public/index.html +2249 -1590
- package/server/relay-tools.md +1 -0
- package/server/repositories/message-repository.mjs +23 -0
- package/server/repositories/session-repository.mjs +19 -0
- package/server/routes/claude-cloud-routes.mjs +19 -0
- package/server/routes/messages-routes.mjs +57 -1
- package/server/routes/sessions-routes.mjs +41 -7
- package/server/server-runtime.mjs +190 -40
- package/server/services/about-service.mjs +22 -53
- package/server/services/claude-billing-swap-service.mjs +7 -20
- package/server/services/claude-cloud-session-service.mjs +47 -5
- package/server/services/remote-relay-dispatcher.mjs +61 -14
- package/server/services/session-activity-tracker.mjs +1 -1
- package/server/services/session-launch-mutex-service.mjs +23 -0
- package/server/services/session-worker-registry-service.mjs +36 -0
- package/server/services/session-worker-websocket-service.mjs +40 -16
- package/server/services/worker-recycle-service.mjs +772 -0
- package/server/services/worker-replacement.mjs +197 -0
- package/shared/build-id.mjs +128 -0
- package/shared/worker-runtime/heartbeat.mjs +5 -0
- package/shared/worker-runtime/simple-steering-state.mjs +38 -0
- package/shared/worker-runtime/worker-websocket-link.mjs +23 -0
- package/shared/worker-runtime/working-directory.mjs +1 -1
|
@@ -75,6 +75,9 @@ import { createDeepSeekUsageImportService } from './services/deepseek-usage-impo
|
|
|
75
75
|
import { createClaudeApiSettingsService } from './services/claude-api-settings-service.mjs';
|
|
76
76
|
import { createClaudeBillingSwitchService, normalizeClaudeBilling } from './services/claude-billing-switch-service.mjs';
|
|
77
77
|
import { createClaudeBillingSwapService } from './services/claude-billing-swap-service.mjs';
|
|
78
|
+
import { createWorkerReplacement, createWorkerReplacementFence } from './services/worker-replacement.mjs';
|
|
79
|
+
import { sessionLaunchMutex } from './services/session-launch-mutex-service.mjs';
|
|
80
|
+
import { createWorkerRecycleService, sweepArchivedWorkers } from './services/worker-recycle-service.mjs';
|
|
78
81
|
import { createClaudeApiUsageService } from './services/claude-api-usage-service.mjs';
|
|
79
82
|
import { createTokenUsageService } from './services/token-usage-service.mjs';
|
|
80
83
|
import { createClaudeApiCapWatch, createClaudeApiPlanUsage, createClaudeApiStopNoteLedger } from './services/plan-usage-claude-api.mjs';
|
|
@@ -119,7 +122,8 @@ import { createActiveDeviceTracker } from './services/push-active-device-service
|
|
|
119
122
|
import { registerPushRoutes } from './routes/push-routes.mjs';
|
|
120
123
|
import { registerAboutRoutes } from './routes/about-routes.mjs';
|
|
121
124
|
import { registerRelayRestartRoutes } from './routes/relay-restart-routes.mjs';
|
|
122
|
-
import { createAboutService
|
|
125
|
+
import { createAboutService } from './services/about-service.mjs';
|
|
126
|
+
import { resolveBuildId } from '../shared/build-id.mjs';
|
|
123
127
|
import { createImageOperationService } from './services/image-operation-service.mjs';
|
|
124
128
|
import {
|
|
125
129
|
normalizeDriveAbsolutePath as _normalizeDriveAbsolutePath,
|
|
@@ -298,7 +302,12 @@ const INSTALL_METHOD = fs.existsSync(path.join(PACKAGE_ROOT, '.git')) ? 'git-che
|
|
|
298
302
|
// When this relay process started, and the commit its code came from (read
|
|
299
303
|
// once: a checkout moved on later still runs the code it started with).
|
|
300
304
|
const RELAY_STARTED_AT = new Date(Date.now() - process.uptime() * 1000).toISOString();
|
|
301
|
-
|
|
305
|
+
// The build id (version plus commit) names that code for the session workers:
|
|
306
|
+
// a worker that states another one started on older code. Read in the same
|
|
307
|
+
// pass as the commit so the two cannot disagree.
|
|
308
|
+
const RELAY_BUILD = resolveBuildId(PACKAGE_ROOT);
|
|
309
|
+
const BUILD_COMMIT = RELAY_BUILD.commit;
|
|
310
|
+
const BUILD_ID = RELAY_BUILD.id;
|
|
302
311
|
const PUBLIC_DIR = path.join(__dirname, 'public');
|
|
303
312
|
const INDEX_HTML_PATH = path.join(PUBLIC_DIR, 'index.html');
|
|
304
313
|
const PWA_MANIFEST_PATH = path.join(PUBLIC_DIR, 'manifest.webmanifest');
|
|
@@ -2006,6 +2015,8 @@ const runtimeTimers = {
|
|
|
2006
2015
|
claudeApiCapClock: null,
|
|
2007
2016
|
memoryLog: null,
|
|
2008
2017
|
memoryWatch: null,
|
|
2018
|
+
// Once, 30 s after the relay listens: archived sessions' workers stopped.
|
|
2019
|
+
archivedWorkerSweep: null,
|
|
2009
2020
|
};
|
|
2010
2021
|
|
|
2011
2022
|
function normalizeRemotePath(value) {
|
|
@@ -6550,6 +6561,23 @@ async function requestSessionWorkerSocketDelivery({ sessionId, pid, reason = 'wo
|
|
|
6550
6561
|
restartOrchestrator: relayRestartOrchestrator?.getState?.() || null,
|
|
6551
6562
|
};
|
|
6552
6563
|
}
|
|
6564
|
+
// The process asking is being replaced (the worker recycle passed its holds
|
|
6565
|
+
// for it): no work, as for a killed session; the worker stays ready, and
|
|
6566
|
+
// what waits goes to the new one. Unlike the killed marker above, this
|
|
6567
|
+
// survives the marker's grace and every clearRestartSchedule.
|
|
6568
|
+
if (workerReplacementFence.isHeld(requesterSessionId, pid)) {
|
|
6569
|
+
return {
|
|
6570
|
+
message: null,
|
|
6571
|
+
routing: {
|
|
6572
|
+
enabled: true,
|
|
6573
|
+
requesterSessionId,
|
|
6574
|
+
blockedReason: 'worker-replacement',
|
|
6575
|
+
lifecycle: sessionWorkerSupervisor?.getLifecycleState?.(requesterSessionId) || null,
|
|
6576
|
+
fallbackRestart: null,
|
|
6577
|
+
},
|
|
6578
|
+
restartOrchestrator: relayRestartOrchestrator?.getState?.() || null,
|
|
6579
|
+
};
|
|
6580
|
+
}
|
|
6553
6581
|
sessionWorkerSupervisor?.noteSessionHeartbeat?.(requesterSessionId);
|
|
6554
6582
|
if (runtimeState.relayPaused) {
|
|
6555
6583
|
return { message: null, paused: true, reason: 'relay_paused' };
|
|
@@ -6629,6 +6657,9 @@ async function requestSessionWorkerSocketDelivery({ sessionId, pid, reason = 'wo
|
|
|
6629
6657
|
workerKind: workerKind || sessionWorkerRegistry?.getWorker?.(requesterSessionId)?.kind || null,
|
|
6630
6658
|
claudeApiClaimCheck,
|
|
6631
6659
|
onBillingFence: handleClaudeBillingFence,
|
|
6660
|
+
// The fence check above, asked again before every claim: ensureWorker
|
|
6661
|
+
// and a transient error's backoff both wait.
|
|
6662
|
+
isReplacementFenced: () => workerReplacementFence.isHeld(requesterSessionId, pid),
|
|
6632
6663
|
});
|
|
6633
6664
|
if (!dequeueResult?.message) {
|
|
6634
6665
|
return {
|
|
@@ -6942,48 +6973,38 @@ async function sweepDeadWorkerProcessingRowsOnce() {
|
|
|
6942
6973
|
// claude-billing-swap-service.mjs): a Claude chat's next row of the other
|
|
6943
6974
|
// billing is refused to the worker it has, which is replaced, once idle, by
|
|
6944
6975
|
// one of the row's kind. The old one is stopped like the kill route does, and
|
|
6945
|
-
// the new one launched only once every process of the session is gone
|
|
6976
|
+
// the new one launched only once every process of the session is gone
|
|
6977
|
+
// (services/worker-replacement.mjs, which the worker recycle below shares).
|
|
6978
|
+
// The two get one instance each so a failed stop's warning names the caller.
|
|
6979
|
+
// They share one replacement fence, the one the deliveries check (the socket
|
|
6980
|
+
// delivery above and /api/pending): a process being replaced is handed no
|
|
6981
|
+
// work. The worker recycle raises it; the billing swap does not, since its
|
|
6982
|
+
// own delivery fence already keeps the other billing's rows from the old
|
|
6983
|
+
// worker.
|
|
6984
|
+
const workerReplacementFence = createWorkerReplacementFence();
|
|
6985
|
+
function createRuntimeWorkerReplacement(tag) {
|
|
6986
|
+
return createWorkerReplacement({
|
|
6987
|
+
registry: sessionWorkerRegistry,
|
|
6988
|
+
supervisor: sessionWorkerSupervisor,
|
|
6989
|
+
processInspector: sessionWorkerProcessInspector,
|
|
6990
|
+
stopSessionWorkerProcesses,
|
|
6991
|
+
isProcessAlive,
|
|
6992
|
+
killTmuxSession,
|
|
6993
|
+
logger: console,
|
|
6994
|
+
logPrefix: () => `${runtimeLogPrefix()}${tag} `,
|
|
6995
|
+
fence: workerReplacementFence,
|
|
6996
|
+
});
|
|
6997
|
+
}
|
|
6998
|
+
// A stop that fails backs off (the swap service retries it) and clears the
|
|
6999
|
+
// killed marker, so the session's worker, alive or relaunched, keeps asking
|
|
7000
|
+
// for work meanwhile. The fence still keeps the other billing's rows from it.
|
|
7001
|
+
const billingSwapWorkerReplacement = createRuntimeWorkerReplacement('BILLING SWAP');
|
|
6946
7002
|
const claudeBillingSwapService = createClaudeBillingSwapService({
|
|
6947
7003
|
getWorker: (sid) => sessionWorkerRegistry?.getWorker?.(sid) || null,
|
|
6948
7004
|
// A row the session claimed and has not finished: never stopped under it.
|
|
6949
7005
|
hasRunningTurn: (sid) => (stmts.listProcessingRowsForOwner?.all?.(sid) || []).length > 0,
|
|
6950
|
-
stopWorker:
|
|
6951
|
-
|
|
6952
|
-
// Nothing may start the session while it is being stopped: a launch then
|
|
6953
|
-
// would adopt the old process, kind and all.
|
|
6954
|
-
sessionWorkerSupervisor?.markKilled?.(sid);
|
|
6955
|
-
// A stop that fails backs off (the swap service retries it): the killed
|
|
6956
|
-
// marker goes with it, so the session's worker, alive or relaunched,
|
|
6957
|
-
// keeps asking for work meanwhile. The fence still keeps the other
|
|
6958
|
-
// billing's rows from it.
|
|
6959
|
-
const backOff = (error, remainingPids = []) => {
|
|
6960
|
-
sessionWorkerSupervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
|
|
6961
|
-
console.warn(`${runtimeLogPrefix()}BILLING SWAP STOP FAILED session=${sid.slice(0, 8)} ${error}${remainingPids.length ? ` remaining=${remainingPids.join(',')}` : ''} — kill marker cleared, retrying with a backoff`);
|
|
6962
|
-
return { ok: false, error, remainingPids };
|
|
6963
|
-
};
|
|
6964
|
-
let stopped = null;
|
|
6965
|
-
try {
|
|
6966
|
-
await sessionWorkerSupervisor?.cancelPendingStart?.(sid, { wait: true });
|
|
6967
|
-
stopped = await stopSessionWorkerProcesses({
|
|
6968
|
-
sdkSessionId: sid,
|
|
6969
|
-
worker,
|
|
6970
|
-
processInspector: sessionWorkerProcessInspector,
|
|
6971
|
-
isPidAliveImpl: isProcessAlive,
|
|
6972
|
-
killTmuxSessionImpl: killTmuxSession,
|
|
6973
|
-
});
|
|
6974
|
-
} catch (error) {
|
|
6975
|
-
return backOff(String(error?.message || error || 'worker-stop-failed'));
|
|
6976
|
-
}
|
|
6977
|
-
if (!stopped.ok) return backOff(stopped.error || 'worker-survived-stop', stopped.remainingPids || []);
|
|
6978
|
-
sessionWorkerRegistry?.removeWorker?.(sid);
|
|
6979
|
-
sessionWorkerSupervisor?.resetHealth?.(sid, { clearFailureCount: false });
|
|
6980
|
-
return { ok: true };
|
|
6981
|
-
},
|
|
6982
|
-
launchWorker: async (sid) => {
|
|
6983
|
-
sessionWorkerSupervisor?.clearRestartSchedule?.(sid, { resetKilledMarker: true });
|
|
6984
|
-
const launched = await sessionWorkerSupervisor?.ensureWorker?.(sid, { allowProcessReuse: false });
|
|
6985
|
-
return { ok: launched?.ok === true, error: launched?.error || null };
|
|
6986
|
-
},
|
|
7006
|
+
stopWorker: billingSwapWorkerReplacement.stopWorker,
|
|
7007
|
+
launchWorker: billingSwapWorkerReplacement.launchWorker,
|
|
6987
7008
|
onStuck: ({ conversationId, rowBilling, attempts, reason }) => {
|
|
6988
7009
|
if (!conversationId) return;
|
|
6989
7010
|
const at = new Date().toISOString();
|
|
@@ -7015,6 +7036,95 @@ const claudeBillingSwapService = createClaudeBillingSwapService({
|
|
|
7015
7036
|
logger: console,
|
|
7016
7037
|
});
|
|
7017
7038
|
|
|
7039
|
+
// Whether a worker's session is one the relay keeps no worker for: its
|
|
7040
|
+
// conversation is archived (or deleted), or there is no conversation behind
|
|
7041
|
+
// the session at all. A conversation in use found either way (the runtime
|
|
7042
|
+
// session's, or the one naming the session id) keeps its worker: a session
|
|
7043
|
+
// is never stopped for good on a missing link alone.
|
|
7044
|
+
function isWorkerSessionArchived(sid) {
|
|
7045
|
+
const runtime = stmts.getRuntimeSessionBySdkSessionId?.get?.(sid) || null;
|
|
7046
|
+
const viaRuntime = runtime?.conversation_id ? (stmts.getConvAnyStatus?.get?.(runtime.conversation_id) || null) : null;
|
|
7047
|
+
const viaSessionId = stmts.getConvBySdkSessionId?.get?.(sid) || null;
|
|
7048
|
+
const inUse = (conversation) => Boolean(conversation)
|
|
7049
|
+
&& Number(conversation.archived || 0) !== 1
|
|
7050
|
+
&& String(conversation.status || '').trim().toLowerCase() !== 'deleted';
|
|
7051
|
+
return !inUse(viaRuntime) && !inUse(viaSessionId);
|
|
7052
|
+
}
|
|
7053
|
+
|
|
7054
|
+
// The worker recycle (services/worker-recycle-service.mjs): a worker that
|
|
7055
|
+
// runs older code than this relay (another build id, or none stated) is
|
|
7056
|
+
// replaced between turns by one on this relay's code; an archived session's
|
|
7057
|
+
// is stopped for good. Its own replacement instance, so a failed stop says
|
|
7058
|
+
// WORKER RECYCLE rather than BILLING SWAP.
|
|
7059
|
+
const recycleWorkerReplacement = createRuntimeWorkerReplacement('WORKER RECYCLE');
|
|
7060
|
+
const workerRecycleService = createWorkerRecycleService({
|
|
7061
|
+
relayBuildId: BUILD_ID,
|
|
7062
|
+
listWorkers: () => sessionWorkerRegistry.listWorkers(),
|
|
7063
|
+
getWorker: (sid) => sessionWorkerRegistry.getWorker(sid),
|
|
7064
|
+
// A row the session claimed and has not finished: never stopped under it.
|
|
7065
|
+
hasRunningTurn: (sid) => (stmts.listProcessingRowsForOwner?.all?.(sid) || []).length > 0,
|
|
7066
|
+
// From the supervisor's lifecycle, not its firstObservedHeartbeatAt: that
|
|
7067
|
+
// is set only for a worker it launched itself, and the workers that
|
|
7068
|
+
// reconnect after a relay restart (the ones most likely stale) never get it.
|
|
7069
|
+
hasHeartbeated: (sid) => {
|
|
7070
|
+
const lc = sessionWorkerSupervisor.getLifecycleState(sid);
|
|
7071
|
+
return !!lc?.lastHeartbeatAt && !lc?.awaitingHeartbeat;
|
|
7072
|
+
},
|
|
7073
|
+
isArchived: (sid) => isWorkerSessionArchived(sid),
|
|
7074
|
+
// The session's last delivery or turn end, for a worker that does not say
|
|
7075
|
+
// whether its CLI child runs (message-repository getOwnerLastActivityAt).
|
|
7076
|
+
quietSince: (sid) => {
|
|
7077
|
+
const at = Date.parse(stmts.getOwnerLastActivityAt?.get?.(sid)?.last_activity_at || '');
|
|
7078
|
+
return Number.isFinite(at) ? at : null;
|
|
7079
|
+
},
|
|
7080
|
+
stopWorker: recycleWorkerReplacement.stopWorker,
|
|
7081
|
+
launchWorker: recycleWorkerReplacement.launchWorker,
|
|
7082
|
+
fence: recycleWorkerReplacement.fence,
|
|
7083
|
+
// The stop and the launch under the per-session launch mutex the Change
|
|
7084
|
+
// CWD relaunch and the launch routes take (sessions-routes.mjs).
|
|
7085
|
+
runExclusive: (sid, fn) => sessionLaunchMutex.runExclusive(sid, fn),
|
|
7086
|
+
logger: console,
|
|
7087
|
+
});
|
|
7088
|
+
|
|
7089
|
+
// Once at start, after the restart's own worker reconnects settled: what
|
|
7090
|
+
// still runs for an archived conversation is stopped and stays stopped.
|
|
7091
|
+
// Through the recycle's replacement, one session at a time; never throws.
|
|
7092
|
+
async function sweepArchivedSessionWorkersAtStart() {
|
|
7093
|
+
if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED !== true) return null;
|
|
7094
|
+
if (runtimeShutdownStarted) return null;
|
|
7095
|
+
// The first lookup reads the process list afresh; the rest share it for the
|
|
7096
|
+
// inspector's cache moment, so a long archive costs one read, not one each.
|
|
7097
|
+
let freshRead = true;
|
|
7098
|
+
return sweepArchivedWorkers({
|
|
7099
|
+
listArchivedSessionIds: () => (stmts.listArchivedRuntimeSessionIds?.all?.() || []).map((row) => row?.sdk_session_id),
|
|
7100
|
+
hasLiveProcess: async (sid) => {
|
|
7101
|
+
if (runtimeShutdownStarted) return false;
|
|
7102
|
+
const pid = Number(sessionWorkerRegistry?.getWorker?.(sid)?.pid);
|
|
7103
|
+
if (Number.isInteger(pid) && pid > 0 && isProcessAlive(pid)) return true;
|
|
7104
|
+
const fresh = freshRead;
|
|
7105
|
+
freshRead = false;
|
|
7106
|
+
return Boolean(await sessionWorkerProcessInspector?.findProcessForSessionAsync?.(sid, { fresh }));
|
|
7107
|
+
},
|
|
7108
|
+
isArchived: (sid) => isWorkerSessionArchived(sid),
|
|
7109
|
+
// The process the fence keeps deliveries from during the stop: the
|
|
7110
|
+
// registry's live pid, else the one the process list shows, else none
|
|
7111
|
+
// (every process of the session is fenced then).
|
|
7112
|
+
workerPid: async (sid) => {
|
|
7113
|
+
const pid = Number(sessionWorkerRegistry?.getWorker?.(sid)?.pid);
|
|
7114
|
+
if (Number.isInteger(pid) && pid > 0 && isProcessAlive(pid)) return pid;
|
|
7115
|
+
const found = await sessionWorkerProcessInspector?.findProcessForSessionAsync?.(sid);
|
|
7116
|
+
return Number(found?.processId) || null;
|
|
7117
|
+
},
|
|
7118
|
+
stopWorker: recycleWorkerReplacement.stopWorker,
|
|
7119
|
+
// Guarded as the recycle's archived stop is: a message that unarchives
|
|
7120
|
+
// the session meanwhile must not start a turn on the process being
|
|
7121
|
+
// stopped.
|
|
7122
|
+
fence: workerReplacementFence,
|
|
7123
|
+
runExclusive: (sid, fn) => sessionLaunchMutex.runExclusive(sid, fn),
|
|
7124
|
+
logger: console,
|
|
7125
|
+
});
|
|
7126
|
+
}
|
|
7127
|
+
|
|
7018
7128
|
// The Claude API monthly cap (plan decision 12): `{ reached, error? }`. The
|
|
7019
7129
|
// one seam the claim check below, the billing switch, an agent's send and a
|
|
7020
7130
|
// new Claude API chat ask: this month's priced estimate in
|
|
@@ -7187,6 +7297,16 @@ const sessionWorkerWebSocketService = createSessionWorkerWebSocketService({
|
|
|
7187
7297
|
const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
|
|
7188
7298
|
if (worker && worker.kind !== kind) sessionWorkerRegistry?.upsertWorker?.({ ...worker, kind });
|
|
7189
7299
|
},
|
|
7300
|
+
// The build id a worker says it started on: the registry keeps it so a
|
|
7301
|
+
// worker on older code can be told from one on the relay's.
|
|
7302
|
+
onWorkerBuildId: (sessionId, buildId) => {
|
|
7303
|
+
const sid = String(sessionId || '').trim();
|
|
7304
|
+
if (!sid) return;
|
|
7305
|
+
const worker = sessionWorkerRegistry?.getWorker?.(sid) || null;
|
|
7306
|
+
if (worker && worker.buildId !== buildId) sessionWorkerRegistry?.upsertWorker?.({ ...worker, buildId });
|
|
7307
|
+
},
|
|
7308
|
+
// Sent in server.hello, so a worker on other code can say so in its log.
|
|
7309
|
+
buildId: BUILD_ID,
|
|
7190
7310
|
isWorkerProcessAlive: (pid) => isProcessAlive(Number(pid)),
|
|
7191
7311
|
pathPrefix: remotePath,
|
|
7192
7312
|
pollIntervalMs: 500,
|
|
@@ -7198,6 +7318,10 @@ runtimeTimers.deadWorkerSweep = setInterval(() => {
|
|
|
7198
7318
|
});
|
|
7199
7319
|
}, 30_000);
|
|
7200
7320
|
if (typeof runtimeTimers.deadWorkerSweep.unref === 'function') runtimeTimers.deadWorkerSweep.unref();
|
|
7321
|
+
// Workers on older code are recycled only where the relay launches them;
|
|
7322
|
+
// with routing off the service never starts. Its timer is its own (unref'd)
|
|
7323
|
+
// and stops with the runtime timers.
|
|
7324
|
+
if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true) workerRecycleService.start();
|
|
7201
7325
|
const tmuxInspectorAccessPolicy = createTmuxInspectorAccessPolicy({
|
|
7202
7326
|
sessionWorkerRegistry,
|
|
7203
7327
|
sessionWorkerSupervisor,
|
|
@@ -7357,6 +7481,13 @@ let relayPaused = false;
|
|
|
7357
7481
|
const relayBridgeOwnerService = createRelayBridgeOwnerService();
|
|
7358
7482
|
|
|
7359
7483
|
const runtimeState = {
|
|
7484
|
+
// The relay's own build id, read once at start; null when package.json
|
|
7485
|
+
// names no version.
|
|
7486
|
+
buildId: BUILD_ID,
|
|
7487
|
+
// The worker recycle's status() (the relay's build id, the stale count,
|
|
7488
|
+
// the recycles in flight and done, who waits on what): a function, so a
|
|
7489
|
+
// route reads it when asked and needs no wiring of its own.
|
|
7490
|
+
workerRecycle: () => workerRecycleService.status(),
|
|
7360
7491
|
get cliOnline() { return cliOnline; },
|
|
7361
7492
|
set cliOnline(value) { cliOnline = value; },
|
|
7362
7493
|
get relayPaused() { return relayPaused; },
|
|
@@ -7693,6 +7824,8 @@ const sharedRouteDeps = {
|
|
|
7693
7824
|
sessionWorkerSupervisor,
|
|
7694
7825
|
sessionWorkerRegistry,
|
|
7695
7826
|
sessionWorkerProcessInspector,
|
|
7827
|
+
// /api/pending refuses work to a process the worker recycle is replacing.
|
|
7828
|
+
workerReplacementFence,
|
|
7696
7829
|
buildContextResponseText,
|
|
7697
7830
|
readContextFromSessionEvents,
|
|
7698
7831
|
sessionHistoryRefreshService,
|
|
@@ -7859,6 +7992,9 @@ registerAboutRoutes(app, {
|
|
|
7859
7992
|
dbPath: DB_PATH,
|
|
7860
7993
|
version: RUNNING_VERSION,
|
|
7861
7994
|
commit: BUILD_COMMIT,
|
|
7995
|
+
// The workers' build ids are counted against this one (the About modal's
|
|
7996
|
+
// "Workers: 14, 3 on an older build").
|
|
7997
|
+
buildId: BUILD_ID,
|
|
7862
7998
|
installMethod: INSTALL_METHOD,
|
|
7863
7999
|
startedAt: RELAY_STARTED_AT,
|
|
7864
8000
|
packageRoot: PACKAGE_ROOT,
|
|
@@ -8494,6 +8630,7 @@ function clearRuntimeTimers() {
|
|
|
8494
8630
|
try { clearTimeout(timer); } catch {}
|
|
8495
8631
|
}
|
|
8496
8632
|
subagentRecordRetention?.stop();
|
|
8633
|
+
try { workerRecycleService.stop(); } catch {}
|
|
8497
8634
|
try { taskHealthPoller.stop(); } catch {}
|
|
8498
8635
|
try { taskOutputRooms.stop(); } catch {}
|
|
8499
8636
|
runtimeTimers.cliStatus = null;
|
|
@@ -8503,6 +8640,7 @@ function clearRuntimeTimers() {
|
|
|
8503
8640
|
runtimeTimers.pendingWorkerPrime = null;
|
|
8504
8641
|
runtimeTimers.sharedViewerPrune = null;
|
|
8505
8642
|
runtimeTimers.restoredTaskSettle = null;
|
|
8643
|
+
runtimeTimers.archivedWorkerSweep = null;
|
|
8506
8644
|
try { relayRestartQueue.dispose(); } catch {}
|
|
8507
8645
|
}
|
|
8508
8646
|
|
|
@@ -8714,4 +8852,16 @@ httpServer.listen(config.port, listenHost, () => {
|
|
|
8714
8852
|
// Deferred and unref'd (a few seconds behind the startup import's own CLI
|
|
8715
8853
|
// spawn); a relay without an installed Copilot CLI skips it entirely.
|
|
8716
8854
|
copilotModelDiscoveryService.scheduleStartupRefresh();
|
|
8855
|
+
// Archived conversations' workers: stopped once, 30 s in, so the workers
|
|
8856
|
+
// that reconnect after the restart and the first prime have settled. With
|
|
8857
|
+
// routing off the relay launches no workers, so there is nothing to sweep.
|
|
8858
|
+
if (featureFlags?.SESSION_WORKER_ROUTING_ENABLED === true) {
|
|
8859
|
+
runtimeTimers.archivedWorkerSweep = setTimeout(() => {
|
|
8860
|
+
runtimeTimers.archivedWorkerSweep = null;
|
|
8861
|
+
sweepArchivedSessionWorkersAtStart().catch((error) => {
|
|
8862
|
+
console.warn(`${runtimeLogPrefix()}WORKER RECYCLE archived sweep failed: ${error?.message || error}`);
|
|
8863
|
+
});
|
|
8864
|
+
}, 30_000);
|
|
8865
|
+
runtimeTimers.archivedWorkerSweep.unref?.();
|
|
8866
|
+
}
|
|
8717
8867
|
});
|
|
@@ -3,6 +3,14 @@
|
|
|
3
3
|
import fs from 'node:fs';
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
|
|
6
|
+
import { normalizeBuildId, readGitCommit, resolveBuildCommit } from '../../shared/build-id.mjs';
|
|
7
|
+
import { isWorkerOnOtherBuild } from './worker-recycle-service.mjs';
|
|
8
|
+
|
|
9
|
+
// The commit readers moved to shared/ so the workers can state their build id
|
|
10
|
+
// too; they are re-exported here so the routes and tests importing them keep
|
|
11
|
+
// their import.
|
|
12
|
+
export { readGitCommit, resolveBuildCommit };
|
|
13
|
+
|
|
6
14
|
// What the About OAR modal shows: the running build, the providers that are
|
|
7
15
|
// on, what this relay has stored over its life, and how it is doing now.
|
|
8
16
|
//
|
|
@@ -64,58 +72,6 @@ function readFileText(fsImpl, filePath) {
|
|
|
64
72
|
}
|
|
65
73
|
}
|
|
66
74
|
|
|
67
|
-
const COMMIT_PATTERN = /^[0-9a-f]{40}$/i;
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* The commit a git checkout has checked out, read from its .git files rather
|
|
71
|
-
* than from a `git` process: HEAD, then the loose ref, then packed-refs. A
|
|
72
|
-
* linked worktree's .git is a file naming its git dir, whose refs live in the
|
|
73
|
-
* common dir. Anything unexpected reads as null. `pathImpl` is the host's path
|
|
74
|
-
* module; tests pass path.posix and path.win32.
|
|
75
|
-
*/
|
|
76
|
-
export function readGitCommit(root, { fsImpl = fs, pathImpl = path } = {}) {
|
|
77
|
-
if (!root) return null;
|
|
78
|
-
const dotGit = pathImpl.join(root, '.git');
|
|
79
|
-
let gitDir = dotGit;
|
|
80
|
-
const pointer = readFileText(fsImpl, dotGit);
|
|
81
|
-
if (pointer !== null) {
|
|
82
|
-
const match = pointer.match(/^gitdir:\s*(.+)$/m);
|
|
83
|
-
if (!match) return null;
|
|
84
|
-
gitDir = pathImpl.resolve(root, match[1].trim());
|
|
85
|
-
}
|
|
86
|
-
const head = readFileText(fsImpl, pathImpl.join(gitDir, 'HEAD'));
|
|
87
|
-
if (head === null) return null;
|
|
88
|
-
const headText = head.trim();
|
|
89
|
-
if (COMMIT_PATTERN.test(headText)) return headText.toLowerCase();
|
|
90
|
-
const refMatch = headText.match(/^ref:\s*(\S+)$/);
|
|
91
|
-
if (!refMatch) return null;
|
|
92
|
-
const ref = refMatch[1];
|
|
93
|
-
const commonDirText = readFileText(fsImpl, pathImpl.join(gitDir, 'commondir'));
|
|
94
|
-
const commonDir = commonDirText ? pathImpl.resolve(gitDir, commonDirText.trim()) : gitDir;
|
|
95
|
-
for (const dir of new Set([gitDir, commonDir])) {
|
|
96
|
-
const loose = readFileText(fsImpl, pathImpl.join(dir, ...ref.split('/')));
|
|
97
|
-
if (loose && COMMIT_PATTERN.test(loose.trim())) return loose.trim().toLowerCase();
|
|
98
|
-
}
|
|
99
|
-
const packed = readFileText(fsImpl, pathImpl.join(commonDir, 'packed-refs')) || '';
|
|
100
|
-
for (const line of packed.split(/\r?\n/)) {
|
|
101
|
-
const [hash, name] = line.trim().split(/\s+/);
|
|
102
|
-
if (name === ref && COMMIT_PATTERN.test(hash || '')) return hash.toLowerCase();
|
|
103
|
-
}
|
|
104
|
-
return null;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/** The commit this build came from: the checkout's HEAD, else a `gitHead` stamp in package.json. */
|
|
108
|
-
export function resolveBuildCommit(packageRoot, { fsImpl = fs, pathImpl = path } = {}) {
|
|
109
|
-
const fromGit = readGitCommit(packageRoot, { fsImpl, pathImpl });
|
|
110
|
-
if (fromGit) return fromGit;
|
|
111
|
-
try {
|
|
112
|
-
const stamp = String(JSON.parse(readFileText(fsImpl, pathImpl.join(packageRoot, 'package.json')) || '{}')?.gitHead || '').trim();
|
|
113
|
-
return COMMIT_PATTERN.test(stamp) ? stamp.toLowerCase() : null;
|
|
114
|
-
} catch {
|
|
115
|
-
return null;
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
|
|
119
75
|
export function readInstalledPackageVersion(packageRoot, packageName, { fsImpl = fs } = {}) {
|
|
120
76
|
if (!packageRoot || !packageName) return null;
|
|
121
77
|
try {
|
|
@@ -338,6 +294,7 @@ async function fileSize(fsImpl, filePath) {
|
|
|
338
294
|
* @param {string|null} [deps.dbPath] the database file; its -wal file is counted too
|
|
339
295
|
* @param {string} deps.version
|
|
340
296
|
* @param {string|null} [deps.commit]
|
|
297
|
+
* @param {string|null} [deps.buildId] the relay's build id (shared/build-id.mjs), to tell the workers on other code
|
|
341
298
|
* @param {string} [deps.installMethod]
|
|
342
299
|
* @param {string} deps.startedAt ISO time this relay process started
|
|
343
300
|
* @param {() => object} [deps.getProviderSettings] `{ github: { enabled, engine }, claude: { enabled }, … }`
|
|
@@ -354,6 +311,7 @@ export function createAboutService({
|
|
|
354
311
|
describeBackups = null,
|
|
355
312
|
version = '0.0.0',
|
|
356
313
|
commit = null,
|
|
314
|
+
buildId = null,
|
|
357
315
|
installMethod = null,
|
|
358
316
|
startedAt,
|
|
359
317
|
packageRoot = null,
|
|
@@ -397,20 +355,30 @@ export function createAboutService({
|
|
|
397
355
|
return totalsInFlight;
|
|
398
356
|
}
|
|
399
357
|
|
|
358
|
+
// The running workers, by provider and by build: after a deploy the
|
|
359
|
+
// workers keep the code they started with until the recycle replaces them,
|
|
360
|
+
// and About shows how many still do. A relay that does not know its own
|
|
361
|
+
// build id cannot tell, so both build counts are null then.
|
|
400
362
|
function runningWorkersByProvider() {
|
|
401
363
|
const counts = new Map();
|
|
402
364
|
let total = 0;
|
|
365
|
+
const relayBuildId = normalizeBuildId(buildId);
|
|
366
|
+
let onOtherBuild = 0;
|
|
403
367
|
for (const worker of listWorkers() || []) {
|
|
404
368
|
const status = String(worker?.status || '').trim().toLowerCase();
|
|
405
369
|
if (!ACTIVE_WORKER_STATUSES.has(status) || !isPidAlive(worker?.pid)) continue;
|
|
406
370
|
total += 1;
|
|
371
|
+
if (isWorkerOnOtherBuild(worker, relayBuildId)) onOtherBuild += 1;
|
|
407
372
|
const conversationId = String(worker?.conversationId || '').trim();
|
|
408
373
|
const provider = conversationId
|
|
409
374
|
? normalizeProviderId(providerForConversation.get(conversationId)?.provider_type)
|
|
410
375
|
: 'unassigned';
|
|
411
376
|
counts.set(provider, (counts.get(provider) || 0) + 1);
|
|
412
377
|
}
|
|
413
|
-
|
|
378
|
+
const builds = relayBuildId
|
|
379
|
+
? { total, onThisBuild: total - onOtherBuild, onOtherBuild }
|
|
380
|
+
: { total, onThisBuild: null, onOtherBuild: null };
|
|
381
|
+
return { total, counts, builds };
|
|
414
382
|
}
|
|
415
383
|
|
|
416
384
|
function describeProviders(workerCounts) {
|
|
@@ -474,6 +442,7 @@ export function createAboutService({
|
|
|
474
442
|
startedAt: startedAt || null,
|
|
475
443
|
uptimeSeconds,
|
|
476
444
|
providers: describeProviders(workers.counts),
|
|
445
|
+
workers: workers.builds,
|
|
477
446
|
stats: {
|
|
478
447
|
conversations: totals.conversations.total,
|
|
479
448
|
archivedConversations: totals.conversations.archived,
|
|
@@ -28,29 +28,16 @@
|
|
|
28
28
|
* the fence lets them through, and the waiting state ages out.
|
|
29
29
|
*/
|
|
30
30
|
|
|
31
|
+
import { workerReleaseHold } from './worker-replacement.mjs';
|
|
32
|
+
|
|
31
33
|
export const CLAUDE_BILLING_SWAP_WAIT_TTL_MS = 15_000;
|
|
32
34
|
|
|
33
35
|
/**
|
|
34
|
-
* What keeps a worker from being replaced now
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* from the rest of it.
|
|
36
|
+
* What keeps a worker from being replaced now (worker-replacement.mjs, which
|
|
37
|
+
* the worker recycle shares). Kept under its old name for the importers that
|
|
38
|
+
* know it by that.
|
|
38
39
|
*/
|
|
39
|
-
export
|
|
40
|
-
if (!worker) return null;
|
|
41
|
-
const steering = worker.steering || null;
|
|
42
|
-
if (steering && typeof steering.releasable === 'boolean') {
|
|
43
|
-
return steering.releasable ? null : (steering.releaseHold || 'turn');
|
|
44
|
-
}
|
|
45
|
-
if (steering) {
|
|
46
|
-
if (steering.holdReason === 'question') return 'question';
|
|
47
|
-
if (steering.holdReason === 'compaction') return 'compaction';
|
|
48
|
-
if (steering.turnActive) return 'turn';
|
|
49
|
-
if (steering.autoCompactWindow?.backgroundWork) return 'background';
|
|
50
|
-
return null;
|
|
51
|
-
}
|
|
52
|
-
return String(worker.status || '') === 'processing' ? 'turn' : null;
|
|
53
|
-
}
|
|
40
|
+
export const claudeWorkerReleaseHold = workerReleaseHold;
|
|
54
41
|
|
|
55
42
|
export function createClaudeBillingSwapService({
|
|
56
43
|
// sid → the registry's entry for the session's worker, or null.
|
|
@@ -208,7 +195,7 @@ export function createClaudeBillingSwapService({
|
|
|
208
195
|
running = true;
|
|
209
196
|
noteProbeFailure(sid, state, fence, error);
|
|
210
197
|
}
|
|
211
|
-
const hold = running ? 'turn' :
|
|
198
|
+
const hold = running ? 'turn' : workerReleaseHold(worker);
|
|
212
199
|
if (hold) {
|
|
213
200
|
setPending(sid, state, {
|
|
214
201
|
to: fence.rowBilling,
|
|
@@ -157,12 +157,9 @@ export function buildClaudeCloudDelivery({ conversation = null, runtimeSession =
|
|
|
157
157
|
if (!isClaudeCloudProviderType(runtimeSession?.provider_type)) return null;
|
|
158
158
|
const source = parseCloudSourceJson(conversation?.cloud_source_json);
|
|
159
159
|
if (!source) return null;
|
|
160
|
-
const sessionId = toText(runtimeSession?.claude_cloud_session_id);
|
|
161
|
-
const lastSequence = toText(runtimeSession?.claude_cloud_last_sequence);
|
|
162
160
|
const fallback = toText(fallbackEnvironmentId);
|
|
163
161
|
return {
|
|
164
|
-
|
|
165
|
-
lastSequence: SEQUENCE_PATTERN.test(lastSequence) ? lastSequence : null,
|
|
162
|
+
...readStoredClaudeCloudBinding(runtimeSession),
|
|
166
163
|
repoUrl: source.repoUrl,
|
|
167
164
|
branch: source.branch,
|
|
168
165
|
environmentId: source.environmentId || (isSafeClaudeCloudId(fallback) ? fallback : null),
|
|
@@ -170,6 +167,20 @@ export function buildClaudeCloudDelivery({ conversation = null, runtimeSession =
|
|
|
170
167
|
};
|
|
171
168
|
}
|
|
172
169
|
|
|
170
|
+
/**
|
|
171
|
+
* The cloud session a runtime session row is bound to and the cursor its
|
|
172
|
+
* worker reads on from: `{ sessionId, lastSequence }`, each null when absent
|
|
173
|
+
* or not in the shape the worker may put into a request path.
|
|
174
|
+
*/
|
|
175
|
+
export function readStoredClaudeCloudBinding(runtimeSession = null) {
|
|
176
|
+
const sessionId = toText(runtimeSession?.claude_cloud_session_id);
|
|
177
|
+
const lastSequence = toText(runtimeSession?.claude_cloud_last_sequence);
|
|
178
|
+
return {
|
|
179
|
+
sessionId: isSafeClaudeCloudId(sessionId) ? sessionId : null,
|
|
180
|
+
lastSequence: SEQUENCE_PATTERN.test(lastSequence) ? lastSequence : null,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
|
|
173
184
|
/** The larger of two sequence numbers (strings of digits), so a late report never moves the cursor back. */
|
|
174
185
|
function laterSequence(stored, reported) {
|
|
175
186
|
const a = SEQUENCE_PATTERN.test(toText(stored)) ? toText(stored) : null;
|
|
@@ -264,6 +275,31 @@ export function createClaudeCloudSessionService({
|
|
|
264
275
|
return { ok: true, cloud };
|
|
265
276
|
}
|
|
266
277
|
|
|
278
|
+
/**
|
|
279
|
+
* GET /api/claude-cloud-session: the binding a starting worker attaches to
|
|
280
|
+
* before any delivery (so it follows the turns the session starts by
|
|
281
|
+
* itself, as the worker it replaces did). `id` is the conversation id or
|
|
282
|
+
* the runtime session's sdk session id, which is all a worker knows of its
|
|
283
|
+
* conversation. A known conversation that has no cloud session (none yet,
|
|
284
|
+
* or another provider) answers `sessionId: null`; only an unknown id is
|
|
285
|
+
* 404. Returns `{ ok, statusCode, error }` or `{ ok: true, sessionId,
|
|
286
|
+
* lastSequence }`.
|
|
287
|
+
*/
|
|
288
|
+
function describeStoredBinding(id = '') {
|
|
289
|
+
const key = toText(id);
|
|
290
|
+
if (!key) return { ok: false, statusCode: 400, error: 'Missing conversationId' };
|
|
291
|
+
const runtimeSession = stmts.getRuntimeSessionByConversation?.get?.(key)
|
|
292
|
+
|| stmts.getRuntimeSessionBySdkSessionId?.get?.(key)
|
|
293
|
+
|| null;
|
|
294
|
+
if (!runtimeSession && !stmts.getConvAnyStatus?.get?.(key)) {
|
|
295
|
+
return { ok: false, statusCode: 404, error: 'Conversation not found' };
|
|
296
|
+
}
|
|
297
|
+
if (!runtimeSession || !isClaudeCloudProviderType(runtimeSession.provider_type)) {
|
|
298
|
+
return { ok: true, sessionId: null, lastSequence: null };
|
|
299
|
+
}
|
|
300
|
+
return { ok: true, ...readStoredClaudeCloudBinding(runtimeSession) };
|
|
301
|
+
}
|
|
302
|
+
|
|
267
303
|
/**
|
|
268
304
|
* Archives the cloud session of a conversation that is being deleted or
|
|
269
305
|
* archived here. Best effort: it returns at once (true when an archive was
|
|
@@ -322,5 +358,11 @@ export function createClaudeCloudSessionService({
|
|
|
322
358
|
return true;
|
|
323
359
|
}
|
|
324
360
|
|
|
325
|
-
return {
|
|
361
|
+
return {
|
|
362
|
+
cloudPayloadFor,
|
|
363
|
+
recordWorkerReport,
|
|
364
|
+
describeStoredBinding,
|
|
365
|
+
archiveSessionInBackground,
|
|
366
|
+
unarchiveSessionInBackground,
|
|
367
|
+
};
|
|
326
368
|
}
|