@adhdev/daemon-core 0.9.82-rc.366 → 0.9.82-rc.368
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/chat-commands.d.ts +27 -0
- package/dist/commands/router.d.ts +34 -0
- package/dist/index.js +220 -26
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +220 -26
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
- package/src/commands/chat-commands.ts +96 -2
- package/src/commands/cli-manager.ts +22 -0
- package/src/commands/router.ts +135 -8
- package/src/mesh/mesh-events-coordinator.ts +144 -32
- package/src/mesh/mesh-reconcile-loop.ts +38 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.368",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.368",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
import type { ChatMessage } from '../types.js';
|
|
31
31
|
import type { SessionTransport } from '../shared-types.js';
|
|
32
32
|
import { filterUserFacingChatMessages, isActivityChatMessage, isUserFacingChatMessage, normalizeChatMessages } from '../providers/chat-message-normalization.js';
|
|
33
|
+
import { normalizeMeshWorkspaceForCompare } from '@adhdev/mesh-shared';
|
|
33
34
|
|
|
34
35
|
const RECENT_SEND_WINDOW_MS = 1200;
|
|
35
36
|
export const READ_CHAT_PROVIDER_EVAL_TIMEOUT_MS = 25_000;
|
|
@@ -989,6 +990,53 @@ function normalizeComparableWorkspace(value: unknown): string {
|
|
|
989
990
|
return path.resolve(text);
|
|
990
991
|
}
|
|
991
992
|
|
|
993
|
+
/**
|
|
994
|
+
* read_chat node scope verdict. One physical daemon hosts a base node plus several
|
|
995
|
+
* worktree nodes; mesh_read_chat always dispatches read_chat with the requested
|
|
996
|
+
* node's workspace (`args.workspace`). When the resolved target session actually
|
|
997
|
+
* lives in a DIFFERENT worktree, returning its transcript — or worse, letting the
|
|
998
|
+
* native-history-by-workspace fallback splice sibling worktree turns into the
|
|
999
|
+
* reply — makes the coordinator believe one session received every worktree's
|
|
1000
|
+
* work. This guard refuses a CONFIRMED cross-workspace read instead of mixing.
|
|
1001
|
+
*
|
|
1002
|
+
* Conservative by design (mirrors the WTCLAIM fix-B "unknown → allow" rule): only
|
|
1003
|
+
* a session id that resolves to a known workspace which is unequal to a known
|
|
1004
|
+
* intended workspace blocks. When either side is unknown — no targetSessionId, no
|
|
1005
|
+
* args.workspace, an unregistered session, the coordinator self-session, or a
|
|
1006
|
+
* plain dashboard read that never passes a node workspace — the read proceeds
|
|
1007
|
+
* untouched, so base-node and same-daemon coordinator reads never regress.
|
|
1008
|
+
*/
|
|
1009
|
+
export function evaluateReadChatNodeWorkspaceScope(args: {
|
|
1010
|
+
targetSessionId?: string;
|
|
1011
|
+
intendedWorkspace?: string;
|
|
1012
|
+
sessionWorkspace?: string;
|
|
1013
|
+
}): { scoped: false } | { scoped: true; intended: string; actual: string } {
|
|
1014
|
+
const targetSessionId = typeof args.targetSessionId === 'string' ? args.targetSessionId.trim() : '';
|
|
1015
|
+
if (!targetSessionId) return { scoped: false };
|
|
1016
|
+
const intended = normalizeMeshWorkspaceForCompare(args.intendedWorkspace);
|
|
1017
|
+
const actual = normalizeMeshWorkspaceForCompare(args.sessionWorkspace);
|
|
1018
|
+
if (!intended || !actual) return { scoped: false };
|
|
1019
|
+
if (intended === actual) return { scoped: false };
|
|
1020
|
+
return { scoped: true, intended, actual };
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/**
|
|
1024
|
+
* Resolve the target session's ACTUAL workspace from the most authoritative source
|
|
1025
|
+
* available on this daemon: the session registry record (stamped at register
|
|
1026
|
+
* time), then the live CLI adapter's working directory, then the bound instance
|
|
1027
|
+
* state. Returns '' when nothing knows the session's workspace — the caller treats
|
|
1028
|
+
* that as "unknown" and does not block.
|
|
1029
|
+
*/
|
|
1030
|
+
function resolveTargetSessionActualWorkspace(h: CommandHelpers, targetSessionId: string): string {
|
|
1031
|
+
const registryWorkspace = (h.ctx?.sessionRegistry?.get?.(targetSessionId) as any)?.workspace;
|
|
1032
|
+
if (typeof registryWorkspace === 'string' && registryWorkspace.trim()) return registryWorkspace;
|
|
1033
|
+
const adapter = h.getCliAdapter?.(targetSessionId);
|
|
1034
|
+
if (adapter && typeof adapter.workingDir === 'string' && adapter.workingDir.trim()) return adapter.workingDir;
|
|
1035
|
+
const instanceWorkspace = (getTargetInstance(h, { targetSessionId })?.getState?.() as any)?.workspace;
|
|
1036
|
+
if (typeof instanceWorkspace === 'string' && instanceWorkspace.trim()) return instanceWorkspace;
|
|
1037
|
+
return '';
|
|
1038
|
+
}
|
|
1039
|
+
|
|
992
1040
|
function isCurrentRuntimePtySafelyAttributed(args: {
|
|
993
1041
|
adapter: CliAdapter;
|
|
994
1042
|
helpers: CommandHelpers;
|
|
@@ -2100,6 +2148,29 @@ export async function handleChatHistory(h: CommandHelpers, args: any): Promise<C
|
|
|
2100
2148
|
}
|
|
2101
2149
|
|
|
2102
2150
|
export async function handleReadChat(h: CommandHelpers, args: any): Promise<CommandResult> {
|
|
2151
|
+
// Node scope guard: a daemon hosting a base node + several worktree nodes must
|
|
2152
|
+
// not serve worktree A's transcript (or splice sibling worktree turns via the
|
|
2153
|
+
// native-history-by-workspace fallback) when a coordinator scoped the read to
|
|
2154
|
+
// worktree B. mesh_read_chat always passes the requested node's workspace as
|
|
2155
|
+
// args.workspace; refuse a CONFIRMED cross-workspace read rather than mix.
|
|
2156
|
+
{
|
|
2157
|
+
const guardSessionId = typeof args?.targetSessionId === 'string' ? args.targetSessionId.trim() : '';
|
|
2158
|
+
if (guardSessionId && typeof args?.workspace === 'string' && args.workspace.trim()) {
|
|
2159
|
+
const verdict = evaluateReadChatNodeWorkspaceScope({
|
|
2160
|
+
targetSessionId: guardSessionId,
|
|
2161
|
+
intendedWorkspace: args.workspace,
|
|
2162
|
+
sessionWorkspace: resolveTargetSessionActualWorkspace(h, guardSessionId),
|
|
2163
|
+
});
|
|
2164
|
+
if (verdict.scoped) {
|
|
2165
|
+
LOG.info('Command', `[read_chat] node scope mismatch: session ${guardSessionId} workspace "${verdict.actual}" ≠ requested node workspace "${verdict.intended}" — refusing cross-worktree transcript`);
|
|
2166
|
+
return {
|
|
2167
|
+
success: false,
|
|
2168
|
+
code: 'read_chat_session_node_scope_mismatch',
|
|
2169
|
+
error: `Session ${guardSessionId} belongs to a different worktree (workspace "${verdict.actual}") than the requested node (workspace "${verdict.intended}"). Refusing to return a cross-worktree transcript — target the node that owns this session.`,
|
|
2170
|
+
};
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
}
|
|
2103
2174
|
// Resolve provider in order: explicit agentType/providerType > registered session.
|
|
2104
2175
|
// Without this fallback, callers that only have a sessionId (e.g. a chat tail
|
|
2105
2176
|
// controller that just got handed a session ID over WS) get an empty result
|
|
@@ -2649,10 +2720,33 @@ export async function handleReadChat(h: CommandHelpers, args: any): Promise<Comm
|
|
|
2649
2720
|
});
|
|
2650
2721
|
|
|
2651
2722
|
if (supportsNative && !decision.nativeSelected) {
|
|
2723
|
+
// Dead-end: we are in the history-only path (no live PTY/ACP
|
|
2724
|
+
// adapter was found for this target session) AND provider-native
|
|
2725
|
+
// history is not safely mappable to the requested session
|
|
2726
|
+
// (no historySessionId stamp / workspace mismatch). Previously
|
|
2727
|
+
// this returned `success:false`, which the command logger emits
|
|
2728
|
+
// at warn level on EVERY poll (handler.ts logCommandEnd) —
|
|
2729
|
+
// mesh coordinators poll read_chat continuously, so a worker whose
|
|
2730
|
+
// transcript can never be safely mapped produced a 100% warn-log
|
|
2731
|
+
// storm with no recovery. Switch to a SOFT response: success with
|
|
2732
|
+
// empty messages + pending:true so the coordinator treats it as
|
|
2733
|
+
// "no live messages readable yet" rather than a hard failure, and
|
|
2734
|
+
// carry the machine-readable reason for debuggability. The normal
|
|
2735
|
+
// live-adapter path (above) and the safe-native return (below) are
|
|
2736
|
+
// unaffected — this is strictly the both-absent dead end.
|
|
2737
|
+
LOG.debug('Command', `[read_chat] soft pending: no live adapter and native history not safely mappable target=${String(args?.targetSessionId || '')} provider=${agentStr} reason=native_history_not_safely_available`);
|
|
2652
2738
|
return {
|
|
2653
|
-
success:
|
|
2739
|
+
success: true,
|
|
2740
|
+
pending: true,
|
|
2741
|
+
// Both signals are true here: we reached the history-only path
|
|
2742
|
+
// because no live adapter was found (`live_adapter_not_found`),
|
|
2743
|
+
// and native history is not safely mappable
|
|
2744
|
+
// (`native_history_not_safely_available`).
|
|
2745
|
+
reason: 'native_history_not_safely_available',
|
|
2746
|
+
reasons: ['live_adapter_not_found', 'native_history_not_safely_available'],
|
|
2654
2747
|
code: 'native_history_not_safely_available',
|
|
2655
|
-
|
|
2748
|
+
messages: [],
|
|
2749
|
+
status: 'idle',
|
|
2656
2750
|
providerSessionId: historyProviderSessionId,
|
|
2657
2751
|
messageSource: decision.messageSource,
|
|
2658
2752
|
transcriptProvenance: decision.messageSource,
|
|
@@ -706,6 +706,28 @@ export class DaemonCliManager {
|
|
|
706
706
|
}
|
|
707
707
|
|
|
708
708
|
this.adapters.set(key, cliInstance.getAdapter());
|
|
709
|
+
|
|
710
|
+
// WTDISPATCH (no_node_binding): a coordinator-launched worker carries its mesh node
|
|
711
|
+
// binding on the CLI-instance settings, but the session-host RECORD meta was never
|
|
712
|
+
// stamped with it — `updateRuntimeSettings` only mutates in-memory runtime settings and
|
|
713
|
+
// `updateRuntimeMeta` was only ever called with providerSessionId. So mesh_cleanup_sessions
|
|
714
|
+
// matched these worker sessions to a node by workspace ALONE
|
|
715
|
+
// (`live_session_matched_by_workspace_only_no_node_binding`), which on a daemon hosting
|
|
716
|
+
// sibling worktree nodes cannot tell two co-located clones apart. Push the launch-time node
|
|
717
|
+
// binding to the record meta so the record is 1:1 bound to its node (the spawned pty exists
|
|
718
|
+
// by now, so updateMeta reaches the session-host store). Best-effort; guarded.
|
|
719
|
+
const launchMeshNodeId = typeof settings?.meshNodeId === 'string' ? settings.meshNodeId.trim() : '';
|
|
720
|
+
const launchMeshNodeFor = typeof settings?.meshNodeFor === 'string' ? settings.meshNodeFor.trim() : '';
|
|
721
|
+
if (launchMeshNodeId || launchMeshNodeFor) {
|
|
722
|
+
try {
|
|
723
|
+
cliInstance.getAdapter().updateRuntimeMeta?.({
|
|
724
|
+
...(launchMeshNodeId ? { meshNodeId: launchMeshNodeId } : {}),
|
|
725
|
+
...(launchMeshNodeFor ? { meshNodeFor: launchMeshNodeFor } : {}),
|
|
726
|
+
...(settings?.launchedByCoordinator === true ? { launchedByCoordinator: true } : {}),
|
|
727
|
+
});
|
|
728
|
+
} catch { /* best-effort — record-meta stamp is cleanup hygiene, not on the dispatch path */ }
|
|
729
|
+
}
|
|
730
|
+
|
|
709
731
|
this.startCliExitMonitor(key, cliType);
|
|
710
732
|
}
|
|
711
733
|
|
package/src/commands/router.ts
CHANGED
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
normalizeMeshNodeId,
|
|
48
48
|
meshNodeIdMatches,
|
|
49
49
|
daemonIdsEquivalent,
|
|
50
|
+
meshWorkspacesEquivalent,
|
|
50
51
|
} from '@adhdev/mesh-shared';
|
|
51
52
|
import { SessionRegistry } from '../sessions/registry.js';
|
|
52
53
|
import { LOG } from '../logging/logger.js';
|
|
@@ -1130,6 +1131,21 @@ function readMeshTimeoutEnvMs(name: string, defaultMs: number): number {
|
|
|
1130
1131
|
// (still under the P2P REQUEST_TIMEOUT of 30s) and made env-overridable.
|
|
1131
1132
|
export const MESH_DIRECT_PROBE_TIMEOUT_MS = readMeshTimeoutEnvMs('MESH_DIRECT_PROBE_TIMEOUT_MS', 25_000);
|
|
1132
1133
|
export const MESH_DIRECT_PROBE_RETRY_TIMEOUT_MS = readMeshTimeoutEnvMs('MESH_DIRECT_PROBE_RETRY_TIMEOUT_MS', 25_000);
|
|
1134
|
+
// Cold-open warmup budget for the FIRST direct-peer probe to a peer whose mesh
|
|
1135
|
+
// DataChannel is not open yet. A fresh cross-machine, TURN-relayed handshake
|
|
1136
|
+
// (ICE gather + TURN allocation + DTLS across two residential networks) routinely
|
|
1137
|
+
// needs many seconds. Charging that warmup against the response deadline
|
|
1138
|
+
// (MESH_DIRECT_PROBE_TIMEOUT_MS) made the very first git_status to a cold peer
|
|
1139
|
+
// false-timeout, after which the warm retry — reusing the now-open channel —
|
|
1140
|
+
// succeeded: the classic cold-open signature. This budget bounds ONLY the
|
|
1141
|
+
// "channel not open yet" phase; once the channel opens the response deadline
|
|
1142
|
+
// governs the round trip. A genuine connect failure still rejects immediately —
|
|
1143
|
+
// the mesh manager fails the peer the instant its PeerConnection state goes
|
|
1144
|
+
// terminal, and isMeshConnectionDefinitivelyDown pre-gates an already-dead peer —
|
|
1145
|
+
// so this never masks a real failure for the whole window; it only grants a
|
|
1146
|
+
// still-handshaking peer the time it legitimately needs. Matches the daemon-cloud
|
|
1147
|
+
// DaemonMeshManager CONNECT_TIMEOUT_MS (45s). Env-overridable for very slow links.
|
|
1148
|
+
export const MESH_DIRECT_PROBE_CONNECT_TIMEOUT_MS = readMeshTimeoutEnvMs('MESH_DIRECT_PROBE_CONNECT_TIMEOUT_MS', 45_000);
|
|
1133
1149
|
// How long a successful per-peer git_status probe stays fresh enough to be
|
|
1134
1150
|
// reused instead of issuing another blocking `refreshUpstream:true` fan-out.
|
|
1135
1151
|
// A slow (TURN-relayed) peer's probe can take 9-23s, and the dashboard's
|
|
@@ -1198,17 +1214,120 @@ export class MeshGitProbeCache {
|
|
|
1198
1214
|
}
|
|
1199
1215
|
}
|
|
1200
1216
|
|
|
1217
|
+
/**
|
|
1218
|
+
* Await `work` under a warmup-aware deadline so a cold-open DataChannel handshake
|
|
1219
|
+
* is NOT charged against the command response budget — the root cause of the
|
|
1220
|
+
* "first mesh probe to a cold peer false-times-out, the warm retry succeeds"
|
|
1221
|
+
* signature. Two budgets, switched by the live peer connection state:
|
|
1222
|
+
*
|
|
1223
|
+
* - While `isConnected()` returns false the peer's channel is still opening; the
|
|
1224
|
+
* cold-open `connectTimeoutMs` budget applies. This phase is deliberately
|
|
1225
|
+
* generous because a TURN-relayed cross-machine handshake legitimately needs
|
|
1226
|
+
* many seconds — but a genuine connect *failure* is surfaced by `work`
|
|
1227
|
+
* rejecting on its own (the mesh manager fails the peer the instant its
|
|
1228
|
+
* PeerConnection state goes terminal), so a real failure is never masked for
|
|
1229
|
+
* the whole window.
|
|
1230
|
+
* - The first time `isConnected()` returns true the channel is warm; from that
|
|
1231
|
+
* instant the tight `responseTimeoutMs` governs how long the handler may take.
|
|
1232
|
+
* Warm-channel callers therefore see behavior identical to the old single
|
|
1233
|
+
* `Promise.race(work, responseTimeoutMs)`.
|
|
1234
|
+
*
|
|
1235
|
+
* Rejects with `Error('timeout')` when either budget is exhausted, mirroring the
|
|
1236
|
+
* previous single-race contract. Pure except for timers + the injected
|
|
1237
|
+
* `isConnected` probe, so it is unit-testable under fake timers without any real
|
|
1238
|
+
* WebRTC. When no connection getter is wired `isConnected` should be `() => true`
|
|
1239
|
+
* (the caller's choice) so the response deadline governs from t0 — the legacy
|
|
1240
|
+
* single-budget behavior, never a combined connect+response window.
|
|
1241
|
+
*/
|
|
1242
|
+
export function awaitWithWarmupDeadline<T>(
|
|
1243
|
+
work: Promise<T>,
|
|
1244
|
+
opts: {
|
|
1245
|
+
isConnected: () => boolean;
|
|
1246
|
+
connectTimeoutMs: number;
|
|
1247
|
+
responseTimeoutMs: number;
|
|
1248
|
+
pollIntervalMs?: number;
|
|
1249
|
+
},
|
|
1250
|
+
): Promise<T> {
|
|
1251
|
+
const pollMs = Math.max(1, Math.min(opts.pollIntervalMs ?? 200, opts.connectTimeoutMs));
|
|
1252
|
+
return new Promise<T>((resolve, reject) => {
|
|
1253
|
+
let done = false;
|
|
1254
|
+
let poll: ReturnType<typeof setInterval> | undefined;
|
|
1255
|
+
let responseTimer: ReturnType<typeof setTimeout> | undefined;
|
|
1256
|
+
const startedAt = Date.now();
|
|
1257
|
+
const cleanup = () => {
|
|
1258
|
+
if (poll) { clearInterval(poll); poll = undefined; }
|
|
1259
|
+
if (responseTimer) { clearTimeout(responseTimer); responseTimer = undefined; }
|
|
1260
|
+
};
|
|
1261
|
+
const settle = (fn: () => void) => {
|
|
1262
|
+
if (done) return;
|
|
1263
|
+
done = true;
|
|
1264
|
+
cleanup();
|
|
1265
|
+
fn();
|
|
1266
|
+
};
|
|
1267
|
+
// Arm the response deadline exactly once, the moment the channel is warm.
|
|
1268
|
+
const armResponse = () => {
|
|
1269
|
+
if (responseTimer || done) return;
|
|
1270
|
+
responseTimer = setTimeout(
|
|
1271
|
+
() => settle(() => reject(new Error('timeout'))),
|
|
1272
|
+
opts.responseTimeoutMs,
|
|
1273
|
+
);
|
|
1274
|
+
if (typeof responseTimer.unref === 'function') responseTimer.unref();
|
|
1275
|
+
};
|
|
1276
|
+
const onPoll = () => {
|
|
1277
|
+
if (done) return;
|
|
1278
|
+
if (opts.isConnected()) {
|
|
1279
|
+
if (poll) { clearInterval(poll); poll = undefined; }
|
|
1280
|
+
armResponse();
|
|
1281
|
+
return;
|
|
1282
|
+
}
|
|
1283
|
+
if (Date.now() - startedAt >= opts.connectTimeoutMs) {
|
|
1284
|
+
settle(() => reject(new Error('timeout')));
|
|
1285
|
+
}
|
|
1286
|
+
};
|
|
1287
|
+
if (opts.isConnected()) {
|
|
1288
|
+
// Already warm (e.g. a retry over an open channel) — skip the warmup
|
|
1289
|
+
// phase entirely and let the response deadline govern from t0.
|
|
1290
|
+
armResponse();
|
|
1291
|
+
} else {
|
|
1292
|
+
poll = setInterval(onPoll, pollMs);
|
|
1293
|
+
if (typeof poll.unref === 'function') poll.unref();
|
|
1294
|
+
}
|
|
1295
|
+
work.then(
|
|
1296
|
+
(val) => settle(() => resolve(val)),
|
|
1297
|
+
(err) => settle(() => reject(err)),
|
|
1298
|
+
);
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1201
1302
|
async function probeRemoteMeshGitStatus(args: {
|
|
1202
1303
|
dispatchMeshCommand?: (daemonId: string, cmd: string, args: Record<string, unknown>) => Promise<unknown>;
|
|
1203
1304
|
daemonId: string;
|
|
1204
1305
|
workspace: string;
|
|
1205
|
-
|
|
1306
|
+
// Response deadline — applies only once the peer's DataChannel is open (warm).
|
|
1307
|
+
responseTimeoutMs: number;
|
|
1308
|
+
// Cold-open warmup budget — applies only while the channel is still opening.
|
|
1309
|
+
connectTimeoutMs: number;
|
|
1310
|
+
// Live peer connection snapshot getter; lets the deadline tell "still warming
|
|
1311
|
+
// up" apart from "warm but slow". Absent → behave as if always warm (the
|
|
1312
|
+
// response deadline governs from t0, i.e. the legacy single-budget behavior).
|
|
1313
|
+
getConnection?: (daemonId: string) => Record<string, unknown> | null;
|
|
1206
1314
|
}): Promise<Record<string, unknown> | null> {
|
|
1207
1315
|
if (!args.dispatchMeshCommand) return null;
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1316
|
+
// Fire the dispatch first — this is what drives the mesh manager to ensure /
|
|
1317
|
+
// open the peer connection. The warmup-aware deadline then charges the
|
|
1318
|
+
// cold-open handshake to the connect budget and only the warm round trip to
|
|
1319
|
+
// the response budget, so the first probe to a cold peer is no longer
|
|
1320
|
+
// false-timed-out before its channel has even opened.
|
|
1321
|
+
const dispatch = args.dispatchMeshCommand(args.daemonId, 'git_status', { workspace: args.workspace, refreshUpstream: true });
|
|
1322
|
+
const getConnection = args.getConnection;
|
|
1323
|
+
const isConnected = getConnection
|
|
1324
|
+
? () => readMeshConnectionState(getConnection(args.daemonId)) === 'connected'
|
|
1325
|
+
: () => true;
|
|
1326
|
+
const remoteResult = await awaitWithWarmupDeadline(dispatch, {
|
|
1327
|
+
isConnected,
|
|
1328
|
+
connectTimeoutMs: args.connectTimeoutMs,
|
|
1329
|
+
responseTimeoutMs: args.responseTimeoutMs,
|
|
1330
|
+
}) as any;
|
|
1212
1331
|
const remoteGit = remoteResult?.status ?? remoteResult?.git ?? remoteResult;
|
|
1213
1332
|
if (!remoteGit || typeof remoteGit !== 'object' || typeof remoteGit.isGitRepo !== 'boolean') return null;
|
|
1214
1333
|
// The member daemon stamps its own platform/arch onto the git_status result
|
|
@@ -1269,6 +1388,8 @@ export async function probeRemoteMeshGitStatusWithRetry(args: {
|
|
|
1269
1388
|
timeoutMs: number;
|
|
1270
1389
|
/** Per-attempt timeout for retries (attempts > 0); defaults to timeoutMs. */
|
|
1271
1390
|
retryTimeoutMs?: number;
|
|
1391
|
+
/** Cold-open warmup budget per attempt; defaults to MESH_DIRECT_PROBE_CONNECT_TIMEOUT_MS. */
|
|
1392
|
+
connectTimeoutMs?: number;
|
|
1272
1393
|
getConnection?: (daemonId: string) => Record<string, unknown> | null;
|
|
1273
1394
|
onConnection?: (connection: Record<string, unknown>) => void;
|
|
1274
1395
|
}): Promise<Record<string, unknown> | null> {
|
|
@@ -1300,7 +1421,9 @@ export async function probeRemoteMeshGitStatusWithRetry(args: {
|
|
|
1300
1421
|
dispatchMeshCommand: args.dispatchMeshCommand,
|
|
1301
1422
|
daemonId: args.daemonId,
|
|
1302
1423
|
workspace: args.workspace,
|
|
1303
|
-
|
|
1424
|
+
responseTimeoutMs: attempt === 0 ? args.timeoutMs : (args.retryTimeoutMs ?? args.timeoutMs),
|
|
1425
|
+
connectTimeoutMs: args.connectTimeoutMs ?? MESH_DIRECT_PROBE_CONNECT_TIMEOUT_MS,
|
|
1426
|
+
getConnection: args.getConnection,
|
|
1304
1427
|
});
|
|
1305
1428
|
if (remoteGit) return remoteGit;
|
|
1306
1429
|
} catch {
|
|
@@ -1447,6 +1570,7 @@ export async function hydrateInlineMeshDirectTruth(args: {
|
|
|
1447
1570
|
workspace,
|
|
1448
1571
|
timeoutMs: MESH_DIRECT_PROBE_TIMEOUT_MS,
|
|
1449
1572
|
retryTimeoutMs: MESH_DIRECT_PROBE_RETRY_TIMEOUT_MS,
|
|
1573
|
+
connectTimeoutMs: MESH_DIRECT_PROBE_CONNECT_TIMEOUT_MS,
|
|
1450
1574
|
getConnection: args.getMeshPeerConnectionStatus,
|
|
1451
1575
|
});
|
|
1452
1576
|
const remoteGit = args.probeCache
|
|
@@ -1512,14 +1636,17 @@ function liveSessionRecordMatchesMeshNode(record: any, meshId: string, nodeId: s
|
|
|
1512
1636
|
if (!recordNodeId || recordNodeId !== nodeId) return false;
|
|
1513
1637
|
if (nodeIsMissingLocalWorktree) return false;
|
|
1514
1638
|
const recordWorkspace = readStringValue(record?.workspace);
|
|
1515
|
-
|
|
1639
|
+
// Normalized compare (shared WTCLAIM rule): a base node and a co-located worktree
|
|
1640
|
+
// clone differ ONLY by workspace root, so a separator/case-skewed exact compare
|
|
1641
|
+
// could wrongly keep a sibling worktree's session attached to this node.
|
|
1642
|
+
if (nodeWorkspace && recordWorkspace && !meshWorkspacesEquivalent(recordWorkspace, nodeWorkspace)) return false;
|
|
1516
1643
|
const recordMeshId = readStringValue(record?.meta?.meshNodeFor);
|
|
1517
1644
|
return !recordMeshId || recordMeshId === meshId;
|
|
1518
1645
|
}
|
|
1519
1646
|
|
|
1520
1647
|
function liveSessionRecordMatchesMeshWorkspace(record: any, meshId: string, workspace: string): boolean {
|
|
1521
1648
|
const recordWorkspace = readStringValue(record?.workspace);
|
|
1522
|
-
if (!recordWorkspace || !workspace || recordWorkspace
|
|
1649
|
+
if (!recordWorkspace || !workspace || !meshWorkspacesEquivalent(recordWorkspace, workspace)) return false;
|
|
1523
1650
|
|
|
1524
1651
|
const recordMeshId = readStringValue(record?.meta?.meshNodeFor);
|
|
1525
1652
|
if (recordMeshId) return recordMeshId === meshId;
|
|
@@ -19,7 +19,7 @@ import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
|
19
19
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
20
20
|
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
21
21
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
22
|
-
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
22
|
+
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
23
23
|
import {
|
|
24
24
|
findRecentTerminalLedgerEvidence,
|
|
25
25
|
hasDispatchAfterTerminal,
|
|
@@ -457,16 +457,10 @@ function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: Delive
|
|
|
457
457
|
});
|
|
458
458
|
}
|
|
459
459
|
|
|
460
|
-
// WTCLAIM:
|
|
461
|
-
//
|
|
462
|
-
//
|
|
463
|
-
//
|
|
464
|
-
// told apart. Kept local (the cli-manager copy is module-private) so the comparison rule
|
|
465
|
-
// stays identical to the one fix-B already uses on the worker side.
|
|
466
|
-
function normalizeMeshWorkspaceForCompare(dir?: string): string {
|
|
467
|
-
if (typeof dir !== 'string') return '';
|
|
468
|
-
return dir.trim().replace(/[\\/]+/g, '/').replace(/\/+$/, '').toLowerCase();
|
|
469
|
-
}
|
|
460
|
+
// WTCLAIM: workspace normalization for base-vs-worktree comparison now lives in
|
|
461
|
+
// @adhdev/mesh-shared (normalizeMeshWorkspaceForCompare) so the enqueue→claim path,
|
|
462
|
+
// the mesh_status per-node session filter, and the read_chat node scope guard all
|
|
463
|
+
// share one comparison rule instead of drifting module-private copies.
|
|
470
464
|
|
|
471
465
|
export function tryAssignQueueTask(
|
|
472
466
|
components: DaemonComponents,
|
|
@@ -493,14 +487,52 @@ export function tryAssignQueueTask(
|
|
|
493
487
|
// getRemoteIdleSessions). Conservative by design: when either workspace is unknown we do NOT
|
|
494
488
|
// skip, so a node with no declared workspace keeps its prior behavior and no legitimate claim
|
|
495
489
|
// is starved.
|
|
490
|
+
// WTDISPATCH (residual of WTCLAIM): the cross-node claim guard must reach EVERY claiming
|
|
491
|
+
// session this daemon can observe — not only those whose adapter happens to be in
|
|
492
|
+
// cliManager.adapters. An auto-launched worker session can carry its node binding on the
|
|
493
|
+
// CLI-instance settings while its session-host record shows no_node_binding, and the
|
|
494
|
+
// event-driven / remote-idle drain (agent:ready → setRemoteIdleSession → tryAssignQueueTask)
|
|
495
|
+
// can pass a nodeId that does NOT belong to the claiming session — a sibling worktree node
|
|
496
|
+
// on the SAME daemon. The adapter-only WTCLAIM check (rc.361/4c5b30b1) never engaged for a
|
|
497
|
+
// session observed solely via instanceManager, so session A could pull node B's task and
|
|
498
|
+
// node A's task was left with no session to claim it (no task_dispatched — it never dispatches).
|
|
499
|
+
//
|
|
500
|
+
// Resolve the claiming session's REAL identity from the adapter workingDir, then fall back to
|
|
501
|
+
// the live CLI instance's workspace + its stamped meshNodeId, and refuse a claim that
|
|
502
|
+
// contradicts EITHER (fail-closed). Reuses the shared meshWorkspacesEquivalent / meshNodeIdMatches
|
|
503
|
+
// comparators — no new comparison logic. Conservative: when neither the workspace NOR the stamp
|
|
504
|
+
// is resolvable we do NOT refuse, so a node with no declared workspace keeps prior behavior and
|
|
505
|
+
// a genuinely remote (cross-daemon) candidate stays nodeId-matched from getRemoteIdleSessions.
|
|
496
506
|
const localClaimAdapter = components.cliManager?.adapters?.get(sessionId) as { workingDir?: string } | undefined;
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
507
|
+
let claimInstanceWorkspace = '';
|
|
508
|
+
let claimStampedNodeId = '';
|
|
509
|
+
try {
|
|
510
|
+
const claimState = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
|
|
511
|
+
claimInstanceWorkspace = readNonEmptyString(claimState?.workspace);
|
|
512
|
+
const claimSettings = (claimState?.settings as Record<string, unknown>) || {};
|
|
513
|
+
claimStampedNodeId = readNonEmptyString(claimSettings.meshNodeId);
|
|
514
|
+
} catch { /* best-effort — fall through to the conservative (no refuse) path */ }
|
|
515
|
+
|
|
516
|
+
const nodeWorkspaceRaw = readNonEmptyString(node?.workspace);
|
|
517
|
+
const sessionWorkspaceRaw = readNonEmptyString(localClaimAdapter?.workingDir) || claimInstanceWorkspace;
|
|
518
|
+
|
|
519
|
+
if (claimStampedNodeId && nodeId) {
|
|
520
|
+
// The session carries its OWN meshNodeId stamp — its authoritative node identity, set when
|
|
521
|
+
// the coordinator launched/dispatched it (mesh-routing trusts this stamp FIRST). When it
|
|
522
|
+
// matches the claim target the session genuinely belongs to this node, so the stamp settles
|
|
523
|
+
// it and the workspace heuristic is skipped (a base/worktree pair can legitimately share a
|
|
524
|
+
// workspace). When it does NOT match, the claim is a cross-node leak — refuse, fail-closed.
|
|
525
|
+
if (!meshNodeIdMatches({ id: claimStampedNodeId } as MeshNodeIdentified, nodeId)) {
|
|
526
|
+
LOG.info('MeshQueue', `WTDISPATCH: refusing claim for node ${nodeId} (${sessionId}) — session is bound to node "${claimStampedNodeId}" (cross-node claim blocked)`);
|
|
502
527
|
return false;
|
|
503
528
|
}
|
|
529
|
+
} else if (sessionWorkspaceRaw && nodeWorkspaceRaw && !meshWorkspacesEquivalent(sessionWorkspaceRaw, nodeWorkspaceRaw)) {
|
|
530
|
+
// No stamp (the no_node_binding worker) — fall back to the workspace to tell two co-located
|
|
531
|
+
// sibling worktree sessions apart. WTCLAIM, now reaching instanceManager-observable sessions
|
|
532
|
+
// too. Conservative: unknown workspace on either side → do NOT refuse (no legitimate claim
|
|
533
|
+
// starved; a genuinely remote cross-daemon candidate stays nodeId-matched as before).
|
|
534
|
+
LOG.info('MeshQueue', `WTCLAIM: refusing claim for node ${nodeId} (${sessionId}) — session workspace "${normalizeMeshWorkspaceForCompare(sessionWorkspaceRaw)}" ≠ node workspace "${normalizeMeshWorkspaceForCompare(nodeWorkspaceRaw)}" (cross-workspace dispatch blocked)`);
|
|
535
|
+
return false;
|
|
504
536
|
}
|
|
505
537
|
|
|
506
538
|
const capabilityTags = buildMeshNodeCapabilityTags(node, providerType);
|
|
@@ -1844,7 +1876,26 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1844
1876
|
// re-attributed to the latest task (the normal task_completed path below).
|
|
1845
1877
|
const supersedesWeakTerminal = isWeakTerminalLedgerPayload(terminal.payload)
|
|
1846
1878
|
&& isGenuineCompletionEvidence(args.metadataEvent);
|
|
1847
|
-
|
|
1879
|
+
// CANON-B (direct-dispatch completion race): a FAST direct dispatch (mesh_send_task)
|
|
1880
|
+
// to an already-idle, previously-used session can have its genuine completion reach
|
|
1881
|
+
// this coordinator handler BEFORE the dispatching side records the new task's dispatch
|
|
1882
|
+
// row / task_dispatched ledger entry — insertDirectDispatch + appendLedgerEntry both run
|
|
1883
|
+
// AFTER the agent_command await resolves, while the worker may already be done. In that
|
|
1884
|
+
// window sessionHasActiveAssignment is false (no active dispatch row, no unterminal
|
|
1885
|
+
// ledger entry yet), so this prior-terminal dedup engages; and because providerSessionId
|
|
1886
|
+
// is STABLE across a reused session's turns, the providerSessionId/finalSummary match
|
|
1887
|
+
// below would suppress the NEW task's completion as a duplicate of the PRIOR task —
|
|
1888
|
+
// silently losing it (the observed intermittent miss; fresh enqueue/autoLaunch is immune
|
|
1889
|
+
// because a fresh session has no prior same-providerSessionId terminal and the queue row
|
|
1890
|
+
// is claimed atomically before dispatch). The echoed taskId is the authoritative
|
|
1891
|
+
// discriminator: when the completion names a DIFFERENT task than the recorded terminal,
|
|
1892
|
+
// it is a genuinely new task's completion, never a duplicate — let it through so it is
|
|
1893
|
+
// attributed to its own taskId. A same-task re-arrival (taskId equal) or a taskId-less
|
|
1894
|
+
// legacy event still falls through to the providerSessionId/finalSummary dedup.
|
|
1895
|
+
const terminalTaskId = readNonEmptyString(terminal.payload.taskId);
|
|
1896
|
+
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId);
|
|
1897
|
+
const distinctTaskCompletion = !!eventTaskId && !!terminalTaskId && eventTaskId !== terminalTaskId;
|
|
1898
|
+
if (!newDispatchAfterTerminal && !supersedesWeakTerminal && !distinctTaskCompletion) {
|
|
1848
1899
|
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
1849
1900
|
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
1850
1901
|
const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
|
|
@@ -2408,6 +2459,43 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
2408
2459
|
});
|
|
2409
2460
|
}
|
|
2410
2461
|
|
|
2462
|
+
// ---------------------------------------------------------------------------
|
|
2463
|
+
// Per-coordinator forward serialization (P2P send-backpressure relief).
|
|
2464
|
+
//
|
|
2465
|
+
// When several workers finish at once, each completion runs forwardUnresolvedDelegate
|
|
2466
|
+
// Event and fires its own `mesh_forward_event` push. Firing the whole burst
|
|
2467
|
+
// concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
|
|
2468
|
+
// which starves the rpc_ack/rpc_res replies the same channel must carry — a
|
|
2469
|
+
// coordinator's inbound `git_status` then times out even though the worker's own
|
|
2470
|
+
// forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
|
|
2471
|
+
// pushes per coordinator: at most one push is in flight to a given coordinator at a
|
|
2472
|
+
// time, the rest run in arrival order behind it. A lone event (idle lane) still
|
|
2473
|
+
// dispatches immediately — only a genuine burst is paced. Durability is unchanged:
|
|
2474
|
+
// every event is already persisted to the outbox before the push runs, so serializing
|
|
2475
|
+
// only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
|
|
2476
|
+
// with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
|
|
2477
|
+
// which is the hard guarantee; this throttle keeps the burst from piling up there.
|
|
2478
|
+
interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
|
|
2479
|
+
const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
|
|
2480
|
+
function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
|
|
2481
|
+
let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
|
|
2482
|
+
if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
|
|
2483
|
+
const wasIdle = lane.depth === 0;
|
|
2484
|
+
lane.depth += 1;
|
|
2485
|
+
const dec = (): void => { lane!.depth -= 1; };
|
|
2486
|
+
if (wasIdle) {
|
|
2487
|
+
// Idle lane → dispatch synchronously, so a lone completion (the common case) has
|
|
2488
|
+
// ZERO added latency and the push call happens in-line. Only a genuine burst —
|
|
2489
|
+
// events arriving while a push is still in flight — is paced (else branch).
|
|
2490
|
+
lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
|
|
2491
|
+
} else {
|
|
2492
|
+
// Burst: queue behind the in-flight push(es) in arrival order so the whole burst
|
|
2493
|
+
// is not dumped into the shared DataChannel buffer at once. The tail is guarded
|
|
2494
|
+
// so one rejecting push never wedges the lane for the next.
|
|
2495
|
+
lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
|
|
2496
|
+
}
|
|
2497
|
+
}
|
|
2498
|
+
|
|
2411
2499
|
// ---------------------------------------------------------------------------
|
|
2412
2500
|
// Worker-side fallback forward for unresolved-mesh delegates.
|
|
2413
2501
|
//
|
|
@@ -2467,6 +2555,24 @@ function forwardUnresolvedDelegateEvent(
|
|
|
2467
2555
|
workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
|
|
2468
2556
|
};
|
|
2469
2557
|
|
|
2558
|
+
// Self-addressed fallback: the resolved coordinator IS this daemon (a self-
|
|
2559
|
+
// coordinating / single-node mesh, or a delegate whose coordinator anchor resolved
|
|
2560
|
+
// to our own id). A cross-daemon mesh_forward_event to our own id is REFUSED by the
|
|
2561
|
+
// dispatch self-dial guard ("route via the local router instead"), so persisting it
|
|
2562
|
+
// to the outbox would only loop forever in PHASE 0's retry, never acked. Honour the
|
|
2563
|
+
// guard's advice: route the event straight through the local receiver — the exact
|
|
2564
|
+
// path the coordinator runs on receiving a remote push — and skip the outbox entirely.
|
|
2565
|
+
const selfDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
2566
|
+
if (selfDaemonIds.some(self => daemonIdsEquivalent(self, coordinatorDaemonId))) {
|
|
2567
|
+
try {
|
|
2568
|
+
handleMeshForwardEvent(components, payload);
|
|
2569
|
+
LOG.info('MeshEvents', `Self-addressed unresolved-delegate ${eventName} routed via local router (coordinator ${coordinatorDaemonId} is self) — outbox skipped`);
|
|
2570
|
+
} catch (e: any) {
|
|
2571
|
+
LOG.warn('MeshEvents', `Local route of self-addressed unresolved-delegate ${eventName} failed: ${e?.message || e}`);
|
|
2572
|
+
}
|
|
2573
|
+
return true;
|
|
2574
|
+
}
|
|
2575
|
+
|
|
2470
2576
|
// 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
|
|
2471
2577
|
// does not duplicate the outbox row. If persistence fails we still attempt the
|
|
2472
2578
|
// push below (degrades to the old at-most-once behaviour rather than dropping
|
|
@@ -2485,21 +2591,27 @@ function forwardUnresolvedDelegateEvent(
|
|
|
2485
2591
|
// 2) Best-effort immediate push for low latency. On success, ack the outbox row so
|
|
2486
2592
|
// the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
|
|
2487
2593
|
traceMeshEventStage('forward_send', fwdTraceCtx, 'immediate push');
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2594
|
+
// Serialize per coordinator so a multi-worker completion burst is paced rather than
|
|
2595
|
+
// dumped concurrently into the shared P2P DataChannel buffer (see coordinator
|
|
2596
|
+
// ForwardLanes). dispatchMeshCommand was null-checked above; capture it for the
|
|
2597
|
+
// deferred closure.
|
|
2598
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
2599
|
+
enqueueCoordinatorForwardPush(coordinatorDaemonId, () =>
|
|
2600
|
+
Promise.resolve(dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
|
|
2601
|
+
.then((result: any) => {
|
|
2602
|
+
if (result && result.success === false) {
|
|
2603
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
2604
|
+
traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
|
|
2605
|
+
return;
|
|
2606
|
+
}
|
|
2607
|
+
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
2608
|
+
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
2609
|
+
})
|
|
2610
|
+
.catch((e: any) => {
|
|
2611
|
+
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
2612
|
+
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
2613
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
2614
|
+
}));
|
|
2503
2615
|
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
2504
2616
|
return true;
|
|
2505
2617
|
}
|