@triflux/core 10.51.0 → 10.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hub/bridge.mjs +69 -1197
- package/hub/cli-adapter-base.mjs +4 -104
- package/hub/codex-adapter.mjs +7 -116
- package/hub/index.mjs +3 -20
- package/hub/lib/process-utils.mjs +1 -33
- package/hub/team/claude-daemon-control.mjs +1 -2
- package/hub/team/claude-session-projection.mjs +5 -73
- package/hub/team/claude-transcript.mjs +19 -0
- package/hub/team/nativeProxy.mjs +759 -0
- package/hub/team/session-context.mjs +1 -1
- package/hud/constants.mjs +9 -22
- package/hud/hud-qos-status.mjs +16 -19
- package/hud/providers/codex-probe.mjs +2 -31
- package/hud/providers/codex.mjs +16 -58
- package/hud/providers/gemini.mjs +20 -0
- package/hud/renderers.mjs +101 -77
- package/hud/utils.mjs +9 -13
- package/package.json +3 -6
- package/scripts/lib/agent-route-policy.mjs +0 -67
- package/scripts/lib/cli-claude.mjs +0 -12
- package/scripts/lib/env-probe.mjs +0 -70
- package/scripts/lib/legacy-mcp-cleanup.mjs +283 -0
- package/scripts/lib/mcp-guard-engine.mjs +42 -101
- package/scripts/lib/session-presence.mjs +1 -99
- package/hub/account-broker.mjs +0 -1221
- package/hub/adaptive-diagnostic.mjs +0 -350
- package/hub/adaptive-inject.mjs +0 -224
- package/hub/adaptive-memory.mjs +0 -277
- package/hub/adaptive.mjs +0 -172
- package/hub/assign-callbacks.mjs +0 -170
- package/hub/delegator/contracts.mjs +0 -38
- package/hub/delegator/index.mjs +0 -14
- package/hub/delegator/schema/delegator-tools.schema.json +0 -250
- package/hub/delegator/service.mjs +0 -336
- package/hub/delegator/tool-definitions.mjs +0 -35
- package/hub/hitl.mjs +0 -202
- package/hub/lib/known-errors.json +0 -72
- package/hub/lib/memory-store.mjs +0 -1001
- package/hub/lib/phase-manager.mjs +0 -493
- package/hub/lib/state-snapshot.mjs +0 -293
- package/hub/lib/trace-recorder.mjs +0 -153
- package/hub/lib/uuidv7.mjs +0 -45
- package/hub/memory-doctor.mjs +0 -640
- package/hub/middleware/quota-middleware.mjs +0 -70
- package/hub/middleware/request-logger.mjs +0 -191
- package/hub/paths.mjs +0 -24
- package/hub/pipeline/index.mjs +0 -163
- package/hub/pipeline/state.mjs +0 -191
- package/hub/pipeline/transitions.mjs +0 -131
- package/hub/reflexion.mjs +0 -290
- package/hub/role-contract.mjs +0 -312
- package/hub/router.mjs +0 -2026
- package/hub/session-fingerprint.mjs +0 -418
- package/hub/state.mjs +0 -300
- package/hub/team/synapse-http.mjs +0 -311
- package/hub/team-bridge.mjs +0 -27
- package/scripts/lib/claudemd-scanner.mjs +0 -26
- package/scripts/lib/context.mjs +0 -51
- package/scripts/lib/handoff.mjs +0 -187
- package/scripts/lib/logger.mjs +0 -122
- package/scripts/lib/stealth-fetch.mjs +0 -176
package/hub/router.mjs
DELETED
|
@@ -1,2026 +0,0 @@
|
|
|
1
|
-
// hub/router.mjs — 실시간 라우팅/수신함 상태 관리자
|
|
2
|
-
// SQLite는 감사 로그만 담당하고, 실제 배달 상태는 메모리에서 관리한다.
|
|
3
|
-
import { EventEmitter, once } from "node:events";
|
|
4
|
-
import {
|
|
5
|
-
clampDurationMs,
|
|
6
|
-
DEFAULT_ASSIGN_TIMEOUT_MS,
|
|
7
|
-
DEFAULT_ASSIGN_TTL_MS,
|
|
8
|
-
DEFAULT_HANDOFF_TTL_MS,
|
|
9
|
-
resolveHardCeilingMs,
|
|
10
|
-
} from "./lib/timeout-defaults.mjs";
|
|
11
|
-
import { uuidv7 } from "./lib/uuidv7.mjs";
|
|
12
|
-
import { normalizeRoleKey } from "./role-contract.mjs";
|
|
13
|
-
|
|
14
|
-
const ASSIGN_PENDING_STATUSES = new Set(["queued", "running"]);
|
|
15
|
-
const ROLE_TOPICS = new Set(["cto"]);
|
|
16
|
-
const ROLE_SOURCE_RANK = Object.freeze({
|
|
17
|
-
explicit: 2,
|
|
18
|
-
"topic-fallback": 1,
|
|
19
|
-
});
|
|
20
|
-
const TRANSPORT_LOCATOR_SCHEMA_VERSION = "tfx.transport-locator.v1";
|
|
21
|
-
const TRANSPORT_KINDS = new Set(["claude-uds", "tmux"]);
|
|
22
|
-
const TRANSPORT_CAPABILITIES = new Set([
|
|
23
|
-
"probe",
|
|
24
|
-
"wake",
|
|
25
|
-
"activate",
|
|
26
|
-
"interrupt",
|
|
27
|
-
]);
|
|
28
|
-
const HOST_ID_RE = /^hst_[A-Za-z0-9_-]+$/u;
|
|
29
|
-
|
|
30
|
-
function registrationError(code, message = code) {
|
|
31
|
-
const error = new Error(message);
|
|
32
|
-
error.code = code;
|
|
33
|
-
return error;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
function registrationString(value, code, { maxLength = 256 } = {}) {
|
|
37
|
-
if (typeof value !== "string" || !value.trim() || value.length > maxLength) {
|
|
38
|
-
throw registrationError(code);
|
|
39
|
-
}
|
|
40
|
-
return value.trim();
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
function registrationLogicalRef(value, code, { maxLength = 128 } = {}) {
|
|
44
|
-
const normalized = registrationString(value, code, { maxLength });
|
|
45
|
-
if (!/^[A-Za-z0-9._:-]+$/u.test(normalized)) {
|
|
46
|
-
throw registrationError(code);
|
|
47
|
-
}
|
|
48
|
-
return normalized;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
function assertExactFields(value, allowed, code) {
|
|
52
|
-
if (
|
|
53
|
-
value === null ||
|
|
54
|
-
typeof value !== "object" ||
|
|
55
|
-
Array.isArray(value) ||
|
|
56
|
-
Object.keys(value).some((field) => !allowed.has(field))
|
|
57
|
-
) {
|
|
58
|
-
throw registrationError(code);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function normalizeTransportLocator(
|
|
63
|
-
value,
|
|
64
|
-
{ cli, hostId, now, leaseExpiresMs },
|
|
65
|
-
) {
|
|
66
|
-
assertExactFields(
|
|
67
|
-
value,
|
|
68
|
-
new Set([
|
|
69
|
-
"schema_version",
|
|
70
|
-
"transport_id",
|
|
71
|
-
"kind",
|
|
72
|
-
"host_id",
|
|
73
|
-
"capabilities",
|
|
74
|
-
"priority",
|
|
75
|
-
"expires_at_ms",
|
|
76
|
-
"locator",
|
|
77
|
-
]),
|
|
78
|
-
"TRANSPORT_LOCATOR_INVALID",
|
|
79
|
-
);
|
|
80
|
-
if (value.schema_version !== TRANSPORT_LOCATOR_SCHEMA_VERSION) {
|
|
81
|
-
throw registrationError("TRANSPORT_LOCATOR_SCHEMA_UNSUPPORTED");
|
|
82
|
-
}
|
|
83
|
-
const transportId = registrationLogicalRef(
|
|
84
|
-
value.transport_id,
|
|
85
|
-
"TRANSPORT_ID_INVALID",
|
|
86
|
-
{ maxLength: 128 },
|
|
87
|
-
);
|
|
88
|
-
if (!TRANSPORT_KINDS.has(value.kind)) {
|
|
89
|
-
throw registrationError("TRANSPORT_KIND_UNSUPPORTED");
|
|
90
|
-
}
|
|
91
|
-
if (cli === "codex" && value.kind !== "tmux") {
|
|
92
|
-
throw registrationError("CODEX_TRANSPORT_UNSUPPORTED");
|
|
93
|
-
}
|
|
94
|
-
const locatorHostId = registrationString(value.host_id, "HOST_ID_INVALID", {
|
|
95
|
-
maxLength: 128,
|
|
96
|
-
});
|
|
97
|
-
if (!HOST_ID_RE.test(locatorHostId) || (hostId && locatorHostId !== hostId)) {
|
|
98
|
-
throw registrationError("HOST_ID_INVALID");
|
|
99
|
-
}
|
|
100
|
-
if (!Array.isArray(value.capabilities)) {
|
|
101
|
-
throw registrationError("TRANSPORT_CAPABILITIES_INVALID");
|
|
102
|
-
}
|
|
103
|
-
const capabilities = uniqueStrings(value.capabilities);
|
|
104
|
-
if (
|
|
105
|
-
capabilities.length !== value.capabilities.length ||
|
|
106
|
-
capabilities.some((capability) => !TRANSPORT_CAPABILITIES.has(capability))
|
|
107
|
-
) {
|
|
108
|
-
throw registrationError("TRANSPORT_CAPABILITIES_INVALID");
|
|
109
|
-
}
|
|
110
|
-
if (!Number.isSafeInteger(value.priority)) {
|
|
111
|
-
throw registrationError("TRANSPORT_PRIORITY_INVALID");
|
|
112
|
-
}
|
|
113
|
-
if (
|
|
114
|
-
!Number.isSafeInteger(value.expires_at_ms) ||
|
|
115
|
-
value.expires_at_ms <= now ||
|
|
116
|
-
value.expires_at_ms > leaseExpiresMs
|
|
117
|
-
) {
|
|
118
|
-
throw registrationError("TRANSPORT_EXPIRY_INVALID");
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
let locator;
|
|
122
|
-
if (value.kind === "claude-uds") {
|
|
123
|
-
assertExactFields(
|
|
124
|
-
value.locator,
|
|
125
|
-
new Set(["session_id", "short", "bridge_id"]),
|
|
126
|
-
"TRANSPORT_LOCATOR_INVALID",
|
|
127
|
-
);
|
|
128
|
-
const sessionId = value.locator.session_id;
|
|
129
|
-
const short = value.locator.short;
|
|
130
|
-
if (!sessionId && !short) {
|
|
131
|
-
throw registrationError("CLAUDE_UDS_SESSION_REQUIRED");
|
|
132
|
-
}
|
|
133
|
-
locator = {
|
|
134
|
-
...(sessionId
|
|
135
|
-
? {
|
|
136
|
-
session_id: registrationLogicalRef(
|
|
137
|
-
sessionId,
|
|
138
|
-
"CLAUDE_UDS_SESSION_INVALID",
|
|
139
|
-
),
|
|
140
|
-
}
|
|
141
|
-
: {}),
|
|
142
|
-
...(short
|
|
143
|
-
? {
|
|
144
|
-
short: registrationLogicalRef(short, "CLAUDE_UDS_SHORT_INVALID", {
|
|
145
|
-
maxLength: 128,
|
|
146
|
-
}),
|
|
147
|
-
}
|
|
148
|
-
: {}),
|
|
149
|
-
bridge_id: registrationLogicalRef(
|
|
150
|
-
value.locator.bridge_id,
|
|
151
|
-
"CLAUDE_UDS_BRIDGE_INVALID",
|
|
152
|
-
{ maxLength: 128 },
|
|
153
|
-
),
|
|
154
|
-
};
|
|
155
|
-
} else {
|
|
156
|
-
assertExactFields(
|
|
157
|
-
value.locator,
|
|
158
|
-
new Set(["session_name", "mux_server_id"]),
|
|
159
|
-
"TRANSPORT_LOCATOR_INVALID",
|
|
160
|
-
);
|
|
161
|
-
locator = {
|
|
162
|
-
session_name: registrationLogicalRef(
|
|
163
|
-
value.locator.session_name,
|
|
164
|
-
"TMUX_SESSION_INVALID",
|
|
165
|
-
{ maxLength: 128 },
|
|
166
|
-
),
|
|
167
|
-
mux_server_id: registrationLogicalRef(
|
|
168
|
-
value.locator.mux_server_id,
|
|
169
|
-
"TMUX_SERVER_INVALID",
|
|
170
|
-
{ maxLength: 128 },
|
|
171
|
-
),
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
return {
|
|
176
|
-
schema_version: TRANSPORT_LOCATOR_SCHEMA_VERSION,
|
|
177
|
-
transport_id: transportId,
|
|
178
|
-
kind: value.kind,
|
|
179
|
-
host_id: locatorHostId,
|
|
180
|
-
capabilities,
|
|
181
|
-
priority: value.priority,
|
|
182
|
-
expires_at_ms: value.expires_at_ms,
|
|
183
|
-
locator,
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
export function normalizeRegistrationIdentity(args = {}, now = Date.now()) {
|
|
188
|
-
const hasProjectId = Object.hasOwn(args, "project_id");
|
|
189
|
-
const hasSessionId = Object.hasOwn(args, "session_id");
|
|
190
|
-
const hasHostId = Object.hasOwn(args, "host_id");
|
|
191
|
-
const hasLocatorJson = Object.hasOwn(args, "transport_locators_json");
|
|
192
|
-
const hasLocators = Object.hasOwn(args, "transport_locators");
|
|
193
|
-
if (
|
|
194
|
-
!hasProjectId &&
|
|
195
|
-
!hasSessionId &&
|
|
196
|
-
!hasHostId &&
|
|
197
|
-
!hasLocatorJson &&
|
|
198
|
-
!hasLocators
|
|
199
|
-
) {
|
|
200
|
-
return null;
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
const projectId = hasProjectId
|
|
204
|
-
? normalizeRoleKey({ project_id: args.project_id, role_kind: "cto" })
|
|
205
|
-
.project_id
|
|
206
|
-
: null;
|
|
207
|
-
const sessionId = hasSessionId
|
|
208
|
-
? registrationString(args.session_id, "SESSION_ID_INVALID")
|
|
209
|
-
: null;
|
|
210
|
-
const hostId = hasHostId
|
|
211
|
-
? registrationString(args.host_id, "HOST_ID_INVALID", { maxLength: 128 })
|
|
212
|
-
: null;
|
|
213
|
-
if (hostId && !HOST_ID_RE.test(hostId)) {
|
|
214
|
-
throw registrationError("HOST_ID_INVALID");
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
let rawLocators = [];
|
|
218
|
-
if (hasLocatorJson) {
|
|
219
|
-
if (Array.isArray(args.transport_locators_json)) {
|
|
220
|
-
rawLocators = args.transport_locators_json;
|
|
221
|
-
} else if (typeof args.transport_locators_json === "string") {
|
|
222
|
-
try {
|
|
223
|
-
rawLocators = JSON.parse(args.transport_locators_json);
|
|
224
|
-
} catch {
|
|
225
|
-
throw registrationError("TRANSPORT_LOCATORS_JSON_INVALID");
|
|
226
|
-
}
|
|
227
|
-
} else {
|
|
228
|
-
throw registrationError("TRANSPORT_LOCATORS_JSON_INVALID");
|
|
229
|
-
}
|
|
230
|
-
} else if (hasLocators) {
|
|
231
|
-
rawLocators = args.transport_locators;
|
|
232
|
-
}
|
|
233
|
-
if (!Array.isArray(rawLocators)) {
|
|
234
|
-
throw registrationError("TRANSPORT_LOCATORS_INVALID");
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
const ttlMs = Number(args.heartbeat_ttl_ms ?? 30000);
|
|
238
|
-
if (rawLocators.length > 0 && (!Number.isFinite(ttlMs) || ttlMs <= 0)) {
|
|
239
|
-
throw registrationError("HEARTBEAT_TTL_INVALID");
|
|
240
|
-
}
|
|
241
|
-
const leaseExpiresMs = now + ttlMs;
|
|
242
|
-
const normalizedLocators = rawLocators
|
|
243
|
-
.map((locator) =>
|
|
244
|
-
normalizeTransportLocator(locator, {
|
|
245
|
-
cli: args.cli,
|
|
246
|
-
hostId,
|
|
247
|
-
now,
|
|
248
|
-
leaseExpiresMs,
|
|
249
|
-
}),
|
|
250
|
-
)
|
|
251
|
-
.sort(
|
|
252
|
-
(left, right) =>
|
|
253
|
-
right.priority - left.priority ||
|
|
254
|
-
left.transport_id.localeCompare(right.transport_id),
|
|
255
|
-
);
|
|
256
|
-
const resolvedHostId = hostId || normalizedLocators[0]?.host_id || null;
|
|
257
|
-
if (
|
|
258
|
-
resolvedHostId &&
|
|
259
|
-
normalizedLocators.some((locator) => locator.host_id !== resolvedHostId)
|
|
260
|
-
) {
|
|
261
|
-
throw registrationError("HOST_ID_INVALID");
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
return {
|
|
265
|
-
project_id: projectId,
|
|
266
|
-
session_id: sessionId,
|
|
267
|
-
host_id: resolvedHostId,
|
|
268
|
-
transport_locators_json:
|
|
269
|
-
hasLocatorJson || hasLocators ? JSON.stringify(normalizedLocators) : null,
|
|
270
|
-
};
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
function uniqueStrings(values = []) {
|
|
274
|
-
return Array.from(
|
|
275
|
-
new Set(
|
|
276
|
-
(values || []).map((value) => String(value || "").trim()).filter(Boolean),
|
|
277
|
-
),
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
function normalizeRoleName(value) {
|
|
282
|
-
const role = String(value || "")
|
|
283
|
-
.trim()
|
|
284
|
-
.toLowerCase();
|
|
285
|
-
return ROLE_TOPICS.has(role) ? role : "";
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
function normalizeRoleValues(value) {
|
|
289
|
-
if (Array.isArray(value))
|
|
290
|
-
return uniqueStrings(value).map((item) => item.toLowerCase());
|
|
291
|
-
if (typeof value === "string") {
|
|
292
|
-
return uniqueStrings(value.split(/[,\s]+/u)).map((item) =>
|
|
293
|
-
item.toLowerCase(),
|
|
294
|
-
);
|
|
295
|
-
}
|
|
296
|
-
return [];
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
function rolePriority(metadata = {}, roleName = "") {
|
|
300
|
-
const candidates = [
|
|
301
|
-
metadata?.[`${roleName}_priority`],
|
|
302
|
-
metadata?.role_priority,
|
|
303
|
-
metadata?.priority,
|
|
304
|
-
];
|
|
305
|
-
for (const value of candidates) {
|
|
306
|
-
const numeric = Number(value);
|
|
307
|
-
if (Number.isFinite(numeric)) return numeric;
|
|
308
|
-
}
|
|
309
|
-
return 0;
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
function normalizeAssignTerminalStatus(input, metadata = {}) {
|
|
313
|
-
const status = String(input || "")
|
|
314
|
-
.trim()
|
|
315
|
-
.toLowerCase();
|
|
316
|
-
const resultTag = String(
|
|
317
|
-
metadata?.result ?? metadata?.status ?? metadata?.outcome ?? "",
|
|
318
|
-
)
|
|
319
|
-
.trim()
|
|
320
|
-
.toLowerCase();
|
|
321
|
-
|
|
322
|
-
if (status === "queued") return "queued";
|
|
323
|
-
if (status === "running" || status === "in_progress") return "running";
|
|
324
|
-
if (status === "timed_out" || status === "timeout") return "timed_out";
|
|
325
|
-
if (status === "failed" || status === "error") return "failed";
|
|
326
|
-
if (status === "succeeded" || status === "success") return "succeeded";
|
|
327
|
-
|
|
328
|
-
if (status === "completed") {
|
|
329
|
-
if (resultTag === "failed" || resultTag === "error") return "failed";
|
|
330
|
-
if (resultTag === "timed_out" || resultTag === "timeout")
|
|
331
|
-
return "timed_out";
|
|
332
|
-
return "succeeded";
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
if (resultTag === "failed" || resultTag === "error") return "failed";
|
|
336
|
-
if (resultTag === "timed_out" || resultTag === "timeout") return "timed_out";
|
|
337
|
-
if (resultTag === "succeeded" || resultTag === "success") return "succeeded";
|
|
338
|
-
return "succeeded";
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
function normalizeAgentTopics(store, agentId, runtimeTopics) {
|
|
342
|
-
const topics = new Set(runtimeTopics || []);
|
|
343
|
-
const persisted = store.getAgent(agentId)?.topics || [];
|
|
344
|
-
for (const topic of persisted) topics.add(topic);
|
|
345
|
-
return Array.from(topics);
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
function waitForEmitterOnce(emitter, eventName, timeoutMs) {
|
|
349
|
-
let timer = null;
|
|
350
|
-
const timeout = Math.max(0, Math.min(Number(timeoutMs) || 0, 30000));
|
|
351
|
-
return Promise.race([
|
|
352
|
-
once(emitter, eventName),
|
|
353
|
-
new Promise((_, reject) => {
|
|
354
|
-
timer = setTimeout(() => {
|
|
355
|
-
reject(new Error(`timed out waiting for ${eventName}`));
|
|
356
|
-
}, timeout);
|
|
357
|
-
}),
|
|
358
|
-
]).finally(() => {
|
|
359
|
-
if (timer) clearTimeout(timer);
|
|
360
|
-
});
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
/**
|
|
364
|
-
* 라우터 생성
|
|
365
|
-
* @param {object} store
|
|
366
|
-
*/
|
|
367
|
-
export function createRouter(store) {
|
|
368
|
-
let sweepTimer = null;
|
|
369
|
-
let staleTimer = null;
|
|
370
|
-
const responseEmitter = new EventEmitter();
|
|
371
|
-
const deliveryEmitter = new EventEmitter();
|
|
372
|
-
responseEmitter.setMaxListeners(200);
|
|
373
|
-
deliveryEmitter.setMaxListeners(200);
|
|
374
|
-
|
|
375
|
-
const runtimeTopics = new Map();
|
|
376
|
-
const queuesByAgent = new Map();
|
|
377
|
-
const liveMessages = new Map();
|
|
378
|
-
const roleStates = new Map();
|
|
379
|
-
const hubInstanceId = uuidv7();
|
|
380
|
-
const storeFingerprint =
|
|
381
|
-
store.fingerprint || `${store.type || "unknown"}:unknown`;
|
|
382
|
-
const updateRegistrationIdentity = store.db?.prepare
|
|
383
|
-
? store.db.prepare(`
|
|
384
|
-
UPDATE agents SET
|
|
385
|
-
project_id=COALESCE(@project_id, project_id),
|
|
386
|
-
session_id=COALESCE(@session_id, session_id),
|
|
387
|
-
host_id=COALESCE(@host_id, host_id),
|
|
388
|
-
transport_locators_json=COALESCE(
|
|
389
|
-
@transport_locators_json,
|
|
390
|
-
transport_locators_json
|
|
391
|
-
)
|
|
392
|
-
WHERE agent_id=@agent_id`)
|
|
393
|
-
: null;
|
|
394
|
-
const MAX_LATENCY_SAMPLES = 100;
|
|
395
|
-
let latencyIdx = 0;
|
|
396
|
-
const deliveryLatencies = new Array(MAX_LATENCY_SAMPLES).fill(0);
|
|
397
|
-
|
|
398
|
-
function hubIdentity() {
|
|
399
|
-
return {
|
|
400
|
-
hub_instance_id: hubInstanceId,
|
|
401
|
-
store_fingerprint: storeFingerprint,
|
|
402
|
-
};
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
function validateCallerIdentity(input = {}) {
|
|
406
|
-
if (
|
|
407
|
-
input.hub_instance_id != null &&
|
|
408
|
-
input.hub_instance_id !== hubInstanceId
|
|
409
|
-
) {
|
|
410
|
-
return {
|
|
411
|
-
ok: false,
|
|
412
|
-
error: {
|
|
413
|
-
code: "STALE_HUB_INSTANCE",
|
|
414
|
-
message: "caller is bound to a previous hub instance",
|
|
415
|
-
},
|
|
416
|
-
data: hubIdentity(),
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
if (
|
|
420
|
-
input.store_fingerprint != null &&
|
|
421
|
-
input.store_fingerprint !== storeFingerprint
|
|
422
|
-
) {
|
|
423
|
-
return {
|
|
424
|
-
ok: false,
|
|
425
|
-
error: {
|
|
426
|
-
code: "STORE_FINGERPRINT_MISMATCH",
|
|
427
|
-
message: "caller store fingerprint does not match this hub",
|
|
428
|
-
},
|
|
429
|
-
data: hubIdentity(),
|
|
430
|
-
};
|
|
431
|
-
}
|
|
432
|
-
return null;
|
|
433
|
-
}
|
|
434
|
-
|
|
435
|
-
function isControlPlaneRegistration(args = {}) {
|
|
436
|
-
const metadata = args.metadata || {};
|
|
437
|
-
const values = [
|
|
438
|
-
...(Array.isArray(args.capabilities) ? args.capabilities : []),
|
|
439
|
-
...(Array.isArray(args.topics) ? args.topics : []),
|
|
440
|
-
metadata.role,
|
|
441
|
-
...(Array.isArray(metadata.roles) ? metadata.roles : []),
|
|
442
|
-
]
|
|
443
|
-
.flat()
|
|
444
|
-
.map((value) => String(value || "").toLowerCase());
|
|
445
|
-
return (
|
|
446
|
-
metadata.control_plane === true ||
|
|
447
|
-
values.some((value) => value === "cto" || value === "role:cto")
|
|
448
|
-
);
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
function staleOwnerFailure(agentId, result) {
|
|
452
|
-
return {
|
|
453
|
-
ok: false,
|
|
454
|
-
error: {
|
|
455
|
-
code: "STALE_PRESENCE_OWNER",
|
|
456
|
-
message: `stale presence owner: ${agentId}`,
|
|
457
|
-
},
|
|
458
|
-
data: { ...result, ...hubIdentity() },
|
|
459
|
-
};
|
|
460
|
-
}
|
|
461
|
-
|
|
462
|
-
function ensureAgentQueue(agentId) {
|
|
463
|
-
let queue = queuesByAgent.get(agentId);
|
|
464
|
-
if (!queue) {
|
|
465
|
-
queue = new Map();
|
|
466
|
-
queuesByAgent.set(agentId, queue);
|
|
467
|
-
}
|
|
468
|
-
return queue;
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
function recordLatency(ms) {
|
|
472
|
-
deliveryLatencies[latencyIdx % MAX_LATENCY_SAMPLES] = ms;
|
|
473
|
-
latencyIdx++;
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
function upsertRuntimeTopics(agentId, topics, { replace = true } = {}) {
|
|
477
|
-
const normalized = uniqueStrings(topics);
|
|
478
|
-
const current = replace
|
|
479
|
-
? new Set()
|
|
480
|
-
: new Set(runtimeTopics.get(agentId) || []);
|
|
481
|
-
for (const topic of normalized) current.add(topic);
|
|
482
|
-
runtimeTopics.set(agentId, current);
|
|
483
|
-
store.updateAgentTopics(agentId, Array.from(current));
|
|
484
|
-
return Array.from(current);
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
function listRuntimeTopics(agentId) {
|
|
488
|
-
return normalizeAgentTopics(store, agentId, runtimeTopics.get(agentId));
|
|
489
|
-
}
|
|
490
|
-
|
|
491
|
-
function ensureRoleState(roleName) {
|
|
492
|
-
const role = normalizeRoleName(roleName);
|
|
493
|
-
if (!role) return null;
|
|
494
|
-
if (!roleStates.has(role)) {
|
|
495
|
-
roleStates.set(role, {
|
|
496
|
-
role,
|
|
497
|
-
leaderAgentId: null,
|
|
498
|
-
previousLeaderAgentId: null,
|
|
499
|
-
leaderEpoch: 0,
|
|
500
|
-
leaderSource: null,
|
|
501
|
-
status: "offline",
|
|
502
|
-
candidates: new Map(),
|
|
503
|
-
lastTransitionMs: null,
|
|
504
|
-
lastReason: "init",
|
|
505
|
-
transferredCount: 0,
|
|
506
|
-
});
|
|
507
|
-
}
|
|
508
|
-
return roleStates.get(role);
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
function hasRoleCapability(capabilities = [], roleName) {
|
|
512
|
-
const normalized = normalizeRoleValues(capabilities);
|
|
513
|
-
return normalized.some(
|
|
514
|
-
(capability) =>
|
|
515
|
-
capability === roleName ||
|
|
516
|
-
capability === `role:${roleName}` ||
|
|
517
|
-
capability === `${roleName}:leader` ||
|
|
518
|
-
capability === `${roleName}:candidate`,
|
|
519
|
-
);
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
function getRoleCandidateSource(agent, topics, roleName) {
|
|
523
|
-
if (!agent) return null;
|
|
524
|
-
const metadata = agent.metadata || {};
|
|
525
|
-
const roles = [
|
|
526
|
-
...normalizeRoleValues(metadata.role),
|
|
527
|
-
...normalizeRoleValues(metadata.roles),
|
|
528
|
-
];
|
|
529
|
-
const explicit =
|
|
530
|
-
roles.includes(roleName) ||
|
|
531
|
-
metadata?.[roleName] === true ||
|
|
532
|
-
metadata?.is_cto === true ||
|
|
533
|
-
hasRoleCapability(agent.capabilities, roleName);
|
|
534
|
-
if (explicit) return "explicit";
|
|
535
|
-
return topics.includes(roleName) ? "topic-fallback" : null;
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
function buildRoleCandidate(agentId, roleName) {
|
|
539
|
-
const role = normalizeRoleName(roleName);
|
|
540
|
-
if (!agentId || !role) return null;
|
|
541
|
-
const agent = store.getAgent(agentId);
|
|
542
|
-
if (!agent) return null;
|
|
543
|
-
const topics = listRuntimeTopics(agentId);
|
|
544
|
-
const source = getRoleCandidateSource(agent, topics, role);
|
|
545
|
-
if (!source) return null;
|
|
546
|
-
return {
|
|
547
|
-
agent_id: agent.agent_id,
|
|
548
|
-
status: agent.status,
|
|
549
|
-
source,
|
|
550
|
-
priority: rolePriority(agent.metadata, role),
|
|
551
|
-
last_seen_ms: agent.last_seen_ms || 0,
|
|
552
|
-
lease_expires_ms: agent.lease_expires_ms || 0,
|
|
553
|
-
topics,
|
|
554
|
-
capabilities: agent.capabilities || [],
|
|
555
|
-
};
|
|
556
|
-
}
|
|
557
|
-
|
|
558
|
-
function refreshRoleCandidateForAgent(agentId) {
|
|
559
|
-
if (!agentId) return;
|
|
560
|
-
for (const roleName of ROLE_TOPICS) {
|
|
561
|
-
const role = ensureRoleState(roleName);
|
|
562
|
-
const candidate = buildRoleCandidate(agentId, roleName);
|
|
563
|
-
if (candidate) role.candidates.set(agentId, candidate);
|
|
564
|
-
else role.candidates.delete(agentId);
|
|
565
|
-
}
|
|
566
|
-
}
|
|
567
|
-
|
|
568
|
-
function refreshRoleCandidates(roleName) {
|
|
569
|
-
const role = ensureRoleState(roleName);
|
|
570
|
-
if (!role) return [];
|
|
571
|
-
const ids = new Set(role.candidates.keys());
|
|
572
|
-
for (const [agentId, topics] of runtimeTopics) {
|
|
573
|
-
if (topics.has(role.role)) ids.add(agentId);
|
|
574
|
-
}
|
|
575
|
-
for (const agent of store.getAgentsByTopic(role.role)) {
|
|
576
|
-
ids.add(agent.agent_id);
|
|
577
|
-
}
|
|
578
|
-
if (typeof store.listAllAgents === "function") {
|
|
579
|
-
for (const agent of store.listAllAgents()) ids.add(agent.agent_id);
|
|
580
|
-
}
|
|
581
|
-
for (const agentId of ids) refreshRoleCandidateForAgent(agentId);
|
|
582
|
-
return Array.from(role.candidates.values());
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
function isCandidateLive(candidate, now = Date.now()) {
|
|
586
|
-
return (
|
|
587
|
-
candidate?.status === "online" && Number(candidate.lease_expires_ms) > now
|
|
588
|
-
);
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
function registerPresenceIsEffective(result) {
|
|
592
|
-
const effective = result?.effective;
|
|
593
|
-
return (
|
|
594
|
-
effective?.online === true &&
|
|
595
|
-
effective.status === "online" &&
|
|
596
|
-
Number(effective.lease_expires_ms) === Number(result?.lease_expires_ms)
|
|
597
|
-
);
|
|
598
|
-
}
|
|
599
|
-
|
|
600
|
-
function registerPresenceFailure(agentId, result) {
|
|
601
|
-
return {
|
|
602
|
-
ok: false,
|
|
603
|
-
error: {
|
|
604
|
-
code: "REGISTER_PRESENCE_NOT_UPDATED",
|
|
605
|
-
message: `${agentId} register did not update effective presence`,
|
|
606
|
-
},
|
|
607
|
-
data: { ...result, hub_pid: process.pid },
|
|
608
|
-
};
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
function sortRoleCandidates(candidates = []) {
|
|
612
|
-
return [...candidates].sort((left, right) => {
|
|
613
|
-
const sourceDelta =
|
|
614
|
-
(ROLE_SOURCE_RANK[right.source] || 0) -
|
|
615
|
-
(ROLE_SOURCE_RANK[left.source] || 0);
|
|
616
|
-
if (sourceDelta !== 0) return sourceDelta;
|
|
617
|
-
const priorityDelta =
|
|
618
|
-
Number(right.priority || 0) - Number(left.priority || 0);
|
|
619
|
-
if (priorityDelta !== 0) return priorityDelta;
|
|
620
|
-
const seenDelta =
|
|
621
|
-
Number(right.last_seen_ms || 0) - Number(left.last_seen_ms || 0);
|
|
622
|
-
if (seenDelta !== 0) return seenDelta;
|
|
623
|
-
return String(left.agent_id).localeCompare(String(right.agent_id));
|
|
624
|
-
});
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
function chooseRoleLeader(roleName) {
|
|
628
|
-
const now = Date.now();
|
|
629
|
-
return (
|
|
630
|
-
sortRoleCandidates(refreshRoleCandidates(roleName)).find((candidate) =>
|
|
631
|
-
isCandidateLive(candidate, now),
|
|
632
|
-
) || null
|
|
633
|
-
);
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
function messageMatchesRole(record, roleName) {
|
|
637
|
-
const message = record?.message || {};
|
|
638
|
-
const to = message.to_agent ?? message.to;
|
|
639
|
-
return to === `topic:${roleName}`;
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
function roleTopicForMessage(message = {}) {
|
|
643
|
-
const to = message?.to_agent ?? message?.to;
|
|
644
|
-
if (!String(to || "").startsWith("topic:")) return "";
|
|
645
|
-
return normalizeRoleName(String(to).slice(6));
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
function isReplayAllowedForAgent(agentId, message = {}) {
|
|
649
|
-
const roleName = roleTopicForMessage(message);
|
|
650
|
-
if (!roleName) return true;
|
|
651
|
-
const targetAgentId = String(agentId || "").trim();
|
|
652
|
-
if (!targetAgentId) return false;
|
|
653
|
-
const role = getRoleSnapshot(roleName);
|
|
654
|
-
return role?.leader_agent_id === targetAgentId;
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
function shouldTrackWithoutRecipients(message) {
|
|
658
|
-
const to = message?.to_agent ?? message?.to;
|
|
659
|
-
if (!String(to || "").startsWith("topic:")) return false;
|
|
660
|
-
return ROLE_TOPICS.has(String(to).slice(6));
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
function isRoleMessageFullyHandled(record, roleName) {
|
|
664
|
-
if (!messageMatchesRole(record, roleName)) return true;
|
|
665
|
-
return (
|
|
666
|
-
record.recipients.size > 0 &&
|
|
667
|
-
record.ackedBy.size >= record.recipients.size
|
|
668
|
-
);
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
function isRoleRecipient(agentId, roleName, previousLeaderAgentId = null) {
|
|
672
|
-
if (!agentId) return false;
|
|
673
|
-
if (agentId === previousLeaderAgentId) return true;
|
|
674
|
-
return Boolean(buildRoleCandidate(agentId, roleName));
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
function countPendingForRole(roleName) {
|
|
678
|
-
const now = Date.now();
|
|
679
|
-
let count = 0;
|
|
680
|
-
for (const record of liveMessages.values()) {
|
|
681
|
-
if (!messageMatchesRole(record, roleName)) continue;
|
|
682
|
-
if (record.message.expires_at_ms <= now) continue;
|
|
683
|
-
if (isRoleMessageFullyHandled(record, roleName)) continue;
|
|
684
|
-
count += 1;
|
|
685
|
-
}
|
|
686
|
-
return count;
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
function transferRoleBacklog(
|
|
690
|
-
roleName,
|
|
691
|
-
newLeaderAgentId,
|
|
692
|
-
previousLeaderAgentId,
|
|
693
|
-
) {
|
|
694
|
-
const role = ensureRoleState(roleName);
|
|
695
|
-
const now = Date.now();
|
|
696
|
-
const stats = { transferred_count: 0, skipped_count: 0, removed_count: 0 };
|
|
697
|
-
if (!role || !newLeaderAgentId) return stats;
|
|
698
|
-
|
|
699
|
-
for (const record of liveMessages.values()) {
|
|
700
|
-
const { message, recipients, ackedBy } = record;
|
|
701
|
-
if (!messageMatchesRole(record, role.role)) continue;
|
|
702
|
-
if (message.expires_at_ms <= now) {
|
|
703
|
-
stats.skipped_count += 1;
|
|
704
|
-
continue;
|
|
705
|
-
}
|
|
706
|
-
if (isRoleMessageFullyHandled(record, role.role)) {
|
|
707
|
-
stats.skipped_count += 1;
|
|
708
|
-
continue;
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
for (const recipient of Array.from(recipients)) {
|
|
712
|
-
if (
|
|
713
|
-
recipient !== newLeaderAgentId &&
|
|
714
|
-
isRoleRecipient(recipient, role.role, previousLeaderAgentId)
|
|
715
|
-
) {
|
|
716
|
-
queuesByAgent.get(recipient)?.delete(message.id);
|
|
717
|
-
recipients.delete(recipient);
|
|
718
|
-
ackedBy.delete(recipient);
|
|
719
|
-
stats.removed_count += 1;
|
|
720
|
-
}
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
recipients.add(newLeaderAgentId);
|
|
724
|
-
const alreadyQueued = queuesByAgent
|
|
725
|
-
.get(newLeaderAgentId)
|
|
726
|
-
?.has(message.id);
|
|
727
|
-
if (!alreadyQueued && !ackedBy.has(newLeaderAgentId)) {
|
|
728
|
-
queueMessage(newLeaderAgentId, message);
|
|
729
|
-
stats.transferred_count += 1;
|
|
730
|
-
} else {
|
|
731
|
-
stats.skipped_count += 1;
|
|
732
|
-
}
|
|
733
|
-
message.status = "delivered";
|
|
734
|
-
store.updateMessageStatus(message.id, "delivered");
|
|
735
|
-
}
|
|
736
|
-
|
|
737
|
-
role.transferredCount += stats.transferred_count;
|
|
738
|
-
return stats;
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
function buildRoleSnapshot(roleName, { refreshCandidates = true } = {}) {
|
|
742
|
-
const role = ensureRoleState(roleName);
|
|
743
|
-
if (!role) return null;
|
|
744
|
-
const now = Date.now();
|
|
745
|
-
const candidates = sortRoleCandidates(
|
|
746
|
-
refreshCandidates
|
|
747
|
-
? refreshRoleCandidates(role.role)
|
|
748
|
-
: Array.from(role.candidates.values()),
|
|
749
|
-
);
|
|
750
|
-
const leader =
|
|
751
|
-
role.leaderAgentId && buildRoleCandidate(role.leaderAgentId, role.role);
|
|
752
|
-
const leaderLive = isCandidateLive(leader, now);
|
|
753
|
-
const liveCandidates = candidates.filter((candidate) =>
|
|
754
|
-
isCandidateLive(candidate, now),
|
|
755
|
-
);
|
|
756
|
-
return {
|
|
757
|
-
role: role.role,
|
|
758
|
-
status: leaderLive
|
|
759
|
-
? "active"
|
|
760
|
-
: liveCandidates.length
|
|
761
|
-
? "electable"
|
|
762
|
-
: "offline",
|
|
763
|
-
leader_agent_id: leaderLive ? role.leaderAgentId : null,
|
|
764
|
-
previous_leader_agent_id: role.previousLeaderAgentId || null,
|
|
765
|
-
leader_epoch: role.leaderEpoch,
|
|
766
|
-
lease_expires_ms: leaderLive ? leader.lease_expires_ms : null,
|
|
767
|
-
candidate_source: leaderLive
|
|
768
|
-
? leader.source
|
|
769
|
-
: liveCandidates[0]?.source || null,
|
|
770
|
-
candidate_count: candidates.length,
|
|
771
|
-
live_candidate_count: liveCandidates.length,
|
|
772
|
-
pending_count: countPendingForRole(role.role),
|
|
773
|
-
transferred_count: role.transferredCount,
|
|
774
|
-
last_transition_ms: role.lastTransitionMs,
|
|
775
|
-
last_reason: role.lastReason,
|
|
776
|
-
candidates: candidates.slice(0, 16).map((candidate) => ({
|
|
777
|
-
agent_id: candidate.agent_id,
|
|
778
|
-
status: isCandidateLive(candidate, now) ? "online" : candidate.status,
|
|
779
|
-
source: candidate.source,
|
|
780
|
-
priority: candidate.priority,
|
|
781
|
-
last_seen_ms: candidate.last_seen_ms,
|
|
782
|
-
lease_expires_ms: candidate.lease_expires_ms,
|
|
783
|
-
})),
|
|
784
|
-
};
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
function ensureRoleLeader(
|
|
788
|
-
roleName,
|
|
789
|
-
{ reason = "ensure", transferBacklog = true } = {},
|
|
790
|
-
) {
|
|
791
|
-
const role = ensureRoleState(roleName);
|
|
792
|
-
if (!role) return null;
|
|
793
|
-
const now = Date.now();
|
|
794
|
-
const current =
|
|
795
|
-
role.leaderAgentId && buildRoleCandidate(role.leaderAgentId, role.role);
|
|
796
|
-
const best = chooseRoleLeader(role.role);
|
|
797
|
-
const currentLive = isCandidateLive(current, now);
|
|
798
|
-
const shouldPreemptFallback =
|
|
799
|
-
currentLive &&
|
|
800
|
-
current?.source === "topic-fallback" &&
|
|
801
|
-
best?.source === "explicit" &&
|
|
802
|
-
best.agent_id !== current.agent_id;
|
|
803
|
-
|
|
804
|
-
if (currentLive && !shouldPreemptFallback) {
|
|
805
|
-
role.status = "active";
|
|
806
|
-
role.leaderSource = current.source;
|
|
807
|
-
return buildRoleSnapshot(role.role);
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
if (!best) {
|
|
811
|
-
if (role.leaderAgentId) {
|
|
812
|
-
role.previousLeaderAgentId = role.leaderAgentId;
|
|
813
|
-
role.leaderAgentId = null;
|
|
814
|
-
role.leaderSource = null;
|
|
815
|
-
role.leaderEpoch += 1;
|
|
816
|
-
role.lastTransitionMs = Date.now();
|
|
817
|
-
role.lastReason = reason;
|
|
818
|
-
}
|
|
819
|
-
role.status = "offline";
|
|
820
|
-
return buildRoleSnapshot(role.role);
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
if (role.leaderAgentId !== best.agent_id) {
|
|
824
|
-
const previousLeaderAgentId = role.leaderAgentId;
|
|
825
|
-
role.previousLeaderAgentId =
|
|
826
|
-
previousLeaderAgentId || role.previousLeaderAgentId;
|
|
827
|
-
role.leaderAgentId = best.agent_id;
|
|
828
|
-
role.leaderSource = best.source;
|
|
829
|
-
role.leaderEpoch += 1;
|
|
830
|
-
role.status = "active";
|
|
831
|
-
role.lastTransitionMs = Date.now();
|
|
832
|
-
role.lastReason = reason;
|
|
833
|
-
if (transferBacklog) {
|
|
834
|
-
transferRoleBacklog(role.role, best.agent_id, previousLeaderAgentId);
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
return buildRoleSnapshot(role.role);
|
|
839
|
-
}
|
|
840
|
-
|
|
841
|
-
function getRoleSnapshot(roleName) {
|
|
842
|
-
return buildRoleSnapshot(roleName, { refreshCandidates: true });
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
function trackMessage(message, recipients) {
|
|
846
|
-
liveMessages.set(message.id, {
|
|
847
|
-
message,
|
|
848
|
-
recipients: new Set(recipients),
|
|
849
|
-
ackedBy: new Set(),
|
|
850
|
-
});
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
function getMessageRecord(messageId) {
|
|
854
|
-
return liveMessages.get(messageId) || null;
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
function removeMessage(messageId) {
|
|
858
|
-
const record = liveMessages.get(messageId);
|
|
859
|
-
if (!record) return;
|
|
860
|
-
for (const agentId of record.recipients) {
|
|
861
|
-
queuesByAgent.get(agentId)?.delete(messageId);
|
|
862
|
-
}
|
|
863
|
-
liveMessages.delete(messageId);
|
|
864
|
-
}
|
|
865
|
-
|
|
866
|
-
function queueMessage(agentId, message) {
|
|
867
|
-
const queue = ensureAgentQueue(agentId);
|
|
868
|
-
queue.set(message.id, {
|
|
869
|
-
message,
|
|
870
|
-
attempts: 0,
|
|
871
|
-
delivered_at_ms: null,
|
|
872
|
-
acked_at_ms: null,
|
|
873
|
-
});
|
|
874
|
-
deliveryEmitter.emit("message", agentId, message);
|
|
875
|
-
}
|
|
876
|
-
|
|
877
|
-
function resolveRecipients(msg) {
|
|
878
|
-
const to = msg.to_agent ?? msg.to;
|
|
879
|
-
if (!to?.startsWith("topic:")) {
|
|
880
|
-
return [to];
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
const topic = to.slice(6);
|
|
884
|
-
if (ROLE_TOPICS.has(topic)) {
|
|
885
|
-
const role = ensureRoleLeader(topic, {
|
|
886
|
-
reason: "route",
|
|
887
|
-
transferBacklog: true,
|
|
888
|
-
});
|
|
889
|
-
return role?.leader_agent_id ? [role.leader_agent_id] : [];
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
const recipients = new Set();
|
|
893
|
-
for (const [agentId, topics] of runtimeTopics) {
|
|
894
|
-
if (topics.has(topic)) recipients.add(agentId);
|
|
895
|
-
}
|
|
896
|
-
for (const agent of store.getAgentsByTopic(topic)) {
|
|
897
|
-
recipients.add(agent.agent_id);
|
|
898
|
-
}
|
|
899
|
-
return Array.from(recipients);
|
|
900
|
-
}
|
|
901
|
-
|
|
902
|
-
function sortedPending(
|
|
903
|
-
agentId,
|
|
904
|
-
{ max_messages = 20, include_topics = null } = {},
|
|
905
|
-
) {
|
|
906
|
-
const queue = ensureAgentQueue(agentId);
|
|
907
|
-
const topicFilter = include_topics?.length ? new Set(include_topics) : null;
|
|
908
|
-
const now = Date.now();
|
|
909
|
-
const pending = [];
|
|
910
|
-
|
|
911
|
-
for (const delivery of queue.values()) {
|
|
912
|
-
const { message } = delivery;
|
|
913
|
-
if (delivery.acked_at_ms) continue;
|
|
914
|
-
if (message.expires_at_ms <= now) continue;
|
|
915
|
-
if (topicFilter && !topicFilter.has(message.topic)) continue;
|
|
916
|
-
pending.push(message);
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
pending.sort((a, b) => {
|
|
920
|
-
if (b.priority !== a.priority) return b.priority - a.priority;
|
|
921
|
-
return a.created_at_ms - b.created_at_ms;
|
|
922
|
-
});
|
|
923
|
-
return pending.slice(0, max_messages);
|
|
924
|
-
}
|
|
925
|
-
|
|
926
|
-
function markDelivered(agentId, messageId) {
|
|
927
|
-
const delivery = queuesByAgent.get(agentId)?.get(messageId);
|
|
928
|
-
const record = getMessageRecord(messageId);
|
|
929
|
-
if (!delivery || !record) return false;
|
|
930
|
-
|
|
931
|
-
delivery.attempts += 1;
|
|
932
|
-
if (!delivery.delivered_at_ms) {
|
|
933
|
-
delivery.delivered_at_ms = Date.now();
|
|
934
|
-
record.message.status = "delivered";
|
|
935
|
-
store.updateMessageStatus(messageId, "delivered");
|
|
936
|
-
recordLatency(delivery.delivered_at_ms - record.message.created_at_ms);
|
|
937
|
-
rearmAssignDeadlineOnDispatch(record.message, delivery.delivered_at_ms);
|
|
938
|
-
return true;
|
|
939
|
-
}
|
|
940
|
-
return false;
|
|
941
|
-
}
|
|
942
|
-
|
|
943
|
-
function rearmAssignDeadlineOnDispatch(message, deliveredAtMs) {
|
|
944
|
-
if (message?.payload?.kind !== "assign.job") return;
|
|
945
|
-
const jobId = message.payload.assign_job_id;
|
|
946
|
-
if (!jobId) return;
|
|
947
|
-
const job = store.getAssign(jobId);
|
|
948
|
-
if (
|
|
949
|
-
!job ||
|
|
950
|
-
!ASSIGN_PENDING_STATUSES.has(job.status) ||
|
|
951
|
-
job.dispatched_at_ms
|
|
952
|
-
)
|
|
953
|
-
return;
|
|
954
|
-
if (Number(message.payload.attempt) !== job.attempt) return;
|
|
955
|
-
store.updateAssignStatus(job.job_id, job.status, {
|
|
956
|
-
dispatched_at_ms: deliveredAtMs,
|
|
957
|
-
deadline_ms:
|
|
958
|
-
deliveredAtMs + clampDurationMs(job.timeout_ms, job.timeout_ms),
|
|
959
|
-
});
|
|
960
|
-
}
|
|
961
|
-
|
|
962
|
-
function ackMessages(ids, agentId) {
|
|
963
|
-
const now = Date.now();
|
|
964
|
-
let count = 0;
|
|
965
|
-
|
|
966
|
-
for (const id of ids || []) {
|
|
967
|
-
const delivery = queuesByAgent.get(agentId)?.get(id);
|
|
968
|
-
const record = getMessageRecord(id);
|
|
969
|
-
if (!delivery || !record || delivery.acked_at_ms) continue;
|
|
970
|
-
|
|
971
|
-
delivery.acked_at_ms = now;
|
|
972
|
-
record.ackedBy.add(agentId);
|
|
973
|
-
count += 1;
|
|
974
|
-
|
|
975
|
-
if (record.ackedBy.size >= record.recipients.size) {
|
|
976
|
-
record.message.status = "acked";
|
|
977
|
-
store.updateMessageStatus(id, "acked");
|
|
978
|
-
removeMessage(id);
|
|
979
|
-
}
|
|
980
|
-
}
|
|
981
|
-
|
|
982
|
-
return count;
|
|
983
|
-
}
|
|
984
|
-
|
|
985
|
-
function dispatchMessage({
|
|
986
|
-
type,
|
|
987
|
-
from,
|
|
988
|
-
to,
|
|
989
|
-
topic,
|
|
990
|
-
priority = 5,
|
|
991
|
-
ttl_ms = 300000,
|
|
992
|
-
payload = {},
|
|
993
|
-
trace_id,
|
|
994
|
-
correlation_id,
|
|
995
|
-
}) {
|
|
996
|
-
const msg = store.auditLog({
|
|
997
|
-
type,
|
|
998
|
-
from,
|
|
999
|
-
to,
|
|
1000
|
-
topic,
|
|
1001
|
-
priority,
|
|
1002
|
-
ttl_ms,
|
|
1003
|
-
payload,
|
|
1004
|
-
trace_id,
|
|
1005
|
-
correlation_id,
|
|
1006
|
-
});
|
|
1007
|
-
const recipients = uniqueStrings(resolveRecipients(msg));
|
|
1008
|
-
if (recipients.length || shouldTrackWithoutRecipients(msg)) {
|
|
1009
|
-
trackMessage(msg, recipients);
|
|
1010
|
-
}
|
|
1011
|
-
if (recipients.length) {
|
|
1012
|
-
for (const agentId of recipients) {
|
|
1013
|
-
queueMessage(agentId, msg);
|
|
1014
|
-
}
|
|
1015
|
-
msg.status = "delivered";
|
|
1016
|
-
store.updateMessageStatus(msg.id, "delivered");
|
|
1017
|
-
}
|
|
1018
|
-
if (msg.type === "response") {
|
|
1019
|
-
responseEmitter.emit(msg.correlation_id, msg.payload);
|
|
1020
|
-
}
|
|
1021
|
-
return { msg, recipients };
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
function buildAssignSnapshot(job, extra = {}) {
|
|
1025
|
-
if (!job) return null;
|
|
1026
|
-
return {
|
|
1027
|
-
job_id: job.job_id,
|
|
1028
|
-
supervisor_agent: job.supervisor_agent,
|
|
1029
|
-
worker_agent: job.worker_agent,
|
|
1030
|
-
topic: job.topic,
|
|
1031
|
-
task: job.task,
|
|
1032
|
-
status: job.status,
|
|
1033
|
-
attempt: job.attempt,
|
|
1034
|
-
retry_count: job.retry_count,
|
|
1035
|
-
max_retries: job.max_retries,
|
|
1036
|
-
timeout_ms: job.timeout_ms,
|
|
1037
|
-
deadline_ms: job.deadline_ms,
|
|
1038
|
-
trace_id: job.trace_id,
|
|
1039
|
-
correlation_id: job.correlation_id,
|
|
1040
|
-
last_message_id: job.last_message_id,
|
|
1041
|
-
result: job.result,
|
|
1042
|
-
error: job.error,
|
|
1043
|
-
updated_at_ms: job.updated_at_ms,
|
|
1044
|
-
completed_at_ms: job.completed_at_ms,
|
|
1045
|
-
...extra,
|
|
1046
|
-
};
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
function notifyAssignSupervisor(job, event, extra = {}) {
|
|
1050
|
-
if (!job?.supervisor_agent) return null;
|
|
1051
|
-
const { msg } = dispatchMessage({
|
|
1052
|
-
type: "event",
|
|
1053
|
-
from: job.worker_agent || "assign-router",
|
|
1054
|
-
to: job.supervisor_agent,
|
|
1055
|
-
topic: "assign.result",
|
|
1056
|
-
priority: Math.max(5, job.priority || 5),
|
|
1057
|
-
ttl_ms: job.ttl_ms || job.timeout_ms || DEFAULT_ASSIGN_TTL_MS,
|
|
1058
|
-
payload: {
|
|
1059
|
-
event,
|
|
1060
|
-
...buildAssignSnapshot(job),
|
|
1061
|
-
...extra,
|
|
1062
|
-
},
|
|
1063
|
-
trace_id: job.trace_id,
|
|
1064
|
-
correlation_id: job.correlation_id,
|
|
1065
|
-
});
|
|
1066
|
-
return msg;
|
|
1067
|
-
}
|
|
1068
|
-
|
|
1069
|
-
function dispatchAssignJob(job, reason = "dispatch") {
|
|
1070
|
-
const { msg, recipients } = dispatchMessage({
|
|
1071
|
-
type: "handoff",
|
|
1072
|
-
from: job.supervisor_agent,
|
|
1073
|
-
to: job.worker_agent,
|
|
1074
|
-
topic: job.topic || "assign.job",
|
|
1075
|
-
priority: job.priority || 5,
|
|
1076
|
-
ttl_ms: job.ttl_ms || job.timeout_ms || DEFAULT_ASSIGN_TTL_MS,
|
|
1077
|
-
payload: {
|
|
1078
|
-
kind: "assign.job",
|
|
1079
|
-
reason,
|
|
1080
|
-
assign_job_id: job.job_id,
|
|
1081
|
-
attempt: job.attempt,
|
|
1082
|
-
retry_count: job.retry_count,
|
|
1083
|
-
max_retries: job.max_retries,
|
|
1084
|
-
timeout_ms: job.timeout_ms,
|
|
1085
|
-
supervisor_agent: job.supervisor_agent,
|
|
1086
|
-
worker_agent: job.worker_agent,
|
|
1087
|
-
task: job.task,
|
|
1088
|
-
payload: job.payload || {},
|
|
1089
|
-
},
|
|
1090
|
-
trace_id: job.trace_id,
|
|
1091
|
-
correlation_id: job.correlation_id,
|
|
1092
|
-
});
|
|
1093
|
-
|
|
1094
|
-
const updated = store.updateAssignStatus(job.job_id, job.status, {
|
|
1095
|
-
last_message_id: msg.id,
|
|
1096
|
-
});
|
|
1097
|
-
return { job: updated || job, recipients, message_id: msg.id };
|
|
1098
|
-
}
|
|
1099
|
-
|
|
1100
|
-
function scheduleAssignRetry(
|
|
1101
|
-
job,
|
|
1102
|
-
reason,
|
|
1103
|
-
error = null,
|
|
1104
|
-
requested_by = "system",
|
|
1105
|
-
) {
|
|
1106
|
-
if (!job) {
|
|
1107
|
-
return {
|
|
1108
|
-
ok: false,
|
|
1109
|
-
error: { code: "ASSIGN_NOT_FOUND", message: "assign job not found" },
|
|
1110
|
-
};
|
|
1111
|
-
}
|
|
1112
|
-
if (job.retry_count >= job.max_retries) {
|
|
1113
|
-
return {
|
|
1114
|
-
ok: false,
|
|
1115
|
-
error: {
|
|
1116
|
-
code: "ASSIGN_RETRY_EXHAUSTED",
|
|
1117
|
-
message: `retry exhausted for ${job.job_id}`,
|
|
1118
|
-
},
|
|
1119
|
-
};
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
const queued = store.retryAssign(job.job_id, {
|
|
1123
|
-
error,
|
|
1124
|
-
timeout_ms: job.timeout_ms,
|
|
1125
|
-
ttl_ms: job.ttl_ms,
|
|
1126
|
-
});
|
|
1127
|
-
const dispatched = dispatchAssignJob(queued, "retry");
|
|
1128
|
-
notifyAssignSupervisor(dispatched.job, "retry_scheduled", {
|
|
1129
|
-
retry_reason: reason,
|
|
1130
|
-
requested_by,
|
|
1131
|
-
});
|
|
1132
|
-
return {
|
|
1133
|
-
ok: true,
|
|
1134
|
-
data: {
|
|
1135
|
-
retried: true,
|
|
1136
|
-
...buildAssignSnapshot(dispatched.job, {
|
|
1137
|
-
retry_reason: reason,
|
|
1138
|
-
requested_by,
|
|
1139
|
-
}),
|
|
1140
|
-
},
|
|
1141
|
-
};
|
|
1142
|
-
}
|
|
1143
|
-
|
|
1144
|
-
function handleAssignTimeout(job) {
|
|
1145
|
-
const timedOut = store.updateAssignStatus(job.job_id, "timed_out", {
|
|
1146
|
-
error: job.error ?? { message: "assign job timed out" },
|
|
1147
|
-
});
|
|
1148
|
-
if (timedOut?.worker_agent) {
|
|
1149
|
-
dispatchMessage({
|
|
1150
|
-
type: "event",
|
|
1151
|
-
from: "assign-router",
|
|
1152
|
-
to: timedOut.worker_agent,
|
|
1153
|
-
topic: "assign.cancel",
|
|
1154
|
-
priority: 8,
|
|
1155
|
-
ttl_ms: clampDurationMs(timedOut.ttl_ms, DEFAULT_ASSIGN_TTL_MS),
|
|
1156
|
-
payload: {
|
|
1157
|
-
kind: "assign.cancel",
|
|
1158
|
-
assign_job_id: timedOut.job_id,
|
|
1159
|
-
attempt: timedOut.attempt,
|
|
1160
|
-
reason: "timed_out",
|
|
1161
|
-
},
|
|
1162
|
-
trace_id: timedOut.trace_id,
|
|
1163
|
-
correlation_id: timedOut.correlation_id,
|
|
1164
|
-
});
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
if (timedOut.retry_count < timedOut.max_retries) {
|
|
1168
|
-
return scheduleAssignRetry(
|
|
1169
|
-
timedOut,
|
|
1170
|
-
"timed_out",
|
|
1171
|
-
timedOut.error,
|
|
1172
|
-
"sweeper",
|
|
1173
|
-
);
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
notifyAssignSupervisor(timedOut, "completed", {
|
|
1177
|
-
completion_reason: "timed_out",
|
|
1178
|
-
});
|
|
1179
|
-
return {
|
|
1180
|
-
ok: true,
|
|
1181
|
-
data: buildAssignSnapshot(timedOut, { completion_reason: "timed_out" }),
|
|
1182
|
-
};
|
|
1183
|
-
}
|
|
1184
|
-
|
|
1185
|
-
const router = {
|
|
1186
|
-
responseEmitter,
|
|
1187
|
-
deliveryEmitter,
|
|
1188
|
-
|
|
1189
|
-
registerAgent(args) {
|
|
1190
|
-
const identityFailure = validateCallerIdentity(args);
|
|
1191
|
-
if (identityFailure) return identityFailure;
|
|
1192
|
-
if (isControlPlaneRegistration(args) && store.type !== "sqlite") {
|
|
1193
|
-
return {
|
|
1194
|
-
ok: false,
|
|
1195
|
-
error: {
|
|
1196
|
-
code: "CONTROL_PLANE_STORE_DEGRADED",
|
|
1197
|
-
message: "control-plane presence requires durable SQLite storage",
|
|
1198
|
-
},
|
|
1199
|
-
data: { ...hubIdentity(), store_type: store.type || "unknown" },
|
|
1200
|
-
};
|
|
1201
|
-
}
|
|
1202
|
-
const identity = normalizeRegistrationIdentity(args);
|
|
1203
|
-
const result = store.registerAgent(args);
|
|
1204
|
-
if (!registerPresenceIsEffective(result)) {
|
|
1205
|
-
return registerPresenceFailure(args.agent_id, result);
|
|
1206
|
-
}
|
|
1207
|
-
if (identity && updateRegistrationIdentity) {
|
|
1208
|
-
updateRegistrationIdentity.run({
|
|
1209
|
-
agent_id: args.agent_id,
|
|
1210
|
-
...identity,
|
|
1211
|
-
});
|
|
1212
|
-
}
|
|
1213
|
-
upsertRuntimeTopics(args.agent_id, args.topics || [], { replace: true });
|
|
1214
|
-
refreshRoleCandidateForAgent(args.agent_id);
|
|
1215
|
-
for (const roleName of ROLE_TOPICS) {
|
|
1216
|
-
ensureRoleLeader(roleName, {
|
|
1217
|
-
reason: "register",
|
|
1218
|
-
transferBacklog: true,
|
|
1219
|
-
});
|
|
1220
|
-
}
|
|
1221
|
-
return {
|
|
1222
|
-
ok: true,
|
|
1223
|
-
data: { ...result, hub_pid: process.pid, ...hubIdentity() },
|
|
1224
|
-
};
|
|
1225
|
-
},
|
|
1226
|
-
|
|
1227
|
-
refreshAgentLease(
|
|
1228
|
-
agentId,
|
|
1229
|
-
ttlMs = 30000,
|
|
1230
|
-
presenceGeneration,
|
|
1231
|
-
callerIdentity = {},
|
|
1232
|
-
) {
|
|
1233
|
-
const identityFailure = validateCallerIdentity(callerIdentity);
|
|
1234
|
-
if (identityFailure) return identityFailure;
|
|
1235
|
-
const result = store.refreshLease(agentId, ttlMs, presenceGeneration);
|
|
1236
|
-
if (result?.effective?.updated === false)
|
|
1237
|
-
return staleOwnerFailure(agentId, result);
|
|
1238
|
-
refreshRoleCandidateForAgent(agentId);
|
|
1239
|
-
for (const roleName of ROLE_TOPICS) {
|
|
1240
|
-
ensureRoleLeader(roleName, {
|
|
1241
|
-
reason: "heartbeat",
|
|
1242
|
-
transferBacklog: true,
|
|
1243
|
-
});
|
|
1244
|
-
}
|
|
1245
|
-
return { ok: true, data: result };
|
|
1246
|
-
},
|
|
1247
|
-
|
|
1248
|
-
subscribeAgent(agentId, topics, { replace = false } = {}) {
|
|
1249
|
-
const nextTopics = upsertRuntimeTopics(agentId, topics, { replace });
|
|
1250
|
-
refreshRoleCandidateForAgent(agentId);
|
|
1251
|
-
for (const roleName of ROLE_TOPICS) {
|
|
1252
|
-
ensureRoleLeader(roleName, {
|
|
1253
|
-
reason: "subscribe",
|
|
1254
|
-
transferBacklog: true,
|
|
1255
|
-
});
|
|
1256
|
-
}
|
|
1257
|
-
return { agent_id: agentId, topics: nextTopics };
|
|
1258
|
-
},
|
|
1259
|
-
|
|
1260
|
-
getSubscribedTopics(agentId) {
|
|
1261
|
-
return listRuntimeTopics(agentId);
|
|
1262
|
-
},
|
|
1263
|
-
|
|
1264
|
-
updateAgentStatus(agentId, status, presenceGeneration) {
|
|
1265
|
-
if (status === "offline") {
|
|
1266
|
-
runtimeTopics.delete(agentId);
|
|
1267
|
-
}
|
|
1268
|
-
const updated = store.updateAgentStatus(
|
|
1269
|
-
agentId,
|
|
1270
|
-
status,
|
|
1271
|
-
presenceGeneration,
|
|
1272
|
-
);
|
|
1273
|
-
refreshRoleCandidateForAgent(agentId);
|
|
1274
|
-
for (const roleName of ROLE_TOPICS) {
|
|
1275
|
-
ensureRoleLeader(roleName, {
|
|
1276
|
-
reason: `status:${status}`,
|
|
1277
|
-
transferBacklog: true,
|
|
1278
|
-
});
|
|
1279
|
-
}
|
|
1280
|
-
return updated;
|
|
1281
|
-
},
|
|
1282
|
-
|
|
1283
|
-
transitionAgentStatus(
|
|
1284
|
-
agentId,
|
|
1285
|
-
status,
|
|
1286
|
-
presenceGeneration,
|
|
1287
|
-
callerIdentity = {},
|
|
1288
|
-
) {
|
|
1289
|
-
const identityFailure = validateCallerIdentity(callerIdentity);
|
|
1290
|
-
if (identityFailure) return identityFailure;
|
|
1291
|
-
const updated = store.updateAgentStatus(
|
|
1292
|
-
agentId,
|
|
1293
|
-
status,
|
|
1294
|
-
presenceGeneration,
|
|
1295
|
-
);
|
|
1296
|
-
if (!updated) return staleOwnerFailure(agentId, { status });
|
|
1297
|
-
if (status === "offline") runtimeTopics.delete(agentId);
|
|
1298
|
-
refreshRoleCandidateForAgent(agentId);
|
|
1299
|
-
for (const roleName of ROLE_TOPICS) {
|
|
1300
|
-
ensureRoleLeader(roleName, {
|
|
1301
|
-
reason: `status:${status}`,
|
|
1302
|
-
transferBacklog: true,
|
|
1303
|
-
});
|
|
1304
|
-
}
|
|
1305
|
-
return {
|
|
1306
|
-
ok: true,
|
|
1307
|
-
data: {
|
|
1308
|
-
agent_id: agentId,
|
|
1309
|
-
status,
|
|
1310
|
-
presence_generation: presenceGeneration,
|
|
1311
|
-
...hubIdentity(),
|
|
1312
|
-
},
|
|
1313
|
-
};
|
|
1314
|
-
},
|
|
1315
|
-
|
|
1316
|
-
route(msg) {
|
|
1317
|
-
const recipients = uniqueStrings(resolveRecipients(msg));
|
|
1318
|
-
if (!recipients.length) {
|
|
1319
|
-
if (shouldTrackWithoutRecipients(msg) && !getMessageRecord(msg.id)) {
|
|
1320
|
-
trackMessage(msg, []);
|
|
1321
|
-
}
|
|
1322
|
-
return 0;
|
|
1323
|
-
}
|
|
1324
|
-
if (!getMessageRecord(msg.id)) {
|
|
1325
|
-
trackMessage(msg, recipients);
|
|
1326
|
-
}
|
|
1327
|
-
for (const agentId of recipients) {
|
|
1328
|
-
queueMessage(agentId, msg);
|
|
1329
|
-
}
|
|
1330
|
-
store.updateMessageStatus(msg.id, "delivered");
|
|
1331
|
-
return recipients.length;
|
|
1332
|
-
},
|
|
1333
|
-
|
|
1334
|
-
getPendingMessages(agentId, options = {}) {
|
|
1335
|
-
return sortedPending(agentId, options);
|
|
1336
|
-
},
|
|
1337
|
-
|
|
1338
|
-
canReplayMessageForAgent(agentId, message) {
|
|
1339
|
-
return isReplayAllowedForAgent(agentId, message);
|
|
1340
|
-
},
|
|
1341
|
-
|
|
1342
|
-
countPendingMessages(agentId) {
|
|
1343
|
-
const queue = ensureAgentQueue(agentId);
|
|
1344
|
-
const now = Date.now();
|
|
1345
|
-
let count = 0;
|
|
1346
|
-
for (const delivery of queue.values()) {
|
|
1347
|
-
if (delivery.acked_at_ms) continue;
|
|
1348
|
-
if (delivery.message.expires_at_ms <= now) continue;
|
|
1349
|
-
count++;
|
|
1350
|
-
}
|
|
1351
|
-
return count;
|
|
1352
|
-
},
|
|
1353
|
-
|
|
1354
|
-
markMessagePushed(agentId, messageId) {
|
|
1355
|
-
return markDelivered(agentId, messageId);
|
|
1356
|
-
},
|
|
1357
|
-
|
|
1358
|
-
drainAgent(
|
|
1359
|
-
agentId,
|
|
1360
|
-
{ max_messages = 20, include_topics = null, auto_ack = false } = {},
|
|
1361
|
-
) {
|
|
1362
|
-
const messages = sortedPending(agentId, { max_messages, include_topics });
|
|
1363
|
-
for (const message of messages) {
|
|
1364
|
-
markDelivered(agentId, message.id);
|
|
1365
|
-
}
|
|
1366
|
-
if (auto_ack && messages.length) {
|
|
1367
|
-
ackMessages(
|
|
1368
|
-
messages.map((message) => message.id),
|
|
1369
|
-
agentId,
|
|
1370
|
-
);
|
|
1371
|
-
}
|
|
1372
|
-
return messages;
|
|
1373
|
-
},
|
|
1374
|
-
|
|
1375
|
-
ackMessages(ids, agentId) {
|
|
1376
|
-
return ackMessages(ids, agentId);
|
|
1377
|
-
},
|
|
1378
|
-
|
|
1379
|
-
async handleAsk({
|
|
1380
|
-
from,
|
|
1381
|
-
to,
|
|
1382
|
-
topic,
|
|
1383
|
-
question,
|
|
1384
|
-
context_refs,
|
|
1385
|
-
payload = {},
|
|
1386
|
-
priority = 5,
|
|
1387
|
-
ttl_ms = 300000,
|
|
1388
|
-
await_response_ms = 0,
|
|
1389
|
-
trace_id,
|
|
1390
|
-
correlation_id,
|
|
1391
|
-
}) {
|
|
1392
|
-
const cid = correlation_id || uuidv7();
|
|
1393
|
-
const tid = trace_id || uuidv7();
|
|
1394
|
-
|
|
1395
|
-
const { msg } = dispatchMessage({
|
|
1396
|
-
type: "request",
|
|
1397
|
-
from,
|
|
1398
|
-
to,
|
|
1399
|
-
topic,
|
|
1400
|
-
priority,
|
|
1401
|
-
ttl_ms,
|
|
1402
|
-
payload: { question, context_refs, ...payload },
|
|
1403
|
-
correlation_id: cid,
|
|
1404
|
-
trace_id: tid,
|
|
1405
|
-
});
|
|
1406
|
-
|
|
1407
|
-
if (await_response_ms <= 0) {
|
|
1408
|
-
return {
|
|
1409
|
-
ok: true,
|
|
1410
|
-
data: {
|
|
1411
|
-
request_message_id: msg.id,
|
|
1412
|
-
correlation_id: cid,
|
|
1413
|
-
trace_id: tid,
|
|
1414
|
-
state: "queued",
|
|
1415
|
-
},
|
|
1416
|
-
};
|
|
1417
|
-
}
|
|
1418
|
-
|
|
1419
|
-
try {
|
|
1420
|
-
const [response] = await waitForEmitterOnce(
|
|
1421
|
-
responseEmitter,
|
|
1422
|
-
cid,
|
|
1423
|
-
await_response_ms,
|
|
1424
|
-
);
|
|
1425
|
-
return {
|
|
1426
|
-
ok: true,
|
|
1427
|
-
data: {
|
|
1428
|
-
request_message_id: msg.id,
|
|
1429
|
-
correlation_id: cid,
|
|
1430
|
-
trace_id: tid,
|
|
1431
|
-
state: "answered",
|
|
1432
|
-
response,
|
|
1433
|
-
},
|
|
1434
|
-
};
|
|
1435
|
-
} catch {
|
|
1436
|
-
const resp = store.getResponseByCorrelation(cid);
|
|
1437
|
-
if (resp) {
|
|
1438
|
-
return {
|
|
1439
|
-
ok: true,
|
|
1440
|
-
data: {
|
|
1441
|
-
request_message_id: msg.id,
|
|
1442
|
-
correlation_id: cid,
|
|
1443
|
-
trace_id: tid,
|
|
1444
|
-
state: "answered",
|
|
1445
|
-
response: resp.payload,
|
|
1446
|
-
},
|
|
1447
|
-
};
|
|
1448
|
-
}
|
|
1449
|
-
return {
|
|
1450
|
-
ok: true,
|
|
1451
|
-
data: {
|
|
1452
|
-
request_message_id: msg.id,
|
|
1453
|
-
correlation_id: cid,
|
|
1454
|
-
trace_id: tid,
|
|
1455
|
-
state: "delivered",
|
|
1456
|
-
},
|
|
1457
|
-
};
|
|
1458
|
-
}
|
|
1459
|
-
},
|
|
1460
|
-
|
|
1461
|
-
handlePublish({
|
|
1462
|
-
from,
|
|
1463
|
-
to,
|
|
1464
|
-
topic,
|
|
1465
|
-
priority = 5,
|
|
1466
|
-
ttl_ms = 300000,
|
|
1467
|
-
payload = {},
|
|
1468
|
-
trace_id,
|
|
1469
|
-
correlation_id,
|
|
1470
|
-
message_type,
|
|
1471
|
-
}) {
|
|
1472
|
-
const type = message_type || (correlation_id ? "response" : "event");
|
|
1473
|
-
const { msg, recipients } = dispatchMessage({
|
|
1474
|
-
type,
|
|
1475
|
-
from,
|
|
1476
|
-
to,
|
|
1477
|
-
topic,
|
|
1478
|
-
priority,
|
|
1479
|
-
ttl_ms,
|
|
1480
|
-
payload,
|
|
1481
|
-
trace_id: trace_id || uuidv7(),
|
|
1482
|
-
correlation_id: correlation_id || uuidv7(),
|
|
1483
|
-
});
|
|
1484
|
-
return {
|
|
1485
|
-
ok: true,
|
|
1486
|
-
data: {
|
|
1487
|
-
message_id: msg.id,
|
|
1488
|
-
fanout_count: recipients.length,
|
|
1489
|
-
expires_at_ms: msg.expires_at_ms,
|
|
1490
|
-
role: to === "topic:cto" ? getRoleSnapshot("cto") : undefined,
|
|
1491
|
-
},
|
|
1492
|
-
};
|
|
1493
|
-
},
|
|
1494
|
-
|
|
1495
|
-
takeoverRole({
|
|
1496
|
-
role = "cto",
|
|
1497
|
-
agent_id,
|
|
1498
|
-
reason = "manual",
|
|
1499
|
-
requested_by = "manual",
|
|
1500
|
-
hub_instance_id,
|
|
1501
|
-
store_fingerprint,
|
|
1502
|
-
} = {}) {
|
|
1503
|
-
const identityFailure = validateCallerIdentity({
|
|
1504
|
-
hub_instance_id,
|
|
1505
|
-
store_fingerprint,
|
|
1506
|
-
});
|
|
1507
|
-
if (identityFailure) return identityFailure;
|
|
1508
|
-
const roleName = normalizeRoleName(role);
|
|
1509
|
-
if (!roleName) {
|
|
1510
|
-
return {
|
|
1511
|
-
ok: false,
|
|
1512
|
-
error: {
|
|
1513
|
-
code: "ROLE_NOT_SUPPORTED",
|
|
1514
|
-
message: `unsupported role: ${role}`,
|
|
1515
|
-
},
|
|
1516
|
-
};
|
|
1517
|
-
}
|
|
1518
|
-
const targetAgentId = String(agent_id || "").trim();
|
|
1519
|
-
if (!targetAgentId) {
|
|
1520
|
-
return {
|
|
1521
|
-
ok: false,
|
|
1522
|
-
error: {
|
|
1523
|
-
code: "AGENT_ID_REQUIRED",
|
|
1524
|
-
message: "agent_id required",
|
|
1525
|
-
},
|
|
1526
|
-
};
|
|
1527
|
-
}
|
|
1528
|
-
|
|
1529
|
-
refreshRoleCandidates(roleName);
|
|
1530
|
-
const roleState = ensureRoleState(roleName);
|
|
1531
|
-
const candidate = buildRoleCandidate(targetAgentId, roleName);
|
|
1532
|
-
if (!isCandidateLive(candidate)) {
|
|
1533
|
-
return {
|
|
1534
|
-
ok: false,
|
|
1535
|
-
error: {
|
|
1536
|
-
code: "ROLE_CANDIDATE_UNAVAILABLE",
|
|
1537
|
-
message: `${targetAgentId} is not a live ${roleName} candidate`,
|
|
1538
|
-
},
|
|
1539
|
-
data: {
|
|
1540
|
-
role: buildRoleSnapshot(roleName),
|
|
1541
|
-
},
|
|
1542
|
-
};
|
|
1543
|
-
}
|
|
1544
|
-
|
|
1545
|
-
const previousLeaderAgentId =
|
|
1546
|
-
roleState.leaderAgentId && roleState.leaderAgentId !== targetAgentId
|
|
1547
|
-
? roleState.leaderAgentId
|
|
1548
|
-
: roleState.previousLeaderAgentId || null;
|
|
1549
|
-
const changed = roleState.leaderAgentId !== targetAgentId;
|
|
1550
|
-
if (changed) {
|
|
1551
|
-
roleState.previousLeaderAgentId = previousLeaderAgentId || null;
|
|
1552
|
-
roleState.leaderAgentId = targetAgentId;
|
|
1553
|
-
roleState.leaderSource = candidate.source;
|
|
1554
|
-
roleState.leaderEpoch += 1;
|
|
1555
|
-
roleState.status = "active";
|
|
1556
|
-
roleState.lastTransitionMs = Date.now();
|
|
1557
|
-
roleState.lastReason = reason || "manual";
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
|
-
const transfer = transferRoleBacklog(
|
|
1561
|
-
roleName,
|
|
1562
|
-
targetAgentId,
|
|
1563
|
-
previousLeaderAgentId,
|
|
1564
|
-
);
|
|
1565
|
-
const snapshot = buildRoleSnapshot(roleName);
|
|
1566
|
-
return {
|
|
1567
|
-
ok: true,
|
|
1568
|
-
data: {
|
|
1569
|
-
role: roleName,
|
|
1570
|
-
requested_by,
|
|
1571
|
-
reason,
|
|
1572
|
-
changed,
|
|
1573
|
-
previous_leader_agent_id: previousLeaderAgentId,
|
|
1574
|
-
leader_agent_id: snapshot.leader_agent_id,
|
|
1575
|
-
leader_epoch: snapshot.leader_epoch,
|
|
1576
|
-
...transfer,
|
|
1577
|
-
status: snapshot,
|
|
1578
|
-
...hubIdentity(),
|
|
1579
|
-
},
|
|
1580
|
-
};
|
|
1581
|
-
},
|
|
1582
|
-
|
|
1583
|
-
handleHandoff({
|
|
1584
|
-
from,
|
|
1585
|
-
to,
|
|
1586
|
-
topic,
|
|
1587
|
-
task,
|
|
1588
|
-
acceptance_criteria,
|
|
1589
|
-
context_refs,
|
|
1590
|
-
priority = 5,
|
|
1591
|
-
ttl_ms = DEFAULT_HANDOFF_TTL_MS,
|
|
1592
|
-
trace_id,
|
|
1593
|
-
correlation_id,
|
|
1594
|
-
}) {
|
|
1595
|
-
const { msg } = dispatchMessage({
|
|
1596
|
-
type: "handoff",
|
|
1597
|
-
from,
|
|
1598
|
-
to,
|
|
1599
|
-
topic,
|
|
1600
|
-
priority,
|
|
1601
|
-
ttl_ms,
|
|
1602
|
-
payload: { task, acceptance_criteria, context_refs },
|
|
1603
|
-
trace_id: trace_id || uuidv7(),
|
|
1604
|
-
correlation_id: correlation_id || uuidv7(),
|
|
1605
|
-
});
|
|
1606
|
-
return {
|
|
1607
|
-
ok: true,
|
|
1608
|
-
data: { handoff_message_id: msg.id, state: "queued", assigned_to: to },
|
|
1609
|
-
};
|
|
1610
|
-
},
|
|
1611
|
-
|
|
1612
|
-
assignAsync({
|
|
1613
|
-
supervisor_agent,
|
|
1614
|
-
worker_agent,
|
|
1615
|
-
topic = "assign.job",
|
|
1616
|
-
task = "",
|
|
1617
|
-
payload = {},
|
|
1618
|
-
priority = 5,
|
|
1619
|
-
ttl_ms = DEFAULT_ASSIGN_TTL_MS,
|
|
1620
|
-
timeout_ms = DEFAULT_ASSIGN_TIMEOUT_MS,
|
|
1621
|
-
max_retries = 0,
|
|
1622
|
-
trace_id,
|
|
1623
|
-
correlation_id,
|
|
1624
|
-
}) {
|
|
1625
|
-
const job = store.createAssign({
|
|
1626
|
-
supervisor_agent,
|
|
1627
|
-
worker_agent,
|
|
1628
|
-
topic,
|
|
1629
|
-
task,
|
|
1630
|
-
payload,
|
|
1631
|
-
priority,
|
|
1632
|
-
ttl_ms,
|
|
1633
|
-
timeout_ms,
|
|
1634
|
-
max_retries,
|
|
1635
|
-
trace_id,
|
|
1636
|
-
correlation_id,
|
|
1637
|
-
});
|
|
1638
|
-
const dispatched = dispatchAssignJob(job, "create");
|
|
1639
|
-
return {
|
|
1640
|
-
ok: true,
|
|
1641
|
-
data: {
|
|
1642
|
-
assigned_to: worker_agent,
|
|
1643
|
-
...buildAssignSnapshot(dispatched.job),
|
|
1644
|
-
},
|
|
1645
|
-
};
|
|
1646
|
-
},
|
|
1647
|
-
|
|
1648
|
-
reportAssignResult({
|
|
1649
|
-
job_id,
|
|
1650
|
-
worker_agent,
|
|
1651
|
-
status,
|
|
1652
|
-
attempt,
|
|
1653
|
-
result,
|
|
1654
|
-
error,
|
|
1655
|
-
payload = {},
|
|
1656
|
-
metadata = {},
|
|
1657
|
-
}) {
|
|
1658
|
-
const job = store.getAssign(job_id);
|
|
1659
|
-
if (!job) {
|
|
1660
|
-
return {
|
|
1661
|
-
ok: false,
|
|
1662
|
-
error: {
|
|
1663
|
-
code: "ASSIGN_NOT_FOUND",
|
|
1664
|
-
message: `assign job not found: ${job_id}`,
|
|
1665
|
-
},
|
|
1666
|
-
};
|
|
1667
|
-
}
|
|
1668
|
-
if (worker_agent && worker_agent !== job.worker_agent) {
|
|
1669
|
-
return {
|
|
1670
|
-
ok: false,
|
|
1671
|
-
error: {
|
|
1672
|
-
code: "ASSIGN_WORKER_MISMATCH",
|
|
1673
|
-
message: `worker mismatch: ${worker_agent}`,
|
|
1674
|
-
},
|
|
1675
|
-
};
|
|
1676
|
-
}
|
|
1677
|
-
const mergedMetadata = {
|
|
1678
|
-
...(payload?.metadata || {}),
|
|
1679
|
-
...(metadata || {}),
|
|
1680
|
-
};
|
|
1681
|
-
const normalizedStatus = normalizeAssignTerminalStatus(
|
|
1682
|
-
status || payload?.status,
|
|
1683
|
-
mergedMetadata,
|
|
1684
|
-
);
|
|
1685
|
-
const nextResult =
|
|
1686
|
-
result ??
|
|
1687
|
-
(Object.hasOwn(payload || {}, "result") ? payload.result : payload);
|
|
1688
|
-
const nextError = error ?? payload?.error ?? null;
|
|
1689
|
-
|
|
1690
|
-
if (Number.isFinite(Number(attempt)) && Number(attempt) !== job.attempt) {
|
|
1691
|
-
const preserved = notifyAssignSupervisor(job, "late_result", {
|
|
1692
|
-
stale_attempt: Number(attempt),
|
|
1693
|
-
late_status: normalizedStatus,
|
|
1694
|
-
late_result: nextResult,
|
|
1695
|
-
late_error: nextError,
|
|
1696
|
-
reported_by: worker_agent || job.worker_agent,
|
|
1697
|
-
});
|
|
1698
|
-
return {
|
|
1699
|
-
ok: false,
|
|
1700
|
-
error: {
|
|
1701
|
-
code: "ASSIGN_ATTEMPT_MISMATCH",
|
|
1702
|
-
message: `stale assign result for attempt ${attempt} (current ${job.attempt})`,
|
|
1703
|
-
details: {
|
|
1704
|
-
preserved: Boolean(preserved),
|
|
1705
|
-
preserved_message_id: preserved?.id ?? null,
|
|
1706
|
-
},
|
|
1707
|
-
},
|
|
1708
|
-
};
|
|
1709
|
-
}
|
|
1710
|
-
|
|
1711
|
-
if (normalizedStatus === "running") {
|
|
1712
|
-
const nowMs = Date.now();
|
|
1713
|
-
const ceilingAtMs =
|
|
1714
|
-
(job.dispatched_at_ms || job.created_at_ms) + resolveHardCeilingMs();
|
|
1715
|
-
const running = store.updateAssignStatus(job.job_id, "running", {
|
|
1716
|
-
started_at_ms: job.started_at_ms || nowMs,
|
|
1717
|
-
deadline_ms: Math.min(
|
|
1718
|
-
nowMs + clampDurationMs(job.timeout_ms, job.timeout_ms),
|
|
1719
|
-
ceilingAtMs,
|
|
1720
|
-
),
|
|
1721
|
-
result: nextResult,
|
|
1722
|
-
error: nextError,
|
|
1723
|
-
});
|
|
1724
|
-
notifyAssignSupervisor(running, "progress");
|
|
1725
|
-
return { ok: true, data: buildAssignSnapshot(running) };
|
|
1726
|
-
}
|
|
1727
|
-
|
|
1728
|
-
const finalized = store.updateAssignStatus(job.job_id, normalizedStatus, {
|
|
1729
|
-
result: nextResult,
|
|
1730
|
-
error: nextError,
|
|
1731
|
-
});
|
|
1732
|
-
|
|
1733
|
-
if (
|
|
1734
|
-
(normalizedStatus === "failed" || normalizedStatus === "timed_out") &&
|
|
1735
|
-
finalized.retry_count < finalized.max_retries
|
|
1736
|
-
) {
|
|
1737
|
-
return scheduleAssignRetry(
|
|
1738
|
-
finalized,
|
|
1739
|
-
normalizedStatus,
|
|
1740
|
-
nextError,
|
|
1741
|
-
worker_agent || finalized.worker_agent,
|
|
1742
|
-
);
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
notifyAssignSupervisor(finalized, "completed");
|
|
1746
|
-
return { ok: true, data: buildAssignSnapshot(finalized) };
|
|
1747
|
-
},
|
|
1748
|
-
|
|
1749
|
-
getAssignStatus({ job_id, ...filters } = {}) {
|
|
1750
|
-
if (job_id) {
|
|
1751
|
-
const job = store.getAssign(job_id);
|
|
1752
|
-
return job
|
|
1753
|
-
? { ok: true, data: buildAssignSnapshot(job) }
|
|
1754
|
-
: {
|
|
1755
|
-
ok: false,
|
|
1756
|
-
error: {
|
|
1757
|
-
code: "ASSIGN_NOT_FOUND",
|
|
1758
|
-
message: `assign job not found: ${job_id}`,
|
|
1759
|
-
},
|
|
1760
|
-
};
|
|
1761
|
-
}
|
|
1762
|
-
return {
|
|
1763
|
-
ok: true,
|
|
1764
|
-
data: {
|
|
1765
|
-
assigns: store
|
|
1766
|
-
.listAssigns(filters)
|
|
1767
|
-
.map((job) => buildAssignSnapshot(job)),
|
|
1768
|
-
},
|
|
1769
|
-
};
|
|
1770
|
-
},
|
|
1771
|
-
|
|
1772
|
-
retryAssign(job_id, { reason = "manual", requested_by = "manual" } = {}) {
|
|
1773
|
-
const job = store.getAssign(job_id);
|
|
1774
|
-
if (!job) {
|
|
1775
|
-
return {
|
|
1776
|
-
ok: false,
|
|
1777
|
-
error: {
|
|
1778
|
-
code: "ASSIGN_NOT_FOUND",
|
|
1779
|
-
message: `assign job not found: ${job_id}`,
|
|
1780
|
-
},
|
|
1781
|
-
};
|
|
1782
|
-
}
|
|
1783
|
-
return scheduleAssignRetry(job, reason, job.error, requested_by);
|
|
1784
|
-
},
|
|
1785
|
-
|
|
1786
|
-
sweepExpired({ now_ms = Date.now(), source = "manual" } = {}) {
|
|
1787
|
-
const result = store.sweepExpired({ now_ms });
|
|
1788
|
-
for (const [messageId, record] of Array.from(liveMessages.entries())) {
|
|
1789
|
-
if (record.message.expires_at_ms > now_ms) continue;
|
|
1790
|
-
removeMessage(messageId);
|
|
1791
|
-
}
|
|
1792
|
-
for (const transition of result.transitions) {
|
|
1793
|
-
if (!transition.dlq_inserted) continue;
|
|
1794
|
-
const { message } = transition;
|
|
1795
|
-
if (
|
|
1796
|
-
!["handoff", "request"].includes(message.type) ||
|
|
1797
|
-
message.payload?.kind === "assign.job"
|
|
1798
|
-
)
|
|
1799
|
-
continue;
|
|
1800
|
-
if (
|
|
1801
|
-
!message.from_agent ||
|
|
1802
|
-
String(message.from_agent).startsWith("topic:")
|
|
1803
|
-
)
|
|
1804
|
-
continue;
|
|
1805
|
-
dispatchMessage({
|
|
1806
|
-
type: "event",
|
|
1807
|
-
from: "hub-sweeper",
|
|
1808
|
-
to: message.from_agent,
|
|
1809
|
-
topic: "handoff.dead_letter",
|
|
1810
|
-
priority: 7,
|
|
1811
|
-
ttl_ms: 300000,
|
|
1812
|
-
payload: {
|
|
1813
|
-
kind: "handoff.dead_letter",
|
|
1814
|
-
message_id: message.id,
|
|
1815
|
-
original_topic: message.topic,
|
|
1816
|
-
original_to: message.to_agent,
|
|
1817
|
-
reason: "ttl_expired",
|
|
1818
|
-
task: message.payload?.task ?? null,
|
|
1819
|
-
expired_at_ms: message.expires_at_ms,
|
|
1820
|
-
failed_at_ms: transition.failed_at_ms,
|
|
1821
|
-
recovery_source: source,
|
|
1822
|
-
},
|
|
1823
|
-
trace_id: message.trace_id,
|
|
1824
|
-
correlation_id: message.correlation_id,
|
|
1825
|
-
});
|
|
1826
|
-
}
|
|
1827
|
-
return result;
|
|
1828
|
-
},
|
|
1829
|
-
|
|
1830
|
-
sweepTimedOutAssigns() {
|
|
1831
|
-
const expiredAssigns = store.listAssigns({
|
|
1832
|
-
statuses: Array.from(ASSIGN_PENDING_STATUSES),
|
|
1833
|
-
active_before_ms: Date.now(),
|
|
1834
|
-
limit: 100,
|
|
1835
|
-
});
|
|
1836
|
-
let timed_out = 0;
|
|
1837
|
-
let retried = 0;
|
|
1838
|
-
|
|
1839
|
-
for (const job of expiredAssigns) {
|
|
1840
|
-
const result = handleAssignTimeout(job);
|
|
1841
|
-
timed_out += 1;
|
|
1842
|
-
if (result?.data?.retried) retried += 1;
|
|
1843
|
-
}
|
|
1844
|
-
|
|
1845
|
-
return { timed_out, retried };
|
|
1846
|
-
},
|
|
1847
|
-
|
|
1848
|
-
startSweeper({
|
|
1849
|
-
run_immediately = false,
|
|
1850
|
-
interval_ms = 10000,
|
|
1851
|
-
onSweepError,
|
|
1852
|
-
} = {}) {
|
|
1853
|
-
if (sweepTimer) return;
|
|
1854
|
-
const reportSweepError = (source, error, failures) => {
|
|
1855
|
-
if (typeof onSweepError === "function") {
|
|
1856
|
-
onSweepError({ source, error, consecutive_failures: failures });
|
|
1857
|
-
}
|
|
1858
|
-
};
|
|
1859
|
-
let sweepFailures = 0;
|
|
1860
|
-
let assignFailures = 0;
|
|
1861
|
-
const runSweep = () => {
|
|
1862
|
-
try {
|
|
1863
|
-
router.sweepExpired({ source: "periodic" });
|
|
1864
|
-
sweepFailures = 0;
|
|
1865
|
-
} catch (error) {
|
|
1866
|
-
sweepFailures += 1;
|
|
1867
|
-
reportSweepError("periodic", error, sweepFailures);
|
|
1868
|
-
}
|
|
1869
|
-
try {
|
|
1870
|
-
router.sweepTimedOutAssigns();
|
|
1871
|
-
assignFailures = 0;
|
|
1872
|
-
} catch (error) {
|
|
1873
|
-
assignFailures += 1;
|
|
1874
|
-
reportSweepError("assign_timeout", error, assignFailures);
|
|
1875
|
-
}
|
|
1876
|
-
};
|
|
1877
|
-
if (run_immediately) runSweep();
|
|
1878
|
-
sweepTimer = setInterval(() => {
|
|
1879
|
-
runSweep();
|
|
1880
|
-
}, interval_ms);
|
|
1881
|
-
let staleFailures = 0;
|
|
1882
|
-
staleTimer = setInterval(() => {
|
|
1883
|
-
try {
|
|
1884
|
-
store.sweepStaleAgents();
|
|
1885
|
-
router.reelectStaleRoles();
|
|
1886
|
-
staleFailures = 0;
|
|
1887
|
-
} catch (error) {
|
|
1888
|
-
staleFailures += 1;
|
|
1889
|
-
reportSweepError("stale_agents", error, staleFailures);
|
|
1890
|
-
}
|
|
1891
|
-
}, 120000);
|
|
1892
|
-
sweepTimer.unref();
|
|
1893
|
-
staleTimer.unref();
|
|
1894
|
-
},
|
|
1895
|
-
|
|
1896
|
-
stopSweeper() {
|
|
1897
|
-
if (sweepTimer) {
|
|
1898
|
-
clearInterval(sweepTimer);
|
|
1899
|
-
sweepTimer = null;
|
|
1900
|
-
}
|
|
1901
|
-
if (staleTimer) {
|
|
1902
|
-
clearInterval(staleTimer);
|
|
1903
|
-
staleTimer = null;
|
|
1904
|
-
}
|
|
1905
|
-
},
|
|
1906
|
-
|
|
1907
|
-
reelectStaleRoles({ reason = "sweep" } = {}) {
|
|
1908
|
-
for (const roleName of ROLE_TOPICS) {
|
|
1909
|
-
const role = roleStates.get(roleName);
|
|
1910
|
-
const leader = role?.leaderAgentId
|
|
1911
|
-
? buildRoleCandidate(role.leaderAgentId, roleName)
|
|
1912
|
-
: null;
|
|
1913
|
-
if (!isCandidateLive(leader)) {
|
|
1914
|
-
ensureRoleLeader(roleName, { reason });
|
|
1915
|
-
}
|
|
1916
|
-
}
|
|
1917
|
-
},
|
|
1918
|
-
|
|
1919
|
-
getQueueDepths() {
|
|
1920
|
-
const counts = { urgent: 0, normal: 0, dlq: store.getAuditStats().dlq };
|
|
1921
|
-
for (const record of liveMessages.values()) {
|
|
1922
|
-
const pending = record.recipients.size > record.ackedBy.size;
|
|
1923
|
-
if (!pending) continue;
|
|
1924
|
-
if (record.message.priority >= 7) counts.urgent += 1;
|
|
1925
|
-
else counts.normal += 1;
|
|
1926
|
-
}
|
|
1927
|
-
return counts;
|
|
1928
|
-
},
|
|
1929
|
-
|
|
1930
|
-
getDeliveryStats() {
|
|
1931
|
-
if (latencyIdx === 0) {
|
|
1932
|
-
return { total_deliveries: 0, avg_delivery_ms: 0 };
|
|
1933
|
-
}
|
|
1934
|
-
const filled = Math.min(latencyIdx, MAX_LATENCY_SAMPLES);
|
|
1935
|
-
const total = deliveryLatencies
|
|
1936
|
-
.slice(0, filled)
|
|
1937
|
-
.reduce((sum, ms) => sum + ms, 0);
|
|
1938
|
-
return {
|
|
1939
|
-
total_deliveries: latencyIdx,
|
|
1940
|
-
avg_delivery_ms: Math.round(total / filled),
|
|
1941
|
-
};
|
|
1942
|
-
},
|
|
1943
|
-
|
|
1944
|
-
getStatus(
|
|
1945
|
-
scope = "hub",
|
|
1946
|
-
{
|
|
1947
|
-
agent_id,
|
|
1948
|
-
trace_id,
|
|
1949
|
-
include_metrics = true,
|
|
1950
|
-
hub_instance_id,
|
|
1951
|
-
store_fingerprint,
|
|
1952
|
-
} = {},
|
|
1953
|
-
) {
|
|
1954
|
-
const identityFailure = validateCallerIdentity({
|
|
1955
|
-
hub_instance_id,
|
|
1956
|
-
store_fingerprint,
|
|
1957
|
-
});
|
|
1958
|
-
if (identityFailure) return identityFailure;
|
|
1959
|
-
const data = {};
|
|
1960
|
-
|
|
1961
|
-
if (scope === "hub" || scope === "queue") {
|
|
1962
|
-
data.hub = {
|
|
1963
|
-
state: "healthy",
|
|
1964
|
-
uptime_ms: (process.uptime() * 1000) | 0,
|
|
1965
|
-
realtime_transport: "named-pipe",
|
|
1966
|
-
audit_store: store.type || "sqlite",
|
|
1967
|
-
...hubIdentity(),
|
|
1968
|
-
};
|
|
1969
|
-
data.roles = {
|
|
1970
|
-
cto: getRoleSnapshot("cto"),
|
|
1971
|
-
};
|
|
1972
|
-
if (include_metrics) {
|
|
1973
|
-
const depths = router.getQueueDepths();
|
|
1974
|
-
const stats = router.getDeliveryStats();
|
|
1975
|
-
const auditStats = store.getAuditStats();
|
|
1976
|
-
data.queues = {
|
|
1977
|
-
urgent_depth: depths.urgent,
|
|
1978
|
-
normal_depth: depths.normal,
|
|
1979
|
-
dlq_depth: depths.dlq,
|
|
1980
|
-
avg_delivery_ms: stats.avg_delivery_ms,
|
|
1981
|
-
};
|
|
1982
|
-
data.assigns = {
|
|
1983
|
-
queued: auditStats.assign_queued,
|
|
1984
|
-
running: auditStats.assign_running,
|
|
1985
|
-
failed: auditStats.assign_failed,
|
|
1986
|
-
timed_out: auditStats.assign_timed_out,
|
|
1987
|
-
};
|
|
1988
|
-
}
|
|
1989
|
-
}
|
|
1990
|
-
|
|
1991
|
-
if (scope === "agent" && agent_id) {
|
|
1992
|
-
const agent = store.getAgent(agent_id);
|
|
1993
|
-
if (agent) {
|
|
1994
|
-
refreshRoleCandidateForAgent(agent_id);
|
|
1995
|
-
data.agent = {
|
|
1996
|
-
agent_id: agent.agent_id,
|
|
1997
|
-
status: agent.status,
|
|
1998
|
-
pending: sortedPending(agent_id, { max_messages: 1000 }).length,
|
|
1999
|
-
last_seen_ms: agent.last_seen_ms,
|
|
2000
|
-
topics: listRuntimeTopics(agent_id),
|
|
2001
|
-
};
|
|
2002
|
-
const roles = {};
|
|
2003
|
-
for (const roleName of ROLE_TOPICS) {
|
|
2004
|
-
const candidate = buildRoleCandidate(agent_id, roleName);
|
|
2005
|
-
if (candidate) {
|
|
2006
|
-
roles[roleName] = {
|
|
2007
|
-
candidate_source: candidate.source,
|
|
2008
|
-
priority: candidate.priority,
|
|
2009
|
-
leader: getRoleSnapshot(roleName)?.leader_agent_id === agent_id,
|
|
2010
|
-
};
|
|
2011
|
-
}
|
|
2012
|
-
}
|
|
2013
|
-
if (Object.keys(roles).length) data.agent.roles = roles;
|
|
2014
|
-
}
|
|
2015
|
-
}
|
|
2016
|
-
|
|
2017
|
-
if (scope === "trace" && trace_id) {
|
|
2018
|
-
data.trace = store.getMessagesByTrace(trace_id);
|
|
2019
|
-
}
|
|
2020
|
-
|
|
2021
|
-
return { ok: true, data: { ...data, ...hubIdentity() } };
|
|
2022
|
-
},
|
|
2023
|
-
};
|
|
2024
|
-
|
|
2025
|
-
return router;
|
|
2026
|
-
}
|