@llblab/pi-telegram 0.27.12 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +150 -258
- package/BACKLOG.md +1 -169
- package/CHANGELOG.md +393 -443
- package/README.md +9 -6
- package/api/updates.ts +5 -0
- package/docs/architecture.md +71 -26
- package/docs/multi-instance-bus.md +23 -5
- package/docs/public-api.md +7 -6
- package/docs/ui-style.md +6 -6
- package/docs/updates.md +29 -11
- package/index.ts +356 -246
- package/lib/activity-verbosity.ts +26 -0
- package/lib/bindings.ts +240 -2
- package/lib/bus-follower.ts +436 -238
- package/lib/bus-leader.ts +395 -42
- package/lib/bus.ts +994 -153
- package/lib/commands.ts +184 -30
- package/lib/config.ts +23 -2
- package/lib/journal.ts +3140 -0
- package/lib/lifecycle.ts +4 -0
- package/lib/locks.ts +4 -2
- package/lib/media.ts +71 -32
- package/lib/menu-queue.ts +31 -17
- package/lib/menu.ts +5 -3
- package/lib/model.ts +51 -24
- package/lib/ownership.ts +42 -7
- package/lib/paths.ts +35 -0
- package/lib/polling.ts +591 -106
- package/lib/prompts.ts +17 -0
- package/lib/queue.ts +732 -143
- package/lib/routing.ts +291 -64
- package/lib/runtime.ts +26 -10
- package/lib/status.ts +257 -18
- package/lib/sync.ts +131 -5
- package/lib/telegram-api.ts +41 -11
- package/lib/text-groups.ts +75 -35
- package/lib/threads.ts +112 -4
- package/lib/turns.ts +79 -14
- package/lib/updates.ts +3771 -223
- package/package.json +3 -3
- package/scripts/check-downgrade.mjs +435 -0
package/lib/bus-leader.ts
CHANGED
|
@@ -16,6 +16,9 @@ import * as Threads from "./threads.ts";
|
|
|
16
16
|
import {
|
|
17
17
|
createTelegramBusLocalServer,
|
|
18
18
|
createUnauthorizedBusAck,
|
|
19
|
+
getTelegramBusEnvelopeTrafficClass,
|
|
20
|
+
getTelegramBusProtocolCompatibility,
|
|
21
|
+
hasTelegramBusCapability,
|
|
19
22
|
isTelegramBusEnvelopeAuthorized,
|
|
20
23
|
getTelegramBusFollowerSocketPath,
|
|
21
24
|
sendTelegramBusLocalEnvelope,
|
|
@@ -24,13 +27,32 @@ import {
|
|
|
24
27
|
type TelegramBusFollowerRegistry,
|
|
25
28
|
type TelegramBusFollowerView,
|
|
26
29
|
type TelegramBusInstanceRegistration,
|
|
30
|
+
type TelegramBusProtocolIdentity,
|
|
27
31
|
type TelegramBusSocketPathSource,
|
|
32
|
+
TELEGRAM_BUS_CAPABILITY_DURABLE_FOLLOWER_ADMISSION,
|
|
33
|
+
TELEGRAM_BUS_CAPABILITY_QUEUE_HANDOFF,
|
|
28
34
|
} from "./bus.ts";
|
|
29
35
|
import { getTelegramBusTransportRetryPolicy } from "./bus-transport.ts";
|
|
36
|
+
import type { TelegramQueueHandoffPayload } from "./queue.ts";
|
|
30
37
|
|
|
31
38
|
export interface TelegramBusLeaderRuntime<TContext> {
|
|
32
39
|
startPolling: (ctx: TContext) => Promise<void>;
|
|
33
40
|
stopPolling: () => Promise<void>;
|
|
41
|
+
routeQueueHandoff: (input: {
|
|
42
|
+
requestId: string;
|
|
43
|
+
auth?: string;
|
|
44
|
+
recipientInstanceId: string;
|
|
45
|
+
recipientRegistrationGeneration: string;
|
|
46
|
+
donorInstanceId: string;
|
|
47
|
+
donorProcessId: number;
|
|
48
|
+
donorProcessBirthId: string;
|
|
49
|
+
donorSessionGeneration: number;
|
|
50
|
+
donorAcquisitionId: string;
|
|
51
|
+
donorAcquiredAtMs: number;
|
|
52
|
+
handoffToken: string;
|
|
53
|
+
payload: TelegramQueueHandoffPayload;
|
|
54
|
+
sentAtMs: number;
|
|
55
|
+
}) => Promise<TelegramBusEnvelope>;
|
|
34
56
|
}
|
|
35
57
|
|
|
36
58
|
export interface TelegramBusFollowerLifecycleAnnouncement {
|
|
@@ -249,6 +271,7 @@ export interface TelegramBusLeaderRuntimeDeps<TContext> {
|
|
|
249
271
|
commitEndpointPublication?: (commit: () => void) => boolean;
|
|
250
272
|
followerRegistry: TelegramBusFollowerRegistry;
|
|
251
273
|
authSecret?: string;
|
|
274
|
+
protocolIdentity: TelegramBusProtocolIdentity;
|
|
252
275
|
startPolling: (ctx: TContext) => void | Promise<void>;
|
|
253
276
|
stopPolling: () => void | Promise<void>;
|
|
254
277
|
callApi?: (method: string, args: unknown[]) => Promise<unknown> | unknown;
|
|
@@ -272,12 +295,20 @@ export interface TelegramBusLeaderRuntimeDeps<TContext> {
|
|
|
272
295
|
{ kind: "follower.routeAgentMessage" }
|
|
273
296
|
>["message"],
|
|
274
297
|
) => Promise<void> | void;
|
|
298
|
+
routeQueueHandoff?: (
|
|
299
|
+
follower: TelegramBusFollowerView,
|
|
300
|
+
envelope: Extract<
|
|
301
|
+
TelegramBusEnvelope,
|
|
302
|
+
{ kind: "follower.offerQueueHandoff" }
|
|
303
|
+
>,
|
|
304
|
+
) => Promise<unknown> | unknown;
|
|
275
305
|
provisionFollowerTarget?: (
|
|
276
306
|
registration: TelegramBusInstanceRegistration,
|
|
277
307
|
) => Promise<TelegramTarget | undefined> | TelegramTarget | undefined;
|
|
278
308
|
getCurrentLeaderEpoch?: () => number | string | undefined;
|
|
279
309
|
provisionLeaderTarget?: (ctx: TContext) => Promise<void> | void;
|
|
280
310
|
getNowMs?: () => number;
|
|
311
|
+
timeoutMs?: number;
|
|
281
312
|
followerPruneIntervalMs?: number;
|
|
282
313
|
followerStaleAfterMs?: number;
|
|
283
314
|
isFollowerProcessAlive?: (pid: number) => boolean;
|
|
@@ -312,9 +343,16 @@ const TELEGRAM_BUS_SLOW_FOLLOWER_REGISTRATION_MS = 1000;
|
|
|
312
343
|
|
|
313
344
|
function scheduleTelegramBusLeaderBackgroundTask(
|
|
314
345
|
task: () => Promise<void>,
|
|
346
|
+
onError: (error: unknown) => void,
|
|
315
347
|
): void {
|
|
316
348
|
const timer = setTimeout(() => {
|
|
317
|
-
void task()
|
|
349
|
+
void task().catch((error) => {
|
|
350
|
+
try {
|
|
351
|
+
onError(error);
|
|
352
|
+
} catch {
|
|
353
|
+
// Background diagnostics cannot create an unhandled timer rejection.
|
|
354
|
+
}
|
|
355
|
+
});
|
|
318
356
|
}, 0);
|
|
319
357
|
timer.unref?.();
|
|
320
358
|
}
|
|
@@ -721,6 +759,11 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
721
759
|
target: result.target,
|
|
722
760
|
reused: result.reused,
|
|
723
761
|
});
|
|
762
|
+
}, (error) => {
|
|
763
|
+
deps.recordRuntimeEvent("bus", error, {
|
|
764
|
+
phase: "follower-register-background-owner",
|
|
765
|
+
instanceId: registration.instanceId,
|
|
766
|
+
});
|
|
724
767
|
});
|
|
725
768
|
return {
|
|
726
769
|
...result.target,
|
|
@@ -1044,6 +1087,7 @@ function createTelegramBusFollowerMutationRunner(): TelegramBusFollowerMutationR
|
|
|
1044
1087
|
export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
1045
1088
|
followerRegistry: TelegramBusFollowerRegistry;
|
|
1046
1089
|
authSecret?: string;
|
|
1090
|
+
protocolIdentity: TelegramBusProtocolIdentity;
|
|
1047
1091
|
getNowMs?: () => number;
|
|
1048
1092
|
timeoutMs?: number;
|
|
1049
1093
|
callApi?: (method: string, args: unknown[]) => Promise<unknown> | unknown;
|
|
@@ -1067,6 +1111,13 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1067
1111
|
{ kind: "follower.routeAgentMessage" }
|
|
1068
1112
|
>["message"],
|
|
1069
1113
|
) => Promise<void> | void;
|
|
1114
|
+
routeQueueHandoff?: (
|
|
1115
|
+
follower: TelegramBusFollowerView,
|
|
1116
|
+
envelope: Extract<
|
|
1117
|
+
TelegramBusEnvelope,
|
|
1118
|
+
{ kind: "follower.offerQueueHandoff" }
|
|
1119
|
+
>,
|
|
1120
|
+
) => Promise<unknown> | unknown;
|
|
1070
1121
|
provisionFollowerTarget?: (
|
|
1071
1122
|
registration: TelegramBusInstanceRegistration,
|
|
1072
1123
|
) =>
|
|
@@ -1107,7 +1158,7 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1107
1158
|
};
|
|
1108
1159
|
}
|
|
1109
1160
|
if (
|
|
1110
|
-
follower.registrationGeneration
|
|
1161
|
+
!follower.registrationGeneration ||
|
|
1111
1162
|
envelope.registrationGeneration !== follower.registrationGeneration
|
|
1112
1163
|
) {
|
|
1113
1164
|
return {
|
|
@@ -1148,6 +1199,130 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1148
1199
|
await deps.routeAgentMessage(follower, envelope.message);
|
|
1149
1200
|
return { kind: "bus.ack", requestId: envelope.requestId, ok: true };
|
|
1150
1201
|
};
|
|
1202
|
+
const routeQueueHandoff = async (
|
|
1203
|
+
envelope: Extract<
|
|
1204
|
+
TelegramBusEnvelope,
|
|
1205
|
+
{ kind: "follower.offerQueueHandoff" }
|
|
1206
|
+
>,
|
|
1207
|
+
): Promise<TelegramBusEnvelope> => {
|
|
1208
|
+
const donor = deps.followerRegistry.get(envelope.instanceId);
|
|
1209
|
+
if (!donor) {
|
|
1210
|
+
return {
|
|
1211
|
+
kind: "bus.ack",
|
|
1212
|
+
requestId: envelope.requestId,
|
|
1213
|
+
ok: false,
|
|
1214
|
+
message: "Unknown Telegram bus follower instance.",
|
|
1215
|
+
};
|
|
1216
|
+
}
|
|
1217
|
+
if (
|
|
1218
|
+
!donor.registrationGeneration ||
|
|
1219
|
+
envelope.registrationGeneration !== donor.registrationGeneration
|
|
1220
|
+
) {
|
|
1221
|
+
return {
|
|
1222
|
+
kind: "bus.ack",
|
|
1223
|
+
requestId: envelope.requestId,
|
|
1224
|
+
ok: false,
|
|
1225
|
+
message: "Stale Telegram bus follower registration generation.",
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1228
|
+
const recipient = deps.followerRegistry.get(envelope.recipientInstanceId);
|
|
1229
|
+
if (
|
|
1230
|
+
!hasTelegramBusCapability(
|
|
1231
|
+
deps.protocolIdentity,
|
|
1232
|
+
TELEGRAM_BUS_CAPABILITY_QUEUE_HANDOFF,
|
|
1233
|
+
) ||
|
|
1234
|
+
!hasTelegramBusCapability(
|
|
1235
|
+
donor.protocol,
|
|
1236
|
+
TELEGRAM_BUS_CAPABILITY_QUEUE_HANDOFF,
|
|
1237
|
+
) ||
|
|
1238
|
+
!hasTelegramBusCapability(
|
|
1239
|
+
recipient?.protocol,
|
|
1240
|
+
TELEGRAM_BUS_CAPABILITY_QUEUE_HANDOFF,
|
|
1241
|
+
)
|
|
1242
|
+
) {
|
|
1243
|
+
return {
|
|
1244
|
+
kind: "bus.ack",
|
|
1245
|
+
requestId: envelope.requestId,
|
|
1246
|
+
ok: false,
|
|
1247
|
+
message: "Telegram queue handoff capability was not negotiated.",
|
|
1248
|
+
};
|
|
1249
|
+
}
|
|
1250
|
+
if (
|
|
1251
|
+
!recipient?.registrationGeneration ||
|
|
1252
|
+
envelope.recipientRegistrationGeneration !==
|
|
1253
|
+
recipient.registrationGeneration
|
|
1254
|
+
) {
|
|
1255
|
+
return {
|
|
1256
|
+
kind: "bus.ack",
|
|
1257
|
+
requestId: envelope.requestId,
|
|
1258
|
+
ok: false,
|
|
1259
|
+
message: "Stale Telegram queue handoff recipient registration generation.",
|
|
1260
|
+
};
|
|
1261
|
+
}
|
|
1262
|
+
if (donor.instanceId === recipient.instanceId) {
|
|
1263
|
+
return {
|
|
1264
|
+
kind: "bus.ack",
|
|
1265
|
+
requestId: envelope.requestId,
|
|
1266
|
+
ok: false,
|
|
1267
|
+
message: "Telegram queue handoff recipient must be another runtime.",
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
if (deps.routeQueueHandoff) {
|
|
1271
|
+
const result = await deps.routeQueueHandoff(donor, envelope);
|
|
1272
|
+
return {
|
|
1273
|
+
kind: "bus.ack",
|
|
1274
|
+
requestId: envelope.requestId,
|
|
1275
|
+
ok: true,
|
|
1276
|
+
...(result !== undefined ? { result } : {}),
|
|
1277
|
+
};
|
|
1278
|
+
}
|
|
1279
|
+
const recipientSocketPath =
|
|
1280
|
+
recipient.busSocketPath ??
|
|
1281
|
+
getTelegramBusFollowerSocketPath(recipient.instanceId);
|
|
1282
|
+
const response = await sendTelegramBusLocalEnvelope({
|
|
1283
|
+
socketPath: recipientSocketPath,
|
|
1284
|
+
timeoutMs: deps.timeoutMs,
|
|
1285
|
+
retry: getTelegramBusTransportRetryPolicy({
|
|
1286
|
+
endpoint: recipientSocketPath,
|
|
1287
|
+
operation: "operation",
|
|
1288
|
+
}),
|
|
1289
|
+
envelope: {
|
|
1290
|
+
kind: "leader.offerQueueHandoff",
|
|
1291
|
+
requestId: envelope.requestId,
|
|
1292
|
+
auth: envelope.auth,
|
|
1293
|
+
recipientInstanceId: recipient.instanceId,
|
|
1294
|
+
recipientRegistrationGeneration: recipient.registrationGeneration,
|
|
1295
|
+
donorInstanceId: donor.instanceId,
|
|
1296
|
+
donorProcessId: envelope.donorProcessId,
|
|
1297
|
+
donorProcessBirthId: envelope.donorProcessBirthId,
|
|
1298
|
+
donorSessionGeneration: envelope.donorSessionGeneration,
|
|
1299
|
+
donorAcquisitionId: envelope.donorAcquisitionId,
|
|
1300
|
+
donorAcquiredAtMs: envelope.donorAcquiredAtMs,
|
|
1301
|
+
handoffToken: envelope.handoffToken,
|
|
1302
|
+
payload: envelope.payload,
|
|
1303
|
+
sentAtMs: envelope.sentAtMs,
|
|
1304
|
+
},
|
|
1305
|
+
});
|
|
1306
|
+
if (response?.kind === "bus.ack" && response.ok) {
|
|
1307
|
+
deps.followerRegistry.heartbeat(donor.instanceId, getNowMs());
|
|
1308
|
+
deps.followerRegistry.heartbeat(recipient.instanceId, getNowMs());
|
|
1309
|
+
return {
|
|
1310
|
+
kind: "bus.ack",
|
|
1311
|
+
requestId: envelope.requestId,
|
|
1312
|
+
ok: true,
|
|
1313
|
+
...(response.result !== undefined ? { result: response.result } : {}),
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
return {
|
|
1317
|
+
kind: "bus.ack",
|
|
1318
|
+
requestId: envelope.requestId,
|
|
1319
|
+
ok: false,
|
|
1320
|
+
message:
|
|
1321
|
+
response?.kind === "bus.ack"
|
|
1322
|
+
? response.message
|
|
1323
|
+
: "Telegram queue handoff recipient did not acknowledge staging.",
|
|
1324
|
+
};
|
|
1325
|
+
};
|
|
1151
1326
|
const forwardToFollower = async (
|
|
1152
1327
|
envelope: Extract<
|
|
1153
1328
|
TelegramBusEnvelope,
|
|
@@ -1161,11 +1336,30 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1161
1336
|
>,
|
|
1162
1337
|
): Promise<TelegramBusEnvelope> => {
|
|
1163
1338
|
const follower = deps.followerRegistry.get(envelope.recipientInstanceId);
|
|
1339
|
+
if (!follower) {
|
|
1340
|
+
return {
|
|
1341
|
+
kind: "bus.ack",
|
|
1342
|
+
requestId: envelope.requestId,
|
|
1343
|
+
ok: false,
|
|
1344
|
+
message: "Unknown Telegram bus follower instance.",
|
|
1345
|
+
};
|
|
1346
|
+
}
|
|
1347
|
+
if (
|
|
1348
|
+
!follower.registrationGeneration ||
|
|
1349
|
+
envelope.recipientRegistrationGeneration !==
|
|
1350
|
+
follower.registrationGeneration
|
|
1351
|
+
) {
|
|
1352
|
+
return {
|
|
1353
|
+
kind: "bus.ack",
|
|
1354
|
+
requestId: envelope.requestId,
|
|
1355
|
+
ok: false,
|
|
1356
|
+
message: "Stale Telegram bus follower registration generation.",
|
|
1357
|
+
};
|
|
1358
|
+
}
|
|
1164
1359
|
const followerSocketPath =
|
|
1165
|
-
follower
|
|
1360
|
+
follower.busSocketPath ??
|
|
1166
1361
|
getTelegramBusFollowerSocketPath(envelope.recipientInstanceId);
|
|
1167
|
-
|
|
1168
|
-
deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
|
|
1362
|
+
deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
|
|
1169
1363
|
try {
|
|
1170
1364
|
const response = await sendTelegramBusLocalEnvelope({
|
|
1171
1365
|
socketPath: followerSocketPath,
|
|
@@ -1177,9 +1371,13 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1177
1371
|
}),
|
|
1178
1372
|
});
|
|
1179
1373
|
if (response?.kind === "bus.ack" && response.ok) {
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1374
|
+
deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
|
|
1375
|
+
return {
|
|
1376
|
+
kind: "bus.ack",
|
|
1377
|
+
requestId: envelope.requestId,
|
|
1378
|
+
ok: true,
|
|
1379
|
+
...(response.result !== undefined ? { result: response.result } : {}),
|
|
1380
|
+
};
|
|
1183
1381
|
}
|
|
1184
1382
|
const message =
|
|
1185
1383
|
response?.kind === "bus.ack" ? response.message : undefined;
|
|
@@ -1202,14 +1400,34 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1202
1400
|
}
|
|
1203
1401
|
};
|
|
1204
1402
|
return async (envelope) => {
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1403
|
+
const trafficClass = getTelegramBusEnvelopeTrafficClass(envelope);
|
|
1404
|
+
if (trafficClass === "response") {
|
|
1405
|
+
return {
|
|
1406
|
+
kind: "bus.ack",
|
|
1407
|
+
requestId: envelope.requestId,
|
|
1408
|
+
ok: false,
|
|
1409
|
+
message: "Telegram bus response envelope cannot be used as a request.",
|
|
1410
|
+
};
|
|
1411
|
+
}
|
|
1412
|
+
if (!isTelegramBusEnvelopeAuthorized(envelope, deps.authSecret)) {
|
|
1209
1413
|
return createUnauthorizedBusAck(envelope.requestId);
|
|
1210
1414
|
}
|
|
1211
1415
|
switch (envelope.kind) {
|
|
1212
1416
|
case "follower.register": {
|
|
1417
|
+
const compatibility = getTelegramBusProtocolCompatibility({
|
|
1418
|
+
local: deps.protocolIdentity,
|
|
1419
|
+
remote: envelope.registration.protocol,
|
|
1420
|
+
});
|
|
1421
|
+
if (!compatibility.compatible) {
|
|
1422
|
+
return {
|
|
1423
|
+
kind: "bus.ack" as const,
|
|
1424
|
+
requestId: envelope.requestId,
|
|
1425
|
+
ok: false,
|
|
1426
|
+
protocol: deps.protocolIdentity,
|
|
1427
|
+
error: { code: "incompatible-protocol" as const },
|
|
1428
|
+
message: `Incompatible Telegram bus protocol: ${compatibility.reason}.`,
|
|
1429
|
+
};
|
|
1430
|
+
}
|
|
1213
1431
|
return runFollowerMutation(envelope.registration, async () => {
|
|
1214
1432
|
try {
|
|
1215
1433
|
if (!envelope.registration.registrationGeneration) {
|
|
@@ -1253,6 +1471,7 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1253
1471
|
kind: "bus.ack" as const,
|
|
1254
1472
|
requestId: envelope.requestId,
|
|
1255
1473
|
ok: true,
|
|
1474
|
+
protocol: deps.protocolIdentity,
|
|
1256
1475
|
...(registeredTarget ? { result: registeredTarget } : {}),
|
|
1257
1476
|
};
|
|
1258
1477
|
} catch (error) {
|
|
@@ -1260,6 +1479,7 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1260
1479
|
kind: "bus.ack" as const,
|
|
1261
1480
|
requestId: envelope.requestId,
|
|
1262
1481
|
ok: false,
|
|
1482
|
+
protocol: deps.protocolIdentity,
|
|
1263
1483
|
message:
|
|
1264
1484
|
error instanceof Error
|
|
1265
1485
|
? error.message
|
|
@@ -1269,6 +1489,8 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1269
1489
|
},
|
|
1270
1490
|
);
|
|
1271
1491
|
}
|
|
1492
|
+
case "follower.offerQueueHandoff":
|
|
1493
|
+
return routeQueueHandoff(envelope);
|
|
1272
1494
|
case "follower.disconnect": {
|
|
1273
1495
|
const registeredFollower = deps.followerRegistry.get(envelope.instanceId);
|
|
1274
1496
|
return runFollowerMutation(
|
|
@@ -1318,8 +1540,16 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1318
1540
|
}
|
|
1319
1541
|
case "follower.heartbeat": {
|
|
1320
1542
|
const current = deps.followerRegistry.get(envelope.instanceId);
|
|
1543
|
+
if (!current) {
|
|
1544
|
+
return {
|
|
1545
|
+
kind: "bus.ack",
|
|
1546
|
+
requestId: envelope.requestId,
|
|
1547
|
+
ok: false,
|
|
1548
|
+
message: "Unknown Telegram bus follower instance.",
|
|
1549
|
+
};
|
|
1550
|
+
}
|
|
1321
1551
|
if (
|
|
1322
|
-
current
|
|
1552
|
+
!current.registrationGeneration ||
|
|
1323
1553
|
envelope.registrationGeneration !== current.registrationGeneration
|
|
1324
1554
|
) {
|
|
1325
1555
|
return {
|
|
@@ -1341,6 +1571,15 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1341
1571
|
result: {
|
|
1342
1572
|
eligibleElectionSlots: deps.followerRegistry
|
|
1343
1573
|
.list()
|
|
1574
|
+
.filter(
|
|
1575
|
+
(candidate) =>
|
|
1576
|
+
!deps.protocolIdentity.capabilities.includes(
|
|
1577
|
+
TELEGRAM_BUS_CAPABILITY_DURABLE_FOLLOWER_ADMISSION,
|
|
1578
|
+
) ||
|
|
1579
|
+
candidate.protocol?.capabilities.includes(
|
|
1580
|
+
TELEGRAM_BUS_CAPABILITY_DURABLE_FOLLOWER_ADMISSION,
|
|
1581
|
+
),
|
|
1582
|
+
)
|
|
1344
1583
|
.map((candidate) => candidate.slot)
|
|
1345
1584
|
.filter((slot): slot is string =>
|
|
1346
1585
|
typeof slot === "string" && /^[A-Z]$/.test(slot),
|
|
@@ -1468,7 +1707,7 @@ async function handleFollowerApiCall(
|
|
|
1468
1707
|
};
|
|
1469
1708
|
}
|
|
1470
1709
|
if (
|
|
1471
|
-
follower.registrationGeneration
|
|
1710
|
+
!follower.registrationGeneration ||
|
|
1472
1711
|
envelope.registrationGeneration !== follower.registrationGeneration
|
|
1473
1712
|
) {
|
|
1474
1713
|
return {
|
|
@@ -1602,19 +1841,34 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1602
1841
|
const followerStaleAfterMs = deps.followerStaleAfterMs ?? 5000;
|
|
1603
1842
|
const runFollowerMutation = createTelegramBusFollowerMutationRunner();
|
|
1604
1843
|
let pruneInterval: ReturnType<typeof setInterval> | undefined;
|
|
1844
|
+
let pruneGeneration = 0;
|
|
1845
|
+
let prunePromise: Promise<void> | undefined;
|
|
1605
1846
|
const stopPruning = () => {
|
|
1606
|
-
|
|
1607
|
-
clearInterval(pruneInterval);
|
|
1847
|
+
pruneGeneration += 1;
|
|
1848
|
+
if (pruneInterval) clearInterval(pruneInterval);
|
|
1608
1849
|
pruneInterval = undefined;
|
|
1850
|
+
prunePromise = undefined;
|
|
1851
|
+
};
|
|
1852
|
+
const recordPruneEvent = (
|
|
1853
|
+
error: unknown,
|
|
1854
|
+
details: Record<string, unknown>,
|
|
1855
|
+
): void => {
|
|
1856
|
+
try {
|
|
1857
|
+
deps.recordRuntimeEvent?.("bus", error, details);
|
|
1858
|
+
} catch {
|
|
1859
|
+
// Prune diagnostics cannot replace lifecycle-owned reconciliation.
|
|
1860
|
+
}
|
|
1609
1861
|
};
|
|
1610
|
-
const pruneFollowers = async () => {
|
|
1862
|
+
const pruneFollowers = async (expectedGeneration: number) => {
|
|
1863
|
+
const isCurrent = (): boolean => pruneGeneration === expectedGeneration;
|
|
1611
1864
|
try {
|
|
1612
1865
|
await localServer.ensureEndpoint();
|
|
1613
1866
|
} catch (error) {
|
|
1614
|
-
|
|
1867
|
+
recordPruneEvent(error, {
|
|
1615
1868
|
phase: "leader-endpoint-recovery",
|
|
1616
1869
|
});
|
|
1617
1870
|
}
|
|
1871
|
+
if (!isCurrent()) return;
|
|
1618
1872
|
const removed = deps.followerRegistry.pruneStale(
|
|
1619
1873
|
getNowMs(),
|
|
1620
1874
|
followerStaleAfterMs,
|
|
@@ -1625,7 +1879,7 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1625
1879
|
try {
|
|
1626
1880
|
processConfirmedDead = !deps.isFollowerProcessAlive(follower.pid);
|
|
1627
1881
|
} catch (error) {
|
|
1628
|
-
|
|
1882
|
+
recordPruneEvent(error, {
|
|
1629
1883
|
phase: "follower-process-liveness",
|
|
1630
1884
|
instanceId: follower.instanceId,
|
|
1631
1885
|
pid: follower.pid,
|
|
@@ -1633,8 +1887,7 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1633
1887
|
}
|
|
1634
1888
|
}
|
|
1635
1889
|
if (!processConfirmedDead) {
|
|
1636
|
-
|
|
1637
|
-
"bus",
|
|
1890
|
+
recordPruneEvent(
|
|
1638
1891
|
"Telegram bus follower heartbeat stale; preserving thread binding",
|
|
1639
1892
|
{
|
|
1640
1893
|
phase: "follower-pruned",
|
|
@@ -1652,15 +1905,15 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1652
1905
|
cleanupEnabled =
|
|
1653
1906
|
(await deps.shouldCleanupConfirmedDeadFollower?.()) ?? false;
|
|
1654
1907
|
} catch (error) {
|
|
1655
|
-
|
|
1908
|
+
recordPruneEvent(error, {
|
|
1656
1909
|
phase: "follower-confirmed-dead-cleanup-policy",
|
|
1657
1910
|
instanceId: follower.instanceId,
|
|
1658
1911
|
pid: follower.pid,
|
|
1659
1912
|
});
|
|
1660
1913
|
}
|
|
1914
|
+
if (!isCurrent()) return;
|
|
1661
1915
|
if (!cleanupEnabled || !deps.onFollowerConfirmedDead) {
|
|
1662
|
-
|
|
1663
|
-
"bus",
|
|
1916
|
+
recordPruneEvent(
|
|
1664
1917
|
"Telegram bus follower process confirmed dead; preserving thread binding",
|
|
1665
1918
|
{
|
|
1666
1919
|
phase: "follower-confirmed-dead-preserved",
|
|
@@ -1673,14 +1926,14 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1673
1926
|
}
|
|
1674
1927
|
try {
|
|
1675
1928
|
await runFollowerMutation(follower, async () => {
|
|
1929
|
+
if (!isCurrent()) return;
|
|
1676
1930
|
const replacement = deps.followerRegistry.list().find((candidate) =>
|
|
1677
1931
|
follower.profileKey
|
|
1678
1932
|
? candidate.profileKey === follower.profileKey
|
|
1679
1933
|
: candidate.instanceId === follower.instanceId,
|
|
1680
1934
|
);
|
|
1681
1935
|
if (replacement) {
|
|
1682
|
-
|
|
1683
|
-
"bus",
|
|
1936
|
+
recordPruneEvent(
|
|
1684
1937
|
"Telegram bus follower replaced before confirmed-dead cleanup; preserving thread binding",
|
|
1685
1938
|
{
|
|
1686
1939
|
phase: "follower-confirmed-dead-replaced",
|
|
@@ -1693,7 +1946,7 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1693
1946
|
await deps.onFollowerConfirmedDead!(follower);
|
|
1694
1947
|
});
|
|
1695
1948
|
} catch (error) {
|
|
1696
|
-
|
|
1949
|
+
recordPruneEvent(error, {
|
|
1697
1950
|
phase: "follower-confirmed-dead-cleanup",
|
|
1698
1951
|
instanceId: follower.instanceId,
|
|
1699
1952
|
pid: follower.pid,
|
|
@@ -1703,13 +1956,124 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1703
1956
|
}
|
|
1704
1957
|
}
|
|
1705
1958
|
};
|
|
1959
|
+
const requestPrune = (): Promise<void> => {
|
|
1960
|
+
if (prunePromise) return prunePromise;
|
|
1961
|
+
const expectedGeneration = pruneGeneration;
|
|
1962
|
+
let tracked: Promise<void>;
|
|
1963
|
+
tracked = pruneFollowers(expectedGeneration)
|
|
1964
|
+
.catch((error) => {
|
|
1965
|
+
if (pruneGeneration === expectedGeneration) {
|
|
1966
|
+
recordPruneEvent(error, { phase: "follower-prune-owner" });
|
|
1967
|
+
}
|
|
1968
|
+
})
|
|
1969
|
+
.finally(() => {
|
|
1970
|
+
if (prunePromise === tracked) prunePromise = undefined;
|
|
1971
|
+
});
|
|
1972
|
+
prunePromise = tracked;
|
|
1973
|
+
return tracked;
|
|
1974
|
+
};
|
|
1706
1975
|
const startPruning = () => {
|
|
1707
1976
|
stopPruning();
|
|
1708
1977
|
pruneInterval = setInterval(() => {
|
|
1709
|
-
void
|
|
1978
|
+
void requestPrune();
|
|
1710
1979
|
}, followerPruneIntervalMs);
|
|
1711
1980
|
pruneInterval.unref?.();
|
|
1712
1981
|
};
|
|
1982
|
+
const handleEnvelope = createTelegramBusLeaderEnvelopeHandler({
|
|
1983
|
+
followerRegistry: deps.followerRegistry,
|
|
1984
|
+
authSecret: deps.authSecret,
|
|
1985
|
+
protocolIdentity: deps.protocolIdentity,
|
|
1986
|
+
getNowMs,
|
|
1987
|
+
callApi: deps.callApi,
|
|
1988
|
+
authorizeFollowerApiCall: deps.authorizeFollowerApiCall,
|
|
1989
|
+
recordFollowerMessageOwnership: deps.recordFollowerMessageOwnership,
|
|
1990
|
+
resolveAgentTarget: deps.resolveAgentTarget,
|
|
1991
|
+
routeAgentMessage: deps.routeAgentMessage,
|
|
1992
|
+
routeQueueHandoff: deps.routeQueueHandoff,
|
|
1993
|
+
provisionFollowerTarget: deps.provisionFollowerTarget,
|
|
1994
|
+
onFollowerDisconnected: deps.onFollowerDisconnected,
|
|
1995
|
+
getCurrentLeaderEpoch: deps.getCurrentLeaderEpoch,
|
|
1996
|
+
runFollowerMutation,
|
|
1997
|
+
});
|
|
1998
|
+
const routeQueueHandoffEnvelope = async (
|
|
1999
|
+
input: Parameters<TelegramBusLeaderRuntime<TContext>["routeQueueHandoff"]>[0],
|
|
2000
|
+
): Promise<TelegramBusEnvelope> => {
|
|
2001
|
+
const recipient = deps.followerRegistry.get(input.recipientInstanceId);
|
|
2002
|
+
if (
|
|
2003
|
+
!hasTelegramBusCapability(
|
|
2004
|
+
deps.protocolIdentity,
|
|
2005
|
+
TELEGRAM_BUS_CAPABILITY_QUEUE_HANDOFF,
|
|
2006
|
+
) ||
|
|
2007
|
+
!hasTelegramBusCapability(
|
|
2008
|
+
recipient?.protocol,
|
|
2009
|
+
TELEGRAM_BUS_CAPABILITY_QUEUE_HANDOFF,
|
|
2010
|
+
)
|
|
2011
|
+
) {
|
|
2012
|
+
return {
|
|
2013
|
+
kind: "bus.ack",
|
|
2014
|
+
requestId: input.requestId,
|
|
2015
|
+
ok: false,
|
|
2016
|
+
message: "Telegram queue handoff capability was not negotiated.",
|
|
2017
|
+
};
|
|
2018
|
+
}
|
|
2019
|
+
if (
|
|
2020
|
+
!recipient?.registrationGeneration ||
|
|
2021
|
+
input.recipientRegistrationGeneration !==
|
|
2022
|
+
recipient.registrationGeneration
|
|
2023
|
+
) {
|
|
2024
|
+
return {
|
|
2025
|
+
kind: "bus.ack",
|
|
2026
|
+
requestId: input.requestId,
|
|
2027
|
+
ok: false,
|
|
2028
|
+
message: "Stale Telegram queue handoff recipient registration generation.",
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
const recipientSocketPath =
|
|
2032
|
+
recipient.busSocketPath ??
|
|
2033
|
+
getTelegramBusFollowerSocketPath(recipient.instanceId);
|
|
2034
|
+
const response = await sendTelegramBusLocalEnvelope({
|
|
2035
|
+
socketPath: recipientSocketPath,
|
|
2036
|
+
timeoutMs: deps.timeoutMs,
|
|
2037
|
+
retry: getTelegramBusTransportRetryPolicy({
|
|
2038
|
+
endpoint: recipientSocketPath,
|
|
2039
|
+
operation: "operation",
|
|
2040
|
+
}),
|
|
2041
|
+
envelope: {
|
|
2042
|
+
kind: "leader.offerQueueHandoff",
|
|
2043
|
+
requestId: input.requestId,
|
|
2044
|
+
auth: input.auth,
|
|
2045
|
+
recipientInstanceId: recipient.instanceId,
|
|
2046
|
+
recipientRegistrationGeneration: recipient.registrationGeneration,
|
|
2047
|
+
donorInstanceId: input.donorInstanceId,
|
|
2048
|
+
donorProcessId: input.donorProcessId,
|
|
2049
|
+
donorProcessBirthId: input.donorProcessBirthId,
|
|
2050
|
+
donorSessionGeneration: input.donorSessionGeneration,
|
|
2051
|
+
donorAcquisitionId: input.donorAcquisitionId,
|
|
2052
|
+
donorAcquiredAtMs: input.donorAcquiredAtMs,
|
|
2053
|
+
handoffToken: input.handoffToken,
|
|
2054
|
+
payload: input.payload,
|
|
2055
|
+
sentAtMs: input.sentAtMs,
|
|
2056
|
+
},
|
|
2057
|
+
});
|
|
2058
|
+
if (response?.kind === "bus.ack" && response.ok) {
|
|
2059
|
+
deps.followerRegistry.heartbeat(recipient.instanceId, getNowMs());
|
|
2060
|
+
return {
|
|
2061
|
+
kind: "bus.ack",
|
|
2062
|
+
requestId: input.requestId,
|
|
2063
|
+
ok: true,
|
|
2064
|
+
...(response.result !== undefined ? { result: response.result } : {}),
|
|
2065
|
+
};
|
|
2066
|
+
}
|
|
2067
|
+
return {
|
|
2068
|
+
kind: "bus.ack",
|
|
2069
|
+
requestId: input.requestId,
|
|
2070
|
+
ok: false,
|
|
2071
|
+
message:
|
|
2072
|
+
response?.kind === "bus.ack"
|
|
2073
|
+
? response.message
|
|
2074
|
+
: "Telegram queue handoff recipient did not acknowledge staging.",
|
|
2075
|
+
};
|
|
2076
|
+
};
|
|
1713
2077
|
const localServer = createTelegramBusLocalServer({
|
|
1714
2078
|
socketPath: deps.socketPath,
|
|
1715
2079
|
commitEndpointPublication: deps.commitEndpointPublication,
|
|
@@ -1719,22 +2083,11 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1719
2083
|
...details,
|
|
1720
2084
|
});
|
|
1721
2085
|
},
|
|
1722
|
-
handleEnvelope
|
|
1723
|
-
followerRegistry: deps.followerRegistry,
|
|
1724
|
-
authSecret: deps.authSecret,
|
|
1725
|
-
getNowMs,
|
|
1726
|
-
callApi: deps.callApi,
|
|
1727
|
-
authorizeFollowerApiCall: deps.authorizeFollowerApiCall,
|
|
1728
|
-
recordFollowerMessageOwnership: deps.recordFollowerMessageOwnership,
|
|
1729
|
-
resolveAgentTarget: deps.resolveAgentTarget,
|
|
1730
|
-
routeAgentMessage: deps.routeAgentMessage,
|
|
1731
|
-
provisionFollowerTarget: deps.provisionFollowerTarget,
|
|
1732
|
-
onFollowerDisconnected: deps.onFollowerDisconnected,
|
|
1733
|
-
getCurrentLeaderEpoch: deps.getCurrentLeaderEpoch,
|
|
1734
|
-
runFollowerMutation,
|
|
1735
|
-
}),
|
|
2086
|
+
handleEnvelope,
|
|
1736
2087
|
});
|
|
1737
2088
|
return {
|
|
2089
|
+
routeQueueHandoff: (envelope) =>
|
|
2090
|
+
routeQueueHandoffEnvelope(envelope),
|
|
1738
2091
|
startPolling: async (ctx) => {
|
|
1739
2092
|
// Replay durable cleanup before publishing the follower endpoint so a
|
|
1740
2093
|
// replacement registration cannot reclaim a target while it is deleted.
|