@llblab/pi-telegram 0.21.1 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +7 -4
- package/BACKLOG.md +29 -1
- package/CHANGELOG.md +479 -452
- package/docs/architecture.md +12 -10
- package/docs/delivery.md +2 -1
- package/docs/locks.md +13 -5
- package/docs/multi-instance-bus.md +5 -5
- package/index.ts +311 -633
- package/lib/bindings.ts +59 -21
- package/lib/bus-api.ts +12 -2
- package/lib/bus-follower.ts +327 -83
- package/lib/bus-leader.ts +201 -121
- package/lib/bus.ts +345 -37
- package/lib/config.ts +168 -28
- package/lib/delivery.ts +148 -61
- package/lib/lifecycle.ts +199 -8
- package/lib/locks.ts +854 -57
- package/lib/logs.ts +185 -22
- package/lib/media.ts +79 -24
- package/lib/model.ts +5 -10
- package/lib/ownership.ts +150 -6
- package/lib/polling.ts +156 -7
- package/lib/preview.ts +15 -3
- package/lib/queue.ts +167 -20
- package/lib/routing.ts +68 -12
- package/lib/sync.ts +105 -32
- package/lib/telegram-api.ts +86 -10
- package/lib/text-groups.ts +71 -15
- package/lib/thread-reconciler.ts +49 -36
- package/lib/threads.ts +505 -118
- package/package.json +1 -1
package/lib/bus-follower.ts
CHANGED
|
@@ -10,10 +10,17 @@ import { basename } from "node:path";
|
|
|
10
10
|
|
|
11
11
|
import * as Sync from "./sync.ts";
|
|
12
12
|
import * as Threads from "./threads.ts";
|
|
13
|
-
import type { TelegramLockState } from "./locks.ts";
|
|
13
|
+
import type { TelegramLockEntry, TelegramLockState } from "./locks.ts";
|
|
14
14
|
import type { TelegramTarget } from "./target.ts";
|
|
15
15
|
import {
|
|
16
|
+
isTelegramApiMethodRetrySafe,
|
|
17
|
+
TelegramApiCommitUnknownError,
|
|
18
|
+
} from "./telegram-api.ts";
|
|
19
|
+
import {
|
|
20
|
+
createTelegramBusFollowerTargetController,
|
|
21
|
+
createTelegramBusForeignOwnedUpdateForwarder,
|
|
16
22
|
createTelegramBusLocalServer,
|
|
23
|
+
createTelegramBusRequestIdFactory,
|
|
17
24
|
createUnauthorizedBusAck,
|
|
18
25
|
getTelegramBusSocketPath,
|
|
19
26
|
resolveTelegramBusSocketPath,
|
|
@@ -46,8 +53,7 @@ export interface TelegramFollowerSessionHandoff {
|
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
export function getTelegramFollowerSessionHandoff():
|
|
49
|
-
|
|
|
50
|
-
| undefined {
|
|
56
|
+
TelegramFollowerSessionHandoff | undefined {
|
|
51
57
|
const value = (globalThis as Record<string, unknown>)[
|
|
52
58
|
TELEGRAM_FOLLOWER_SESSION_HANDOFF_KEY
|
|
53
59
|
];
|
|
@@ -118,6 +124,7 @@ export interface TelegramBusFollowerSessionRefreshHookDeps<TContext> {
|
|
|
118
124
|
"registerWithLeader" | "setContext"
|
|
119
125
|
>;
|
|
120
126
|
getLeaderState: () => TelegramLockState;
|
|
127
|
+
isSessionActive?: (ctx: TContext) => boolean;
|
|
121
128
|
updateStatus: (ctx: TContext) => void;
|
|
122
129
|
recordRuntimeEvent: (
|
|
123
130
|
category: string,
|
|
@@ -131,10 +138,11 @@ export interface TelegramBusFollowerRegistrationState {
|
|
|
131
138
|
getTarget: () => TelegramTarget | undefined;
|
|
132
139
|
getSlot: () => string | undefined;
|
|
133
140
|
getThreadName: () => string | undefined;
|
|
141
|
+
getGeneration: () => string | undefined;
|
|
134
142
|
setRegistered: (
|
|
135
143
|
registered: boolean,
|
|
136
144
|
target?: TelegramTarget,
|
|
137
|
-
metadata?: { slot?: string; threadName?: string },
|
|
145
|
+
metadata?: { slot?: string; threadName?: string; generation?: string },
|
|
138
146
|
) => void;
|
|
139
147
|
}
|
|
140
148
|
|
|
@@ -143,11 +151,21 @@ export interface TelegramBusForwardedUpdateReceiverRuntime {
|
|
|
143
151
|
stop: () => Promise<void>;
|
|
144
152
|
}
|
|
145
153
|
|
|
154
|
+
export interface TelegramBusFollowerClientRuntimeDeps {
|
|
155
|
+
socketPath: TelegramBusSocketPathSource;
|
|
156
|
+
instanceId: string;
|
|
157
|
+
getApiAuthSecret?: () => string | undefined;
|
|
158
|
+
getForwardingAuthSecret?: () => string | undefined;
|
|
159
|
+
getRegistrationGeneration?: () => string | undefined;
|
|
160
|
+
timeoutMs?: number;
|
|
161
|
+
}
|
|
162
|
+
|
|
146
163
|
export interface TelegramBusFollowerApiCallerDeps {
|
|
147
164
|
socketPath: TelegramBusSocketPathSource;
|
|
148
165
|
instanceId: string;
|
|
149
166
|
createRequestId: () => string;
|
|
150
167
|
getAuthSecret?: () => string | undefined;
|
|
168
|
+
getRegistrationGeneration?: () => string | undefined;
|
|
151
169
|
getNowMs?: () => number;
|
|
152
170
|
timeoutMs?: number;
|
|
153
171
|
}
|
|
@@ -165,6 +183,7 @@ export interface TelegramBusFollowerRegistrationRuntimeDeps<
|
|
|
165
183
|
startReceiving?: () => Promise<void>;
|
|
166
184
|
stopReceiving?: () => Promise<void> | void;
|
|
167
185
|
registrationState?: TelegramBusFollowerRegistrationState;
|
|
186
|
+
isContextActive?: (ctx: TContext) => boolean;
|
|
168
187
|
getProfileKey?: (ctx: TContext) => string | undefined;
|
|
169
188
|
getThreadName?: (ctx: TContext) => string | undefined;
|
|
170
189
|
getNowMs?: () => number;
|
|
@@ -194,10 +213,15 @@ export function createTelegramManualFollowerProfileKeyResolver(input: {
|
|
|
194
213
|
});
|
|
195
214
|
}
|
|
196
215
|
|
|
216
|
+
export interface TelegramBusFollowerElection {
|
|
217
|
+
expectedOwner?: TelegramLockEntry;
|
|
218
|
+
}
|
|
219
|
+
|
|
197
220
|
export type TelegramBusFollowerPromotionHandler<TContext> = (
|
|
198
221
|
ctx: TContext,
|
|
199
222
|
binding: TelegramBusFollowerPromotedBinding,
|
|
200
|
-
|
|
223
|
+
election: TelegramBusFollowerElection,
|
|
224
|
+
) => Promise<boolean>;
|
|
201
225
|
|
|
202
226
|
export function createTelegramBusFollowerPromotionHandler<
|
|
203
227
|
TContext extends { cwd: string },
|
|
@@ -205,38 +229,43 @@ export function createTelegramBusFollowerPromotionHandler<
|
|
|
205
229
|
topicTargetStore: Threads.TelegramTopicTargetStore;
|
|
206
230
|
instanceId: string;
|
|
207
231
|
getActiveProfileName: () => string | undefined;
|
|
208
|
-
startLeader: (
|
|
232
|
+
startLeader: (
|
|
233
|
+
ctx: TContext,
|
|
234
|
+
election: TelegramBusFollowerElection,
|
|
235
|
+
onAcquired: () => Promise<void>,
|
|
236
|
+
) => Promise<boolean>;
|
|
209
237
|
recordRuntimeEvent: (
|
|
210
238
|
category: string,
|
|
211
239
|
error: unknown,
|
|
212
240
|
details?: Record<string, unknown>,
|
|
213
241
|
) => void;
|
|
214
242
|
}): TelegramBusFollowerPromotionHandler<TContext> {
|
|
215
|
-
return async (ctx, binding) =>
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
243
|
+
return async (ctx, binding, election) =>
|
|
244
|
+
input.startLeader(ctx, election, async () => {
|
|
245
|
+
const promotedRecord =
|
|
246
|
+
await Threads.promoteTelegramFollowerBindingToLeader({
|
|
247
|
+
store: input.topicTargetStore,
|
|
248
|
+
instanceId: input.instanceId,
|
|
249
|
+
cwd: ctx.cwd,
|
|
250
|
+
telegramProfile: input.getActiveProfileName(),
|
|
251
|
+
target: binding.target,
|
|
252
|
+
slot: binding.slot,
|
|
253
|
+
threadName: binding.threadName,
|
|
254
|
+
});
|
|
255
|
+
if (promotedRecord) {
|
|
256
|
+
input.recordRuntimeEvent(
|
|
257
|
+
"bus",
|
|
258
|
+
"Follower thread binding promoted to leader",
|
|
259
|
+
{
|
|
260
|
+
phase: "follower-promoted-binding",
|
|
261
|
+
chatId: promotedRecord.target.chatId,
|
|
262
|
+
threadId: promotedRecord.target.threadId,
|
|
263
|
+
slot: promotedRecord.slot,
|
|
264
|
+
threadName: promotedRecord.threadName,
|
|
265
|
+
},
|
|
266
|
+
);
|
|
267
|
+
}
|
|
224
268
|
});
|
|
225
|
-
if (promotedRecord) {
|
|
226
|
-
input.recordRuntimeEvent(
|
|
227
|
-
"bus",
|
|
228
|
-
"Follower thread binding promoted to leader",
|
|
229
|
-
{
|
|
230
|
-
phase: "follower-promoted-binding",
|
|
231
|
-
chatId: promotedRecord.target.chatId,
|
|
232
|
-
threadId: promotedRecord.target.threadId,
|
|
233
|
-
slot: promotedRecord.slot,
|
|
234
|
-
threadName: promotedRecord.threadName,
|
|
235
|
-
},
|
|
236
|
-
);
|
|
237
|
-
}
|
|
238
|
-
await input.startLeader(ctx);
|
|
239
|
-
};
|
|
240
269
|
}
|
|
241
270
|
|
|
242
271
|
export interface TelegramBusFollowerTargetReplacementHandlerDeps<TContext> {
|
|
@@ -247,7 +276,8 @@ export interface TelegramBusFollowerTargetReplacementHandlerDeps<TContext> {
|
|
|
247
276
|
registrationState: Pick<
|
|
248
277
|
TelegramBusFollowerRegistrationState,
|
|
249
278
|
"getTarget" | "setRegistered"
|
|
250
|
-
|
|
279
|
+
> &
|
|
280
|
+
Partial<Pick<TelegramBusFollowerRegistrationState, "getGeneration">>;
|
|
251
281
|
instanceId: string;
|
|
252
282
|
getManualFollowerProfileKey: () => string;
|
|
253
283
|
manualFollowerOwnerId: string;
|
|
@@ -268,10 +298,7 @@ export type TelegramBusFollowerLeaderState =
|
|
|
268
298
|
| { kind: "active-elsewhere"; lock: TelegramBusFollowerLeaderLock }
|
|
269
299
|
| { kind: "stale"; lock: TelegramBusFollowerLeaderLock };
|
|
270
300
|
|
|
271
|
-
export
|
|
272
|
-
busSocketPath?: string;
|
|
273
|
-
busSecret?: string;
|
|
274
|
-
}
|
|
301
|
+
export type TelegramBusFollowerLeaderLock = TelegramLockEntry;
|
|
275
302
|
|
|
276
303
|
export interface TelegramBusFollowerPromotedBinding {
|
|
277
304
|
target?: TelegramTarget;
|
|
@@ -291,9 +318,12 @@ export interface TelegramBusFollowerHeartbeatRecoveryHandlerDeps<TContext> {
|
|
|
291
318
|
promoteToLeader: (
|
|
292
319
|
ctx: TContext,
|
|
293
320
|
binding: TelegramBusFollowerPromotedBinding,
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
321
|
+
election: TelegramBusFollowerElection,
|
|
322
|
+
) => Promise<boolean>;
|
|
323
|
+
sleep?: (ms: number) => Promise<void>;
|
|
324
|
+
scheduleRetry?: (retry: () => void, delayMs: number) => void;
|
|
325
|
+
getActiveContext?: () => TContext | undefined;
|
|
326
|
+
promotionGraceMs?: number;
|
|
297
327
|
recordRuntimeEvent: (
|
|
298
328
|
category: string,
|
|
299
329
|
error: unknown,
|
|
@@ -310,6 +340,7 @@ export interface TelegramBusForwardedUpdateReceiverRuntimeDeps<
|
|
|
310
340
|
socketPath: TelegramBusSocketPathSource;
|
|
311
341
|
instanceId: string;
|
|
312
342
|
getAuthSecret?: () => string | undefined;
|
|
343
|
+
getRegistrationGeneration?: () => string | undefined;
|
|
313
344
|
getContext: () => TContext | undefined;
|
|
314
345
|
handleForwardedCallback: (
|
|
315
346
|
query: TCallbackQuery,
|
|
@@ -373,6 +404,47 @@ export interface TelegramBusFollowerRuntimeAssembly<TContext> {
|
|
|
373
404
|
registration: TelegramBusFollowerRegistrationRuntime<TContext>;
|
|
374
405
|
}
|
|
375
406
|
|
|
407
|
+
export function createTelegramBusForwardedRouteHandlers<
|
|
408
|
+
TContext,
|
|
409
|
+
TReactionUpdate,
|
|
410
|
+
TCallbackQuery,
|
|
411
|
+
TMessage,
|
|
412
|
+
>(route: {
|
|
413
|
+
handleUpdate(
|
|
414
|
+
update: {
|
|
415
|
+
callback_query?: TCallbackQuery;
|
|
416
|
+
message?: TMessage;
|
|
417
|
+
edited_message?: TMessage;
|
|
418
|
+
},
|
|
419
|
+
ctx: TContext,
|
|
420
|
+
): Promise<void> | void;
|
|
421
|
+
handleAuthorizedReactionUpdate(
|
|
422
|
+
reactionUpdate: TReactionUpdate,
|
|
423
|
+
ctx: TContext,
|
|
424
|
+
): Promise<void> | void;
|
|
425
|
+
}): Pick<
|
|
426
|
+
TelegramBusForwardedUpdateReceiverRuntimeDeps<
|
|
427
|
+
TContext,
|
|
428
|
+
TReactionUpdate,
|
|
429
|
+
TCallbackQuery,
|
|
430
|
+
TMessage
|
|
431
|
+
>,
|
|
432
|
+
| "handleForwardedCallback"
|
|
433
|
+
| "handleForwardedReaction"
|
|
434
|
+
| "handleForwardedMessage"
|
|
435
|
+
| "handleForwardedEditedMessage"
|
|
436
|
+
> {
|
|
437
|
+
return {
|
|
438
|
+
handleForwardedCallback: (query, ctx) =>
|
|
439
|
+
route.handleUpdate({ callback_query: query }, ctx),
|
|
440
|
+
handleForwardedReaction: route.handleAuthorizedReactionUpdate,
|
|
441
|
+
handleForwardedMessage: (message, ctx) =>
|
|
442
|
+
route.handleUpdate({ message }, ctx),
|
|
443
|
+
handleForwardedEditedMessage: (message, ctx) =>
|
|
444
|
+
route.handleUpdate({ edited_message: message }, ctx),
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
|
|
376
448
|
export function createTelegramBusFollowerRuntimeAssembly<
|
|
377
449
|
TContext extends { cwd?: string },
|
|
378
450
|
TReactionUpdate,
|
|
@@ -388,10 +460,11 @@ export function createTelegramBusFollowerRuntimeAssembly<
|
|
|
388
460
|
): TelegramBusFollowerRuntimeAssembly<TContext> {
|
|
389
461
|
const receiver = createTelegramBusForwardedUpdateReceiverRuntime({
|
|
390
462
|
...deps.receiver,
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
463
|
+
getRegistrationGeneration:
|
|
464
|
+
deps.targetReplacement.registrationState.getGeneration,
|
|
465
|
+
handleReplaceTarget: createTelegramBusFollowerTargetReplacementHandler(
|
|
466
|
+
deps.targetReplacement,
|
|
467
|
+
),
|
|
395
468
|
});
|
|
396
469
|
let registration: TelegramBusFollowerRegistrationRuntime<TContext>;
|
|
397
470
|
const recovery = createTelegramBusFollowerHeartbeatRecoveryHandler({
|
|
@@ -458,7 +531,11 @@ export function createTelegramBusFollowerTargetReplacementHandler<TContext>(
|
|
|
458
531
|
threadName: currentRecord?.threadName,
|
|
459
532
|
rerouteConfirmedAtMs: nowMs,
|
|
460
533
|
});
|
|
461
|
-
deps.registrationState.setRegistered(true, input.target
|
|
534
|
+
deps.registrationState.setRegistered(true, input.target, {
|
|
535
|
+
slot: currentRecord?.slot,
|
|
536
|
+
threadName: currentRecord?.threadName,
|
|
537
|
+
generation: deps.registrationState.getGeneration?.(),
|
|
538
|
+
});
|
|
462
539
|
deps.setSyncState(
|
|
463
540
|
Sync.markTelegramSyncSliceFresh(deps.getSyncState(), "target-bindings", {
|
|
464
541
|
nowMs,
|
|
@@ -482,6 +559,42 @@ export function createTelegramBusFollowerTargetReplacementHandler<TContext>(
|
|
|
482
559
|
};
|
|
483
560
|
}
|
|
484
561
|
|
|
562
|
+
export function createTelegramBusFollowerClientRuntime<
|
|
563
|
+
TContext,
|
|
564
|
+
TReactionUpdate,
|
|
565
|
+
TCallbackQuery,
|
|
566
|
+
TMessage = unknown,
|
|
567
|
+
>(deps: TelegramBusFollowerClientRuntimeDeps) {
|
|
568
|
+
const createRequestId = createTelegramBusRequestIdFactory(deps.instanceId);
|
|
569
|
+
const sharedClientDeps = {
|
|
570
|
+
socketPath: deps.socketPath,
|
|
571
|
+
createRequestId,
|
|
572
|
+
timeoutMs: deps.timeoutMs,
|
|
573
|
+
};
|
|
574
|
+
return {
|
|
575
|
+
createRequestId,
|
|
576
|
+
callApi: createTelegramBusFollowerApiCaller({
|
|
577
|
+
...sharedClientDeps,
|
|
578
|
+
instanceId: deps.instanceId,
|
|
579
|
+
getAuthSecret: deps.getApiAuthSecret,
|
|
580
|
+
getRegistrationGeneration: deps.getRegistrationGeneration,
|
|
581
|
+
}),
|
|
582
|
+
foreignOwnedUpdateForwarder: createTelegramBusForeignOwnedUpdateForwarder<
|
|
583
|
+
TContext,
|
|
584
|
+
TReactionUpdate,
|
|
585
|
+
TCallbackQuery,
|
|
586
|
+
TMessage
|
|
587
|
+
>({
|
|
588
|
+
...sharedClientDeps,
|
|
589
|
+
getAuthSecret: deps.getForwardingAuthSecret,
|
|
590
|
+
}),
|
|
591
|
+
targetController: createTelegramBusFollowerTargetController({
|
|
592
|
+
...sharedClientDeps,
|
|
593
|
+
getAuthSecret: deps.getForwardingAuthSecret,
|
|
594
|
+
}),
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
|
|
485
598
|
export function createTelegramBusFollowerApiCaller(
|
|
486
599
|
deps: TelegramBusFollowerApiCallerDeps,
|
|
487
600
|
): (method: string, args: unknown[]) => Promise<unknown> {
|
|
@@ -489,28 +602,55 @@ export function createTelegramBusFollowerApiCaller(
|
|
|
489
602
|
const timeoutMs = deps.timeoutMs ?? 30000;
|
|
490
603
|
return async (method, args) => {
|
|
491
604
|
const socketPath = resolveTelegramBusSocketPath(deps.socketPath);
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
605
|
+
let response: TelegramBusEnvelope | undefined;
|
|
606
|
+
try {
|
|
607
|
+
response = await sendTelegramBusLocalEnvelope({
|
|
608
|
+
socketPath,
|
|
609
|
+
timeoutMs,
|
|
610
|
+
retry: getTelegramBusTransportRetryPolicy({
|
|
611
|
+
endpoint: socketPath,
|
|
612
|
+
operation: "operation",
|
|
613
|
+
}),
|
|
614
|
+
envelope: {
|
|
615
|
+
kind: "follower.callApi",
|
|
616
|
+
requestId: deps.createRequestId(),
|
|
617
|
+
auth: deps.getAuthSecret?.(),
|
|
618
|
+
instanceId: deps.instanceId,
|
|
619
|
+
...(deps.getRegistrationGeneration?.()
|
|
620
|
+
? {
|
|
621
|
+
registrationGeneration: deps.getRegistrationGeneration?.(),
|
|
622
|
+
}
|
|
623
|
+
: {}),
|
|
624
|
+
method,
|
|
625
|
+
args,
|
|
626
|
+
sentAtMs: getNowMs(),
|
|
627
|
+
},
|
|
628
|
+
});
|
|
629
|
+
} catch (error) {
|
|
630
|
+
const apiMethod =
|
|
631
|
+
(method === "call" || method === "callMultipart") &&
|
|
632
|
+
typeof args[0] === "string"
|
|
633
|
+
? args[0]
|
|
634
|
+
: method;
|
|
635
|
+
if (!isTelegramApiMethodRetrySafe(apiMethod)) {
|
|
636
|
+
throw new TelegramApiCommitUnknownError(apiMethod, error);
|
|
637
|
+
}
|
|
638
|
+
throw error;
|
|
639
|
+
}
|
|
509
640
|
if (response?.kind === "bus.ack" && response.ok) return response.result;
|
|
510
641
|
const message =
|
|
511
642
|
response?.kind === "bus.ack"
|
|
512
643
|
? response.message
|
|
513
644
|
: "Telegram bus API call did not return an acknowledgement.";
|
|
645
|
+
if (
|
|
646
|
+
response?.kind === "bus.ack" &&
|
|
647
|
+
response.error?.code === "commit-unknown"
|
|
648
|
+
) {
|
|
649
|
+
throw new TelegramApiCommitUnknownError(
|
|
650
|
+
response.error.method ?? method,
|
|
651
|
+
new Error(message ?? "Telegram bus API call result is ambiguous."),
|
|
652
|
+
);
|
|
653
|
+
}
|
|
514
654
|
throw new Error(message ?? "Telegram bus API call failed.");
|
|
515
655
|
};
|
|
516
656
|
}
|
|
@@ -558,6 +698,7 @@ export function createTelegramBusFollowerSessionRefreshHook<TContext>(
|
|
|
558
698
|
deps: TelegramBusFollowerSessionRefreshHookDeps<TContext>,
|
|
559
699
|
): (_event: unknown, ctx: TContext) => Promise<void> {
|
|
560
700
|
return async (_event, ctx) => {
|
|
701
|
+
if (deps.isSessionActive && !deps.isSessionActive(ctx)) return;
|
|
561
702
|
if (!deps.registrationState.isRegistered()) {
|
|
562
703
|
const handoff = getTelegramFollowerSessionHandoff();
|
|
563
704
|
const lockState = deps.getLeaderState();
|
|
@@ -569,6 +710,7 @@ export function createTelegramBusFollowerSessionRefreshHook<TContext>(
|
|
|
569
710
|
lockState.lock,
|
|
570
711
|
{ target: handoff.target },
|
|
571
712
|
);
|
|
713
|
+
if (deps.isSessionActive && !deps.isSessionActive(ctx)) return;
|
|
572
714
|
if (restored) {
|
|
573
715
|
setTelegramFollowerSessionHandoff(undefined);
|
|
574
716
|
deps.updateStatus(ctx);
|
|
@@ -592,6 +734,7 @@ export function createTelegramBusFollowerSessionRefreshHook<TContext>(
|
|
|
592
734
|
}
|
|
593
735
|
}
|
|
594
736
|
if (!deps.registrationState.isRegistered()) return;
|
|
737
|
+
if (deps.isSessionActive && !deps.isSessionActive(ctx)) return;
|
|
595
738
|
deps.registrationRuntime.setContext(ctx);
|
|
596
739
|
deps.updateStatus(ctx);
|
|
597
740
|
deps.recordRuntimeEvent(
|
|
@@ -607,16 +750,19 @@ export function createTelegramBusFollowerRegistrationState(): TelegramBusFollowe
|
|
|
607
750
|
let target: TelegramTarget | undefined;
|
|
608
751
|
let slot: string | undefined;
|
|
609
752
|
let threadName: string | undefined;
|
|
753
|
+
let generation: string | undefined;
|
|
610
754
|
return {
|
|
611
755
|
isRegistered: () => registered,
|
|
612
756
|
getTarget: () => (target ? { ...target } : undefined),
|
|
613
757
|
getSlot: () => slot,
|
|
614
758
|
getThreadName: () => threadName,
|
|
759
|
+
getGeneration: () => generation,
|
|
615
760
|
setRegistered: (next, nextTarget, metadata) => {
|
|
616
761
|
registered = next;
|
|
617
762
|
target = next ? (nextTarget ? { ...nextTarget } : undefined) : undefined;
|
|
618
763
|
slot = next ? metadata?.slot : undefined;
|
|
619
764
|
threadName = next ? metadata?.threadName : undefined;
|
|
765
|
+
generation = next ? metadata?.generation : undefined;
|
|
620
766
|
},
|
|
621
767
|
};
|
|
622
768
|
}
|
|
@@ -624,6 +770,17 @@ export function createTelegramBusFollowerRegistrationState(): TelegramBusFollowe
|
|
|
624
770
|
export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
625
771
|
deps: TelegramBusFollowerHeartbeatRecoveryHandlerDeps<TContext>,
|
|
626
772
|
): (error: unknown, ctx: TContext) => Promise<void> {
|
|
773
|
+
const promotionGraceMs =
|
|
774
|
+
deps.promotionGraceMs ?? TELEGRAM_BUS_FOLLOWER_PROMOTION_GRACE_MS;
|
|
775
|
+
const sleep =
|
|
776
|
+
deps.sleep ??
|
|
777
|
+
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
778
|
+
const scheduleRetry =
|
|
779
|
+
deps.scheduleRetry ??
|
|
780
|
+
((retry: () => void, delayMs: number) => {
|
|
781
|
+
const timer = setTimeout(retry, delayMs);
|
|
782
|
+
timer.unref?.();
|
|
783
|
+
});
|
|
627
784
|
let promotionPending = false;
|
|
628
785
|
const safeUpdateStatus = (ctx: TContext) => {
|
|
629
786
|
try {
|
|
@@ -643,11 +800,16 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
643
800
|
ctx: TContext,
|
|
644
801
|
leader: TelegramBusFollowerLeaderLock,
|
|
645
802
|
phase: string,
|
|
803
|
+
binding?: TelegramBusFollowerPromotedBinding,
|
|
646
804
|
) => {
|
|
647
805
|
try {
|
|
648
806
|
const restored = await deps
|
|
649
807
|
.getRegistrationRuntime()
|
|
650
|
-
.registerWithLeader(
|
|
808
|
+
.registerWithLeader(
|
|
809
|
+
ctx,
|
|
810
|
+
leader,
|
|
811
|
+
binding?.target ? { target: binding.target } : undefined,
|
|
812
|
+
);
|
|
651
813
|
if (!restored) return false;
|
|
652
814
|
deps.setLifecyclePhase(undefined);
|
|
653
815
|
safeUpdateStatus(ctx);
|
|
@@ -670,11 +832,34 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
670
832
|
slot: deps.registrationState.getSlot(),
|
|
671
833
|
threadName: deps.registrationState.getThreadName(),
|
|
672
834
|
});
|
|
835
|
+
const scheduleRecovery = (
|
|
836
|
+
reason: unknown,
|
|
837
|
+
fallbackCtx: TContext,
|
|
838
|
+
binding: TelegramBusFollowerPromotedBinding,
|
|
839
|
+
) => {
|
|
840
|
+
const retry = () => {
|
|
841
|
+
const activeCtx = deps.getActiveContext
|
|
842
|
+
? deps.getActiveContext()
|
|
843
|
+
: fallbackCtx;
|
|
844
|
+
if (!activeCtx) {
|
|
845
|
+
scheduleRetry(retry, promotionGraceMs);
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
void recover(reason, activeCtx, binding);
|
|
849
|
+
};
|
|
850
|
+
scheduleRetry(retry, promotionGraceMs);
|
|
851
|
+
};
|
|
673
852
|
const promoteToLeader = async (
|
|
674
853
|
reason: unknown,
|
|
675
854
|
ctx: TContext,
|
|
676
|
-
binding
|
|
855
|
+
binding: TelegramBusFollowerPromotedBinding,
|
|
856
|
+
election: TelegramBusFollowerElection,
|
|
677
857
|
) => {
|
|
858
|
+
const activeCtx = deps.getActiveContext?.();
|
|
859
|
+
if (deps.getActiveContext && activeCtx !== ctx) {
|
|
860
|
+
scheduleRecovery(reason, ctx, binding);
|
|
861
|
+
return;
|
|
862
|
+
}
|
|
678
863
|
deps.setLifecyclePhase("electing");
|
|
679
864
|
safeUpdateStatus(ctx);
|
|
680
865
|
deps.recordRuntimeEvent("bus", reason, {
|
|
@@ -686,18 +871,31 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
686
871
|
deps.recordRuntimeEvent("bus", "Telegram follower elected for promotion", {
|
|
687
872
|
phase: "follower-promotion-electing",
|
|
688
873
|
});
|
|
689
|
-
await deps.promoteToLeader(ctx, binding);
|
|
874
|
+
const promoted = await deps.promoteToLeader(ctx, binding, election);
|
|
690
875
|
deps.setLifecyclePhase(undefined);
|
|
691
876
|
safeUpdateStatus(ctx);
|
|
692
|
-
deps.recordRuntimeEvent(
|
|
693
|
-
|
|
694
|
-
|
|
877
|
+
deps.recordRuntimeEvent(
|
|
878
|
+
"bus",
|
|
879
|
+
promoted
|
|
880
|
+
? "Telegram follower promotion completed"
|
|
881
|
+
: "Telegram follower promotion lost election",
|
|
882
|
+
{
|
|
883
|
+
phase: promoted
|
|
884
|
+
? "follower-promotion-complete"
|
|
885
|
+
: "follower-promotion-lost",
|
|
886
|
+
},
|
|
887
|
+
);
|
|
888
|
+
if (!promoted) scheduleRecovery(reason, ctx, binding);
|
|
695
889
|
};
|
|
696
|
-
|
|
890
|
+
const recover = async (
|
|
891
|
+
error: unknown,
|
|
892
|
+
ctx: TContext,
|
|
893
|
+
carriedBinding?: TelegramBusFollowerPromotedBinding,
|
|
894
|
+
): Promise<void> => {
|
|
697
895
|
if (promotionPending) return;
|
|
698
896
|
promotionPending = true;
|
|
699
897
|
try {
|
|
700
|
-
const initialBinding = snapshotBinding();
|
|
898
|
+
const initialBinding = carriedBinding ?? snapshotBinding();
|
|
701
899
|
const state = deps.getLeaderState();
|
|
702
900
|
if (state.kind === "active-elsewhere") {
|
|
703
901
|
clearRegisteredState(ctx);
|
|
@@ -706,6 +904,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
706
904
|
ctx,
|
|
707
905
|
state.lock,
|
|
708
906
|
"follower-register-restore",
|
|
907
|
+
initialBinding,
|
|
709
908
|
)
|
|
710
909
|
) {
|
|
711
910
|
return;
|
|
@@ -717,7 +916,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
717
916
|
"Telegram follower waiting for leader reload recovery",
|
|
718
917
|
{ phase: "follower-promotion-grace" },
|
|
719
918
|
);
|
|
720
|
-
await
|
|
919
|
+
await sleep(promotionGraceMs);
|
|
721
920
|
const graceState = deps.getLeaderState();
|
|
722
921
|
if (graceState.kind === "active-elsewhere") {
|
|
723
922
|
if (
|
|
@@ -725,20 +924,37 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
725
924
|
ctx,
|
|
726
925
|
graceState.lock,
|
|
727
926
|
"follower-register-restore-grace",
|
|
927
|
+
initialBinding,
|
|
728
928
|
)
|
|
729
929
|
) {
|
|
730
930
|
return;
|
|
731
931
|
}
|
|
732
|
-
|
|
932
|
+
deps.setLifecyclePhase(undefined);
|
|
933
|
+
safeUpdateStatus(ctx);
|
|
934
|
+
deps.recordRuntimeEvent(
|
|
935
|
+
"bus",
|
|
936
|
+
"Telegram follower promotion blocked by live leader lease",
|
|
937
|
+
{
|
|
938
|
+
phase: "follower-promotion-live-owner",
|
|
939
|
+
leaderInstanceId: graceState.lock.instanceId,
|
|
940
|
+
leaderEpoch: graceState.lock.leaderEpoch,
|
|
941
|
+
},
|
|
942
|
+
);
|
|
943
|
+
scheduleRecovery(error, ctx, initialBinding);
|
|
733
944
|
return;
|
|
734
945
|
}
|
|
735
946
|
if (graceState.kind === "stale" || graceState.kind === "inactive") {
|
|
736
|
-
await promoteToLeader(error, ctx, initialBinding
|
|
947
|
+
await promoteToLeader(error, ctx, initialBinding, {
|
|
948
|
+
expectedOwner:
|
|
949
|
+
graceState.kind === "stale" ? graceState.lock : undefined,
|
|
950
|
+
});
|
|
737
951
|
}
|
|
738
952
|
return;
|
|
739
953
|
}
|
|
740
954
|
if (state.kind === "stale" || state.kind === "inactive") {
|
|
741
|
-
await promoteToLeader(error, ctx, initialBinding
|
|
955
|
+
await promoteToLeader(error, ctx, initialBinding, {
|
|
956
|
+
expectedOwner: state.kind === "stale" ? state.lock : undefined,
|
|
957
|
+
});
|
|
742
958
|
}
|
|
743
959
|
} catch (promotionError) {
|
|
744
960
|
deps.setLifecyclePhase(undefined);
|
|
@@ -754,6 +970,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
754
970
|
promotionPending = false;
|
|
755
971
|
}
|
|
756
972
|
};
|
|
973
|
+
return recover;
|
|
757
974
|
}
|
|
758
975
|
|
|
759
976
|
export function createTelegramBusFollowerRegistrationRuntime<
|
|
@@ -775,6 +992,7 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
775
992
|
let heartbeatInterval: ReturnType<typeof setInterval> | undefined;
|
|
776
993
|
let activeLeaderSocketPath: string | undefined;
|
|
777
994
|
let activeAuthSecret: string | undefined;
|
|
995
|
+
let activeRegistrationGeneration: string | undefined;
|
|
778
996
|
let activeContext: TContext | undefined;
|
|
779
997
|
let lastKnownTarget: TelegramTarget | undefined;
|
|
780
998
|
let lastKnownSlot: string | undefined;
|
|
@@ -787,6 +1005,7 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
787
1005
|
const stop = () => {
|
|
788
1006
|
stopHeartbeat();
|
|
789
1007
|
activeAuthSecret = undefined;
|
|
1008
|
+
activeRegistrationGeneration = undefined;
|
|
790
1009
|
deps.setActiveAuthSecret?.(undefined);
|
|
791
1010
|
deps.registrationState?.setRegistered(false);
|
|
792
1011
|
lastKnownTarget = undefined;
|
|
@@ -810,6 +1029,9 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
810
1029
|
requestId: deps.createRequestId(),
|
|
811
1030
|
auth: activeAuthSecret,
|
|
812
1031
|
instanceId: deps.instanceId,
|
|
1032
|
+
...(activeRegistrationGeneration
|
|
1033
|
+
? { registrationGeneration: activeRegistrationGeneration }
|
|
1034
|
+
: {}),
|
|
813
1035
|
sentAtMs: getNowMs(),
|
|
814
1036
|
},
|
|
815
1037
|
});
|
|
@@ -840,12 +1062,13 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
840
1062
|
await deps.startReceiving?.();
|
|
841
1063
|
activeAuthSecret = deps.getLeaderAuthSecret?.(leader);
|
|
842
1064
|
deps.setActiveAuthSecret?.(activeAuthSecret);
|
|
843
|
-
const
|
|
1065
|
+
const registrationGeneration = deps.createRequestId();
|
|
1066
|
+
const registrationEnvelope: Extract<
|
|
844
1067
|
TelegramBusEnvelope,
|
|
845
1068
|
{ kind: "follower.register" }
|
|
846
|
-
>
|
|
1069
|
+
> = {
|
|
847
1070
|
kind: "follower.register",
|
|
848
|
-
requestId:
|
|
1071
|
+
requestId: registrationGeneration,
|
|
849
1072
|
auth: activeAuthSecret,
|
|
850
1073
|
registration: {
|
|
851
1074
|
instanceId: deps.instanceId,
|
|
@@ -857,7 +1080,7 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
857
1080
|
lastKnownThreadName ??
|
|
858
1081
|
deps.getThreadName?.(ctx) ??
|
|
859
1082
|
(ctx.cwd ? basename(ctx.cwd) : undefined),
|
|
860
|
-
...(deps.registrationState?.getSlot() ?? lastKnownSlot
|
|
1083
|
+
...((deps.registrationState?.getSlot() ?? lastKnownSlot)
|
|
861
1084
|
? { slot: deps.registrationState?.getSlot() ?? lastKnownSlot }
|
|
862
1085
|
: {}),
|
|
863
1086
|
cwd: ctx.cwd,
|
|
@@ -868,15 +1091,16 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
868
1091
|
lastKnownTarget,
|
|
869
1092
|
busSocketPath:
|
|
870
1093
|
deps.getFollowerBusSocketPath?.() ?? deps.followerBusSocketPath,
|
|
1094
|
+
registrationGeneration,
|
|
871
1095
|
connectedAtMs: getNowMs(),
|
|
872
1096
|
},
|
|
873
|
-
}
|
|
1097
|
+
};
|
|
874
1098
|
let response: TelegramBusEnvelope | undefined;
|
|
875
1099
|
try {
|
|
876
1100
|
response = await sendTelegramBusLocalEnvelope({
|
|
877
1101
|
socketPath: leaderSocketPath,
|
|
878
1102
|
timeoutMs: registrationTimeoutMs,
|
|
879
|
-
envelope:
|
|
1103
|
+
envelope: registrationEnvelope,
|
|
880
1104
|
retry: getTelegramBusTransportRetryPolicy({
|
|
881
1105
|
endpoint: leaderSocketPath,
|
|
882
1106
|
operation: "registration",
|
|
@@ -901,6 +1125,14 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
901
1125
|
await deps.stopReceiving?.();
|
|
902
1126
|
throw error;
|
|
903
1127
|
}
|
|
1128
|
+
if (deps.isContextActive && !deps.isContextActive(ctx)) {
|
|
1129
|
+
stopHeartbeat();
|
|
1130
|
+
activeLeaderSocketPath = undefined;
|
|
1131
|
+
activeAuthSecret = undefined;
|
|
1132
|
+
deps.setActiveAuthSecret?.(undefined);
|
|
1133
|
+
await deps.stopReceiving?.();
|
|
1134
|
+
return false;
|
|
1135
|
+
}
|
|
904
1136
|
if (response?.kind === "bus.ack" && !response.ok) {
|
|
905
1137
|
stopHeartbeat();
|
|
906
1138
|
activeLeaderSocketPath = undefined;
|
|
@@ -915,15 +1147,15 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
915
1147
|
}
|
|
916
1148
|
if (response?.kind === "bus.ack" && response.ok) {
|
|
917
1149
|
const registrationResult = parseRegistrationResult(response.result);
|
|
918
|
-
deps.registrationState?.setRegistered(
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
);
|
|
1150
|
+
deps.registrationState?.setRegistered(true, registrationResult.target, {
|
|
1151
|
+
...registrationResult,
|
|
1152
|
+
generation: registrationGeneration,
|
|
1153
|
+
});
|
|
923
1154
|
lastKnownTarget = registrationResult.target;
|
|
924
1155
|
lastKnownSlot = registrationResult.slot;
|
|
925
1156
|
lastKnownThreadName = registrationResult.threadName;
|
|
926
1157
|
activeLeaderSocketPath = leaderSocketPath;
|
|
1158
|
+
activeRegistrationGeneration = registrationGeneration;
|
|
927
1159
|
activeContext = ctx;
|
|
928
1160
|
await sendHeartbeat();
|
|
929
1161
|
startHeartbeat(leaderSocketPath);
|
|
@@ -985,6 +1217,18 @@ export function createTelegramBusForwardedUpdateReceiverRuntime<
|
|
|
985
1217
|
message: "Telegram bus receiver cannot handle this envelope.",
|
|
986
1218
|
};
|
|
987
1219
|
}
|
|
1220
|
+
const registrationGeneration = deps.getRegistrationGeneration?.();
|
|
1221
|
+
if (
|
|
1222
|
+
registrationGeneration &&
|
|
1223
|
+
envelope.recipientRegistrationGeneration !== registrationGeneration
|
|
1224
|
+
) {
|
|
1225
|
+
return {
|
|
1226
|
+
kind: "bus.ack",
|
|
1227
|
+
requestId: envelope.requestId,
|
|
1228
|
+
ok: false,
|
|
1229
|
+
message: "Stale Telegram bus follower registration generation.",
|
|
1230
|
+
};
|
|
1231
|
+
}
|
|
988
1232
|
const ctx = deps.getContext();
|
|
989
1233
|
if (!ctx) {
|
|
990
1234
|
return {
|