@llblab/pi-telegram 0.22.1 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +24 -13
- package/BACKLOG.md +1 -51
- package/CHANGELOG.md +41 -19
- package/README.md +3 -1
- package/docs/README.md +1 -1
- package/docs/activity.md +8 -0
- package/docs/architecture.md +16 -15
- package/docs/locks.md +21 -15
- package/docs/multi-instance-bus.md +17 -16
- package/docs/outbound.md +16 -0
- package/docs/public-api.md +9 -4
- package/index.ts +76 -65
- package/lib/activity.ts +100 -3
- package/lib/bindings.ts +86 -15
- package/lib/bus-follower.ts +205 -17
- package/lib/bus-leader.ts +333 -244
- package/lib/bus.ts +82 -19
- package/lib/commands.ts +28 -4
- package/lib/config.ts +44 -33
- package/lib/media.ts +102 -1
- package/lib/menu-settings.ts +4 -1
- package/lib/outbound-attachments.ts +150 -1
- package/lib/outbound.ts +106 -1
- package/lib/polling.ts +5 -0
- package/lib/queue.ts +41 -48
- package/lib/routing.ts +88 -1
- package/lib/sync.ts +40 -3
- package/lib/telegram-api.ts +77 -13
- package/lib/text-groups.ts +183 -16
- package/lib/thread-reconciler.ts +69 -93
- package/lib/threads.ts +131 -66
- package/lib/turns.ts +102 -13
- package/lib/updates.ts +2 -0
- package/package.json +3 -2
package/lib/bus-leader.ts
CHANGED
|
@@ -109,17 +109,10 @@ export interface TelegramBusFollowerTargetProvisionerDeps {
|
|
|
109
109
|
) => void;
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
export interface
|
|
112
|
+
export interface TelegramBusFollowerDisconnectHandlerDeps {
|
|
113
113
|
topicTargetStore: Pick<
|
|
114
114
|
Threads.TelegramTopicTargetStore,
|
|
115
|
-
| "
|
|
116
|
-
| "getActiveByInstanceId"
|
|
117
|
-
| "list"
|
|
118
|
-
| "listPendingProvisions"
|
|
119
|
-
| "markStaleByTarget"
|
|
120
|
-
| "markOfflineByInstanceId"
|
|
121
|
-
| "persist"
|
|
122
|
-
| "removePendingProvision"
|
|
115
|
+
"markOfflineByInstanceId" | "persist"
|
|
123
116
|
>;
|
|
124
117
|
callApi: <TResponse>(
|
|
125
118
|
method: string,
|
|
@@ -161,10 +154,9 @@ export interface TelegramBusLeaderRuntimeAssemblyDeps<TContext> {
|
|
|
161
154
|
runtime: Omit<
|
|
162
155
|
TelegramBusLeaderRuntimeDeps<TContext>,
|
|
163
156
|
| "callApi"
|
|
164
|
-
| "
|
|
157
|
+
| "onFollowerDisconnected"
|
|
165
158
|
| "provisionFollowerTarget"
|
|
166
159
|
| "provisionLeaderTarget"
|
|
167
|
-
| "reconcileFollowerBindings"
|
|
168
160
|
| "recordRuntimeEvent"
|
|
169
161
|
>;
|
|
170
162
|
getAllowedUserId: () => number | undefined;
|
|
@@ -217,19 +209,13 @@ export function createTelegramBusLeaderRuntimeAssembly<TContext>(
|
|
|
217
209
|
recordThreadReconciliationPlan: deps.recordThreadReconciliationPlan,
|
|
218
210
|
setLeaderTarget: deps.setLeaderTarget,
|
|
219
211
|
}),
|
|
220
|
-
|
|
212
|
+
onFollowerDisconnected: createTelegramBusFollowerDisconnectHandler({
|
|
221
213
|
...provisionerPorts,
|
|
222
214
|
}),
|
|
223
215
|
provisionFollowerTarget: createTelegramBusFollowerTargetProvisioner({
|
|
224
216
|
...provisionerPorts,
|
|
225
217
|
}),
|
|
226
218
|
getCurrentLeaderEpoch: deps.getCurrentLeaderEpoch,
|
|
227
|
-
reconcileFollowerBindings:
|
|
228
|
-
createTelegramBusFollowerBindingRealityReconciler({
|
|
229
|
-
topicTargetStore: deps.topicTargetStore,
|
|
230
|
-
followerRegistry: deps.runtime.followerRegistry,
|
|
231
|
-
recordRuntimeEvent: deps.recordRuntimeEvent,
|
|
232
|
-
}),
|
|
233
219
|
callApi: createTelegramBusLeaderApiProxy({
|
|
234
220
|
call: deps.callApi,
|
|
235
221
|
callMultipart: deps.callMultipart,
|
|
@@ -240,29 +226,6 @@ export function createTelegramBusLeaderRuntimeAssembly<TContext>(
|
|
|
240
226
|
});
|
|
241
227
|
}
|
|
242
228
|
|
|
243
|
-
export const TELEGRAM_BUS_RECENT_FOLLOWER_BINDING_GRACE_MS = 10_000;
|
|
244
|
-
|
|
245
|
-
export interface TelegramBusFollowerBindingRealityDeps {
|
|
246
|
-
topicTargetStore: Pick<
|
|
247
|
-
Threads.TelegramTopicTargetStore,
|
|
248
|
-
| "load"
|
|
249
|
-
| "list"
|
|
250
|
-
| "markOfflineByInstanceId"
|
|
251
|
-
| "forgetIdentityByProfileKey"
|
|
252
|
-
| "getBotState"
|
|
253
|
-
| "persist"
|
|
254
|
-
| "setBotState"
|
|
255
|
-
>;
|
|
256
|
-
followerRegistry: Pick<TelegramBusFollowerRegistry, "list">;
|
|
257
|
-
getNowMs?: () => number;
|
|
258
|
-
recentBindingGraceMs?: number;
|
|
259
|
-
recordRuntimeEvent: (
|
|
260
|
-
category: string,
|
|
261
|
-
error: unknown,
|
|
262
|
-
details?: Record<string, unknown>,
|
|
263
|
-
) => void;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
229
|
export interface TelegramBusFollowerMessageOwnershipRecord {
|
|
267
230
|
follower: TelegramBusFollowerView;
|
|
268
231
|
chatId: number;
|
|
@@ -292,13 +255,11 @@ export interface TelegramBusLeaderRuntimeDeps<TContext> {
|
|
|
292
255
|
registration: TelegramBusInstanceRegistration,
|
|
293
256
|
) => Promise<TelegramTarget | undefined> | TelegramTarget | undefined;
|
|
294
257
|
getCurrentLeaderEpoch?: () => number | string | undefined;
|
|
295
|
-
reconcileFollowerBindings?: () => Promise<unknown> | unknown;
|
|
296
258
|
provisionLeaderTarget?: (ctx: TContext) => Promise<void> | void;
|
|
297
259
|
getNowMs?: () => number;
|
|
298
260
|
followerPruneIntervalMs?: number;
|
|
299
261
|
followerStaleAfterMs?: number;
|
|
300
|
-
|
|
301
|
-
onFollowerPruned?: (
|
|
262
|
+
onFollowerDisconnected?: (
|
|
302
263
|
follower: TelegramBusFollowerView,
|
|
303
264
|
) => Promise<void> | void;
|
|
304
265
|
recordRuntimeEvent?: (
|
|
@@ -308,76 +269,11 @@ export interface TelegramBusLeaderRuntimeDeps<TContext> {
|
|
|
308
269
|
) => void;
|
|
309
270
|
}
|
|
310
271
|
|
|
311
|
-
export function createTelegramBusFollowerBindingRealityReconciler(
|
|
312
|
-
deps: TelegramBusFollowerBindingRealityDeps,
|
|
313
|
-
): () => Promise<number> {
|
|
314
|
-
const getNowMs = deps.getNowMs ?? Date.now;
|
|
315
|
-
const recentBindingGraceMs =
|
|
316
|
-
deps.recentBindingGraceMs ?? TELEGRAM_BUS_RECENT_FOLLOWER_BINDING_GRACE_MS;
|
|
317
|
-
return async () => {
|
|
318
|
-
await deps.topicTargetStore.load();
|
|
319
|
-
const nowMs = getNowMs();
|
|
320
|
-
const liveInstanceIds = new Set(
|
|
321
|
-
deps.followerRegistry.list().map((follower) => follower.instanceId),
|
|
322
|
-
);
|
|
323
|
-
const staleRecords = deps.topicTargetStore.list().filter((record) => {
|
|
324
|
-
return (
|
|
325
|
-
record.owner?.kind === "manual-follower" &&
|
|
326
|
-
!!record.instanceId &&
|
|
327
|
-
!liveInstanceIds.has(record.instanceId) &&
|
|
328
|
-
nowMs - record.updatedAtMs > recentBindingGraceMs
|
|
329
|
-
);
|
|
330
|
-
});
|
|
331
|
-
const staleInstanceIds = new Set(
|
|
332
|
-
staleRecords.flatMap((record) =>
|
|
333
|
-
record.instanceId ? [record.instanceId] : [],
|
|
334
|
-
),
|
|
335
|
-
);
|
|
336
|
-
let removed = 0;
|
|
337
|
-
for (const instanceId of staleInstanceIds) {
|
|
338
|
-
removed += deps.topicTargetStore.markOfflineByInstanceId(instanceId);
|
|
339
|
-
}
|
|
340
|
-
for (const record of staleRecords) {
|
|
341
|
-
const profileKey =
|
|
342
|
-
record.profileKey ??
|
|
343
|
-
(record.owner
|
|
344
|
-
? Threads.getTelegramThreadOwnerKey(record.owner)
|
|
345
|
-
: undefined);
|
|
346
|
-
if (profileKey)
|
|
347
|
-
deps.topicTargetStore.forgetIdentityByProfileKey(profileKey);
|
|
348
|
-
}
|
|
349
|
-
if (removed === 0) return 0;
|
|
350
|
-
const cursorRealigned = Threads.reconcileTelegramFreshAllocationCursor(
|
|
351
|
-
deps.topicTargetStore,
|
|
352
|
-
);
|
|
353
|
-
await deps.topicTargetStore.persist();
|
|
354
|
-
deps.recordRuntimeEvent(
|
|
355
|
-
"bus",
|
|
356
|
-
"Historical follower bindings removed from live state",
|
|
357
|
-
{
|
|
358
|
-
phase: "follower-binding-reality",
|
|
359
|
-
removed,
|
|
360
|
-
},
|
|
361
|
-
);
|
|
362
|
-
if (cursorRealigned) {
|
|
363
|
-
deps.recordRuntimeEvent(
|
|
364
|
-
"bus",
|
|
365
|
-
"Fresh allocation cursor realigned after live-state compaction",
|
|
366
|
-
{
|
|
367
|
-
phase: "follower-binding-reality-cursor",
|
|
368
|
-
lastSlot: deps.topicTargetStore.getBotState().lastSlot,
|
|
369
|
-
},
|
|
370
|
-
);
|
|
371
|
-
}
|
|
372
|
-
return removed;
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
|
-
|
|
376
272
|
export function createTelegramBusInstanceLifecycleAnnouncement(input: {
|
|
377
273
|
target: TelegramTarget & { threadId: number };
|
|
378
274
|
threadName?: string;
|
|
379
275
|
slot?: string;
|
|
380
|
-
state: "connected"
|
|
276
|
+
state: "connected";
|
|
381
277
|
}): TelegramBusFollowerLifecycleAnnouncement {
|
|
382
278
|
return {
|
|
383
279
|
target: { ...input.target },
|
|
@@ -386,23 +282,6 @@ export function createTelegramBusInstanceLifecycleAnnouncement(input: {
|
|
|
386
282
|
};
|
|
387
283
|
}
|
|
388
284
|
|
|
389
|
-
export function createTelegramBusFollowerDisconnectedAnnouncement(input: {
|
|
390
|
-
follower: TelegramBusFollowerView;
|
|
391
|
-
threadName?: string;
|
|
392
|
-
slot?: string;
|
|
393
|
-
}): TelegramBusFollowerLifecycleAnnouncement | undefined {
|
|
394
|
-
if (!input.follower.target?.threadId) return undefined;
|
|
395
|
-
return createTelegramBusInstanceLifecycleAnnouncement({
|
|
396
|
-
target: {
|
|
397
|
-
chatId: input.follower.target.chatId,
|
|
398
|
-
threadId: input.follower.target.threadId,
|
|
399
|
-
},
|
|
400
|
-
threadName: input.threadName,
|
|
401
|
-
slot: input.slot,
|
|
402
|
-
state: "disconnected",
|
|
403
|
-
});
|
|
404
|
-
}
|
|
405
|
-
|
|
406
285
|
const TELEGRAM_BUS_SLOW_FOLLOWER_REGISTRATION_MS = 1000;
|
|
407
286
|
|
|
408
287
|
function scheduleTelegramBusLeaderBackgroundTask(
|
|
@@ -473,29 +352,29 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
473
352
|
const recordsBeforeProvision = deps.topicTargetStore.list();
|
|
474
353
|
const followerProfileKey =
|
|
475
354
|
registration.profileKey ?? `manual:${registration.instanceId}`;
|
|
476
|
-
const
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
355
|
+
const requestedTarget = registration.target;
|
|
356
|
+
const reconnectRecord = recordsBeforeProvision.find((record) => {
|
|
357
|
+
return (
|
|
358
|
+
record.owner?.kind === "manual-follower" &&
|
|
359
|
+
(record.instanceId === registration.instanceId ||
|
|
360
|
+
record.profileKey === followerProfileKey) &&
|
|
361
|
+
(!requestedTarget ||
|
|
362
|
+
(record.target.chatId === requestedTarget.chatId &&
|
|
363
|
+
record.target.threadId === requestedTarget.threadId))
|
|
364
|
+
);
|
|
365
|
+
});
|
|
487
366
|
const followerOwner =
|
|
488
367
|
Threads.getTelegramThreadOwnerFromProfileKey(followerProfileKey);
|
|
489
368
|
const recoverableTarget =
|
|
490
369
|
!reconnectRecord &&
|
|
491
|
-
|
|
492
|
-
|
|
370
|
+
requestedTarget?.chatId === chatId &&
|
|
371
|
+
requestedTarget.threadId !== undefined &&
|
|
493
372
|
!recordsBeforeProvision.some(
|
|
494
373
|
(record) =>
|
|
495
|
-
record.target.chatId ===
|
|
496
|
-
record.target.threadId ===
|
|
374
|
+
record.target.chatId === requestedTarget.chatId &&
|
|
375
|
+
record.target.threadId === requestedTarget.threadId,
|
|
497
376
|
)
|
|
498
|
-
?
|
|
377
|
+
? requestedTarget
|
|
499
378
|
: undefined;
|
|
500
379
|
const recoveryHint = recoverableTarget
|
|
501
380
|
? deps.topicTargetStore.getFollowerRecoveryHintByTarget?.(
|
|
@@ -526,13 +405,14 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
526
405
|
};
|
|
527
406
|
const recoverRequestedTarget = async () => {
|
|
528
407
|
const nowMs = getNowMs();
|
|
408
|
+
const carriedThreadName = registration.threadName;
|
|
529
409
|
const requestedThreadName =
|
|
530
|
-
|
|
410
|
+
carriedThreadName &&
|
|
531
411
|
Threads.isTelegramTopicThreadNameValidForSlot(
|
|
532
|
-
|
|
412
|
+
carriedThreadName,
|
|
533
413
|
registration.slot,
|
|
534
414
|
)
|
|
535
|
-
?
|
|
415
|
+
? carriedThreadName
|
|
536
416
|
: recoveryHint?.threadName &&
|
|
537
417
|
Threads.isTelegramTopicThreadNameValidForSlot(
|
|
538
418
|
recoveryHint.threadName,
|
|
@@ -540,7 +420,7 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
540
420
|
)
|
|
541
421
|
? recoveryHint.threadName
|
|
542
422
|
: undefined;
|
|
543
|
-
let recoveredRecord =
|
|
423
|
+
let recoveredRecord: Threads.TelegramTopicTargetRecord = {
|
|
544
424
|
profileKey: followerProfileKey,
|
|
545
425
|
owner:
|
|
546
426
|
followerOwner.kind === "manual-follower"
|
|
@@ -560,19 +440,15 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
560
440
|
...(requestedThreadName ? { threadName: requestedThreadName } : {}),
|
|
561
441
|
lastSyncObservedAtMs: nowMs,
|
|
562
442
|
lastReconcileAction: "follower-live-target-recovery",
|
|
563
|
-
}
|
|
443
|
+
};
|
|
564
444
|
const requestedSlot = registration.slot ?? recoveryHint?.slot;
|
|
565
445
|
const requestedSlotAvailable =
|
|
566
446
|
!!requestedSlot &&
|
|
567
447
|
/^[A-Z]$/.test(requestedSlot) &&
|
|
568
448
|
!recordsBeforeProvision.some((record) => record.slot === requestedSlot);
|
|
569
449
|
if (requestedSlotAvailable) {
|
|
570
|
-
recoveredRecord =
|
|
571
|
-
...recoveredRecord,
|
|
572
|
-
slot: requestedSlot,
|
|
573
|
-
});
|
|
450
|
+
recoveredRecord = { ...recoveredRecord, slot: requestedSlot };
|
|
574
451
|
}
|
|
575
|
-
await deps.topicTargetStore.persist();
|
|
576
452
|
return {
|
|
577
453
|
target: recoveredRecord.target,
|
|
578
454
|
reused: true,
|
|
@@ -591,25 +467,117 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
591
467
|
: recoverableTarget
|
|
592
468
|
? await recoverRequestedTarget()
|
|
593
469
|
: await provisionTarget();
|
|
594
|
-
|
|
470
|
+
const crossSessionReuse =
|
|
471
|
+
!!reconnectRecord &&
|
|
472
|
+
reconnectRecord.instanceId !== registration.instanceId;
|
|
473
|
+
if (reconnectRecord && !crossSessionReuse) {
|
|
595
474
|
const nowMs = getNowMs();
|
|
596
|
-
const
|
|
475
|
+
const refreshedRecord = deps.topicTargetStore.upsert({
|
|
597
476
|
...reconnectRecord,
|
|
598
477
|
instanceId: registration.instanceId,
|
|
599
478
|
updatedAtMs: nowMs,
|
|
600
479
|
lastSyncObservedAtMs: nowMs,
|
|
601
|
-
lastReconcileAction:
|
|
602
|
-
reconnectRecord.instanceId === registration.instanceId
|
|
603
|
-
? "follower-register-reuse"
|
|
604
|
-
: "follower-session-handoff",
|
|
480
|
+
lastReconcileAction: "follower-register-reuse",
|
|
605
481
|
});
|
|
606
482
|
await deps.topicTargetStore.persist();
|
|
607
483
|
result = {
|
|
608
|
-
target:
|
|
484
|
+
target: refreshedRecord.target,
|
|
609
485
|
reused: true,
|
|
610
|
-
record:
|
|
486
|
+
record: refreshedRecord,
|
|
611
487
|
};
|
|
612
488
|
}
|
|
489
|
+
const probeRequiredRecord =
|
|
490
|
+
reconnectRecord?.status === "probe-required";
|
|
491
|
+
const requiresVisibilityProbe =
|
|
492
|
+
crossSessionReuse ||
|
|
493
|
+
probeRequiredRecord ||
|
|
494
|
+
recoverableTarget !== undefined;
|
|
495
|
+
let connectedAnnouncement =
|
|
496
|
+
!result.reused || requiresVisibilityProbe
|
|
497
|
+
? createTelegramBusInstanceLifecycleAnnouncement({
|
|
498
|
+
target: result.target,
|
|
499
|
+
threadName: result.record.threadName,
|
|
500
|
+
slot: result.record.slot,
|
|
501
|
+
state: "connected",
|
|
502
|
+
})
|
|
503
|
+
: undefined;
|
|
504
|
+
if (requiresVisibilityProbe && connectedAnnouncement) {
|
|
505
|
+
try {
|
|
506
|
+
await deps.callApi("sendMessage", {
|
|
507
|
+
chat_id: connectedAnnouncement.target.chatId,
|
|
508
|
+
message_thread_id: connectedAnnouncement.target.threadId,
|
|
509
|
+
text: connectedAnnouncement.text,
|
|
510
|
+
parse_mode: connectedAnnouncement.parseMode,
|
|
511
|
+
});
|
|
512
|
+
if (recoverableTarget || probeRequiredRecord) {
|
|
513
|
+
const activatedRecord = deps.topicTargetStore.upsert({
|
|
514
|
+
...result.record,
|
|
515
|
+
status: "active",
|
|
516
|
+
updatedAtMs: getNowMs(),
|
|
517
|
+
lastSyncObservedAtMs: getNowMs(),
|
|
518
|
+
lastReconcileAction: "follower-live-target-recovery",
|
|
519
|
+
});
|
|
520
|
+
await deps.topicTargetStore.persist();
|
|
521
|
+
result = {
|
|
522
|
+
target: activatedRecord.target,
|
|
523
|
+
reused: true,
|
|
524
|
+
record: activatedRecord,
|
|
525
|
+
};
|
|
526
|
+
} else if (crossSessionReuse && reconnectRecord) {
|
|
527
|
+
const nowMs = getNowMs();
|
|
528
|
+
const transferredRecord = deps.topicTargetStore.upsert({
|
|
529
|
+
...reconnectRecord,
|
|
530
|
+
instanceId: registration.instanceId,
|
|
531
|
+
updatedAtMs: nowMs,
|
|
532
|
+
lastSyncObservedAtMs: nowMs,
|
|
533
|
+
lastReconcileAction: "follower-session-handoff",
|
|
534
|
+
});
|
|
535
|
+
await deps.topicTargetStore.persist();
|
|
536
|
+
result = {
|
|
537
|
+
target: transferredRecord.target,
|
|
538
|
+
reused: true,
|
|
539
|
+
record: transferredRecord,
|
|
540
|
+
};
|
|
541
|
+
}
|
|
542
|
+
connectedAnnouncement = undefined;
|
|
543
|
+
} catch (error) {
|
|
544
|
+
if (Threads.isTelegramTopicTargetStaleError(error)) {
|
|
545
|
+
deps.topicTargetStore.markStaleByTarget(
|
|
546
|
+
result.target,
|
|
547
|
+
"deleted",
|
|
548
|
+
error instanceof Error ? error.message : String(error),
|
|
549
|
+
);
|
|
550
|
+
await deps.topicTargetStore.persist();
|
|
551
|
+
result = await provisionTarget();
|
|
552
|
+
connectedAnnouncement =
|
|
553
|
+
createTelegramBusInstanceLifecycleAnnouncement({
|
|
554
|
+
target: result.target,
|
|
555
|
+
threadName: result.record.threadName,
|
|
556
|
+
slot: result.record.slot,
|
|
557
|
+
state: "connected",
|
|
558
|
+
});
|
|
559
|
+
} else {
|
|
560
|
+
deps.recordRuntimeEvent("telegram", error, {
|
|
561
|
+
phase: "follower-topic-reuse-probe",
|
|
562
|
+
instanceId: registration.instanceId,
|
|
563
|
+
chatId: result.target.chatId,
|
|
564
|
+
threadId: result.target.threadId,
|
|
565
|
+
});
|
|
566
|
+
if (recoverableTarget) {
|
|
567
|
+
deps.topicTargetStore.upsert({
|
|
568
|
+
...result.record,
|
|
569
|
+
status: "probe-required",
|
|
570
|
+
updatedAtMs: getNowMs(),
|
|
571
|
+
lastSyncError:
|
|
572
|
+
error instanceof Error ? error.message : String(error),
|
|
573
|
+
lastReconcileAction: "follower-visibility-probe-required",
|
|
574
|
+
});
|
|
575
|
+
await deps.topicTargetStore.persist();
|
|
576
|
+
}
|
|
577
|
+
throw error;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
613
581
|
deps.setSyncState(
|
|
614
582
|
Sync.markTelegramSyncSliceFresh(
|
|
615
583
|
deps.getSyncState(),
|
|
@@ -620,14 +588,6 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
620
588
|
},
|
|
621
589
|
),
|
|
622
590
|
);
|
|
623
|
-
const connectedAnnouncement = result.reused
|
|
624
|
-
? undefined
|
|
625
|
-
: createTelegramBusInstanceLifecycleAnnouncement({
|
|
626
|
-
target: result.target,
|
|
627
|
-
threadName: result.record.threadName,
|
|
628
|
-
slot: result.record.slot,
|
|
629
|
-
state: "connected",
|
|
630
|
-
});
|
|
631
591
|
recordSlowTelegramBusFollowerRegistrationStep(deps, {
|
|
632
592
|
phase: "follower-register-critical",
|
|
633
593
|
elapsedMs: Date.now() - registrationStartedAtMs,
|
|
@@ -718,18 +678,55 @@ export function createTelegramBusFollowerTargetProvisioner(
|
|
|
718
678
|
};
|
|
719
679
|
}
|
|
720
680
|
|
|
721
|
-
export function
|
|
722
|
-
deps:
|
|
681
|
+
export function createTelegramBusFollowerDisconnectHandler(
|
|
682
|
+
deps: TelegramBusFollowerDisconnectHandlerDeps,
|
|
723
683
|
): (follower: TelegramBusFollowerView) => Promise<void> {
|
|
724
684
|
return async (follower) => {
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
685
|
+
const target = follower.target;
|
|
686
|
+
if (!target?.threadId) return;
|
|
687
|
+
const leaderEpoch = deps.getCurrentLeaderEpoch?.();
|
|
688
|
+
if (deps.getCurrentLeaderEpoch && leaderEpoch === undefined) {
|
|
689
|
+
throw new Error("Follower disconnect cleanup requires leader ownership.");
|
|
690
|
+
}
|
|
691
|
+
const cleanup = await ThreadReconciler.applyThreadReconciliationPlan(
|
|
692
|
+
ThreadReconciler.planDisconnectedInstanceThreadCleanup({
|
|
693
|
+
target: { chatId: target.chatId, threadId: target.threadId },
|
|
730
694
|
instanceId: follower.instanceId,
|
|
695
|
+
leaderEpoch,
|
|
696
|
+
}),
|
|
697
|
+
{
|
|
698
|
+
callApi: deps.callApi,
|
|
699
|
+
persist: deps.topicTargetStore.persist,
|
|
700
|
+
getCurrentLeaderEpoch: deps.getCurrentLeaderEpoch,
|
|
701
|
+
recordRuntimeEvent: deps.recordRuntimeEvent,
|
|
731
702
|
},
|
|
732
703
|
);
|
|
704
|
+
if (cleanup.incompleteActions?.length) {
|
|
705
|
+
throw new Error(
|
|
706
|
+
"Telegram follower thread deletion was not confirmed; reconnect the leader and retry /telegram-disconnect.",
|
|
707
|
+
);
|
|
708
|
+
}
|
|
709
|
+
if (
|
|
710
|
+
deps.getCurrentLeaderEpoch &&
|
|
711
|
+
deps.getCurrentLeaderEpoch() !== leaderEpoch
|
|
712
|
+
) {
|
|
713
|
+
throw new Error("Follower disconnect cleanup lost leader ownership.");
|
|
714
|
+
}
|
|
715
|
+
const changed =
|
|
716
|
+
deps.topicTargetStore.markOfflineByInstanceId(follower.instanceId) > 0;
|
|
717
|
+
if (changed) await deps.topicTargetStore.persist();
|
|
718
|
+
deps.setSyncState(
|
|
719
|
+
Sync.markTelegramSyncSliceFresh(deps.getSyncState(), "target-bindings", {
|
|
720
|
+
nowMs: (deps.getNowMs ?? Date.now)(),
|
|
721
|
+
action: "manual-follower-disconnect",
|
|
722
|
+
}),
|
|
723
|
+
);
|
|
724
|
+
deps.recordRuntimeEvent("bus", "Telegram bus follower disconnected", {
|
|
725
|
+
phase: "follower-disconnect",
|
|
726
|
+
instanceId: follower.instanceId,
|
|
727
|
+
chatId: target.chatId,
|
|
728
|
+
threadId: target.threadId,
|
|
729
|
+
});
|
|
733
730
|
};
|
|
734
731
|
}
|
|
735
732
|
|
|
@@ -875,12 +872,49 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
875
872
|
recordFollowerMessageOwnership?: TelegramBusFollowerMessageOwnershipRecorder;
|
|
876
873
|
provisionFollowerTarget?: (
|
|
877
874
|
registration: TelegramBusInstanceRegistration,
|
|
878
|
-
) =>
|
|
875
|
+
) =>
|
|
876
|
+
| Promise<
|
|
877
|
+
| (TelegramTarget & { slot?: string; threadName?: string })
|
|
878
|
+
| undefined
|
|
879
|
+
>
|
|
880
|
+
| (TelegramTarget & { slot?: string; threadName?: string })
|
|
881
|
+
| undefined;
|
|
882
|
+
onFollowerDisconnected?: (
|
|
883
|
+
follower: TelegramBusFollowerView,
|
|
884
|
+
) => Promise<void> | void;
|
|
879
885
|
getCurrentLeaderEpoch?: () => number | string | undefined;
|
|
880
886
|
}): (
|
|
881
887
|
envelope: TelegramBusEnvelope,
|
|
882
888
|
) => Promise<TelegramBusEnvelope> | TelegramBusEnvelope {
|
|
883
889
|
const getNowMs = deps.getNowMs ?? Date.now;
|
|
890
|
+
const followerMutationTails = new Map<string, Promise<void>>();
|
|
891
|
+
const getFollowerMutationKey = (follower: {
|
|
892
|
+
instanceId: string;
|
|
893
|
+
profileKey?: string;
|
|
894
|
+
}): string =>
|
|
895
|
+
follower.profileKey
|
|
896
|
+
? `profile:${follower.profileKey}`
|
|
897
|
+
: `instance:${follower.instanceId}`;
|
|
898
|
+
const runFollowerMutation = async <T>(
|
|
899
|
+
mutationKey: string,
|
|
900
|
+
operation: () => Promise<T>,
|
|
901
|
+
): Promise<T> => {
|
|
902
|
+
const previous = followerMutationTails.get(mutationKey);
|
|
903
|
+
let release!: () => void;
|
|
904
|
+
const current = new Promise<void>((resolve) => {
|
|
905
|
+
release = resolve;
|
|
906
|
+
});
|
|
907
|
+
followerMutationTails.set(mutationKey, current);
|
|
908
|
+
if (previous) await previous;
|
|
909
|
+
try {
|
|
910
|
+
return await operation();
|
|
911
|
+
} finally {
|
|
912
|
+
release();
|
|
913
|
+
if (followerMutationTails.get(mutationKey) === current) {
|
|
914
|
+
followerMutationTails.delete(mutationKey);
|
|
915
|
+
}
|
|
916
|
+
}
|
|
917
|
+
};
|
|
884
918
|
const forwardToFollower = async (
|
|
885
919
|
envelope: Extract<
|
|
886
920
|
TelegramBusEnvelope,
|
|
@@ -943,47 +977,115 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
943
977
|
}
|
|
944
978
|
switch (envelope.kind) {
|
|
945
979
|
case "follower.register": {
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
980
|
+
return runFollowerMutation(
|
|
981
|
+
getFollowerMutationKey(envelope.registration),
|
|
982
|
+
async () => {
|
|
983
|
+
try {
|
|
984
|
+
if (!envelope.registration.registrationGeneration) {
|
|
985
|
+
throw new Error(
|
|
986
|
+
"Telegram follower registration requires an exact generation.",
|
|
987
|
+
);
|
|
988
|
+
}
|
|
989
|
+
const leaderEpoch = deps.getCurrentLeaderEpoch?.();
|
|
990
|
+
if (deps.getCurrentLeaderEpoch && leaderEpoch === undefined) {
|
|
991
|
+
throw new Error(
|
|
992
|
+
"Telegram follower registration requires leader ownership.",
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
const target = await deps.provisionFollowerTarget?.(
|
|
996
|
+
envelope.registration,
|
|
951
997
|
);
|
|
998
|
+
if (
|
|
999
|
+
deps.getCurrentLeaderEpoch &&
|
|
1000
|
+
deps.getCurrentLeaderEpoch() !== leaderEpoch
|
|
1001
|
+
) {
|
|
1002
|
+
throw new Error(
|
|
1003
|
+
"Telegram follower registration lost leader ownership.",
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
1006
|
+
const registeredTarget = target ?? envelope.registration.target;
|
|
1007
|
+
deps.followerRegistry.register({
|
|
1008
|
+
...envelope.registration,
|
|
1009
|
+
connectedAtMs: getNowMs(),
|
|
1010
|
+
target: registeredTarget,
|
|
1011
|
+
...((target?.slot ?? envelope.registration.slot)
|
|
1012
|
+
? { slot: target?.slot ?? envelope.registration.slot }
|
|
1013
|
+
: {}),
|
|
1014
|
+
...((target?.threadName ?? envelope.registration.threadName)
|
|
1015
|
+
? {
|
|
1016
|
+
threadName:
|
|
1017
|
+
target?.threadName ?? envelope.registration.threadName,
|
|
1018
|
+
}
|
|
1019
|
+
: {}),
|
|
1020
|
+
});
|
|
1021
|
+
return {
|
|
1022
|
+
kind: "bus.ack" as const,
|
|
1023
|
+
requestId: envelope.requestId,
|
|
1024
|
+
ok: true,
|
|
1025
|
+
...(registeredTarget ? { result: registeredTarget } : {}),
|
|
1026
|
+
};
|
|
1027
|
+
} catch (error) {
|
|
1028
|
+
return {
|
|
1029
|
+
kind: "bus.ack" as const,
|
|
1030
|
+
requestId: envelope.requestId,
|
|
1031
|
+
ok: false,
|
|
1032
|
+
message:
|
|
1033
|
+
error instanceof Error
|
|
1034
|
+
? error.message
|
|
1035
|
+
: "Telegram bus follower target provisioning failed.",
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
},
|
|
1039
|
+
);
|
|
1040
|
+
}
|
|
1041
|
+
case "follower.disconnect": {
|
|
1042
|
+
const registeredFollower = deps.followerRegistry.get(envelope.instanceId);
|
|
1043
|
+
return runFollowerMutation(
|
|
1044
|
+
getFollowerMutationKey(
|
|
1045
|
+
registeredFollower ?? { instanceId: envelope.instanceId },
|
|
1046
|
+
),
|
|
1047
|
+
async () => {
|
|
1048
|
+
const follower = deps.followerRegistry.get(envelope.instanceId);
|
|
1049
|
+
if (!follower) {
|
|
1050
|
+
return {
|
|
1051
|
+
kind: "bus.ack" as const,
|
|
1052
|
+
requestId: envelope.requestId,
|
|
1053
|
+
ok: false,
|
|
1054
|
+
message: "Unknown Telegram bus follower instance.",
|
|
1055
|
+
};
|
|
952
1056
|
}
|
|
953
|
-
const target = await deps.provisionFollowerTarget?.(
|
|
954
|
-
envelope.registration,
|
|
955
|
-
);
|
|
956
1057
|
if (
|
|
957
|
-
|
|
958
|
-
|
|
1058
|
+
!follower.registrationGeneration ||
|
|
1059
|
+
!envelope.registrationGeneration ||
|
|
1060
|
+
envelope.registrationGeneration !== follower.registrationGeneration
|
|
959
1061
|
) {
|
|
960
|
-
|
|
961
|
-
"
|
|
962
|
-
|
|
1062
|
+
return {
|
|
1063
|
+
kind: "bus.ack" as const,
|
|
1064
|
+
requestId: envelope.requestId,
|
|
1065
|
+
ok: false,
|
|
1066
|
+
message: "Stale Telegram bus follower registration generation.",
|
|
1067
|
+
};
|
|
963
1068
|
}
|
|
964
|
-
|
|
965
|
-
deps.followerRegistry.
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1069
|
+
await deps.onFollowerDisconnected?.(follower);
|
|
1070
|
+
const current = deps.followerRegistry.get(follower.instanceId);
|
|
1071
|
+
if (
|
|
1072
|
+
current?.registrationGeneration !== follower.registrationGeneration
|
|
1073
|
+
) {
|
|
1074
|
+
return {
|
|
1075
|
+
kind: "bus.ack" as const,
|
|
1076
|
+
requestId: envelope.requestId,
|
|
1077
|
+
ok: false,
|
|
1078
|
+
message: "Stale Telegram bus follower registration generation.",
|
|
1079
|
+
};
|
|
1080
|
+
}
|
|
1081
|
+
deps.followerRegistry.remove(follower.instanceId);
|
|
970
1082
|
return {
|
|
971
|
-
kind: "bus.ack",
|
|
1083
|
+
kind: "bus.ack" as const,
|
|
972
1084
|
requestId: envelope.requestId,
|
|
973
1085
|
ok: true,
|
|
974
|
-
...(registeredTarget ? { result: registeredTarget } : {}),
|
|
975
|
-
};
|
|
976
|
-
} catch (error) {
|
|
977
|
-
return {
|
|
978
|
-
kind: "bus.ack",
|
|
979
|
-
requestId: envelope.requestId,
|
|
980
|
-
ok: false,
|
|
981
|
-
message:
|
|
982
|
-
error instanceof Error
|
|
983
|
-
? error.message
|
|
984
|
-
: "Telegram bus follower target provisioning failed.",
|
|
985
1086
|
};
|
|
986
|
-
|
|
1087
|
+
},
|
|
1088
|
+
);
|
|
987
1089
|
}
|
|
988
1090
|
case "follower.heartbeat": {
|
|
989
1091
|
const current = deps.followerRegistry.get(envelope.instanceId);
|
|
@@ -1003,7 +1105,20 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
|
|
|
1003
1105
|
getNowMs(),
|
|
1004
1106
|
);
|
|
1005
1107
|
return follower
|
|
1006
|
-
? {
|
|
1108
|
+
? {
|
|
1109
|
+
kind: "bus.ack",
|
|
1110
|
+
requestId: envelope.requestId,
|
|
1111
|
+
ok: true,
|
|
1112
|
+
result: {
|
|
1113
|
+
eligibleElectionSlots: deps.followerRegistry
|
|
1114
|
+
.list()
|
|
1115
|
+
.map((candidate) => candidate.slot)
|
|
1116
|
+
.filter((slot): slot is string =>
|
|
1117
|
+
typeof slot === "string" && /^[A-Z]$/.test(slot),
|
|
1118
|
+
)
|
|
1119
|
+
.sort(),
|
|
1120
|
+
},
|
|
1121
|
+
}
|
|
1007
1122
|
: {
|
|
1008
1123
|
kind: "bus.ack",
|
|
1009
1124
|
requestId: envelope.requestId,
|
|
@@ -1253,32 +1368,12 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1253
1368
|
const getNowMs = deps.getNowMs ?? Date.now;
|
|
1254
1369
|
const followerPruneIntervalMs = deps.followerPruneIntervalMs ?? 1000;
|
|
1255
1370
|
const followerStaleAfterMs = deps.followerStaleAfterMs ?? 5000;
|
|
1256
|
-
const followerRecoveryGraceMs = deps.followerRecoveryGraceMs ?? 5000;
|
|
1257
1371
|
let pruneInterval: ReturnType<typeof setInterval> | undefined;
|
|
1258
|
-
let followerRealityTimer: ReturnType<typeof setTimeout> | undefined;
|
|
1259
1372
|
const stopPruning = () => {
|
|
1260
1373
|
if (!pruneInterval) return;
|
|
1261
1374
|
clearInterval(pruneInterval);
|
|
1262
1375
|
pruneInterval = undefined;
|
|
1263
1376
|
};
|
|
1264
|
-
const stopFollowerRealityTimer = () => {
|
|
1265
|
-
if (!followerRealityTimer) return;
|
|
1266
|
-
clearTimeout(followerRealityTimer);
|
|
1267
|
-
followerRealityTimer = undefined;
|
|
1268
|
-
};
|
|
1269
|
-
const scheduleFollowerBindingReality = () => {
|
|
1270
|
-
stopFollowerRealityTimer();
|
|
1271
|
-
if (!deps.reconcileFollowerBindings) return;
|
|
1272
|
-
followerRealityTimer = setTimeout(() => {
|
|
1273
|
-
followerRealityTimer = undefined;
|
|
1274
|
-
void Promise.resolve(deps.reconcileFollowerBindings?.()).catch((error) =>
|
|
1275
|
-
deps.recordRuntimeEvent?.("bus", error, {
|
|
1276
|
-
phase: "follower-binding-reality",
|
|
1277
|
-
}),
|
|
1278
|
-
);
|
|
1279
|
-
}, followerRecoveryGraceMs);
|
|
1280
|
-
followerRealityTimer.unref?.();
|
|
1281
|
-
};
|
|
1282
1377
|
const pruneFollowers = async () => {
|
|
1283
1378
|
try {
|
|
1284
1379
|
await localServer.ensureEndpoint();
|
|
@@ -1292,18 +1387,14 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1292
1387
|
followerStaleAfterMs,
|
|
1293
1388
|
);
|
|
1294
1389
|
for (const follower of removed) {
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
phase: "follower-
|
|
1390
|
+
deps.recordRuntimeEvent?.(
|
|
1391
|
+
"bus",
|
|
1392
|
+
"Telegram bus follower heartbeat stale; preserving thread binding",
|
|
1393
|
+
{
|
|
1394
|
+
phase: "follower-pruned",
|
|
1300
1395
|
instanceId: follower.instanceId,
|
|
1301
|
-
}
|
|
1302
|
-
|
|
1303
|
-
deps.recordRuntimeEvent?.("bus", "Telegram bus follower timed out", {
|
|
1304
|
-
phase: "follower-prune",
|
|
1305
|
-
instanceId: follower.instanceId,
|
|
1306
|
-
});
|
|
1396
|
+
},
|
|
1397
|
+
);
|
|
1307
1398
|
}
|
|
1308
1399
|
};
|
|
1309
1400
|
const startPruning = () => {
|
|
@@ -1330,27 +1421,25 @@ export function createTelegramBusLeaderRuntime<TContext>(
|
|
|
1330
1421
|
authorizeFollowerApiCall: deps.authorizeFollowerApiCall,
|
|
1331
1422
|
recordFollowerMessageOwnership: deps.recordFollowerMessageOwnership,
|
|
1332
1423
|
provisionFollowerTarget: deps.provisionFollowerTarget,
|
|
1424
|
+
onFollowerDisconnected: deps.onFollowerDisconnected,
|
|
1333
1425
|
getCurrentLeaderEpoch: deps.getCurrentLeaderEpoch,
|
|
1334
1426
|
}),
|
|
1335
1427
|
});
|
|
1336
1428
|
return {
|
|
1337
1429
|
startPolling: async (ctx) => {
|
|
1338
1430
|
await localServer.start();
|
|
1339
|
-
scheduleFollowerBindingReality();
|
|
1340
1431
|
startPruning();
|
|
1341
1432
|
try {
|
|
1342
1433
|
await deps.provisionLeaderTarget?.(ctx);
|
|
1343
1434
|
await deps.startPolling(ctx);
|
|
1344
1435
|
} catch (error) {
|
|
1345
1436
|
stopPruning();
|
|
1346
|
-
stopFollowerRealityTimer();
|
|
1347
1437
|
await localServer.stop();
|
|
1348
1438
|
throw error;
|
|
1349
1439
|
}
|
|
1350
1440
|
},
|
|
1351
1441
|
stopPolling: async () => {
|
|
1352
1442
|
stopPruning();
|
|
1353
|
-
stopFollowerRealityTimer();
|
|
1354
1443
|
try {
|
|
1355
1444
|
await deps.stopPolling();
|
|
1356
1445
|
} finally {
|