@llblab/pi-telegram 0.18.4 → 0.18.6

package/lib/bus-follower.ts

@@ -19,11 +19,17 @@ import {
   sendTelegramBusLocalEnvelope,
   type TelegramBusEnvelope,
 } from "./bus.ts";
+import {
+  getTelegramBusTransportRetryPolicy,
+  TELEGRAM_BUS_REGISTRATION_RETRY,
+} from "./bus-transport.ts";
 
 export const TELEGRAM_BUS_FOLLOWER_PROMOTION_GRACE_MS = 2_500;
 export const TELEGRAM_FOLLOWER_SESSION_HANDOFF_TTL_MS = 30_000;
-export const TELEGRAM_BUS_FOLLOWER_REGISTRATION_RETRY_ATTEMPTS = 10;
-export const TELEGRAM_BUS_FOLLOWER_REGISTRATION_RETRY_DELAY_MS = 150;
+export const TELEGRAM_BUS_FOLLOWER_REGISTRATION_RETRY_ATTEMPTS =
+  TELEGRAM_BUS_REGISTRATION_RETRY.attempts;
+export const TELEGRAM_BUS_FOLLOWER_REGISTRATION_RETRY_DELAY_MS =
+  TELEGRAM_BUS_REGISTRATION_RETRY.delayMs;
 
 const TELEGRAM_FOLLOWER_SESSION_HANDOFF_KEY =
   "__piTelegramFollowerSessionHandoff";
@@ -197,16 +203,25 @@ export interface TelegramBusFollowerLeaderLock {
   busSecret?: string;
 }
 
+export interface TelegramBusFollowerPromotedBinding {
+  target?: TelegramTarget;
+  slot?: string;
+  threadName?: string;
+}
+
 export interface TelegramBusFollowerHeartbeatRecoveryHandlerDeps<TContext> {
   registrationState: Pick<
     TelegramBusFollowerRegistrationState,
-    "setRegistered"
+    "getTarget" | "getSlot" | "getThreadName" | "setRegistered"
   >;
   getRegistrationRuntime: () => TelegramBusFollowerRegistrationRuntime<TContext>;
   getLeaderState: () => TelegramBusFollowerLeaderState;
   setLifecyclePhase: (phase: "electing" | undefined) => void;
   updateStatus: (ctx: TContext) => void;
-  promoteToLeader: (ctx: TContext) => Promise<void> | void;
+  promoteToLeader: (
+    ctx: TContext,
+    binding: TelegramBusFollowerPromotedBinding,
+  ) => Promise<void> | void;
   sleep: (ms: number) => Promise<void>;
   promotionGraceMs: number;
   recordRuntimeEvent: (
@@ -341,6 +356,10 @@ export function createTelegramBusFollowerApiCaller(
     const response = await sendTelegramBusLocalEnvelope({
       socketPath: deps.socketPath,
       timeoutMs,
+      retry: getTelegramBusTransportRetryPolicy({
+        endpoint: deps.socketPath,
+        operation: "operation",
+      }),
       envelope: {
         kind: "follower.callApi",
         requestId: deps.createRequestId(),
@@ -368,19 +387,6 @@ function isTelegramStaleContextError(error: unknown): boolean {
   );
 }
 
-function isRetryableTelegramBusRegistrationError(error: unknown): boolean {
-  if (!(error instanceof Error)) return false;
-  const code = (error as NodeJS.ErrnoException).code;
-  return code === "ENOENT" || code === "ECONNREFUSED" || code === "EPIPE";
-}
-
-function delayTelegramBusFollowerRegistration(ms: number): Promise<void> {
-  return new Promise((resolve) => {
-    const timer = setTimeout(resolve, ms);
-    timer.unref?.();
-  });
-}
-
 export function createTelegramBusFollowerSessionReplacementSuspender(
   deps: TelegramBusFollowerSessionReplacementSuspenderDeps,
 ): () => Promise<void> {
@@ -514,7 +520,16 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
       return false;
     }
   };
-  const promoteToLeader = async (reason: unknown, ctx: TContext) => {
+  const snapshotBinding = (): TelegramBusFollowerPromotedBinding => ({
+    target: deps.registrationState.getTarget(),
+    slot: deps.registrationState.getSlot(),
+    threadName: deps.registrationState.getThreadName(),
+  });
+  const promoteToLeader = async (
+    reason: unknown,
+    ctx: TContext,
+    binding = snapshotBinding(),
+  ) => {
     deps.setLifecyclePhase("electing");
     safeUpdateStatus(ctx);
     deps.recordRuntimeEvent("bus", reason, {
@@ -526,7 +541,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
     deps.recordRuntimeEvent("bus", "Telegram follower elected for promotion", {
       phase: "follower-promotion-electing",
     });
-    await deps.promoteToLeader(ctx);
+    await deps.promoteToLeader(ctx, binding);
     deps.setLifecyclePhase(undefined);
     safeUpdateStatus(ctx);
     deps.recordRuntimeEvent("bus", "Telegram follower promotion completed", {
@@ -537,6 +552,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
     if (promotionPending) return;
     promotionPending = true;
     try {
+      const initialBinding = snapshotBinding();
       const state = deps.getLeaderState();
       if (state.kind === "active-elsewhere") {
         clearRegisteredState(ctx);
@@ -568,16 +584,16 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
           ) {
             return;
           }
-          await promoteToLeader(error, ctx);
+          await promoteToLeader(error, ctx, initialBinding);
           return;
         }
         if (graceState.kind === "stale" || graceState.kind === "inactive") {
-          await promoteToLeader(error, ctx);
+          await promoteToLeader(error, ctx, initialBinding);
         }
         return;
       }
       if (state.kind === "stale" || state.kind === "inactive") {
-        await promoteToLeader(error, ctx);
+        await promoteToLeader(error, ctx, initialBinding);
       }
     } catch (promotionError) {
       deps.setLifecyclePhase(undefined);
@@ -634,6 +650,10 @@ export function createTelegramBusFollowerRegistrationRuntime<
       const response = await sendTelegramBusLocalEnvelope({
         socketPath: activeLeaderSocketPath,
         timeoutMs: deps.timeoutMs,
+        retry: getTelegramBusTransportRetryPolicy({
+          endpoint: activeLeaderSocketPath,
+          operation: "operation",
+        }),
         envelope: {
           kind: "follower.heartbeat",
           requestId: deps.createRequestId(),
@@ -692,31 +712,25 @@ export function createTelegramBusFollowerRegistrationRuntime<
       });
       let response: TelegramBusEnvelope | undefined;
       try {
-        for (let attempt = 1; ; attempt += 1) {
-          try {
-            response = await sendTelegramBusLocalEnvelope({
-              socketPath: leaderSocketPath,
-              timeoutMs: registrationTimeoutMs,
-              envelope: createRegistrationEnvelope(),
-            });
-            break;
-          } catch (error) {
-            if (
-              attempt >= registrationRetryAttempts ||
-              !isRetryableTelegramBusRegistrationError(error)
-            ) {
-              throw error;
-            }
-            deps.recordRuntimeEvent?.("bus", error, {
-              phase: "follower-register-retry",
-              attempt,
-              socketPath: leaderSocketPath,
+        response = await sendTelegramBusLocalEnvelope({
+          socketPath: leaderSocketPath,
+          timeoutMs: registrationTimeoutMs,
+          envelope: createRegistrationEnvelope(),
+          retry: getTelegramBusTransportRetryPolicy({
+            endpoint: leaderSocketPath,
+            operation: "registration",
+            overrides: {
+              attempts: registrationRetryAttempts,
+              delayMs: registrationRetryDelayMs,
+            },
+          }),
+          recordTransportEvent(phase, details) {
+            deps.recordRuntimeEvent?.("bus", `Telegram bus ${phase}`, {
+              phase: `follower-register-${phase}`,
+              ...details,
             });
-            await delayTelegramBusFollowerRegistration(
-              registrationRetryDelayMs,
-            );
-          }
-        }
+          },
+        });
       } catch (error) {
         stopHeartbeat();
         activeLeaderSocketPath = undefined;
@@ -781,6 +795,12 @@ export function createTelegramBusForwardedUpdateReceiverRuntime<
 ): TelegramBusForwardedUpdateReceiverRuntime {
   const server = createTelegramBusLocalServer({
     socketPath: deps.socketPath,
+    recordTransportEvent(phase, details) {
+      deps.recordRuntimeEvent?.("bus", `Telegram bus ${phase}`, {
+        phase: `follower-receiver-${phase}`,
+        ...details,
+      });
+    },
     async handleEnvelope(envelope) {
       const authSecret = deps.getAuthSecret?.();
       if (deps.getAuthSecret && (!authSecret || envelope.auth !== authSecret)) {

package/lib/bus-leader.ts

@@ -21,6 +21,7 @@ import {
   type TelegramBusFollowerView,
   type TelegramBusInstanceRegistration,
 } from "./bus.ts";
+import { getTelegramBusTransportRetryPolicy } from "./bus-transport.ts";
 
 export interface TelegramBusLeaderRuntime<TContext> {
   startPolling: (ctx: TContext) => Promise<void>;
@@ -162,6 +163,17 @@ export interface TelegramBusFollowerRegistryRestoreDeps {
   getNowMs?: () => number;
 }
 
+export interface TelegramBusFollowerMessageOwnershipRecord {
+  follower: TelegramBusFollowerView;
+  chatId: number;
+  messageId: number;
+  target?: TelegramTarget;
+}
+
+export type TelegramBusFollowerMessageOwnershipRecorder = (
+  record: TelegramBusFollowerMessageOwnershipRecord,
+) => void;
+
 export interface TelegramBusLeaderRuntimeDeps<TContext> {
   socketPath: string;
   followerRegistry: TelegramBusFollowerRegistry;
@@ -174,6 +186,7 @@ export interface TelegramBusLeaderRuntimeDeps<TContext> {
     method: string;
     args: unknown[];
   }) => boolean;
+  recordFollowerMessageOwnership?: TelegramBusFollowerMessageOwnershipRecorder;
   provisionFollowerTarget?: (
     registration: TelegramBusInstanceRegistration,
   ) => Promise<TelegramTarget | undefined> | TelegramTarget | undefined;
@@ -324,30 +337,74 @@ export function createTelegramBusFollowerTargetProvisioner(
       registration.profileKey ?? `manual:${registration.instanceId}`;
     const followerOwner =
       Threads.getTelegramThreadOwnerFromProfileKey(followerProfileKey);
-    deps.onProvisioningStart?.();
-    let result: Threads.TelegramTopicTargetProvisionResult;
-    try {
-      result = await provision({
-        instanceId: registration.instanceId,
-        owner:
-          followerOwner.kind === "manual-follower"
-            ? followerOwner
-            : {
-                kind: "manual-follower",
-                instanceId: registration.instanceId,
-              },
-        profileKey: followerProfileKey,
-        threadName: registration.threadName,
-      });
-    } finally {
-      deps.onProvisioningEnd?.();
-    }
+    const provisionTarget = async () => {
+      deps.onProvisioningStart?.();
+      try {
+        return await provision({
+          instanceId: registration.instanceId,
+          owner:
+            followerOwner.kind === "manual-follower"
+              ? followerOwner
+              : {
+                  kind: "manual-follower",
+                  instanceId: registration.instanceId,
+                },
+          profileKey: followerProfileKey,
+          threadName: registration.threadName,
+        });
+      } finally {
+        deps.onProvisioningEnd?.();
+      }
+    };
+    let result = await provisionTarget();
     deps.setSyncState(
       Sync.markTelegramSyncSliceFresh(deps.getSyncState(), "target-bindings", {
         nowMs: getNowMs(),
         action: "follower-register",
       }),
     );
+    let connectedAnnouncement =
+      createTelegramBusInstanceLifecycleAnnouncement({
+        target: result.target,
+        threadName: result.record.threadName,
+        slot: result.record.slot,
+        state: "connected",
+      });
+    let connectedAnnouncementSent = false;
+    if (result.reused && connectedAnnouncement) {
+      try {
+        await deps.callApi("sendMessage", {
+          chat_id: connectedAnnouncement.target.chatId,
+          message_thread_id: connectedAnnouncement.target.threadId,
+          text: connectedAnnouncement.text,
+          parse_mode: connectedAnnouncement.parseMode,
+        });
+        connectedAnnouncementSent = true;
+      } catch (error) {
+        deps.recordRuntimeEvent("telegram", error, {
+          phase: "follower-topic-reuse-probe",
+          instanceId: registration.instanceId,
+          chatId: result.target.chatId,
+          threadId: result.target.threadId,
+        });
+        if (Threads.isTelegramTopicTargetStaleError(error)) {
+          deps.topicTargetStore.markStaleByTarget(
+            result.target,
+            "deleted",
+            "Follower registration found the reusable thread target stale.",
+          );
+          await deps.topicTargetStore.persist();
+          result = await provisionTarget();
+          connectedAnnouncement = createTelegramBusInstanceLifecycleAnnouncement({
+            target: result.target,
+            threadName: result.record.threadName,
+            slot: result.record.slot,
+            state: "connected",
+          });
+          connectedAnnouncementSent = false;
+        }
+      }
+    }
     recordSlowTelegramBusFollowerRegistrationStep(deps, {
       phase: "follower-register-critical",
       elapsedMs: Date.now() - registrationStartedAtMs,
@@ -355,16 +412,9 @@ export function createTelegramBusFollowerTargetProvisioner(
       target: result.target,
       reused: result.reused,
     });
-    const connectedAnnouncement =
-      createTelegramBusInstanceLifecycleAnnouncement({
-        target: result.target,
-        threadName: result.record.threadName,
-        slot: result.record.slot,
-        state: "connected",
-      });
     scheduleTelegramBusLeaderBackgroundTask(async () => {
       const backgroundStartedAtMs = Date.now();
-      if (connectedAnnouncement) {
+      if (connectedAnnouncement && !connectedAnnouncementSent) {
         try {
           await deps.callApi("sendMessage", {
             chat_id: connectedAnnouncement.target.chatId,
@@ -578,6 +628,7 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
     method: string;
     args: unknown[];
   }) => boolean;
+  recordFollowerMessageOwnership?: TelegramBusFollowerMessageOwnershipRecorder;
   provisionFollowerTarget?: (
     registration: TelegramBusInstanceRegistration,
   ) => Promise<TelegramTarget | undefined> | TelegramTarget | undefined;
@@ -598,31 +649,22 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
     >,
   ): Promise<TelegramBusEnvelope> => {
     const follower = deps.followerRegistry.get(envelope.recipientInstanceId);
-    if (!follower) {
-      return {
-        kind: "bus.ack",
-        requestId: envelope.requestId,
-        ok: false,
-        message: "Unknown Telegram bus follower instance.",
-      };
-    }
-    if (!follower.busSocketPath) {
-      return {
-        kind: "bus.ack",
-        requestId: envelope.requestId,
-        ok: false,
-        message: "Telegram bus follower does not expose a receiver socket.",
-      };
-    }
-    deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
+    const followerSocketPath =
+      follower?.busSocketPath ??
+      getTelegramBusFollowerSocketPath(envelope.recipientInstanceId);
+    if (follower) deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
     try {
       const response = await sendTelegramBusLocalEnvelope({
-        socketPath: follower.busSocketPath,
+        socketPath: followerSocketPath,
         envelope,
         timeoutMs: deps.timeoutMs,
+        retry: getTelegramBusTransportRetryPolicy({
+          endpoint: followerSocketPath,
+          operation: "operation",
+        }),
       });
       if (response?.kind === "bus.ack" && response.ok) {
-        deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
+        if (follower) deps.followerRegistry.heartbeat(follower.instanceId, getNowMs());
         return { kind: "bus.ack", requestId: envelope.requestId, ok: true };
       }
       const message =
@@ -714,6 +756,72 @@ export function createTelegramBusLeaderEnvelopeHandler(deps: {
   };
 }
 
+function asRecord(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function asInteger(value: unknown): number | undefined {
+  if (typeof value === "number" && Number.isInteger(value)) return value;
+  if (typeof value !== "string" || value.trim() === "") return undefined;
+  const parsed = Number(value);
+  return Number.isInteger(parsed) ? parsed : undefined;
+}
+
+function getFollowerApiMethodAndBody(envelope: Extract<TelegramBusEnvelope, { kind: "follower.callApi" }>): {
+  apiMethod: string;
+  body?: Record<string, unknown>;
+} {
+  if (envelope.method === "call" || envelope.method === "callMultipart") {
+    return {
+      apiMethod:
+        typeof envelope.args[0] === "string" ? envelope.args[0] : "",
+      body: asRecord(envelope.args[1]),
+    };
+  }
+  return { apiMethod: envelope.method, body: asRecord(envelope.args[0]) };
+}
+
+function getSentMessageIds(result: unknown): number[] {
+  const values = Array.isArray(result) ? result : [result];
+  return values
+    .map((value) => asInteger(asRecord(value)?.message_id))
+    .filter((messageId): messageId is number => messageId !== undefined);
+}
+
+function recordFollowerApiMessageOwnership(input: {
+  envelope: Extract<TelegramBusEnvelope, { kind: "follower.callApi" }>;
+  follower: TelegramBusFollowerView;
+  result: unknown;
+  record?: TelegramBusFollowerMessageOwnershipRecorder;
+}): void {
+  if (!input.record) return;
+  const { apiMethod, body } = getFollowerApiMethodAndBody(input.envelope);
+  if (
+    apiMethod !== "sendMessage" &&
+    apiMethod !== "sendRichMessage" &&
+    apiMethod !== "sendPhoto" &&
+    apiMethod !== "sendDocument" &&
+    apiMethod !== "sendVoice" &&
+    apiMethod !== "sendMediaGroup"
+  ) {
+    return;
+  }
+  const chatId = asInteger(body?.chat_id) ?? input.follower.target?.chatId;
+  if (chatId === undefined) return;
+  const threadId = asInteger(body?.message_thread_id) ?? input.follower.target?.threadId;
+  const target = threadId !== undefined ? { chatId, threadId } : { chatId };
+  for (const messageId of getSentMessageIds(input.result)) {
+    input.record({
+      follower: input.follower,
+      chatId,
+      messageId,
+      target,
+    });
+  }
+}
+
 async function handleFollowerApiCall(
   envelope: Extract<TelegramBusEnvelope, { kind: "follower.callApi" }>,
   deps: {
@@ -725,6 +833,7 @@ async function handleFollowerApiCall(
       method: string;
       args: unknown[];
     }) => boolean;
+    recordFollowerMessageOwnership?: TelegramBusFollowerMessageOwnershipRecorder;
   },
 ): Promise<TelegramBusEnvelope> {
   const follower = deps.followerRegistry.get(envelope.instanceId);
@@ -761,11 +870,18 @@ async function handleFollowerApiCall(
     };
   }
   try {
+    const result = await deps.callApi(envelope.method, envelope.args);
+    recordFollowerApiMessageOwnership({
+      envelope,
+      follower,
+      result,
+      record: deps.recordFollowerMessageOwnership,
+    });
     return {
       kind: "bus.ack",
       requestId: envelope.requestId,
       ok: true,
-      result: await deps.callApi(envelope.method, envelope.args),
+      result,
     };
   } catch (error) {
     return {
@@ -842,7 +958,7 @@ export function createTelegramBusLeaderRuntime<TContext>(
 ): TelegramBusLeaderRuntime<TContext> {
   const getNowMs = deps.getNowMs ?? Date.now;
   const followerPruneIntervalMs = deps.followerPruneIntervalMs ?? 1000;
-  const followerStaleAfterMs = deps.followerStaleAfterMs ?? 2000;
+  const followerStaleAfterMs = deps.followerStaleAfterMs ?? 5000;
   let pruneInterval: ReturnType<typeof setInterval> | undefined;
   const stopPruning = () => {
     if (!pruneInterval) return;
@@ -878,12 +994,19 @@ export function createTelegramBusLeaderRuntime<TContext>(
   };
   const localServer = createTelegramBusLocalServer({
     socketPath: deps.socketPath,
+    recordTransportEvent(phase, details) {
+      deps.recordRuntimeEvent?.("bus", `Telegram bus ${phase}`, {
+        phase: `leader-${phase}`,
+        ...details,
+      });
+    },
     handleEnvelope: createTelegramBusLeaderEnvelopeHandler({
       followerRegistry: deps.followerRegistry,
       authSecret: deps.authSecret,
       getNowMs,
       callApi: deps.callApi,
       authorizeFollowerApiCall: deps.authorizeFollowerApiCall,
+      recordFollowerMessageOwnership: deps.recordFollowerMessageOwnership,
       provisionFollowerTarget: deps.provisionFollowerTarget,
     }),
   });
@@ -917,6 +1040,7 @@ export function createTelegramBusLeaderRuntime<TContext>(
           .catch((error) =>
             deps.recordRuntimeEvent?.("bus", error, { phase: "stop" }),
           );
+        deps.followerRegistry.clear();
       }
     },
   };