@llblab/pi-telegram 0.54.3 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -2
- package/BACKLOG.md +1 -1
- package/CHANGELOG.md +21 -0
- package/README.md +5 -3
- package/dist/lib/activity.d.ts +5 -1
- package/dist/lib/activity.js +31 -2
- package/dist/lib/bindings.d.ts +5 -1
- package/dist/lib/bindings.js +9 -2
- package/dist/lib/bus-api.d.ts +2 -2
- package/dist/lib/bus-api.js +7 -2
- package/dist/lib/bus-follower.d.ts +49 -3
- package/dist/lib/bus-follower.js +313 -146
- package/dist/lib/bus-leader.d.ts +3 -0
- package/dist/lib/bus-leader.js +99 -18
- package/dist/lib/bus.d.ts +5 -3
- package/dist/lib/bus.js +13 -10
- package/dist/lib/delivery.d.ts +4 -0
- package/dist/lib/delivery.js +48 -15
- package/dist/lib/extension.js +56 -6
- package/dist/lib/lifecycle.d.ts +68 -0
- package/dist/lib/lifecycle.js +178 -58
- package/dist/lib/machine.d.ts +23 -0
- package/dist/lib/machine.js +37 -0
- package/dist/lib/media.d.ts +114 -28
- package/dist/lib/media.js +292 -119
- package/dist/lib/model.d.ts +54 -15
- package/dist/lib/model.js +98 -53
- package/dist/lib/polling.js +2 -0
- package/dist/lib/preview.d.ts +128 -113
- package/dist/lib/preview.js +448 -366
- package/dist/lib/queue.d.ts +2 -2
- package/dist/lib/queue.js +3 -3
- package/dist/lib/routing.d.ts +14 -2
- package/dist/lib/routing.js +546 -207
- package/dist/lib/runtime.d.ts +77 -11
- package/dist/lib/runtime.js +174 -92
- package/dist/lib/sections.js +0 -1
- package/dist/lib/telegram-api.d.ts +11 -9
- package/dist/lib/telegram-api.js +53 -4
- package/dist/lib/text-groups.d.ts +0 -26
- package/dist/lib/text-groups.js +51 -98
- package/dist/lib/thread-display.js +14 -6
- package/dist/lib/threads.d.ts +20 -5
- package/dist/lib/threads.js +111 -32
- package/dist/lib/turns.d.ts +2 -0
- package/dist/lib/turns.js +6 -1
- package/dist/lib/updates.d.ts +11 -4
- package/dist/lib/updates.js +54 -19
- package/dist/package.json +1 -1
- package/docs/activity.md +1 -1
- package/docs/architecture.md +16 -15
- package/docs/delivery.md +7 -5
- package/docs/multi-instance-bus.md +18 -8
- package/docs/public-api.md +2 -2
- package/docs/ui-style.md +8 -6
- package/lib/activity.ts +29 -5
- package/lib/bindings.ts +13 -1
- package/lib/bus-api.ts +11 -3
- package/lib/bus-follower.ts +417 -203
- package/lib/bus-leader.ts +88 -14
- package/lib/bus.ts +17 -15
- package/lib/delivery.ts +47 -19
- package/lib/extension.ts +59 -9
- package/lib/lifecycle.ts +229 -55
- package/lib/machine.ts +60 -0
- package/lib/media.ts +438 -172
- package/lib/model.ts +169 -74
- package/lib/polling.ts +2 -0
- package/lib/preview.ts +715 -566
- package/lib/queue.ts +5 -5
- package/lib/routing.ts +644 -283
- package/lib/runtime.ts +260 -104
- package/lib/sections.ts +0 -2
- package/lib/telegram-api.ts +61 -8
- package/lib/text-groups.ts +75 -136
- package/lib/thread-display.ts +15 -4
- package/lib/threads.ts +130 -38
- package/lib/turns.ts +8 -1
- package/lib/updates.ts +58 -21
- package/package.json +1 -1
package/lib/bus-follower.ts
CHANGED
|
@@ -44,6 +44,7 @@ import {
|
|
|
44
44
|
TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_COMMAND_SET,
|
|
45
45
|
TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_SAVE,
|
|
46
46
|
TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_SETTLE,
|
|
47
|
+
TELEGRAM_BUS_CAPABILITY_PRE_DISPATCH_REBIND_EXECUTION,
|
|
47
48
|
TELEGRAM_BUS_CAPABILITY_SELECTED_MENU_DELIVERY,
|
|
48
49
|
TELEGRAM_BUS_CAPABILITY_SESSION_REPLACEMENT_INTENT,
|
|
49
50
|
TELEGRAM_BUS_CAPABILITY_THREAD_DISPLAY_MODE,
|
|
@@ -82,6 +83,7 @@ import {
|
|
|
82
83
|
type TelegramLockEntry,
|
|
83
84
|
type TelegramLockState,
|
|
84
85
|
} from "./locks.ts";
|
|
86
|
+
import * as Machine from "./machine.ts";
|
|
85
87
|
import { isPiStaleContextError } from "./pi.ts";
|
|
86
88
|
import type { TelegramQueueHandoffStageResult } from "./queue.ts";
|
|
87
89
|
import {
|
|
@@ -90,9 +92,11 @@ import {
|
|
|
90
92
|
} from "./target.ts";
|
|
91
93
|
import {
|
|
92
94
|
isTelegramApiMethodRetrySafe,
|
|
95
|
+
assertTelegramApiCallAuthority,
|
|
93
96
|
TelegramApiAuthorityError,
|
|
94
97
|
TelegramApiCommitUnknownError,
|
|
95
98
|
TelegramApiStaleTargetError,
|
|
99
|
+
type TelegramApiCallOptions,
|
|
96
100
|
} from "./telegram-api.ts";
|
|
97
101
|
import * as Threads from "./threads.ts";
|
|
98
102
|
import { isWireRecord as isRecord } from "./wire.ts";
|
|
@@ -709,10 +713,10 @@ export function createTelegramBusFollowerRuntimeAssembly<
|
|
|
709
713
|
sessionId: string | undefined;
|
|
710
714
|
}
|
|
711
715
|
| undefined;
|
|
712
|
-
const isReadyContext = (
|
|
716
|
+
const isReadyContext = (): boolean =>
|
|
713
717
|
Boolean(
|
|
714
718
|
readyContext &&
|
|
715
|
-
(
|
|
719
|
+
ports.receiver.getContext() === readyContext.ctx &&
|
|
716
720
|
ports.registrationState.getGeneration() === readyContext.generation &&
|
|
717
721
|
ports.registration.getSessionGeneration?.() ===
|
|
718
722
|
readyContext.sessionGeneration &&
|
|
@@ -724,24 +728,42 @@ export function createTelegramBusFollowerRuntimeAssembly<
|
|
|
724
728
|
const generation = ports.registrationState.getGeneration();
|
|
725
729
|
const sessionGeneration = ports.registration.getSessionGeneration?.();
|
|
726
730
|
const sessionId = ports.registration.getSessionId?.(ctx);
|
|
731
|
+
const executionContext = ports.receiver.getContext();
|
|
727
732
|
readyContext = undefined;
|
|
728
733
|
if (
|
|
729
734
|
!generation ||
|
|
735
|
+
executionContext === undefined ||
|
|
736
|
+
ports.registration.isContextActive?.(ctx) === false ||
|
|
737
|
+
ports.registration.isContextActive?.(executionContext) === false ||
|
|
738
|
+
ports.registration.getSessionId?.(executionContext) !== sessionId ||
|
|
739
|
+
(executionContext !== ctx &&
|
|
740
|
+
(sessionId === undefined ||
|
|
741
|
+
!ports.registration.isContextActive ||
|
|
742
|
+
!ports.registration.getSessionGeneration)) ||
|
|
730
743
|
(sessionId !== undefined &&
|
|
731
744
|
ports.registrationState.getSessionId(sessionId) !== sessionId)
|
|
732
745
|
)
|
|
733
746
|
throw new Error(
|
|
734
|
-
"Telegram follower session change requires acknowledged leader registration.",
|
|
747
|
+
"Telegram follower session change requires acknowledged leader registration and a current canonical context.",
|
|
735
748
|
);
|
|
736
|
-
|
|
749
|
+
// Connect has a command context; inbound work uses the session's canonical object.
|
|
750
|
+
await ports.registration.onRegistered?.(executionContext);
|
|
737
751
|
if (
|
|
738
752
|
generation &&
|
|
739
753
|
ports.registrationState.getGeneration() === generation &&
|
|
740
754
|
ports.registration.getSessionGeneration?.() === sessionGeneration &&
|
|
755
|
+
ports.receiver.getContext() === executionContext &&
|
|
741
756
|
ports.registration.getSessionId?.(ctx) === sessionId &&
|
|
742
|
-
ports.registration.
|
|
757
|
+
ports.registration.getSessionId?.(executionContext) === sessionId &&
|
|
758
|
+
ports.registration.isContextActive?.(ctx) !== false &&
|
|
759
|
+
ports.registration.isContextActive?.(executionContext) !== false
|
|
743
760
|
)
|
|
744
|
-
readyContext = {
|
|
761
|
+
readyContext = {
|
|
762
|
+
generation,
|
|
763
|
+
ctx: executionContext,
|
|
764
|
+
sessionGeneration,
|
|
765
|
+
sessionId,
|
|
766
|
+
};
|
|
745
767
|
};
|
|
746
768
|
const sharedRuntimeDeps = {
|
|
747
769
|
instanceId: ports.instanceId,
|
|
@@ -792,7 +814,7 @@ export function createTelegramBusFollowerRuntimeAssembly<
|
|
|
792
814
|
ports.registration.isContextActive?.(ctx) === false
|
|
793
815
|
)
|
|
794
816
|
return;
|
|
795
|
-
if (!isReadyContext(
|
|
817
|
+
if (!isReadyContext()) await prepareContext(ctx);
|
|
796
818
|
},
|
|
797
819
|
};
|
|
798
820
|
return {
|
|
@@ -929,7 +951,7 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
929
951
|
isCurrent(): boolean;
|
|
930
952
|
/** Captured prepared-branch target; equality must precede local apply or release grants. */
|
|
931
953
|
expectedTarget?: TelegramTarget & { threadId: number };
|
|
932
|
-
release?: (canRelease: () => boolean) => void;
|
|
954
|
+
release?: (canRelease: () => boolean, target: TelegramTarget & { threadId: number }) => void;
|
|
933
955
|
observeWork?: (
|
|
934
956
|
oldTarget: TelegramTarget & { threadId: number },
|
|
935
957
|
) => void;
|
|
@@ -952,6 +974,9 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
952
974
|
workObserver = scope && ports!.observeWork?.bind(scope);
|
|
953
975
|
const captured = deps.getContextAuthority(ctx);
|
|
954
976
|
if (!captured) throw new Error("Stale Telegram follower Restore context.");
|
|
977
|
+
const snapshotStore = deps.topicTargetStore,
|
|
978
|
+
liveSnapshot = snapshotStore.withWorkspaceLiveRebindSnapshot,
|
|
979
|
+
restoreSnapshot = snapshotStore.withWorkspaceRestoreSnapshot;
|
|
955
980
|
const authority = structuredClone(captured);
|
|
956
981
|
const request = structuredClone({
|
|
957
982
|
operationId: input.operationId,
|
|
@@ -1015,6 +1040,10 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
1015
1040
|
const assertCurrent = (): void => {
|
|
1016
1041
|
if (
|
|
1017
1042
|
!isDeepStrictEqual(deps.getContextAuthority(ctx), authority) ||
|
|
1043
|
+
deps.topicTargetStore !== snapshotStore ||
|
|
1044
|
+
(liveCurrent
|
|
1045
|
+
? snapshotStore.withWorkspaceLiveRebindSnapshot !== liveSnapshot
|
|
1046
|
+
: snapshotStore.withWorkspaceRestoreSnapshot !== restoreSnapshot) ||
|
|
1018
1047
|
liveCurrent?.() === false ||
|
|
1019
1048
|
(scope &&
|
|
1020
1049
|
(input.liveRebind !== scope ||
|
|
@@ -1042,6 +1071,7 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
1042
1071
|
(liveCurrent
|
|
1043
1072
|
? !("kind" in intent) ||
|
|
1044
1073
|
intent.kind !== "live-rebind" ||
|
|
1074
|
+
(intent.inputRelease !== undefined && !hasTelegramBusCapability(authority.leaderProtocol, TELEGRAM_BUS_CAPABILITY_PRE_DISPATCH_REBIND_EXECUTION)) ||
|
|
1045
1075
|
!isDeepStrictEqual(
|
|
1046
1076
|
[...intent.request.source.updateIds].sort((a, b) => a - b),
|
|
1047
1077
|
sourceUpdateIds,
|
|
@@ -1071,7 +1101,7 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
1071
1101
|
"Telegram follower Restore intent does not match this recipient.",
|
|
1072
1102
|
);
|
|
1073
1103
|
const expected = structuredClone(intent);
|
|
1074
|
-
await
|
|
1104
|
+
await snapshotStore.load();
|
|
1075
1105
|
assertCurrent();
|
|
1076
1106
|
const observe = (
|
|
1077
1107
|
canonical: Readonly<
|
|
@@ -1159,13 +1189,19 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
1159
1189
|
ready,
|
|
1160
1190
|
};
|
|
1161
1191
|
};
|
|
1192
|
+
const withSnapshot = (observer: Parameters<typeof restoreSnapshot>[1]): void => {
|
|
1193
|
+
assertCurrent();
|
|
1194
|
+
if ("kind" in expected) {
|
|
1195
|
+
if (!liveSnapshot)
|
|
1196
|
+
throw new Error("Live-rebind read-only Workspace snapshot is unavailable.");
|
|
1197
|
+
liveSnapshot.call(snapshotStore, expected, observer);
|
|
1198
|
+
} else restoreSnapshot.call(snapshotStore, expected, observer);
|
|
1199
|
+
assertCurrent();
|
|
1200
|
+
};
|
|
1162
1201
|
let result: TelegramBusWorkspaceRestoreObservation | undefined;
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
result = observe(snapshot);
|
|
1167
|
-
},
|
|
1168
|
-
);
|
|
1202
|
+
withSnapshot((snapshot) => {
|
|
1203
|
+
result = observe(snapshot);
|
|
1204
|
+
});
|
|
1169
1205
|
if (!result)
|
|
1170
1206
|
throw new Error(
|
|
1171
1207
|
"Telegram follower Restore observation was not completed.",
|
|
@@ -1174,40 +1210,35 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
1174
1210
|
if (
|
|
1175
1211
|
!("kind" in expected) ||
|
|
1176
1212
|
expected.kind !== "live-rebind" ||
|
|
1177
|
-
|
|
1213
|
+
(release ? !Threads.isTelegramLiveRebindInputAuthorized(expected)
|
|
1214
|
+
: !(Threads.isTelegramLiveRebindInputAuthorized(expected) || expected.phase === "released" && expected.inputRelease === "held" && !expected.cleanup)) ||
|
|
1178
1215
|
!result.ready ||
|
|
1179
1216
|
request.mode !== "inspect"
|
|
1180
1217
|
)
|
|
1181
1218
|
throw new Error(
|
|
1182
1219
|
"Live-rebind dispatch has no canonical released/local-applied grant.",
|
|
1183
1220
|
);
|
|
1184
|
-
//
|
|
1221
|
+
// Live barriers reobserve read-only Workspace evidence; journal reads never borrow publication authority.
|
|
1185
1222
|
if (workObserver)
|
|
1186
1223
|
sampleWork = () => {
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
assertCurrent();
|
|
1196
|
-
},
|
|
1197
|
-
);
|
|
1224
|
+
withSnapshot((snapshot) => {
|
|
1225
|
+
if (!observe(snapshot).ready)
|
|
1226
|
+
throw new Error(
|
|
1227
|
+
"Live-rebind work observation lost local target application.",
|
|
1228
|
+
);
|
|
1229
|
+
workObserver(expected.request.binding.target);
|
|
1230
|
+
assertCurrent();
|
|
1231
|
+
});
|
|
1198
1232
|
};
|
|
1199
1233
|
release?.(() => {
|
|
1200
1234
|
assertCurrent();
|
|
1201
1235
|
let ready = false;
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
ready = observe(snapshot).ready;
|
|
1206
|
-
},
|
|
1207
|
-
);
|
|
1236
|
+
withSnapshot((snapshot) => {
|
|
1237
|
+
ready = observe(snapshot).ready;
|
|
1238
|
+
});
|
|
1208
1239
|
assertCurrent();
|
|
1209
1240
|
return ready;
|
|
1210
|
-
});
|
|
1241
|
+
}, { ...expected.request.target });
|
|
1211
1242
|
}
|
|
1212
1243
|
return result;
|
|
1213
1244
|
},
|
|
@@ -1352,8 +1383,7 @@ export function createTelegramBusFollowerWorkspaceRestoreHandler<
|
|
|
1352
1383
|
if (
|
|
1353
1384
|
!value ||
|
|
1354
1385
|
value.kind !== "live-rebind" ||
|
|
1355
|
-
value
|
|
1356
|
-
value.cleanup ||
|
|
1386
|
+
!Threads.isTelegramLiveRebindInputAuthorized(value) ||
|
|
1357
1387
|
value.request.operationId !== expected.operationId ||
|
|
1358
1388
|
value.recipient.kind !== "follower" ||
|
|
1359
1389
|
value.recipient.instanceId !== instanceId ||
|
|
@@ -1846,14 +1876,18 @@ export function createTelegramBusFollowerSelectedMenuCaller<TContext>(deps: {
|
|
|
1846
1876
|
|
|
1847
1877
|
export function createTelegramBusFollowerApiCaller(
|
|
1848
1878
|
deps: TelegramBusFollowerApiCallerDeps,
|
|
1849
|
-
): (method: string, args: unknown[]) => Promise<unknown> {
|
|
1879
|
+
): (method: string, args: unknown[], localOptions?: Pick<TelegramApiCallOptions, "assertAuthority">) => Promise<unknown> {
|
|
1850
1880
|
const getNowMs = deps.getNowMs ?? Date.now;
|
|
1851
1881
|
const timeoutMs = deps.timeoutMs ?? TELEGRAM_BUS_FOLLOWER_CLIENT_TIMEOUT_MS;
|
|
1852
|
-
return async (method, args) => {
|
|
1882
|
+
return async (method, args, localOptions) => {
|
|
1883
|
+
const assertion = localOptions?.assertAuthority;
|
|
1884
|
+
const assertCurrent = () => assertTelegramApiCallAuthority(assertion, false);
|
|
1885
|
+
assertCurrent();
|
|
1853
1886
|
const registration = await resolveTelegramBusFollowerRegistration(
|
|
1854
1887
|
deps,
|
|
1855
1888
|
timeoutMs,
|
|
1856
1889
|
);
|
|
1890
|
+
assertCurrent();
|
|
1857
1891
|
const socketPath = resolveTelegramBusSocketPath(deps.socketPath);
|
|
1858
1892
|
let response: TelegramBusEnvelope | undefined;
|
|
1859
1893
|
try {
|
|
@@ -1870,8 +1904,10 @@ export function createTelegramBusFollowerApiCaller(
|
|
|
1870
1904
|
args,
|
|
1871
1905
|
sentAtMs: getNowMs(),
|
|
1872
1906
|
},
|
|
1907
|
+
assertion ? assertCurrent : undefined,
|
|
1873
1908
|
);
|
|
1874
1909
|
} catch (error) {
|
|
1910
|
+
if (error instanceof TelegramBusLocalAuthorityError) throw new TelegramApiAuthorityError(error.requestIssued);
|
|
1875
1911
|
const apiMethod =
|
|
1876
1912
|
(method === "call" || method === "callMultipart") &&
|
|
1877
1913
|
typeof args[0] === "string"
|
|
@@ -2223,186 +2259,345 @@ export type TelegramBusFollowerHeartbeatRecoveryHandler<TContext> = ((
|
|
|
2223
2259
|
ctx: TContext,
|
|
2224
2260
|
) => Promise<void>) & { halt: () => void; resume: () => void };
|
|
2225
2261
|
|
|
2262
|
+
/** The whole recovery lifecycle state; only `transitionTelegramBusFollowerRecovery` produces a new one. */
|
|
2263
|
+
export interface TelegramBusFollowerRecoveryState {
|
|
2264
|
+
/** Bumped by halt and resume, so a recovery or retry from before an operator stop or connect is inert. */
|
|
2265
|
+
readonly epoch: number;
|
|
2266
|
+
readonly halted: boolean;
|
|
2267
|
+
/** One recovery runs at a time; it belongs to the epoch it started in. */
|
|
2268
|
+
readonly recovering: boolean;
|
|
2269
|
+
readonly retryId: number | null;
|
|
2270
|
+
readonly nextRetryId: number;
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
export type TelegramBusFollowerRecoveryEvent =
|
|
2274
|
+
| { type: "recoverRequested" }
|
|
2275
|
+
| { type: "recoverySettled"; epoch: number }
|
|
2276
|
+
| { type: "retryRequested"; epoch: number }
|
|
2277
|
+
| { type: "retryFired"; retryId: number }
|
|
2278
|
+
| { type: "halted" }
|
|
2279
|
+
| { type: "resumed" };
|
|
2280
|
+
|
|
2281
|
+
export type TelegramBusFollowerRecoveryEffect =
|
|
2282
|
+
| { type: "runRecovery"; epoch: number }
|
|
2283
|
+
| { type: "armRetry"; retryId: number }
|
|
2284
|
+
| { type: "cancelRetry"; retryId: number };
|
|
2285
|
+
|
|
2286
|
+
export function createTelegramBusFollowerRecoveryState(): TelegramBusFollowerRecoveryState {
|
|
2287
|
+
return {
|
|
2288
|
+
epoch: 0,
|
|
2289
|
+
halted: false,
|
|
2290
|
+
recovering: false,
|
|
2291
|
+
retryId: null,
|
|
2292
|
+
nextRetryId: 0,
|
|
2293
|
+
};
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
/**
|
|
2297
|
+
* Pure follower-recovery lifecycle. An operator stop halts it and a connect resumes it, each in a new epoch: work
|
|
2298
|
+
* from the old epoch, a recovery still awaiting or a retry still armed, can no longer register or promote.
|
|
2299
|
+
*/
|
|
2300
|
+
export function transitionTelegramBusFollowerRecovery(
|
|
2301
|
+
state: TelegramBusFollowerRecoveryState,
|
|
2302
|
+
event: TelegramBusFollowerRecoveryEvent,
|
|
2303
|
+
): Machine.TelegramMachineStep<
|
|
2304
|
+
TelegramBusFollowerRecoveryState,
|
|
2305
|
+
TelegramBusFollowerRecoveryEffect
|
|
2306
|
+
> {
|
|
2307
|
+
const cancelRetry: TelegramBusFollowerRecoveryEffect[] =
|
|
2308
|
+
state.retryId === null
|
|
2309
|
+
? []
|
|
2310
|
+
: [{ type: "cancelRetry", retryId: state.retryId }];
|
|
2311
|
+
const start = (
|
|
2312
|
+
from: TelegramBusFollowerRecoveryState,
|
|
2313
|
+
): Machine.TelegramMachineStep<
|
|
2314
|
+
TelegramBusFollowerRecoveryState,
|
|
2315
|
+
TelegramBusFollowerRecoveryEffect
|
|
2316
|
+
> =>
|
|
2317
|
+
from.halted || from.recovering
|
|
2318
|
+
? { state: from, effects: [] }
|
|
2319
|
+
: {
|
|
2320
|
+
state: { ...from, recovering: true },
|
|
2321
|
+
effects: [{ type: "runRecovery", epoch: from.epoch }],
|
|
2322
|
+
};
|
|
2323
|
+
switch (event.type) {
|
|
2324
|
+
case "recoverRequested":
|
|
2325
|
+
return start(state);
|
|
2326
|
+
case "recoverySettled":
|
|
2327
|
+
return event.epoch === state.epoch
|
|
2328
|
+
? { state: { ...state, recovering: false }, effects: [] }
|
|
2329
|
+
: { state, effects: [] };
|
|
2330
|
+
case "retryRequested": {
|
|
2331
|
+
if (event.epoch !== state.epoch || state.halted)
|
|
2332
|
+
return { state, effects: [] };
|
|
2333
|
+
const retryId = state.nextRetryId + 1;
|
|
2334
|
+
return {
|
|
2335
|
+
state: { ...state, retryId, nextRetryId: retryId },
|
|
2336
|
+
effects: [...cancelRetry, { type: "armRetry", retryId }],
|
|
2337
|
+
};
|
|
2338
|
+
}
|
|
2339
|
+
case "retryFired":
|
|
2340
|
+
if (state.retryId !== event.retryId) return { state, effects: [] };
|
|
2341
|
+
return start({ ...state, retryId: null });
|
|
2342
|
+
case "halted":
|
|
2343
|
+
case "resumed":
|
|
2344
|
+
return {
|
|
2345
|
+
state: {
|
|
2346
|
+
...state,
|
|
2347
|
+
epoch: state.epoch + 1,
|
|
2348
|
+
halted: event.type === "halted",
|
|
2349
|
+
recovering: false,
|
|
2350
|
+
retryId: null,
|
|
2351
|
+
},
|
|
2352
|
+
effects: cancelRetry,
|
|
2353
|
+
};
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2226
2357
|
export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
2227
2358
|
deps: TelegramBusFollowerHeartbeatRecoveryHandlerDeps<TContext>,
|
|
2228
2359
|
): TelegramBusFollowerHeartbeatRecoveryHandler<TContext> {
|
|
2229
2360
|
// Promotion stops registration internally and must keep retrying; only an operator stop or a confirmed
|
|
2230
2361
|
// switch to classic mode halts recovery, so it never re-registers behind a disconnect.
|
|
2231
|
-
let halted = false;
|
|
2232
2362
|
const promotionGraceMs =
|
|
2233
2363
|
deps.promotionGraceMs ?? TELEGRAM_BUS_FOLLOWER_PROMOTION_GRACE_MS;
|
|
2234
2364
|
const sleep =
|
|
2235
2365
|
deps.sleep ??
|
|
2236
2366
|
((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
2367
|
+
// A cancelled retry still fires through an injected scheduler; its id no longer matches and it does nothing.
|
|
2237
2368
|
const scheduleRetry =
|
|
2238
2369
|
deps.scheduleRetry ??
|
|
2239
2370
|
((retry: () => void, delayMs: number) => {
|
|
2240
2371
|
const timer = setTimeout(retry, delayMs);
|
|
2241
2372
|
timer.unref?.();
|
|
2242
2373
|
});
|
|
2243
|
-
|
|
2244
|
-
|
|
2374
|
+
/** The recovery request carried by each retry; the machine decides whether the retry still counts. */
|
|
2375
|
+
type RecoveryRequest = {
|
|
2376
|
+
reason: unknown;
|
|
2377
|
+
ctx: TContext;
|
|
2378
|
+
binding?: TelegramBusFollowerPromotedBinding;
|
|
2379
|
+
};
|
|
2380
|
+
const retryRequests = new Map<number, RecoveryRequest>();
|
|
2381
|
+
let pendingRetry: RecoveryRequest | undefined;
|
|
2382
|
+
let request: RecoveryRequest | undefined;
|
|
2383
|
+
let running: Promise<void> = Promise.resolve();
|
|
2384
|
+
const machine = Machine.createTelegramMachineRunner<
|
|
2385
|
+
TelegramBusFollowerRecoveryState,
|
|
2386
|
+
TelegramBusFollowerRecoveryEvent,
|
|
2387
|
+
TelegramBusFollowerRecoveryEffect
|
|
2388
|
+
>({
|
|
2389
|
+
initial: createTelegramBusFollowerRecoveryState(),
|
|
2390
|
+
transition: transitionTelegramBusFollowerRecovery,
|
|
2391
|
+
run(effect) {
|
|
2392
|
+
switch (effect.type) {
|
|
2393
|
+
case "runRecovery": {
|
|
2394
|
+
const started = request!;
|
|
2395
|
+
running = runRecovery(started, effect.epoch).finally(() =>
|
|
2396
|
+
machine.dispatch({ type: "recoverySettled", epoch: effect.epoch }),
|
|
2397
|
+
);
|
|
2398
|
+
return;
|
|
2399
|
+
}
|
|
2400
|
+
case "armRetry": {
|
|
2401
|
+
const retryId = effect.retryId;
|
|
2402
|
+
retryRequests.set(retryId, pendingRetry!);
|
|
2403
|
+
const fire = () => {
|
|
2404
|
+
const carried = retryRequests.get(retryId);
|
|
2405
|
+
if (!carried || machine.state.retryId !== retryId) return;
|
|
2406
|
+
const activeCtx = deps.getActiveContext
|
|
2407
|
+
? deps.getActiveContext()
|
|
2408
|
+
: carried.ctx;
|
|
2409
|
+
if (!activeCtx) {
|
|
2410
|
+
scheduleRetry(fire, promotionGraceMs);
|
|
2411
|
+
return;
|
|
2412
|
+
}
|
|
2413
|
+
retryRequests.delete(retryId);
|
|
2414
|
+
void requestRecovery(
|
|
2415
|
+
{ ...carried, ctx: activeCtx },
|
|
2416
|
+
{
|
|
2417
|
+
type: "retryFired",
|
|
2418
|
+
retryId,
|
|
2419
|
+
},
|
|
2420
|
+
);
|
|
2421
|
+
};
|
|
2422
|
+
scheduleRetry(fire, promotionGraceMs);
|
|
2423
|
+
return;
|
|
2424
|
+
}
|
|
2425
|
+
case "cancelRetry":
|
|
2426
|
+
retryRequests.delete(effect.retryId);
|
|
2427
|
+
return;
|
|
2428
|
+
}
|
|
2429
|
+
},
|
|
2430
|
+
});
|
|
2431
|
+
const requestRecovery = (
|
|
2432
|
+
next: RecoveryRequest,
|
|
2433
|
+
event: TelegramBusFollowerRecoveryEvent,
|
|
2434
|
+
): Promise<void> => {
|
|
2435
|
+
const before = running;
|
|
2436
|
+
request = next;
|
|
2245
2437
|
try {
|
|
2246
|
-
|
|
2247
|
-
}
|
|
2248
|
-
|
|
2249
|
-
deps.recordRuntimeEvent("bus", error, {
|
|
2250
|
-
phase: "follower-stale-context-status",
|
|
2251
|
-
});
|
|
2438
|
+
machine.dispatch(event);
|
|
2439
|
+
} finally {
|
|
2440
|
+
request = undefined;
|
|
2252
2441
|
}
|
|
2442
|
+
return running === before ? Promise.resolve() : running;
|
|
2253
2443
|
};
|
|
2254
|
-
const
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
.
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2444
|
+
const runRecovery = async (
|
|
2445
|
+
{ reason: error, ctx, binding: carriedBinding }: RecoveryRequest,
|
|
2446
|
+
epoch: number,
|
|
2447
|
+
): Promise<void> => {
|
|
2448
|
+
const isCurrent = (): boolean =>
|
|
2449
|
+
machine.state.epoch === epoch && !machine.state.halted;
|
|
2450
|
+
const scheduleRecovery = (
|
|
2451
|
+
reason: unknown,
|
|
2452
|
+
fallbackCtx: TContext,
|
|
2453
|
+
binding: TelegramBusFollowerPromotedBinding,
|
|
2454
|
+
) => {
|
|
2455
|
+
pendingRetry = { reason, ctx: fallbackCtx, binding };
|
|
2456
|
+
try {
|
|
2457
|
+
machine.dispatch({ type: "retryRequested", epoch });
|
|
2458
|
+
} finally {
|
|
2459
|
+
pendingRetry = undefined;
|
|
2460
|
+
}
|
|
2461
|
+
};
|
|
2462
|
+
const safeUpdateStatus = (ctx: TContext) => {
|
|
2463
|
+
try {
|
|
2464
|
+
deps.updateStatus(ctx);
|
|
2465
|
+
} catch (error) {
|
|
2466
|
+
if (!isPiStaleContextError(error)) throw error;
|
|
2467
|
+
deps.recordRuntimeEvent("bus", error, {
|
|
2468
|
+
phase: "follower-stale-context-status",
|
|
2469
|
+
});
|
|
2470
|
+
}
|
|
2471
|
+
};
|
|
2472
|
+
const clearRegisteredState = (ctx: TContext) => {
|
|
2473
|
+
deps.registrationState.setRegistered(false);
|
|
2474
|
+
safeUpdateStatus(ctx);
|
|
2475
|
+
};
|
|
2476
|
+
const tryRegisterWithLeader = async (
|
|
2477
|
+
ctx: TContext,
|
|
2478
|
+
leader: TelegramBusFollowerLeaderLock,
|
|
2479
|
+
phase: string,
|
|
2480
|
+
binding?: TelegramBusFollowerPromotedBinding,
|
|
2481
|
+
) => {
|
|
2482
|
+
if (!isCurrent()) return false;
|
|
2483
|
+
try {
|
|
2484
|
+
const restored = await deps
|
|
2485
|
+
.getRegistrationRuntime()
|
|
2486
|
+
.registerWithLeader(
|
|
2487
|
+
ctx,
|
|
2488
|
+
leader,
|
|
2489
|
+
binding?.target ? { target: binding.target } : undefined,
|
|
2490
|
+
);
|
|
2491
|
+
// Registration owns its attempt cleanup; an obsolete recovery cannot stop the shared replacement runtime.
|
|
2492
|
+
if (!restored || !isCurrent()) return false;
|
|
2493
|
+
deps.setLifecyclePhase(undefined);
|
|
2494
|
+
safeUpdateStatus(ctx);
|
|
2495
|
+
deps.recordRuntimeEvent(
|
|
2496
|
+
"bus",
|
|
2497
|
+
"Telegram follower registration restored",
|
|
2498
|
+
{
|
|
2499
|
+
phase,
|
|
2500
|
+
},
|
|
2271
2501
|
);
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2502
|
+
return true;
|
|
2503
|
+
} catch (error) {
|
|
2504
|
+
if (!isCurrent()) return false;
|
|
2505
|
+
clearRegisteredState(ctx);
|
|
2506
|
+
deps.recordRuntimeEvent("bus", error, { phase });
|
|
2275
2507
|
return false;
|
|
2276
2508
|
}
|
|
2509
|
+
};
|
|
2510
|
+
const snapshotBinding = (): TelegramBusFollowerPromotedBinding => ({
|
|
2511
|
+
target: deps.registrationState.getTarget(),
|
|
2512
|
+
slot: deps.registrationState.getSlot(),
|
|
2513
|
+
threadName: deps.registrationState.getThreadName(),
|
|
2514
|
+
});
|
|
2515
|
+
const promoteToLeader = async (
|
|
2516
|
+
reason: unknown,
|
|
2517
|
+
ctx: TContext,
|
|
2518
|
+
binding: TelegramBusFollowerPromotedBinding,
|
|
2519
|
+
election: TelegramBusFollowerElection,
|
|
2520
|
+
) => {
|
|
2521
|
+
if (!isCurrent()) return;
|
|
2522
|
+
const activeCtx = deps.getActiveContext?.();
|
|
2523
|
+
if (deps.getActiveContext && activeCtx !== ctx) {
|
|
2524
|
+
scheduleRecovery(reason, ctx, binding);
|
|
2525
|
+
return;
|
|
2526
|
+
}
|
|
2527
|
+
deps.setLifecyclePhase("electing");
|
|
2528
|
+
safeUpdateStatus(ctx);
|
|
2529
|
+
deps.recordRuntimeEvent("bus", reason, {
|
|
2530
|
+
phase: "follower-promotion-electing",
|
|
2531
|
+
});
|
|
2532
|
+
deps.getRegistrationRuntime().stop();
|
|
2533
|
+
deps.setLifecyclePhase("electing");
|
|
2534
|
+
safeUpdateStatus(ctx);
|
|
2535
|
+
deps.recordRuntimeEvent("bus", "Telegram follower attempting promotion", {
|
|
2536
|
+
phase: "follower-promotion-electing",
|
|
2537
|
+
});
|
|
2538
|
+
const promoted = await deps.promoteToLeader(ctx, binding, election);
|
|
2539
|
+
if (!isCurrent()) return;
|
|
2277
2540
|
deps.setLifecyclePhase(undefined);
|
|
2278
2541
|
safeUpdateStatus(ctx);
|
|
2279
2542
|
deps.recordRuntimeEvent(
|
|
2280
2543
|
"bus",
|
|
2281
|
-
|
|
2544
|
+
promoted
|
|
2545
|
+
? "Telegram follower promotion completed"
|
|
2546
|
+
: "Telegram follower promotion lost election",
|
|
2282
2547
|
{
|
|
2283
|
-
phase
|
|
2548
|
+
phase: promoted
|
|
2549
|
+
? "follower-promotion-complete"
|
|
2550
|
+
: "follower-promotion-lost",
|
|
2284
2551
|
},
|
|
2285
2552
|
);
|
|
2286
|
-
|
|
2287
|
-
} catch (error) {
|
|
2288
|
-
clearRegisteredState(ctx);
|
|
2289
|
-
deps.recordRuntimeEvent("bus", error, { phase });
|
|
2290
|
-
return false;
|
|
2291
|
-
}
|
|
2292
|
-
};
|
|
2293
|
-
const snapshotBinding = (): TelegramBusFollowerPromotedBinding => ({
|
|
2294
|
-
target: deps.registrationState.getTarget(),
|
|
2295
|
-
slot: deps.registrationState.getSlot(),
|
|
2296
|
-
threadName: deps.registrationState.getThreadName(),
|
|
2297
|
-
});
|
|
2298
|
-
const scheduleRecovery = (
|
|
2299
|
-
reason: unknown,
|
|
2300
|
-
fallbackCtx: TContext,
|
|
2301
|
-
binding: TelegramBusFollowerPromotedBinding,
|
|
2302
|
-
) => {
|
|
2303
|
-
const retry = () => {
|
|
2304
|
-
if (halted) return;
|
|
2305
|
-
const activeCtx = deps.getActiveContext
|
|
2306
|
-
? deps.getActiveContext()
|
|
2307
|
-
: fallbackCtx;
|
|
2308
|
-
if (!activeCtx) {
|
|
2309
|
-
scheduleRetry(retry, promotionGraceMs);
|
|
2310
|
-
return;
|
|
2311
|
-
}
|
|
2312
|
-
void recover(reason, activeCtx, binding);
|
|
2553
|
+
if (!promoted) scheduleRecovery(reason, ctx, binding);
|
|
2313
2554
|
};
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
);
|
|
2352
|
-
if (!promoted) scheduleRecovery(reason, ctx, binding);
|
|
2353
|
-
};
|
|
2354
|
-
const attemptPreferredPromotion = async (
|
|
2355
|
-
reason: unknown,
|
|
2356
|
-
ctx: TContext,
|
|
2357
|
-
binding: TelegramBusFollowerPromotedBinding,
|
|
2358
|
-
candidateState: TelegramBusFollowerLeaderState,
|
|
2359
|
-
): Promise<void> => {
|
|
2360
|
-
const slot = binding.slot;
|
|
2361
|
-
const lowerEligibleSlot = slot
|
|
2362
|
-
? deps.registrationState
|
|
2363
|
-
.getEligibleElectionSlots()
|
|
2364
|
-
.find((candidate) => candidate < slot)
|
|
2365
|
-
: undefined;
|
|
2366
|
-
if (lowerEligibleSlot) {
|
|
2367
|
-
deps.recordRuntimeEvent(
|
|
2368
|
-
"bus",
|
|
2369
|
-
"Telegram follower deferring to a lower-slot election candidate",
|
|
2370
|
-
{
|
|
2371
|
-
phase: "follower-promotion-slot-priority",
|
|
2372
|
-
slot,
|
|
2373
|
-
lowerEligibleSlot,
|
|
2374
|
-
},
|
|
2375
|
-
);
|
|
2376
|
-
await sleep(promotionGraceMs);
|
|
2377
|
-
candidateState = deps.getLeaderState();
|
|
2378
|
-
if (candidateState.kind === "active-elsewhere") {
|
|
2379
|
-
if (
|
|
2380
|
-
!(await tryRegisterWithLeader(
|
|
2381
|
-
ctx,
|
|
2382
|
-
candidateState.lock,
|
|
2383
|
-
"follower-register-preferred-successor",
|
|
2384
|
-
binding,
|
|
2385
|
-
))
|
|
2386
|
-
) {
|
|
2387
|
-
scheduleRecovery(reason, ctx, binding);
|
|
2555
|
+
const attemptPreferredPromotion = async (
|
|
2556
|
+
reason: unknown,
|
|
2557
|
+
ctx: TContext,
|
|
2558
|
+
binding: TelegramBusFollowerPromotedBinding,
|
|
2559
|
+
candidateState: TelegramBusFollowerLeaderState,
|
|
2560
|
+
): Promise<void> => {
|
|
2561
|
+
const slot = binding.slot;
|
|
2562
|
+
const lowerEligibleSlot = slot
|
|
2563
|
+
? deps.registrationState
|
|
2564
|
+
.getEligibleElectionSlots()
|
|
2565
|
+
.find((candidate) => candidate < slot)
|
|
2566
|
+
: undefined;
|
|
2567
|
+
if (lowerEligibleSlot) {
|
|
2568
|
+
deps.recordRuntimeEvent(
|
|
2569
|
+
"bus",
|
|
2570
|
+
"Telegram follower deferring to a lower-slot election candidate",
|
|
2571
|
+
{
|
|
2572
|
+
phase: "follower-promotion-slot-priority",
|
|
2573
|
+
slot,
|
|
2574
|
+
lowerEligibleSlot,
|
|
2575
|
+
},
|
|
2576
|
+
);
|
|
2577
|
+
await sleep(promotionGraceMs);
|
|
2578
|
+
if (!isCurrent()) return;
|
|
2579
|
+
candidateState = deps.getLeaderState();
|
|
2580
|
+
if (candidateState.kind === "active-elsewhere") {
|
|
2581
|
+
if (
|
|
2582
|
+
!(await tryRegisterWithLeader(
|
|
2583
|
+
ctx,
|
|
2584
|
+
candidateState.lock,
|
|
2585
|
+
"follower-register-preferred-successor",
|
|
2586
|
+
binding,
|
|
2587
|
+
))
|
|
2588
|
+
) {
|
|
2589
|
+
scheduleRecovery(reason, ctx, binding);
|
|
2590
|
+
}
|
|
2591
|
+
return;
|
|
2388
2592
|
}
|
|
2389
|
-
return;
|
|
2390
2593
|
}
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
}
|
|
2398
|
-
};
|
|
2399
|
-
const recover = async (
|
|
2400
|
-
error: unknown,
|
|
2401
|
-
ctx: TContext,
|
|
2402
|
-
carriedBinding?: TelegramBusFollowerPromotedBinding,
|
|
2403
|
-
): Promise<void> => {
|
|
2404
|
-
if (promotionPending || halted) return;
|
|
2405
|
-
promotionPending = true;
|
|
2594
|
+
if (candidateState.kind !== "stale" && candidateState.kind !== "inactive")
|
|
2595
|
+
return;
|
|
2596
|
+
await promoteToLeader(reason, ctx, binding, {
|
|
2597
|
+
expectedOwner:
|
|
2598
|
+
candidateState.kind === "stale" ? candidateState.lock : undefined,
|
|
2599
|
+
});
|
|
2600
|
+
};
|
|
2406
2601
|
deps.registrationState.beginRecovery();
|
|
2407
2602
|
const initialBinding = carriedBinding ?? snapshotBinding();
|
|
2408
2603
|
try {
|
|
@@ -2414,8 +2609,9 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
2414
2609
|
phase: "follower-thread-mode-read",
|
|
2415
2610
|
});
|
|
2416
2611
|
}
|
|
2612
|
+
if (!isCurrent()) return;
|
|
2417
2613
|
if (threadModeDisabled) {
|
|
2418
|
-
halted
|
|
2614
|
+
machine.dispatch({ type: "halted" });
|
|
2419
2615
|
deps.getRegistrationRuntime().stop();
|
|
2420
2616
|
deps.setLifecyclePhase(undefined);
|
|
2421
2617
|
safeUpdateStatus(ctx);
|
|
@@ -2439,6 +2635,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
2439
2635
|
) {
|
|
2440
2636
|
return;
|
|
2441
2637
|
}
|
|
2638
|
+
if (!isCurrent()) return;
|
|
2442
2639
|
deps.setLifecyclePhase("electing");
|
|
2443
2640
|
safeUpdateStatus(ctx);
|
|
2444
2641
|
deps.recordRuntimeEvent(
|
|
@@ -2447,6 +2644,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
2447
2644
|
{ phase: "follower-promotion-grace" },
|
|
2448
2645
|
);
|
|
2449
2646
|
await sleep(promotionGraceMs);
|
|
2647
|
+
if (!isCurrent()) return;
|
|
2450
2648
|
const graceState = deps.getLeaderState();
|
|
2451
2649
|
if (graceState.kind === "active-elsewhere") {
|
|
2452
2650
|
if (
|
|
@@ -2459,7 +2657,11 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
2459
2657
|
) {
|
|
2460
2658
|
return;
|
|
2461
2659
|
}
|
|
2462
|
-
|
|
2660
|
+
const unresponsive = await deps.proveLeaderUnresponsive?.(
|
|
2661
|
+
graceState.lock,
|
|
2662
|
+
);
|
|
2663
|
+
if (!isCurrent()) return;
|
|
2664
|
+
if (unresponsive) {
|
|
2463
2665
|
await promoteToLeader(error, ctx, initialBinding, {
|
|
2464
2666
|
expectedOwner: graceState.lock,
|
|
2465
2667
|
unresponsive: true,
|
|
@@ -2485,6 +2687,7 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
2485
2687
|
}
|
|
2486
2688
|
await attemptPreferredPromotion(error, ctx, initialBinding, state);
|
|
2487
2689
|
} catch (promotionError) {
|
|
2690
|
+
if (!isCurrent()) return;
|
|
2488
2691
|
deps.setLifecyclePhase(undefined);
|
|
2489
2692
|
safeUpdateStatus(ctx);
|
|
2490
2693
|
if (isPiStaleContextError(promotionError)) {
|
|
@@ -2497,18 +2700,20 @@ export function createTelegramBusFollowerHeartbeatRecoveryHandler<TContext>(
|
|
|
2497
2700
|
phase: "follower-promotion-failed",
|
|
2498
2701
|
});
|
|
2499
2702
|
scheduleRecovery(promotionError, ctx, initialBinding);
|
|
2500
|
-
} finally {
|
|
2501
|
-
promotionPending = false;
|
|
2502
2703
|
}
|
|
2503
2704
|
};
|
|
2504
|
-
return Object.assign(
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2705
|
+
return Object.assign(
|
|
2706
|
+
(error: unknown, ctx: TContext): Promise<void> =>
|
|
2707
|
+
requestRecovery({ reason: error, ctx }, { type: "recoverRequested" }),
|
|
2708
|
+
{
|
|
2709
|
+
halt() {
|
|
2710
|
+
machine.dispatch({ type: "halted" });
|
|
2711
|
+
},
|
|
2712
|
+
resume() {
|
|
2713
|
+
machine.dispatch({ type: "resumed" });
|
|
2714
|
+
},
|
|
2510
2715
|
},
|
|
2511
|
-
|
|
2716
|
+
);
|
|
2512
2717
|
}
|
|
2513
2718
|
|
|
2514
2719
|
export function createTelegramBusFollowerRegistrationRuntime<
|
|
@@ -2590,6 +2795,7 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
2590
2795
|
const leaderSocketPath = activeLeaderSocketPath;
|
|
2591
2796
|
const registrationGeneration = activeRegistrationGeneration;
|
|
2592
2797
|
const heartbeatContext = activeContext;
|
|
2798
|
+
const heartbeatTarget = deps.registrationState?.getTarget();
|
|
2593
2799
|
if (!leaderSocketPath || !registrationGeneration) return;
|
|
2594
2800
|
const isCurrentHeartbeat = (): boolean =>
|
|
2595
2801
|
activeLeaderSocketPath === leaderSocketPath &&
|
|
@@ -2621,6 +2827,7 @@ export function createTelegramBusFollowerRegistrationRuntime<
|
|
|
2621
2827
|
deps.registrationState?.setEligibleElectionSlots(slots);
|
|
2622
2828
|
if (
|
|
2623
2829
|
typeof heartbeatResult?.displayTitle === "string" &&
|
|
2830
|
+
isDeepStrictEqual(deps.registrationState?.getTarget(), heartbeatTarget) &&
|
|
2624
2831
|
deps.registrationState?.setDisplayTitle(
|
|
2625
2832
|
heartbeatResult.displayTitle,
|
|
2626
2833
|
registrationGeneration,
|
|
@@ -3276,12 +3483,14 @@ function sendTelegramBusOperation(
|
|
|
3276
3483
|
socketPath: string,
|
|
3277
3484
|
timeoutMs: number,
|
|
3278
3485
|
envelope: TelegramBusEnvelope,
|
|
3486
|
+
assertAuthority?: () => void,
|
|
3279
3487
|
) {
|
|
3280
3488
|
return sendTelegramBusLocalEnvelope({
|
|
3281
3489
|
socketPath,
|
|
3282
3490
|
timeoutMs,
|
|
3283
3491
|
envelope,
|
|
3284
|
-
|
|
3492
|
+
assertAuthority,
|
|
3493
|
+
retry: assertAuthority ? { attempts: 1, delayMs: 0 } : getTelegramBusTransportRetryPolicy({
|
|
3285
3494
|
endpoint: socketPath,
|
|
3286
3495
|
operation: "operation",
|
|
3287
3496
|
}),
|
|
@@ -3817,9 +4026,10 @@ export function createTelegramBusFollowerLiveRebindRuntime<TContext>(deps: {
|
|
|
3817
4026
|
!matchesLiveRebindCarrier(record.observation, envelope)
|
|
3818
4027
|
)
|
|
3819
4028
|
throw new Error("Live-rebind settlement has no exact current carrier.");
|
|
3820
|
-
if (envelope.mode === "observe") {
|
|
4029
|
+
if (envelope.mode === "observe" || envelope.mode === "observe-held") {
|
|
4030
|
+
const beforeRelease = envelope.mode === "observe-held";
|
|
3821
4031
|
if (
|
|
3822
|
-
record.outcome?.status !== "released" ||
|
|
4032
|
+
(beforeRelease ? !!record.outcome || !record.applyIssued || !record.held.confirmSaved() : record.outcome?.status !== "released") ||
|
|
3823
4033
|
!deps.applyTarget ||
|
|
3824
4034
|
!deps.observeWork ||
|
|
3825
4035
|
!envelope.oldTarget
|
|
@@ -3864,7 +4074,7 @@ export function createTelegramBusFollowerLiveRebindRuntime<TContext>(deps: {
|
|
|
3864
4074
|
if (!current() || !work)
|
|
3865
4075
|
throw new Error("Live-rebind work observation is unconfirmed.");
|
|
3866
4076
|
// Unfinished or unconfirmed selected execution stays unknown; later settled delivery cannot clear it.
|
|
3867
|
-
if (commandWork && status!.result !== "completed")
|
|
4077
|
+
if (!beforeRelease && commandWork && status!.result !== "completed")
|
|
3868
4078
|
work.unknown = true;
|
|
3869
4079
|
if (!current())
|
|
3870
4080
|
throw new Error("Live-rebind work observer authority changed.");
|
|
@@ -3917,10 +4127,12 @@ export function createTelegramBusFollowerLiveRebindRuntime<TContext>(deps: {
|
|
|
3917
4127
|
sourceUpdateIds: record.observation.sourceUpdateIds,
|
|
3918
4128
|
isCurrent: current,
|
|
3919
4129
|
...copyLiveRebindCommandTarget(record.observation),
|
|
3920
|
-
release(canRelease) {
|
|
4130
|
+
release(canRelease, target) {
|
|
4131
|
+
if (!target) throw new Error("Live-rebind release target is unconfirmed.");
|
|
3921
4132
|
record.outcome = observation("unknown");
|
|
3922
4133
|
record.outcome = record.held.release(
|
|
3923
4134
|
() => current() && canRelease(),
|
|
4135
|
+
target,
|
|
3924
4136
|
)
|
|
3925
4137
|
? observation("released")
|
|
3926
4138
|
: undefined;
|
|
@@ -4327,6 +4539,8 @@ export function createTelegramBusForwardedUpdateReceiverRuntime<TContext>(
|
|
|
4327
4539
|
TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_SAVE,
|
|
4328
4540
|
...(apply ? [TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_APPLY] : []),
|
|
4329
4541
|
...(settle ? [TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_SETTLE] : []),
|
|
4542
|
+
...(envelope.kind === "leader.settleLiveRebind" && envelope.mode === "observe-held"
|
|
4543
|
+
? [TELEGRAM_BUS_CAPABILITY_PRE_DISPATCH_REBIND_EXECUTION] : []),
|
|
4330
4544
|
...(commandInput
|
|
4331
4545
|
? [
|
|
4332
4546
|
TELEGRAM_BUS_CAPABILITY_LIVE_REBIND_APPLY,
|