@llblab/pi-telegram 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -4
- package/CHANGELOG.md +474 -452
- package/docs/architecture.md +12 -10
- package/docs/delivery.md +2 -1
- package/docs/locks.md +9 -5
- package/docs/multi-instance-bus.md +5 -5
- package/index.ts +311 -633
- package/lib/bindings.ts +59 -21
- package/lib/bus-api.ts +12 -2
- package/lib/bus-follower.ts +327 -83
- package/lib/bus-leader.ts +201 -121
- package/lib/bus.ts +345 -37
- package/lib/config.ts +168 -28
- package/lib/delivery.ts +148 -61
- package/lib/lifecycle.ts +199 -8
- package/lib/locks.ts +569 -56
- package/lib/logs.ts +178 -19
- package/lib/media.ts +79 -24
- package/lib/model.ts +5 -10
- package/lib/ownership.ts +150 -6
- package/lib/polling.ts +156 -7
- package/lib/preview.ts +15 -3
- package/lib/queue.ts +167 -20
- package/lib/routing.ts +68 -12
- package/lib/sync.ts +105 -32
- package/lib/telegram-api.ts +86 -10
- package/lib/text-groups.ts +71 -15
- package/lib/thread-reconciler.ts +49 -36
- package/lib/threads.ts +505 -118
- package/package.json +1 -1
package/lib/locks.ts
CHANGED
|
@@ -6,12 +6,14 @@
|
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
8
|
existsSync,
|
|
9
|
+
linkSync,
|
|
9
10
|
mkdirSync,
|
|
10
11
|
readFileSync,
|
|
11
12
|
renameSync,
|
|
12
13
|
unlinkSync,
|
|
13
14
|
writeFileSync,
|
|
14
15
|
} from "node:fs";
|
|
16
|
+
import { randomUUID } from "node:crypto";
|
|
15
17
|
import { dirname } from "node:path";
|
|
16
18
|
import { resolveTelegramLocksPath } from "./paths.ts";
|
|
17
19
|
|
|
@@ -19,6 +21,22 @@ export const TELEGRAM_LOCK_KEY = "@llblab/pi-telegram";
|
|
|
19
21
|
export const TELEGRAM_BUS_LEADER_STALE_HEARTBEAT_MS = 5_000;
|
|
20
22
|
const TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS = 5;
|
|
21
23
|
const TELEGRAM_LOCK_WRITE_RETRY_DELAY_MS = 25;
|
|
24
|
+
const TELEGRAM_LOCK_TRANSACTION_ATTEMPTS = 80;
|
|
25
|
+
const TELEGRAM_LOCK_TRANSACTION_RETRY_DELAY_MS = 25;
|
|
26
|
+
const TELEGRAM_LOCK_RUNTIME_GENERATION_KEY =
|
|
27
|
+
"__piTelegramLockRuntimeGeneration__";
|
|
28
|
+
|
|
29
|
+
function allocateTelegramLockRuntimeGeneration(): number {
|
|
30
|
+
const globals = globalThis as Record<string, unknown>;
|
|
31
|
+
const previous = globals[TELEGRAM_LOCK_RUNTIME_GENERATION_KEY];
|
|
32
|
+
const previousGeneration =
|
|
33
|
+
typeof previous === "number" && Number.isSafeInteger(previous)
|
|
34
|
+
? previous
|
|
35
|
+
: 0;
|
|
36
|
+
const generation = Math.max(Date.now(), previousGeneration + 1);
|
|
37
|
+
globals[TELEGRAM_LOCK_RUNTIME_GENERATION_KEY] = generation;
|
|
38
|
+
return generation;
|
|
39
|
+
}
|
|
22
40
|
|
|
23
41
|
function getLocksPath(): string {
|
|
24
42
|
return resolveTelegramLocksPath();
|
|
@@ -51,7 +69,8 @@ export interface TelegramLockEntry {
|
|
|
51
69
|
cwd?: string;
|
|
52
70
|
instanceId?: string;
|
|
53
71
|
heartbeatMs?: number;
|
|
54
|
-
leaderEpoch?: number;
|
|
72
|
+
leaderEpoch?: number | string;
|
|
73
|
+
runtimeGeneration?: number;
|
|
55
74
|
busSocketPath?: string;
|
|
56
75
|
busSecret?: string;
|
|
57
76
|
}
|
|
@@ -68,6 +87,8 @@ export type TelegramLockState =
|
|
|
68
87
|
|
|
69
88
|
export interface TelegramLockAcquireOptions {
|
|
70
89
|
force?: boolean;
|
|
90
|
+
expectedOwner?: TelegramLockEntry;
|
|
91
|
+
election?: boolean;
|
|
71
92
|
}
|
|
72
93
|
|
|
73
94
|
export type TelegramLockAcquireResult =
|
|
@@ -82,7 +103,9 @@ export interface TelegramLockRuntime<TContext extends TelegramLockContext> {
|
|
|
82
103
|
release: () => TelegramLockState;
|
|
83
104
|
getState: () => TelegramLockState;
|
|
84
105
|
getStatusLabel: () => string;
|
|
106
|
+
getOwnedLeaderEpoch: () => number | string | undefined;
|
|
85
107
|
owns: (ctx?: TelegramLockContext) => boolean;
|
|
108
|
+
commitIfOwned: (commit: () => void) => boolean;
|
|
86
109
|
refresh: (ctx?: TelegramLockContext) => boolean;
|
|
87
110
|
}
|
|
88
111
|
|
|
@@ -107,6 +130,8 @@ export interface TelegramLockRuntimeOptions {
|
|
|
107
130
|
busSocketPath?: string;
|
|
108
131
|
busSecret?: string;
|
|
109
132
|
getNowMs?: () => number;
|
|
133
|
+
mintLeaderEpoch?: () => number | string;
|
|
134
|
+
runtimeGeneration?: number;
|
|
110
135
|
staleHeartbeatMs?: number;
|
|
111
136
|
}
|
|
112
137
|
|
|
@@ -122,6 +147,21 @@ export function readLocks(path = getLocksPath()): Record<string, unknown> {
|
|
|
122
147
|
}
|
|
123
148
|
}
|
|
124
149
|
|
|
150
|
+
function readLocksForTransaction(path: string): Record<string, unknown> {
|
|
151
|
+
let source: string;
|
|
152
|
+
try {
|
|
153
|
+
source = readFileSync(path, "utf8");
|
|
154
|
+
} catch (error) {
|
|
155
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return {};
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
const value: unknown = JSON.parse(source);
|
|
159
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
160
|
+
throw new Error(`Invalid Telegram lock registry: ${path}`);
|
|
161
|
+
}
|
|
162
|
+
return value as Record<string, unknown>;
|
|
163
|
+
}
|
|
164
|
+
|
|
125
165
|
function isRetryableLockWriteError(error: unknown): boolean {
|
|
126
166
|
const code = (error as { code?: unknown })?.code;
|
|
127
167
|
return code === "EPERM" || code === "EBUSY" || code === "EACCES";
|
|
@@ -132,11 +172,217 @@ function sleepSync(ms: number): void {
|
|
|
132
172
|
Atomics.wait(new Int32Array(buffer), 0, 0, ms);
|
|
133
173
|
}
|
|
134
174
|
|
|
175
|
+
interface TelegramLockTransactionOwner {
|
|
176
|
+
pid: number;
|
|
177
|
+
acquiredAtMs: number;
|
|
178
|
+
generation: string;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function readLockTransactionOwner(
|
|
182
|
+
path: string,
|
|
183
|
+
): TelegramLockTransactionOwner | undefined {
|
|
184
|
+
try {
|
|
185
|
+
const value = JSON.parse(readFileSync(path, "utf8")) as Record<
|
|
186
|
+
string,
|
|
187
|
+
unknown
|
|
188
|
+
>;
|
|
189
|
+
if (
|
|
190
|
+
typeof value.pid !== "number" ||
|
|
191
|
+
typeof value.acquiredAtMs !== "number" ||
|
|
192
|
+
typeof value.generation !== "string"
|
|
193
|
+
) {
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
pid: value.pid,
|
|
198
|
+
acquiredAtMs: value.acquiredAtMs,
|
|
199
|
+
generation: value.generation,
|
|
200
|
+
};
|
|
201
|
+
} catch {
|
|
202
|
+
return undefined;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function createLockTransactionGuard(
|
|
207
|
+
path: string,
|
|
208
|
+
): TelegramLockTransactionOwner {
|
|
209
|
+
const owner: TelegramLockTransactionOwner = {
|
|
210
|
+
pid: process.pid,
|
|
211
|
+
acquiredAtMs: Date.now(),
|
|
212
|
+
generation: randomUUID(),
|
|
213
|
+
};
|
|
214
|
+
const stagedPath = `${path}.${owner.generation}.tmp`;
|
|
215
|
+
try {
|
|
216
|
+
writeFileSync(stagedPath, `${JSON.stringify(owner)}\n`, {
|
|
217
|
+
encoding: "utf8",
|
|
218
|
+
flag: "wx",
|
|
219
|
+
mode: 0o600,
|
|
220
|
+
});
|
|
221
|
+
linkSync(stagedPath, path);
|
|
222
|
+
return owner;
|
|
223
|
+
} finally {
|
|
224
|
+
try {
|
|
225
|
+
unlinkSync(stagedPath);
|
|
226
|
+
} catch {
|
|
227
|
+
/* best effort */
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function releaseLockTransactionGuard(
|
|
233
|
+
path: string,
|
|
234
|
+
owner: TelegramLockTransactionOwner,
|
|
235
|
+
): void {
|
|
236
|
+
const current = readLockTransactionOwner(path);
|
|
237
|
+
if (!current) {
|
|
238
|
+
if (!existsSync(path)) return;
|
|
239
|
+
throw new Error(`Cannot verify Telegram lock transaction guard: ${path}`);
|
|
240
|
+
}
|
|
241
|
+
if (
|
|
242
|
+
current.pid !== owner.pid ||
|
|
243
|
+
current.generation !== owner.generation ||
|
|
244
|
+
current.acquiredAtMs !== owner.acquiredAtMs
|
|
245
|
+
) {
|
|
246
|
+
throw new Error(
|
|
247
|
+
`Telegram lock transaction guard changed ownership: ${path}`,
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
for (
|
|
251
|
+
let attempt = 0;
|
|
252
|
+
attempt < TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS;
|
|
253
|
+
attempt += 1
|
|
254
|
+
) {
|
|
255
|
+
try {
|
|
256
|
+
unlinkSync(path);
|
|
257
|
+
return;
|
|
258
|
+
} catch (error) {
|
|
259
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return;
|
|
260
|
+
if (
|
|
261
|
+
!isRetryableLockWriteError(error) ||
|
|
262
|
+
attempt === TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS - 1
|
|
263
|
+
) {
|
|
264
|
+
throw error;
|
|
265
|
+
}
|
|
266
|
+
sleepSync(TELEGRAM_LOCK_WRITE_RETRY_DELAY_MS * (attempt + 1));
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function isAbandonedLockTransaction(path: string): boolean {
|
|
272
|
+
const owner = readLockTransactionOwner(path);
|
|
273
|
+
return owner ? !isProcessAlive(owner.pid) : false;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function recoverAbandonedLockTransaction(
|
|
277
|
+
path: string,
|
|
278
|
+
): TelegramLockTransactionOwner | undefined {
|
|
279
|
+
if (!isAbandonedLockTransaction(path)) return undefined;
|
|
280
|
+
const recoveryGuardPath = `${path}.recovery`;
|
|
281
|
+
let recoveryOwner: TelegramLockTransactionOwner;
|
|
282
|
+
try {
|
|
283
|
+
recoveryOwner = createLockTransactionGuard(recoveryGuardPath);
|
|
284
|
+
} catch (error) {
|
|
285
|
+
if ((error as { code?: unknown })?.code === "EEXIST") return undefined;
|
|
286
|
+
throw error;
|
|
287
|
+
}
|
|
288
|
+
let recoveredOwner: TelegramLockTransactionOwner | undefined;
|
|
289
|
+
try {
|
|
290
|
+
if (!isAbandonedLockTransaction(path)) return undefined;
|
|
291
|
+
const stalePath = `${path}.stale.${process.pid}.${Date.now()}`;
|
|
292
|
+
try {
|
|
293
|
+
renameSync(path, stalePath);
|
|
294
|
+
} catch (error) {
|
|
295
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return undefined;
|
|
296
|
+
throw error;
|
|
297
|
+
}
|
|
298
|
+
try {
|
|
299
|
+
unlinkSync(stalePath);
|
|
300
|
+
} catch {
|
|
301
|
+
/* best effort */
|
|
302
|
+
}
|
|
303
|
+
try {
|
|
304
|
+
recoveredOwner = createLockTransactionGuard(path);
|
|
305
|
+
return recoveredOwner;
|
|
306
|
+
} catch (error) {
|
|
307
|
+
if ((error as { code?: unknown })?.code === "EEXIST") return undefined;
|
|
308
|
+
throw error;
|
|
309
|
+
}
|
|
310
|
+
} finally {
|
|
311
|
+
try {
|
|
312
|
+
releaseLockTransactionGuard(recoveryGuardPath, recoveryOwner);
|
|
313
|
+
} catch (error) {
|
|
314
|
+
if (recoveredOwner) {
|
|
315
|
+
try {
|
|
316
|
+
releaseLockTransactionGuard(path, recoveredOwner);
|
|
317
|
+
} catch {
|
|
318
|
+
/* preserve the recovery cleanup failure */
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
throw error;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function acquireLockTransaction(path: string): TelegramLockTransactionOwner {
|
|
327
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
328
|
+
for (
|
|
329
|
+
let attempt = 0;
|
|
330
|
+
attempt < TELEGRAM_LOCK_TRANSACTION_ATTEMPTS;
|
|
331
|
+
attempt += 1
|
|
332
|
+
) {
|
|
333
|
+
try {
|
|
334
|
+
return createLockTransactionGuard(path);
|
|
335
|
+
} catch (error) {
|
|
336
|
+
if ((error as { code?: unknown })?.code !== "EEXIST") throw error;
|
|
337
|
+
const recoveredOwner = recoverAbandonedLockTransaction(path);
|
|
338
|
+
if (recoveredOwner !== undefined) return recoveredOwner;
|
|
339
|
+
if (attempt === TELEGRAM_LOCK_TRANSACTION_ATTEMPTS - 1) {
|
|
340
|
+
throw new Error(
|
|
341
|
+
`Timed out acquiring Telegram lock transaction: ${path}`,
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
sleepSync(TELEGRAM_LOCK_TRANSACTION_RETRY_DELAY_MS);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
throw new Error(`Failed to acquire Telegram lock transaction: ${path}`);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
export function withTelegramFileTransaction<T>(
|
|
351
|
+
transactionPath: string,
|
|
352
|
+
operation: () => T,
|
|
353
|
+
): T {
|
|
354
|
+
const owner = acquireLockTransaction(transactionPath);
|
|
355
|
+
try {
|
|
356
|
+
return operation();
|
|
357
|
+
} finally {
|
|
358
|
+
releaseLockTransactionGuard(transactionPath, owner);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function withLockTransaction<T>(
|
|
363
|
+
locksPath: string,
|
|
364
|
+
mutate: (locks: Record<string, unknown>) => {
|
|
365
|
+
result: T;
|
|
366
|
+
changed: boolean;
|
|
367
|
+
},
|
|
368
|
+
): T {
|
|
369
|
+
return withTelegramFileTransaction(`${locksPath}.transaction`, () => {
|
|
370
|
+
const locks = readLocksForTransaction(locksPath);
|
|
371
|
+
const outcome = mutate(locks);
|
|
372
|
+
if (outcome.changed) writeLocks(locksPath, locks);
|
|
373
|
+
return outcome.result;
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
|
|
135
377
|
export function writeLocks(path: string, locks: Record<string, unknown>): void {
|
|
136
378
|
mkdirSync(dirname(path), { recursive: true });
|
|
137
379
|
const payload = `${JSON.stringify(locks, null, 2)}\n`;
|
|
138
380
|
let lastError: unknown;
|
|
139
|
-
for (
|
|
381
|
+
for (
|
|
382
|
+
let attempt = 0;
|
|
383
|
+
attempt < TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS;
|
|
384
|
+
attempt += 1
|
|
385
|
+
) {
|
|
140
386
|
const tempPath = `${path}.${process.pid}.${Date.now()}.${attempt}.tmp`;
|
|
141
387
|
try {
|
|
142
388
|
writeFileSync(tempPath, payload, {
|
|
@@ -179,7 +425,14 @@ export function parseTelegramLockEntry(
|
|
|
179
425
|
heartbeatMs:
|
|
180
426
|
typeof record.heartbeatMs === "number" ? record.heartbeatMs : undefined,
|
|
181
427
|
leaderEpoch:
|
|
182
|
-
typeof record.leaderEpoch === "number"
|
|
428
|
+
typeof record.leaderEpoch === "number" ||
|
|
429
|
+
typeof record.leaderEpoch === "string"
|
|
430
|
+
? record.leaderEpoch
|
|
431
|
+
: undefined,
|
|
432
|
+
runtimeGeneration:
|
|
433
|
+
typeof record.runtimeGeneration === "number"
|
|
434
|
+
? record.runtimeGeneration
|
|
435
|
+
: undefined,
|
|
183
436
|
busSocketPath:
|
|
184
437
|
typeof record.busSocketPath === "string"
|
|
185
438
|
? record.busSocketPath
|
|
@@ -243,6 +496,40 @@ function ownsLockContext(
|
|
|
243
496
|
return !lock.cwd || !ctx || lock.cwd === ctx.cwd;
|
|
244
497
|
}
|
|
245
498
|
|
|
499
|
+
function hasSameLockOwner(
|
|
500
|
+
current: TelegramLockEntry | undefined,
|
|
501
|
+
expected: TelegramLockEntry | undefined,
|
|
502
|
+
): boolean {
|
|
503
|
+
if (!current || !expected) return false;
|
|
504
|
+
return (
|
|
505
|
+
current.pid === expected.pid &&
|
|
506
|
+
current.cwd === expected.cwd &&
|
|
507
|
+
current.instanceId === expected.instanceId &&
|
|
508
|
+
current.leaderEpoch === expected.leaderEpoch &&
|
|
509
|
+
current.runtimeGeneration === expected.runtimeGeneration
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
function canSupersedeSameProcessOwner(
|
|
514
|
+
current: TelegramLockEntry,
|
|
515
|
+
pid: number,
|
|
516
|
+
ctx: TelegramLockContext,
|
|
517
|
+
instanceId: string | undefined,
|
|
518
|
+
runtimeGeneration: number,
|
|
519
|
+
): boolean {
|
|
520
|
+
if (
|
|
521
|
+
current.pid !== pid ||
|
|
522
|
+
(current.cwd !== undefined && current.cwd !== ctx.cwd) ||
|
|
523
|
+
!instanceId
|
|
524
|
+
) {
|
|
525
|
+
return false;
|
|
526
|
+
}
|
|
527
|
+
return (
|
|
528
|
+
current.runtimeGeneration === undefined ||
|
|
529
|
+
runtimeGeneration > current.runtimeGeneration
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
|
|
246
533
|
function createLockEntry(
|
|
247
534
|
pid: number,
|
|
248
535
|
ctx: TelegramLockContext,
|
|
@@ -251,6 +538,8 @@ function createLockEntry(
|
|
|
251
538
|
busSocketPath?: string;
|
|
252
539
|
busSecret?: string;
|
|
253
540
|
getNowMs?: () => number;
|
|
541
|
+
mintLeaderEpoch?: () => number | string;
|
|
542
|
+
runtimeGeneration?: number;
|
|
254
543
|
},
|
|
255
544
|
): TelegramLockEntry {
|
|
256
545
|
const lock: TelegramLockEntry = { pid, cwd: ctx.cwd };
|
|
@@ -258,7 +547,8 @@ function createLockEntry(
|
|
|
258
547
|
const nowMs = options.getNowMs?.();
|
|
259
548
|
lock.instanceId = options.instanceId;
|
|
260
549
|
lock.heartbeatMs = nowMs;
|
|
261
|
-
lock.leaderEpoch =
|
|
550
|
+
lock.leaderEpoch = options.mintLeaderEpoch?.() ?? randomUUID();
|
|
551
|
+
lock.runtimeGeneration = options.runtimeGeneration;
|
|
262
552
|
}
|
|
263
553
|
if (options.busSocketPath) lock.busSocketPath = options.busSocketPath;
|
|
264
554
|
if (options.busSecret) lock.busSecret = options.busSecret;
|
|
@@ -286,6 +576,10 @@ export function createTelegramLockRuntime<TContext extends TelegramLockContext>(
|
|
|
286
576
|
const pid = options.pid ?? process.pid;
|
|
287
577
|
const isAlive = options.isProcessAlive ?? isProcessAlive;
|
|
288
578
|
const getNowMs = options.getNowMs ?? Date.now;
|
|
579
|
+
const runtimeGeneration =
|
|
580
|
+
options.runtimeGeneration ?? allocateTelegramLockRuntimeGeneration();
|
|
581
|
+
let ownedLockKey: string | undefined;
|
|
582
|
+
let ownedLock: TelegramLockEntry | undefined;
|
|
289
583
|
const stateOptions = () => ({
|
|
290
584
|
nowMs: getNowMs(),
|
|
291
585
|
staleHeartbeatMs: options.staleHeartbeatMs,
|
|
@@ -298,56 +592,196 @@ export function createTelegramLockRuntime<TContext extends TelegramLockContext>(
|
|
|
298
592
|
const effectiveKey = resolveEffectiveKey();
|
|
299
593
|
return parseTelegramLockEntry(readLocks(locksPath)[effectiveKey]);
|
|
300
594
|
};
|
|
301
|
-
const
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
595
|
+
const adoptCompatibleOwnedLock = (
|
|
596
|
+
effectiveKey: string,
|
|
597
|
+
lock: TelegramLockEntry | undefined,
|
|
598
|
+
ctx?: TelegramLockContext,
|
|
599
|
+
): TelegramLockEntry | undefined => {
|
|
600
|
+
if (ownedLock) {
|
|
601
|
+
return ownedLockKey === effectiveKey ? ownedLock : undefined;
|
|
602
|
+
}
|
|
603
|
+
if (!ownsLockContext(lock, pid, ctx)) return undefined;
|
|
604
|
+
if (
|
|
605
|
+
(lock?.instanceId !== undefined &&
|
|
606
|
+
lock.instanceId !== options.instanceId) ||
|
|
607
|
+
(lock?.runtimeGeneration !== undefined &&
|
|
608
|
+
lock.runtimeGeneration !== runtimeGeneration)
|
|
609
|
+
) {
|
|
610
|
+
return undefined;
|
|
611
|
+
}
|
|
612
|
+
ownedLockKey = effectiveKey;
|
|
613
|
+
ownedLock = lock;
|
|
614
|
+
return ownedLock;
|
|
306
615
|
};
|
|
307
616
|
return {
|
|
308
|
-
acquire: (ctx, acquireOptions = {}) =>
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
617
|
+
acquire: (ctx, acquireOptions = {}) =>
|
|
618
|
+
withLockTransaction<TelegramLockAcquireResult>(locksPath, (locks) => {
|
|
619
|
+
const effectiveKey = resolveEffectiveKey();
|
|
620
|
+
const current = parseTelegramLockEntry(locks[effectiveKey]);
|
|
621
|
+
const state = getLockState(current, pid, isAlive, stateOptions());
|
|
622
|
+
const expectedOwned = adoptCompatibleOwnedLock(
|
|
623
|
+
effectiveKey,
|
|
624
|
+
current,
|
|
625
|
+
ctx,
|
|
626
|
+
);
|
|
627
|
+
if (
|
|
628
|
+
state.kind === "active-here" &&
|
|
629
|
+
hasSameLockOwner(current, expectedOwned)
|
|
630
|
+
) {
|
|
631
|
+
return {
|
|
632
|
+
result: {
|
|
633
|
+
ok: true,
|
|
634
|
+
lock: current!,
|
|
635
|
+
replacedStale: false,
|
|
636
|
+
} as const,
|
|
637
|
+
changed: false,
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
if (acquireOptions.election && current) {
|
|
641
|
+
if (
|
|
642
|
+
state.kind !== "stale" ||
|
|
643
|
+
!hasSameLockOwner(current, acquireOptions.expectedOwner)
|
|
644
|
+
) {
|
|
645
|
+
return {
|
|
646
|
+
result: { ok: false, lock: current } as const,
|
|
647
|
+
changed: false,
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
const expectedReplacementMatches = hasSameLockOwner(
|
|
652
|
+
state.kind === "active-here" || state.kind === "active-elsewhere"
|
|
653
|
+
? state.lock
|
|
654
|
+
: undefined,
|
|
655
|
+
acquireOptions.expectedOwner,
|
|
656
|
+
);
|
|
657
|
+
const canReplaceCurrent =
|
|
658
|
+
state.kind === "active-elsewhere" ||
|
|
659
|
+
(state.kind === "active-here" &&
|
|
660
|
+
canSupersedeSameProcessOwner(
|
|
661
|
+
state.lock,
|
|
662
|
+
pid,
|
|
663
|
+
ctx,
|
|
664
|
+
options.instanceId,
|
|
665
|
+
runtimeGeneration,
|
|
666
|
+
));
|
|
667
|
+
if (
|
|
668
|
+
!acquireOptions.election &&
|
|
669
|
+
(state.kind === "active-here" || state.kind === "active-elsewhere") &&
|
|
670
|
+
(!acquireOptions.force ||
|
|
671
|
+
!expectedReplacementMatches ||
|
|
672
|
+
!canReplaceCurrent)
|
|
673
|
+
) {
|
|
674
|
+
return {
|
|
675
|
+
result: { ok: false, lock: state.lock } as const,
|
|
676
|
+
changed: false,
|
|
677
|
+
};
|
|
678
|
+
}
|
|
679
|
+
const lock = createLockEntry(pid, ctx, {
|
|
680
|
+
instanceId: options.instanceId,
|
|
681
|
+
busSocketPath: options.busSocketPath,
|
|
682
|
+
busSecret: options.busSecret,
|
|
683
|
+
getNowMs,
|
|
684
|
+
mintLeaderEpoch: options.mintLeaderEpoch,
|
|
685
|
+
runtimeGeneration,
|
|
686
|
+
});
|
|
687
|
+
locks[effectiveKey] = lock;
|
|
688
|
+
ownedLockKey = effectiveKey;
|
|
689
|
+
ownedLock = lock;
|
|
690
|
+
return {
|
|
691
|
+
result: {
|
|
692
|
+
ok: true,
|
|
693
|
+
lock,
|
|
694
|
+
replacedStale: state.kind === "stale",
|
|
695
|
+
} as const,
|
|
696
|
+
changed: true,
|
|
697
|
+
};
|
|
698
|
+
}),
|
|
699
|
+
release: () =>
|
|
700
|
+
withLockTransaction(locksPath, (locks) => {
|
|
701
|
+
const effectiveKey = resolveEffectiveKey();
|
|
702
|
+
const state = getLockState(
|
|
703
|
+
parseTelegramLockEntry(locks[effectiveKey]),
|
|
704
|
+
pid,
|
|
705
|
+
isAlive,
|
|
706
|
+
stateOptions(),
|
|
707
|
+
);
|
|
708
|
+
const changed =
|
|
709
|
+
ownedLockKey === effectiveKey &&
|
|
710
|
+
hasSameLockOwner(
|
|
711
|
+
parseTelegramLockEntry(locks[effectiveKey]),
|
|
712
|
+
ownedLock,
|
|
713
|
+
);
|
|
714
|
+
if (changed) {
|
|
715
|
+
delete locks[effectiveKey];
|
|
716
|
+
ownedLockKey = undefined;
|
|
717
|
+
ownedLock = undefined;
|
|
718
|
+
}
|
|
719
|
+
return { result: state, changed };
|
|
720
|
+
}),
|
|
330
721
|
getState: () => getLockState(readLock(), pid, isAlive, stateOptions()),
|
|
331
722
|
getStatusLabel: () =>
|
|
332
723
|
formatLockState(getLockState(readLock(), pid, isAlive, stateOptions())),
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
const lock =
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
: {}),
|
|
347
|
-
busSecret: options.busSecret ?? lock.busSecret,
|
|
348
|
-
});
|
|
349
|
-
return true;
|
|
724
|
+
getOwnedLeaderEpoch: () => {
|
|
725
|
+
const effectiveKey = resolveEffectiveKey();
|
|
726
|
+
const lock = parseTelegramLockEntry(readLocks(locksPath)[effectiveKey]);
|
|
727
|
+
const exactOwner = adoptCompatibleOwnedLock(effectiveKey, lock);
|
|
728
|
+
return hasSameLockOwner(lock, exactOwner) ? lock?.leaderEpoch : undefined;
|
|
729
|
+
},
|
|
730
|
+
owns: (ctx) => {
|
|
731
|
+
const effectiveKey = resolveEffectiveKey();
|
|
732
|
+
const lock = parseTelegramLockEntry(readLocks(locksPath)[effectiveKey]);
|
|
733
|
+
return hasSameLockOwner(
|
|
734
|
+
lock,
|
|
735
|
+
adoptCompatibleOwnedLock(effectiveKey, lock, ctx),
|
|
736
|
+
);
|
|
350
737
|
},
|
|
738
|
+
commitIfOwned: (commit) =>
|
|
739
|
+
withLockTransaction(locksPath, (locks) => {
|
|
740
|
+
const effectiveKey = resolveEffectiveKey();
|
|
741
|
+
const lock = parseTelegramLockEntry(locks[effectiveKey]);
|
|
742
|
+
const exactOwner =
|
|
743
|
+
ownedLockKey === effectiveKey && hasSameLockOwner(lock, ownedLock);
|
|
744
|
+
if (!exactOwner) {
|
|
745
|
+
if (ownedLockKey === effectiveKey) {
|
|
746
|
+
ownedLockKey = undefined;
|
|
747
|
+
ownedLock = undefined;
|
|
748
|
+
}
|
|
749
|
+
return { result: false, changed: false };
|
|
750
|
+
}
|
|
751
|
+
commit();
|
|
752
|
+
return { result: true, changed: false };
|
|
753
|
+
}),
|
|
754
|
+
refresh: (ctx) =>
|
|
755
|
+
withLockTransaction(locksPath, (locks) => {
|
|
756
|
+
const effectiveKey = resolveEffectiveKey();
|
|
757
|
+
const lock = parseTelegramLockEntry(locks[effectiveKey]);
|
|
758
|
+
const expectedOwner = adoptCompatibleOwnedLock(effectiveKey, lock, ctx);
|
|
759
|
+
if (!lock || !hasSameLockOwner(lock, expectedOwner)) {
|
|
760
|
+
if (ownedLockKey === effectiveKey) {
|
|
761
|
+
ownedLockKey = undefined;
|
|
762
|
+
ownedLock = undefined;
|
|
763
|
+
}
|
|
764
|
+
return { result: false, changed: false };
|
|
765
|
+
}
|
|
766
|
+
if (!options.instanceId) return { result: true, changed: false };
|
|
767
|
+
const refreshedLock: TelegramLockEntry = {
|
|
768
|
+
pid: lock.pid,
|
|
769
|
+
...(lock.cwd ? { cwd: lock.cwd } : {}),
|
|
770
|
+
instanceId: options.instanceId,
|
|
771
|
+
heartbeatMs: getNowMs(),
|
|
772
|
+
leaderEpoch:
|
|
773
|
+
lock.leaderEpoch ?? options.mintLeaderEpoch?.() ?? randomUUID(),
|
|
774
|
+
runtimeGeneration: lock.runtimeGeneration ?? runtimeGeneration,
|
|
775
|
+
...(options.busSocketPath
|
|
776
|
+
? { busSocketPath: options.busSocketPath }
|
|
777
|
+
: {}),
|
|
778
|
+
busSecret: options.busSecret ?? lock.busSecret,
|
|
779
|
+
};
|
|
780
|
+
locks[effectiveKey] = refreshedLock;
|
|
781
|
+
ownedLockKey = effectiveKey;
|
|
782
|
+
ownedLock = refreshedLock;
|
|
783
|
+
return { result: true, changed: true };
|
|
784
|
+
}),
|
|
351
785
|
};
|
|
352
786
|
}
|
|
353
787
|
|
|
@@ -374,6 +808,8 @@ export function createTelegramDirectDeliveryOwnershipChecker<
|
|
|
374
808
|
export interface TelegramLockedPollingStartOptions {
|
|
375
809
|
force?: boolean;
|
|
376
810
|
forceFreshLeaderThread?: boolean;
|
|
811
|
+
election?: { expectedOwner?: TelegramLockEntry };
|
|
812
|
+
onAcquired?: () => Promise<void> | void;
|
|
377
813
|
}
|
|
378
814
|
|
|
379
815
|
export type TelegramLockedPollingStartResult =
|
|
@@ -434,6 +870,7 @@ export function createTelegramLockedPollingRuntime<
|
|
|
434
870
|
): TelegramLockedPollingRuntime<TContext> {
|
|
435
871
|
let ownershipInterval: ReturnType<typeof setInterval> | undefined;
|
|
436
872
|
let ownershipStop: Promise<void> | undefined;
|
|
873
|
+
let takeoverCandidate: TelegramLockEntry | undefined;
|
|
437
874
|
let sessionAutoStartRun: Promise<void> | undefined;
|
|
438
875
|
let sessionAutoStartGeneration = 0;
|
|
439
876
|
const ownershipCheckMs = deps.ownershipCheckMs ?? 1000;
|
|
@@ -480,6 +917,36 @@ export function createTelegramLockedPollingRuntime<
|
|
|
480
917
|
}, ownershipCheckMs);
|
|
481
918
|
ownershipInterval.unref?.();
|
|
482
919
|
};
|
|
920
|
+
const runOwnedPollingStart = async (
|
|
921
|
+
ctx: TContext,
|
|
922
|
+
options: TelegramLockedPollingStartOptions,
|
|
923
|
+
): Promise<boolean> => {
|
|
924
|
+
startOwnershipWatcher(ctx);
|
|
925
|
+
try {
|
|
926
|
+
if (!deps.lock.refresh(snapshotLockContext(ctx))) {
|
|
927
|
+
stopOwnershipWatcher();
|
|
928
|
+
return false;
|
|
929
|
+
}
|
|
930
|
+
await options.onAcquired?.();
|
|
931
|
+
await deps.startPolling(ctx, options);
|
|
932
|
+
} catch (error) {
|
|
933
|
+
stopOwnershipWatcher();
|
|
934
|
+
try {
|
|
935
|
+
await deps.stopPolling();
|
|
936
|
+
} catch (stopError) {
|
|
937
|
+
deps.recordRuntimeEvent?.("lock", stopError, {
|
|
938
|
+
phase: "startup-rollback",
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
deps.lock.release();
|
|
942
|
+
throw error;
|
|
943
|
+
}
|
|
944
|
+
if (deps.lock.owns(ctx)) return true;
|
|
945
|
+
stopOwnershipWatcher();
|
|
946
|
+
if (ownershipStop) await ownershipStop;
|
|
947
|
+
await deps.stopPolling();
|
|
948
|
+
return false;
|
|
949
|
+
};
|
|
483
950
|
const canStartPolling = (ctx: TContext): boolean =>
|
|
484
951
|
deps.canStartPolling?.(ctx) ?? true;
|
|
485
952
|
const formatStartBlockedMessage = (ctx: TContext): string =>
|
|
@@ -493,8 +960,35 @@ export function createTelegramLockedPollingRuntime<
|
|
|
493
960
|
if (!canStartPolling(ctx)) {
|
|
494
961
|
return { ok: false, message: formatStartBlockedMessage(ctx) };
|
|
495
962
|
}
|
|
496
|
-
|
|
963
|
+
let acquired = deps.lock.acquire(ctx, {
|
|
964
|
+
force: options.force,
|
|
965
|
+
expectedOwner:
|
|
966
|
+
options.election?.expectedOwner ??
|
|
967
|
+
(options.force ? takeoverCandidate : undefined),
|
|
968
|
+
election: options.election !== undefined,
|
|
969
|
+
});
|
|
970
|
+
if (!acquired.ok && !options.election) {
|
|
971
|
+
const currentState = deps.lock.getState();
|
|
972
|
+
if (
|
|
973
|
+
currentState.kind === "active-here" &&
|
|
974
|
+
hasSameLockOwner(currentState.lock, acquired.lock)
|
|
975
|
+
) {
|
|
976
|
+
acquired = deps.lock.acquire(ctx, {
|
|
977
|
+
force: true,
|
|
978
|
+
expectedOwner: acquired.lock,
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
}
|
|
497
982
|
if (!acquired.ok) {
|
|
983
|
+
takeoverCandidate = acquired.lock;
|
|
984
|
+
if (options.election) {
|
|
985
|
+
return {
|
|
986
|
+
ok: false,
|
|
987
|
+
canTakeover: false,
|
|
988
|
+
owner: formatTelegramLockEntry(acquired.lock),
|
|
989
|
+
message: "Telegram leadership election lost to another live owner.",
|
|
990
|
+
};
|
|
991
|
+
}
|
|
498
992
|
if (deps.registerFollowerWithOwner) {
|
|
499
993
|
let failureMessage: string | undefined;
|
|
500
994
|
try {
|
|
@@ -532,8 +1026,14 @@ export function createTelegramLockedPollingRuntime<
|
|
|
532
1026
|
message: `Telegram bridge is active in another Pi instance (${owner}).`,
|
|
533
1027
|
};
|
|
534
1028
|
}
|
|
535
|
-
|
|
536
|
-
|
|
1029
|
+
takeoverCandidate = undefined;
|
|
1030
|
+
if (!(await runOwnedPollingStart(ctx, options))) {
|
|
1031
|
+
return {
|
|
1032
|
+
ok: false,
|
|
1033
|
+
canTakeover: false,
|
|
1034
|
+
message: "Telegram leadership changed during polling startup.",
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
537
1037
|
deps.updateStatus(ctx);
|
|
538
1038
|
const staleSuffix = acquired.replacedStale ? " Replaced stale lock." : "";
|
|
539
1039
|
return { ok: true, message: `Telegram bridge connected.${staleSuffix}` };
|
|
@@ -557,7 +1057,16 @@ export function createTelegramLockedPollingRuntime<
|
|
|
557
1057
|
const state = ownsCurrentLock ? undefined : deps.lock.getState();
|
|
558
1058
|
const canResumeStaleSameCwd =
|
|
559
1059
|
state?.kind === "stale" && state.lock.cwd === ctx.cwd;
|
|
560
|
-
|
|
1060
|
+
const canHandoffSameProcess =
|
|
1061
|
+
state?.kind === "active-here" &&
|
|
1062
|
+
(!state.lock.cwd || state.lock.cwd === ctx.cwd);
|
|
1063
|
+
if (
|
|
1064
|
+
!ownsCurrentLock &&
|
|
1065
|
+
!canResumeStaleSameCwd &&
|
|
1066
|
+
!canHandoffSameProcess
|
|
1067
|
+
) {
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
561
1070
|
sessionAutoStartGeneration += 1;
|
|
562
1071
|
const generation = sessionAutoStartGeneration;
|
|
563
1072
|
const startedAtMs = Date.now();
|
|
@@ -567,14 +1076,18 @@ export function createTelegramLockedPollingRuntime<
|
|
|
567
1076
|
const run = (async () => {
|
|
568
1077
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
569
1078
|
if (generation !== sessionAutoStartGeneration) return;
|
|
570
|
-
if (canResumeStaleSameCwd) {
|
|
571
|
-
const acquired = deps.lock.acquire(
|
|
1079
|
+
if (canResumeStaleSameCwd || canHandoffSameProcess) {
|
|
1080
|
+
const acquired = deps.lock.acquire(
|
|
1081
|
+
ctx,
|
|
1082
|
+
canHandoffSameProcess
|
|
1083
|
+
? { force: true, expectedOwner: state?.lock }
|
|
1084
|
+
: undefined,
|
|
1085
|
+
);
|
|
572
1086
|
if (!acquired.ok) return;
|
|
573
1087
|
}
|
|
574
1088
|
if (generation !== sessionAutoStartGeneration) return;
|
|
575
|
-
await
|
|
1089
|
+
if (!(await runOwnedPollingStart(ctx, {}))) return;
|
|
576
1090
|
if (generation !== sessionAutoStartGeneration) return;
|
|
577
|
-
startOwnershipWatcher(ctx);
|
|
578
1091
|
deps.updateStatus(ctx);
|
|
579
1092
|
deps.recordRuntimeEvent?.("lock", "Telegram auto-start completed", {
|
|
580
1093
|
phase: "auto-start-complete",
|