@llblab/pi-telegram 0.21.1 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +7 -4
- package/BACKLOG.md +29 -1
- package/CHANGELOG.md +479 -452
- package/docs/architecture.md +12 -10
- package/docs/delivery.md +2 -1
- package/docs/locks.md +13 -5
- package/docs/multi-instance-bus.md +5 -5
- package/index.ts +311 -633
- package/lib/bindings.ts +59 -21
- package/lib/bus-api.ts +12 -2
- package/lib/bus-follower.ts +327 -83
- package/lib/bus-leader.ts +201 -121
- package/lib/bus.ts +345 -37
- package/lib/config.ts +168 -28
- package/lib/delivery.ts +148 -61
- package/lib/lifecycle.ts +199 -8
- package/lib/locks.ts +854 -57
- package/lib/logs.ts +185 -22
- package/lib/media.ts +79 -24
- package/lib/model.ts +5 -10
- package/lib/ownership.ts +150 -6
- package/lib/polling.ts +156 -7
- package/lib/preview.ts +15 -3
- package/lib/queue.ts +167 -20
- package/lib/routing.ts +68 -12
- package/lib/sync.ts +105 -32
- package/lib/telegram-api.ts +86 -10
- package/lib/text-groups.ts +71 -15
- package/lib/thread-reconciler.ts +49 -36
- package/lib/threads.ts +505 -118
- package/package.json +1 -1
package/lib/locks.ts
CHANGED
|
@@ -5,20 +5,42 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
|
+
chmodSync,
|
|
8
9
|
existsSync,
|
|
10
|
+
lstatSync,
|
|
9
11
|
mkdirSync,
|
|
12
|
+
mkdtempSync,
|
|
10
13
|
readFileSync,
|
|
14
|
+
readdirSync,
|
|
11
15
|
renameSync,
|
|
16
|
+
rmSync,
|
|
12
17
|
unlinkSync,
|
|
13
18
|
writeFileSync,
|
|
14
19
|
} from "node:fs";
|
|
15
|
-
import {
|
|
20
|
+
import { randomUUID } from "node:crypto";
|
|
21
|
+
import { basename, dirname, join } from "node:path";
|
|
16
22
|
import { resolveTelegramLocksPath } from "./paths.ts";
|
|
17
23
|
|
|
18
24
|
export const TELEGRAM_LOCK_KEY = "@llblab/pi-telegram";
|
|
19
25
|
export const TELEGRAM_BUS_LEADER_STALE_HEARTBEAT_MS = 5_000;
|
|
20
26
|
const TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS = 5;
|
|
21
27
|
const TELEGRAM_LOCK_WRITE_RETRY_DELAY_MS = 25;
|
|
28
|
+
const TELEGRAM_LOCK_TRANSACTION_ATTEMPTS = 80;
|
|
29
|
+
const TELEGRAM_LOCK_TRANSACTION_RETRY_DELAY_MS = 25;
|
|
30
|
+
const TELEGRAM_LOCK_RUNTIME_GENERATION_KEY =
|
|
31
|
+
"__piTelegramLockRuntimeGeneration__";
|
|
32
|
+
|
|
33
|
+
function allocateTelegramLockRuntimeGeneration(): number {
|
|
34
|
+
const globals = globalThis as Record<string, unknown>;
|
|
35
|
+
const previous = globals[TELEGRAM_LOCK_RUNTIME_GENERATION_KEY];
|
|
36
|
+
const previousGeneration =
|
|
37
|
+
typeof previous === "number" && Number.isSafeInteger(previous)
|
|
38
|
+
? previous
|
|
39
|
+
: 0;
|
|
40
|
+
const generation = Math.max(Date.now(), previousGeneration + 1);
|
|
41
|
+
globals[TELEGRAM_LOCK_RUNTIME_GENERATION_KEY] = generation;
|
|
42
|
+
return generation;
|
|
43
|
+
}
|
|
22
44
|
|
|
23
45
|
function getLocksPath(): string {
|
|
24
46
|
return resolveTelegramLocksPath();
|
|
@@ -51,7 +73,8 @@ export interface TelegramLockEntry {
|
|
|
51
73
|
cwd?: string;
|
|
52
74
|
instanceId?: string;
|
|
53
75
|
heartbeatMs?: number;
|
|
54
|
-
leaderEpoch?: number;
|
|
76
|
+
leaderEpoch?: number | string;
|
|
77
|
+
runtimeGeneration?: number;
|
|
55
78
|
busSocketPath?: string;
|
|
56
79
|
busSecret?: string;
|
|
57
80
|
}
|
|
@@ -68,6 +91,8 @@ export type TelegramLockState =
|
|
|
68
91
|
|
|
69
92
|
export interface TelegramLockAcquireOptions {
|
|
70
93
|
force?: boolean;
|
|
94
|
+
expectedOwner?: TelegramLockEntry;
|
|
95
|
+
election?: boolean;
|
|
71
96
|
}
|
|
72
97
|
|
|
73
98
|
export type TelegramLockAcquireResult =
|
|
@@ -82,7 +107,9 @@ export interface TelegramLockRuntime<TContext extends TelegramLockContext> {
|
|
|
82
107
|
release: () => TelegramLockState;
|
|
83
108
|
getState: () => TelegramLockState;
|
|
84
109
|
getStatusLabel: () => string;
|
|
110
|
+
getOwnedLeaderEpoch: () => number | string | undefined;
|
|
85
111
|
owns: (ctx?: TelegramLockContext) => boolean;
|
|
112
|
+
commitIfOwned: (commit: () => void) => boolean;
|
|
86
113
|
refresh: (ctx?: TelegramLockContext) => boolean;
|
|
87
114
|
}
|
|
88
115
|
|
|
@@ -107,6 +134,8 @@ export interface TelegramLockRuntimeOptions {
|
|
|
107
134
|
busSocketPath?: string;
|
|
108
135
|
busSecret?: string;
|
|
109
136
|
getNowMs?: () => number;
|
|
137
|
+
mintLeaderEpoch?: () => number | string;
|
|
138
|
+
runtimeGeneration?: number;
|
|
110
139
|
staleHeartbeatMs?: number;
|
|
111
140
|
}
|
|
112
141
|
|
|
@@ -122,6 +151,21 @@ export function readLocks(path = getLocksPath()): Record<string, unknown> {
|
|
|
122
151
|
}
|
|
123
152
|
}
|
|
124
153
|
|
|
154
|
+
function readLocksForTransaction(path: string): Record<string, unknown> {
|
|
155
|
+
let source: string;
|
|
156
|
+
try {
|
|
157
|
+
source = readFileSync(path, "utf8");
|
|
158
|
+
} catch (error) {
|
|
159
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return {};
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
const value: unknown = JSON.parse(source);
|
|
163
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
164
|
+
throw new Error(`Invalid Telegram lock registry: ${path}`);
|
|
165
|
+
}
|
|
166
|
+
return value as Record<string, unknown>;
|
|
167
|
+
}
|
|
168
|
+
|
|
125
169
|
function isRetryableLockWriteError(error: unknown): boolean {
|
|
126
170
|
const code = (error as { code?: unknown })?.code;
|
|
127
171
|
return code === "EPERM" || code === "EBUSY" || code === "EACCES";
|
|
@@ -132,11 +176,497 @@ function sleepSync(ms: number): void {
|
|
|
132
176
|
Atomics.wait(new Int32Array(buffer), 0, 0, ms);
|
|
133
177
|
}
|
|
134
178
|
|
|
179
|
+
interface TelegramLockTransactionOwner {
|
|
180
|
+
pid: number;
|
|
181
|
+
acquiredAtMs: number;
|
|
182
|
+
generation: string;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const TELEGRAM_TRANSACTION_OWNER_PATTERN =
|
|
186
|
+
/^owner\.([A-Za-z0-9-]+)\.json$/u;
|
|
187
|
+
|
|
188
|
+
function getLockTransactionOwnerFile(generation: string): string {
|
|
189
|
+
return `owner.${generation}.json`;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function getLockTransactionOwnerPath(path: string): string {
|
|
193
|
+
const stat = lstatSync(path);
|
|
194
|
+
if (stat.isDirectory()) {
|
|
195
|
+
const entries = readdirSync(path);
|
|
196
|
+
if (
|
|
197
|
+
entries.length === 1 &&
|
|
198
|
+
(TELEGRAM_TRANSACTION_OWNER_PATTERN.test(entries[0]) ||
|
|
199
|
+
TELEGRAM_TRANSACTION_RECLAIM_PATTERN.test(entries[0]))
|
|
200
|
+
) {
|
|
201
|
+
return join(path, entries[0]);
|
|
202
|
+
}
|
|
203
|
+
throw new Error(`Unverifiable Telegram lock transaction guard: ${path}`);
|
|
204
|
+
}
|
|
205
|
+
if (stat.isFile()) return path;
|
|
206
|
+
throw new Error(`Unsupported Telegram lock transaction guard: ${path}`);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
function readLockTransactionOwner(
|
|
210
|
+
path: string,
|
|
211
|
+
): TelegramLockTransactionOwner | undefined {
|
|
212
|
+
try {
|
|
213
|
+
const value = JSON.parse(
|
|
214
|
+
readFileSync(getLockTransactionOwnerPath(path), "utf8"),
|
|
215
|
+
) as Record<string, unknown>;
|
|
216
|
+
if (
|
|
217
|
+
typeof value.pid !== "number" ||
|
|
218
|
+
typeof value.acquiredAtMs !== "number" ||
|
|
219
|
+
typeof value.generation !== "string"
|
|
220
|
+
) {
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
const ownerMatch = TELEGRAM_TRANSACTION_OWNER_PATTERN.exec(
|
|
224
|
+
basename(getLockTransactionOwnerPath(path)),
|
|
225
|
+
);
|
|
226
|
+
if (ownerMatch && ownerMatch[1] !== value.generation) return undefined;
|
|
227
|
+
return {
|
|
228
|
+
pid: value.pid,
|
|
229
|
+
acquiredAtMs: value.acquiredAtMs,
|
|
230
|
+
generation: value.generation,
|
|
231
|
+
};
|
|
232
|
+
} catch {
|
|
233
|
+
return undefined;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function createLockTransactionContentionError(path: string): Error {
|
|
238
|
+
return Object.assign(
|
|
239
|
+
new Error(`Telegram lock transaction guard already exists: ${path}`),
|
|
240
|
+
{ code: "EEXIST" },
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function isLockTransactionContentionError(
|
|
245
|
+
error: unknown,
|
|
246
|
+
path: string,
|
|
247
|
+
): boolean {
|
|
248
|
+
const code = (error as { code?: unknown })?.code;
|
|
249
|
+
if (
|
|
250
|
+
code === "EEXIST" ||
|
|
251
|
+
code === "ENOTEMPTY" ||
|
|
252
|
+
code === "ENOTDIR" ||
|
|
253
|
+
code === "EISDIR"
|
|
254
|
+
) {
|
|
255
|
+
return true;
|
|
256
|
+
}
|
|
257
|
+
return existsSync(path) && (code === "EPERM" || code === "EACCES");
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function removeLockTransactionGuard(path: string): void {
|
|
261
|
+
rmSync(path, { recursive: true, force: true });
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function createLockTransactionGuard(
|
|
265
|
+
path: string,
|
|
266
|
+
): TelegramLockTransactionOwner {
|
|
267
|
+
const owner: TelegramLockTransactionOwner = {
|
|
268
|
+
pid: process.pid,
|
|
269
|
+
acquiredAtMs: Date.now(),
|
|
270
|
+
generation: randomUUID(),
|
|
271
|
+
};
|
|
272
|
+
const stagedPath = mkdtempSync(`${path}.staged.`);
|
|
273
|
+
try {
|
|
274
|
+
chmodSync(stagedPath, 0o700);
|
|
275
|
+
writeFileSync(
|
|
276
|
+
join(stagedPath, getLockTransactionOwnerFile(owner.generation)),
|
|
277
|
+
`${JSON.stringify(owner)}\n`,
|
|
278
|
+
{ encoding: "utf8", flag: "wx", mode: 0o600 },
|
|
279
|
+
);
|
|
280
|
+
if (existsSync(path)) throw createLockTransactionContentionError(path);
|
|
281
|
+
renameSync(stagedPath, path);
|
|
282
|
+
return owner;
|
|
283
|
+
} finally {
|
|
284
|
+
try {
|
|
285
|
+
removeLockTransactionGuard(stagedPath);
|
|
286
|
+
} catch {
|
|
287
|
+
/* best effort */
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function releaseLockTransactionGuard(
|
|
293
|
+
path: string,
|
|
294
|
+
owner: TelegramLockTransactionOwner,
|
|
295
|
+
): void {
|
|
296
|
+
const current = readLockTransactionOwner(path);
|
|
297
|
+
if (!current) {
|
|
298
|
+
if (!existsSync(path)) return;
|
|
299
|
+
throw new Error(`Cannot verify Telegram lock transaction guard: ${path}`);
|
|
300
|
+
}
|
|
301
|
+
if (
|
|
302
|
+
current.pid !== owner.pid ||
|
|
303
|
+
current.generation !== owner.generation ||
|
|
304
|
+
current.acquiredAtMs !== owner.acquiredAtMs
|
|
305
|
+
) {
|
|
306
|
+
throw new Error(
|
|
307
|
+
`Telegram lock transaction guard changed ownership: ${path}`,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
const releasedPath = `${path}.released.${randomUUID()}`;
|
|
311
|
+
for (
|
|
312
|
+
let attempt = 0;
|
|
313
|
+
attempt < TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS;
|
|
314
|
+
attempt += 1
|
|
315
|
+
) {
|
|
316
|
+
try {
|
|
317
|
+
renameSync(path, releasedPath);
|
|
318
|
+
try {
|
|
319
|
+
removeLockTransactionGuard(releasedPath);
|
|
320
|
+
} catch {
|
|
321
|
+
/* released debris cannot retain transaction authority */
|
|
322
|
+
}
|
|
323
|
+
return;
|
|
324
|
+
} catch (error) {
|
|
325
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return;
|
|
326
|
+
if (
|
|
327
|
+
!isRetryableLockWriteError(error) ||
|
|
328
|
+
attempt === TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS - 1
|
|
329
|
+
) {
|
|
330
|
+
throw error;
|
|
331
|
+
}
|
|
332
|
+
sleepSync(TELEGRAM_LOCK_WRITE_RETRY_DELAY_MS * (attempt + 1));
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function isAbandonedLockTransaction(path: string): boolean {
|
|
338
|
+
const owner = readLockTransactionOwner(path);
|
|
339
|
+
return owner ? !isProcessAlive(owner.pid) : false;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const TELEGRAM_TRANSACTION_RECLAIM_PATTERN =
|
|
343
|
+
/^owner\.reclaim\.(\d+)\.([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.json$/u;
|
|
344
|
+
const TELEGRAM_ACTIVE_TRANSACTION_RECLAIMS = Symbol.for(
|
|
345
|
+
"@llblab/pi-telegram/active-transaction-reclaims",
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
type TelegramTransactionGlobal = typeof globalThis & {
|
|
349
|
+
[TELEGRAM_ACTIVE_TRANSACTION_RECLAIMS]?: Set<string>;
|
|
350
|
+
};
|
|
351
|
+
|
|
352
|
+
export interface TelegramFileTransactionOptions {
|
|
353
|
+
recoveryRename?: typeof renameSync;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
function getActiveTransactionReclaims(): Set<string> {
|
|
357
|
+
const root = globalThis as TelegramTransactionGlobal;
|
|
358
|
+
return (root[TELEGRAM_ACTIVE_TRANSACTION_RECLAIMS] ??= new Set());
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function reclaimAbandonedDirectoryGuard(
|
|
362
|
+
path: string,
|
|
363
|
+
options: TelegramFileTransactionOptions = {},
|
|
364
|
+
): boolean {
|
|
365
|
+
try {
|
|
366
|
+
if (!lstatSync(path).isDirectory()) return false;
|
|
367
|
+
} catch {
|
|
368
|
+
return false;
|
|
369
|
+
}
|
|
370
|
+
const entries = readdirSync(path);
|
|
371
|
+
if (entries.length !== 1) return false;
|
|
372
|
+
const entry = entries[0];
|
|
373
|
+
let observedPid: number;
|
|
374
|
+
let observedReclaimGeneration: string | undefined;
|
|
375
|
+
if (TELEGRAM_TRANSACTION_OWNER_PATTERN.test(entry)) {
|
|
376
|
+
const owner = readLockTransactionOwner(path);
|
|
377
|
+
if (!owner) return false;
|
|
378
|
+
observedPid = owner.pid;
|
|
379
|
+
} else {
|
|
380
|
+
const match = TELEGRAM_TRANSACTION_RECLAIM_PATTERN.exec(entry);
|
|
381
|
+
if (!match) return false;
|
|
382
|
+
observedPid = Number.parseInt(match[1], 10);
|
|
383
|
+
observedReclaimGeneration = match[2];
|
|
384
|
+
}
|
|
385
|
+
const activeReclaims = getActiveTransactionReclaims();
|
|
386
|
+
if (
|
|
387
|
+
observedPid === process.pid &&
|
|
388
|
+
observedReclaimGeneration !== undefined
|
|
389
|
+
) {
|
|
390
|
+
if (activeReclaims.has(observedReclaimGeneration)) return false;
|
|
391
|
+
} else if (isProcessAlive(observedPid)) {
|
|
392
|
+
return false;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const renameRecovery = options.recoveryRename ?? renameSync;
|
|
396
|
+
const sourcePath = join(path, entry);
|
|
397
|
+
const reclaimGeneration = randomUUID();
|
|
398
|
+
const reclaimPath = join(
|
|
399
|
+
path,
|
|
400
|
+
`owner.reclaim.${process.pid}.${reclaimGeneration}.json`,
|
|
401
|
+
);
|
|
402
|
+
try {
|
|
403
|
+
// Claim inside the still-occupied guard before making its stable path free.
|
|
404
|
+
renameRecovery(sourcePath, reclaimPath);
|
|
405
|
+
} catch (error) {
|
|
406
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return false;
|
|
407
|
+
throw error;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
const renameWithRetry = (fromPath: string, toPath: string): boolean => {
|
|
411
|
+
for (
|
|
412
|
+
let attempt = 0;
|
|
413
|
+
attempt < TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS;
|
|
414
|
+
attempt += 1
|
|
415
|
+
) {
|
|
416
|
+
try {
|
|
417
|
+
renameRecovery(fromPath, toPath);
|
|
418
|
+
return true;
|
|
419
|
+
} catch (error) {
|
|
420
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return false;
|
|
421
|
+
if (
|
|
422
|
+
!isRetryableLockWriteError(error) ||
|
|
423
|
+
attempt === TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS - 1
|
|
424
|
+
) {
|
|
425
|
+
throw error;
|
|
426
|
+
}
|
|
427
|
+
sleepSync(TELEGRAM_LOCK_WRITE_RETRY_DELAY_MS * (attempt + 1));
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return false;
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
activeReclaims.add(reclaimGeneration);
|
|
434
|
+
const stalePath = `${path}.stale.${process.pid}.${randomUUID()}`;
|
|
435
|
+
try {
|
|
436
|
+
try {
|
|
437
|
+
if (!renameWithRetry(path, stalePath)) return false;
|
|
438
|
+
} catch (renameError) {
|
|
439
|
+
try {
|
|
440
|
+
if (!renameWithRetry(reclaimPath, sourcePath)) throw renameError;
|
|
441
|
+
} catch (rollbackError) {
|
|
442
|
+
throw new AggregateError(
|
|
443
|
+
[renameError, rollbackError],
|
|
444
|
+
`Failed to reclaim or restore Telegram lock transaction guard: ${path}`,
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
throw renameError;
|
|
448
|
+
}
|
|
449
|
+
} finally {
|
|
450
|
+
activeReclaims.delete(reclaimGeneration);
|
|
451
|
+
}
|
|
452
|
+
try {
|
|
453
|
+
removeLockTransactionGuard(stalePath);
|
|
454
|
+
} catch {
|
|
455
|
+
/* stale debris cannot retain transaction authority */
|
|
456
|
+
}
|
|
457
|
+
return true;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function acquireRecoverableDirectoryGuard(
|
|
461
|
+
path: string,
|
|
462
|
+
options: TelegramFileTransactionOptions = {},
|
|
463
|
+
): TelegramLockTransactionOwner | undefined {
|
|
464
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
465
|
+
try {
|
|
466
|
+
return createLockTransactionGuard(path);
|
|
467
|
+
} catch (error) {
|
|
468
|
+
if (!isLockTransactionContentionError(error, path)) throw error;
|
|
469
|
+
if (!reclaimAbandonedDirectoryGuard(path, options)) return undefined;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
return undefined;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
function removeAbandonedLegacyRecoveryGuard(
|
|
476
|
+
path: string,
|
|
477
|
+
options: TelegramFileTransactionOptions = {},
|
|
478
|
+
): boolean {
|
|
479
|
+
try {
|
|
480
|
+
if (!lstatSync(path).isFile() || !isAbandonedLockTransaction(path))
|
|
481
|
+
return false;
|
|
482
|
+
} catch {
|
|
483
|
+
return false;
|
|
484
|
+
}
|
|
485
|
+
const migrationGuardPath = `${path}.migration`;
|
|
486
|
+
const migrationOwner = acquireRecoverableDirectoryGuard(
|
|
487
|
+
migrationGuardPath,
|
|
488
|
+
options,
|
|
489
|
+
);
|
|
490
|
+
if (!migrationOwner) return false;
|
|
491
|
+
try {
|
|
492
|
+
try {
|
|
493
|
+
if (!lstatSync(path).isFile() || !isAbandonedLockTransaction(path))
|
|
494
|
+
return false;
|
|
495
|
+
} catch {
|
|
496
|
+
return false;
|
|
497
|
+
}
|
|
498
|
+
const stalePath = `${path}.stale.${process.pid}.${randomUUID()}`;
|
|
499
|
+
try {
|
|
500
|
+
renameSync(path, stalePath);
|
|
501
|
+
} catch (error) {
|
|
502
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return false;
|
|
503
|
+
throw error;
|
|
504
|
+
}
|
|
505
|
+
try {
|
|
506
|
+
removeLockTransactionGuard(stalePath);
|
|
507
|
+
} catch {
|
|
508
|
+
/* stale debris cannot retain transaction authority */
|
|
509
|
+
}
|
|
510
|
+
return true;
|
|
511
|
+
} finally {
|
|
512
|
+
releaseLockTransactionGuard(migrationGuardPath, migrationOwner);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function acquireLegacyRecoveryGuard(
|
|
517
|
+
path: string,
|
|
518
|
+
options: TelegramFileTransactionOptions = {},
|
|
519
|
+
): TelegramLockTransactionOwner | undefined {
|
|
520
|
+
let owner = acquireRecoverableDirectoryGuard(path, options);
|
|
521
|
+
if (owner) return owner;
|
|
522
|
+
if (!removeAbandonedLegacyRecoveryGuard(path, options)) return undefined;
|
|
523
|
+
owner = acquireRecoverableDirectoryGuard(path, options);
|
|
524
|
+
return owner;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
function createRecoveredLockTransactionGuard(
|
|
528
|
+
path: string,
|
|
529
|
+
): TelegramLockTransactionOwner | undefined {
|
|
530
|
+
try {
|
|
531
|
+
return createLockTransactionGuard(path);
|
|
532
|
+
} catch (error) {
|
|
533
|
+
if (isLockTransactionContentionError(error, path)) return undefined;
|
|
534
|
+
throw error;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function recoverAbandonedLockTransaction(
|
|
539
|
+
path: string,
|
|
540
|
+
options: TelegramFileTransactionOptions = {},
|
|
541
|
+
): TelegramLockTransactionOwner | undefined {
|
|
542
|
+
if (!isAbandonedLockTransaction(path)) return undefined;
|
|
543
|
+
let isDirectory: boolean;
|
|
544
|
+
try {
|
|
545
|
+
isDirectory = lstatSync(path).isDirectory();
|
|
546
|
+
} catch {
|
|
547
|
+
return undefined;
|
|
548
|
+
}
|
|
549
|
+
if (isDirectory) {
|
|
550
|
+
if (!reclaimAbandonedDirectoryGuard(path, options)) return undefined;
|
|
551
|
+
const recoveredOwner = createRecoveredLockTransactionGuard(path);
|
|
552
|
+
try {
|
|
553
|
+
reclaimAbandonedDirectoryGuard(`${path}.recovery`, options);
|
|
554
|
+
return recoveredOwner;
|
|
555
|
+
} catch (error) {
|
|
556
|
+
if (recoveredOwner) {
|
|
557
|
+
try {
|
|
558
|
+
releaseLockTransactionGuard(path, recoveredOwner);
|
|
559
|
+
} catch {
|
|
560
|
+
/* preserve the recovery cleanup failure */
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
throw error;
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const recoveryGuardPath = `${path}.recovery`;
|
|
568
|
+
const recoveryOwner = acquireLegacyRecoveryGuard(
|
|
569
|
+
recoveryGuardPath,
|
|
570
|
+
options,
|
|
571
|
+
);
|
|
572
|
+
if (!recoveryOwner) return undefined;
|
|
573
|
+
let recoveredOwner: TelegramLockTransactionOwner | undefined;
|
|
574
|
+
try {
|
|
575
|
+
if (!isAbandonedLockTransaction(path)) return undefined;
|
|
576
|
+
const stalePath = `${path}.stale.${process.pid}.${randomUUID()}`;
|
|
577
|
+
try {
|
|
578
|
+
renameSync(path, stalePath);
|
|
579
|
+
} catch (error) {
|
|
580
|
+
if ((error as { code?: unknown })?.code === "ENOENT") return undefined;
|
|
581
|
+
throw error;
|
|
582
|
+
}
|
|
583
|
+
try {
|
|
584
|
+
removeLockTransactionGuard(stalePath);
|
|
585
|
+
} catch {
|
|
586
|
+
/* stale debris cannot retain transaction authority */
|
|
587
|
+
}
|
|
588
|
+
recoveredOwner = createRecoveredLockTransactionGuard(path);
|
|
589
|
+
return recoveredOwner;
|
|
590
|
+
} finally {
|
|
591
|
+
try {
|
|
592
|
+
releaseLockTransactionGuard(recoveryGuardPath, recoveryOwner);
|
|
593
|
+
} catch (error) {
|
|
594
|
+
if (recoveredOwner) {
|
|
595
|
+
try {
|
|
596
|
+
releaseLockTransactionGuard(path, recoveredOwner);
|
|
597
|
+
} catch {
|
|
598
|
+
/* preserve the recovery cleanup failure */
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
throw error;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
function acquireLockTransaction(
|
|
607
|
+
path: string,
|
|
608
|
+
options: TelegramFileTransactionOptions = {},
|
|
609
|
+
): TelegramLockTransactionOwner {
|
|
610
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
611
|
+
for (
|
|
612
|
+
let attempt = 0;
|
|
613
|
+
attempt < TELEGRAM_LOCK_TRANSACTION_ATTEMPTS;
|
|
614
|
+
attempt += 1
|
|
615
|
+
) {
|
|
616
|
+
try {
|
|
617
|
+
return createLockTransactionGuard(path);
|
|
618
|
+
} catch (error) {
|
|
619
|
+
if (!isLockTransactionContentionError(error, path)) throw error;
|
|
620
|
+
const recoveredOwner = recoverAbandonedLockTransaction(path, options);
|
|
621
|
+
if (recoveredOwner !== undefined) return recoveredOwner;
|
|
622
|
+
if (attempt === TELEGRAM_LOCK_TRANSACTION_ATTEMPTS - 1) {
|
|
623
|
+
throw new Error(
|
|
624
|
+
`Timed out acquiring Telegram lock transaction: ${path}`,
|
|
625
|
+
);
|
|
626
|
+
}
|
|
627
|
+
sleepSync(TELEGRAM_LOCK_TRANSACTION_RETRY_DELAY_MS);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
throw new Error(`Failed to acquire Telegram lock transaction: ${path}`);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
export function withTelegramFileTransaction<T>(
|
|
634
|
+
transactionPath: string,
|
|
635
|
+
operation: () => T,
|
|
636
|
+
options: TelegramFileTransactionOptions = {},
|
|
637
|
+
): T {
|
|
638
|
+
const owner = acquireLockTransaction(transactionPath, options);
|
|
639
|
+
try {
|
|
640
|
+
return operation();
|
|
641
|
+
} finally {
|
|
642
|
+
releaseLockTransactionGuard(transactionPath, owner);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
function withLockTransaction<T>(
|
|
647
|
+
locksPath: string,
|
|
648
|
+
mutate: (locks: Record<string, unknown>) => {
|
|
649
|
+
result: T;
|
|
650
|
+
changed: boolean;
|
|
651
|
+
},
|
|
652
|
+
): T {
|
|
653
|
+
return withTelegramFileTransaction(`${locksPath}.transaction`, () => {
|
|
654
|
+
const locks = readLocksForTransaction(locksPath);
|
|
655
|
+
const outcome = mutate(locks);
|
|
656
|
+
if (outcome.changed) writeLocks(locksPath, locks);
|
|
657
|
+
return outcome.result;
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
|
|
135
661
|
export function writeLocks(path: string, locks: Record<string, unknown>): void {
|
|
136
662
|
mkdirSync(dirname(path), { recursive: true });
|
|
137
663
|
const payload = `${JSON.stringify(locks, null, 2)}\n`;
|
|
138
664
|
let lastError: unknown;
|
|
139
|
-
for (
|
|
665
|
+
for (
|
|
666
|
+
let attempt = 0;
|
|
667
|
+
attempt < TELEGRAM_LOCK_WRITE_RETRY_ATTEMPTS;
|
|
668
|
+
attempt += 1
|
|
669
|
+
) {
|
|
140
670
|
const tempPath = `${path}.${process.pid}.${Date.now()}.${attempt}.tmp`;
|
|
141
671
|
try {
|
|
142
672
|
writeFileSync(tempPath, payload, {
|
|
@@ -179,7 +709,14 @@ export function parseTelegramLockEntry(
|
|
|
179
709
|
heartbeatMs:
|
|
180
710
|
typeof record.heartbeatMs === "number" ? record.heartbeatMs : undefined,
|
|
181
711
|
leaderEpoch:
|
|
182
|
-
typeof record.leaderEpoch === "number"
|
|
712
|
+
typeof record.leaderEpoch === "number" ||
|
|
713
|
+
typeof record.leaderEpoch === "string"
|
|
714
|
+
? record.leaderEpoch
|
|
715
|
+
: undefined,
|
|
716
|
+
runtimeGeneration:
|
|
717
|
+
typeof record.runtimeGeneration === "number"
|
|
718
|
+
? record.runtimeGeneration
|
|
719
|
+
: undefined,
|
|
183
720
|
busSocketPath:
|
|
184
721
|
typeof record.busSocketPath === "string"
|
|
185
722
|
? record.busSocketPath
|
|
@@ -243,6 +780,40 @@ function ownsLockContext(
|
|
|
243
780
|
return !lock.cwd || !ctx || lock.cwd === ctx.cwd;
|
|
244
781
|
}
|
|
245
782
|
|
|
783
|
+
function hasSameLockOwner(
|
|
784
|
+
current: TelegramLockEntry | undefined,
|
|
785
|
+
expected: TelegramLockEntry | undefined,
|
|
786
|
+
): boolean {
|
|
787
|
+
if (!current || !expected) return false;
|
|
788
|
+
return (
|
|
789
|
+
current.pid === expected.pid &&
|
|
790
|
+
current.cwd === expected.cwd &&
|
|
791
|
+
current.instanceId === expected.instanceId &&
|
|
792
|
+
current.leaderEpoch === expected.leaderEpoch &&
|
|
793
|
+
current.runtimeGeneration === expected.runtimeGeneration
|
|
794
|
+
);
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
function canSupersedeSameProcessOwner(
|
|
798
|
+
current: TelegramLockEntry,
|
|
799
|
+
pid: number,
|
|
800
|
+
ctx: TelegramLockContext,
|
|
801
|
+
instanceId: string | undefined,
|
|
802
|
+
runtimeGeneration: number,
|
|
803
|
+
): boolean {
|
|
804
|
+
if (
|
|
805
|
+
current.pid !== pid ||
|
|
806
|
+
(current.cwd !== undefined && current.cwd !== ctx.cwd) ||
|
|
807
|
+
!instanceId
|
|
808
|
+
) {
|
|
809
|
+
return false;
|
|
810
|
+
}
|
|
811
|
+
return (
|
|
812
|
+
current.runtimeGeneration === undefined ||
|
|
813
|
+
runtimeGeneration > current.runtimeGeneration
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
|
|
246
817
|
function createLockEntry(
|
|
247
818
|
pid: number,
|
|
248
819
|
ctx: TelegramLockContext,
|
|
@@ -251,6 +822,8 @@ function createLockEntry(
|
|
|
251
822
|
busSocketPath?: string;
|
|
252
823
|
busSecret?: string;
|
|
253
824
|
getNowMs?: () => number;
|
|
825
|
+
mintLeaderEpoch?: () => number | string;
|
|
826
|
+
runtimeGeneration?: number;
|
|
254
827
|
},
|
|
255
828
|
): TelegramLockEntry {
|
|
256
829
|
const lock: TelegramLockEntry = { pid, cwd: ctx.cwd };
|
|
@@ -258,7 +831,8 @@ function createLockEntry(
|
|
|
258
831
|
const nowMs = options.getNowMs?.();
|
|
259
832
|
lock.instanceId = options.instanceId;
|
|
260
833
|
lock.heartbeatMs = nowMs;
|
|
261
|
-
lock.leaderEpoch =
|
|
834
|
+
lock.leaderEpoch = options.mintLeaderEpoch?.() ?? randomUUID();
|
|
835
|
+
lock.runtimeGeneration = options.runtimeGeneration;
|
|
262
836
|
}
|
|
263
837
|
if (options.busSocketPath) lock.busSocketPath = options.busSocketPath;
|
|
264
838
|
if (options.busSecret) lock.busSecret = options.busSecret;
|
|
@@ -286,6 +860,10 @@ export function createTelegramLockRuntime<TContext extends TelegramLockContext>(
|
|
|
286
860
|
const pid = options.pid ?? process.pid;
|
|
287
861
|
const isAlive = options.isProcessAlive ?? isProcessAlive;
|
|
288
862
|
const getNowMs = options.getNowMs ?? Date.now;
|
|
863
|
+
const runtimeGeneration =
|
|
864
|
+
options.runtimeGeneration ?? allocateTelegramLockRuntimeGeneration();
|
|
865
|
+
let ownedLockKey: string | undefined;
|
|
866
|
+
let ownedLock: TelegramLockEntry | undefined;
|
|
289
867
|
const stateOptions = () => ({
|
|
290
868
|
nowMs: getNowMs(),
|
|
291
869
|
staleHeartbeatMs: options.staleHeartbeatMs,
|
|
@@ -298,56 +876,196 @@ export function createTelegramLockRuntime<TContext extends TelegramLockContext>(
|
|
|
298
876
|
const effectiveKey = resolveEffectiveKey();
|
|
299
877
|
return parseTelegramLockEntry(readLocks(locksPath)[effectiveKey]);
|
|
300
878
|
};
|
|
301
|
-
const
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
879
|
+
const adoptCompatibleOwnedLock = (
|
|
880
|
+
effectiveKey: string,
|
|
881
|
+
lock: TelegramLockEntry | undefined,
|
|
882
|
+
ctx?: TelegramLockContext,
|
|
883
|
+
): TelegramLockEntry | undefined => {
|
|
884
|
+
if (ownedLock) {
|
|
885
|
+
return ownedLockKey === effectiveKey ? ownedLock : undefined;
|
|
886
|
+
}
|
|
887
|
+
if (!ownsLockContext(lock, pid, ctx)) return undefined;
|
|
888
|
+
if (
|
|
889
|
+
(lock?.instanceId !== undefined &&
|
|
890
|
+
lock.instanceId !== options.instanceId) ||
|
|
891
|
+
(lock?.runtimeGeneration !== undefined &&
|
|
892
|
+
lock.runtimeGeneration !== runtimeGeneration)
|
|
893
|
+
) {
|
|
894
|
+
return undefined;
|
|
895
|
+
}
|
|
896
|
+
ownedLockKey = effectiveKey;
|
|
897
|
+
ownedLock = lock;
|
|
898
|
+
return ownedLock;
|
|
306
899
|
};
|
|
307
900
|
return {
|
|
308
|
-
acquire: (ctx, acquireOptions = {}) =>
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
901
|
+
acquire: (ctx, acquireOptions = {}) =>
|
|
902
|
+
withLockTransaction<TelegramLockAcquireResult>(locksPath, (locks) => {
|
|
903
|
+
const effectiveKey = resolveEffectiveKey();
|
|
904
|
+
const current = parseTelegramLockEntry(locks[effectiveKey]);
|
|
905
|
+
const state = getLockState(current, pid, isAlive, stateOptions());
|
|
906
|
+
const expectedOwned = adoptCompatibleOwnedLock(
|
|
907
|
+
effectiveKey,
|
|
908
|
+
current,
|
|
909
|
+
ctx,
|
|
910
|
+
);
|
|
911
|
+
if (
|
|
912
|
+
state.kind === "active-here" &&
|
|
913
|
+
hasSameLockOwner(current, expectedOwned)
|
|
914
|
+
) {
|
|
915
|
+
return {
|
|
916
|
+
result: {
|
|
917
|
+
ok: true,
|
|
918
|
+
lock: current!,
|
|
919
|
+
replacedStale: false,
|
|
920
|
+
} as const,
|
|
921
|
+
changed: false,
|
|
922
|
+
};
|
|
923
|
+
}
|
|
924
|
+
if (acquireOptions.election && current) {
|
|
925
|
+
if (
|
|
926
|
+
state.kind !== "stale" ||
|
|
927
|
+
!hasSameLockOwner(current, acquireOptions.expectedOwner)
|
|
928
|
+
) {
|
|
929
|
+
return {
|
|
930
|
+
result: { ok: false, lock: current } as const,
|
|
931
|
+
changed: false,
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
const expectedReplacementMatches = hasSameLockOwner(
|
|
936
|
+
state.kind === "active-here" || state.kind === "active-elsewhere"
|
|
937
|
+
? state.lock
|
|
938
|
+
: undefined,
|
|
939
|
+
acquireOptions.expectedOwner,
|
|
940
|
+
);
|
|
941
|
+
const canReplaceCurrent =
|
|
942
|
+
state.kind === "active-elsewhere" ||
|
|
943
|
+
(state.kind === "active-here" &&
|
|
944
|
+
canSupersedeSameProcessOwner(
|
|
945
|
+
state.lock,
|
|
946
|
+
pid,
|
|
947
|
+
ctx,
|
|
948
|
+
options.instanceId,
|
|
949
|
+
runtimeGeneration,
|
|
950
|
+
));
|
|
951
|
+
if (
|
|
952
|
+
!acquireOptions.election &&
|
|
953
|
+
(state.kind === "active-here" || state.kind === "active-elsewhere") &&
|
|
954
|
+
(!acquireOptions.force ||
|
|
955
|
+
!expectedReplacementMatches ||
|
|
956
|
+
!canReplaceCurrent)
|
|
957
|
+
) {
|
|
958
|
+
return {
|
|
959
|
+
result: { ok: false, lock: state.lock } as const,
|
|
960
|
+
changed: false,
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
const lock = createLockEntry(pid, ctx, {
|
|
964
|
+
instanceId: options.instanceId,
|
|
965
|
+
busSocketPath: options.busSocketPath,
|
|
966
|
+
busSecret: options.busSecret,
|
|
967
|
+
getNowMs,
|
|
968
|
+
mintLeaderEpoch: options.mintLeaderEpoch,
|
|
969
|
+
runtimeGeneration,
|
|
970
|
+
});
|
|
971
|
+
locks[effectiveKey] = lock;
|
|
972
|
+
ownedLockKey = effectiveKey;
|
|
973
|
+
ownedLock = lock;
|
|
974
|
+
return {
|
|
975
|
+
result: {
|
|
976
|
+
ok: true,
|
|
977
|
+
lock,
|
|
978
|
+
replacedStale: state.kind === "stale",
|
|
979
|
+
} as const,
|
|
980
|
+
changed: true,
|
|
981
|
+
};
|
|
982
|
+
}),
|
|
983
|
+
release: () =>
|
|
984
|
+
withLockTransaction(locksPath, (locks) => {
|
|
985
|
+
const effectiveKey = resolveEffectiveKey();
|
|
986
|
+
const state = getLockState(
|
|
987
|
+
parseTelegramLockEntry(locks[effectiveKey]),
|
|
988
|
+
pid,
|
|
989
|
+
isAlive,
|
|
990
|
+
stateOptions(),
|
|
991
|
+
);
|
|
992
|
+
const changed =
|
|
993
|
+
ownedLockKey === effectiveKey &&
|
|
994
|
+
hasSameLockOwner(
|
|
995
|
+
parseTelegramLockEntry(locks[effectiveKey]),
|
|
996
|
+
ownedLock,
|
|
997
|
+
);
|
|
998
|
+
if (changed) {
|
|
999
|
+
delete locks[effectiveKey];
|
|
1000
|
+
ownedLockKey = undefined;
|
|
1001
|
+
ownedLock = undefined;
|
|
1002
|
+
}
|
|
1003
|
+
return { result: state, changed };
|
|
1004
|
+
}),
|
|
330
1005
|
getState: () => getLockState(readLock(), pid, isAlive, stateOptions()),
|
|
331
1006
|
getStatusLabel: () =>
|
|
332
1007
|
formatLockState(getLockState(readLock(), pid, isAlive, stateOptions())),
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
const lock =
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
writeLock({
|
|
339
|
-
pid: lock.pid,
|
|
340
|
-
...(lock.cwd ? { cwd: lock.cwd } : {}),
|
|
341
|
-
instanceId: options.instanceId,
|
|
342
|
-
heartbeatMs: getNowMs(),
|
|
343
|
-
leaderEpoch: lock.leaderEpoch,
|
|
344
|
-
...(options.busSocketPath
|
|
345
|
-
? { busSocketPath: options.busSocketPath }
|
|
346
|
-
: {}),
|
|
347
|
-
busSecret: options.busSecret ?? lock.busSecret,
|
|
348
|
-
});
|
|
349
|
-
return true;
|
|
1008
|
+
getOwnedLeaderEpoch: () => {
|
|
1009
|
+
const effectiveKey = resolveEffectiveKey();
|
|
1010
|
+
const lock = parseTelegramLockEntry(readLocks(locksPath)[effectiveKey]);
|
|
1011
|
+
const exactOwner = adoptCompatibleOwnedLock(effectiveKey, lock);
|
|
1012
|
+
return hasSameLockOwner(lock, exactOwner) ? lock?.leaderEpoch : undefined;
|
|
350
1013
|
},
|
|
1014
|
+
owns: (ctx) => {
|
|
1015
|
+
const effectiveKey = resolveEffectiveKey();
|
|
1016
|
+
const lock = parseTelegramLockEntry(readLocks(locksPath)[effectiveKey]);
|
|
1017
|
+
return hasSameLockOwner(
|
|
1018
|
+
lock,
|
|
1019
|
+
adoptCompatibleOwnedLock(effectiveKey, lock, ctx),
|
|
1020
|
+
);
|
|
1021
|
+
},
|
|
1022
|
+
commitIfOwned: (commit) =>
|
|
1023
|
+
withLockTransaction(locksPath, (locks) => {
|
|
1024
|
+
const effectiveKey = resolveEffectiveKey();
|
|
1025
|
+
const lock = parseTelegramLockEntry(locks[effectiveKey]);
|
|
1026
|
+
const exactOwner =
|
|
1027
|
+
ownedLockKey === effectiveKey && hasSameLockOwner(lock, ownedLock);
|
|
1028
|
+
if (!exactOwner) {
|
|
1029
|
+
if (ownedLockKey === effectiveKey) {
|
|
1030
|
+
ownedLockKey = undefined;
|
|
1031
|
+
ownedLock = undefined;
|
|
1032
|
+
}
|
|
1033
|
+
return { result: false, changed: false };
|
|
1034
|
+
}
|
|
1035
|
+
commit();
|
|
1036
|
+
return { result: true, changed: false };
|
|
1037
|
+
}),
|
|
1038
|
+
refresh: (ctx) =>
|
|
1039
|
+
withLockTransaction(locksPath, (locks) => {
|
|
1040
|
+
const effectiveKey = resolveEffectiveKey();
|
|
1041
|
+
const lock = parseTelegramLockEntry(locks[effectiveKey]);
|
|
1042
|
+
const expectedOwner = adoptCompatibleOwnedLock(effectiveKey, lock, ctx);
|
|
1043
|
+
if (!lock || !hasSameLockOwner(lock, expectedOwner)) {
|
|
1044
|
+
if (ownedLockKey === effectiveKey) {
|
|
1045
|
+
ownedLockKey = undefined;
|
|
1046
|
+
ownedLock = undefined;
|
|
1047
|
+
}
|
|
1048
|
+
return { result: false, changed: false };
|
|
1049
|
+
}
|
|
1050
|
+
if (!options.instanceId) return { result: true, changed: false };
|
|
1051
|
+
const refreshedLock: TelegramLockEntry = {
|
|
1052
|
+
pid: lock.pid,
|
|
1053
|
+
...(lock.cwd ? { cwd: lock.cwd } : {}),
|
|
1054
|
+
instanceId: options.instanceId,
|
|
1055
|
+
heartbeatMs: getNowMs(),
|
|
1056
|
+
leaderEpoch:
|
|
1057
|
+
lock.leaderEpoch ?? options.mintLeaderEpoch?.() ?? randomUUID(),
|
|
1058
|
+
runtimeGeneration: lock.runtimeGeneration ?? runtimeGeneration,
|
|
1059
|
+
...(options.busSocketPath
|
|
1060
|
+
? { busSocketPath: options.busSocketPath }
|
|
1061
|
+
: {}),
|
|
1062
|
+
busSecret: options.busSecret ?? lock.busSecret,
|
|
1063
|
+
};
|
|
1064
|
+
locks[effectiveKey] = refreshedLock;
|
|
1065
|
+
ownedLockKey = effectiveKey;
|
|
1066
|
+
ownedLock = refreshedLock;
|
|
1067
|
+
return { result: true, changed: true };
|
|
1068
|
+
}),
|
|
351
1069
|
};
|
|
352
1070
|
}
|
|
353
1071
|
|
|
@@ -374,6 +1092,8 @@ export function createTelegramDirectDeliveryOwnershipChecker<
|
|
|
374
1092
|
export interface TelegramLockedPollingStartOptions {
|
|
375
1093
|
force?: boolean;
|
|
376
1094
|
forceFreshLeaderThread?: boolean;
|
|
1095
|
+
election?: { expectedOwner?: TelegramLockEntry };
|
|
1096
|
+
onAcquired?: () => Promise<void> | void;
|
|
377
1097
|
}
|
|
378
1098
|
|
|
379
1099
|
export type TelegramLockedPollingStartResult =
|
|
@@ -434,6 +1154,7 @@ export function createTelegramLockedPollingRuntime<
|
|
|
434
1154
|
): TelegramLockedPollingRuntime<TContext> {
|
|
435
1155
|
let ownershipInterval: ReturnType<typeof setInterval> | undefined;
|
|
436
1156
|
let ownershipStop: Promise<void> | undefined;
|
|
1157
|
+
let takeoverCandidate: TelegramLockEntry | undefined;
|
|
437
1158
|
let sessionAutoStartRun: Promise<void> | undefined;
|
|
438
1159
|
let sessionAutoStartGeneration = 0;
|
|
439
1160
|
const ownershipCheckMs = deps.ownershipCheckMs ?? 1000;
|
|
@@ -480,6 +1201,36 @@ export function createTelegramLockedPollingRuntime<
|
|
|
480
1201
|
}, ownershipCheckMs);
|
|
481
1202
|
ownershipInterval.unref?.();
|
|
482
1203
|
};
|
|
1204
|
+
const runOwnedPollingStart = async (
|
|
1205
|
+
ctx: TContext,
|
|
1206
|
+
options: TelegramLockedPollingStartOptions,
|
|
1207
|
+
): Promise<boolean> => {
|
|
1208
|
+
startOwnershipWatcher(ctx);
|
|
1209
|
+
try {
|
|
1210
|
+
if (!deps.lock.refresh(snapshotLockContext(ctx))) {
|
|
1211
|
+
stopOwnershipWatcher();
|
|
1212
|
+
return false;
|
|
1213
|
+
}
|
|
1214
|
+
await options.onAcquired?.();
|
|
1215
|
+
await deps.startPolling(ctx, options);
|
|
1216
|
+
} catch (error) {
|
|
1217
|
+
stopOwnershipWatcher();
|
|
1218
|
+
try {
|
|
1219
|
+
await deps.stopPolling();
|
|
1220
|
+
} catch (stopError) {
|
|
1221
|
+
deps.recordRuntimeEvent?.("lock", stopError, {
|
|
1222
|
+
phase: "startup-rollback",
|
|
1223
|
+
});
|
|
1224
|
+
}
|
|
1225
|
+
deps.lock.release();
|
|
1226
|
+
throw error;
|
|
1227
|
+
}
|
|
1228
|
+
if (deps.lock.owns(ctx)) return true;
|
|
1229
|
+
stopOwnershipWatcher();
|
|
1230
|
+
if (ownershipStop) await ownershipStop;
|
|
1231
|
+
await deps.stopPolling();
|
|
1232
|
+
return false;
|
|
1233
|
+
};
|
|
483
1234
|
const canStartPolling = (ctx: TContext): boolean =>
|
|
484
1235
|
deps.canStartPolling?.(ctx) ?? true;
|
|
485
1236
|
const formatStartBlockedMessage = (ctx: TContext): string =>
|
|
@@ -493,8 +1244,35 @@ export function createTelegramLockedPollingRuntime<
|
|
|
493
1244
|
if (!canStartPolling(ctx)) {
|
|
494
1245
|
return { ok: false, message: formatStartBlockedMessage(ctx) };
|
|
495
1246
|
}
|
|
496
|
-
|
|
1247
|
+
let acquired = deps.lock.acquire(ctx, {
|
|
1248
|
+
force: options.force,
|
|
1249
|
+
expectedOwner:
|
|
1250
|
+
options.election?.expectedOwner ??
|
|
1251
|
+
(options.force ? takeoverCandidate : undefined),
|
|
1252
|
+
election: options.election !== undefined,
|
|
1253
|
+
});
|
|
1254
|
+
if (!acquired.ok && !options.election) {
|
|
1255
|
+
const currentState = deps.lock.getState();
|
|
1256
|
+
if (
|
|
1257
|
+
currentState.kind === "active-here" &&
|
|
1258
|
+
hasSameLockOwner(currentState.lock, acquired.lock)
|
|
1259
|
+
) {
|
|
1260
|
+
acquired = deps.lock.acquire(ctx, {
|
|
1261
|
+
force: true,
|
|
1262
|
+
expectedOwner: acquired.lock,
|
|
1263
|
+
});
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
497
1266
|
if (!acquired.ok) {
|
|
1267
|
+
takeoverCandidate = acquired.lock;
|
|
1268
|
+
if (options.election) {
|
|
1269
|
+
return {
|
|
1270
|
+
ok: false,
|
|
1271
|
+
canTakeover: false,
|
|
1272
|
+
owner: formatTelegramLockEntry(acquired.lock),
|
|
1273
|
+
message: "Telegram leadership election lost to another live owner.",
|
|
1274
|
+
};
|
|
1275
|
+
}
|
|
498
1276
|
if (deps.registerFollowerWithOwner) {
|
|
499
1277
|
let failureMessage: string | undefined;
|
|
500
1278
|
try {
|
|
@@ -532,8 +1310,14 @@ export function createTelegramLockedPollingRuntime<
|
|
|
532
1310
|
message: `Telegram bridge is active in another Pi instance (${owner}).`,
|
|
533
1311
|
};
|
|
534
1312
|
}
|
|
535
|
-
|
|
536
|
-
|
|
1313
|
+
takeoverCandidate = undefined;
|
|
1314
|
+
if (!(await runOwnedPollingStart(ctx, options))) {
|
|
1315
|
+
return {
|
|
1316
|
+
ok: false,
|
|
1317
|
+
canTakeover: false,
|
|
1318
|
+
message: "Telegram leadership changed during polling startup.",
|
|
1319
|
+
};
|
|
1320
|
+
}
|
|
537
1321
|
deps.updateStatus(ctx);
|
|
538
1322
|
const staleSuffix = acquired.replacedStale ? " Replaced stale lock." : "";
|
|
539
1323
|
return { ok: true, message: `Telegram bridge connected.${staleSuffix}` };
|
|
@@ -557,7 +1341,16 @@ export function createTelegramLockedPollingRuntime<
|
|
|
557
1341
|
const state = ownsCurrentLock ? undefined : deps.lock.getState();
|
|
558
1342
|
const canResumeStaleSameCwd =
|
|
559
1343
|
state?.kind === "stale" && state.lock.cwd === ctx.cwd;
|
|
560
|
-
|
|
1344
|
+
const canHandoffSameProcess =
|
|
1345
|
+
state?.kind === "active-here" &&
|
|
1346
|
+
(!state.lock.cwd || state.lock.cwd === ctx.cwd);
|
|
1347
|
+
if (
|
|
1348
|
+
!ownsCurrentLock &&
|
|
1349
|
+
!canResumeStaleSameCwd &&
|
|
1350
|
+
!canHandoffSameProcess
|
|
1351
|
+
) {
|
|
1352
|
+
return;
|
|
1353
|
+
}
|
|
561
1354
|
sessionAutoStartGeneration += 1;
|
|
562
1355
|
const generation = sessionAutoStartGeneration;
|
|
563
1356
|
const startedAtMs = Date.now();
|
|
@@ -567,14 +1360,18 @@ export function createTelegramLockedPollingRuntime<
|
|
|
567
1360
|
const run = (async () => {
|
|
568
1361
|
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
569
1362
|
if (generation !== sessionAutoStartGeneration) return;
|
|
570
|
-
if (canResumeStaleSameCwd) {
|
|
571
|
-
const acquired = deps.lock.acquire(
|
|
1363
|
+
if (canResumeStaleSameCwd || canHandoffSameProcess) {
|
|
1364
|
+
const acquired = deps.lock.acquire(
|
|
1365
|
+
ctx,
|
|
1366
|
+
canHandoffSameProcess
|
|
1367
|
+
? { force: true, expectedOwner: state?.lock }
|
|
1368
|
+
: undefined,
|
|
1369
|
+
);
|
|
572
1370
|
if (!acquired.ok) return;
|
|
573
1371
|
}
|
|
574
1372
|
if (generation !== sessionAutoStartGeneration) return;
|
|
575
|
-
await
|
|
1373
|
+
if (!(await runOwnedPollingStart(ctx, {}))) return;
|
|
576
1374
|
if (generation !== sessionAutoStartGeneration) return;
|
|
577
|
-
startOwnershipWatcher(ctx);
|
|
578
1375
|
deps.updateStatus(ctx);
|
|
579
1376
|
deps.recordRuntimeEvent?.("lock", "Telegram auto-start completed", {
|
|
580
1377
|
phase: "auto-start-complete",
|