@cr1ms0n/pi-subagent 0.8.9 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -1
- package/README.md +218 -115
- package/docs/ARCHITECTURE.md +56 -13
- package/docs/COST-ACCOUNTING.md +116 -66
- package/docs/RELEASING.md +32 -32
- package/docs/SECURITY.md +42 -5
- package/docs/UX.md +158 -141
- package/package.json +2 -2
- package/skills/subagent/SKILL.md +78 -49
- package/src/backends/pi.ts +164 -94
- package/src/child-preflight.ts +166 -0
- package/src/config.ts +254 -252
- package/src/dispatch-preflight.ts +87 -0
- package/src/dispatch-routing.ts +56 -0
- package/src/extension.ts +366 -158
- package/src/format.ts +436 -365
- package/src/jev-router.ts +1036 -0
- package/src/orchestrator.ts +75 -19
- package/src/persistence.ts +643 -335
- package/src/policy.ts +120 -89
- package/src/process-lock.ts +730 -687
- package/src/protocol.ts +320 -290
- package/src/registry.ts +730 -632
- package/src/routing-policy.ts +268 -0
- package/src/routing-types.ts +217 -0
- package/src/runner.ts +1299 -850
- package/src/schema.ts +10 -10
- package/src/startup-check.ts +481 -0
- package/src/types.ts +208 -198
- package/src/usage.ts +316 -274
- package/src/model-policy.ts +0 -169
package/src/process-lock.ts
CHANGED
|
@@ -1,687 +1,730 @@
|
|
|
1
|
-
import { execFileSync, spawn } from "node:child_process";
|
|
2
|
-
import * as fs from "node:fs";
|
|
3
|
-
import * as fsp from "node:fs/promises";
|
|
4
|
-
import * as os from "node:os";
|
|
5
|
-
import * as path from "node:path";
|
|
6
|
-
import { defaultConfig } from "./config.js";
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Durable, crash-surviving coordination primitives under `~/.pi/subagent-locks/`.
|
|
10
|
-
*
|
|
11
|
-
* Provides:
|
|
12
|
-
* - Per-child-session exclusive resume locks (file lock via O_EXCL lock directory)
|
|
13
|
-
* - Machine-wide concurrency tokens (slot files)
|
|
14
|
-
* - Run process identity records (PID + startTime + pgid) for orphan reconcile
|
|
15
|
-
*
|
|
16
|
-
* Locks use mkdir atomicity (POSIX + Node) and embed owner identity so stale
|
|
17
|
-
* locks from dead processes can be reclaimed conservatively (PID birth-time
|
|
18
|
-
* checked where available).
|
|
19
|
-
*/
|
|
20
|
-
|
|
21
|
-
export interface ProcessIdentity {
|
|
22
|
-
pid: number;
|
|
23
|
-
/** Process start time in ms since epoch when known; 0 when unknown. */
|
|
24
|
-
startTime: number;
|
|
25
|
-
pgid?: number;
|
|
26
|
-
hostname: string;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export interface SessionLockOwner {
|
|
30
|
-
ownerId: string;
|
|
31
|
-
runId: string;
|
|
32
|
-
parentSessionKey: string;
|
|
33
|
-
process: ProcessIdentity;
|
|
34
|
-
acquiredAt: number;
|
|
35
|
-
leaseExpiresAt: number;
|
|
36
|
-
childSessionId: string;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export interface
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
function
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
if (
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
fs.
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
private
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
);
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
if (
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
const
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
1
|
+
import { execFileSync, spawn } from "node:child_process";
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import * as fsp from "node:fs/promises";
|
|
4
|
+
import * as os from "node:os";
|
|
5
|
+
import * as path from "node:path";
|
|
6
|
+
import { defaultConfig } from "./config.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Durable, crash-surviving coordination primitives under `~/.pi/subagent-locks/`.
|
|
10
|
+
*
|
|
11
|
+
* Provides:
|
|
12
|
+
* - Per-child-session exclusive resume locks (file lock via O_EXCL lock directory)
|
|
13
|
+
* - Machine-wide concurrency tokens (slot files)
|
|
14
|
+
* - Run process identity records (PID + startTime + pgid) for orphan reconcile
|
|
15
|
+
*
|
|
16
|
+
* Locks use mkdir atomicity (POSIX + Node) and embed owner identity so stale
|
|
17
|
+
* locks from dead processes can be reclaimed conservatively (PID birth-time
|
|
18
|
+
* checked where available).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export interface ProcessIdentity {
|
|
22
|
+
pid: number;
|
|
23
|
+
/** Process start time in ms since epoch when known; 0 when unknown. */
|
|
24
|
+
startTime: number;
|
|
25
|
+
pgid?: number;
|
|
26
|
+
hostname: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface SessionLockOwner {
|
|
30
|
+
ownerId: string;
|
|
31
|
+
runId: string;
|
|
32
|
+
parentSessionKey: string;
|
|
33
|
+
process: ProcessIdentity;
|
|
34
|
+
acquiredAt: number;
|
|
35
|
+
leaseExpiresAt: number;
|
|
36
|
+
childSessionId: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface SessionLockAvailability {
|
|
40
|
+
childSessionId: string;
|
|
41
|
+
/** `reclaimable` means a demonstrably stale lock that a later acquire may reclaim. */
|
|
42
|
+
status: "free" | "held" | "reclaimable";
|
|
43
|
+
owner?: SessionLockOwner;
|
|
44
|
+
reason?: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface RunProcessRecord {
|
|
48
|
+
runId: string;
|
|
49
|
+
parentSessionKey: string;
|
|
50
|
+
childSessionId?: string;
|
|
51
|
+
/** Live worktree checkout of this run; shields it from machine-wide GC sweeps. */
|
|
52
|
+
worktreeCwd?: string;
|
|
53
|
+
process: ProcessIdentity;
|
|
54
|
+
startedAt: number;
|
|
55
|
+
state: "running" | "terminal";
|
|
56
|
+
terminalState?: string;
|
|
57
|
+
updatedAt: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface SlotToken {
|
|
61
|
+
slotId: string;
|
|
62
|
+
path: string;
|
|
63
|
+
runId: string;
|
|
64
|
+
released: boolean;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface ProcessLockOptions {
|
|
68
|
+
rootDir?: string;
|
|
69
|
+
/** Soft lease for session locks; current holders renew while active. */
|
|
70
|
+
leaseMs?: number;
|
|
71
|
+
/** Max concurrent child Pi processes machine-wide. 0 = no global limit. */
|
|
72
|
+
maxGlobalActive?: number;
|
|
73
|
+
/**
|
|
74
|
+
* Nesting ceiling used for depth-tiered global slots. Defaults to
|
|
75
|
+
* `defaultConfig.maxDepth` so shallow tiers reserve room for deeper ones.
|
|
76
|
+
*/
|
|
77
|
+
maxDepth?: number;
|
|
78
|
+
/** Now override for tests. */
|
|
79
|
+
now?: () => number;
|
|
80
|
+
/** isAlive override for tests. */
|
|
81
|
+
isAlive?: (identity: ProcessIdentity) => boolean;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const DEFAULT_LEASE_MS = 60_000;
|
|
85
|
+
const HOSTNAME = (() => {
|
|
86
|
+
try {
|
|
87
|
+
return os.hostname();
|
|
88
|
+
} catch {
|
|
89
|
+
return "unknown";
|
|
90
|
+
}
|
|
91
|
+
})();
|
|
92
|
+
|
|
93
|
+
function lockRoot(root?: string): string {
|
|
94
|
+
return root ?? path.join(defaultConfig.sessionDir, "..", "subagent-locks");
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function ensureDirSync(dir: string): void {
|
|
98
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function sessionLockPath(root: string, childSessionId: string): string {
|
|
102
|
+
// Nested dirs: keep file names short and FS-safe.
|
|
103
|
+
const safe = childSessionId.replace(/[^a-zA-Z0-9._-]+/g, "_").slice(0, 120);
|
|
104
|
+
return path.join(root, "sessions", `${safe}.lock`);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function slotDir(root: string): string {
|
|
108
|
+
return path.join(root, "slots");
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function runRecordPath(root: string, runId: string): string {
|
|
112
|
+
// Parallel/synthesis IDs contain ':', which is not a valid Windows filename.
|
|
113
|
+
// Encode '%' as well to keep IDs distinct; preserve existing POSIX records.
|
|
114
|
+
const filename = process.platform === "win32" ? encodeURIComponent(runId) : runId;
|
|
115
|
+
return path.join(root, "runs", `${filename}.json`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function readJsonSync<T>(file: string): T | undefined {
|
|
119
|
+
try {
|
|
120
|
+
return JSON.parse(fs.readFileSync(file, "utf8")) as T;
|
|
121
|
+
} catch {
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function writeJsonAtomicSync(file: string, value: unknown): void {
|
|
127
|
+
ensureDirSync(path.dirname(file));
|
|
128
|
+
const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
|
|
129
|
+
fs.writeFileSync(tmp, JSON.stringify(value, null, 2), "utf8");
|
|
130
|
+
fs.renameSync(tmp, file);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Best-effort process start-time identity. On Linux uses `/proc/<pid>/stat`
|
|
135
|
+
* field 22 (starttime in clock ticks). On macOS/BSD uses `ps -o lstart=`
|
|
136
|
+
* (epoch seconds). Elsewhere falls back to 0 (we still check `kill(pid, 0)`
|
|
137
|
+
* for liveness, but PID-reuse protection is weaker).
|
|
138
|
+
*/
|
|
139
|
+
export function processStartTime(pid: number): number {
|
|
140
|
+
if (process.platform === "linux") {
|
|
141
|
+
try {
|
|
142
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
143
|
+
// Proc name may contain spaces/parens; starttime is the 22nd field after the closing ')'.
|
|
144
|
+
const close = stat.lastIndexOf(")");
|
|
145
|
+
if (close >= 0) {
|
|
146
|
+
const fields = stat.slice(close + 2).trim().split(/\s+/);
|
|
147
|
+
const startTicks = Number(fields[19]); // field 22 absolute = index 19 after cmd
|
|
148
|
+
if (Number.isFinite(startTicks)) return startTicks;
|
|
149
|
+
}
|
|
150
|
+
} catch {
|
|
151
|
+
/* fall through */
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (process.platform === "darwin" || process.platform === "freebsd") {
|
|
155
|
+
try {
|
|
156
|
+
const out = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], {
|
|
157
|
+
encoding: "utf8",
|
|
158
|
+
timeout: 2_000,
|
|
159
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
160
|
+
}).trim();
|
|
161
|
+
if (out) {
|
|
162
|
+
const epoch = Math.floor(new Date(out).getTime() / 1000);
|
|
163
|
+
if (Number.isFinite(epoch) && epoch > 0) return epoch;
|
|
164
|
+
}
|
|
165
|
+
} catch {
|
|
166
|
+
/* fall through */
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return 0;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function currentProcessIdentity(): ProcessIdentity {
|
|
173
|
+
let pgid: number | undefined;
|
|
174
|
+
try {
|
|
175
|
+
// Node exposes getpgrp on POSIX but the type package has not always
|
|
176
|
+
// declared it. Access reflectively for portability.
|
|
177
|
+
const getpgrp = (process as NodeJS.Process & { getpgrp?: () => number }).getpgrp;
|
|
178
|
+
if (process.platform !== "win32" && typeof getpgrp === "function") {
|
|
179
|
+
pgid = getpgrp.call(process);
|
|
180
|
+
}
|
|
181
|
+
} catch {
|
|
182
|
+
pgid = undefined;
|
|
183
|
+
}
|
|
184
|
+
return {
|
|
185
|
+
pid: process.pid,
|
|
186
|
+
startTime: processStartTime(process.pid),
|
|
187
|
+
pgid,
|
|
188
|
+
hostname: HOSTNAME,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
export function isProcessAlive(identity: ProcessIdentity): boolean {
|
|
193
|
+
if (!identity.pid || identity.pid <= 0) return false;
|
|
194
|
+
// Only check processes on this host; cross-host records are treated as dead so
|
|
195
|
+
// local reconcile does not wait forever.
|
|
196
|
+
if (identity.hostname && identity.hostname !== HOSTNAME) return false;
|
|
197
|
+
try {
|
|
198
|
+
process.kill(identity.pid, 0);
|
|
199
|
+
} catch (error: any) {
|
|
200
|
+
if (error?.code === "ESRCH") return false;
|
|
201
|
+
// EPERM means the process exists but we cannot signal it.
|
|
202
|
+
if (error?.code === "EPERM") {
|
|
203
|
+
// Still verify startTime if known.
|
|
204
|
+
} else {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
if (identity.startTime > 0) {
|
|
209
|
+
const live = processStartTime(identity.pid);
|
|
210
|
+
// If we can read startTime and it differs, this is a recycled PID.
|
|
211
|
+
if (live > 0 && live !== identity.startTime) return false;
|
|
212
|
+
}
|
|
213
|
+
return true;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
export function killProcessTree(identity: ProcessIdentity, signal: NodeJS.Signals = "SIGTERM"): void {
|
|
217
|
+
if (!identity.pid) return;
|
|
218
|
+
try {
|
|
219
|
+
if (process.platform === "win32") {
|
|
220
|
+
// Use taskkill for the process tree.
|
|
221
|
+
// Spawned fire-and-forget: callers that need confirmation re-check liveness.
|
|
222
|
+
const force = signal === "SIGKILL";
|
|
223
|
+
const args = ["/pid", String(identity.pid), "/T", ...(force ? ["/F"] : [])];
|
|
224
|
+
const killer = spawn("taskkill", args, { shell: false, stdio: "ignore" });
|
|
225
|
+
killer.unref();
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const target = identity.pgid && identity.pgid > 0 ? -identity.pgid : -identity.pid;
|
|
229
|
+
try {
|
|
230
|
+
process.kill(target, signal);
|
|
231
|
+
} catch (error: any) {
|
|
232
|
+
if (error?.code !== "ESRCH") {
|
|
233
|
+
try {
|
|
234
|
+
process.kill(identity.pid, signal);
|
|
235
|
+
} catch {
|
|
236
|
+
/* best effort */
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
} catch {
|
|
241
|
+
/* best effort */
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
export class ProcessLockManager {
|
|
246
|
+
private readonly root: string;
|
|
247
|
+
private readonly leaseMs: number;
|
|
248
|
+
private readonly maxGlobalActive: number;
|
|
249
|
+
private readonly maxDepth: number;
|
|
250
|
+
private readonly now: () => number;
|
|
251
|
+
private readonly isAlive: (identity: ProcessIdentity) => boolean;
|
|
252
|
+
private renewTimers = new Map<string, NodeJS.Timeout>();
|
|
253
|
+
|
|
254
|
+
constructor(options: ProcessLockOptions = {}) {
|
|
255
|
+
this.root = lockRoot(options.rootDir);
|
|
256
|
+
this.leaseMs = options.leaseMs ?? DEFAULT_LEASE_MS;
|
|
257
|
+
this.maxGlobalActive = options.maxGlobalActive ?? 0;
|
|
258
|
+
this.maxDepth = options.maxDepth ?? defaultConfig.maxDepth;
|
|
259
|
+
this.now = options.now ?? Date.now;
|
|
260
|
+
this.isAlive = options.isAlive ?? isProcessAlive;
|
|
261
|
+
ensureDirSync(this.root);
|
|
262
|
+
ensureDirSync(path.join(this.root, "sessions"));
|
|
263
|
+
ensureDirSync(path.join(this.root, "slots"));
|
|
264
|
+
ensureDirSync(path.join(this.root, "runs"));
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
get rootDir(): string {
|
|
268
|
+
return this.root;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ---- Session resume locks ------------------------------------------------
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Acquire exclusive ownership of a child session for direct resume.
|
|
275
|
+
* Returns the owner record on success, or the current conflict owner.
|
|
276
|
+
*/
|
|
277
|
+
acquireSessionLock(
|
|
278
|
+
childSessionId: string,
|
|
279
|
+
owner: { ownerId: string; runId: string; parentSessionKey: string },
|
|
280
|
+
): { ok: true; lock: SessionLockOwner } | { ok: false; conflict: SessionLockOwner } {
|
|
281
|
+
const file = sessionLockPath(this.root, childSessionId);
|
|
282
|
+
ensureDirSync(path.dirname(file));
|
|
283
|
+
// Attempt up to 2 times: reclaim a demonstrably-stale lock then retry.
|
|
284
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
285
|
+
let created = false;
|
|
286
|
+
let fd: number | undefined;
|
|
287
|
+
let identity: fs.Stats | undefined;
|
|
288
|
+
try {
|
|
289
|
+
// Exclusive create. Existence of the file == lock held.
|
|
290
|
+
fd = fs.openSync(file, "wx");
|
|
291
|
+
created = true;
|
|
292
|
+
identity = fs.fstatSync(fd);
|
|
293
|
+
const record: SessionLockOwner = {
|
|
294
|
+
ownerId: owner.ownerId,
|
|
295
|
+
runId: owner.runId,
|
|
296
|
+
parentSessionKey: owner.parentSessionKey,
|
|
297
|
+
process: currentProcessIdentity(),
|
|
298
|
+
acquiredAt: this.now(),
|
|
299
|
+
leaseExpiresAt: this.now() + this.leaseMs,
|
|
300
|
+
childSessionId,
|
|
301
|
+
};
|
|
302
|
+
fs.writeFileSync(fd, JSON.stringify(record, null, 2), "utf8");
|
|
303
|
+
fs.closeSync(fd);
|
|
304
|
+
fd = undefined;
|
|
305
|
+
this.startLeaseRenewal(childSessionId, record);
|
|
306
|
+
return { ok: true, lock: record };
|
|
307
|
+
} catch (error: any) {
|
|
308
|
+
if (created) {
|
|
309
|
+
if (fd !== undefined) { try { fs.closeSync(fd); } catch { /* Preserve the original initialization error. */ } }
|
|
310
|
+
this.stopLeaseRenewal(childSessionId);
|
|
311
|
+
// Only clean up the file we just created, never another owner's replacement.
|
|
312
|
+
try {
|
|
313
|
+
const current = fs.statSync(file);
|
|
314
|
+
if (identity && current.dev === identity.dev && current.ino === identity.ino) fs.unlinkSync(file);
|
|
315
|
+
} catch { /* Cleanup cannot mask the original I/O failure. */ }
|
|
316
|
+
throw error;
|
|
317
|
+
}
|
|
318
|
+
if (error?.code !== "EEXIST") throw error;
|
|
319
|
+
const existing = readJsonSync<SessionLockOwner>(file);
|
|
320
|
+
if (!existing) {
|
|
321
|
+
// Another process may be between exclusive creation and its first write.
|
|
322
|
+
// An unreadable existing record is ambiguous, never proof of a stale owner.
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
// Same owner re-acquiring is fine (idempotent).
|
|
326
|
+
if (existing.ownerId === owner.ownerId || existing.runId === owner.runId) {
|
|
327
|
+
existing.leaseExpiresAt = this.now() + this.leaseMs;
|
|
328
|
+
writeJsonAtomicSync(file, existing);
|
|
329
|
+
this.startLeaseRenewal(childSessionId, existing);
|
|
330
|
+
return { ok: true, lock: existing };
|
|
331
|
+
}
|
|
332
|
+
// Stale evaluation: see isSessionLockStale (host × identity × lease matrix).
|
|
333
|
+
const stale = this.isSessionLockStale(existing);
|
|
334
|
+
if (stale && attempt === 0) {
|
|
335
|
+
try {
|
|
336
|
+
fs.unlinkSync(file);
|
|
337
|
+
} catch {
|
|
338
|
+
/* raced with another reclaim */
|
|
339
|
+
}
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
return { ok: false, conflict: existing };
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
const conflict = readJsonSync<SessionLockOwner>(sessionLockPath(this.root, childSessionId));
|
|
346
|
+
return {
|
|
347
|
+
ok: false,
|
|
348
|
+
conflict: conflict ?? {
|
|
349
|
+
ownerId: "unknown",
|
|
350
|
+
runId: "unknown",
|
|
351
|
+
parentSessionKey: "unknown",
|
|
352
|
+
process: { pid: 0, startTime: 0, hostname: HOSTNAME },
|
|
353
|
+
acquiredAt: 0,
|
|
354
|
+
leaseExpiresAt: 0,
|
|
355
|
+
childSessionId,
|
|
356
|
+
},
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
releaseSessionLock(childSessionId: string, ownerIdOrRunId?: string): void {
|
|
361
|
+
this.stopLeaseRenewal(childSessionId);
|
|
362
|
+
const file = sessionLockPath(this.root, childSessionId);
|
|
363
|
+
const existing = readJsonSync<SessionLockOwner>(file);
|
|
364
|
+
if (!existing) return;
|
|
365
|
+
if (
|
|
366
|
+
ownerIdOrRunId &&
|
|
367
|
+
existing.ownerId !== ownerIdOrRunId &&
|
|
368
|
+
existing.runId !== ownerIdOrRunId
|
|
369
|
+
) {
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
try {
|
|
373
|
+
fs.unlinkSync(file);
|
|
374
|
+
} catch {
|
|
375
|
+
/* already gone */
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Read-only availability probe for a direct-resume lock. Uses the same
|
|
381
|
+
* host × identity × lease staleness matrix as acquisition, but **never** opens,
|
|
382
|
+
* unlinks, writes, renews or reaps anything. Live/ambiguous owners report `held`
|
|
383
|
+
* (the caller must reject); a demonstrably stale owner reports `reclaimable` and
|
|
384
|
+
* is left untouched for the authoritative atomic acquire at launch.
|
|
385
|
+
*/
|
|
386
|
+
checkResumeAvailability(childSessionId: string): SessionLockAvailability {
|
|
387
|
+
const file = sessionLockPath(this.root, childSessionId);
|
|
388
|
+
const owner = readJsonSync<SessionLockOwner>(file);
|
|
389
|
+
if (!owner) {
|
|
390
|
+
// Distinguish "no lock" from an unreadable/ambiguous record that still exists.
|
|
391
|
+
if (fs.existsSync(file)) {
|
|
392
|
+
return { childSessionId, status: "held", reason: "lock record is unreadable or ambiguous" };
|
|
393
|
+
}
|
|
394
|
+
return { childSessionId, status: "free" };
|
|
395
|
+
}
|
|
396
|
+
if (this.isSessionLockStale(owner)) {
|
|
397
|
+
return { childSessionId, status: "reclaimable", owner, reason: "stale owner (dead process or expired lease); reclaimable at acquire" };
|
|
398
|
+
}
|
|
399
|
+
return { childSessionId, status: "held", owner, reason: "live or ambiguous owner" };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
private startLeaseRenewal(childSessionId: string, record: SessionLockOwner): void {
|
|
403
|
+
this.stopLeaseRenewal(childSessionId);
|
|
404
|
+
const interval = Math.max(1_000, Math.floor(this.leaseMs / 3));
|
|
405
|
+
const timer = setInterval(() => {
|
|
406
|
+
const file = sessionLockPath(this.root, childSessionId);
|
|
407
|
+
const current = readJsonSync<SessionLockOwner>(file);
|
|
408
|
+
if (!current || (current.ownerId !== record.ownerId && current.runId !== record.runId)) {
|
|
409
|
+
this.stopLeaseRenewal(childSessionId);
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
current.leaseExpiresAt = this.now() + this.leaseMs;
|
|
413
|
+
current.process = currentProcessIdentity();
|
|
414
|
+
try {
|
|
415
|
+
writeJsonAtomicSync(file, current);
|
|
416
|
+
} catch {
|
|
417
|
+
this.stopLeaseRenewal(childSessionId);
|
|
418
|
+
}
|
|
419
|
+
}, interval);
|
|
420
|
+
timer.unref?.();
|
|
421
|
+
this.renewTimers.set(childSessionId, timer);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
private stopLeaseRenewal(childSessionId: string): void {
|
|
425
|
+
const timer = this.renewTimers.get(childSessionId);
|
|
426
|
+
if (timer) {
|
|
427
|
+
clearInterval(timer);
|
|
428
|
+
this.renewTimers.delete(childSessionId);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// ---- Global concurrency slots -------------------------------------------
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* Slots held back for strictly deeper nesting tiers so a shallow fan-out cannot
|
|
436
|
+
* exhaust the machine-wide cap while nested parents wait on their children.
|
|
437
|
+
*
|
|
438
|
+
* With defaults (cap 16, maxDepth 2): depth-0 may hold at most 15 slots, so
|
|
439
|
+
* depth-1 spawns always have ≥1 free. Single-level fan-out at maxDepth 1
|
|
440
|
+
* reserves 0 and can still fill the full cap.
|
|
441
|
+
*
|
|
442
|
+
* PLAN 3.1 numeric contract: reserve `max(0, maxDepth - 1 - depth)` (the
|
|
443
|
+
* written `min(depth, maxDepth-1)` form is inverted relative to the example).
|
|
444
|
+
*/
|
|
445
|
+
private reservedFor(depth: number): number {
|
|
446
|
+
if (this.maxDepth <= 1) return 0;
|
|
447
|
+
return Math.max(0, this.maxDepth - 1 - Math.max(0, depth));
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Try to claim a machine-wide concurrency slot. Returns undefined when the
|
|
452
|
+
* global cap is 0 (disabled) or when a slot is granted. Throws when the
|
|
453
|
+
* tiered budget for `depth` is exhausted.
|
|
454
|
+
*
|
|
455
|
+
* Slot records store `depth` (migration: missing field counts as depth 0).
|
|
456
|
+
* Admission: `activeAtOrBelowDepth(depth) < maxGlobalActive - reservedFor(depth)`.
|
|
457
|
+
*/
|
|
458
|
+
tryAcquireGlobalSlot(runId: string, depth = 0): SlotToken | undefined {
|
|
459
|
+
if (!this.maxGlobalActive || this.maxGlobalActive <= 0) return undefined;
|
|
460
|
+
const normalizedDepth = Number.isFinite(depth) && depth > 0 ? Math.floor(depth) : 0;
|
|
461
|
+
ensureDirSync(slotDir(this.root));
|
|
462
|
+
this.reapDeadSlots();
|
|
463
|
+
const limit = this.maxGlobalActive - this.reservedFor(normalizedDepth);
|
|
464
|
+
const activeAtOrBelow = this.countLiveSlotsAtOrBelow(normalizedDepth);
|
|
465
|
+
if (activeAtOrBelow >= limit || limit <= 0) {
|
|
466
|
+
throw new Error(
|
|
467
|
+
`Machine-wide subagent process limit reached (${this.maxGlobalActive}` +
|
|
468
|
+
`; depth ${normalizedDepth} budget ${Math.max(0, limit)}). ` +
|
|
469
|
+
`Wait for other Pi sessions to finish, raise PI_SUBAGENT_MAX_GLOBAL_ACTIVE, or cancel running subagents.`,
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
const slotId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
473
|
+
const file = path.join(slotDir(this.root), `${slotId}.slot`);
|
|
474
|
+
const token: SlotToken = { slotId, path: file, runId, released: false };
|
|
475
|
+
writeJsonAtomicSync(file, {
|
|
476
|
+
slotId,
|
|
477
|
+
runId,
|
|
478
|
+
depth: normalizedDepth,
|
|
479
|
+
process: currentProcessIdentity(),
|
|
480
|
+
acquiredAt: this.now(),
|
|
481
|
+
});
|
|
482
|
+
return token;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
releaseGlobalSlot(token: SlotToken | undefined): void {
|
|
486
|
+
if (!token || token.released) return;
|
|
487
|
+
token.released = true;
|
|
488
|
+
try {
|
|
489
|
+
fs.unlinkSync(token.path);
|
|
490
|
+
} catch {
|
|
491
|
+
/* already gone */
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** Count live slot files whose recorded depth is ≤ the given depth (missing → 0). */
|
|
496
|
+
private countLiveSlotsAtOrBelow(depth: number): number {
|
|
497
|
+
let entries: string[] = [];
|
|
498
|
+
try {
|
|
499
|
+
entries = fs.readdirSync(slotDir(this.root));
|
|
500
|
+
} catch {
|
|
501
|
+
return 0;
|
|
502
|
+
}
|
|
503
|
+
let count = 0;
|
|
504
|
+
for (const name of entries) {
|
|
505
|
+
if (!name.endsWith(".slot")) continue;
|
|
506
|
+
const file = path.join(slotDir(this.root), name);
|
|
507
|
+
const data = readJsonSync<{ depth?: number; process?: ProcessIdentity }>(file);
|
|
508
|
+
if (!data?.process || !this.isAlive(data.process)) continue;
|
|
509
|
+
const slotDepth = typeof data.depth === "number" && Number.isFinite(data.depth) ? data.depth : 0;
|
|
510
|
+
if (slotDepth <= depth) count++;
|
|
511
|
+
}
|
|
512
|
+
return count;
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
private reapDeadSlots(): void {
|
|
516
|
+
let entries: string[] = [];
|
|
517
|
+
try {
|
|
518
|
+
entries = fs.readdirSync(slotDir(this.root));
|
|
519
|
+
} catch {
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
for (const name of entries) {
|
|
523
|
+
if (!name.endsWith(".slot")) continue;
|
|
524
|
+
const file = path.join(slotDir(this.root), name);
|
|
525
|
+
const data = readJsonSync<{ process?: ProcessIdentity }>(file);
|
|
526
|
+
if (!data?.process || !this.isAlive(data.process)) {
|
|
527
|
+
try {
|
|
528
|
+
fs.unlinkSync(file);
|
|
529
|
+
} catch {
|
|
530
|
+
/* raced */
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* Whether an existing session lock may be unlinked and reclaimed.
|
|
538
|
+
*
|
|
539
|
+
* Matrix (host match × identity-verifiable × lease):
|
|
540
|
+
* - Same host, start-time identity match, process dead → reclaim immediately.
|
|
541
|
+
* - Same host, identity unknown (no startTime) → reclaim after **2×** lease
|
|
542
|
+
* (clock skew / PID recycle both plausible on one machine). Live PIDs
|
|
543
|
+
* with unknown identity still need the 2× window even if the soft lease
|
|
544
|
+
* expired once.
|
|
545
|
+
* - Different host (or otherwise not locally verifiable across hosts) →
|
|
546
|
+
* reclaim after **one** lease period (`leaseExpiresAt < now`). Local
|
|
547
|
+
* process checks are meaningless cross-host, so dead-looking hosts are
|
|
548
|
+
* not treated as immediate free-for-all.
|
|
549
|
+
*
|
|
550
|
+
* Clock-skew assumption (inline): on the same host, clocks are shared so a
|
|
551
|
+
* single expired lease could be skew or a stall; double the window before
|
|
552
|
+
* trusting expiry alone. Across hosts, we only have the lease wall-clock
|
|
553
|
+
* and resume after one period once expired — we never assume foreign
|
|
554
|
+
* host liveness.
|
|
555
|
+
*/
|
|
556
|
+
private isSessionLockStale(owner: SessionLockOwner): boolean {
|
|
557
|
+
const now = this.now();
|
|
558
|
+
const leaseExpiresAt = owner.leaseExpiresAt;
|
|
559
|
+
const leaseExpired = leaseExpiresAt > 0 && leaseExpiresAt < now;
|
|
560
|
+
const doubleLeaseExpired = leaseExpiresAt > 0 && leaseExpiresAt + this.leaseMs < now;
|
|
561
|
+
const sameHost = !owner.process.hostname || owner.process.hostname === HOSTNAME;
|
|
562
|
+
// Non-zero startTime lets isAlive distinguish recycled PIDs from the owner.
|
|
563
|
+
const identityVerifiable = owner.process.pid > 0 && owner.process.startTime > 0;
|
|
564
|
+
|
|
565
|
+
if (sameHost) {
|
|
566
|
+
if (identityVerifiable) {
|
|
567
|
+
// Verifiably dead with start-time match → immediate. Live owner keeps the lock
|
|
568
|
+
// even if a renewal ticked late (degrade open on runtime).
|
|
569
|
+
return !this.isAlive(owner.process);
|
|
570
|
+
}
|
|
571
|
+
// Identity unknown: ESRCH means nothing is at that pid → free immediately.
|
|
572
|
+
// If something answers kill(0), only the 2× lease window can reclaim.
|
|
573
|
+
if (!this.isAlive(owner.process)) return true;
|
|
574
|
+
return doubleLeaseExpired;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// Cross-host / foreign host: do **not** consult local isAlive (default
|
|
578
|
+
// treats other hostnames as dead). Reclaim only after one lease period.
|
|
579
|
+
return leaseExpired;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// ---- Run process records (orphan reconcile) ------------------------------
|
|
583
|
+
|
|
584
|
+
writeRunRecord(record: RunProcessRecord): void {
|
|
585
|
+
writeJsonAtomicSync(runRecordPath(this.root, record.runId), record);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
readRunRecord(runId: string): RunProcessRecord | undefined {
|
|
589
|
+
return readJsonSync<RunProcessRecord>(runRecordPath(this.root, runId));
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
markRunTerminal(runId: string, terminalState: string): void {
|
|
593
|
+
const existing = this.readRunRecord(runId);
|
|
594
|
+
const next: RunProcessRecord = existing
|
|
595
|
+
? { ...existing, state: "terminal", terminalState, updatedAt: this.now() }
|
|
596
|
+
: {
|
|
597
|
+
runId,
|
|
598
|
+
parentSessionKey: "",
|
|
599
|
+
process: { pid: 0, startTime: 0, hostname: HOSTNAME },
|
|
600
|
+
startedAt: this.now(),
|
|
601
|
+
state: "terminal",
|
|
602
|
+
terminalState,
|
|
603
|
+
updatedAt: this.now(),
|
|
604
|
+
};
|
|
605
|
+
this.writeRunRecord(next);
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
deleteRunRecord(runId: string): void {
|
|
609
|
+
try {
|
|
610
|
+
fs.unlinkSync(runRecordPath(this.root, runId));
|
|
611
|
+
} catch {
|
|
612
|
+
/* gone */
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
listRunRecords(): RunProcessRecord[] {
|
|
617
|
+
let entries: string[] = [];
|
|
618
|
+
try {
|
|
619
|
+
entries = fs.readdirSync(path.join(this.root, "runs"));
|
|
620
|
+
} catch {
|
|
621
|
+
return [];
|
|
622
|
+
}
|
|
623
|
+
const out: RunProcessRecord[] = [];
|
|
624
|
+
for (const name of entries) {
|
|
625
|
+
if (!name.endsWith(".json")) continue;
|
|
626
|
+
const record = readJsonSync<RunProcessRecord>(path.join(this.root, "runs", name));
|
|
627
|
+
if (record) out.push(record);
|
|
628
|
+
}
|
|
629
|
+
return out;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Kill any still-alive process trees recorded as running (orphan reclaim).
|
|
634
|
+
* Returns the set of runIds that were reaped.
|
|
635
|
+
*/
|
|
636
|
+
async reconcileOrphans(options: {
|
|
637
|
+
/** Grace between SIGTERM and SIGKILL. */
|
|
638
|
+
killGraceMs?: number;
|
|
639
|
+
/** Only reconcile records for this parent session key (optional). */
|
|
640
|
+
parentSessionKey?: string;
|
|
641
|
+
/** Skip run ids that the current registry still owns live. */
|
|
642
|
+
skipRunIds?: ReadonlySet<string>;
|
|
643
|
+
} = {}): Promise<{ reaped: string[]; stillAlive: string[]; alreadyDead: string[] }> {
|
|
644
|
+
const killGraceMs = options.killGraceMs ?? 3_000;
|
|
645
|
+
const report = { reaped: [] as string[], stillAlive: [] as string[], alreadyDead: [] as string[] };
|
|
646
|
+
const sleep = (ms: number) => new Promise<void>((resolve) => {
|
|
647
|
+
const t = setTimeout(resolve, ms);
|
|
648
|
+
t.unref?.();
|
|
649
|
+
});
|
|
650
|
+
for (const record of this.listRunRecords()) {
|
|
651
|
+
if (record.state === "terminal") {
|
|
652
|
+
// Keep terminal records briefly for diagnostics; sweep later.
|
|
653
|
+
continue;
|
|
654
|
+
}
|
|
655
|
+
if (options.parentSessionKey && record.parentSessionKey !== options.parentSessionKey) continue;
|
|
656
|
+
if (options.skipRunIds?.has(record.runId)) continue;
|
|
657
|
+
if (!this.isAlive(record.process)) {
|
|
658
|
+
this.markRunTerminal(record.runId, "orphaned-dead");
|
|
659
|
+
report.alreadyDead.push(record.runId);
|
|
660
|
+
if (record.childSessionId) this.releaseSessionLock(record.childSessionId, record.runId);
|
|
661
|
+
continue;
|
|
662
|
+
}
|
|
663
|
+
// Still alive: force-kill so "lost" is an honest terminal fact.
|
|
664
|
+
killProcessTree(record.process, "SIGTERM");
|
|
665
|
+
const deadline = this.now() + killGraceMs;
|
|
666
|
+
while (this.now() < deadline && this.isAlive(record.process)) {
|
|
667
|
+
await sleep(50);
|
|
668
|
+
}
|
|
669
|
+
if (this.isAlive(record.process)) {
|
|
670
|
+
killProcessTree(record.process, "SIGKILL");
|
|
671
|
+
await sleep(100);
|
|
672
|
+
}
|
|
673
|
+
this.markRunTerminal(record.runId, "orphaned-killed");
|
|
674
|
+
if (record.childSessionId) this.releaseSessionLock(record.childSessionId, record.runId);
|
|
675
|
+
if (this.isAlive(record.process)) report.stillAlive.push(record.runId);
|
|
676
|
+
else report.reaped.push(record.runId);
|
|
677
|
+
}
|
|
678
|
+
return report;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Remove terminal run records older than retentionMs and dead session locks.
|
|
683
|
+
*/
|
|
684
|
+
sweep(retentionMs = 7 * 24 * 60 * 60_000): { removedRuns: number; removedLocks: number } {
|
|
685
|
+
let removedRuns = 0;
|
|
686
|
+
let removedLocks = 0;
|
|
687
|
+
const cutoff = this.now() - retentionMs;
|
|
688
|
+
for (const record of this.listRunRecords()) {
|
|
689
|
+
if (record.state === "terminal" && record.updatedAt < cutoff) {
|
|
690
|
+
this.deleteRunRecord(record.runId);
|
|
691
|
+
removedRuns++;
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
let locks: string[] = [];
|
|
695
|
+
try {
|
|
696
|
+
locks = fs.readdirSync(path.join(this.root, "sessions"));
|
|
697
|
+
} catch {
|
|
698
|
+
locks = [];
|
|
699
|
+
}
|
|
700
|
+
for (const name of locks) {
|
|
701
|
+
if (!name.endsWith(".lock")) continue;
|
|
702
|
+
const file = path.join(this.root, "sessions", name);
|
|
703
|
+
const owner = readJsonSync<SessionLockOwner>(file);
|
|
704
|
+
if (owner && this.isSessionLockStale(owner)) {
|
|
705
|
+
try {
|
|
706
|
+
fs.unlinkSync(file);
|
|
707
|
+
removedLocks++;
|
|
708
|
+
} catch {
|
|
709
|
+
/* raced */
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
this.reapDeadSlots();
|
|
714
|
+
return { removedRuns, removedLocks };
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
dispose(): void {
|
|
718
|
+
for (const childSessionId of [...this.renewTimers.keys()]) this.stopLeaseRenewal(childSessionId);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/** Async mkdir variation used when callers need await. */
|
|
723
|
+
export async function ensureLockRoot(root?: string): Promise<string> {
|
|
724
|
+
const dir = lockRoot(root);
|
|
725
|
+
await fsp.mkdir(dir, { recursive: true });
|
|
726
|
+
await fsp.mkdir(path.join(dir, "sessions"), { recursive: true });
|
|
727
|
+
await fsp.mkdir(path.join(dir, "slots"), { recursive: true });
|
|
728
|
+
await fsp.mkdir(path.join(dir, "runs"), { recursive: true });
|
|
729
|
+
return dir;
|
|
730
|
+
}
|