@bridge4dev/runner 0.55.0 → 0.56.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -144,6 +144,11 @@ export interface RewindPreview {
144
144
  *
145
145
  * Never throws for an ordinary failure: a checkpoint that could not be taken
146
146
  * must not stop the message it was taken for from reaching the agent.
147
+ *
148
+ * The store is held for the whole of it (#388): until the closing `update-ref`
149
+ * nothing names the objects being written, and a collection running in the
150
+ * same store would take them for garbage — which is what they are, right up
151
+ * until they are not.
147
152
  */
148
153
  export declare function createCheckpoint(input: CreateCheckpointInput): Promise<CreateCheckpointResult>;
149
154
  export declare function listCheckpoints(worktreePath: string, sessionId: string): Promise<CheckpointRecord[]>;
@@ -156,7 +161,13 @@ export declare function listCheckpoints(worktreePath: string, sessionId: string)
156
161
  * a cap on a courtesy must degrade, never reject.
157
162
  */
158
163
  export declare const MAX_BUSY_SESSIONS = 10;
159
- /** What a rewind to this checkpoint would do, without doing any of it. */
164
+ /**
165
+ * What a rewind to this checkpoint would do, without doing any of it.
166
+ *
167
+ * Holds the store (#388): building the preview writes a tree of «where we are
168
+ * now», and that tree is named by no ref ever — a collection running beside it
169
+ * takes it, and `diff-tree` then fails on the oid it was just handed.
170
+ */
160
171
  export declare function previewRewind(input: {
161
172
  worktreePath: string;
162
173
  sessionId: string;
@@ -149,9 +149,154 @@ async function ensureStore(worktreePath) {
149
149
  }
150
150
  return store;
151
151
  }
152
+ /**
153
+ * One store, one thing at a time: snapshots OR collection (#388).
154
+ *
155
+ * The store is keyed by REPOSITORY, so every session working in one folder
156
+ * writes into the same objects directory — and `pruneCheckpoints` collects in
157
+ * it. Between the first `update-index --add` and the closing `update-ref` the
158
+ * objects of a snapshot are named by nothing, and `gc --prune=now` collects
159
+ * exactly what nothing names. Measured on production 07.09.2026: a reconnect
160
+ * fired the collection while a turn was taking its point, and `write-tree`
161
+ * died on its own blobs («invalid object … error building trees»). A rewind
162
+ * preview is the same shape — its tree is never named by a ref at all.
163
+ *
164
+ * Shared for the writers, exclusive for the collection. Three properties are
165
+ * load-bearing:
166
+ *
167
+ * 1. Writers do not exclude each other. Two sessions in one folder take their
168
+ * points at the same time, as they always did — `tempIndexFile` is what
169
+ * keeps them apart, and this gate must not quietly serialise them.
170
+ * 2. The queue is fair: a later writer never overtakes a waiting collection.
171
+ * A busy folder takes a point every few seconds, and a collection that can
172
+ * be overtaken is a collection that never runs — i.e. the disk it exists to
173
+ * give back is never given back.
174
+ * 3. A collection WAITS; it never gives up and runs anyway. Waiting costs it
175
+ * nothing (nobody awaits it — `reconcile` fires it and moves on), while
176
+ * running anyway is precisely the defect this closes.
177
+ *
178
+ * In-process, deliberately. The directory is shared per MACHINE, but the runner
179
+ * daemon is the only process that ever opens it — this module has exactly one
180
+ * importer — so ordering inside this process is ordering, full stop. A second
181
+ * runner started by hand beside the service would need a lock in the
182
+ * filesystem; that is not a shape this product has.
183
+ */
184
+ class StoreGate {
185
+ /** Snapshots and previews in flight. */
186
+ writing = 0;
187
+ /** A collection has the store to itself. */
188
+ collecting = false;
189
+ queue = [];
190
+ async enter(exclusive) {
191
+ // The empty-queue test is the fairness rule: with somebody already waiting,
192
+ // even a writer that could go now takes its place at the back.
193
+ if (this.queue.length === 0 && this.free(exclusive)) {
194
+ this.take(exclusive);
195
+ return;
196
+ }
197
+ await new Promise((admit) => {
198
+ this.queue.push({ exclusive, admit });
199
+ });
200
+ }
201
+ leave(exclusive) {
202
+ if (exclusive)
203
+ this.collecting = false;
204
+ else
205
+ this.writing -= 1;
206
+ while (this.queue.length > 0) {
207
+ const next = this.queue[0];
208
+ if (!next || !this.free(next.exclusive))
209
+ return;
210
+ this.queue.shift();
211
+ this.take(next.exclusive);
212
+ next.admit();
213
+ // A collection is alone in there; the writers behind it wait for its turn
214
+ // to end.
215
+ if (next.exclusive)
216
+ return;
217
+ }
218
+ }
219
+ /** Nobody holds it and nobody is waiting — the entry can be forgotten. */
220
+ idle() {
221
+ return this.writing === 0 && !this.collecting && this.queue.length === 0;
222
+ }
223
+ free(exclusive) {
224
+ return exclusive ? !this.collecting && this.writing === 0 : !this.collecting;
225
+ }
226
+ take(exclusive) {
227
+ if (exclusive)
228
+ this.collecting = true;
229
+ else
230
+ this.writing += 1;
231
+ }
232
+ }
233
+ const storeGates = new Map();
234
+ /**
235
+ * Hold this store while `fn` runs.
236
+ *
237
+ * `writing` for anything that puts objects in the store OR reads objects a
238
+ * collection could take; `collecting` for the collection itself.
239
+ *
240
+ * ONE lease per operation, taken at the entry point and never inside it. The
241
+ * rewind is why: it previews, takes a safety point and reads the checkpoint's
242
+ * tree, and if each of those took its own lease, a collection queuing between
243
+ * two of them would be waiting for a lease the rewind cannot release until the
244
+ * collection lets it continue. That is a deadlock, and the fair queue in the
245
+ * point above is exactly what makes it possible — so the fairness and the
246
+ * single lease are one decision, not two.
247
+ */
248
+ async function withStore(store, mode, fn) {
249
+ // Resolved, because the two sides name the store from different ends: the
250
+ // writers build it out of `storeFor`, the collection out of a directory
251
+ // listing.
252
+ const key = path.resolve(store);
253
+ const exclusive = mode === 'collecting';
254
+ const gate = storeGates.get(key) ?? new StoreGate();
255
+ storeGates.set(key, gate);
256
+ // No `await` between the lookup and the claim — `enter` takes its place
257
+ // synchronously, so the entry cannot be swept out from under it below.
258
+ const askedAt = Date.now();
259
+ await gate.enter(exclusive);
260
+ const waitedMs = Date.now() - askedAt;
261
+ if (waitedMs > 1_000) {
262
+ // The one thing this gate can do that is felt from outside: a restore point
263
+ // — and with it the message in front of it — waiting for a collection to
264
+ // finish. Unlogged, that is a delay nobody can explain afterwards.
265
+ log.info('checkpoints: waited for the store to be free', {
266
+ store: path.basename(key),
267
+ mode,
268
+ waitedMs,
269
+ });
270
+ }
271
+ try {
272
+ return await fn();
273
+ }
274
+ finally {
275
+ gate.leave(exclusive);
276
+ if (gate.idle() && storeGates.get(key) === gate)
277
+ storeGates.delete(key);
278
+ }
279
+ }
152
280
  function indexFileFor(sessionId) {
153
281
  return path.join(checkpointsDir(), 'index', `${sessionId}.idx`);
154
282
  }
283
+ /**
284
+ * Drop a temporary index, and never fail because of it.
285
+ *
286
+ * These calls sit in `finally` blocks, and a throw from there escapes PAST the
287
+ * catch that classifies failures — which would turn «could not take a restore
288
+ * point» into a rejected promise, and `createCheckpoint` promises never to
289
+ * throw. The file is scratch; losing the ability to delete it is not worth a
290
+ * message that never reaches the agent.
291
+ */
292
+ function removeIndex(indexFile) {
293
+ try {
294
+ fs.rmSync(indexFile, { force: true });
295
+ }
296
+ catch (error) {
297
+ log.warn('checkpoints: could not remove a temporary index', { error: String(error) });
298
+ }
299
+ }
155
300
  /**
156
301
  * A private index file for ONE operation.
157
302
  *
@@ -383,17 +528,42 @@ function decodeMeta(message) {
383
528
  *
384
529
  * Never throws for an ordinary failure: a checkpoint that could not be taken
385
530
  * must not stop the message it was taken for from reaching the agent.
531
+ *
532
+ * The store is held for the whole of it (#388): until the closing `update-ref`
533
+ * nothing names the objects being written, and a collection running in the
534
+ * same store would take them for garbage — which is what they are, right up
535
+ * until they are not.
386
536
  */
387
537
  export async function createCheckpoint(input) {
538
+ let store;
539
+ try {
540
+ store = await ensureStore(input.worktreePath);
541
+ }
542
+ catch (error) {
543
+ // Almost always «this folder is not a git repository» — the store is built
544
+ // from the repo's own common dir, so there is nothing to open.
545
+ return checkpointRefusal(input.sessionId, error);
546
+ }
547
+ return withStore(store, 'writing', () => takeCheckpoint(store, input));
548
+ }
549
+ /** An error on the way to a restore point, read as a reason to report. */
550
+ function checkpointRefusal(sessionId, error) {
551
+ const detail = String(error instanceof Error ? error.message : error).slice(0, 300);
552
+ if (/not a git repository|ambiguous argument 'HEAD'|unknown revision/i.test(detail)) {
553
+ return { created: false, reason: 'not-a-repo', detail };
554
+ }
555
+ log.warn('checkpoints: could not create a restore point', { sessionId, error: detail });
556
+ return { created: false, reason: 'failed', detail };
557
+ }
558
+ /** The point itself. The caller holds the store; this never takes it (#388). */
559
+ async function takeCheckpoint(store, input) {
388
560
  const { worktreePath, sessionId, kind } = input;
561
+ const indexFile = tempIndexFile(sessionId, 'create');
389
562
  try {
390
- const store = await ensureStore(worktreePath);
391
- const indexFile = tempIndexFile(sessionId, 'create');
392
563
  // Both ends of the shutter — see `CreateCheckpointInput.busySessions`.
393
564
  const busyBefore = input.busySessions?.() ?? [];
394
565
  const built = await buildIndex(store, worktreePath, indexFile);
395
566
  if (built.tooLarge) {
396
- fs.rmSync(indexFile, { force: true });
397
567
  return { created: false, reason: 'too-large' };
398
568
  }
399
569
  const { headSha, included, excluded: skippedFiles, byteCount } = built;
@@ -417,7 +587,6 @@ export async function createCheckpoint(input) {
417
587
  const commit = await gitStore(store, worktreePath, indexFile, 'commit-tree', tree, '-m', encodeMeta(meta));
418
588
  const ordinal = await nextOrdinal(store, worktreePath, sessionId);
419
589
  await gitStore(store, worktreePath, indexFile, 'update-ref', refFor(sessionId, ordinal), commit);
420
- fs.rmSync(indexFile, { force: true });
421
590
  return {
422
591
  created: true,
423
592
  record: { ordinal, commit, ...meta },
@@ -426,12 +595,12 @@ export async function createCheckpoint(input) {
426
595
  };
427
596
  }
428
597
  catch (error) {
429
- const detail = String(error instanceof Error ? error.message : error).slice(0, 300);
430
- if (/not a git repository|ambiguous argument 'HEAD'|unknown revision/i.test(detail)) {
431
- return { created: false, reason: 'not-a-repo', detail };
432
- }
433
- log.warn('checkpoints: could not create a restore point', { sessionId, error: detail });
434
- return { created: false, reason: 'failed', detail };
598
+ return checkpointRefusal(sessionId, error);
599
+ }
600
+ finally {
601
+ // In `finally` rather than on the way out: every refusal above used to
602
+ // leave its index file behind, and `checkpoints/index/` only ever grew.
603
+ removeIndex(indexFile);
435
604
  }
436
605
  }
437
606
  async function readCheckpoint(store, worktreePath, sessionId, ordinal) {
@@ -492,11 +661,28 @@ async function currentTree(store, worktreePath, indexFile) {
492
661
  */
493
662
  export const MAX_BUSY_SESSIONS = 10;
494
663
  const MAX_PREVIEW_ENTRIES = 5_000;
495
- /** What a rewind to this checkpoint would do, without doing any of it. */
664
+ /**
665
+ * What a rewind to this checkpoint would do, without doing any of it.
666
+ *
667
+ * Holds the store (#388): building the preview writes a tree of «where we are
668
+ * now», and that tree is named by no ref ever — a collection running beside it
669
+ * takes it, and `diff-tree` then fails on the oid it was just handed.
670
+ */
496
671
  export async function previewRewind(input) {
672
+ const store = await ensureStore(input.worktreePath);
673
+ const indexFile = tempIndexFile(input.sessionId, 'preview');
674
+ try {
675
+ return await withStore(store, 'writing', () => buildPreview(store, indexFile, input));
676
+ }
677
+ finally {
678
+ // Named out here so that every way out of the preview — including the two
679
+ // that used to walk past the cleanup — leaves the index behind it.
680
+ removeIndex(indexFile);
681
+ }
682
+ }
683
+ /** The preview itself. The caller holds the store; this never takes it (#388). */
684
+ async function buildPreview(store, indexFile, input) {
497
685
  const { worktreePath, sessionId, ordinal } = input;
498
- const store = await ensureStore(worktreePath);
499
- const indexFile = tempIndexFile(sessionId, 'preview');
500
686
  const record = await readCheckpoint(store, worktreePath, sessionId, ordinal);
501
687
  const { tree, headSha } = await currentTree(store, worktreePath, indexFile);
502
688
  if (!record) {
@@ -511,7 +697,6 @@ export async function previewRewind(input) {
511
697
  };
512
698
  }
513
699
  const raw = await gitStore(store, worktreePath, indexFile, 'diff-tree', '-r', '--no-renames', '--name-status', '-z', `${record.commit}^{tree}`, tree);
514
- fs.rmSync(indexFile, { force: true });
515
700
  const restore = [];
516
701
  const remove = [];
517
702
  const recreate = [];
@@ -597,12 +782,53 @@ async function mergeInProgress(worktreePath) {
597
782
  * by list, and a rule is exactly what nobody confirmed.
598
783
  */
599
784
  export async function applyRewind(input) {
600
- const { worktreePath, sessionId, ordinal, confirmDeletes, expectedTreeOid } = input;
785
+ const { worktreePath } = input;
601
786
  const store = await ensureStore(worktreePath);
787
+ // ONE lease over the whole of it (#388), and it covers the READS as much as
788
+ // the writes: the collection unlinks refs and then collects, so a checkpoint
789
+ // whose ref ages out mid-rewind would lose its tree between the safety point
790
+ // and the `read-tree` that restores it — a rewind that fails after it has
791
+ // already promised. Nesting a second lease inside this one would deadlock
792
+ // against a waiting collection, which is why `buildPreview` and
793
+ // `takeCheckpoint` are called here rather than their exported wrappers.
794
+ const { record, preview, safety } = await withStore(store, 'writing', () => rewindToPoint(store, input));
795
+ // Outside the lease from here on: this is the worktree and the PROJECT's
796
+ // index, and nothing in the checkpoint store depends on it.
797
+ //
798
+ // `read-tree --reset -u` removes the files that are in the seeded index and
799
+ // not in the checkpoint — which is the same set the user just confirmed,
800
+ // because both come from the same tree diff. This pass is the belt to that
801
+ // brace: it names each path explicitly, re-checks it against the worktree
802
+ // root, and reports what is actually gone. Nothing here deletes by rule, and
803
+ // there is no `git clean` anywhere in this file.
804
+ await deletePaths(worktreePath, preview.delete);
805
+ const deleted = preview.delete.filter((rel) => !fs.existsSync(path.join(worktreePath, rel))).length;
806
+ await reconcileIndex(worktreePath, preview, record.stagedPaths);
807
+ return {
808
+ restored: preview.restore.length,
809
+ deleted,
810
+ recreated: preview.recreate.length,
811
+ safety,
812
+ rewoundToKind: record.kind,
813
+ };
814
+ }
815
+ /**
816
+ * Everything the rewind does INSIDE the store: check, take the safety point,
817
+ * put the tree back. The caller holds the lease; nothing here takes one (#388).
818
+ */
819
+ async function rewindToPoint(store, input) {
820
+ const { worktreePath, sessionId, ordinal, confirmDeletes, expectedTreeOid } = input;
602
821
  const record = await readCheckpoint(store, worktreePath, sessionId, ordinal);
603
822
  if (!record)
604
823
  throw new Error('This restore point is no longer available');
605
- const preview = await previewRewind({ worktreePath, sessionId, ordinal });
824
+ const previewIndex = tempIndexFile(sessionId, 'preview');
825
+ let preview;
826
+ try {
827
+ preview = await buildPreview(store, previewIndex, { worktreePath, sessionId, ordinal });
828
+ }
829
+ finally {
830
+ removeIndex(previewIndex);
831
+ }
606
832
  if (preview.blockedReason) {
607
833
  throw new Error(rewindBlockMessage(preview.blockedReason));
608
834
  }
@@ -624,7 +850,7 @@ export async function applyRewind(input) {
624
850
  if (expected.length !== echoed.length || expected.some((p, i) => p !== echoed[i])) {
625
851
  throw new Error(MOVED);
626
852
  }
627
- const safetyResult = await createCheckpoint({
853
+ const safetyResult = await takeCheckpoint(store, {
628
854
  worktreePath,
629
855
  sessionId,
630
856
  kind: 'SAFETY',
@@ -636,29 +862,18 @@ export async function applyRewind(input) {
636
862
  : 'Could not take a safety point before the rewind — nothing was changed');
637
863
  }
638
864
  const indexFile = tempIndexFile(sessionId, 'rewind');
639
- // Seed the index with the CURRENT state so `read-tree --reset -u` only
640
- // touches files that actually differ. Against an empty index git rewrites
641
- // every file in the repository, and an mtime bump on a whole tree is a full
642
- // rebuild for every watcher on the machine.
643
- await currentTree(store, worktreePath, indexFile);
644
- await gitStore(store, worktreePath, indexFile, 'read-tree', '--reset', '-u', `${record.commit}^{tree}`);
645
- fs.rmSync(indexFile, { force: true });
646
- // `read-tree --reset -u` removes the files that are in the seeded index and
647
- // not in the checkpoint — which is the same set the user just confirmed,
648
- // because both come from the same tree diff. This pass is the belt to that
649
- // brace: it names each path explicitly, re-checks it against the worktree
650
- // root, and reports what is actually gone. Nothing here deletes by rule, and
651
- // there is no `git clean` anywhere in this file.
652
- await deletePaths(worktreePath, preview.delete);
653
- const deleted = preview.delete.filter((rel) => !fs.existsSync(path.join(worktreePath, rel))).length;
654
- await reconcileIndex(worktreePath, preview, record.stagedPaths);
655
- return {
656
- restored: preview.restore.length,
657
- deleted,
658
- recreated: preview.recreate.length,
659
- safety: safetyResult.record,
660
- rewoundToKind: record.kind,
661
- };
865
+ try {
866
+ // Seed the index with the CURRENT state so `read-tree --reset -u` only
867
+ // touches files that actually differ. Against an empty index git rewrites
868
+ // every file in the repository, and an mtime bump on a whole tree is a full
869
+ // rebuild for every watcher on the machine.
870
+ await currentTree(store, worktreePath, indexFile);
871
+ await gitStore(store, worktreePath, indexFile, 'read-tree', '--reset', '-u', `${record.commit}^{tree}`);
872
+ }
873
+ finally {
874
+ removeIndex(indexFile);
875
+ }
876
+ return { record, preview, safety: safetyResult.record };
662
877
  }
663
878
  /**
664
879
  * Said when the files of a restore point cannot be trusted (#310). Its own
@@ -852,8 +1067,23 @@ export async function pruneCheckpoints(input) {
852
1067
  // Dropping a ref only unlinks it; the trees and blobs it named stay on
853
1068
  // disk until they are collected. Skipping this would make "retention"
854
1069
  // mean nothing at all for the thing that actually takes the space.
855
- await gitRefs(store, 'reflog', 'expire', '--expire=now', '--all');
856
- await gitRefs(store, 'gc', '--prune=now', '--quiet');
1070
+ //
1071
+ // Under the store's lease, and taken HERE rather than around the walk
1072
+ // above (#388): `--prune=now` deletes everything no ref names, and a
1073
+ // snapshot being written in this same store is a set of objects no ref
1074
+ // names YET. The lease is claimed per store and only around these two
1075
+ // commands, so the ref walk of every other store stays outside it —
1076
+ // though the loop itself is sequential, so a folder that keeps this one
1077
+ // waiting does postpone the stores after it. That is acceptable and
1078
+ // was already true of `gc` itself: collection is best-effort and comes
1079
+ // round again on the next reconnect.
1080
+ //
1081
+ // It WAITS rather than skipping: a collection that runs anyway is the
1082
+ // whole defect.
1083
+ await withStore(store, 'collecting', async () => {
1084
+ await gitRefs(store, 'reflog', 'expire', '--expire=now', '--all');
1085
+ await gitRefs(store, 'gc', '--prune=now', '--quiet');
1086
+ });
857
1087
  }
858
1088
  }
859
1089
  catch (error) {
package/dist/index.js CHANGED
@@ -1443,10 +1443,10 @@ async function cmdDoctor(args) {
1443
1443
  print(` ${devbridgeSliceOverridePath()}`);
1444
1444
  if (cage.mode === 'scope') {
1445
1445
  const mb = (bytes) => bytes === null ? 'infinity' : `${Math.round(bytes / 1024 / 1024)} MB`;
1446
- print(` per session MemoryHigh ${mb(cage.memoryHighBytes)} (brake: slows, never kills), MemoryMax ${mb(cage.memoryMaxBytes)} (wall), MemorySwapMax ${mb(cage.swapMaxBytes)}, TasksMax ${SESSION_TASKS_MAX}`);
1447
- print(` at the wall ${cage.oomContinue
1448
- ? 'OOMPolicy=continue — the kernel kills the hungriest process, the session stays up'
1449
- : 'this systemd does not take OOMPolicy on a scope (it is newer than the rest of the cage), so what happens at the wall is its own default — on systemd 253+ that is «stop the whole session»'}`);
1446
+ print(` per session share ${mb(cage.memoryHighBytes)} (MemoryHigh — allocations are throttled above it), ceiling ${mb(cage.memoryMaxBytes)} (MemoryMax), swap ${mb(cage.swapMaxBytes)} (MemorySwapMax), TasksMax ${SESSION_TASKS_MAX}`);
1447
+ print(` at the ceiling ${cage.oomContinue
1448
+ ? 'OOMPolicy=continue — the kernel stops the largest process in the session'
1449
+ : 'this systemd does not take OOMPolicy on a scope, so its own default applies — on systemd 253+ that stops the whole session'}`);
1450
1450
  // What systemd has IN FORCE on the slice, beside the paths of the files
1451
1451
  // that were supposed to put it there. «В файле 7680M, действует 6.0G» is the
1452
1452
  // lesson of §5.5, and the drop-in of a slice can fail to land in exactly the
@@ -1457,7 +1457,7 @@ async function cmdDoctor(args) {
1457
1457
  print(' NOT CAPPED — the drop-in above has not been applied.');
1458
1458
  print(` Apply with: ${systemctlHint('daemon-reload')}`);
1459
1459
  }
1460
- print(` service ceiling ${mb(cage.serviceMemoryMaxBytes)} (the per-session brake is half of the containing ceiling, capped at 2.5 GiB; the wall IS the containing ceiling)`);
1460
+ print(` service ceiling ${mb(cage.serviceMemoryMaxBytes)} (a session's share is a third of the containing ceiling, its own ceiling is that minus one share)`);
1461
1461
  print(` expand-env flag ${cage.expandEnvironmentFlag ? 'passed (systemd ≥ 254)' : 'not passed — this systemd does not know it, and --scope does not expand anyway'}`);
1462
1462
  }
1463
1463
  else {
@@ -433,6 +433,12 @@ export declare function parseScopeMemoryStatus(files: {
433
433
  */
434
434
  export declare function readScopeMemoryStatus(unit: string, readFile?: (p: string) => string): ScopeMemoryStatus | null;
435
435
  export declare function markScopeOomKillsSeen(id: string, oomKills: number): void;
436
+ /**
437
+ * «a process in it was stopped» / «3 processes in it were stopped», so the verb
438
+ * agrees. Shared with the supervisor's feed notices, which count the same
439
+ * thing and must not word it differently.
440
+ */
441
+ export declare function stoppedProcesses(count: number, where?: 'it' | 'this one'): string;
436
442
  /**
437
443
  * Snapshot the cgroup's verdict before systemd can take it away.
438
444
  *
@@ -891,6 +891,16 @@ const oomKillsSeen = new Map();
891
891
  export function markScopeOomKillsSeen(id, oomKills) {
892
892
  oomKillsSeen.set(id, oomKills);
893
893
  }
894
+ /**
895
+ * «a process in it was stopped» / «3 processes in it were stopped», so the verb
896
+ * agrees. Shared with the supervisor's feed notices, which count the same
897
+ * thing and must not word it differently.
898
+ */
899
+ export function stoppedProcesses(count, where = 'it') {
900
+ return count === 1
901
+ ? `a process in ${where} was stopped`
902
+ : `${count} processes in ${where} were stopped`;
903
+ }
894
904
  const deaths = new Map();
895
905
  /**
896
906
  * Snapshot the cgroup's verdict before systemd can take it away.
@@ -909,7 +919,7 @@ export function rememberDeath(id, unit, readStatus = readScopeMemoryStatus) {
909
919
  maxBytes: status.maxBytes,
910
920
  // `oom` moves only where the limit that was hit belongs. A kill with our
911
921
  // own counter still at zero came from an ancestor — the pool.
912
- ownWall: status.ownLimitOom > 0,
922
+ reached: status.ownLimitOom > 0 ? 'own-ceiling' : 'shared-pool',
913
923
  });
914
924
  }
915
925
  /**
@@ -928,9 +938,9 @@ function rememberDeathFromResult(id) {
928
938
  deaths.set(id, {
929
939
  oomKills: 1,
930
940
  maxBytes: sessionCage().memoryMaxBytes,
931
- // systemd stopped the unit for an OOM inside it; which limit was hit is not
932
- // in `Result`, and «your own wall» is the claim that must not be guessed.
933
- ownWall: false,
941
+ // systemd stopped the unit for an OOM inside it; which limit was reached is
942
+ // not in `Result`, and neither claim may be guessed.
943
+ reached: 'unknown',
934
944
  });
935
945
  }
936
946
  /**
@@ -947,16 +957,21 @@ export function explainMemoryDeath(id) {
947
957
  if (!death)
948
958
  return null;
949
959
  deaths.delete(id);
950
- const killed = death.oomKills === 1 ? 'a process' : `${death.oomKills} processes`;
951
- const wall = death.maxBytes === null ? '' : ` of ${Math.round(death.maxBytes / MIB)} MB`;
952
- if (death.ownWall) {
953
- return (`The session ran out of memory: it went over its own memory ceiling${wall} and the kernel killed ` +
954
- `${killed} inside it. ` +
955
- 'Resume it and avoid re-running the command that was in flight, or give the machine more memory or swap.');
960
+ const stopped = stoppedProcesses(death.oomKills);
961
+ const ceiling = death.maxBytes === null ? '' : ` of ${Math.round(death.maxBytes / MIB)} MB`;
962
+ if (death.reached === 'own-ceiling') {
963
+ return (`This session reached its memory ceiling${ceiling} and ${stopped} — ` +
964
+ 'send a message to carry on; if it keeps happening, the machine needs more memory.');
965
+ }
966
+ if (death.reached === 'shared-pool') {
967
+ return (`The machine ran out of memory for agent sessions and ${stoppedProcesses(death.oomKills, 'this one')} — ` +
968
+ 'send a message to carry on; if it keeps happening, run fewer sessions at once or give the machine more memory.');
956
969
  }
957
- return (`The machine ran out of memory for agent sessions — all sessions on it share one ceiling — and the kernel killed ` +
958
- `${killed} in this one. This session was not necessarily the greedy one. ` +
959
- 'Resume it; if it keeps happening, run fewer sessions at once or give the machine more memory or swap.');
970
+ // Which ceiling was reached is unknowable here, so the sentence names no
971
+ // culprit: it has to be true whether this session was the greedy one or a
972
+ // bystander.
973
+ return (`There was not enough memory for this session and ${stopped} — ` +
974
+ 'send a message to carry on; if it keeps happening, run fewer sessions at once or give the machine more memory.');
960
975
  }
961
976
  const realSystemctl = async (args) => {
962
977
  const { stdout, stderr } = await execFileAsync('systemctl', ['--user', ...args], {
@@ -227,6 +227,8 @@ export declare class Supervisor {
227
227
  * which of the two is running.
228
228
  */
229
229
  private installInFlight;
230
+ /** A restore-point collection is running; a second reconnect must not start another (#388). */
231
+ private checkpointGcInFlight;
230
232
  /** Session 14: one project-recipe run per machine, and its verdict queue. */
231
233
  private readonly verify;
232
234
  private readonly verifyReports;
@@ -890,24 +892,37 @@ export declare class Supervisor {
890
892
  * a second session opened. The neighbours are recorded on the point instead —
891
893
  * the conversation can always be rewound to it, the files cannot.
892
894
  *
893
- * Every refusal is now audible. A restore point that was never taken is
894
- * invisible until the day somebody reaches for it, and «the button is not
895
- * there» is not a sentence anybody can act on.
895
+ * A refusal is audible when it is a refusal — when the person can do
896
+ * something about it, or when a way back they might reach for is not there.
897
+ * A restore point that was never taken is invisible until the day somebody
898
+ * reaches for it, and «the button is not there» is not a sentence anybody can
899
+ * act on.
900
+ *
901
+ * «This session was still answering» is the exception, and it is the only one
902
+ * (#384). It is not a fault and not a state to act on: it is what a follow-up
903
+ * note to a working agent looks like from in here, thirty times in a day on
904
+ * one machine, and it costs almost nothing — the point in front of the turn
905
+ * already stands, and rewinding to it takes back the files AND the
906
+ * conversation, including the note. So it goes to the runner's own log, where
907
+ * support can answer «why is there no point for that step», and not into the
908
+ * feed, where it read as breakage.
896
909
  */
897
910
  private captureCheckpoint;
898
911
  /**
899
912
  * Say something once per BUSY PERIOD, not once per message (#310).
900
913
  *
901
- * A folder held by a neighbour stays held for minutes, and a session mid-turn
902
- * can be sent three follow-up notes inside one answer. Keying this on the
903
- * message seq would have counted each of those as its own turn and said the
904
- * same sentence three times — the noise the frequency policy exists to
905
- * prevent. The set is cleared when the session next comes to rest
906
- * (`reportStatus`), which is exactly when the reason stops being true.
914
+ * A repository held by another git command stays held for as long as that
915
+ * command runs, and three follow-up notes can arrive inside one answer.
916
+ * Keying this on the message seq would have counted each of those as its own
917
+ * turn and said the same sentence three times — the noise the frequency
918
+ * policy exists to prevent. The set is cleared when the session next comes to
919
+ * rest (`reportStatus`), which is exactly when the reason stops being true.
907
920
  *
908
921
  * A SET of keys, not the last one said: two different reasons can both come
909
922
  * up inside one period, and remembering only the most recent would let them
910
- * take turns re-announcing each other.
923
+ * take turns re-announcing each other. One key uses this today — «another git
924
+ * command holds this repository» — and the set stays a set for that reason,
925
+ * not out of habit: the mid-turn key left when it stopped being said (#384).
911
926
  */
912
927
  private noticeOncePerTurn;
913
928
  /**