agents-can-communicate 0.6.3 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +3 -1
  2. package/bin/entrypoints/hook-output.mjs +12 -0
  3. package/docs/ARCHITECTURE.md +5 -0
  4. package/docs/CAPABILITIES.md +1 -1
  5. package/docs/CLI.md +50 -2
  6. package/docs/CONCEPTS.md +7 -0
  7. package/docs/GETTING_STARTED.md +2 -2
  8. package/docs/PROTOCOL.md +31 -0
  9. package/docs/TROUBLESHOOTING.md +4 -3
  10. package/node_modules/@agents-can-communicate/adapter-antigravity/package.json +1 -1
  11. package/node_modules/@agents-can-communicate/adapter-antigravity/plugin/plugin.json +1 -1
  12. package/node_modules/@agents-can-communicate/adapter-antigravity/plugin/skills/acc/SKILL.md +8 -0
  13. package/node_modules/@agents-can-communicate/adapter-antigravity/src/install.mjs +2 -1
  14. package/node_modules/@agents-can-communicate/adapter-antigravity/src/native-delivery.mjs +57 -11
  15. package/node_modules/@agents-can-communicate/adapter-claude-code/package.json +1 -1
  16. package/node_modules/@agents-can-communicate/adapter-claude-code/plugin/skills/acc/SKILL.md +8 -0
  17. package/node_modules/@agents-can-communicate/adapter-codex/package.json +1 -1
  18. package/node_modules/@agents-can-communicate/adapter-codex/plugin/skills/acc/SKILL.md +8 -0
  19. package/node_modules/@agents-can-communicate/adapter-gemini-cli/extension/gemini-extension.json +1 -1
  20. package/node_modules/@agents-can-communicate/adapter-gemini-cli/extension/skills/acc/SKILL.md +8 -0
  21. package/node_modules/@agents-can-communicate/adapter-gemini-cli/package.json +1 -1
  22. package/node_modules/@agents-can-communicate/adapter-grok/package.json +1 -1
  23. package/node_modules/@agents-can-communicate/adapter-grok/plugin/skills/acc/SKILL.md +8 -0
  24. package/node_modules/@agents-can-communicate/adapter-kimi/package.json +1 -1
  25. package/node_modules/@agents-can-communicate/adapter-kimi/plugin/skills/acc/SKILL.md +8 -0
  26. package/node_modules/@agents-can-communicate/adapter-sdk/package.json +1 -1
  27. package/node_modules/@agents-can-communicate/adapter-sdk/src/context-projector.mjs +15 -2
  28. package/node_modules/@agents-can-communicate/adapter-sdk/src/native-attempt-writer.mjs +1 -1
  29. package/node_modules/@agents-can-communicate/adapter-sdk/src/native-attempt.mjs +3 -1
  30. package/node_modules/@agents-can-communicate/cli/package.json +1 -1
  31. package/node_modules/@agents-can-communicate/cli/src/args.mjs +6 -0
  32. package/node_modules/@agents-can-communicate/cli/src/doctor-command.mjs +13 -1
  33. package/node_modules/@agents-can-communicate/cli/src/help.mjs +2 -1
  34. package/node_modules/@agents-can-communicate/cli/src/main.mjs +33 -8
  35. package/node_modules/@agents-can-communicate/core/package.json +1 -1
  36. package/node_modules/@agents-can-communicate/core/src/attention.mjs +7 -2
  37. package/node_modules/@agents-can-communicate/core/src/conversations.mjs +2 -2
  38. package/node_modules/@agents-can-communicate/core/src/intents.mjs +16 -0
  39. package/node_modules/@agents-can-communicate/core/src/next-turn-delivery.mjs +21 -4
  40. package/node_modules/@agents-can-communicate/core/src/prune.mjs +241 -0
  41. package/node_modules/@agents-can-communicate/core/src/receipts.mjs +66 -10
  42. package/node_modules/@agents-can-communicate/core/src/service.mjs +3 -0
  43. package/node_modules/@agents-can-communicate/core/src/status.mjs +25 -1
  44. package/node_modules/@agents-can-communicate/core/src/sync.mjs +15 -2
  45. package/node_modules/@agents-can-communicate/delivery-router/package.json +1 -1
  46. package/node_modules/@agents-can-communicate/hook-runner/package.json +1 -1
  47. package/node_modules/@agents-can-communicate/hook-runner/src/native-attempt.mjs +60 -7
  48. package/node_modules/@agents-can-communicate/hook-runner/src/runner.mjs +62 -27
  49. package/node_modules/@agents-can-communicate/installer/package.json +1 -1
  50. package/node_modules/@agents-can-communicate/mcp-server/package.json +1 -1
  51. package/node_modules/@agents-can-communicate/mcp-server/src/tools.mjs +5 -3
  52. package/node_modules/@agents-can-communicate/protocol/package.json +1 -1
  53. package/node_modules/@agents-can-communicate/storage-filesystem/package.json +1 -1
  54. package/node_modules/@agents-can-communicate/storage-filesystem/src/atomic-json.mjs +26 -7
  55. package/node_modules/@agents-can-communicate/storage-filesystem/src/doomed-directory.mjs +98 -0
  56. package/node_modules/@agents-can-communicate/storage-filesystem/src/event-floor.mjs +52 -0
  57. package/node_modules/@agents-can-communicate/storage-filesystem/src/reclaim.mjs +272 -0
  58. package/node_modules/@agents-can-communicate/storage-filesystem/src/recovery.mjs +54 -7
  59. package/node_modules/@agents-can-communicate/storage-filesystem/src/stage-sweep.mjs +174 -0
  60. package/node_modules/@agents-can-communicate/storage-filesystem/src/store.mjs +97 -5
  61. package/package.json +1 -1
@@ -0,0 +1,272 @@
1
+ import path from "node:path";
2
+
3
+ import { AccError, EXIT } from "@agents-can-communicate/protocol";
4
+
5
+ import { readActiveJournal } from "./active-journal.mjs";
6
+ import { listDirectoryEntries, readJsonIfPresent } from "./atomic-json.mjs";
7
+ import { condemn, detachDoomed, discard, discardLeftovers, expired } from "./doomed-directory.mjs";
8
+ import { assertSequence, raiseEventFloor, readEventFloor } from "./event-floor.mjs";
9
+ import { statePath } from "./record-id.mjs";
10
+
11
+ // Shared with the stage sweep that calls this, so one pass cannot spend two
12
+ // budgets. Exported for the caller that wants to reclaim without a bound.
13
+ export const RECLAIM_BUDGET = 512;
14
+
15
+ const jsonNames = async (directory, root) =>
16
+ (await listDirectoryEntries(directory, { root }))
17
+ .filter(entry => entry.isFile() && entry.name.endsWith(".json"))
18
+ .map(entry => entry.name);
19
+
20
+ /**
21
+ * Which transaction the store is in the middle of, or null when it cannot tell.
22
+ *
23
+ * `readActiveJournal` throws on a store with no authority record. Not knowing
24
+ * which entry is active is not a licence to guess, so that answer suppresses
25
+ * journal reclamation rather than failing the store open it runs inside.
26
+ *
27
+ * It suppresses it without holding the pass open. A store missing its authority
28
+ * is one `openFilesystemStore` would have initialised, so it is broken rather
29
+ * than busy, and doctor is what reports a broken store. Reporting work as
30
+ * remaining would re-run this pass on every open forever, to keep finding the
31
+ * same answer about files that are inert either way.
32
+ */
33
+ async function activeTransaction(paths, root) {
34
+ try {
35
+ const active = await readActiveJournal(paths, root);
36
+ return { known: true, transactionId: active.state === "open" ? active.transactionId : null };
37
+ } catch {
38
+ return { known: false, transactionId: null };
39
+ }
40
+ }
41
+
42
+ /**
43
+ * Condemn every journal entry that has been retired, with its completion marker.
44
+ *
45
+ * `retireJournalEntry` calls `retainFile`, which by construction never unlinks,
46
+ * so a completed entry was kept forever; `completeJournal` then wrote a marker
47
+ * that nothing reads. Both halves go together: a marker outliving its entry
48
+ * records the completion of a file nothing can find.
49
+ *
50
+ * The marker directory is listed once rather than read per entry. A workspace
51
+ * in daily use reaches thousands of retired transactions, and one open each
52
+ * would be the cost this pass exists to remove.
53
+ */
54
+ async function condemnRetiredJournals(paths, root, doomed, budget, deadlineAt) {
55
+ let spent = 0;
56
+ const active = await activeTransaction(paths, root);
57
+ if (!active.known) return { spent, drained: true };
58
+
59
+ const markerDirectory = path.join(paths.retained, "journal");
60
+ const completed = new Set(await jsonNames(markerDirectory, root));
61
+ for (const name of await jsonNames(paths.journal, root)) {
62
+ if (spent + 2 > budget || expired(deadlineAt)) return { spent, drained: false };
63
+ if (!completed.has(name)) continue;
64
+ // The marker is written before the pointer goes idle, so a crash between
65
+ // the two leaves a completed marker on the entry recovery still owns.
66
+ if (path.basename(name, ".json") === active.transactionId) continue;
67
+ await condemn(path.join(paths.journal, name), doomed);
68
+ await condemn(path.join(markerDirectory, name), doomed);
69
+ spent += 2;
70
+ }
71
+ return { spent, drained: true };
72
+ }
73
+
74
+ // Ordered as numbers, not as strings: a sequence can outgrow its padding, which
75
+ // is the same reason latestEphemeralMarker sorts this way. A name that is not a
76
+ // sequence is left alone rather than guessed at.
77
+ function supersededNames(names) {
78
+ const sequenced = names.map(name => {
79
+ const digits = path.basename(name, ".json");
80
+ return /^\d+$/.test(digits) ? { name, order: BigInt(digits) } : null;
81
+ }).filter(Boolean).sort((left, right) => (left.order < right.order ? 1 : -1));
82
+ return sequenced.slice(1).map(entry => entry.name);
83
+ }
84
+
85
+ /**
86
+ * Condemn every ephemeral retention marker except the newest of each record.
87
+ *
88
+ * `latestEphemeralMarker` reads only the newest and lists the rest to find it,
89
+ * so an older marker is pure listing cost on a path that runs inside hooks.
90
+ */
91
+ async function condemnSupersededMarkers(paths, root, doomed, budget, deadlineAt) {
92
+ let spent = 0;
93
+ const area = path.join(paths.retained, "ephemeral");
94
+ for (const kind of await listDirectoryEntries(area, { root })) {
95
+ if (!kind.isDirectory()) continue;
96
+ const kindDirectory = path.join(area, kind.name);
97
+ for (const record of await listDirectoryEntries(kindDirectory, { root })) {
98
+ if (!record.isDirectory()) continue;
99
+ const directory = path.join(kindDirectory, record.name);
100
+ for (const name of supersededNames(await jsonNames(directory, root))) {
101
+ if (spent >= budget || expired(deadlineAt)) return { spent, drained: false };
102
+ await condemn(path.join(directory, name), doomed);
103
+ spent += 1;
104
+ }
105
+ }
106
+ }
107
+ return { spent, drained: true };
108
+ }
109
+
110
+ /**
111
+ * Reclaim what the store keeps and no longer reads.
112
+ *
113
+ * Both classes are provably unread - a retired journal entry has no reader at
114
+ * all, and a superseded ephemeral marker is skipped by the only function that
115
+ * looks - which is what lets this run without an operator asking. Anything that
116
+ * carries meaning waits for `acc prune`.
117
+ *
118
+ * @returns {Promise<{ reclaimed: number, remaining: boolean }>}
119
+ */
120
+ export async function reclaimRetired(paths,
121
+ { root, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
122
+ if (expired(deadlineAt)) return { reclaimed: 0, remaining: true };
123
+ let spent = 0;
124
+
125
+ const leftovers = await discardLeftovers(root, limit, deadlineAt);
126
+ spent += leftovers.spent;
127
+ // What a previous pass already moved has left its live name; removing it is
128
+ // what reclaims it, so it counts here exactly as this pass's own work does.
129
+ let reclaimed = leftovers.spent;
130
+ if (!leftovers.drained) return { reclaimed, remaining: true };
131
+
132
+ const doomed = await detachDoomed(root);
133
+ const journals = await condemnRetiredJournals(paths, root, doomed, limit - spent, deadlineAt);
134
+ spent += journals.spent;
135
+ const markers = journals.drained
136
+ ? await condemnSupersededMarkers(paths, root, doomed, limit - spent, deadlineAt)
137
+ : { spent: 0, drained: false };
138
+ spent += markers.spent;
139
+
140
+ const removed = await discard(doomed, root, limit - spent, deadlineAt);
141
+ reclaimed += removed.spent;
142
+ return { reclaimed,
143
+ remaining: !journals.drained || !markers.drained || !removed.drained };
144
+ }
145
+
146
+ /**
147
+ * Reclaim named state records, each with every retention marker it owns.
148
+ *
149
+ * `tx.remove` is the wrong instrument here and the difference matters. It
150
+ * publishes a deletion marker, which hides the record and *adds* a file, and
151
+ * `listState` then reads the record before asking whether that marker exists.
152
+ * Removing records that way would make the store larger and its listings
153
+ * slower, which is the opposite of what an operator ran this for. A marker only
154
+ * exists because the store cannot unlink a live name; when the record itself
155
+ * can be moved out, the marker has nothing left to say.
156
+ *
157
+ * `generation` is the caller's optimistic lock. Eligibility is decided from a
158
+ * snapshot and applied later, and a claim can be renewed in between, which
159
+ * writes a new generation onto the same id. A record that no longer matches is
160
+ * skipped rather than removed: it is no longer the record that was judged.
161
+ *
162
+ * @param {{kind: string, id: string, generation: string}[]} entries
163
+ * @returns {Promise<{ reclaimed: number, skipped: number, remaining: boolean }>}
164
+ */
165
+ export async function reclaimStateRecords(paths, entries,
166
+ { root, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
167
+ if (expired(deadlineAt)) return { reclaimed: 0, skipped: 0, remaining: entries.length > 0 };
168
+ let spent = 0;
169
+ let skipped = 0;
170
+
171
+ const leftovers = await discardLeftovers(root, limit, deadlineAt);
172
+ spent += leftovers.spent;
173
+ if (!leftovers.drained) {
174
+ return { reclaimed: leftovers.spent, skipped, remaining: true };
175
+ }
176
+
177
+ const doomed = await detachDoomed(root);
178
+ let condemned = 0;
179
+ let drained = true;
180
+ for (const entry of entries) {
181
+ // Two condemnations per record, asked for together, so a budget never
182
+ // leaves a record half removed - its markers gone and the record itself
183
+ // still listed, which would read as a live record whose deletion history
184
+ // had been erased.
185
+ if (spent + 2 > limit || expired(deadlineAt)) {
186
+ drained = false;
187
+ break;
188
+ }
189
+ const filePath = statePath(paths, entry.kind, entry.id);
190
+ const found = await readJsonIfPresent(filePath, root).catch(() => null);
191
+ if (found === null || found.value?.generation !== entry.generation) {
192
+ skipped += 1;
193
+ continue;
194
+ }
195
+ await condemn(filePath, doomed);
196
+ spent += 1;
197
+ condemned += 1;
198
+ try {
199
+ await condemn(path.join(paths.retained, "state", entry.kind, entry.id), doomed);
200
+ spent += 1;
201
+ // Counted only when it moved. Most records own no marker directory at
202
+ // all, and counting one anyway made `remaining` compare what was
203
+ // condemned against a number larger than could ever be discarded - so a
204
+ // finished pass reported work left and asked to be run again forever.
205
+ condemned += 1;
206
+ } catch (error) {
207
+ // A record whose generation was never superseded owns no markers.
208
+ // Nothing to move is the ordinary case, not a fault.
209
+ if (error.code !== "ENOENT") throw error;
210
+ }
211
+ }
212
+
213
+ const removed = await discard(doomed, root, limit - spent, deadlineAt);
214
+ return { reclaimed: leftovers.spent + removed.spent, skipped,
215
+ remaining: !drained || !removed.drained || removed.spent < condemned };
216
+ }
217
+
218
+ /**
219
+ * Trim the event log to a boundary, reclaiming every event at or below it.
220
+ *
221
+ * The floor is raised before a single file moves, and that order is the whole
222
+ * safety argument. A floor ahead of the trim reports a boundary for events that
223
+ * are still present, which costs a reader nothing. A trim ahead of the floor is
224
+ * a silent gap, which is the failure the floor exists to prevent.
225
+ *
226
+ * @returns {Promise<{ reclaimed: number, trimmedThrough: string, remaining: boolean }>}
227
+ */
228
+ export async function trimEventLog(paths, boundary,
229
+ { root, publish, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
230
+ assertSequence(boundary, "before");
231
+ if (expired(deadlineAt)) {
232
+ return { reclaimed: 0, trimmedThrough: (await readEventFloor(paths, root)) ?? boundary,
233
+ remaining: true };
234
+ }
235
+
236
+ const names = (await jsonNames(paths.events, root)).sort();
237
+ const newest = names.at(-1);
238
+ const allocated = newest === undefined
239
+ ? await readEventFloor(paths, root)
240
+ : path.basename(newest, ".json");
241
+ // A boundary past what this store has ever issued is not a point in its
242
+ // history. Persisting one would raise the floor above every sequence that can
243
+ // still be allocated, so a single mistyped cursor would leave a workspace
244
+ // that can never publish again. Refused rather than clamped: an operator who
245
+ // named the wrong cursor should hear about it.
246
+ if (allocated === null || boundary > allocated) {
247
+ throw new AccError(EXIT.USAGE,
248
+ "before is past the newest event this workspace has issued",
249
+ { before: boundary, newest: allocated });
250
+ }
251
+ const trimmedThrough = await raiseEventFloor(paths, { ...publish, root }, boundary);
252
+
253
+ let spent = 0;
254
+ let drained = true;
255
+ const doomed = await detachDoomed(root);
256
+ for (const name of names) {
257
+ // Sequences are fixed-width, so comparing the names is comparing the
258
+ // numbers, and the sorted listing lets this stop at the boundary rather
259
+ // than walk a log that is mostly newer than it.
260
+ if (path.basename(name, ".json") > boundary) break;
261
+ if (spent >= limit || expired(deadlineAt)) {
262
+ drained = false;
263
+ break;
264
+ }
265
+ await condemn(path.join(paths.events, name), doomed);
266
+ spent += 1;
267
+ }
268
+
269
+ const removed = await discard(doomed, root, limit - spent, deadlineAt);
270
+ return { reclaimed: removed.spent, trimmedThrough,
271
+ remaining: !drained || !removed.drained };
272
+ }
@@ -3,22 +3,62 @@ import path from "node:path";
3
3
  import { AccError, EXIT, validateRecord } from "@agents-can-communicate/protocol";
4
4
 
5
5
  import { listDirectoryEntries, listJsonFiles, readJsonIfPresent } from "./atomic-json.mjs";
6
+ import { readEventFloor } from "./event-floor.mjs";
6
7
  import { readStoreIdentity } from "./identity.mjs";
7
8
  import { readOpenJournals, rollForward } from "./journal.mjs";
8
9
  import { assertEventBinding, assertStateBinding } from "./record-id.mjs";
10
+ import { reclaimRetired } from "./reclaim.mjs";
11
+ import { sweepAcceptedStages } from "./stage-sweep.mjs";
9
12
  import { storePaths } from "./store.mjs";
10
13
  import { withWriterMutex } from "./writer-mutex.mjs";
11
14
 
12
15
  /** @typedef {{ healthy: boolean, repaired: string[], blocked: string[],
13
- * corrupt: string[] }} RepairReport */
16
+ * corrupt: string[], swept: number, staged: number, partials: number, retired: number,
17
+ * trimmedThrough: string|null }} RepairReport */
14
18
 
15
- const report = ({ repaired = [], blocked = [], corrupt = [] }) => ({
19
+ const report = ({ repaired = [], blocked = [], corrupt = [], swept = 0, staged = 0,
20
+ partials = 0, retired = 0, trimmedThrough = null }) => ({
16
21
  healthy: blocked.length === 0 && corrupt.length === 0,
17
22
  repaired: [...repaired].sort(),
18
23
  blocked: [...blocked].sort(),
19
24
  corrupt: [...corrupt].sort(),
25
+ swept,
26
+ staged,
27
+ partials,
28
+ retired,
29
+ trimmedThrough,
20
30
  });
21
31
 
32
+ // Counting, never touching. A stage held in tmp by an older version is an
33
+ // accepted stage wherever it sits, so both places are counted as staged, while
34
+ // a partial is only ever the unfinished temporary a failed publication left.
35
+ async function countStaging(paths, root) {
36
+ const entries = async directory => {
37
+ try {
38
+ return await listDirectoryEntries(directory, { root });
39
+ } catch (error) {
40
+ if (error.code === "ENOENT") return [];
41
+ throw error;
42
+ }
43
+ };
44
+ const accepted = name => name.endsWith(".published");
45
+ const staged = (await entries(paths.stage)).filter(entry => entry.isFile()
46
+ && accepted(entry.name)).length;
47
+ const inTmp = await entries(paths.tmp);
48
+ // A retired journal entry is one the store keeps and never reads again, so
49
+ // counting them is what tells an operator the automatic pass has work left.
50
+ const completed = new Set((await entries(path.join(paths.retained, "journal")))
51
+ .filter(entry => entry.isFile()).map(entry => entry.name));
52
+ const retired = (await entries(paths.journal))
53
+ .filter(entry => entry.isFile() && completed.has(entry.name)).length;
54
+ return {
55
+ staged: staged + inTmp.filter(entry => accepted(entry.name)).length,
56
+ partials: inTmp.filter(entry => entry.name.endsWith(".tmp")).length,
57
+ retired,
58
+ trimmedThrough: await readEventFloor(paths, root),
59
+ };
60
+ }
61
+
22
62
  async function inspect(root) {
23
63
  const paths = storePaths(root);
24
64
  const blocked = [];
@@ -82,6 +122,7 @@ export async function diagnoseFilesystemStore({ root }) {
82
122
  blocked: state.blocked,
83
123
  corrupt: state.corrupt,
84
124
  repaired: state.pending.map(entry => entry.transactionId),
125
+ ...await countStaging(state.paths, root),
85
126
  });
86
127
  }
87
128
 
@@ -92,18 +133,24 @@ export async function repairFilesystemStore({ root, clock }) {
92
133
  // would turn an ambiguous store into a confidently wrong one.
93
134
  return report({ blocked: state.blocked, corrupt: state.corrupt });
94
135
  }
95
- if (state.pending.length === 0) return report({});
96
-
97
- const repaired = await withWriterMutex(state.paths, { root, tmpDir: state.paths.tmp, clock },
136
+ // No early return for an empty journal any more: a store with nothing to roll
137
+ // forward is exactly the store whose staging directory needs reclaiming.
138
+ return withWriterMutex(state.paths, { root, tmpDir: state.paths.tmp, clock },
98
139
  async () => {
99
140
  const completed = [];
100
141
  for (const entry of await readOpenJournals(state.paths, root)) {
101
142
  await rollForward(state.paths, { root, tmpDir: state.paths.tmp, clock }, entry);
102
143
  completed.push(entry.transactionId);
103
144
  }
104
- return completed;
145
+ // An operator is waiting on this and a hook is not, so the sweep runs
146
+ // without the per-pass bound the automatic one obeys.
147
+ // An operator is waiting on this, and a hook is not, so neither pass is
148
+ // bounded here.
149
+ const { swept } = await sweepAcceptedStages(state.paths, { root, limit: Infinity });
150
+ const reclaimed = await reclaimRetired(state.paths, { root, limit: Infinity });
151
+ return report({ repaired: completed, swept: swept + reclaimed.reclaimed,
152
+ ...await countStaging(state.paths, root) });
105
153
  });
106
- return report({ repaired });
107
154
  }
108
155
 
109
156
  export function assertRepairable(reportValue) {
@@ -0,0 +1,174 @@
1
+ import { rename } from "node:fs/promises";
2
+ import path from "node:path";
3
+
4
+ import { EXIT } from "@agents-can-communicate/protocol";
5
+
6
+ import { encode, listDirectoryEntries, publishAtomic, readJsonIfPresent } from "./atomic-json.mjs";
7
+ import { discard, discardLeftovers, doomedName, expired, syncDirectory }
8
+ from "./doomed-directory.mjs";
9
+ import { reclaimRetired } from "./reclaim.mjs";
10
+ import { ensureManagedDirectory } from "./safe-directory.mjs";
11
+
12
+ // One pass never touches more than this many entries, counting both the legacy
13
+ // entries it moves and the entries it discards. A store left unopened can hold
14
+ // as many accepted stages as an unswept tmp/ did - tens of thousands - and an
15
+ // unbounded removal inside a hook is the budget failure this exists to avoid.
16
+ export const SWEEP_BUDGET = 512;
17
+
18
+ // A day between passes. A count-based threshold costs either a full listing of
19
+ // the directory this keeps small, or an fsync'ed write on every transaction. A
20
+ // timestamp costs one read.
21
+ export const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000;
22
+
23
+ // An older ACC keeps renaming accepted stages onto tmp, because it does not
24
+ // know stage exists, so this is a standing rule rather than a one-off
25
+ // migration. They are moved rather than unlinked: joining the doomed directory
26
+ // costs one rename and keeps the no-unlink rule intact. A partial from a failed
27
+ // publication does not end in .published and never moves.
28
+ async function reclaimLegacy(paths, root, detached, budget, deadlineAt) {
29
+ let spent = 0;
30
+ for (const entry of await listDirectoryEntries(paths.tmp, { root })) {
31
+ if (spent >= budget || expired(deadlineAt)) return { spent, drained: false };
32
+ if (!entry.isFile() || !entry.name.endsWith(".published")) continue;
33
+ await rename(path.join(paths.tmp, entry.name), path.join(detached, entry.name));
34
+ spent += 1;
35
+ }
36
+ return { spent, drained: true };
37
+ }
38
+
39
+ /**
40
+ * Empty the store's accepted staging directory.
41
+ *
42
+ * `spent` is what the budget bought, which the caller needs to give whatever
43
+ * runs next the part of the budget this pass did not use.
44
+ *
45
+ * @returns {Promise<{ swept: number, remaining: boolean, spent: number }>}
46
+ */
47
+ export async function sweepAcceptedStages(paths,
48
+ { root, limit = SWEEP_BUDGET, deadlineAt } = {}) {
49
+ // Two counters, because they answer different questions. `spent` is what the
50
+ // budget bought: moving a legacy entry and removing one both cost a syscall
51
+ // on the same directory. `swept` is what left the store, and a legacy entry
52
+ // that was only moved has not left it yet. Counting a move as a sweep would
53
+ // report two reclaimed records where one record was reclaimed.
54
+ let spent = 0;
55
+ let swept = 0;
56
+ let remaining = false;
57
+
58
+ // Finish what an interrupted pass left before starting another one. This runs
59
+ // first for a second reason: a pass that stops here never detaches a further
60
+ // directory, so an exhausted budget cannot make them accumulate.
61
+ const leftovers = await discardLeftovers(root, limit, deadlineAt);
62
+ spent += leftovers.spent;
63
+ swept += leftovers.spent;
64
+ if (!leftovers.drained) return { swept, remaining: true, spent };
65
+
66
+ const detached = doomedName(root);
67
+ // A store written by an older version has no stage directory at all, because
68
+ // nothing ever published into one. Creating it before the detach makes that
69
+ // the ordinary case rather than an error, and such a store is exactly the one
70
+ // whose accepted stages are all still in tmp for the reclamation below.
71
+ await ensureManagedDirectory(root, paths.stage);
72
+ await rename(paths.stage, detached);
73
+ await ensureManagedDirectory(root, paths.stage);
74
+ await syncDirectory(root);
75
+
76
+ const reclaimed = await reclaimLegacy(paths, root, detached, limit - spent, deadlineAt);
77
+ spent += reclaimed.spent;
78
+ if (!reclaimed.drained) return { swept, remaining: true, spent };
79
+
80
+ const removed = await discard(detached, root, limit - spent, deadlineAt);
81
+ spent += removed.spent;
82
+ swept += removed.spent;
83
+ if (!removed.drained) remaining = true;
84
+ return { swept, remaining, spent };
85
+ }
86
+
87
+ const markerPath = paths => path.join(paths.locks, "stage-sweep.json");
88
+
89
+ // A lost lock and a lapsed budget are the same answer: not now, next open. They
90
+ // both arrive as CONFLICT, and neither is a reason to fail the caller.
91
+ async function withLockTolerantly(withLock, operation) {
92
+ try {
93
+ return await withLock(operation);
94
+ } catch (error) {
95
+ if (error?.code !== EXIT.CONFLICT) throw error;
96
+ return { swept: 0, remaining: true };
97
+ }
98
+ }
99
+
100
+ // An unreadable marker means the interval is unknown, and an unknown interval
101
+ // is treated as due: maintenance that refuses to run because its own bookkeeping
102
+ // is damaged is the failure it exists to prevent.
103
+ async function sweptAt(paths, root) {
104
+ const found = await readJsonIfPresent(markerPath(paths), root).catch(() => null);
105
+ const value = Date.parse(found?.value?.sweptAt ?? "");
106
+ return Number.isNaN(value) ? null : value;
107
+ }
108
+
109
+ /**
110
+ * Sweep only when the recorded pass is older than SWEEP_INTERVAL_MS.
111
+ *
112
+ * The due check is one read and it happens before the lock. Sweeping is a
113
+ * write and needs the writer mutex, but taking that mutex is itself a mkdir, a
114
+ * write, an fsync and a rename, and this runs on every store open - which means
115
+ * inside every hook. Paying for the lock on the days there is nothing to sweep
116
+ * would put that cost on every hook to do nothing, which is the shape of the
117
+ * slowdown that made hooks time out in 0.6.1.
118
+ *
119
+ * `withLock` is the seam: production passes the writer mutex, tests count calls.
120
+ *
121
+ * @returns {Promise<{ swept: number, remaining: boolean }>}
122
+ */
123
+ export async function sweepIfDue(paths,
124
+ { root, clock, limit = SWEEP_BUDGET, deadlineAt, withLock = operation => operation() } = {}) {
125
+ // Nothing below tolerates an expired budget: withWriterMutex refuses one, and
126
+ // so does the publishAtomic that writes the marker. Leaving the work for the
127
+ // next open is the whole point of a bounded pass, so leave before the first
128
+ // call that would throw instead of failing the open this runs inside.
129
+ if (expired(deadlineAt)) return { swept: 0, remaining: true };
130
+
131
+ const now = Date.parse(clock.now());
132
+ const last = await sweptAt(paths, root);
133
+ if (last !== null && now - last < SWEEP_INTERVAL_MS) return { swept: 0, remaining: false };
134
+
135
+ // The budget can lapse after the check above: the lock may be granted late,
136
+ // or granted to someone else first. Both arrive as CONFLICT, from the mutex
137
+ // or from the publishAtomic that writes the marker. Leaving that pass for the
138
+ // next open is the bounded behaviour working, so it is answered here rather
139
+ // than left for whoever called - maintenance never decides whether a store
140
+ // can be opened. Every other fault is a real store problem and propagates.
141
+ return withLockTolerantly(withLock, async () => {
142
+ // Re-read under the lock: another process may have swept while this one
143
+ // waited for it, and two sweeps in a row would detach an empty directory
144
+ // for nothing.
145
+ const holder = await sweptAt(paths, root);
146
+ if (holder !== null && holder !== last && Date.parse(clock.now()) - holder < SWEEP_INTERVAL_MS) {
147
+ return { swept: 0, remaining: false };
148
+ }
149
+ const stages = await sweepAcceptedStages(paths, { root, limit, deadlineAt });
150
+ // One budget for the whole pass. The reclaimer gets what the stage sweep
151
+ // did not spend, so opening a store never pays the bound twice, and a stage
152
+ // sweep that ran out leaves the reclaimer for the next open rather than
153
+ // borrowing against it.
154
+ const retired = stages.remaining
155
+ ? { reclaimed: 0, remaining: true }
156
+ : await reclaimRetired(paths, { root, limit: limit - stages.spent, deadlineAt });
157
+ const result = { swept: stages.swept + retired.reclaimed,
158
+ remaining: stages.remaining || retired.remaining };
159
+ // The marker records a *finished* pass, never an attempted one. A pass that
160
+ // stopped at its entry budget or ran out of time leaves it unwritten, so
161
+ // the next open carries on immediately. Recording an unfinished pass as
162
+ // done would park the remainder for a whole interval: a store holding
163
+ // 17,880 entries would drain over weeks rather than over a few opens.
164
+ //
165
+ // Writing it after an expired deadline would also throw, since
166
+ // publishAtomic refuses one - out of the store open this runs inside.
167
+ if (result.remaining || expired(deadlineAt)) {
168
+ return { swept: result.swept, remaining: true };
169
+ }
170
+ await publishAtomic(markerPath(paths), encode({ sweptAt: clock.now() }),
171
+ { root, tmpDir: paths.tmp, stageDir: paths.stage, replace: true, deadlineAt });
172
+ return result;
173
+ });
174
+ }