agents-can-communicate 0.6.3 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/bin/entrypoints/hook-output.mjs +12 -0
- package/docs/ARCHITECTURE.md +5 -0
- package/docs/CAPABILITIES.md +1 -1
- package/docs/CLI.md +50 -2
- package/docs/CONCEPTS.md +7 -0
- package/docs/GETTING_STARTED.md +2 -2
- package/docs/PROTOCOL.md +31 -0
- package/docs/TROUBLESHOOTING.md +4 -3
- package/node_modules/@agents-can-communicate/adapter-antigravity/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-antigravity/plugin/plugin.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-antigravity/plugin/skills/acc/SKILL.md +8 -0
- package/node_modules/@agents-can-communicate/adapter-antigravity/src/install.mjs +2 -1
- package/node_modules/@agents-can-communicate/adapter-antigravity/src/native-delivery.mjs +57 -11
- package/node_modules/@agents-can-communicate/adapter-claude-code/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-claude-code/plugin/skills/acc/SKILL.md +8 -0
- package/node_modules/@agents-can-communicate/adapter-codex/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-codex/plugin/skills/acc/SKILL.md +8 -0
- package/node_modules/@agents-can-communicate/adapter-gemini-cli/extension/gemini-extension.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-gemini-cli/extension/skills/acc/SKILL.md +8 -0
- package/node_modules/@agents-can-communicate/adapter-gemini-cli/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-grok/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-grok/plugin/skills/acc/SKILL.md +8 -0
- package/node_modules/@agents-can-communicate/adapter-kimi/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-kimi/plugin/skills/acc/SKILL.md +8 -0
- package/node_modules/@agents-can-communicate/adapter-sdk/package.json +1 -1
- package/node_modules/@agents-can-communicate/adapter-sdk/src/context-projector.mjs +15 -2
- package/node_modules/@agents-can-communicate/adapter-sdk/src/native-attempt-writer.mjs +1 -1
- package/node_modules/@agents-can-communicate/adapter-sdk/src/native-attempt.mjs +3 -1
- package/node_modules/@agents-can-communicate/cli/package.json +1 -1
- package/node_modules/@agents-can-communicate/cli/src/args.mjs +6 -0
- package/node_modules/@agents-can-communicate/cli/src/doctor-command.mjs +13 -1
- package/node_modules/@agents-can-communicate/cli/src/help.mjs +2 -1
- package/node_modules/@agents-can-communicate/cli/src/main.mjs +33 -8
- package/node_modules/@agents-can-communicate/core/package.json +1 -1
- package/node_modules/@agents-can-communicate/core/src/attention.mjs +7 -2
- package/node_modules/@agents-can-communicate/core/src/conversations.mjs +2 -2
- package/node_modules/@agents-can-communicate/core/src/intents.mjs +16 -0
- package/node_modules/@agents-can-communicate/core/src/next-turn-delivery.mjs +21 -4
- package/node_modules/@agents-can-communicate/core/src/prune.mjs +241 -0
- package/node_modules/@agents-can-communicate/core/src/receipts.mjs +66 -10
- package/node_modules/@agents-can-communicate/core/src/service.mjs +3 -0
- package/node_modules/@agents-can-communicate/core/src/status.mjs +25 -1
- package/node_modules/@agents-can-communicate/core/src/sync.mjs +15 -2
- package/node_modules/@agents-can-communicate/delivery-router/package.json +1 -1
- package/node_modules/@agents-can-communicate/hook-runner/package.json +1 -1
- package/node_modules/@agents-can-communicate/hook-runner/src/native-attempt.mjs +60 -7
- package/node_modules/@agents-can-communicate/hook-runner/src/runner.mjs +62 -27
- package/node_modules/@agents-can-communicate/installer/package.json +1 -1
- package/node_modules/@agents-can-communicate/mcp-server/package.json +1 -1
- package/node_modules/@agents-can-communicate/mcp-server/src/tools.mjs +5 -3
- package/node_modules/@agents-can-communicate/protocol/package.json +1 -1
- package/node_modules/@agents-can-communicate/storage-filesystem/package.json +1 -1
- package/node_modules/@agents-can-communicate/storage-filesystem/src/atomic-json.mjs +26 -7
- package/node_modules/@agents-can-communicate/storage-filesystem/src/doomed-directory.mjs +98 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/event-floor.mjs +52 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/reclaim.mjs +272 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/recovery.mjs +54 -7
- package/node_modules/@agents-can-communicate/storage-filesystem/src/stage-sweep.mjs +174 -0
- package/node_modules/@agents-can-communicate/storage-filesystem/src/store.mjs +97 -5
- package/package.json +1 -1
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
|
|
3
|
+
import { AccError, EXIT } from "@agents-can-communicate/protocol";
|
|
4
|
+
|
|
5
|
+
import { readActiveJournal } from "./active-journal.mjs";
|
|
6
|
+
import { listDirectoryEntries, readJsonIfPresent } from "./atomic-json.mjs";
|
|
7
|
+
import { condemn, detachDoomed, discard, discardLeftovers, expired } from "./doomed-directory.mjs";
|
|
8
|
+
import { assertSequence, raiseEventFloor, readEventFloor } from "./event-floor.mjs";
|
|
9
|
+
import { statePath } from "./record-id.mjs";
|
|
10
|
+
|
|
11
|
+
// Shared with the stage sweep that calls this, so one pass cannot spend two
|
|
12
|
+
// budgets. Exported for the caller that wants to reclaim without a bound.
|
|
13
|
+
export const RECLAIM_BUDGET = 512;
|
|
14
|
+
|
|
15
|
+
const jsonNames = async (directory, root) =>
|
|
16
|
+
(await listDirectoryEntries(directory, { root }))
|
|
17
|
+
.filter(entry => entry.isFile() && entry.name.endsWith(".json"))
|
|
18
|
+
.map(entry => entry.name);
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Which transaction the store is in the middle of, or null when it cannot tell.
|
|
22
|
+
*
|
|
23
|
+
* `readActiveJournal` throws on a store with no authority record. Not knowing
|
|
24
|
+
* which entry is active is not a licence to guess, so that answer suppresses
|
|
25
|
+
* journal reclamation rather than failing the store open it runs inside.
|
|
26
|
+
*
|
|
27
|
+
* It suppresses it without holding the pass open. A store missing its authority
|
|
28
|
+
* is one `openFilesystemStore` would have initialised, so it is broken rather
|
|
29
|
+
* than busy, and doctor is what reports a broken store. Reporting work as
|
|
30
|
+
* remaining would re-run this pass on every open forever, to keep finding the
|
|
31
|
+
* same answer about files that are inert either way.
|
|
32
|
+
*/
|
|
33
|
+
async function activeTransaction(paths, root) {
|
|
34
|
+
try {
|
|
35
|
+
const active = await readActiveJournal(paths, root);
|
|
36
|
+
return { known: true, transactionId: active.state === "open" ? active.transactionId : null };
|
|
37
|
+
} catch {
|
|
38
|
+
return { known: false, transactionId: null };
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Condemn every journal entry that has been retired, with its completion marker.
|
|
44
|
+
*
|
|
45
|
+
* `retireJournalEntry` calls `retainFile`, which by construction never unlinks,
|
|
46
|
+
* so a completed entry was kept forever; `completeJournal` then wrote a marker
|
|
47
|
+
* that nothing reads. Both halves go together: a marker outliving its entry
|
|
48
|
+
* records the completion of a file nothing can find.
|
|
49
|
+
*
|
|
50
|
+
* The marker directory is listed once rather than read per entry. A workspace
|
|
51
|
+
* in daily use reaches thousands of retired transactions, and one open each
|
|
52
|
+
* would be the cost this pass exists to remove.
|
|
53
|
+
*/
|
|
54
|
+
async function condemnRetiredJournals(paths, root, doomed, budget, deadlineAt) {
|
|
55
|
+
let spent = 0;
|
|
56
|
+
const active = await activeTransaction(paths, root);
|
|
57
|
+
if (!active.known) return { spent, drained: true };
|
|
58
|
+
|
|
59
|
+
const markerDirectory = path.join(paths.retained, "journal");
|
|
60
|
+
const completed = new Set(await jsonNames(markerDirectory, root));
|
|
61
|
+
for (const name of await jsonNames(paths.journal, root)) {
|
|
62
|
+
if (spent + 2 > budget || expired(deadlineAt)) return { spent, drained: false };
|
|
63
|
+
if (!completed.has(name)) continue;
|
|
64
|
+
// The marker is written before the pointer goes idle, so a crash between
|
|
65
|
+
// the two leaves a completed marker on the entry recovery still owns.
|
|
66
|
+
if (path.basename(name, ".json") === active.transactionId) continue;
|
|
67
|
+
await condemn(path.join(paths.journal, name), doomed);
|
|
68
|
+
await condemn(path.join(markerDirectory, name), doomed);
|
|
69
|
+
spent += 2;
|
|
70
|
+
}
|
|
71
|
+
return { spent, drained: true };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Ordered as numbers, not as strings: a sequence can outgrow its padding, which
|
|
75
|
+
// is the same reason latestEphemeralMarker sorts this way. A name that is not a
|
|
76
|
+
// sequence is left alone rather than guessed at.
|
|
77
|
+
function supersededNames(names) {
|
|
78
|
+
const sequenced = names.map(name => {
|
|
79
|
+
const digits = path.basename(name, ".json");
|
|
80
|
+
return /^\d+$/.test(digits) ? { name, order: BigInt(digits) } : null;
|
|
81
|
+
}).filter(Boolean).sort((left, right) => (left.order < right.order ? 1 : -1));
|
|
82
|
+
return sequenced.slice(1).map(entry => entry.name);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Condemn every ephemeral retention marker except the newest of each record.
|
|
87
|
+
*
|
|
88
|
+
* `latestEphemeralMarker` reads only the newest and lists the rest to find it,
|
|
89
|
+
* so an older marker is pure listing cost on a path that runs inside hooks.
|
|
90
|
+
*/
|
|
91
|
+
async function condemnSupersededMarkers(paths, root, doomed, budget, deadlineAt) {
|
|
92
|
+
let spent = 0;
|
|
93
|
+
const area = path.join(paths.retained, "ephemeral");
|
|
94
|
+
for (const kind of await listDirectoryEntries(area, { root })) {
|
|
95
|
+
if (!kind.isDirectory()) continue;
|
|
96
|
+
const kindDirectory = path.join(area, kind.name);
|
|
97
|
+
for (const record of await listDirectoryEntries(kindDirectory, { root })) {
|
|
98
|
+
if (!record.isDirectory()) continue;
|
|
99
|
+
const directory = path.join(kindDirectory, record.name);
|
|
100
|
+
for (const name of supersededNames(await jsonNames(directory, root))) {
|
|
101
|
+
if (spent >= budget || expired(deadlineAt)) return { spent, drained: false };
|
|
102
|
+
await condemn(path.join(directory, name), doomed);
|
|
103
|
+
spent += 1;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return { spent, drained: true };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Reclaim what the store keeps and no longer reads.
|
|
112
|
+
*
|
|
113
|
+
* Both classes are provably unread - a retired journal entry has no reader at
|
|
114
|
+
* all, and a superseded ephemeral marker is skipped by the only function that
|
|
115
|
+
* looks - which is what lets this run without an operator asking. Anything that
|
|
116
|
+
* carries meaning waits for `acc prune`.
|
|
117
|
+
*
|
|
118
|
+
* @returns {Promise<{ reclaimed: number, remaining: boolean }>}
|
|
119
|
+
*/
|
|
120
|
+
export async function reclaimRetired(paths,
|
|
121
|
+
{ root, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
|
|
122
|
+
if (expired(deadlineAt)) return { reclaimed: 0, remaining: true };
|
|
123
|
+
let spent = 0;
|
|
124
|
+
|
|
125
|
+
const leftovers = await discardLeftovers(root, limit, deadlineAt);
|
|
126
|
+
spent += leftovers.spent;
|
|
127
|
+
// What a previous pass already moved has left its live name; removing it is
|
|
128
|
+
// what reclaims it, so it counts here exactly as this pass's own work does.
|
|
129
|
+
let reclaimed = leftovers.spent;
|
|
130
|
+
if (!leftovers.drained) return { reclaimed, remaining: true };
|
|
131
|
+
|
|
132
|
+
const doomed = await detachDoomed(root);
|
|
133
|
+
const journals = await condemnRetiredJournals(paths, root, doomed, limit - spent, deadlineAt);
|
|
134
|
+
spent += journals.spent;
|
|
135
|
+
const markers = journals.drained
|
|
136
|
+
? await condemnSupersededMarkers(paths, root, doomed, limit - spent, deadlineAt)
|
|
137
|
+
: { spent: 0, drained: false };
|
|
138
|
+
spent += markers.spent;
|
|
139
|
+
|
|
140
|
+
const removed = await discard(doomed, root, limit - spent, deadlineAt);
|
|
141
|
+
reclaimed += removed.spent;
|
|
142
|
+
return { reclaimed,
|
|
143
|
+
remaining: !journals.drained || !markers.drained || !removed.drained };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Reclaim named state records, each with every retention marker it owns.
|
|
148
|
+
*
|
|
149
|
+
* `tx.remove` is the wrong instrument here and the difference matters. It
|
|
150
|
+
* publishes a deletion marker, which hides the record and *adds* a file, and
|
|
151
|
+
* `listState` then reads the record before asking whether that marker exists.
|
|
152
|
+
* Removing records that way would make the store larger and its listings
|
|
153
|
+
* slower, which is the opposite of what an operator ran this for. A marker only
|
|
154
|
+
* exists because the store cannot unlink a live name; when the record itself
|
|
155
|
+
* can be moved out, the marker has nothing left to say.
|
|
156
|
+
*
|
|
157
|
+
* `generation` is the caller's optimistic lock. Eligibility is decided from a
|
|
158
|
+
* snapshot and applied later, and a claim can be renewed in between, which
|
|
159
|
+
* writes a new generation onto the same id. A record that no longer matches is
|
|
160
|
+
* skipped rather than removed: it is no longer the record that was judged.
|
|
161
|
+
*
|
|
162
|
+
* @param {{kind: string, id: string, generation: string}[]} entries
|
|
163
|
+
* @returns {Promise<{ reclaimed: number, skipped: number, remaining: boolean }>}
|
|
164
|
+
*/
|
|
165
|
+
export async function reclaimStateRecords(paths, entries,
|
|
166
|
+
{ root, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
|
|
167
|
+
if (expired(deadlineAt)) return { reclaimed: 0, skipped: 0, remaining: entries.length > 0 };
|
|
168
|
+
let spent = 0;
|
|
169
|
+
let skipped = 0;
|
|
170
|
+
|
|
171
|
+
const leftovers = await discardLeftovers(root, limit, deadlineAt);
|
|
172
|
+
spent += leftovers.spent;
|
|
173
|
+
if (!leftovers.drained) {
|
|
174
|
+
return { reclaimed: leftovers.spent, skipped, remaining: true };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const doomed = await detachDoomed(root);
|
|
178
|
+
let condemned = 0;
|
|
179
|
+
let drained = true;
|
|
180
|
+
for (const entry of entries) {
|
|
181
|
+
// Two condemnations per record, asked for together, so a budget never
|
|
182
|
+
// leaves a record half removed - its markers gone and the record itself
|
|
183
|
+
// still listed, which would read as a live record whose deletion history
|
|
184
|
+
// had been erased.
|
|
185
|
+
if (spent + 2 > limit || expired(deadlineAt)) {
|
|
186
|
+
drained = false;
|
|
187
|
+
break;
|
|
188
|
+
}
|
|
189
|
+
const filePath = statePath(paths, entry.kind, entry.id);
|
|
190
|
+
const found = await readJsonIfPresent(filePath, root).catch(() => null);
|
|
191
|
+
if (found === null || found.value?.generation !== entry.generation) {
|
|
192
|
+
skipped += 1;
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
await condemn(filePath, doomed);
|
|
196
|
+
spent += 1;
|
|
197
|
+
condemned += 1;
|
|
198
|
+
try {
|
|
199
|
+
await condemn(path.join(paths.retained, "state", entry.kind, entry.id), doomed);
|
|
200
|
+
spent += 1;
|
|
201
|
+
// Counted only when it moved. Most records own no marker directory at
|
|
202
|
+
// all, and counting one anyway made `remaining` compare what was
|
|
203
|
+
// condemned against a number larger than could ever be discarded - so a
|
|
204
|
+
// finished pass reported work left and asked to be run again forever.
|
|
205
|
+
condemned += 1;
|
|
206
|
+
} catch (error) {
|
|
207
|
+
// A record whose generation was never superseded owns no markers.
|
|
208
|
+
// Nothing to move is the ordinary case, not a fault.
|
|
209
|
+
if (error.code !== "ENOENT") throw error;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const removed = await discard(doomed, root, limit - spent, deadlineAt);
|
|
214
|
+
return { reclaimed: leftovers.spent + removed.spent, skipped,
|
|
215
|
+
remaining: !drained || !removed.drained || removed.spent < condemned };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Trim the event log to a boundary, reclaiming every event at or below it.
|
|
220
|
+
*
|
|
221
|
+
* The floor is raised before a single file moves, and that order is the whole
|
|
222
|
+
* safety argument. A floor ahead of the trim reports a boundary for events that
|
|
223
|
+
* are still present, which costs a reader nothing. A trim ahead of the floor is
|
|
224
|
+
* a silent gap, which is the failure the floor exists to prevent.
|
|
225
|
+
*
|
|
226
|
+
* @returns {Promise<{ reclaimed: number, trimmedThrough: string, remaining: boolean }>}
|
|
227
|
+
*/
|
|
228
|
+
export async function trimEventLog(paths, boundary,
|
|
229
|
+
{ root, publish, limit = RECLAIM_BUDGET, deadlineAt } = {}) {
|
|
230
|
+
assertSequence(boundary, "before");
|
|
231
|
+
if (expired(deadlineAt)) {
|
|
232
|
+
return { reclaimed: 0, trimmedThrough: (await readEventFloor(paths, root)) ?? boundary,
|
|
233
|
+
remaining: true };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const names = (await jsonNames(paths.events, root)).sort();
|
|
237
|
+
const newest = names.at(-1);
|
|
238
|
+
const allocated = newest === undefined
|
|
239
|
+
? await readEventFloor(paths, root)
|
|
240
|
+
: path.basename(newest, ".json");
|
|
241
|
+
// A boundary past what this store has ever issued is not a point in its
|
|
242
|
+
// history. Persisting one would raise the floor above every sequence that can
|
|
243
|
+
// still be allocated, so a single mistyped cursor would leave a workspace
|
|
244
|
+
// that can never publish again. Refused rather than clamped: an operator who
|
|
245
|
+
// named the wrong cursor should hear about it.
|
|
246
|
+
if (allocated === null || boundary > allocated) {
|
|
247
|
+
throw new AccError(EXIT.USAGE,
|
|
248
|
+
"before is past the newest event this workspace has issued",
|
|
249
|
+
{ before: boundary, newest: allocated });
|
|
250
|
+
}
|
|
251
|
+
const trimmedThrough = await raiseEventFloor(paths, { ...publish, root }, boundary);
|
|
252
|
+
|
|
253
|
+
let spent = 0;
|
|
254
|
+
let drained = true;
|
|
255
|
+
const doomed = await detachDoomed(root);
|
|
256
|
+
for (const name of names) {
|
|
257
|
+
// Sequences are fixed-width, so comparing the names is comparing the
|
|
258
|
+
// numbers, and the sorted listing lets this stop at the boundary rather
|
|
259
|
+
// than walk a log that is mostly newer than it.
|
|
260
|
+
if (path.basename(name, ".json") > boundary) break;
|
|
261
|
+
if (spent >= limit || expired(deadlineAt)) {
|
|
262
|
+
drained = false;
|
|
263
|
+
break;
|
|
264
|
+
}
|
|
265
|
+
await condemn(path.join(paths.events, name), doomed);
|
|
266
|
+
spent += 1;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const removed = await discard(doomed, root, limit - spent, deadlineAt);
|
|
270
|
+
return { reclaimed: removed.spent, trimmedThrough,
|
|
271
|
+
remaining: !drained || !removed.drained };
|
|
272
|
+
}
|
|
@@ -3,22 +3,62 @@ import path from "node:path";
|
|
|
3
3
|
import { AccError, EXIT, validateRecord } from "@agents-can-communicate/protocol";
|
|
4
4
|
|
|
5
5
|
import { listDirectoryEntries, listJsonFiles, readJsonIfPresent } from "./atomic-json.mjs";
|
|
6
|
+
import { readEventFloor } from "./event-floor.mjs";
|
|
6
7
|
import { readStoreIdentity } from "./identity.mjs";
|
|
7
8
|
import { readOpenJournals, rollForward } from "./journal.mjs";
|
|
8
9
|
import { assertEventBinding, assertStateBinding } from "./record-id.mjs";
|
|
10
|
+
import { reclaimRetired } from "./reclaim.mjs";
|
|
11
|
+
import { sweepAcceptedStages } from "./stage-sweep.mjs";
|
|
9
12
|
import { storePaths } from "./store.mjs";
|
|
10
13
|
import { withWriterMutex } from "./writer-mutex.mjs";
|
|
11
14
|
|
|
12
15
|
/** @typedef {{ healthy: boolean, repaired: string[], blocked: string[],
|
|
13
|
-
* corrupt: string[]
|
|
16
|
+
* corrupt: string[], swept: number, staged: number, partials: number, retired: number,
|
|
17
|
+
* trimmedThrough: string|null }} RepairReport */
|
|
14
18
|
|
|
15
|
-
const report = ({ repaired = [], blocked = [], corrupt = []
|
|
19
|
+
const report = ({ repaired = [], blocked = [], corrupt = [], swept = 0, staged = 0,
|
|
20
|
+
partials = 0, retired = 0, trimmedThrough = null }) => ({
|
|
16
21
|
healthy: blocked.length === 0 && corrupt.length === 0,
|
|
17
22
|
repaired: [...repaired].sort(),
|
|
18
23
|
blocked: [...blocked].sort(),
|
|
19
24
|
corrupt: [...corrupt].sort(),
|
|
25
|
+
swept,
|
|
26
|
+
staged,
|
|
27
|
+
partials,
|
|
28
|
+
retired,
|
|
29
|
+
trimmedThrough,
|
|
20
30
|
});
|
|
21
31
|
|
|
32
|
+
// Counting, never touching. A stage held in tmp by an older version is an
|
|
33
|
+
// accepted stage wherever it sits, so both places are counted as staged, while
|
|
34
|
+
// a partial is only ever the unfinished temporary a failed publication left.
|
|
35
|
+
async function countStaging(paths, root) {
|
|
36
|
+
const entries = async directory => {
|
|
37
|
+
try {
|
|
38
|
+
return await listDirectoryEntries(directory, { root });
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (error.code === "ENOENT") return [];
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
const accepted = name => name.endsWith(".published");
|
|
45
|
+
const staged = (await entries(paths.stage)).filter(entry => entry.isFile()
|
|
46
|
+
&& accepted(entry.name)).length;
|
|
47
|
+
const inTmp = await entries(paths.tmp);
|
|
48
|
+
// A retired journal entry is one the store keeps and never reads again, so
|
|
49
|
+
// counting them is what tells an operator the automatic pass has work left.
|
|
50
|
+
const completed = new Set((await entries(path.join(paths.retained, "journal")))
|
|
51
|
+
.filter(entry => entry.isFile()).map(entry => entry.name));
|
|
52
|
+
const retired = (await entries(paths.journal))
|
|
53
|
+
.filter(entry => entry.isFile() && completed.has(entry.name)).length;
|
|
54
|
+
return {
|
|
55
|
+
staged: staged + inTmp.filter(entry => accepted(entry.name)).length,
|
|
56
|
+
partials: inTmp.filter(entry => entry.name.endsWith(".tmp")).length,
|
|
57
|
+
retired,
|
|
58
|
+
trimmedThrough: await readEventFloor(paths, root),
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
22
62
|
async function inspect(root) {
|
|
23
63
|
const paths = storePaths(root);
|
|
24
64
|
const blocked = [];
|
|
@@ -82,6 +122,7 @@ export async function diagnoseFilesystemStore({ root }) {
|
|
|
82
122
|
blocked: state.blocked,
|
|
83
123
|
corrupt: state.corrupt,
|
|
84
124
|
repaired: state.pending.map(entry => entry.transactionId),
|
|
125
|
+
...await countStaging(state.paths, root),
|
|
85
126
|
});
|
|
86
127
|
}
|
|
87
128
|
|
|
@@ -92,18 +133,24 @@ export async function repairFilesystemStore({ root, clock }) {
|
|
|
92
133
|
// would turn an ambiguous store into a confidently wrong one.
|
|
93
134
|
return report({ blocked: state.blocked, corrupt: state.corrupt });
|
|
94
135
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
136
|
+
// No early return for an empty journal any more: a store with nothing to roll
|
|
137
|
+
// forward is exactly the store whose staging directory needs reclaiming.
|
|
138
|
+
return withWriterMutex(state.paths, { root, tmpDir: state.paths.tmp, clock },
|
|
98
139
|
async () => {
|
|
99
140
|
const completed = [];
|
|
100
141
|
for (const entry of await readOpenJournals(state.paths, root)) {
|
|
101
142
|
await rollForward(state.paths, { root, tmpDir: state.paths.tmp, clock }, entry);
|
|
102
143
|
completed.push(entry.transactionId);
|
|
103
144
|
}
|
|
104
|
-
|
|
145
|
+
// An operator is waiting on this and a hook is not, so the sweep runs
|
|
146
|
+
// without the per-pass bound the automatic one obeys.
|
|
147
|
+
// An operator is waiting on this, and a hook is not, so neither pass is
|
|
148
|
+
// bounded here.
|
|
149
|
+
const { swept } = await sweepAcceptedStages(state.paths, { root, limit: Infinity });
|
|
150
|
+
const reclaimed = await reclaimRetired(state.paths, { root, limit: Infinity });
|
|
151
|
+
return report({ repaired: completed, swept: swept + reclaimed.reclaimed,
|
|
152
|
+
...await countStaging(state.paths, root) });
|
|
105
153
|
});
|
|
106
|
-
return report({ repaired });
|
|
107
154
|
}
|
|
108
155
|
|
|
109
156
|
export function assertRepairable(reportValue) {
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { rename } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
import { EXIT } from "@agents-can-communicate/protocol";
|
|
5
|
+
|
|
6
|
+
import { encode, listDirectoryEntries, publishAtomic, readJsonIfPresent } from "./atomic-json.mjs";
|
|
7
|
+
import { discard, discardLeftovers, doomedName, expired, syncDirectory }
|
|
8
|
+
from "./doomed-directory.mjs";
|
|
9
|
+
import { reclaimRetired } from "./reclaim.mjs";
|
|
10
|
+
import { ensureManagedDirectory } from "./safe-directory.mjs";
|
|
11
|
+
|
|
12
|
+
// One pass never touches more than this many entries, counting both the legacy
|
|
13
|
+
// entries it moves and the entries it discards. A store left unopened can hold
|
|
14
|
+
// as many accepted stages as an unswept tmp/ did - tens of thousands - and an
|
|
15
|
+
// unbounded removal inside a hook is the budget failure this exists to avoid.
|
|
16
|
+
export const SWEEP_BUDGET = 512;
|
|
17
|
+
|
|
18
|
+
// A day between passes. A count-based threshold costs either a full listing of
|
|
19
|
+
// the directory this keeps small, or an fsync'ed write on every transaction. A
|
|
20
|
+
// timestamp costs one read.
|
|
21
|
+
export const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
22
|
+
|
|
23
|
+
// An older ACC keeps renaming accepted stages onto tmp, because it does not
|
|
24
|
+
// know stage exists, so this is a standing rule rather than a one-off
|
|
25
|
+
// migration. They are moved rather than unlinked: joining the doomed directory
|
|
26
|
+
// costs one rename and keeps the no-unlink rule intact. A partial from a failed
|
|
27
|
+
// publication does not end in .published and never moves.
|
|
28
|
+
async function reclaimLegacy(paths, root, detached, budget, deadlineAt) {
|
|
29
|
+
let spent = 0;
|
|
30
|
+
for (const entry of await listDirectoryEntries(paths.tmp, { root })) {
|
|
31
|
+
if (spent >= budget || expired(deadlineAt)) return { spent, drained: false };
|
|
32
|
+
if (!entry.isFile() || !entry.name.endsWith(".published")) continue;
|
|
33
|
+
await rename(path.join(paths.tmp, entry.name), path.join(detached, entry.name));
|
|
34
|
+
spent += 1;
|
|
35
|
+
}
|
|
36
|
+
return { spent, drained: true };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Empty the store's accepted staging directory.
|
|
41
|
+
*
|
|
42
|
+
* `spent` is what the budget bought, which the caller needs to give whatever
|
|
43
|
+
* runs next the part of the budget this pass did not use.
|
|
44
|
+
*
|
|
45
|
+
* @returns {Promise<{ swept: number, remaining: boolean, spent: number }>}
|
|
46
|
+
*/
|
|
47
|
+
export async function sweepAcceptedStages(paths,
|
|
48
|
+
{ root, limit = SWEEP_BUDGET, deadlineAt } = {}) {
|
|
49
|
+
// Two counters, because they answer different questions. `spent` is what the
|
|
50
|
+
// budget bought: moving a legacy entry and removing one both cost a syscall
|
|
51
|
+
// on the same directory. `swept` is what left the store, and a legacy entry
|
|
52
|
+
// that was only moved has not left it yet. Counting a move as a sweep would
|
|
53
|
+
// report two reclaimed records where one record was reclaimed.
|
|
54
|
+
let spent = 0;
|
|
55
|
+
let swept = 0;
|
|
56
|
+
let remaining = false;
|
|
57
|
+
|
|
58
|
+
// Finish what an interrupted pass left before starting another one. This runs
|
|
59
|
+
// first for a second reason: a pass that stops here never detaches a further
|
|
60
|
+
// directory, so an exhausted budget cannot make them accumulate.
|
|
61
|
+
const leftovers = await discardLeftovers(root, limit, deadlineAt);
|
|
62
|
+
spent += leftovers.spent;
|
|
63
|
+
swept += leftovers.spent;
|
|
64
|
+
if (!leftovers.drained) return { swept, remaining: true, spent };
|
|
65
|
+
|
|
66
|
+
const detached = doomedName(root);
|
|
67
|
+
// A store written by an older version has no stage directory at all, because
|
|
68
|
+
// nothing ever published into one. Creating it before the detach makes that
|
|
69
|
+
// the ordinary case rather than an error, and such a store is exactly the one
|
|
70
|
+
// whose accepted stages are all still in tmp for the reclamation below.
|
|
71
|
+
await ensureManagedDirectory(root, paths.stage);
|
|
72
|
+
await rename(paths.stage, detached);
|
|
73
|
+
await ensureManagedDirectory(root, paths.stage);
|
|
74
|
+
await syncDirectory(root);
|
|
75
|
+
|
|
76
|
+
const reclaimed = await reclaimLegacy(paths, root, detached, limit - spent, deadlineAt);
|
|
77
|
+
spent += reclaimed.spent;
|
|
78
|
+
if (!reclaimed.drained) return { swept, remaining: true, spent };
|
|
79
|
+
|
|
80
|
+
const removed = await discard(detached, root, limit - spent, deadlineAt);
|
|
81
|
+
spent += removed.spent;
|
|
82
|
+
swept += removed.spent;
|
|
83
|
+
if (!removed.drained) remaining = true;
|
|
84
|
+
return { swept, remaining, spent };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const markerPath = paths => path.join(paths.locks, "stage-sweep.json");
|
|
88
|
+
|
|
89
|
+
// A lost lock and a lapsed budget are the same answer: not now, next open. They
|
|
90
|
+
// both arrive as CONFLICT, and neither is a reason to fail the caller.
|
|
91
|
+
async function withLockTolerantly(withLock, operation) {
|
|
92
|
+
try {
|
|
93
|
+
return await withLock(operation);
|
|
94
|
+
} catch (error) {
|
|
95
|
+
if (error?.code !== EXIT.CONFLICT) throw error;
|
|
96
|
+
return { swept: 0, remaining: true };
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// An unreadable marker means the interval is unknown, and an unknown interval
|
|
101
|
+
// is treated as due: maintenance that refuses to run because its own bookkeeping
|
|
102
|
+
// is damaged is the failure it exists to prevent.
|
|
103
|
+
async function sweptAt(paths, root) {
|
|
104
|
+
const found = await readJsonIfPresent(markerPath(paths), root).catch(() => null);
|
|
105
|
+
const value = Date.parse(found?.value?.sweptAt ?? "");
|
|
106
|
+
return Number.isNaN(value) ? null : value;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Sweep only when the recorded pass is older than SWEEP_INTERVAL_MS.
|
|
111
|
+
*
|
|
112
|
+
* The due check is one read and it happens before the lock. Sweeping is a
|
|
113
|
+
* write and needs the writer mutex, but taking that mutex is itself a mkdir, a
|
|
114
|
+
* write, an fsync and a rename, and this runs on every store open - which means
|
|
115
|
+
* inside every hook. Paying for the lock on the days there is nothing to sweep
|
|
116
|
+
* would put that cost on every hook to do nothing, which is the shape of the
|
|
117
|
+
* slowdown that made hooks time out in 0.6.1.
|
|
118
|
+
*
|
|
119
|
+
* `withLock` is the seam: production passes the writer mutex, tests count calls.
|
|
120
|
+
*
|
|
121
|
+
* @returns {Promise<{ swept: number, remaining: boolean }>}
|
|
122
|
+
*/
|
|
123
|
+
export async function sweepIfDue(paths,
|
|
124
|
+
{ root, clock, limit = SWEEP_BUDGET, deadlineAt, withLock = operation => operation() } = {}) {
|
|
125
|
+
// Nothing below tolerates an expired budget: withWriterMutex refuses one, and
|
|
126
|
+
// so does the publishAtomic that writes the marker. Leaving the work for the
|
|
127
|
+
// next open is the whole point of a bounded pass, so leave before the first
|
|
128
|
+
// call that would throw instead of failing the open this runs inside.
|
|
129
|
+
if (expired(deadlineAt)) return { swept: 0, remaining: true };
|
|
130
|
+
|
|
131
|
+
const now = Date.parse(clock.now());
|
|
132
|
+
const last = await sweptAt(paths, root);
|
|
133
|
+
if (last !== null && now - last < SWEEP_INTERVAL_MS) return { swept: 0, remaining: false };
|
|
134
|
+
|
|
135
|
+
// The budget can lapse after the check above: the lock may be granted late,
|
|
136
|
+
// or granted to someone else first. Both arrive as CONFLICT, from the mutex
|
|
137
|
+
// or from the publishAtomic that writes the marker. Leaving that pass for the
|
|
138
|
+
// next open is the bounded behaviour working, so it is answered here rather
|
|
139
|
+
// than left for whoever called - maintenance never decides whether a store
|
|
140
|
+
// can be opened. Every other fault is a real store problem and propagates.
|
|
141
|
+
return withLockTolerantly(withLock, async () => {
|
|
142
|
+
// Re-read under the lock: another process may have swept while this one
|
|
143
|
+
// waited for it, and two sweeps in a row would detach an empty directory
|
|
144
|
+
// for nothing.
|
|
145
|
+
const holder = await sweptAt(paths, root);
|
|
146
|
+
if (holder !== null && holder !== last && Date.parse(clock.now()) - holder < SWEEP_INTERVAL_MS) {
|
|
147
|
+
return { swept: 0, remaining: false };
|
|
148
|
+
}
|
|
149
|
+
const stages = await sweepAcceptedStages(paths, { root, limit, deadlineAt });
|
|
150
|
+
// One budget for the whole pass. The reclaimer gets what the stage sweep
|
|
151
|
+
// did not spend, so opening a store never pays the bound twice, and a stage
|
|
152
|
+
// sweep that ran out leaves the reclaimer for the next open rather than
|
|
153
|
+
// borrowing against it.
|
|
154
|
+
const retired = stages.remaining
|
|
155
|
+
? { reclaimed: 0, remaining: true }
|
|
156
|
+
: await reclaimRetired(paths, { root, limit: limit - stages.spent, deadlineAt });
|
|
157
|
+
const result = { swept: stages.swept + retired.reclaimed,
|
|
158
|
+
remaining: stages.remaining || retired.remaining };
|
|
159
|
+
// The marker records a *finished* pass, never an attempted one. A pass that
|
|
160
|
+
// stopped at its entry budget or ran out of time leaves it unwritten, so
|
|
161
|
+
// the next open carries on immediately. Recording an unfinished pass as
|
|
162
|
+
// done would park the remainder for a whole interval: a store holding
|
|
163
|
+
// 17,880 entries would drain over weeks rather than over a few opens.
|
|
164
|
+
//
|
|
165
|
+
// Writing it after an expired deadline would also throw, since
|
|
166
|
+
// publishAtomic refuses one - out of the store open this runs inside.
|
|
167
|
+
if (result.remaining || expired(deadlineAt)) {
|
|
168
|
+
return { swept: result.swept, remaining: true };
|
|
169
|
+
}
|
|
170
|
+
await publishAtomic(markerPath(paths), encode({ sweptAt: clock.now() }),
|
|
171
|
+
{ root, tmpDir: paths.tmp, stageDir: paths.stage, replace: true, deadlineAt });
|
|
172
|
+
return result;
|
|
173
|
+
});
|
|
174
|
+
}
|