@loomwork/plugin-content 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +8 -0
- package/README.md +92 -2
- package/dist/CachingContentStore.d.ts +54 -0
- package/dist/CachingContentStore.d.ts.map +1 -0
- package/dist/CachingContentStore.js +171 -0
- package/dist/CachingContentStore.js.map +1 -0
- package/dist/ContentResolvingEventStore.d.ts +70 -0
- package/dist/ContentResolvingEventStore.d.ts.map +1 -0
- package/dist/ContentResolvingEventStore.js +245 -0
- package/dist/ContentResolvingEventStore.js.map +1 -0
- package/dist/ContentStore.d.ts +186 -0
- package/dist/ContentStore.d.ts.map +1 -0
- package/dist/ContentStore.js +177 -0
- package/dist/ContentStore.js.map +1 -0
- package/dist/ContentTouchingSnapshotStore.d.ts +56 -0
- package/dist/ContentTouchingSnapshotStore.d.ts.map +1 -0
- package/dist/ContentTouchingSnapshotStore.js +85 -0
- package/dist/ContentTouchingSnapshotStore.js.map +1 -0
- package/dist/citations.d.ts +40 -0
- package/dist/citations.d.ts.map +1 -0
- package/dist/citations.js +118 -0
- package/dist/citations.js.map +1 -0
- package/dist/content.bench.d.ts +2 -0
- package/dist/content.bench.d.ts.map +1 -0
- package/dist/content.bench.js +75 -0
- package/dist/content.bench.js.map +1 -0
- package/dist/content.d.ts +40 -0
- package/dist/content.d.ts.map +1 -0
- package/dist/content.js +177 -0
- package/dist/content.js.map +1 -0
- package/dist/derive.d.ts +9 -0
- package/dist/derive.d.ts.map +1 -0
- package/dist/derive.js +40 -0
- package/dist/derive.js.map +1 -0
- package/dist/errors.d.ts +6 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +20 -0
- package/dist/errors.js.map +1 -0
- package/dist/executionEnd.d.ts +22 -0
- package/dist/executionEnd.d.ts.map +1 -0
- package/dist/executionEnd.js +66 -0
- package/dist/executionEnd.js.map +1 -0
- package/dist/gc.d.ts +470 -0
- package/dist/gc.d.ts.map +1 -0
- package/dist/gc.js +819 -0
- package/dist/gc.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +42 -0
- package/dist/index.js.map +1 -0
- package/dist/touchAround.d.ts +46 -0
- package/dist/touchAround.d.ts.map +1 -0
- package/dist/touchAround.js +98 -0
- package/dist/touchAround.js.map +1 -0
- package/package.json +58 -3
- package/src/CachingContentStore.ts +184 -0
- package/src/ContentResolvingEventStore.ts +325 -0
- package/src/ContentStore.ts +317 -0
- package/src/ContentTouchingSnapshotStore.ts +109 -0
- package/src/citations.ts +117 -0
- package/src/content.bench.ts +84 -0
- package/src/content.ts +191 -0
- package/src/derive.ts +40 -0
- package/src/errors.ts +19 -0
- package/src/executionEnd.ts +64 -0
- package/src/gc.ts +1213 -0
- package/src/index.ts +54 -0
- package/src/touchAround.ts +131 -0
package/dist/gc.js
ADDED
|
@@ -0,0 +1,819 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SealedSnapshotScanError = exports.SealedArchiveScanError = exports.ArchiveStoresRequiredError = exports.ReclaimOrphansClockError = exports.ReclaimOrphansMarginError = exports.MIN_ORPHAN_ERASE_MARGIN_MS = exports.ReclaimOrphansTouchError = exports.ReclaimOrphansSnapshotAckError = exports.ReclaimOrphansAckError = void 0;
|
|
4
|
+
exports.pinByEventKinds = pinByEventKinds;
|
|
5
|
+
exports.resetSnapshotUnwrappedWarning = resetSnapshotUnwrappedWarning;
|
|
6
|
+
exports.sweepUnpinnedContent = sweepUnpinnedContent;
|
|
7
|
+
const event_runtime_1 = require("@loomwork/event-runtime");
|
|
8
|
+
const ContentTouchingSnapshotStore_1 = require("./ContentTouchingSnapshotStore");
|
|
9
|
+
const citations_1 = require("./citations");
|
|
10
|
+
const executionEnd_1 = require("./executionEnd");
|
|
11
|
+
// ── K10.6 (ADR 0055) — pinned mark-and-sweep for claim-checked content ────────
|
|
12
|
+
//
|
|
13
|
+
// Blob reclamation was the ADR 0046 deferred gap: `deleteExecution`/
|
|
14
|
+
// `truncatePrefix` reclaim events but never blobs. This sweep closes it for
|
|
15
|
+
// claim-checked results (ADR 0054), with the retention rule the perpetual-agents
|
|
16
|
+
// series fixed jointly: *findings and their evidence are permanent; runs are
|
|
17
|
+
// archivable; raw telemetry is disposable.* Pins are DERIVED from the log on
|
|
18
|
+
// every sweep (mark-and-sweep, not a persisted refcount): the log is the source
|
|
19
|
+
// of truth, there is no mutable index to corrupt or resurrect, and archived
|
|
20
|
+
// executions keep pinning simply by being scanned — pass the archive store
|
|
21
|
+
// alongside the live one. Keyed by content address, so a blob cited by a
|
|
22
|
+
// finding in an archived month-1 run survives any number of rotations.
|
|
23
|
+
//
|
|
24
|
+
// Only a claim-check envelope's own `ref` is TTL-reclaimable (ADR 0055 D4).
|
|
25
|
+
// A bare ref is load-bearing: `ContentResolvingEventStore.read` resolves every
|
|
26
|
+
// bare ref in a payload and throws `ContentMissingError` when its blob is gone,
|
|
27
|
+
// so erasing one bricks the execution (resume, replay, fork, export). The raw
|
|
28
|
+
// log cannot say which bare refs a resolving store will resolve, so every bare
|
|
29
|
+
// ref is retained while any scanned event cites it.
|
|
30
|
+
/**
|
|
31
|
+
* The highest sequence missing below the log's tip, or -1 when it is contiguous from 0. The
|
|
32
|
+
* same test `LogLifecycle` uses to find a compaction checkpoint (ADR 0106). One backward pass.
|
|
33
|
+
*/
|
|
34
|
+
function highestMissingSequence(events) {
|
|
35
|
+
for (let i = events.length - 1; i > 0; i--) {
|
|
36
|
+
const at = events[i].sequence;
|
|
37
|
+
if (at - events[i - 1].sequence > 1)
|
|
38
|
+
return at - 1;
|
|
39
|
+
}
|
|
40
|
+
return events.length > 0 && events[0].sequence !== 0 ? events[0].sequence - 1 : -1;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* A {@link PinPredicate} for `LogLifecycle` (K10.6): pin any execution whose log
|
|
44
|
+
* carries at least one event of the designated kinds — e.g. the app's finding
|
|
45
|
+
* events. Use when no archive is configured and reclaiming the run would destroy
|
|
46
|
+
* the only copy of its durable conclusions.
|
|
47
|
+
*/
|
|
48
|
+
function pinByEventKinds(eventStore, kinds) {
|
|
49
|
+
const pinnedKinds = new Set(kinds);
|
|
50
|
+
return async (executionId) => (await eventStore.read(executionId)).some((event) => pinnedKinds.has(event.eventType));
|
|
51
|
+
}
|
|
52
|
+
/** A safe description of a rejected value: BigInt and circular values make `JSON.stringify` throw. */
|
|
53
|
+
function describeReceived(received) {
|
|
54
|
+
try {
|
|
55
|
+
return JSON.stringify(received) ?? String(received);
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
try {
|
|
59
|
+
return `an unserializable ${typeof received} (${Object.prototype.toString.call(received)})`;
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return `an unserializable ${typeof received}`;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** `reclaimOrphans` was set without `{ completeScan: true }`. Nothing was read or erased. */
|
|
67
|
+
class ReclaimOrphansAckError extends TypeError {
|
|
68
|
+
constructor(received) {
|
|
69
|
+
super(`sweepUnpinnedContent: reclaimOrphans must be { completeScan: true } to erase (received ${describeReceived(received)}). Orphan reclamation erases every blob that no scanned log, snapshot or root cites, so a citing store you forgot to pass loses its blobs while they are still cited. Set { completeScan: true } only when eventStores, snapshotStores and roots cover ALL of them (every retention archive and its snapshot store included), AND scanned in order: each live store BEFORE its archive, or an execution retention moves mid-sweep can be missed by both. A dryRun does not need the acknowledgement. Migration: reclaimOrphans: true -> reclaimOrphans: { completeScan: true }.`);
|
|
70
|
+
this.name = "ReclaimOrphansAckError";
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
exports.ReclaimOrphansAckError = ReclaimOrphansAckError;
|
|
74
|
+
/**
|
|
75
|
+
* An ERASING `reclaimOrphans` pass was given `snapshotStores` without
|
|
76
|
+
* `{ snapshotsTouchContent: true }`. Nothing was read or erased. A compacted parent's prefix refs
|
|
77
|
+
* live only in snapshot state, and a fork copies that state into its seed snapshot with no `put`
|
|
78
|
+
* and no event; only a snapshot store wrapped in `ContentTouchingSnapshotStore` touches them.
|
|
79
|
+
* The sweep cannot see how the writing runtime's snapshot store was built, so, like D6's
|
|
80
|
+
* archive assertion, the host states it. A `dryRun` is exempt. With no `snapshotStores` the
|
|
81
|
+
* acknowledgement is not needed: the pass then refuses any store that can `truncatePrefix`, and
|
|
82
|
+
* a fork of an UNcompacted parent copies every event (refs touched by `copyPrefix`), so its seed
|
|
83
|
+
* state cites nothing the copied events do not.
|
|
84
|
+
*/
|
|
85
|
+
class ReclaimOrphansSnapshotAckError extends TypeError {
|
|
86
|
+
constructor() {
|
|
87
|
+
super("sweepUnpinnedContent: an erasing reclaimOrphans pass given snapshotStores needs { completeScan: true, snapshotsTouchContent: true }. A fork of a compacted parent saves a seed snapshot that cites refs found nowhere else; only a snapshot store wrapped in ContentTouchingSnapshotStore (outside any EncryptingSnapshotStore, over the same content store the log uses) touches them, so a sweep racing the fork erases blobs a live child cites (replay then throws ContentMissingError). Wrap every snapshot store the runtime saves to, then pass snapshotsTouchContent: true, or run with dryRun: true.");
|
|
88
|
+
this.name = "ReclaimOrphansSnapshotAckError";
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
exports.ReclaimOrphansSnapshotAckError = ReclaimOrphansSnapshotAckError;
|
|
92
|
+
/**
|
|
93
|
+
* `reclaimOrphans` was asked to ERASE through a content store without SPI 1.4.0 `touch`.
|
|
94
|
+
* Nothing was read or erased. Without `touch`, a citation that enters a log with no `put` (a
|
|
95
|
+
* fork's `copyPrefix`, a ref an `append` carries) cannot make the conditional delete refuse, so
|
|
96
|
+
* an orphan sweep concurrent with a fork or a delete can erase a blob a live log still cites
|
|
97
|
+
* (replay then throws `ContentMissingError`). This is a deletion safety boundary: it refuses
|
|
98
|
+
* rather than documenting the hazard. A `dryRun` is unaffected.
|
|
99
|
+
*/
|
|
100
|
+
class ReclaimOrphansTouchError extends TypeError {
|
|
101
|
+
constructor() {
|
|
102
|
+
super("sweepUnpinnedContent: reclaimOrphans needs a content store with touch() (ContentStore SPI 1.4.0) to ERASE. Without it a fork (copyPrefix) or an append that carries an existing ref cannot mark the blob re-cited, so an orphan sweep racing a fork or a delete can erase a blob a live log still cites. Add touch() to the store (SqliteContentStore and InMemoryContentStore have it; a CachingContentStore forwards it), or run with dryRun: true.");
|
|
103
|
+
this.name = "ReclaimOrphansTouchError";
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
exports.ReclaimOrphansTouchError = ReclaimOrphansTouchError;
|
|
107
|
+
/**
|
|
108
|
+
* The smallest margin, in ms, between the content store's clock and the guard cutoff that an
|
|
109
|
+
* ERASING orphan pass accepts (ADR 0055 D6 amendment 2). A write's touch-to-commit window must
|
|
110
|
+
* be shorter than the margin for the touch to protect it; 60 s is far above an append and
|
|
111
|
+
* above a fork's prefix copy at any realistic size, and an operator who needs a bigger one
|
|
112
|
+
* simply passes a larger TTL.
|
|
113
|
+
*/
|
|
114
|
+
exports.MIN_ORPHAN_ERASE_MARGIN_MS = 60_000;
|
|
115
|
+
/**
|
|
116
|
+
* An ERASING `reclaimOrphans` pass was configured with a zero/negative `olderThanMs`, or a
|
|
117
|
+
* cutoff (`touchedBefore`) closer to the content store's clock than
|
|
118
|
+
* {@link MIN_ORPHAN_ERASE_MARGIN_MS}. Nothing was read or erased. With no margin the
|
|
119
|
+
* touch-before-write has no window to protect: a write can touch, the sweep read a later cutoff,
|
|
120
|
+
* and the write commit after the scan, so a blob a live log cites is erased.
|
|
121
|
+
*/
|
|
122
|
+
class ReclaimOrphansMarginError extends TypeError {
|
|
123
|
+
marginMs;
|
|
124
|
+
constructor(marginMs) {
|
|
125
|
+
super(`sweepUnpinnedContent: an erasing reclaimOrphans pass needs a guard margin of at least ${exports.MIN_ORPHAN_ERASE_MARGIN_MS}ms between the content store's clock and the cutoff (received ${marginMs}ms: olderThanMs, or touchedBefore's distance from the store's clock). The touch that protects a fork or an append only covers a write whose touch-to-commit window is shorter than that margin; with olderThanMs <= 0 the window is unbounded. Raise olderThanMs / lower touchedBefore, or run with dryRun: true.`);
|
|
126
|
+
this.marginMs = marginMs;
|
|
127
|
+
this.name = "ReclaimOrphansMarginError";
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
exports.ReclaimOrphansMarginError = ReclaimOrphansMarginError;
|
|
131
|
+
/**
|
|
132
|
+
* An ERASING `reclaimOrphans` pass over a content store with `deleteIfUntouchedSince` but no
|
|
133
|
+
* `touchClock()`. Nothing was read or erased. Without the store's clock the sweep cannot measure
|
|
134
|
+
* the margin between "now" and the cutoff (`touchedBefore` is an unchecked caller value, so
|
|
135
|
+
* `touchedBefore: Date.now()` would pass any `olderThanMs`), and the touch-to-commit window is
|
|
136
|
+
* then unbounded. A `dryRun` is exempt.
|
|
137
|
+
*/
|
|
138
|
+
class ReclaimOrphansClockError extends TypeError {
|
|
139
|
+
constructor() {
|
|
140
|
+
super("sweepUnpinnedContent: an erasing reclaimOrphans pass needs a content store with touchClock() so the guard margin can be measured (a store with deleteIfUntouchedSince but no touchClock cannot prove touchedBefore is far enough behind its clock). Add touchClock() to the store, or run with dryRun: true.");
|
|
141
|
+
this.name = "ReclaimOrphansClockError";
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
exports.ReclaimOrphansClockError = ReclaimOrphansClockError;
|
|
145
|
+
let warnedUnwrapped = false;
|
|
146
|
+
/** Test-only: re-arm the once-per-process warning below. Not exported from the package index. */
|
|
147
|
+
function resetSnapshotUnwrappedWarning() {
|
|
148
|
+
warnedUnwrapped = false;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* One process warning for a MIXED set of snapshot stores, ONCE per process (like
|
|
152
|
+
* `LOOM_CONTENT_TOUCH`): a scheduled sweep over the same mixed set would otherwise warn every run.
|
|
153
|
+
* Never throws (best effort, advisory).
|
|
154
|
+
*/
|
|
155
|
+
function emitUnwrappedWarning(touching, unbranded) {
|
|
156
|
+
if (warnedUnwrapped)
|
|
157
|
+
return;
|
|
158
|
+
warnedUnwrapped = true;
|
|
159
|
+
const proc = globalThis
|
|
160
|
+
.process;
|
|
161
|
+
try {
|
|
162
|
+
proc?.emitWarning?.(`sweepUnpinnedContent: ${unbranded} of ${touching + unbranded} snapshot stores are not a ContentTouchingSnapshotStore while ${touching} are. If an unbranded one is a store the runtime SAVES to, a fork of a compacted parent can lose blobs to this erasing orphan pass; if it is only a raw reader over a wrapped writer's tables, ignore this.`, { code: "LOOM_CONTENT_SNAPSHOT_UNWRAPPED" });
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
// advisory only
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
/** Orphan-pass page size (ADR 0055 D6): bounded memory per page. */
|
|
169
|
+
const ORPHAN_PAGE_LIMIT = 500;
|
|
170
|
+
/**
|
|
171
|
+
* An ERASING `reclaimOrphans` pass found compacted logs but was given no `archiveStores`: the events
|
|
172
|
+
* a compaction moved are cited only by the archive, so their blobs would look like orphans and be
|
|
173
|
+
* erased (ADR 0138). Nothing was erased.
|
|
174
|
+
*/
|
|
175
|
+
class ArchiveStoresRequiredError extends TypeError {
|
|
176
|
+
executionIds;
|
|
177
|
+
reason;
|
|
178
|
+
constructor(executionIds,
|
|
179
|
+
/** `"none"`: no archiveStores given. `"uncovered"`: given, but they do not hold what was archived. */
|
|
180
|
+
reason = "none") {
|
|
181
|
+
const shown = executionIds.slice(0, 5).join(", ");
|
|
182
|
+
const more = executionIds.length > 5 ? ` (+${executionIds.length - 5} more)` : "";
|
|
183
|
+
super(reason === "none"
|
|
184
|
+
? `sweepUnpinnedContent: ${executionIds.length} scanned log(s) are compacted (${shown}${more}) and no archiveStores were given. Compaction MOVES the removed events into the compaction archive, which is then the only place that cites their blobs; an erasing orphan pass without it deletes blobs the recoverable history still names. Pass archiveStores: [the compaction archive] (the sealing store when encrypted), or acknowledgeUnarchivedLogs: true if no archive exists (logs compacted before ADR 0138), or run with dryRun: true. Nothing was erased.`
|
|
185
|
+
: `sweepUnpinnedContent: ${executionIds.length} scanned log(s) are compacted but the given archiveStores do not cover the removed range (${shown}${more}): the archive passed is empty, the wrong one, or was wiped. An erasing orphan pass would delete blobs the lost history still names. Pass the real compaction archive (the sealing store when encrypted), restore it, run with dryRun: true, or pass acknowledgeUnarchivedLogs: true if that history is already gone (pre-ADR-0138 logs, fork children). Nothing was erased.`);
|
|
186
|
+
this.executionIds = executionIds;
|
|
187
|
+
this.reason = reason;
|
|
188
|
+
this.name = "ArchiveStoresRequiredError";
|
|
189
|
+
Object.setPrototypeOf(this, ArchiveStoresRequiredError.prototype);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
exports.ArchiveStoresRequiredError = ArchiveStoresRequiredError;
|
|
193
|
+
/**
|
|
194
|
+
* An erasing sweep read an archive segment that is still sealed (count > 0, no envelopes): the RAW
|
|
195
|
+
* inner store of an `EncryptingArchiveStore`. The refs inside are invisible, so live blobs would
|
|
196
|
+
* look unreferenced. Pass the sealing store itself. Nothing was erased.
|
|
197
|
+
*/
|
|
198
|
+
class SealedArchiveScanError extends TypeError {
|
|
199
|
+
executionIds;
|
|
200
|
+
constructor(executionIds) {
|
|
201
|
+
const shown = executionIds.slice(0, 5).join(", ");
|
|
202
|
+
const more = executionIds.length > 5 ? ` (+${executionIds.length - 5} more)` : "";
|
|
203
|
+
super(`sweepUnpinnedContent: ${executionIds.length} archived execution(s) hold sealed segments (${shown}${more}), so the refs they cite are invisible and an erasing pass would delete live blobs: pass the opening archive store (the EncryptingArchiveStore itself, not its inner store) in archiveStores, or run with dryRun: true. Nothing was erased.`);
|
|
204
|
+
this.executionIds = executionIds;
|
|
205
|
+
this.name = "SealedArchiveScanError";
|
|
206
|
+
Object.setPrototypeOf(this, SealedArchiveScanError.prototype);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
exports.SealedArchiveScanError = SealedArchiveScanError;
|
|
210
|
+
/**
|
|
211
|
+
* An erasing sweep read a snapshot whose `state` is one sealed envelope (a `sealState`
|
|
212
|
+
* `EncryptingSnapshotStore` given as its RAW inner store). The scan cannot see the refs inside, so
|
|
213
|
+
* it would treat live blobs as unreferenced and erase them. Nothing was erased.
|
|
214
|
+
*/
|
|
215
|
+
class SealedSnapshotScanError extends TypeError {
|
|
216
|
+
executionIds;
|
|
217
|
+
constructor(executionIds) {
|
|
218
|
+
const shown = executionIds.slice(0, 5).join(", ");
|
|
219
|
+
const more = executionIds.length > 5 ? ` (+${executionIds.length - 5} more)` : "";
|
|
220
|
+
super(`sweepUnpinnedContent: ${executionIds.length} scanned snapshot(s) hold a sealed (whole-state encrypted) state (${shown}${more}), so the refs they cite are invisible and an erasing pass would delete live blobs: pass the decrypting snapshot store (the EncryptingSnapshotStore itself, not its inner store) in snapshotStores, or run with dryRun: true. Nothing was erased.`);
|
|
221
|
+
this.executionIds = executionIds;
|
|
222
|
+
this.name = "SealedSnapshotScanError";
|
|
223
|
+
Object.setPrototypeOf(this, SealedSnapshotScanError.prototype);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
exports.SealedSnapshotScanError = SealedSnapshotScanError;
|
|
227
|
+
/**
|
|
228
|
+
* Mark: walk every event of every execution across the given stores, collecting
|
|
229
|
+
* each ref's latest citation timestamp, pinning refs cited from events of the
|
|
230
|
+
* designated kinds, and marking refs cited anywhere other than as a
|
|
231
|
+
* claim-check envelope's `ref` (a `digest` included) as load-bearing. Sweep:
|
|
232
|
+
* physically erase refs that are neither pinned nor load-bearing and whose
|
|
233
|
+
* effective citation time is older than the TTL — i.e. only refs cited solely as an
|
|
234
|
+
* envelope's `ref`. ADR 0146: a citing execution that has ENDED (newest kernel terminal event, not
|
|
235
|
+
* re-opened) ages its citations from `max(citation, end)`, so with a content TTL at least the log TTL a blob is
|
|
236
|
+
* never reclaimable before retention could delete the log citing it (exceptions: ADR 0146); a
|
|
237
|
+
* running execution (or a reserved stream) keeps the citation clock. Never earlier than the citation. Their erasure is invisible to replay by the claim-check
|
|
238
|
+
* contract (ADR 0054 — the fold never resolves); `resolveClaimCheck` readers get
|
|
239
|
+
* `null`. A bare ref is never reclaimed while cited (ADR 0055 D4): the resolving
|
|
240
|
+
* store would throw `ContentMissingError` on the next read. Idempotent:
|
|
241
|
+
* `ContentStore.delete` is idempotent and a reclaimed ref simply no longer
|
|
242
|
+
* resolves on the next sweep.
|
|
243
|
+
*
|
|
244
|
+
* Race-safe (ADR 0055 D5) when the store has `deleteIfUntouchedSince`: every erase is
|
|
245
|
+
* conditional on the blob being untouched since `now − olderThanMs`, so a `put` that re-cites
|
|
246
|
+
* old bytes between the scan and its own append keeps the blob. That holds while a flow's
|
|
247
|
+
* put→append window is shorter than the TTL. A citation copied without a `put` (a fork's
|
|
248
|
+
* `copyPrefix`, a ref kept inside an `append` payload) is covered by SPI 1.4.0 `touch`, which
|
|
249
|
+
* `reclaimOrphans` requires; see `reclaimOrphans`. `roots` (D7) pins addresses held outside the
|
|
250
|
+
* log; `reclaimOrphans` (D6) also erases stored blobs nothing scanned cites. Cost:
|
|
251
|
+
* O(events + blobs), all membership tests are Set/Map lookups.
|
|
252
|
+
*/
|
|
253
|
+
async function sweepUnpinnedContent(given) {
|
|
254
|
+
const hasTtl = given.olderThanMs !== undefined;
|
|
255
|
+
const hasPolicy = given.policy !== undefined;
|
|
256
|
+
if (hasTtl === hasPolicy) {
|
|
257
|
+
throw new TypeError(hasTtl
|
|
258
|
+
? "sweepUnpinnedContent: pass olderThanMs OR policy, not both - they name the same TTL"
|
|
259
|
+
: "sweepUnpinnedContent: needs a TTL - pass olderThanMs, or policy (a retention policy with a content.unpinnedAfterMs section)");
|
|
260
|
+
}
|
|
261
|
+
let olderThanMs = given.olderThanMs;
|
|
262
|
+
if (hasPolicy)
|
|
263
|
+
olderThanMs = resolvePolicyTtl(given.policy);
|
|
264
|
+
const { policy: _policy, ...rest } = given;
|
|
265
|
+
return sweepResolved({ ...rest, olderThanMs });
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* ADR 0145 D3 - structural re-validation of a retention policy, WITHOUT a dependency on
|
|
269
|
+
* `@loomwork/workflow-runtime`. Each field is read ONCE into a plain snapshot, then the snapshot is
|
|
270
|
+
* checked, so a getter answering differently on a second read cannot pass a rule-breaking value.
|
|
271
|
+
* Checks what the sweep consumes: unknown keys (top level and `content`), `unpinnedAfterMs` a
|
|
272
|
+
* finite integer > 0, and, when `terminalTtlMs` is present, a finite integer >= 0 with
|
|
273
|
+
* `unpinnedAfterMs >= terminalTtlMs`. `snapshots` is allowed and not inspected (the sweep reads
|
|
274
|
+
* nothing from it; `LogLifecycle` validates it).
|
|
275
|
+
*/
|
|
276
|
+
function resolvePolicyTtl(policy) {
|
|
277
|
+
const bad = (m) => {
|
|
278
|
+
throw new TypeError(`sweepUnpinnedContent: policy ${m}`);
|
|
279
|
+
};
|
|
280
|
+
if (typeof policy !== "object" || policy === null || Array.isArray(policy)) {
|
|
281
|
+
return bad("must be a retention policy object");
|
|
282
|
+
}
|
|
283
|
+
for (const k of Object.keys(policy)) {
|
|
284
|
+
if (k !== "terminalTtlMs" && k !== "snapshots" && k !== "content") {
|
|
285
|
+
bad(`has unknown key "${k}" (expected terminalTtlMs, snapshots, content)`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
const o = policy;
|
|
289
|
+
const ttl = o.terminalTtlMs;
|
|
290
|
+
const content = o.content;
|
|
291
|
+
if (ttl !== undefined && (typeof ttl !== "number" || !Number.isInteger(ttl) || ttl < 0)) {
|
|
292
|
+
bad(`terminalTtlMs must be a finite integer >= 0 (milliseconds); got ${String(ttl)}`);
|
|
293
|
+
}
|
|
294
|
+
if (content === undefined) {
|
|
295
|
+
return bad("has no content section - add content.unpinnedAfterMs to the retention policy, or pass olderThanMs");
|
|
296
|
+
}
|
|
297
|
+
if (typeof content !== "object" || content === null || Array.isArray(content)) {
|
|
298
|
+
return bad("content must be an object with unpinnedAfterMs");
|
|
299
|
+
}
|
|
300
|
+
for (const k of Object.keys(content)) {
|
|
301
|
+
if (k !== "unpinnedAfterMs") {
|
|
302
|
+
bad(`has unknown key "content.${k}" (expected unpinnedAfterMs)`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
const u = content.unpinnedAfterMs;
|
|
306
|
+
if (typeof u !== "number" || !Number.isInteger(u) || u <= 0) {
|
|
307
|
+
return bad(`content.unpinnedAfterMs must be a finite integer > 0 (milliseconds); got ${String(u)}`);
|
|
308
|
+
}
|
|
309
|
+
if (typeof ttl === "number" && u < ttl) {
|
|
310
|
+
bad(`content.unpinnedAfterMs (${u}) must be >= terminalTtlMs (${ttl})`);
|
|
311
|
+
}
|
|
312
|
+
return u;
|
|
313
|
+
}
|
|
314
|
+
async function sweepResolved(options) {
|
|
315
|
+
const store = options.contentStore;
|
|
316
|
+
const requested = options.reclaimOrphans;
|
|
317
|
+
if (requested !== undefined && requested !== false) {
|
|
318
|
+
// A dry run erases nothing, so it needs no acknowledgement (any truthy value turns the
|
|
319
|
+
// orphan pass on); the sweep that would ERASE must carry it.
|
|
320
|
+
if (options.dryRun === true
|
|
321
|
+
? !requested
|
|
322
|
+
: typeof requested !== "object" ||
|
|
323
|
+
requested === null ||
|
|
324
|
+
requested.completeScan !== true) {
|
|
325
|
+
throw new ReclaimOrphansAckError(requested);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
const orphans = requested !== undefined && requested !== false;
|
|
329
|
+
if (orphans && store.listPage === undefined) {
|
|
330
|
+
throw new TypeError("sweepUnpinnedContent: reclaimOrphans needs a content store with listPage() (ContentStore SPI 1.3.0) — orphans cannot be found without enumerating the store");
|
|
331
|
+
}
|
|
332
|
+
if (orphans && store.deleteIfUntouchedSince === undefined) {
|
|
333
|
+
throw new TypeError("sweepUnpinnedContent: reclaimOrphans needs a content store with deleteIfUntouchedSince() (ContentStore SPI 1.3.0) — a plain delete of a listed orphan loses a blob re-put since the listing");
|
|
334
|
+
}
|
|
335
|
+
if (orphans && options.dryRun !== true && store.touch === undefined) {
|
|
336
|
+
throw new ReclaimOrphansTouchError();
|
|
337
|
+
}
|
|
338
|
+
if (orphans &&
|
|
339
|
+
options.dryRun !== true &&
|
|
340
|
+
options.snapshotStores !== undefined &&
|
|
341
|
+
requested.snapshotsTouchContent !== true) {
|
|
342
|
+
throw new ReclaimOrphansSnapshotAckError();
|
|
343
|
+
}
|
|
344
|
+
if (orphans && options.dryRun !== true && store.touchClock === undefined) {
|
|
345
|
+
throw new ReclaimOrphansClockError();
|
|
346
|
+
}
|
|
347
|
+
if (orphans &&
|
|
348
|
+
options.snapshotStores === undefined &&
|
|
349
|
+
options.eventStores.some((s) => typeof s.truncatePrefix === "function")) {
|
|
350
|
+
throw new TypeError("sweepUnpinnedContent: reclaimOrphans over an event store that can truncatePrefix needs snapshotStores — a compacted prefix's refs live only in its snapshot, and without it they look like orphans");
|
|
351
|
+
}
|
|
352
|
+
const snapshotList = options.snapshotStores ?? [];
|
|
353
|
+
const snapshotStoresTouching = snapshotList.filter(ContentTouchingSnapshotStore_1.isContentTouchingSnapshotStore).length;
|
|
354
|
+
const snapshotStoresUnbranded = snapshotList.length - snapshotStoresTouching;
|
|
355
|
+
if (orphans &&
|
|
356
|
+
options.dryRun !== true &&
|
|
357
|
+
snapshotStoresTouching > 0 &&
|
|
358
|
+
snapshotStoresUnbranded > 0) {
|
|
359
|
+
emitUnwrappedWarning(snapshotStoresTouching, snapshotStoresUnbranded);
|
|
360
|
+
}
|
|
361
|
+
// ADR 0055 D5 — the guard cutoff is on the STORE's clock, read once at the START: a blob
|
|
362
|
+
// touched after it (a re-citing put the scan could not see yet) is refused. The TTL on the
|
|
363
|
+
// log (`now`) is a different clock and never feeds it.
|
|
364
|
+
const raceSafe = store.deleteIfUntouchedSince !== undefined;
|
|
365
|
+
let cutoff = Number.NEGATIVE_INFINITY;
|
|
366
|
+
// The store's clock as read ONCE for this sweep: the cutoff AND the orphan margin below are
|
|
367
|
+
// measured against the same reading. A second read made the margin `olderThanMs` plus however
|
|
368
|
+
// many milliseconds elapsed between the reads, so a sub-minute TTL could pass the guard.
|
|
369
|
+
let storeNow;
|
|
370
|
+
if (raceSafe) {
|
|
371
|
+
if (options.touchedBefore !== undefined) {
|
|
372
|
+
cutoff = options.touchedBefore;
|
|
373
|
+
}
|
|
374
|
+
else if (store.touchClock !== undefined) {
|
|
375
|
+
storeNow = await store.touchClock();
|
|
376
|
+
cutoff = storeNow - options.olderThanMs;
|
|
377
|
+
}
|
|
378
|
+
else {
|
|
379
|
+
throw new TypeError("sweepUnpinnedContent: the content store has deleteIfUntouchedSince() but no touchClock() — pass touchedBefore (a cutoff on the store's clock) explicitly");
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
const dryRun = options.dryRun === true;
|
|
383
|
+
if (orphans && !dryRun) {
|
|
384
|
+
// The touch protects a write only while its touch-to-commit window is shorter than the
|
|
385
|
+
// margin between the store's clock and the cutoff (module note; ADR 0055 D6 amendment 2).
|
|
386
|
+
if (!(options.olderThanMs > 0))
|
|
387
|
+
throw new ReclaimOrphansMarginError(options.olderThanMs);
|
|
388
|
+
const margin = (storeNow ?? (await store.touchClock())) - cutoff;
|
|
389
|
+
if (!(margin >= exports.MIN_ORPHAN_ERASE_MARGIN_MS))
|
|
390
|
+
throw new ReclaimOrphansMarginError(margin);
|
|
391
|
+
}
|
|
392
|
+
/** Erase one address; `true` iff it is gone because of this sweep (or would be). */
|
|
393
|
+
let keptByTouch = 0;
|
|
394
|
+
const erase = async (address) => {
|
|
395
|
+
if (dryRun)
|
|
396
|
+
return true;
|
|
397
|
+
const ref = { __loomContent: address };
|
|
398
|
+
if (store.deleteIfUntouchedSince !== undefined) {
|
|
399
|
+
const gone = await store.deleteIfUntouchedSince(ref, cutoff);
|
|
400
|
+
if (!gone)
|
|
401
|
+
keptByTouch++;
|
|
402
|
+
return gone;
|
|
403
|
+
}
|
|
404
|
+
await store.delete(ref);
|
|
405
|
+
return true;
|
|
406
|
+
};
|
|
407
|
+
// Mark memory is O(distinct cited addresses): one Map entry per address plus the Sets.
|
|
408
|
+
const pinnedKinds = new Set(options.pinnedEventTypes);
|
|
409
|
+
// ADR 0146: the EFFECTIVE time per ref (end clock). Dry run alone also keeps the raw citation time.
|
|
410
|
+
const lastCited = new Map();
|
|
411
|
+
const rawCited = options.dryRun === true ? new Map() : undefined;
|
|
412
|
+
const viaEnd = new Set();
|
|
413
|
+
// Refs whose newest citation came from a RE-OPENED execution (same tie rule as `viaEnd`).
|
|
414
|
+
const viaReopened = new Set();
|
|
415
|
+
const pinned = new Set();
|
|
416
|
+
const loadBearing = new Set();
|
|
417
|
+
let executionsScanned = 0;
|
|
418
|
+
const uncovered = [];
|
|
419
|
+
const sealedSnapshots = [];
|
|
420
|
+
const shreddedSnapshots = [];
|
|
421
|
+
const compactedLogs = new Set();
|
|
422
|
+
// EVERY compacted log's removed range (the `ckpt-0138-` mark is NOT consulted: it can be lost to
|
|
423
|
+
// pruning or a fork seed, and a heuristic must not gate an ERASE): each must be covered by the
|
|
424
|
+
// given archives' headers, else refused unless `acknowledgeUnarchivedLogs`.
|
|
425
|
+
const archivedRanges = new Map();
|
|
426
|
+
const sealedArchives = [];
|
|
427
|
+
for (const eventStore of options.eventStores) {
|
|
428
|
+
if (eventStore.listExecutions === undefined && eventStore.listExecutionsPage === undefined) {
|
|
429
|
+
throw new TypeError("sweepUnpinnedContent: every event store must support listExecutionsPage() or listExecutions() — a store that cannot be enumerated cannot be swept safely");
|
|
430
|
+
}
|
|
431
|
+
for await (const executionId of (0, event_runtime_1.iterateExecutionIds)(eventStore)) {
|
|
432
|
+
executionsScanned++;
|
|
433
|
+
const events = await eventStore.read(executionId);
|
|
434
|
+
// ADR 0146: a terminal execution's citations age from its END (never earlier than the citation).
|
|
435
|
+
const endedAt = (0, event_runtime_1.isReservedStream)(executionId) ? undefined : (0, executionEnd_1.terminalEndOf)(events);
|
|
436
|
+
const reopened = !(0, event_runtime_1.isReservedStream)(executionId) && endedAt === undefined && (0, executionEnd_1.wasReopened)(events);
|
|
437
|
+
for (const event of events) {
|
|
438
|
+
// Only a finite end strictly after the citation moves the clock; anything else is today's clock exactly.
|
|
439
|
+
const at = endedAt !== undefined && endedAt > event.timestamp ? endedAt : event.timestamp;
|
|
440
|
+
(0, citations_1.visitCitedAddresses)(event.payload, (address, inEnvelope) => {
|
|
441
|
+
const prior = lastCited.get(address);
|
|
442
|
+
if (prior === undefined || at > prior) {
|
|
443
|
+
lastCited.set(address, at);
|
|
444
|
+
if (at > event.timestamp)
|
|
445
|
+
viaEnd.add(address);
|
|
446
|
+
else
|
|
447
|
+
viaEnd.delete(address);
|
|
448
|
+
if (reopened)
|
|
449
|
+
viaReopened.add(address);
|
|
450
|
+
else
|
|
451
|
+
viaReopened.delete(address);
|
|
452
|
+
}
|
|
453
|
+
if (rawCited !== undefined) {
|
|
454
|
+
const rawPrior = rawCited.get(address);
|
|
455
|
+
if (rawPrior === undefined || event.timestamp > rawPrior) {
|
|
456
|
+
rawCited.set(address, event.timestamp);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
if (pinnedKinds.has(event.eventType))
|
|
460
|
+
pinned.add(address);
|
|
461
|
+
if (!inEnvelope)
|
|
462
|
+
loadBearing.add(address);
|
|
463
|
+
}, (0, citations_1.retainRootsOf)(event));
|
|
464
|
+
}
|
|
465
|
+
// Did compaction touch this log, and where is its checkpoint (ADR 0106)? Compaction
|
|
466
|
+
// removes its checkpoint's own slot and, whenever it keeps pinned-kind events, never
|
|
467
|
+
// the tip (the tip rule). So a gap-free log with more than the bootstrap was never
|
|
468
|
+
// compacted and costs no snapshot read; a gapped log's checkpoint sits at its highest
|
|
469
|
+
// missing sequence `gap`; and a bootstrap-only log was compacted through its tip (or
|
|
470
|
+
// never got past its bootstrap), its checkpoint above the tip.
|
|
471
|
+
if (events.length === 0)
|
|
472
|
+
continue;
|
|
473
|
+
const tipSequence = events[events.length - 1].sequence;
|
|
474
|
+
// ADR 0139 - a RESERVED stream (the inbox, its topic streams, the quarantine copies) is never
|
|
475
|
+
// checkpointed: a trim MOVES its consumed prefix to the archive instead. Its removed range is
|
|
476
|
+
// what its sequences skip from its first possible one (a topic stream starts at
|
|
477
|
+
// INBOX_TOPIC_SEQ_BASE, not 0, so a bare "gap" test would call every topic stream compacted), and
|
|
478
|
+
// an erasing pass needs the archive to cover it, exactly as for a compacted log (D24).
|
|
479
|
+
if ((0, event_runtime_1.isReservedStream)(executionId)) {
|
|
480
|
+
const runs = missingRunsFrom(events, isInboxTopicStream(executionId) ? event_runtime_1.INBOX_TOPIC_SEQ_BASE : 1, tipSequence);
|
|
481
|
+
if (runs.length > 0) {
|
|
482
|
+
compactedLogs.add(executionId);
|
|
483
|
+
archivedRanges.set(executionId, runs);
|
|
484
|
+
}
|
|
485
|
+
continue;
|
|
486
|
+
}
|
|
487
|
+
const gap = highestMissingSequence(events);
|
|
488
|
+
const bootstrapOnly = events.length === 1;
|
|
489
|
+
if (gap < 0 && !bootstrapOnly)
|
|
490
|
+
continue;
|
|
491
|
+
// ADR 0138 - a gapped log is compacted: an erasing orphan pass needs its archive, else the
|
|
492
|
+
// moved events' blobs look unreferenced. (A bootstrap-only log counts once a checkpoint shows.)
|
|
493
|
+
if (gap >= 0) {
|
|
494
|
+
compactedLogs.add(executionId);
|
|
495
|
+
archivedRanges.set(executionId, missingRunsWithin(events, gap));
|
|
496
|
+
}
|
|
497
|
+
if (options.snapshotStores === undefined) {
|
|
498
|
+
if (gap >= 0)
|
|
499
|
+
uncovered.push(executionId); // a gap is log evidence
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
const pin = (snapshot) => {
|
|
503
|
+
const st = snapshot.state;
|
|
504
|
+
if (typeof st === "object" && st !== null && "__loomEnc" in st) {
|
|
505
|
+
if (!sealedSnapshots.includes(executionId))
|
|
506
|
+
sealedSnapshots.push(executionId);
|
|
507
|
+
}
|
|
508
|
+
(0, citations_1.visitCitedAddresses)(snapshot.state, (address) => {
|
|
509
|
+
if (!lastCited.has(address))
|
|
510
|
+
lastCited.set(address, Number.POSITIVE_INFINITY);
|
|
511
|
+
pinned.add(address);
|
|
512
|
+
});
|
|
513
|
+
};
|
|
514
|
+
let covered = false;
|
|
515
|
+
for (const snapshots of options.snapshotStores) {
|
|
516
|
+
try {
|
|
517
|
+
const latest = await snapshots.loadLatest(executionId);
|
|
518
|
+
if (latest !== null && latest.sequence >= 1)
|
|
519
|
+
pin(latest);
|
|
520
|
+
if (bootstrapOnly) {
|
|
521
|
+
// Compacted at the tip: the checkpoint IS the latest snapshot.
|
|
522
|
+
if (latest !== null && latest.sequence >= 1) {
|
|
523
|
+
covered = true;
|
|
524
|
+
compactedLogs.add(executionId);
|
|
525
|
+
archivedRanges.set(executionId, [[1, latest.sequence]]);
|
|
526
|
+
}
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
// The checkpoint: the LOWEST snapshot in [gap, tip] — any snapshot at or above the gap
|
|
530
|
+
// covers a lost checkpoint, exactly as `compact` accepts a base at or above it. A
|
|
531
|
+
// compaction's own sits exactly at `gap` (one read). A child forked from a compacted
|
|
532
|
+
// parent copies the gap, and its lowest snapshot is its fork seed, whose sequence the
|
|
533
|
+
// copied envelopes record (`metadata.forkSequence`): one exact read. Only a log with
|
|
534
|
+
// neither walks the snapshots down from the tip.
|
|
535
|
+
let checkpoint = await snapshots.loadLatestBefore(executionId, gap);
|
|
536
|
+
const forkSequence = events[0].metadata
|
|
537
|
+
?.forkSequence;
|
|
538
|
+
if ((checkpoint === null || checkpoint.sequence !== gap) &&
|
|
539
|
+
typeof forkSequence === "number" &&
|
|
540
|
+
forkSequence >= gap) {
|
|
541
|
+
const seed = snapshots.loadAt !== undefined
|
|
542
|
+
? await snapshots.loadAt(executionId, forkSequence)
|
|
543
|
+
: await snapshots.loadLatestBefore(executionId, forkSequence);
|
|
544
|
+
checkpoint = seed !== null && seed.sequence === forkSequence ? seed : checkpoint;
|
|
545
|
+
}
|
|
546
|
+
if (checkpoint === null || checkpoint.sequence < gap) {
|
|
547
|
+
checkpoint = null;
|
|
548
|
+
let s = await snapshots.loadLatestBefore(executionId, tipSequence);
|
|
549
|
+
while (s !== null && s.sequence >= gap) {
|
|
550
|
+
checkpoint = s;
|
|
551
|
+
s =
|
|
552
|
+
s.sequence > gap
|
|
553
|
+
? await snapshots.loadLatestBefore(executionId, s.sequence - 1)
|
|
554
|
+
: null;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
if (checkpoint !== null) {
|
|
558
|
+
pin(checkpoint);
|
|
559
|
+
covered = true;
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
catch (err) {
|
|
563
|
+
// Only a CONFIRMED crypto-shred means the snapshot is erased: the store's error (recognised by the realm-safe
|
|
564
|
+
// brand, never by name) AND the execution's own `PAYLOAD_SHREDDED`
|
|
565
|
+
// tombstone. A node that merely lacks the key can report the same error, so without the
|
|
566
|
+
// tombstone it propagates like anything else (undecryptable, I/O, a key unavailable with no
|
|
567
|
+
// tombstone): dropping refs for data that still exists would delete live blobs. A
|
|
568
|
+
// snapshot store without the raw log reports a shredded subject's PRE-shred snapshot as
|
|
569
|
+
// `KeyUnavailableError` (it cannot see the tombstone above it), so that name is accepted
|
|
570
|
+
// too, but ONLY beside the tombstone.
|
|
571
|
+
const tombstoned = events.some((e) => e.eventType === "PAYLOAD_SHREDDED");
|
|
572
|
+
if (!(err instanceof Error) ||
|
|
573
|
+
(!(0, event_runtime_1.isPayloadShredded)(err) && err.name !== "KeyUnavailableError") ||
|
|
574
|
+
!tombstoned)
|
|
575
|
+
throw err;
|
|
576
|
+
if (!shreddedSnapshots.includes(executionId))
|
|
577
|
+
shreddedSnapshots.push(executionId);
|
|
578
|
+
covered = true;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
if (!covered)
|
|
582
|
+
uncovered.push(executionId);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
// ADR 0138 - what compaction MOVED out of the logs still cites content: pin it permanently. The
|
|
586
|
+
// archive is enumerated (not walked per log), so segments of an execution whose log is gone count.
|
|
587
|
+
const anyLog = async (executionId) => {
|
|
588
|
+
const all = [];
|
|
589
|
+
for (const es of options.eventStores)
|
|
590
|
+
all.push(...(await es.read(executionId)));
|
|
591
|
+
return all;
|
|
592
|
+
};
|
|
593
|
+
for (const archive of options.archiveStores ?? []) {
|
|
594
|
+
let cursor;
|
|
595
|
+
do {
|
|
596
|
+
const page = await archive.listExecutions({ ...(cursor !== undefined && { cursor }) });
|
|
597
|
+
for (const executionId of page.executionIds) {
|
|
598
|
+
// ADR 0139 - a reserved stream that is DELETED (nothing live to derive a gap from) is judged by
|
|
599
|
+
// the archive's own extent: a hole between its first possible sequence and its last archived one
|
|
600
|
+
// (a trim before the archive existed) is history nothing holds, exactly like a live stream's gap.
|
|
601
|
+
if ((0, event_runtime_1.isReservedStream)(executionId) && !executionId.startsWith("__loom_late__:")) {
|
|
602
|
+
if (!archivedRanges.has(executionId)) {
|
|
603
|
+
const headers = await archive.headers(executionId);
|
|
604
|
+
const first = isInboxTopicStream(executionId) ? event_runtime_1.INBOX_TOPIC_SEQ_BASE : 1;
|
|
605
|
+
const extent = headers.length > 0 ? headers[headers.length - 1].through : -1;
|
|
606
|
+
const holes = missingRunsFrom(headers.flatMap((h) => Array.from(new Set([h.from, h.through])).map((sequence) => ({ sequence }))), first, extent).filter(([x, y]) => !rangeCovered(x, y, headers.map((h) => [h.from, h.through])));
|
|
607
|
+
if (holes.length > 0) {
|
|
608
|
+
compactedLogs.add(executionId);
|
|
609
|
+
archivedRanges.set(executionId, holes);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
let segments;
|
|
614
|
+
try {
|
|
615
|
+
segments = await archive.read(executionId);
|
|
616
|
+
}
|
|
617
|
+
catch (err) {
|
|
618
|
+
// Only a CONFIRMED shred (the realm-safe brand AND the execution's own tombstone) means the
|
|
619
|
+
// archived copy is erased; anything else (a missing key on this node, I/O) propagates.
|
|
620
|
+
if ((0, event_runtime_1.isPayloadShredded)(err) &&
|
|
621
|
+
(await anyLog(executionId)).some((e) => e.eventType === "PAYLOAD_SHREDDED")) {
|
|
622
|
+
continue;
|
|
623
|
+
}
|
|
624
|
+
throw err;
|
|
625
|
+
}
|
|
626
|
+
for (const seg of segments) {
|
|
627
|
+
if (seg.count > 0 && seg.envelopes.length === 0) {
|
|
628
|
+
if (!sealedArchives.includes(executionId))
|
|
629
|
+
sealedArchives.push(executionId);
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
for (const event of seg.envelopes) {
|
|
633
|
+
(0, citations_1.visitCitedAddresses)(event.payload, (address) => {
|
|
634
|
+
const prior = lastCited.get(address);
|
|
635
|
+
if (prior === undefined || event.timestamp > prior) {
|
|
636
|
+
lastCited.set(address, event.timestamp);
|
|
637
|
+
viaEnd.delete(address);
|
|
638
|
+
viaReopened.delete(address);
|
|
639
|
+
}
|
|
640
|
+
if (rawCited !== undefined) {
|
|
641
|
+
const rawPrior = rawCited.get(address);
|
|
642
|
+
if (rawPrior === undefined || event.timestamp > rawPrior) {
|
|
643
|
+
rawCited.set(address, event.timestamp);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
pinned.add(address);
|
|
647
|
+
loadBearing.add(address);
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
cursor = page.nextCursor;
|
|
653
|
+
} while (cursor !== undefined);
|
|
654
|
+
}
|
|
655
|
+
if (!dryRun && sealedArchives.length > 0)
|
|
656
|
+
throw new SealedArchiveScanError(sealedArchives);
|
|
657
|
+
const unarchivedAcknowledged = [];
|
|
658
|
+
if (orphans && !dryRun && options.archiveStores === undefined && compactedLogs.size > 0) {
|
|
659
|
+
if (options.acknowledgeUnarchivedLogs !== true) {
|
|
660
|
+
throw new ArchiveStoresRequiredError([...compactedLogs]);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
// Coverage is CHECKED for every compacted log, whatever its checkpoint is called: `[]` or a
|
|
664
|
+
// partial archive never suffices for a gapped log; only the headers (or the explicit
|
|
665
|
+
// acknowledgment, for logs whose history is already gone) get past it.
|
|
666
|
+
if (orphans && !dryRun && archivedRanges.size > 0) {
|
|
667
|
+
const lost = [];
|
|
668
|
+
for (const [executionId, runs] of archivedRanges) {
|
|
669
|
+
const held = [];
|
|
670
|
+
for (const archive of options.archiveStores ?? []) {
|
|
671
|
+
for (const h of await archive.headers(executionId))
|
|
672
|
+
held.push([h.from, h.through]);
|
|
673
|
+
}
|
|
674
|
+
if (!runs.every(([a, b]) => rangeCovered(a, b, held)))
|
|
675
|
+
lost.push(executionId);
|
|
676
|
+
}
|
|
677
|
+
if (lost.length > 0) {
|
|
678
|
+
if (options.acknowledgeUnarchivedLogs !== true) {
|
|
679
|
+
throw new ArchiveStoresRequiredError(lost, "uncovered");
|
|
680
|
+
}
|
|
681
|
+
unarchivedAcknowledged.push(...lost);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
if (!dryRun && sealedSnapshots.length > 0)
|
|
685
|
+
throw new SealedSnapshotScanError(sealedSnapshots);
|
|
686
|
+
if (orphans && uncovered.length > 0) {
|
|
687
|
+
const shown = uncovered.slice(0, 5).join(", ");
|
|
688
|
+
const more = uncovered.length > 5 ? ` (+${uncovered.length - 5} more)` : "";
|
|
689
|
+
throw new TypeError(`sweepUnpinnedContent: ${uncovered.length} execution(s) look compacted but no checkpoint in snapshotStores covers them (${shown}${more}) — their prefix refs cannot be seen, so reclaimOrphans refuses before erasing anything. A run that never got past its bootstrap looks the same: let it progress, or pass the snapshot store its checkpoint went to (each archive's too)`);
|
|
690
|
+
}
|
|
691
|
+
// ADR 0055 D7 — external root sets pin like a pinned event kind does.
|
|
692
|
+
const roots = new Set();
|
|
693
|
+
if (options.roots !== undefined) {
|
|
694
|
+
for await (const address of options.roots())
|
|
695
|
+
roots.add(address);
|
|
696
|
+
}
|
|
697
|
+
const reclaimed = [];
|
|
698
|
+
const heldByEndClock = [];
|
|
699
|
+
const reclaimedWhileReopened = [];
|
|
700
|
+
let refsAgedByEnd = 0;
|
|
701
|
+
let refsOfReopened = 0;
|
|
702
|
+
for (const [address, citedAt] of lastCited) {
|
|
703
|
+
if (pinned.has(address) || loadBearing.has(address) || roots.has(address))
|
|
704
|
+
continue;
|
|
705
|
+
if (viaEnd.has(address))
|
|
706
|
+
refsAgedByEnd++;
|
|
707
|
+
const ofReopened = viaReopened.has(address);
|
|
708
|
+
if (ofReopened)
|
|
709
|
+
refsOfReopened++;
|
|
710
|
+
if (options.now - citedAt < options.olderThanMs) {
|
|
711
|
+
const raw = rawCited?.get(address);
|
|
712
|
+
if (raw !== undefined && options.now - raw >= options.olderThanMs) {
|
|
713
|
+
heldByEndClock.push(address);
|
|
714
|
+
}
|
|
715
|
+
continue;
|
|
716
|
+
}
|
|
717
|
+
if (await erase(address)) {
|
|
718
|
+
reclaimed.push(address);
|
|
719
|
+
if (ofReopened && dryRun)
|
|
720
|
+
reclaimedWhileReopened.push(address);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
// ADR 0055 D6 — orphans: stored, untouched since the cutoff, and cited by nothing scanned.
|
|
724
|
+
// `pinned` and `loadBearing` are subsets of `lastCited`, so two Set lookups decide.
|
|
725
|
+
// Keyset paging on address: erasing entries of a page never skips a later one.
|
|
726
|
+
const orphansReclaimed = [];
|
|
727
|
+
if (orphans) {
|
|
728
|
+
let cursor;
|
|
729
|
+
do {
|
|
730
|
+
const page = await store.listPage({
|
|
731
|
+
...(cursor !== undefined && { cursor }),
|
|
732
|
+
limit: ORPHAN_PAGE_LIMIT,
|
|
733
|
+
touchedBefore: cutoff,
|
|
734
|
+
});
|
|
735
|
+
for (const { address, touchedAt } of page.entries) {
|
|
736
|
+
// 0 = UNKNOWN: the column default a pre-1.3.0 writer leaves during a mixed rollout.
|
|
737
|
+
if (!(touchedAt > 0))
|
|
738
|
+
continue;
|
|
739
|
+
if (lastCited.has(address) || roots.has(address))
|
|
740
|
+
continue;
|
|
741
|
+
if (await erase(address))
|
|
742
|
+
orphansReclaimed.push(address);
|
|
743
|
+
}
|
|
744
|
+
cursor = page.nextCursor;
|
|
745
|
+
} while (cursor !== undefined);
|
|
746
|
+
}
|
|
747
|
+
return {
|
|
748
|
+
executionsScanned,
|
|
749
|
+
refsSeen: lastCited.size,
|
|
750
|
+
refsPinned: pinned.size,
|
|
751
|
+
refsLoadBearing: loadBearing.size,
|
|
752
|
+
refsAgedByEnd,
|
|
753
|
+
...(dryRun && { heldByEndClock }),
|
|
754
|
+
refsOfReopened,
|
|
755
|
+
...(dryRun && { reclaimedWhileReopened }),
|
|
756
|
+
reclaimed,
|
|
757
|
+
keptByTouch,
|
|
758
|
+
...(raceSafe && { touchCutoff: cutoff }),
|
|
759
|
+
raceSafe,
|
|
760
|
+
orphansReclaimed,
|
|
761
|
+
snapshotStoresTouching,
|
|
762
|
+
snapshotStoresUnbranded,
|
|
763
|
+
sealedSnapshots,
|
|
764
|
+
shreddedSnapshots,
|
|
765
|
+
unarchivedAcknowledged,
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
/** The runs of sequences in `[1..extent]` that `events` does not hold. */
|
|
769
|
+
function missingRunsWithin(events, extent) {
|
|
770
|
+
const runs = [];
|
|
771
|
+
let prev = 0;
|
|
772
|
+
for (const e of events) {
|
|
773
|
+
if (e.sequence === 0)
|
|
774
|
+
continue;
|
|
775
|
+
if (e.sequence > extent + 1)
|
|
776
|
+
break;
|
|
777
|
+
if (e.sequence - prev > 1)
|
|
778
|
+
runs.push([prev + 1, e.sequence - 1]);
|
|
779
|
+
prev = e.sequence;
|
|
780
|
+
}
|
|
781
|
+
if (prev < extent)
|
|
782
|
+
runs.push([prev + 1, extent]);
|
|
783
|
+
return runs;
|
|
784
|
+
}
|
|
785
|
+
/** The runs of sequences in `[first..extent]` that `events` does not hold. */
|
|
786
|
+
function missingRunsFrom(events, first, extent) {
|
|
787
|
+
const runs = [];
|
|
788
|
+
let prev = first - 1;
|
|
789
|
+
for (const e of events) {
|
|
790
|
+
if (e.sequence < first)
|
|
791
|
+
continue;
|
|
792
|
+
if (e.sequence > extent)
|
|
793
|
+
break;
|
|
794
|
+
if (e.sequence - prev > 1)
|
|
795
|
+
runs.push([prev + 1, e.sequence - 1]);
|
|
796
|
+
prev = e.sequence;
|
|
797
|
+
}
|
|
798
|
+
if (prev < extent)
|
|
799
|
+
runs.push([prev + 1, extent]);
|
|
800
|
+
return runs;
|
|
801
|
+
}
|
|
802
|
+
/** A per-topic inbox stream (`__loom_inbox__:<chain>#<16 hex>`): its sequences start at INBOX_TOPIC_SEQ_BASE. */
|
|
803
|
+
function isInboxTopicStream(executionId) {
|
|
804
|
+
return executionId.startsWith(event_runtime_1.INBOX_STREAM_PREFIX) && /#[0-9a-f]{16}$/.test(executionId);
|
|
805
|
+
}
|
|
806
|
+
/** Whether the union of `held` ranges (inclusive) covers every sequence of `[a..b]`. */
|
|
807
|
+
function rangeCovered(a, b, held) {
|
|
808
|
+
let next = a;
|
|
809
|
+
for (const [from, through] of [...held].sort((x, y) => x[0] - y[0])) {
|
|
810
|
+
if (from > next)
|
|
811
|
+
break;
|
|
812
|
+
if (through >= next)
|
|
813
|
+
next = through + 1;
|
|
814
|
+
if (next > b)
|
|
815
|
+
return true;
|
|
816
|
+
}
|
|
817
|
+
return next > b;
|
|
818
|
+
}
|
|
819
|
+
//# sourceMappingURL=gc.js.map
|