@loomwork/plugin-content 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +8 -0
- package/README.md +92 -2
- package/dist/CachingContentStore.d.ts +54 -0
- package/dist/CachingContentStore.d.ts.map +1 -0
- package/dist/CachingContentStore.js +171 -0
- package/dist/CachingContentStore.js.map +1 -0
- package/dist/ContentResolvingEventStore.d.ts +70 -0
- package/dist/ContentResolvingEventStore.d.ts.map +1 -0
- package/dist/ContentResolvingEventStore.js +245 -0
- package/dist/ContentResolvingEventStore.js.map +1 -0
- package/dist/ContentStore.d.ts +186 -0
- package/dist/ContentStore.d.ts.map +1 -0
- package/dist/ContentStore.js +177 -0
- package/dist/ContentStore.js.map +1 -0
- package/dist/ContentTouchingSnapshotStore.d.ts +56 -0
- package/dist/ContentTouchingSnapshotStore.d.ts.map +1 -0
- package/dist/ContentTouchingSnapshotStore.js +85 -0
- package/dist/ContentTouchingSnapshotStore.js.map +1 -0
- package/dist/citations.d.ts +40 -0
- package/dist/citations.d.ts.map +1 -0
- package/dist/citations.js +118 -0
- package/dist/citations.js.map +1 -0
- package/dist/content.bench.d.ts +2 -0
- package/dist/content.bench.d.ts.map +1 -0
- package/dist/content.bench.js +75 -0
- package/dist/content.bench.js.map +1 -0
- package/dist/content.d.ts +40 -0
- package/dist/content.d.ts.map +1 -0
- package/dist/content.js +177 -0
- package/dist/content.js.map +1 -0
- package/dist/derive.d.ts +9 -0
- package/dist/derive.d.ts.map +1 -0
- package/dist/derive.js +40 -0
- package/dist/derive.js.map +1 -0
- package/dist/errors.d.ts +6 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +20 -0
- package/dist/errors.js.map +1 -0
- package/dist/executionEnd.d.ts +22 -0
- package/dist/executionEnd.d.ts.map +1 -0
- package/dist/executionEnd.js +66 -0
- package/dist/executionEnd.js.map +1 -0
- package/dist/gc.d.ts +470 -0
- package/dist/gc.d.ts.map +1 -0
- package/dist/gc.js +819 -0
- package/dist/gc.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +42 -0
- package/dist/index.js.map +1 -0
- package/dist/touchAround.d.ts +46 -0
- package/dist/touchAround.d.ts.map +1 -0
- package/dist/touchAround.js +98 -0
- package/dist/touchAround.js.map +1 -0
- package/package.json +58 -3
- package/src/CachingContentStore.ts +184 -0
- package/src/ContentResolvingEventStore.ts +325 -0
- package/src/ContentStore.ts +317 -0
- package/src/ContentTouchingSnapshotStore.ts +109 -0
- package/src/citations.ts +117 -0
- package/src/content.bench.ts +84 -0
- package/src/content.ts +191 -0
- package/src/derive.ts +40 -0
- package/src/errors.ts +19 -0
- package/src/executionEnd.ts +64 -0
- package/src/gc.ts +1213 -0
- package/src/index.ts +54 -0
- package/src/touchAround.ts +131 -0
package/src/gc.ts
ADDED
|
@@ -0,0 +1,1213 @@
|
|
|
1
|
+
import type { EventEnvelope } from "@loomwork/core";
|
|
2
|
+
import type { ArchiveStore, EventStore, ExecutionEnumerable } from "@loomwork/event-runtime";
|
|
3
|
+
import {
|
|
4
|
+
INBOX_STREAM_PREFIX,
|
|
5
|
+
INBOX_TOPIC_SEQ_BASE,
|
|
6
|
+
isPayloadShredded,
|
|
7
|
+
isReservedStream,
|
|
8
|
+
iterateExecutionIds,
|
|
9
|
+
} from "@loomwork/event-runtime";
|
|
10
|
+
import type { ContentStore } from "./ContentStore";
|
|
11
|
+
import { isContentTouchingSnapshotStore } from "./ContentTouchingSnapshotStore";
|
|
12
|
+
import { retainRootsOf, visitCitedAddresses } from "./citations";
|
|
13
|
+
import { terminalEndOf, wasReopened } from "./executionEnd";
|
|
14
|
+
|
|
15
|
+
// ── K10.6 (ADR 0055) — pinned mark-and-sweep for claim-checked content ────────
|
|
16
|
+
//
|
|
17
|
+
// Blob reclamation was the ADR 0046 deferred gap: `deleteExecution`/
|
|
18
|
+
// `truncatePrefix` reclaim events but never blobs. This sweep closes it for
|
|
19
|
+
// claim-checked results (ADR 0054), with the retention rule the perpetual-agents
|
|
20
|
+
// series fixed jointly: *findings and their evidence are permanent; runs are
|
|
21
|
+
// archivable; raw telemetry is disposable.* Pins are DERIVED from the log on
|
|
22
|
+
// every sweep (mark-and-sweep, not a persisted refcount): the log is the source
|
|
23
|
+
// of truth, there is no mutable index to corrupt or resurrect, and archived
|
|
24
|
+
// executions keep pinning simply by being scanned — pass the archive store
|
|
25
|
+
// alongside the live one. Keyed by content address, so a blob cited by a
|
|
26
|
+
// finding in an archived month-1 run survives any number of rotations.
|
|
27
|
+
//
|
|
28
|
+
// Only a claim-check envelope's own `ref` is TTL-reclaimable (ADR 0055 D4).
|
|
29
|
+
// A bare ref is load-bearing: `ContentResolvingEventStore.read` resolves every
|
|
30
|
+
// bare ref in a payload and throws `ContentMissingError` when its blob is gone,
|
|
31
|
+
// so erasing one bricks the execution (resume, replay, fork, export). The raw
|
|
32
|
+
// log cannot say which bare refs a resolving store will resolve, so every bare
|
|
33
|
+
// ref is retained while any scanned event cites it.
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The highest sequence missing below the log's tip, or -1 when it is contiguous from 0. The
|
|
37
|
+
* same test `LogLifecycle` uses to find a compaction checkpoint (ADR 0106). One backward pass.
|
|
38
|
+
*/
|
|
39
|
+
function highestMissingSequence(events: readonly { sequence: number }[]): number {
|
|
40
|
+
for (let i = events.length - 1; i > 0; i--) {
|
|
41
|
+
const at = events[i]!.sequence;
|
|
42
|
+
if (at - events[i - 1]!.sequence > 1) return at - 1;
|
|
43
|
+
}
|
|
44
|
+
return events.length > 0 && events[0]!.sequence !== 0 ? events[0]!.sequence - 1 : -1;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* A {@link PinPredicate} for `LogLifecycle` (K10.6): pin any execution whose log
|
|
49
|
+
* carries at least one event of the designated kinds — e.g. the app's finding
|
|
50
|
+
* events. Use when no archive is configured and reclaiming the run would destroy
|
|
51
|
+
* the only copy of its durable conclusions.
|
|
52
|
+
*/
|
|
53
|
+
export function pinByEventKinds(
|
|
54
|
+
eventStore: Pick<EventStore, "read">,
|
|
55
|
+
kinds: Iterable<string>
|
|
56
|
+
): (executionId: string) => Promise<boolean> {
|
|
57
|
+
const pinnedKinds = new Set(kinds);
|
|
58
|
+
return async (executionId) =>
|
|
59
|
+
(await eventStore.read(executionId)).some((event) => pinnedKinds.has(event.eventType));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Everything but the `dryRun` / `reclaimOrphans` pair, which {@link ContentSweepOptions} ties together. */
|
|
63
|
+
export interface ContentSweepBase {
|
|
64
|
+
/**
|
|
65
|
+
* Every log store whose events may cite content — the live store AND any
|
|
66
|
+
* archive stores. All must be enumerable (`listExecutionsPage()` preferred —
|
|
67
|
+
* SP-2, bounded memory — or `listExecutions()`): a store that cannot be
|
|
68
|
+
* enumerated cannot be swept safely (its pins would be invisible), so the
|
|
69
|
+
* sweep refuses rather than silently under-pinning.
|
|
70
|
+
*
|
|
71
|
+
* List the LIVE store BEFORE its archive. Retention copies an execution to the archive and
|
|
72
|
+
* then deletes it from the live store. Scanned live-first, an execution that moves during
|
|
73
|
+
* the sweep is seen in one store or the other. Scanned archive-first, it can be missed by
|
|
74
|
+
* both, and its blobs then look like orphans.
|
|
75
|
+
*/
|
|
76
|
+
eventStores: readonly (Pick<EventStore, "read"> & ExecutionEnumerable)[];
|
|
77
|
+
/**
|
|
78
|
+
* ADR 0055 D6 — where compaction checkpoints live: the live snapshot store AND EACH
|
|
79
|
+
* ARCHIVE'S snapshot store. `LogLifecycle.compact` checkpoints the fold and truncates the
|
|
80
|
+
* prefix, so a ref the prefix cited survives only in snapshot state.
|
|
81
|
+
*
|
|
82
|
+
* Which logs were compacted (ADR 0106, superseding "no sequence-1 event"): compaction
|
|
83
|
+
* removes its checkpoint's slot and, when it keeps pinned-kind events, never the tip. So a
|
|
84
|
+
* gap-free log with more than the bootstrap was never compacted (no snapshot read); a
|
|
85
|
+
* gapped log's checkpoint sits at its highest missing sequence `gap`; a bootstrap-only log
|
|
86
|
+
* was compacted through its tip. For each compacted execution the sweep pins every ref in:
|
|
87
|
+
* - the COMPACTION CHECKPOINT: the lowest snapshot in `[gap, tip]` — exactly `gap` for a
|
|
88
|
+
* compaction's own, the fork seed (`metadata.forkSequence`, one `loadAt`) for a child
|
|
89
|
+
* forked from a compacted parent — or, for a bootstrap-only log, its latest snapshot.
|
|
90
|
+
* ANY snapshot at or above `gap` covers a lost checkpoint, matching `LogLifecycle.compact`,
|
|
91
|
+
* which accepts a base at or above the gap; below `gap` nothing does. `fork()` from any point after it loads that
|
|
92
|
+
* checkpoint, so its refs stay reachable even after a later snapshot overwrote them.
|
|
93
|
+
* - the LATEST snapshot, which resume loads.
|
|
94
|
+
*
|
|
95
|
+
* Snapshots in between hold only checkpoint- or suffix-derived refs, and the suffix is
|
|
96
|
+
* scanned as log. Anything below the checkpoint is unreachable. The refs are PINNED because
|
|
97
|
+
* compaction folded away the event kinds, so the sweep cannot tell telemetry from evidence.
|
|
98
|
+
* They are released when retention deletes the execution, which deletes its snapshots too.
|
|
99
|
+
*
|
|
100
|
+
* `reclaimOrphans` REFUSES, before erasing anything:
|
|
101
|
+
* - without this option, when any scanned store can `truncatePrefix`;
|
|
102
|
+
* - when an execution looks compacted but no checkpoint is found. That covers a gap with no
|
|
103
|
+
* snapshot in `[gap, tip]`, and a sequence-0-only log with no snapshot, which is either
|
|
104
|
+
* compacted at its tip or a run that never got past its bootstrap. The two are
|
|
105
|
+
* indistinguishable, so both refuse.
|
|
106
|
+
*/
|
|
107
|
+
snapshotStores?: readonly SnapshotLookup[];
|
|
108
|
+
/**
|
|
109
|
+
* ADR 0138 - the compaction archives (`LogLifecycle.compactionArchive`). Compaction MOVES the events
|
|
110
|
+
* it removes into the archive, so a ref only an archived event cites (a step input the fold no
|
|
111
|
+
* longer keeps, an old finding) is cited THERE and nowhere else. Every ref cited by an archived
|
|
112
|
+
* envelope is PINNED for as long as the archive holds it: the sweep never reclaims a blob the
|
|
113
|
+
* recoverable history still names. Not scanning them is the hazard: an erasing orphan pass without
|
|
114
|
+
* the archive of a compacted log erases blobs its audit trail still cites. Read through the same
|
|
115
|
+
* sealing store the lifecycle uses (an `EncryptingArchiveStore` yields plaintext; a crypto-shredded
|
|
116
|
+
* subject's archive is unreadable and cites nothing). The archive itself is never swept or pruned.
|
|
117
|
+
*
|
|
118
|
+
* REQUIRED (fail closed) by an ERASING `reclaimOrphans` pass whenever any scanned log is compacted
|
|
119
|
+
* (a gap below its tip, or only its bootstrap left): {@link ArchiveStoresRequiredError} otherwise,
|
|
120
|
+
* before anything is erased. Coverage is CHECKED for EVERY compacted log, whatever its checkpoint
|
|
121
|
+
* is called (the `ckpt-0138-` id mark is only used to label reports - it can be lost to pruning, so
|
|
122
|
+
* it never gates an erase): the log's removed range must be covered by the given archives' headers
|
|
123
|
+
* (legacy-checkpoint segments included), else {@link ArchiveStoresRequiredError} (`reason:
|
|
124
|
+
* "uncovered"`). `[]` alone therefore no longer passes a gapped log. The only way past it, for logs
|
|
125
|
+
* compacted before ADR 0138 (no archive exists), is {@link acknowledgeUnarchivedLogs}.
|
|
126
|
+
* The archive is ENUMERATED (`listExecutions`), so segments whose log is gone are scanned
|
|
127
|
+
* too. A segment that is still sealed (count > 0 with no envelopes: the RAW inner store of an
|
|
128
|
+
* `EncryptingArchiveStore`) throws {@link SealedArchiveScanError} on an erasing pass.
|
|
129
|
+
*/
|
|
130
|
+
archiveStores?: readonly Pick<ArchiveStore, "read" | "listExecutions" | "headers">[];
|
|
131
|
+
/**
|
|
132
|
+
* Explicit opt-in for an erasing `reclaimOrphans` pass over compacted logs the given
|
|
133
|
+
* `archiveStores` do not cover: "these logs' compacted history is already gone (compacted before
|
|
134
|
+
* ADR 0138 and never re-compacted, or a fork child whose parent's archive is not keyed to it);
|
|
135
|
+
* erasing their orphan blobs is accepted." The affected executions are reported in
|
|
136
|
+
* `unarchivedAcknowledged`. Never a substitute for passing the real archive.
|
|
137
|
+
*/
|
|
138
|
+
acknowledgeUnarchivedLogs?: boolean;
|
|
139
|
+
/**
|
|
140
|
+
* The blob store. With `deleteIfUntouchedSince` (SPI 1.3.0) every erase is conditional on
|
|
141
|
+
* the blob's touch time, which closes the re-cite race for a re-cite THROUGH `put` (ADR 0055
|
|
142
|
+
* D5); without it the sweep falls back to `delete` and reports `raceSafe: false`. A citation
|
|
143
|
+
* copied WITHOUT a `put` (a fork's `copyPrefix`, a ref an `append` keeps) is protected by SPI
|
|
144
|
+
* 1.4.0 `touch`, which `reclaimOrphans` requires; see `reclaimOrphans`. `listPage` enables it.
|
|
145
|
+
*/
|
|
146
|
+
contentStore: Pick<
|
|
147
|
+
ContentStore,
|
|
148
|
+
"delete" | "deleteIfUntouchedSince" | "listPage" | "touchClock" | "touch"
|
|
149
|
+
>;
|
|
150
|
+
/** Event kinds whose cited refs are pinned — retained as long as the citing fact exists. */
|
|
151
|
+
pinnedEventTypes: Iterable<string>;
|
|
152
|
+
/**
|
|
153
|
+
* Reclaim an unpinned ref only when its effective citation time is older than this (ADR 0146:
|
|
154
|
+
* its latest citation, or, for a ref cited by an execution that has ENDED, that execution's end
|
|
155
|
+
* if later). EXACTLY ONE of `olderThanMs` / `policy` must be given.
|
|
156
|
+
*/
|
|
157
|
+
olderThanMs?: number;
|
|
158
|
+
/**
|
|
159
|
+
* ADR 0145 - an app-owned retention policy (`retentionPolicy()` from `@loomwork/workflow-runtime`;
|
|
160
|
+
* typed structurally here so this package takes no dependency on it). `content.unpinnedAfterMs`
|
|
161
|
+
* is used as `olderThanMs`. A policy without a `content` section is an error, not a silent no-op.
|
|
162
|
+
* Re-validated structurally (see `resolvePolicyTtl`): unknown keys are refused, and
|
|
163
|
+
* `unpinnedAfterMs >= terminalTtlMs` when the policy carries one (an advisory consistency rule,
|
|
164
|
+
* ADR 0145 D2).
|
|
165
|
+
*/
|
|
166
|
+
policy?: {
|
|
167
|
+
terminalTtlMs?: number;
|
|
168
|
+
snapshots?: unknown;
|
|
169
|
+
content?: { unpinnedAfterMs: number };
|
|
170
|
+
};
|
|
171
|
+
/**
|
|
172
|
+
* LOG time, in epoch ms: the instant the TTL is measured back from, compared against event
|
|
173
|
+
* `timestamp`s (`now − effective citation time ≥ olderThanMs`, ADR 0146). It is NOT the store's clock. A
|
|
174
|
+
* simulated or replayed log may run far from the store's wall clock; the race guard uses
|
|
175
|
+
* `touchedBefore` for that reason.
|
|
176
|
+
*/
|
|
177
|
+
now: number;
|
|
178
|
+
/**
|
|
179
|
+
* STORE time, in epoch ms, on the clock the content store stamps touch times with: the
|
|
180
|
+
* cutoff for the conditional delete and for orphan eligibility (ADR 0055 D5). Default: the
|
|
181
|
+
* store's `touchClock()` read ONCE at the sweep's start, minus `olderThanMs`. A store with
|
|
182
|
+
* `deleteIfUntouchedSince` but no `touchClock` requires it explicitly.
|
|
183
|
+
*/
|
|
184
|
+
touchedBefore?: number;
|
|
185
|
+
/**
|
|
186
|
+
* DX-2 — report what WOULD be reclaimed without deleting anything (the
|
|
187
|
+
* `loom sweep content` operator default; `--apply` turns it off).
|
|
188
|
+
*/
|
|
189
|
+
dryRun?: boolean;
|
|
190
|
+
/**
|
|
191
|
+
* ADR 0055 D7 — external root sets: content addresses held OUTSIDE the scanned logs (an app
|
|
192
|
+
* table such as a Postgres `experience_refs`). Every yielded address is pinned through both
|
|
193
|
+
* passes. Read once per sweep, after the mark phase. An app that records a new root for
|
|
194
|
+
* bytes the store already holds must `put` them (the put is the touch the conditional
|
|
195
|
+
* delete sees), exactly as a flow does.
|
|
196
|
+
*/
|
|
197
|
+
roots?: () => AsyncIterable<string>;
|
|
198
|
+
/**
|
|
199
|
+
* ADR 0055 D6 — also reclaim ORPHANS: blobs no scanned event and no root cites (their
|
|
200
|
+
* execution was deleted with no archive, a crash between `put` and the append, a truncated
|
|
201
|
+
* prefix), once untouched for the TTL. Requires the store's `listPage` and
|
|
202
|
+
* `deleteIfUntouchedSince`; refuses otherwise.
|
|
203
|
+
*
|
|
204
|
+
* ACKNOWLEDGEMENT REQUIRED (ADR 0055 amendment, fail-closed): to ERASE, pass
|
|
205
|
+
* `{ completeScan: true }` (plus `snapshotsTouchContent: true` when `snapshotStores` is given, below). A bare `true` (or any other value) throws
|
|
206
|
+
* {@link ReclaimOrphansAckError} before anything is read or erased. A `dryRun` erases nothing
|
|
207
|
+
* and is how you check completeness, so under `dryRun: true` any truthy value (`true` included)
|
|
208
|
+
* is accepted; see {@link ContentSweepOptions} for the typing. The acknowledgement is your
|
|
209
|
+
* assertion that `eventStores`, `snapshotStores` and `roots` together cover EVERY place that
|
|
210
|
+
* cites content, scanned in the right ORDER: each live store BEFORE its archive (ADR 0055
|
|
211
|
+
* "Scan order"). `[archive, live]` can miss an execution that retention moves mid-sweep, and
|
|
212
|
+
* its blobs then look like orphans.
|
|
213
|
+
*
|
|
214
|
+
* HAZARD the sweep cannot detect, which is why the assertion exists: a citing store that was
|
|
215
|
+
* NOT passed in `eventStores` or `snapshotStores`. Every blob only that store cites is an
|
|
216
|
+
* orphan to this pass, and it is erased while still cited. Pass every archive
|
|
217
|
+
* (`retention.archive`) and each archive's snapshot store, and every app table through `roots`.
|
|
218
|
+
* Run with `dryRun: true` first and read the list.
|
|
219
|
+
*
|
|
220
|
+
* CONCURRENCY (ADR 0055 D6 amendment 2): D5's conditional delete only sees a touch. A `put`
|
|
221
|
+
* touches; a citation that enters a log WITHOUT one does not by itself:
|
|
222
|
+
* `ContentResolvingEventStore.copyPrefix` (a fork) copies refs verbatim, and an `append`
|
|
223
|
+
* keeps refs already inside its payload (recalled memory, `scriptRef`, refs passed between
|
|
224
|
+
* steps). SPI 1.4.0 `touch` closes that: those two paths touch the refs they carry, so a fork
|
|
225
|
+
* made mid-sweep makes `deleteIfUntouchedSince` refuse. To ERASE, the content store MUST
|
|
226
|
+
* implement `touch` or the sweep throws {@link ReclaimOrphansTouchError} (a `dryRun` need not).
|
|
227
|
+
* The log must be written through a `ContentResolvingEventStore` over that store: a writer
|
|
228
|
+
* that copies refs some other way is outside the guarantee.
|
|
229
|
+
*
|
|
230
|
+
* SNAPSHOTS (second path): a fork of a COMPACTED parent saves a seed snapshot whose state
|
|
231
|
+
* cites refs that exist nowhere else (the prefix was truncated). `ContentResolvingEventStore`
|
|
232
|
+
* cannot see it; only a snapshot store wrapped in `ContentTouchingSnapshotStore`
|
|
233
|
+
* (outside any `EncryptingSnapshotStore`, over the same content store) touches those refs.
|
|
234
|
+
* loom never builds either store for the host, so the wrapping is the host's and is not PROVEN
|
|
235
|
+
* from here (a sweep, and the CLI, read a RAW store over the tables a wrapped one wrote, so an
|
|
236
|
+
* unbranded store is no evidence the writer was unwrapped): an ERASING pass given `snapshotStores` must also pass
|
|
237
|
+
* `{ completeScan: true, snapshotsTouchContent: true }` or it throws
|
|
238
|
+
* {@link ReclaimOrphansSnapshotAckError}. Without `snapshotStores` it is not needed (such a
|
|
239
|
+
* pass already refuses a store that can `truncatePrefix`, and a fork of an uncompacted parent
|
|
240
|
+
* copies every event, so its seed cites nothing the copied events do not). A `dryRun` is exempt.
|
|
241
|
+
* The flag stays an assertion about the WRITER and is ALWAYS required, even when every store
|
|
242
|
+
* passed is a `ContentTouchingSnapshotStore` (it vouches for itself, not for the other stores the
|
|
243
|
+
* runtime saves to). Best-effort detection only WARNS: the report counts branded and unbranded
|
|
244
|
+
* stores (`snapshotStoresTouching` / `snapshotStoresUnbranded`), and a pass that is handed a MIX
|
|
245
|
+
* (some branded, some not: a host wrapped some writers and likely forgot others) emits one
|
|
246
|
+
* `LOOM_CONTENT_SNAPSHOT_UNWRAPPED` process warning. It never refuses, so the CLI's raw SQLite
|
|
247
|
+
* stores are not rejected; it never relaxes the flag either (fail-closed).
|
|
248
|
+
*
|
|
249
|
+
* THE GUARANTEE IS D5'S, NOT THE TOUCH'S. The cutoff is `touchClock() - olderThanMs`. The
|
|
250
|
+
* touch BEFORE a write is what makes a sweep whose cutoff is later refuse; so the race is
|
|
251
|
+
* closed while a write's touch-to-commit window is shorter than the margin
|
|
252
|
+
* `store-now - cutoff` (the TTL, or the distance of an explicit `touchedBefore` from the
|
|
253
|
+
* store's clock). The touch AFTER the write is a best-effort extension of that window (it
|
|
254
|
+
* is not retried and an error is reported through `onTouchError`, not raised); do not
|
|
255
|
+
* count on it. With a zero or tiny margin the window is unbounded: a fork whose before-touch
|
|
256
|
+
* is older than the cutoff and which commits after the scan has passed the child is erased
|
|
257
|
+
* under. So an ERASING orphan pass refuses `olderThanMs <= 0` and any cutoff closer to the
|
|
258
|
+
* store's clock than {@link MIN_ORPHAN_ERASE_MARGIN_MS} with {@link ReclaimOrphansMarginError},
|
|
259
|
+
* and a store with no `touchClock()` (the margin cannot be measured) with
|
|
260
|
+
* {@link ReclaimOrphansClockError}; a `dryRun` is unaffected. Pick a margin well above your longest write (a fork's copy of a
|
|
261
|
+
* long prefix included).
|
|
262
|
+
*/
|
|
263
|
+
reclaimOrphans?: ReclaimOrphansAck | boolean;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Sweep options. `reclaimOrphans` and `dryRun` are tied by a union so the type says what the
|
|
268
|
+
* runtime enforces: a sweep that can ERASE (`dryRun` absent or `false`) accepts only the
|
|
269
|
+
* acknowledgement object (or `false`), so a bare `true` is a compile error there; a
|
|
270
|
+
* `dryRun: true` sweep also accepts a plain `true`, because it erases nothing and is how a host
|
|
271
|
+
* checks completeness. (A `dryRun` typed as a plain `boolean` falls in the first arm, which is
|
|
272
|
+
* the safe one.)
|
|
273
|
+
*/
|
|
274
|
+
export type ContentSweepOptions = Omit<ContentSweepBase, "dryRun" | "reclaimOrphans"> &
|
|
275
|
+
(
|
|
276
|
+
| { dryRun?: false; reclaimOrphans?: ReclaimOrphansAck | false }
|
|
277
|
+
| { dryRun: true; reclaimOrphans?: ReclaimOrphansAck | boolean }
|
|
278
|
+
);
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* The caller's assertion that a `reclaimOrphans` sweep sees every citing store. The literal
|
|
282
|
+
* `true` type makes `{ completeScan: false }` a compile error; the runtime check also refuses
|
|
283
|
+
* untyped callers.
|
|
284
|
+
*/
|
|
285
|
+
export interface ReclaimOrphansAck {
|
|
286
|
+
completeScan: true;
|
|
287
|
+
/**
|
|
288
|
+
* Required (checked at runtime) whenever an ERASING pass is given `snapshotStores`: the
|
|
289
|
+
* caller's assertion that every snapshot store the runtime SAVES to is wrapped in
|
|
290
|
+
* `ContentTouchingSnapshotStore` (outside any `EncryptingSnapshotStore`). Without it a
|
|
291
|
+
* fork of a compacted parent saves a seed snapshot citing refs that live nowhere else, and
|
|
292
|
+
* a sweep can erase them under the live child. See {@link ReclaimOrphansSnapshotAckError}.
|
|
293
|
+
*/
|
|
294
|
+
snapshotsTouchContent?: true;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** A safe description of a rejected value: BigInt and circular values make `JSON.stringify` throw. */
|
|
298
|
+
function describeReceived(received: unknown): string {
|
|
299
|
+
try {
|
|
300
|
+
return JSON.stringify(received) ?? String(received);
|
|
301
|
+
} catch {
|
|
302
|
+
try {
|
|
303
|
+
return `an unserializable ${typeof received} (${Object.prototype.toString.call(received)})`;
|
|
304
|
+
} catch {
|
|
305
|
+
return `an unserializable ${typeof received}`;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** `reclaimOrphans` was set without `{ completeScan: true }`. Nothing was read or erased. */
|
|
311
|
+
export class ReclaimOrphansAckError extends TypeError {
|
|
312
|
+
constructor(received: unknown) {
|
|
313
|
+
super(
|
|
314
|
+
`sweepUnpinnedContent: reclaimOrphans must be { completeScan: true } to erase (received ${describeReceived(received)}). Orphan reclamation erases every blob that no scanned log, snapshot or root cites, so a citing store you forgot to pass loses its blobs while they are still cited. Set { completeScan: true } only when eventStores, snapshotStores and roots cover ALL of them (every retention archive and its snapshot store included), AND scanned in order: each live store BEFORE its archive, or an execution retention moves mid-sweep can be missed by both. A dryRun does not need the acknowledgement. Migration: reclaimOrphans: true -> reclaimOrphans: { completeScan: true }.`
|
|
315
|
+
);
|
|
316
|
+
this.name = "ReclaimOrphansAckError";
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* An ERASING `reclaimOrphans` pass was given `snapshotStores` without
|
|
322
|
+
* `{ snapshotsTouchContent: true }`. Nothing was read or erased. A compacted parent's prefix refs
|
|
323
|
+
* live only in snapshot state, and a fork copies that state into its seed snapshot with no `put`
|
|
324
|
+
* and no event; only a snapshot store wrapped in `ContentTouchingSnapshotStore` touches them.
|
|
325
|
+
* The sweep cannot see how the writing runtime's snapshot store was built, so, like D6's
|
|
326
|
+
* archive assertion, the host states it. A `dryRun` is exempt. With no `snapshotStores` the
|
|
327
|
+
* acknowledgement is not needed: the pass then refuses any store that can `truncatePrefix`, and
|
|
328
|
+
* a fork of an UNcompacted parent copies every event (refs touched by `copyPrefix`), so its seed
|
|
329
|
+
* state cites nothing the copied events do not.
|
|
330
|
+
*/
|
|
331
|
+
export class ReclaimOrphansSnapshotAckError extends TypeError {
|
|
332
|
+
constructor() {
|
|
333
|
+
super(
|
|
334
|
+
"sweepUnpinnedContent: an erasing reclaimOrphans pass given snapshotStores needs { completeScan: true, snapshotsTouchContent: true }. A fork of a compacted parent saves a seed snapshot that cites refs found nowhere else; only a snapshot store wrapped in ContentTouchingSnapshotStore (outside any EncryptingSnapshotStore, over the same content store the log uses) touches them, so a sweep racing the fork erases blobs a live child cites (replay then throws ContentMissingError). Wrap every snapshot store the runtime saves to, then pass snapshotsTouchContent: true, or run with dryRun: true."
|
|
335
|
+
);
|
|
336
|
+
this.name = "ReclaimOrphansSnapshotAckError";
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* `reclaimOrphans` was asked to ERASE through a content store without SPI 1.4.0 `touch`.
|
|
342
|
+
* Nothing was read or erased. Without `touch`, a citation that enters a log with no `put` (a
|
|
343
|
+
* fork's `copyPrefix`, a ref an `append` carries) cannot make the conditional delete refuse, so
|
|
344
|
+
* an orphan sweep concurrent with a fork or a delete can erase a blob a live log still cites
|
|
345
|
+
* (replay then throws `ContentMissingError`). This is a deletion safety boundary: it refuses
|
|
346
|
+
* rather than documenting the hazard. A `dryRun` is unaffected.
|
|
347
|
+
*/
|
|
348
|
+
export class ReclaimOrphansTouchError extends TypeError {
|
|
349
|
+
constructor() {
|
|
350
|
+
super(
|
|
351
|
+
"sweepUnpinnedContent: reclaimOrphans needs a content store with touch() (ContentStore SPI 1.4.0) to ERASE. Without it a fork (copyPrefix) or an append that carries an existing ref cannot mark the blob re-cited, so an orphan sweep racing a fork or a delete can erase a blob a live log still cites. Add touch() to the store (SqliteContentStore and InMemoryContentStore have it; a CachingContentStore forwards it), or run with dryRun: true."
|
|
352
|
+
);
|
|
353
|
+
this.name = "ReclaimOrphansTouchError";
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* The smallest margin, in ms, between the content store's clock and the guard cutoff that an
|
|
359
|
+
* ERASING orphan pass accepts (ADR 0055 D6 amendment 2). A write's touch-to-commit window must
|
|
360
|
+
* be shorter than the margin for the touch to protect it; 60 s is far above an append and
|
|
361
|
+
* above a fork's prefix copy at any realistic size, and an operator who needs a bigger one
|
|
362
|
+
* simply passes a larger TTL.
|
|
363
|
+
*/
|
|
364
|
+
export const MIN_ORPHAN_ERASE_MARGIN_MS = 60_000;
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* An ERASING `reclaimOrphans` pass was configured with a zero/negative `olderThanMs`, or a
|
|
368
|
+
* cutoff (`touchedBefore`) closer to the content store's clock than
|
|
369
|
+
* {@link MIN_ORPHAN_ERASE_MARGIN_MS}. Nothing was read or erased. With no margin the
|
|
370
|
+
* touch-before-write has no window to protect: a write can touch, the sweep read a later cutoff,
|
|
371
|
+
* and the write commit after the scan, so a blob a live log cites is erased.
|
|
372
|
+
*/
|
|
373
|
+
export class ReclaimOrphansMarginError extends TypeError {
|
|
374
|
+
constructor(readonly marginMs: number) {
|
|
375
|
+
super(
|
|
376
|
+
`sweepUnpinnedContent: an erasing reclaimOrphans pass needs a guard margin of at least ${MIN_ORPHAN_ERASE_MARGIN_MS}ms between the content store's clock and the cutoff (received ${marginMs}ms: olderThanMs, or touchedBefore's distance from the store's clock). The touch that protects a fork or an append only covers a write whose touch-to-commit window is shorter than that margin; with olderThanMs <= 0 the window is unbounded. Raise olderThanMs / lower touchedBefore, or run with dryRun: true.`
|
|
377
|
+
);
|
|
378
|
+
this.name = "ReclaimOrphansMarginError";
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* An ERASING `reclaimOrphans` pass over a content store with `deleteIfUntouchedSince` but no
|
|
384
|
+
* `touchClock()`. Nothing was read or erased. Without the store's clock the sweep cannot measure
|
|
385
|
+
* the margin between "now" and the cutoff (`touchedBefore` is an unchecked caller value, so
|
|
386
|
+
* `touchedBefore: Date.now()` would pass any `olderThanMs`), and the touch-to-commit window is
|
|
387
|
+
* then unbounded. A `dryRun` is exempt.
|
|
388
|
+
*/
|
|
389
|
+
export class ReclaimOrphansClockError extends TypeError {
|
|
390
|
+
constructor() {
|
|
391
|
+
super(
|
|
392
|
+
"sweepUnpinnedContent: an erasing reclaimOrphans pass needs a content store with touchClock() so the guard margin can be measured (a store with deleteIfUntouchedSince but no touchClock cannot prove touchedBefore is far enough behind its clock). Add touchClock() to the store, or run with dryRun: true."
|
|
393
|
+
);
|
|
394
|
+
this.name = "ReclaimOrphansClockError";
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** Structural view of a snapshot store: just `loadLatest` (no snapshot-runtime dependency). */
|
|
399
|
+
export interface SnapshotLookup {
|
|
400
|
+
loadLatest(
|
|
401
|
+
executionId: string
|
|
402
|
+
): Promise<{ id?: string; sequence: number; state: unknown } | null>;
|
|
403
|
+
/** Exact-sequence load; used for a forked child's seed (optional — falls back to a walk). */
|
|
404
|
+
loadAt?(
|
|
405
|
+
executionId: string,
|
|
406
|
+
sequence: number
|
|
407
|
+
): Promise<{ id?: string; sequence: number; state: unknown } | null>;
|
|
408
|
+
loadLatestBefore(
|
|
409
|
+
executionId: string,
|
|
410
|
+
sequence: number
|
|
411
|
+
): Promise<{ id?: string; sequence: number; state: unknown } | null>;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
let warnedUnwrapped = false;
|
|
415
|
+
/** Test-only: re-arm the once-per-process warning below. Not exported from the package index. */
|
|
416
|
+
export function resetSnapshotUnwrappedWarning(): void {
|
|
417
|
+
warnedUnwrapped = false;
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* One process warning for a MIXED set of snapshot stores, ONCE per process (like
|
|
421
|
+
* `LOOM_CONTENT_TOUCH`): a scheduled sweep over the same mixed set would otherwise warn every run.
|
|
422
|
+
* Never throws (best effort, advisory).
|
|
423
|
+
*/
|
|
424
|
+
function emitUnwrappedWarning(touching: number, unbranded: number): void {
|
|
425
|
+
if (warnedUnwrapped) return;
|
|
426
|
+
warnedUnwrapped = true;
|
|
427
|
+
const proc = (globalThis as { process?: { emitWarning?: (m: string, o?: object) => void } })
|
|
428
|
+
.process;
|
|
429
|
+
try {
|
|
430
|
+
proc?.emitWarning?.(
|
|
431
|
+
`sweepUnpinnedContent: ${unbranded} of ${touching + unbranded} snapshot stores are not a ContentTouchingSnapshotStore while ${touching} are. If an unbranded one is a store the runtime SAVES to, a fork of a compacted parent can lose blobs to this erasing orphan pass; if it is only a raw reader over a wrapped writer's tables, ignore this.`,
|
|
432
|
+
{ code: "LOOM_CONTENT_SNAPSHOT_UNWRAPPED" }
|
|
433
|
+
);
|
|
434
|
+
} catch {
|
|
435
|
+
// advisory only
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** Orphan-pass page size (ADR 0055 D6): bounded memory per page. */
|
|
440
|
+
const ORPHAN_PAGE_LIMIT = 500;
|
|
441
|
+
|
|
442
|
+
export interface ContentSweepReport {
|
|
443
|
+
executionsScanned: number;
|
|
444
|
+
refsSeen: number;
|
|
445
|
+
refsPinned: number;
|
|
446
|
+
/**
|
|
447
|
+
* ADR 0055 D4 - refs cited at least once anywhere but as a RECLAIMABLE claim-check envelope's
|
|
448
|
+
* `ref` (a digest counts as anywhere). Since ADR 0140 that includes a `retain: true` envelope
|
|
449
|
+
* whose ref sits at a trusted root (the top-level `output` of a STEP_COMPLETED, through a taint
|
|
450
|
+
* wrapper), and a R5 media source `{ kind: "ref", ref: { address } }`. Every address cited by an
|
|
451
|
+
* `archiveStores` segment is counted unconditionally. These are never reclaimed while any scanned
|
|
452
|
+
* event cites them. The set OVERLAPS `refsPinned` (a ref can be in both): the two are not disjoint
|
|
453
|
+
* partitions of `refsSeen`.
|
|
454
|
+
*/
|
|
455
|
+
refsLoadBearing: number;
|
|
456
|
+
/**
|
|
457
|
+
* ADR 0146 - unpinned refs whose age clock came from a citing execution's END rather than from the
|
|
458
|
+
* citation: a ref cited by an execution that ended after the citation ages from the end. Counted
|
|
459
|
+
* on every sweep.
|
|
460
|
+
*/
|
|
461
|
+
refsAgedByEnd: number;
|
|
462
|
+
/**
|
|
463
|
+
* ADR 0146 - DRY RUN ONLY (absent on an erasing sweep, which keeps no second map): refs the old
|
|
464
|
+
* citation clock would have reclaimed that the end clock keeps. Never reclaimed by the end clock
|
|
465
|
+
* that would not also have been by the old one, so this is the whole delta.
|
|
466
|
+
*/
|
|
467
|
+
heldByEndClock?: string[];
|
|
468
|
+
/**
|
|
469
|
+
* Unpinned refs whose age clock stayed on the CITATION because the newest citing execution was
|
|
470
|
+
* RE-OPENED after a terminal event (`resumeAs` onto an ended run, ADR 0104 D1; ADR 0146's exception):
|
|
471
|
+
* they can be reclaimed while that execution's log is still live. Counted on every sweep. A plain
|
|
472
|
+
* running or parked execution that never ended is NOT counted. Decisions are unchanged: visibility only.
|
|
473
|
+
*/
|
|
474
|
+
refsOfReopened: number;
|
|
475
|
+
/**
|
|
476
|
+
* DRY RUN ONLY (absent on an erasing sweep, which keeps no second list): of `reclaimed`, the addresses
|
|
477
|
+
* counted in `refsOfReopened`, so an operator sees them before `--apply`.
|
|
478
|
+
*/
|
|
479
|
+
reclaimedWhileReopened?: string[];
|
|
480
|
+
/** Content addresses erased this sweep — or, under `dryRun`, that WOULD be. */
|
|
481
|
+
reclaimed: string[];
|
|
482
|
+
/**
|
|
483
|
+
* Eligible addresses (refs and orphans) whose conditional erase was REFUSED by the touch/race guard:
|
|
484
|
+
* `deleteIfUntouchedSince` returned false because the blob was touched after `touchCutoff` (a
|
|
485
|
+
* concurrent re-cite, or a content-store clock that is not the one the log's `now` runs on).
|
|
486
|
+
* ALWAYS 0 on a dry run, which never calls the store: a dry run can list N in `reclaimed` while the
|
|
487
|
+
* apply reports `reclaimed: []` and `keptByTouch: N`; when it does, compare `touchCutoff` with the
|
|
488
|
+
* store's touch stamps (see `touchedBefore`). Also 0 when the store is not `raceSafe` (plain delete).
|
|
489
|
+
*/
|
|
490
|
+
keptByTouch: number;
|
|
491
|
+
/**
|
|
492
|
+
* The cutoff the guard compared touch stamps with, on the CONTENT STORE's clock (resolved from
|
|
493
|
+
* `touchedBefore`, else `touchClock() - olderThanMs`). Present on a dry run too (where it was not
|
|
494
|
+
* enforced); `undefined` when the store is not `raceSafe`.
|
|
495
|
+
*/
|
|
496
|
+
touchCutoff?: number;
|
|
497
|
+
/**
|
|
498
|
+
* ADR 0055 D5 — every erase was conditional on the blob's touch time
|
|
499
|
+
* (`deleteIfUntouchedSince`), so a `put` that re-cited a blob during the sweep kept it.
|
|
500
|
+
* `false` = the store lacks it and the sweep used plain `delete`: such a re-cite is lost.
|
|
501
|
+
*/
|
|
502
|
+
raceSafe: boolean;
|
|
503
|
+
/** ADR 0055 D6 — orphans erased (or, under `dryRun`, that WOULD be); `[]` unless `reclaimOrphans`. */
|
|
504
|
+
orphansReclaimed: string[];
|
|
505
|
+
/**
|
|
506
|
+
* Of `snapshotStores`, how many are a `ContentTouchingSnapshotStore` (branded). Informational:
|
|
507
|
+
* an unbranded store may be a raw reader over tables a wrapped writer filled.
|
|
508
|
+
*/
|
|
509
|
+
snapshotStoresTouching: number;
|
|
510
|
+
/** Of `snapshotStores`, how many are NOT branded. See `snapshotStoresTouching`. */
|
|
511
|
+
snapshotStoresUnbranded: number;
|
|
512
|
+
/**
|
|
513
|
+
* Executions whose scanned snapshot `state` is a top-level `__loomEnc` envelope (an
|
|
514
|
+
* `EncryptingSnapshotStore({ sealState })` read RAW): the refs it cites are invisible to this
|
|
515
|
+
* scan. An erasing pass throws {@link SealedSnapshotScanError} instead; a `dryRun` lists them.
|
|
516
|
+
* The check looks at the TOP-LEVEL `__loomEnc` key of `state` only, not a nested envelope.
|
|
517
|
+
*/
|
|
518
|
+
sealedSnapshots: string[];
|
|
519
|
+
/**
|
|
520
|
+
* Executions whose snapshot load threw `PayloadShreddedError` (their subject was crypto-shredded
|
|
521
|
+
* under a `sealState` store): the snapshot is erased, so it cites no readable refs and is skipped.
|
|
522
|
+
* Blobs another live owner cites stay pinned by it. Any OTHER load error still aborts the pass.
|
|
523
|
+
*/
|
|
524
|
+
shreddedSnapshots: string[];
|
|
525
|
+
/**
|
|
526
|
+
* Executions whose compacted history is NOT (fully) in the given archives and whose orphan blobs
|
|
527
|
+
* this erasing pass erased anyway because `acknowledgeUnarchivedLogs: true` accepted it. Empty
|
|
528
|
+
* otherwise (and on a dry run).
|
|
529
|
+
*/
|
|
530
|
+
unarchivedAcknowledged: string[];
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* An ERASING `reclaimOrphans` pass found compacted logs but was given no `archiveStores`: the events
|
|
535
|
+
* a compaction moved are cited only by the archive, so their blobs would look like orphans and be
|
|
536
|
+
* erased (ADR 0138). Nothing was erased.
|
|
537
|
+
*/
|
|
538
|
+
export class ArchiveStoresRequiredError extends TypeError {
|
|
539
|
+
constructor(
|
|
540
|
+
readonly executionIds: readonly string[],
|
|
541
|
+
/** `"none"`: no archiveStores given. `"uncovered"`: given, but they do not hold what was archived. */
|
|
542
|
+
readonly reason: "none" | "uncovered" = "none"
|
|
543
|
+
) {
|
|
544
|
+
const shown = executionIds.slice(0, 5).join(", ");
|
|
545
|
+
const more = executionIds.length > 5 ? ` (+${executionIds.length - 5} more)` : "";
|
|
546
|
+
super(
|
|
547
|
+
reason === "none"
|
|
548
|
+
? `sweepUnpinnedContent: ${executionIds.length} scanned log(s) are compacted (${shown}${more}) and no archiveStores were given. Compaction MOVES the removed events into the compaction archive, which is then the only place that cites their blobs; an erasing orphan pass without it deletes blobs the recoverable history still names. Pass archiveStores: [the compaction archive] (the sealing store when encrypted), or acknowledgeUnarchivedLogs: true if no archive exists (logs compacted before ADR 0138), or run with dryRun: true. Nothing was erased.`
|
|
549
|
+
: `sweepUnpinnedContent: ${executionIds.length} scanned log(s) are compacted but the given archiveStores do not cover the removed range (${shown}${more}): the archive passed is empty, the wrong one, or was wiped. An erasing orphan pass would delete blobs the lost history still names. Pass the real compaction archive (the sealing store when encrypted), restore it, run with dryRun: true, or pass acknowledgeUnarchivedLogs: true if that history is already gone (pre-ADR-0138 logs, fork children). Nothing was erased.`
|
|
550
|
+
);
|
|
551
|
+
this.name = "ArchiveStoresRequiredError";
|
|
552
|
+
Object.setPrototypeOf(this, ArchiveStoresRequiredError.prototype);
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* An erasing sweep read an archive segment that is still sealed (count > 0, no envelopes): the RAW
|
|
558
|
+
* inner store of an `EncryptingArchiveStore`. The refs inside are invisible, so live blobs would
|
|
559
|
+
* look unreferenced. Pass the sealing store itself. Nothing was erased.
|
|
560
|
+
*/
|
|
561
|
+
export class SealedArchiveScanError extends TypeError {
|
|
562
|
+
constructor(readonly executionIds: readonly string[]) {
|
|
563
|
+
const shown = executionIds.slice(0, 5).join(", ");
|
|
564
|
+
const more = executionIds.length > 5 ? ` (+${executionIds.length - 5} more)` : "";
|
|
565
|
+
super(
|
|
566
|
+
`sweepUnpinnedContent: ${executionIds.length} archived execution(s) hold sealed segments (${shown}${more}), so the refs they cite are invisible and an erasing pass would delete live blobs: pass the opening archive store (the EncryptingArchiveStore itself, not its inner store) in archiveStores, or run with dryRun: true. Nothing was erased.`
|
|
567
|
+
);
|
|
568
|
+
this.name = "SealedArchiveScanError";
|
|
569
|
+
Object.setPrototypeOf(this, SealedArchiveScanError.prototype);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* An erasing sweep read a snapshot whose `state` is one sealed envelope (a `sealState`
|
|
575
|
+
* `EncryptingSnapshotStore` given as its RAW inner store). The scan cannot see the refs inside, so
|
|
576
|
+
* it would treat live blobs as unreferenced and erase them. Nothing was erased.
|
|
577
|
+
*/
|
|
578
|
+
export class SealedSnapshotScanError extends TypeError {
|
|
579
|
+
constructor(readonly executionIds: readonly string[]) {
|
|
580
|
+
const shown = executionIds.slice(0, 5).join(", ");
|
|
581
|
+
const more = executionIds.length > 5 ? ` (+${executionIds.length - 5} more)` : "";
|
|
582
|
+
super(
|
|
583
|
+
`sweepUnpinnedContent: ${executionIds.length} scanned snapshot(s) hold a sealed (whole-state encrypted) state (${shown}${more}), so the refs they cite are invisible and an erasing pass would delete live blobs: pass the decrypting snapshot store (the EncryptingSnapshotStore itself, not its inner store) in snapshotStores, or run with dryRun: true. Nothing was erased.`
|
|
584
|
+
);
|
|
585
|
+
this.name = "SealedSnapshotScanError";
|
|
586
|
+
Object.setPrototypeOf(this, SealedSnapshotScanError.prototype);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Mark: walk every event of every execution across the given stores, collecting
|
|
592
|
+
* each ref's latest citation timestamp, pinning refs cited from events of the
|
|
593
|
+
* designated kinds, and marking refs cited anywhere other than as a
|
|
594
|
+
* claim-check envelope's `ref` (a `digest` included) as load-bearing. Sweep:
|
|
595
|
+
* physically erase refs that are neither pinned nor load-bearing and whose
|
|
596
|
+
* effective citation time is older than the TTL — i.e. only refs cited solely as an
|
|
597
|
+
* envelope's `ref`. ADR 0146: a citing execution that has ENDED (newest kernel terminal event, not
|
|
598
|
+
* re-opened) ages its citations from `max(citation, end)`, so with a content TTL at least the log TTL a blob is
|
|
599
|
+
* never reclaimable before retention could delete the log citing it (exceptions: ADR 0146); a
|
|
600
|
+
* running execution (or a reserved stream) keeps the citation clock. Never earlier than the citation. Their erasure is invisible to replay by the claim-check
|
|
601
|
+
* contract (ADR 0054 — the fold never resolves); `resolveClaimCheck` readers get
|
|
602
|
+
* `null`. A bare ref is never reclaimed while cited (ADR 0055 D4): the resolving
|
|
603
|
+
* store would throw `ContentMissingError` on the next read. Idempotent:
|
|
604
|
+
* `ContentStore.delete` is idempotent and a reclaimed ref simply no longer
|
|
605
|
+
* resolves on the next sweep.
|
|
606
|
+
*
|
|
607
|
+
* Race-safe (ADR 0055 D5) when the store has `deleteIfUntouchedSince`: every erase is
|
|
608
|
+
* conditional on the blob being untouched since `now − olderThanMs`, so a `put` that re-cites
|
|
609
|
+
* old bytes between the scan and its own append keeps the blob. That holds while a flow's
|
|
610
|
+
* put→append window is shorter than the TTL. A citation copied without a `put` (a fork's
|
|
611
|
+
* `copyPrefix`, a ref kept inside an `append` payload) is covered by SPI 1.4.0 `touch`, which
|
|
612
|
+
* `reclaimOrphans` requires; see `reclaimOrphans`. `roots` (D7) pins addresses held outside the
|
|
613
|
+
* log; `reclaimOrphans` (D6) also erases stored blobs nothing scanned cites. Cost:
|
|
614
|
+
* O(events + blobs), all membership tests are Set/Map lookups.
|
|
615
|
+
*/
|
|
616
|
+
export async function sweepUnpinnedContent(
|
|
617
|
+
given: ContentSweepOptions
|
|
618
|
+
): Promise<ContentSweepReport> {
|
|
619
|
+
const hasTtl = given.olderThanMs !== undefined;
|
|
620
|
+
const hasPolicy = given.policy !== undefined;
|
|
621
|
+
if (hasTtl === hasPolicy) {
|
|
622
|
+
throw new TypeError(
|
|
623
|
+
hasTtl
|
|
624
|
+
? "sweepUnpinnedContent: pass olderThanMs OR policy, not both - they name the same TTL"
|
|
625
|
+
: "sweepUnpinnedContent: needs a TTL - pass olderThanMs, or policy (a retention policy with a content.unpinnedAfterMs section)"
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
let olderThanMs = given.olderThanMs as number;
|
|
629
|
+
if (hasPolicy) olderThanMs = resolvePolicyTtl(given.policy);
|
|
630
|
+
const { policy: _policy, ...rest } = given;
|
|
631
|
+
return sweepResolved({ ...rest, olderThanMs } as ContentSweepOptions & { olderThanMs: number });
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* ADR 0145 D3 - structural re-validation of a retention policy, WITHOUT a dependency on
|
|
636
|
+
* `@loomwork/workflow-runtime`. Each field is read ONCE into a plain snapshot, then the snapshot is
|
|
637
|
+
* checked, so a getter answering differently on a second read cannot pass a rule-breaking value.
|
|
638
|
+
* Checks what the sweep consumes: unknown keys (top level and `content`), `unpinnedAfterMs` a
|
|
639
|
+
* finite integer > 0, and, when `terminalTtlMs` is present, a finite integer >= 0 with
|
|
640
|
+
* `unpinnedAfterMs >= terminalTtlMs`. `snapshots` is allowed and not inspected (the sweep reads
|
|
641
|
+
* nothing from it; `LogLifecycle` validates it).
|
|
642
|
+
*/
|
|
643
|
+
function resolvePolicyTtl(policy: unknown): number {
|
|
644
|
+
const bad = (m: string): never => {
|
|
645
|
+
throw new TypeError(`sweepUnpinnedContent: policy ${m}`);
|
|
646
|
+
};
|
|
647
|
+
if (typeof policy !== "object" || policy === null || Array.isArray(policy)) {
|
|
648
|
+
return bad("must be a retention policy object");
|
|
649
|
+
}
|
|
650
|
+
for (const k of Object.keys(policy)) {
|
|
651
|
+
if (k !== "terminalTtlMs" && k !== "snapshots" && k !== "content") {
|
|
652
|
+
bad(`has unknown key "${k}" (expected terminalTtlMs, snapshots, content)`);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
const o = policy as Record<string, unknown>;
|
|
656
|
+
const ttl = o.terminalTtlMs;
|
|
657
|
+
const content = o.content;
|
|
658
|
+
if (ttl !== undefined && (typeof ttl !== "number" || !Number.isInteger(ttl) || ttl < 0)) {
|
|
659
|
+
bad(`terminalTtlMs must be a finite integer >= 0 (milliseconds); got ${String(ttl)}`);
|
|
660
|
+
}
|
|
661
|
+
if (content === undefined) {
|
|
662
|
+
return bad(
|
|
663
|
+
"has no content section - add content.unpinnedAfterMs to the retention policy, or pass olderThanMs"
|
|
664
|
+
);
|
|
665
|
+
}
|
|
666
|
+
if (typeof content !== "object" || content === null || Array.isArray(content)) {
|
|
667
|
+
return bad("content must be an object with unpinnedAfterMs");
|
|
668
|
+
}
|
|
669
|
+
for (const k of Object.keys(content)) {
|
|
670
|
+
if (k !== "unpinnedAfterMs") {
|
|
671
|
+
bad(`has unknown key "content.${k}" (expected unpinnedAfterMs)`);
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
const u = (content as Record<string, unknown>).unpinnedAfterMs;
|
|
675
|
+
if (typeof u !== "number" || !Number.isInteger(u) || u <= 0) {
|
|
676
|
+
return bad(
|
|
677
|
+
`content.unpinnedAfterMs must be a finite integer > 0 (milliseconds); got ${String(u)}`
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
if (typeof ttl === "number" && u < ttl) {
|
|
681
|
+
bad(`content.unpinnedAfterMs (${u}) must be >= terminalTtlMs (${ttl})`);
|
|
682
|
+
}
|
|
683
|
+
return u;
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
async function sweepResolved(
|
|
687
|
+
options: ContentSweepOptions & { olderThanMs: number }
|
|
688
|
+
): Promise<ContentSweepReport> {
|
|
689
|
+
const store = options.contentStore;
|
|
690
|
+
const requested = options.reclaimOrphans as unknown;
|
|
691
|
+
if (requested !== undefined && requested !== false) {
|
|
692
|
+
// A dry run erases nothing, so it needs no acknowledgement (any truthy value turns the
|
|
693
|
+
// orphan pass on); the sweep that would ERASE must carry it.
|
|
694
|
+
if (
|
|
695
|
+
options.dryRun === true
|
|
696
|
+
? !requested
|
|
697
|
+
: typeof requested !== "object" ||
|
|
698
|
+
requested === null ||
|
|
699
|
+
(requested as { completeScan?: unknown }).completeScan !== true
|
|
700
|
+
) {
|
|
701
|
+
throw new ReclaimOrphansAckError(requested);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
const orphans = requested !== undefined && requested !== false;
|
|
705
|
+
if (orphans && store.listPage === undefined) {
|
|
706
|
+
throw new TypeError(
|
|
707
|
+
"sweepUnpinnedContent: reclaimOrphans needs a content store with listPage() (ContentStore SPI 1.3.0) — orphans cannot be found without enumerating the store"
|
|
708
|
+
);
|
|
709
|
+
}
|
|
710
|
+
if (orphans && store.deleteIfUntouchedSince === undefined) {
|
|
711
|
+
throw new TypeError(
|
|
712
|
+
"sweepUnpinnedContent: reclaimOrphans needs a content store with deleteIfUntouchedSince() (ContentStore SPI 1.3.0) — a plain delete of a listed orphan loses a blob re-put since the listing"
|
|
713
|
+
);
|
|
714
|
+
}
|
|
715
|
+
if (orphans && options.dryRun !== true && store.touch === undefined) {
|
|
716
|
+
throw new ReclaimOrphansTouchError();
|
|
717
|
+
}
|
|
718
|
+
if (
|
|
719
|
+
orphans &&
|
|
720
|
+
options.dryRun !== true &&
|
|
721
|
+
options.snapshotStores !== undefined &&
|
|
722
|
+
(requested as { snapshotsTouchContent?: unknown }).snapshotsTouchContent !== true
|
|
723
|
+
) {
|
|
724
|
+
throw new ReclaimOrphansSnapshotAckError();
|
|
725
|
+
}
|
|
726
|
+
if (orphans && options.dryRun !== true && store.touchClock === undefined) {
|
|
727
|
+
throw new ReclaimOrphansClockError();
|
|
728
|
+
}
|
|
729
|
+
if (
|
|
730
|
+
orphans &&
|
|
731
|
+
options.snapshotStores === undefined &&
|
|
732
|
+
options.eventStores.some(
|
|
733
|
+
(s) => typeof (s as { truncatePrefix?: unknown }).truncatePrefix === "function"
|
|
734
|
+
)
|
|
735
|
+
) {
|
|
736
|
+
throw new TypeError(
|
|
737
|
+
"sweepUnpinnedContent: reclaimOrphans over an event store that can truncatePrefix needs snapshotStores — a compacted prefix's refs live only in its snapshot, and without it they look like orphans"
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
const snapshotList = options.snapshotStores ?? [];
|
|
741
|
+
const snapshotStoresTouching = snapshotList.filter(isContentTouchingSnapshotStore).length;
|
|
742
|
+
const snapshotStoresUnbranded = snapshotList.length - snapshotStoresTouching;
|
|
743
|
+
if (
|
|
744
|
+
orphans &&
|
|
745
|
+
options.dryRun !== true &&
|
|
746
|
+
snapshotStoresTouching > 0 &&
|
|
747
|
+
snapshotStoresUnbranded > 0
|
|
748
|
+
) {
|
|
749
|
+
emitUnwrappedWarning(snapshotStoresTouching, snapshotStoresUnbranded);
|
|
750
|
+
}
|
|
751
|
+
// ADR 0055 D5 — the guard cutoff is on the STORE's clock, read once at the START: a blob
|
|
752
|
+
// touched after it (a re-citing put the scan could not see yet) is refused. The TTL on the
|
|
753
|
+
// log (`now`) is a different clock and never feeds it.
|
|
754
|
+
const raceSafe = store.deleteIfUntouchedSince !== undefined;
|
|
755
|
+
let cutoff = Number.NEGATIVE_INFINITY;
|
|
756
|
+
// The store's clock as read ONCE for this sweep: the cutoff AND the orphan margin below are
|
|
757
|
+
// measured against the same reading. A second read made the margin `olderThanMs` plus however
|
|
758
|
+
// many milliseconds elapsed between the reads, so a sub-minute TTL could pass the guard.
|
|
759
|
+
let storeNow: number | undefined;
|
|
760
|
+
if (raceSafe) {
|
|
761
|
+
if (options.touchedBefore !== undefined) {
|
|
762
|
+
cutoff = options.touchedBefore;
|
|
763
|
+
} else if (store.touchClock !== undefined) {
|
|
764
|
+
storeNow = await store.touchClock();
|
|
765
|
+
cutoff = storeNow - options.olderThanMs;
|
|
766
|
+
} else {
|
|
767
|
+
throw new TypeError(
|
|
768
|
+
"sweepUnpinnedContent: the content store has deleteIfUntouchedSince() but no touchClock() — pass touchedBefore (a cutoff on the store's clock) explicitly"
|
|
769
|
+
);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
const dryRun = options.dryRun === true;
|
|
773
|
+
if (orphans && !dryRun) {
|
|
774
|
+
// The touch protects a write only while its touch-to-commit window is shorter than the
|
|
775
|
+
// margin between the store's clock and the cutoff (module note; ADR 0055 D6 amendment 2).
|
|
776
|
+
if (!(options.olderThanMs > 0)) throw new ReclaimOrphansMarginError(options.olderThanMs);
|
|
777
|
+
const margin = (storeNow ?? (await store.touchClock!())) - cutoff;
|
|
778
|
+
if (!(margin >= MIN_ORPHAN_ERASE_MARGIN_MS)) throw new ReclaimOrphansMarginError(margin);
|
|
779
|
+
}
|
|
780
|
+
/** Erase one address; `true` iff it is gone because of this sweep (or would be). */
|
|
781
|
+
let keptByTouch = 0;
|
|
782
|
+
const erase = async (address: string): Promise<boolean> => {
|
|
783
|
+
if (dryRun) return true;
|
|
784
|
+
const ref = { __loomContent: address };
|
|
785
|
+
if (store.deleteIfUntouchedSince !== undefined) {
|
|
786
|
+
const gone = await store.deleteIfUntouchedSince(ref, cutoff);
|
|
787
|
+
if (!gone) keptByTouch++;
|
|
788
|
+
return gone;
|
|
789
|
+
}
|
|
790
|
+
await store.delete(ref);
|
|
791
|
+
return true;
|
|
792
|
+
};
|
|
793
|
+
|
|
794
|
+
// Mark memory is O(distinct cited addresses): one Map entry per address plus the Sets.
|
|
795
|
+
const pinnedKinds = new Set(options.pinnedEventTypes);
|
|
796
|
+
// ADR 0146: the EFFECTIVE time per ref (end clock). Dry run alone also keeps the raw citation time.
|
|
797
|
+
const lastCited = new Map<string, number>();
|
|
798
|
+
const rawCited = options.dryRun === true ? new Map<string, number>() : undefined;
|
|
799
|
+
const viaEnd = new Set<string>();
|
|
800
|
+
// Refs whose newest citation came from a RE-OPENED execution (same tie rule as `viaEnd`).
|
|
801
|
+
const viaReopened = new Set<string>();
|
|
802
|
+
const pinned = new Set<string>();
|
|
803
|
+
const loadBearing = new Set<string>();
|
|
804
|
+
let executionsScanned = 0;
|
|
805
|
+
const uncovered: string[] = [];
|
|
806
|
+
const sealedSnapshots: string[] = [];
|
|
807
|
+
const shreddedSnapshots: string[] = [];
|
|
808
|
+
const compactedLogs = new Set<string>();
|
|
809
|
+
// EVERY compacted log's removed range (the `ckpt-0138-` mark is NOT consulted: it can be lost to
|
|
810
|
+
// pruning or a fork seed, and a heuristic must not gate an ERASE): each must be covered by the
|
|
811
|
+
// given archives' headers, else refused unless `acknowledgeUnarchivedLogs`.
|
|
812
|
+
const archivedRanges = new Map<string, Array<[number, number]>>();
|
|
813
|
+
const sealedArchives: string[] = [];
|
|
814
|
+
|
|
815
|
+
for (const eventStore of options.eventStores) {
|
|
816
|
+
if (eventStore.listExecutions === undefined && eventStore.listExecutionsPage === undefined) {
|
|
817
|
+
throw new TypeError(
|
|
818
|
+
"sweepUnpinnedContent: every event store must support listExecutionsPage() or listExecutions() — a store that cannot be enumerated cannot be swept safely"
|
|
819
|
+
);
|
|
820
|
+
}
|
|
821
|
+
for await (const executionId of iterateExecutionIds(eventStore)) {
|
|
822
|
+
executionsScanned++;
|
|
823
|
+
const events = await eventStore.read(executionId);
|
|
824
|
+
// ADR 0146: a terminal execution's citations age from its END (never earlier than the citation).
|
|
825
|
+
const endedAt = isReservedStream(executionId) ? undefined : terminalEndOf(events);
|
|
826
|
+
const reopened =
|
|
827
|
+
!isReservedStream(executionId) && endedAt === undefined && wasReopened(events);
|
|
828
|
+
for (const event of events) {
|
|
829
|
+
// Only a finite end strictly after the citation moves the clock; anything else is today's clock exactly.
|
|
830
|
+
const at = endedAt !== undefined && endedAt > event.timestamp ? endedAt : event.timestamp;
|
|
831
|
+
visitCitedAddresses(
|
|
832
|
+
event.payload,
|
|
833
|
+
(address, inEnvelope) => {
|
|
834
|
+
const prior = lastCited.get(address);
|
|
835
|
+
if (prior === undefined || at > prior) {
|
|
836
|
+
lastCited.set(address, at);
|
|
837
|
+
if (at > event.timestamp) viaEnd.add(address);
|
|
838
|
+
else viaEnd.delete(address);
|
|
839
|
+
if (reopened) viaReopened.add(address);
|
|
840
|
+
else viaReopened.delete(address);
|
|
841
|
+
}
|
|
842
|
+
if (rawCited !== undefined) {
|
|
843
|
+
const rawPrior = rawCited.get(address);
|
|
844
|
+
if (rawPrior === undefined || event.timestamp > rawPrior) {
|
|
845
|
+
rawCited.set(address, event.timestamp);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
if (pinnedKinds.has(event.eventType)) pinned.add(address);
|
|
849
|
+
if (!inEnvelope) loadBearing.add(address);
|
|
850
|
+
},
|
|
851
|
+
retainRootsOf(event)
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
// Did compaction touch this log, and where is its checkpoint (ADR 0106)? Compaction
|
|
855
|
+
// removes its checkpoint's own slot and, whenever it keeps pinned-kind events, never
|
|
856
|
+
// the tip (the tip rule). So a gap-free log with more than the bootstrap was never
|
|
857
|
+
// compacted and costs no snapshot read; a gapped log's checkpoint sits at its highest
|
|
858
|
+
// missing sequence `gap`; and a bootstrap-only log was compacted through its tip (or
|
|
859
|
+
// never got past its bootstrap), its checkpoint above the tip.
|
|
860
|
+
if (events.length === 0) continue;
|
|
861
|
+
const tipSequence = events[events.length - 1]!.sequence;
|
|
862
|
+
// ADR 0139 - a RESERVED stream (the inbox, its topic streams, the quarantine copies) is never
|
|
863
|
+
// checkpointed: a trim MOVES its consumed prefix to the archive instead. Its removed range is
|
|
864
|
+
// what its sequences skip from its first possible one (a topic stream starts at
|
|
865
|
+
// INBOX_TOPIC_SEQ_BASE, not 0, so a bare "gap" test would call every topic stream compacted), and
|
|
866
|
+
// an erasing pass needs the archive to cover it, exactly as for a compacted log (D24).
|
|
867
|
+
if (isReservedStream(executionId)) {
|
|
868
|
+
const runs = missingRunsFrom(
|
|
869
|
+
events,
|
|
870
|
+
isInboxTopicStream(executionId) ? INBOX_TOPIC_SEQ_BASE : 1,
|
|
871
|
+
tipSequence
|
|
872
|
+
);
|
|
873
|
+
if (runs.length > 0) {
|
|
874
|
+
compactedLogs.add(executionId);
|
|
875
|
+
archivedRanges.set(executionId, runs);
|
|
876
|
+
}
|
|
877
|
+
continue;
|
|
878
|
+
}
|
|
879
|
+
const gap = highestMissingSequence(events);
|
|
880
|
+
const bootstrapOnly = events.length === 1;
|
|
881
|
+
if (gap < 0 && !bootstrapOnly) continue;
|
|
882
|
+
// ADR 0138 - a gapped log is compacted: an erasing orphan pass needs its archive, else the
|
|
883
|
+
// moved events' blobs look unreferenced. (A bootstrap-only log counts once a checkpoint shows.)
|
|
884
|
+
if (gap >= 0) {
|
|
885
|
+
compactedLogs.add(executionId);
|
|
886
|
+
archivedRanges.set(executionId, missingRunsWithin(events, gap));
|
|
887
|
+
}
|
|
888
|
+
if (options.snapshotStores === undefined) {
|
|
889
|
+
if (gap >= 0) uncovered.push(executionId); // a gap is log evidence
|
|
890
|
+
continue;
|
|
891
|
+
}
|
|
892
|
+
const pin = (snapshot: { state: unknown }): void => {
|
|
893
|
+
const st = snapshot.state;
|
|
894
|
+
if (typeof st === "object" && st !== null && "__loomEnc" in st) {
|
|
895
|
+
if (!sealedSnapshots.includes(executionId)) sealedSnapshots.push(executionId);
|
|
896
|
+
}
|
|
897
|
+
visitCitedAddresses(snapshot.state, (address) => {
|
|
898
|
+
if (!lastCited.has(address)) lastCited.set(address, Number.POSITIVE_INFINITY);
|
|
899
|
+
pinned.add(address);
|
|
900
|
+
});
|
|
901
|
+
};
|
|
902
|
+
let covered = false;
|
|
903
|
+
for (const snapshots of options.snapshotStores) {
|
|
904
|
+
try {
|
|
905
|
+
const latest = await snapshots.loadLatest(executionId);
|
|
906
|
+
if (latest !== null && latest.sequence >= 1) pin(latest);
|
|
907
|
+
if (bootstrapOnly) {
|
|
908
|
+
// Compacted at the tip: the checkpoint IS the latest snapshot.
|
|
909
|
+
if (latest !== null && latest.sequence >= 1) {
|
|
910
|
+
covered = true;
|
|
911
|
+
compactedLogs.add(executionId);
|
|
912
|
+
archivedRanges.set(executionId, [[1, latest.sequence]]);
|
|
913
|
+
}
|
|
914
|
+
continue;
|
|
915
|
+
}
|
|
916
|
+
// The checkpoint: the LOWEST snapshot in [gap, tip] — any snapshot at or above the gap
|
|
917
|
+
// covers a lost checkpoint, exactly as `compact` accepts a base at or above it. A
|
|
918
|
+
// compaction's own sits exactly at `gap` (one read). A child forked from a compacted
|
|
919
|
+
// parent copies the gap, and its lowest snapshot is its fork seed, whose sequence the
|
|
920
|
+
// copied envelopes record (`metadata.forkSequence`): one exact read. Only a log with
|
|
921
|
+
// neither walks the snapshots down from the tip.
|
|
922
|
+
let checkpoint = await snapshots.loadLatestBefore(executionId, gap);
|
|
923
|
+
const forkSequence = (events[0]!.metadata as { forkSequence?: unknown } | undefined)
|
|
924
|
+
?.forkSequence;
|
|
925
|
+
if (
|
|
926
|
+
(checkpoint === null || checkpoint.sequence !== gap) &&
|
|
927
|
+
typeof forkSequence === "number" &&
|
|
928
|
+
forkSequence >= gap
|
|
929
|
+
) {
|
|
930
|
+
const seed =
|
|
931
|
+
snapshots.loadAt !== undefined
|
|
932
|
+
? await snapshots.loadAt(executionId, forkSequence)
|
|
933
|
+
: await snapshots.loadLatestBefore(executionId, forkSequence);
|
|
934
|
+
checkpoint = seed !== null && seed.sequence === forkSequence ? seed : checkpoint;
|
|
935
|
+
}
|
|
936
|
+
if (checkpoint === null || checkpoint.sequence < gap) {
|
|
937
|
+
checkpoint = null;
|
|
938
|
+
let s = await snapshots.loadLatestBefore(executionId, tipSequence);
|
|
939
|
+
while (s !== null && s.sequence >= gap) {
|
|
940
|
+
checkpoint = s;
|
|
941
|
+
s =
|
|
942
|
+
s.sequence > gap
|
|
943
|
+
? await snapshots.loadLatestBefore(executionId, s.sequence - 1)
|
|
944
|
+
: null;
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
if (checkpoint !== null) {
|
|
948
|
+
pin(checkpoint);
|
|
949
|
+
covered = true;
|
|
950
|
+
}
|
|
951
|
+
} catch (err) {
|
|
952
|
+
// Only a CONFIRMED crypto-shred means the snapshot is erased: the store's error (recognised by the realm-safe
|
|
953
|
+
// brand, never by name) AND the execution's own `PAYLOAD_SHREDDED`
|
|
954
|
+
// tombstone. A node that merely lacks the key can report the same error, so without the
|
|
955
|
+
// tombstone it propagates like anything else (undecryptable, I/O, a key unavailable with no
|
|
956
|
+
// tombstone): dropping refs for data that still exists would delete live blobs. A
|
|
957
|
+
// snapshot store without the raw log reports a shredded subject's PRE-shred snapshot as
|
|
958
|
+
// `KeyUnavailableError` (it cannot see the tombstone above it), so that name is accepted
|
|
959
|
+
// too, but ONLY beside the tombstone.
|
|
960
|
+
const tombstoned = events.some((e) => e.eventType === "PAYLOAD_SHREDDED");
|
|
961
|
+
if (
|
|
962
|
+
!(err instanceof Error) ||
|
|
963
|
+
(!isPayloadShredded(err) && err.name !== "KeyUnavailableError") ||
|
|
964
|
+
!tombstoned
|
|
965
|
+
)
|
|
966
|
+
throw err;
|
|
967
|
+
if (!shreddedSnapshots.includes(executionId)) shreddedSnapshots.push(executionId);
|
|
968
|
+
covered = true;
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
if (!covered) uncovered.push(executionId);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
// ADR 0138 - what compaction MOVED out of the logs still cites content: pin it permanently. The
|
|
975
|
+
// archive is enumerated (not walked per log), so segments of an execution whose log is gone count.
|
|
976
|
+
const anyLog = async (executionId: string): Promise<EventEnvelope[]> => {
|
|
977
|
+
const all: EventEnvelope[] = [];
|
|
978
|
+
for (const es of options.eventStores) all.push(...(await es.read(executionId)));
|
|
979
|
+
return all;
|
|
980
|
+
};
|
|
981
|
+
for (const archive of options.archiveStores ?? []) {
|
|
982
|
+
let cursor: string | undefined;
|
|
983
|
+
do {
|
|
984
|
+
const page = await archive.listExecutions({ ...(cursor !== undefined && { cursor }) });
|
|
985
|
+
for (const executionId of page.executionIds) {
|
|
986
|
+
// ADR 0139 - a reserved stream that is DELETED (nothing live to derive a gap from) is judged by
|
|
987
|
+
// the archive's own extent: a hole between its first possible sequence and its last archived one
|
|
988
|
+
// (a trim before the archive existed) is history nothing holds, exactly like a live stream's gap.
|
|
989
|
+
if (isReservedStream(executionId) && !executionId.startsWith("__loom_late__:")) {
|
|
990
|
+
if (!archivedRanges.has(executionId)) {
|
|
991
|
+
const headers = await archive.headers(executionId);
|
|
992
|
+
const first = isInboxTopicStream(executionId) ? INBOX_TOPIC_SEQ_BASE : 1;
|
|
993
|
+
const extent = headers.length > 0 ? headers[headers.length - 1]!.through : -1;
|
|
994
|
+
const holes = missingRunsFrom(
|
|
995
|
+
headers.flatMap((h) =>
|
|
996
|
+
Array.from(new Set([h.from, h.through])).map((sequence) => ({ sequence }))
|
|
997
|
+
),
|
|
998
|
+
first,
|
|
999
|
+
extent
|
|
1000
|
+
).filter(
|
|
1001
|
+
([x, y]) =>
|
|
1002
|
+
!rangeCovered(
|
|
1003
|
+
x,
|
|
1004
|
+
y,
|
|
1005
|
+
headers.map((h) => [h.from, h.through] as const)
|
|
1006
|
+
)
|
|
1007
|
+
);
|
|
1008
|
+
if (holes.length > 0) {
|
|
1009
|
+
compactedLogs.add(executionId);
|
|
1010
|
+
archivedRanges.set(executionId, holes);
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
let segments: Awaited<ReturnType<ArchiveStore["read"]>>;
|
|
1015
|
+
try {
|
|
1016
|
+
segments = await archive.read(executionId);
|
|
1017
|
+
} catch (err) {
|
|
1018
|
+
// Only a CONFIRMED shred (the realm-safe brand AND the execution's own tombstone) means the
|
|
1019
|
+
// archived copy is erased; anything else (a missing key on this node, I/O) propagates.
|
|
1020
|
+
if (
|
|
1021
|
+
isPayloadShredded(err) &&
|
|
1022
|
+
(await anyLog(executionId)).some((e) => e.eventType === "PAYLOAD_SHREDDED")
|
|
1023
|
+
) {
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
1026
|
+
throw err;
|
|
1027
|
+
}
|
|
1028
|
+
for (const seg of segments) {
|
|
1029
|
+
if (seg.count > 0 && seg.envelopes.length === 0) {
|
|
1030
|
+
if (!sealedArchives.includes(executionId)) sealedArchives.push(executionId);
|
|
1031
|
+
continue;
|
|
1032
|
+
}
|
|
1033
|
+
for (const event of seg.envelopes) {
|
|
1034
|
+
visitCitedAddresses(event.payload, (address) => {
|
|
1035
|
+
const prior = lastCited.get(address);
|
|
1036
|
+
if (prior === undefined || event.timestamp > prior) {
|
|
1037
|
+
lastCited.set(address, event.timestamp);
|
|
1038
|
+
viaEnd.delete(address);
|
|
1039
|
+
viaReopened.delete(address);
|
|
1040
|
+
}
|
|
1041
|
+
if (rawCited !== undefined) {
|
|
1042
|
+
const rawPrior = rawCited.get(address);
|
|
1043
|
+
if (rawPrior === undefined || event.timestamp > rawPrior) {
|
|
1044
|
+
rawCited.set(address, event.timestamp);
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
pinned.add(address);
|
|
1048
|
+
loadBearing.add(address);
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
cursor = page.nextCursor;
|
|
1054
|
+
} while (cursor !== undefined);
|
|
1055
|
+
}
|
|
1056
|
+
if (!dryRun && sealedArchives.length > 0) throw new SealedArchiveScanError(sealedArchives);
|
|
1057
|
+
const unarchivedAcknowledged: string[] = [];
|
|
1058
|
+
if (orphans && !dryRun && options.archiveStores === undefined && compactedLogs.size > 0) {
|
|
1059
|
+
if (options.acknowledgeUnarchivedLogs !== true) {
|
|
1060
|
+
throw new ArchiveStoresRequiredError([...compactedLogs]);
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
// Coverage is CHECKED for every compacted log, whatever its checkpoint is called: `[]` or a
|
|
1064
|
+
// partial archive never suffices for a gapped log; only the headers (or the explicit
|
|
1065
|
+
// acknowledgment, for logs whose history is already gone) get past it.
|
|
1066
|
+
if (orphans && !dryRun && archivedRanges.size > 0) {
|
|
1067
|
+
const lost: string[] = [];
|
|
1068
|
+
for (const [executionId, runs] of archivedRanges) {
|
|
1069
|
+
const held: Array<[number, number]> = [];
|
|
1070
|
+
for (const archive of options.archiveStores ?? []) {
|
|
1071
|
+
for (const h of await archive.headers(executionId)) held.push([h.from, h.through]);
|
|
1072
|
+
}
|
|
1073
|
+
if (!runs.every(([a, b]) => rangeCovered(a, b, held))) lost.push(executionId);
|
|
1074
|
+
}
|
|
1075
|
+
if (lost.length > 0) {
|
|
1076
|
+
if (options.acknowledgeUnarchivedLogs !== true) {
|
|
1077
|
+
throw new ArchiveStoresRequiredError(lost, "uncovered");
|
|
1078
|
+
}
|
|
1079
|
+
unarchivedAcknowledged.push(...lost);
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
if (!dryRun && sealedSnapshots.length > 0) throw new SealedSnapshotScanError(sealedSnapshots);
|
|
1083
|
+
if (orphans && uncovered.length > 0) {
|
|
1084
|
+
const shown = uncovered.slice(0, 5).join(", ");
|
|
1085
|
+
const more = uncovered.length > 5 ? ` (+${uncovered.length - 5} more)` : "";
|
|
1086
|
+
throw new TypeError(
|
|
1087
|
+
`sweepUnpinnedContent: ${uncovered.length} execution(s) look compacted but no checkpoint in snapshotStores covers them (${shown}${more}) — their prefix refs cannot be seen, so reclaimOrphans refuses before erasing anything. A run that never got past its bootstrap looks the same: let it progress, or pass the snapshot store its checkpoint went to (each archive's too)`
|
|
1088
|
+
);
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
// ADR 0055 D7 — external root sets pin like a pinned event kind does.
|
|
1092
|
+
const roots = new Set<string>();
|
|
1093
|
+
if (options.roots !== undefined) {
|
|
1094
|
+
for await (const address of options.roots()) roots.add(address);
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
const reclaimed: string[] = [];
|
|
1098
|
+
const heldByEndClock: string[] = [];
|
|
1099
|
+
const reclaimedWhileReopened: string[] = [];
|
|
1100
|
+
let refsAgedByEnd = 0;
|
|
1101
|
+
let refsOfReopened = 0;
|
|
1102
|
+
for (const [address, citedAt] of lastCited) {
|
|
1103
|
+
if (pinned.has(address) || loadBearing.has(address) || roots.has(address)) continue;
|
|
1104
|
+
if (viaEnd.has(address)) refsAgedByEnd++;
|
|
1105
|
+
const ofReopened = viaReopened.has(address);
|
|
1106
|
+
if (ofReopened) refsOfReopened++;
|
|
1107
|
+
if (options.now - citedAt < options.olderThanMs) {
|
|
1108
|
+
const raw = rawCited?.get(address);
|
|
1109
|
+
if (raw !== undefined && options.now - raw >= options.olderThanMs) {
|
|
1110
|
+
heldByEndClock.push(address);
|
|
1111
|
+
}
|
|
1112
|
+
continue;
|
|
1113
|
+
}
|
|
1114
|
+
if (await erase(address)) {
|
|
1115
|
+
reclaimed.push(address);
|
|
1116
|
+
if (ofReopened && dryRun) reclaimedWhileReopened.push(address);
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// ADR 0055 D6 — orphans: stored, untouched since the cutoff, and cited by nothing scanned.
|
|
1121
|
+
// `pinned` and `loadBearing` are subsets of `lastCited`, so two Set lookups decide.
|
|
1122
|
+
// Keyset paging on address: erasing entries of a page never skips a later one.
|
|
1123
|
+
const orphansReclaimed: string[] = [];
|
|
1124
|
+
if (orphans) {
|
|
1125
|
+
let cursor: string | undefined;
|
|
1126
|
+
do {
|
|
1127
|
+
const page = await store.listPage!({
|
|
1128
|
+
...(cursor !== undefined && { cursor }),
|
|
1129
|
+
limit: ORPHAN_PAGE_LIMIT,
|
|
1130
|
+
touchedBefore: cutoff,
|
|
1131
|
+
});
|
|
1132
|
+
for (const { address, touchedAt } of page.entries) {
|
|
1133
|
+
// 0 = UNKNOWN: the column default a pre-1.3.0 writer leaves during a mixed rollout.
|
|
1134
|
+
if (!(touchedAt > 0)) continue;
|
|
1135
|
+
if (lastCited.has(address) || roots.has(address)) continue;
|
|
1136
|
+
if (await erase(address)) orphansReclaimed.push(address);
|
|
1137
|
+
}
|
|
1138
|
+
cursor = page.nextCursor;
|
|
1139
|
+
} while (cursor !== undefined);
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
return {
|
|
1143
|
+
executionsScanned,
|
|
1144
|
+
refsSeen: lastCited.size,
|
|
1145
|
+
refsPinned: pinned.size,
|
|
1146
|
+
refsLoadBearing: loadBearing.size,
|
|
1147
|
+
refsAgedByEnd,
|
|
1148
|
+
...(dryRun && { heldByEndClock }),
|
|
1149
|
+
refsOfReopened,
|
|
1150
|
+
...(dryRun && { reclaimedWhileReopened }),
|
|
1151
|
+
reclaimed,
|
|
1152
|
+
keptByTouch,
|
|
1153
|
+
...(raceSafe && { touchCutoff: cutoff }),
|
|
1154
|
+
raceSafe,
|
|
1155
|
+
orphansReclaimed,
|
|
1156
|
+
snapshotStoresTouching,
|
|
1157
|
+
snapshotStoresUnbranded,
|
|
1158
|
+
sealedSnapshots,
|
|
1159
|
+
shreddedSnapshots,
|
|
1160
|
+
unarchivedAcknowledged,
|
|
1161
|
+
};
|
|
1162
|
+
}
|
|
1163
|
+
|
|
1164
|
+
/** The runs of sequences in `[1..extent]` that `events` does not hold. */
|
|
1165
|
+
function missingRunsWithin(
|
|
1166
|
+
events: readonly { sequence: number }[],
|
|
1167
|
+
extent: number
|
|
1168
|
+
): Array<[number, number]> {
|
|
1169
|
+
const runs: Array<[number, number]> = [];
|
|
1170
|
+
let prev = 0;
|
|
1171
|
+
for (const e of events) {
|
|
1172
|
+
if (e.sequence === 0) continue;
|
|
1173
|
+
if (e.sequence > extent + 1) break;
|
|
1174
|
+
if (e.sequence - prev > 1) runs.push([prev + 1, e.sequence - 1]);
|
|
1175
|
+
prev = e.sequence;
|
|
1176
|
+
}
|
|
1177
|
+
if (prev < extent) runs.push([prev + 1, extent]);
|
|
1178
|
+
return runs;
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
/** The runs of sequences in `[first..extent]` that `events` does not hold. */
|
|
1182
|
+
function missingRunsFrom(
|
|
1183
|
+
events: readonly { sequence: number }[],
|
|
1184
|
+
first: number,
|
|
1185
|
+
extent: number
|
|
1186
|
+
): Array<[number, number]> {
|
|
1187
|
+
const runs: Array<[number, number]> = [];
|
|
1188
|
+
let prev = first - 1;
|
|
1189
|
+
for (const e of events) {
|
|
1190
|
+
if (e.sequence < first) continue;
|
|
1191
|
+
if (e.sequence > extent) break;
|
|
1192
|
+
if (e.sequence - prev > 1) runs.push([prev + 1, e.sequence - 1]);
|
|
1193
|
+
prev = e.sequence;
|
|
1194
|
+
}
|
|
1195
|
+
if (prev < extent) runs.push([prev + 1, extent]);
|
|
1196
|
+
return runs;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
/** A per-topic inbox stream (`__loom_inbox__:<chain>#<16 hex>`): its sequences start at INBOX_TOPIC_SEQ_BASE. */
|
|
1200
|
+
function isInboxTopicStream(executionId: string): boolean {
|
|
1201
|
+
return executionId.startsWith(INBOX_STREAM_PREFIX) && /#[0-9a-f]{16}$/.test(executionId);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
/** Whether the union of `held` ranges (inclusive) covers every sequence of `[a..b]`. */
|
|
1205
|
+
function rangeCovered(a: number, b: number, held: readonly (readonly [number, number])[]): boolean {
|
|
1206
|
+
let next = a;
|
|
1207
|
+
for (const [from, through] of [...held].sort((x, y) => x[0] - y[0])) {
|
|
1208
|
+
if (from > next) break;
|
|
1209
|
+
if (through >= next) next = through + 1;
|
|
1210
|
+
if (next > b) return true;
|
|
1211
|
+
}
|
|
1212
|
+
return next > b;
|
|
1213
|
+
}
|