@cloudbitmaps/roaring 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -2
- package/dist/bin/compact-segments.js +70 -3
- package/dist/bin/compact-segments.js.map +1 -1
- package/dist/bin/export-segments.js +1 -1
- package/dist/{chunk-DAV4F5WM.js → chunk-D6LIJSGF.js} +125 -15
- package/dist/chunk-D6LIJSGF.js.map +1 -0
- package/dist/index.cjs +122 -12
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +100 -3
- package/dist/index.d.ts +100 -3
- package/dist/index.js +123 -13
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/dist/chunk-DAV4F5WM.js.map +0 -1
package/README.md
CHANGED
|
@@ -48,7 +48,7 @@ bit array, byte for byte what you have now. It just stops paying for the chunks
|
|
|
48
48
|
| `prior = SETBIT` | `claimMany(ids)` — adds a batch and returns only the ids that were **not** already there, so an *"already sent to this id?"* check is exactly-once |
|
|
49
49
|
| `BITCOUNT` | `count()` — exact, served from the index with no payload reads |
|
|
50
50
|
| `BITOP AND` / `OR` / `DIFF` | `intersect` / `union` / `andNot` — and `intersect` skips chunks that cannot contribute |
|
|
51
|
-
| `EXPIRE` |
|
|
51
|
+
| `EXPIRE` | **`setRetention(ref, { expiresAt })` + `retireExpired()`** — a per-segment expiry the writer sets, and a sweep **you** schedule (this library starts no timer, so it works the same in a Lambda and a server). No per-**id** TTL: a bitmap stores ids, not timestamps. And never enable your backend's own row expiry on the warm table — those rows are un-compacted deltas, so it discards writes silently |
|
|
52
52
|
|
|
53
53
|
**It is not a drop-in replacement, and two limits are worth knowing before you port anything.**
|
|
54
54
|
|
|
@@ -67,6 +67,39 @@ against. Raw bit-position import/export is unbuilt;
|
|
|
67
67
|
[say so in an issue](https://github.com/cloudbitmaps/cloudbitmaps/issues) if you need it, because that is what
|
|
68
68
|
decides whether it gets built.
|
|
69
69
|
|
|
70
|
+
## Retiring data: a per-segment expiry, and a sweep you schedule
|
|
71
|
+
|
|
72
|
+
A **segment** can expire; an individual **id** cannot (a bitmap stores ids, not `(id, timestamp)` pairs — a
|
|
73
|
+
timestamp per id costs more than the compression saves).
|
|
74
|
+
|
|
75
|
+
```ts
|
|
76
|
+
const DAY = 86_400_000;
|
|
77
|
+
const ref = { namespace: 'active-daily', segment: '2026-08-05' };
|
|
78
|
+
|
|
79
|
+
await store.setRetention(ref, { expiresAt: Date.now() + 30 * DAY }); // once, when you create the bucket
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`expiresAt` is an absolute instant **you** compute, not a duration derived from anything observed: compaction
|
|
83
|
+
rewrites every basis such a duration could use, so "30 days since the last write" would keep a busy bucket alive
|
|
84
|
+
precisely *because* the daemon was keeping it cheap.
|
|
85
|
+
|
|
86
|
+
Then, from whatever schedule your deployment already has — an EventBridge rule, a Kubernetes `CronJob`, `cron`, a
|
|
87
|
+
queue job — run the sweep. **This library starts no background timer**, deliberately: the same code has to behave
|
|
88
|
+
identically in a Lambda, an edge isolate and a long-lived server, and a timer that only works in one of those is
|
|
89
|
+
worse than none.
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
93
|
+
for (const e of swept.entries) if (e.action === 'skipped') console.warn(e.segment, e.reason);
|
|
94
|
+
if (swept.limited) scheduleAnotherPassSoon(); // the per-cycle cap deferred some; re-run
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Each retirement goes through `dropSegment`, so the Warm → registry → Cold ordering is one implementation rather
|
|
98
|
+
than two. The sweep is bounded (`limit`, default 100), previewable (`dryRun`), and returns a per-segment ledger
|
|
99
|
+
rather than throwing — a fault on one segment must not decide the fate of the other ninety-nine. Once a day is
|
|
100
|
+
enough for daily buckets. Full walkthrough:
|
|
101
|
+
[getting-started §13.5](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/getting-started.md#135-retention-ttl-and-pruning--what-exists-and-what-doesnt).
|
|
102
|
+
|
|
70
103
|
## No seed step, and compaction is optional
|
|
71
104
|
|
|
72
105
|
A brand-new segment is usable the moment you construct the store: `addMany`, `has`, `remove`, `count`, `iterate`
|
|
@@ -75,7 +108,8 @@ merges `(cold ∪ warm.adds) \ warm.removes` and an absent cold tier just makes
|
|
|
75
108
|
**import** path for data you already have elsewhere, not an initialization step.
|
|
76
109
|
|
|
77
110
|
Compaction is a **cost** optimization, never a correctness requirement — it buys cheaper storage, smaller rewrites,
|
|
78
|
-
and a free index-only `count()`. For a write-once dated bucket you retire with `dropSegment`,
|
|
111
|
+
and a free index-only `count()`. For a write-once dated bucket you retire (with `dropSegment`, or by recording an
|
|
112
|
+
expiry as above), skip it entirely.
|
|
79
113
|
|
|
80
114
|
Redis stays first-class as a **warm tier** underneath this (`@cloudbitmaps/roaring/redis`) — the point above is
|
|
81
115
|
about replacing `SETBIT`-on-one-giant-key as your *data model*, not replacing Redis as infrastructure.
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { SystemClock, src_exports, runCompactionCycle } from '../chunk-
|
|
2
|
+
import { SystemClock, src_exports, runCompactionCycle } from '../chunk-D6LIJSGF.js';
|
|
3
3
|
import { hostname } from 'os';
|
|
4
4
|
import { join } from 'path';
|
|
5
5
|
import { pathToFileURL } from 'url';
|
|
@@ -44,7 +44,22 @@ function parseConfig(env) {
|
|
|
44
44
|
owner: env.CR_COMPACT_OWNER ?? `${hostname()}:${process.pid}`,
|
|
45
45
|
shard,
|
|
46
46
|
totalShards,
|
|
47
|
-
maxSegments: intOpt("CR_COMPACT_MAX_SEGMENTS", 1)
|
|
47
|
+
maxSegments: intOpt("CR_COMPACT_MAX_SEGMENTS", 1),
|
|
48
|
+
retire: env.CR_RETIRE === "1" ? parseRetireConfig(env, num, intOpt) : void 0
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
function parseRetireConfig(env, num, intOpt) {
|
|
52
|
+
return {
|
|
53
|
+
// `intOpt(name, 1)`, not `num`: `num` accepts 0 and 1.5, and `retireExpired` then throws a ValidationError
|
|
54
|
+
// from *inside* the cycle — swallowed into a generic `{"event":"error"}` line every 30 s in loop mode, so
|
|
55
|
+
// compaction keeps reporting healthy cycles while retention silently never runs. `parseConfig` promises to
|
|
56
|
+
// throw a clear error on misconfiguration; this is what keeps that promise. (CR_COMPACT_MAX_SEGMENTS, the
|
|
57
|
+
// identical concept, already used `intOpt`.)
|
|
58
|
+
limit: intOpt("CR_RETIRE_LIMIT", 1) ?? src_exports.DEFAULT_RETIRE_LIMIT,
|
|
59
|
+
dryRun: env.CR_RETIRE_DRY_RUN === "1",
|
|
60
|
+
intervalMs: num("CR_RETIRE_INTERVAL_MS", src_exports.DEFAULT_TOMBSTONE_GRACE_MS),
|
|
61
|
+
purgeTombstones: env.CR_RETIRE_KEEP_TOMBSTONES !== "1",
|
|
62
|
+
tombstoneGraceMs: num("CR_RETIRE_TOMBSTONE_GRACE_MS", src_exports.DEFAULT_TOMBSTONE_GRACE_MS)
|
|
48
63
|
};
|
|
49
64
|
}
|
|
50
65
|
function localFsDeps(root, metrics) {
|
|
@@ -69,6 +84,50 @@ async function runOnce(deps, config) {
|
|
|
69
84
|
maxSegments: config.maxSegments
|
|
70
85
|
});
|
|
71
86
|
}
|
|
87
|
+
async function retireOnce(deps, config, lastSweptAt) {
|
|
88
|
+
const retire = config.retire;
|
|
89
|
+
if (retire === void 0) return void 0;
|
|
90
|
+
if (lastSweptAt !== void 0 && deps.clock.now() - lastSweptAt < retire.intervalMs) {
|
|
91
|
+
return void 0;
|
|
92
|
+
}
|
|
93
|
+
return (0, src_exports.retireExpired)(
|
|
94
|
+
{ registry: deps.registry, warm: deps.warm, cold: deps.cold },
|
|
95
|
+
{
|
|
96
|
+
namespace: config.namespace,
|
|
97
|
+
now: deps.clock.now(),
|
|
98
|
+
limit: retire.limit,
|
|
99
|
+
dryRun: retire.dryRun,
|
|
100
|
+
purgeTombstones: retire.purgeTombstones,
|
|
101
|
+
tombstoneGraceMs: retire.tombstoneGraceMs
|
|
102
|
+
}
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
var LOG_SKIPPED_SAMPLE = 20;
|
|
106
|
+
function logRetire(result) {
|
|
107
|
+
const skipped = result.entries.filter((e) => e.action === "skipped");
|
|
108
|
+
const byReason = {};
|
|
109
|
+
for (const e of skipped) {
|
|
110
|
+
const key = e.reason.startsWith("failed:") ? "failed" : e.reason;
|
|
111
|
+
byReason[key] = (byReason[key] ?? 0) + 1;
|
|
112
|
+
}
|
|
113
|
+
const notReclaimed = result.entries.filter(
|
|
114
|
+
(e) => e.action === "retired" && e.result.generationsRemaining.length > 0
|
|
115
|
+
);
|
|
116
|
+
log({
|
|
117
|
+
event: "retire",
|
|
118
|
+
dryRun: result.dryRun,
|
|
119
|
+
scanned: result.scanned,
|
|
120
|
+
eligible: result.eligible,
|
|
121
|
+
retired: result.retired,
|
|
122
|
+
wouldRetire: result.wouldRetire,
|
|
123
|
+
tombstonesPurged: result.tombstonesPurged,
|
|
124
|
+
limited: result.limited,
|
|
125
|
+
skippedByReason: byReason,
|
|
126
|
+
skippedSample: skipped.slice(0, LOG_SKIPPED_SAMPLE).map((e) => ({ segment: e.segment, namespace: e.namespace, reason: e.reason })),
|
|
127
|
+
notFullyReclaimed: notReclaimed.slice(0, LOG_SKIPPED_SAMPLE).map((e) => e.segment),
|
|
128
|
+
notFullyReclaimedCount: notReclaimed.length
|
|
129
|
+
});
|
|
130
|
+
}
|
|
72
131
|
var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
73
132
|
var log = (obj) => {
|
|
74
133
|
process.stdout.write(`${JSON.stringify(obj)}
|
|
@@ -95,6 +154,8 @@ async function main(env = process.env) {
|
|
|
95
154
|
deferred: cycle.deferred,
|
|
96
155
|
results: cycle.results
|
|
97
156
|
});
|
|
157
|
+
const swept = await retireOnce(deps, config);
|
|
158
|
+
if (swept !== void 0) logRetire(swept);
|
|
98
159
|
return;
|
|
99
160
|
}
|
|
100
161
|
let stop = false;
|
|
@@ -104,6 +165,7 @@ async function main(env = process.env) {
|
|
|
104
165
|
process.on("SIGINT", onSignal);
|
|
105
166
|
process.on("SIGTERM", onSignal);
|
|
106
167
|
log({ event: "start", mode: "loop", intervalMs: config.intervalMs, owner: config.owner });
|
|
168
|
+
let lastSweptAt;
|
|
107
169
|
while (!stop) {
|
|
108
170
|
try {
|
|
109
171
|
const cycle = await runOnce(deps, config);
|
|
@@ -114,6 +176,11 @@ async function main(env = process.env) {
|
|
|
114
176
|
deferred: cycle.deferred,
|
|
115
177
|
results: cycle.results
|
|
116
178
|
});
|
|
179
|
+
const swept = await retireOnce(deps, config, lastSweptAt);
|
|
180
|
+
if (swept !== void 0) {
|
|
181
|
+
lastSweptAt = deps.clock.now();
|
|
182
|
+
logRetire(swept);
|
|
183
|
+
}
|
|
117
184
|
} catch (err) {
|
|
118
185
|
log({ event: "error", message: err.message });
|
|
119
186
|
}
|
|
@@ -129,6 +196,6 @@ if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv
|
|
|
129
196
|
});
|
|
130
197
|
}
|
|
131
198
|
|
|
132
|
-
export { localFsDeps, main, parseConfig, runOnce };
|
|
199
|
+
export { localFsDeps, main, parseConfig, retireOnce, runOnce };
|
|
133
200
|
//# sourceMappingURL=compact-segments.js.map
|
|
134
201
|
//# sourceMappingURL=compact-segments.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/bin/compact-segments.ts"],"names":[],"mappings":";;;;;;AAyDO,SAAS,YAAY,GAAA,EAAwD;AAClF,EAAA,MAAM,OAAO,GAAA,CAAI,eAAA;AACjB,EAAA,IAAI,IAAA,KAAS,MAAA,IAAa,IAAA,KAAS,EAAA,EAAI;AACrC,IAAA,MAAM,IAAI,MAAM,iEAAiE,CAAA;AAAA,EACnF;AACA,EAAA,MAAM,IAAA,GAAO,IAAI,eAAA,IAAmB,MAAA;AACpC,EAAA,IAAI,IAAA,KAAS,MAAA,IAAU,IAAA,KAAS,MAAA,EAAQ;AACtC,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,8CAAA,EAAiD,IAAI,CAAA,CAAE,CAAA;AAAA,EACzE;AACA,EAAA,MAAM,GAAA,GAAM,CAAC,IAAA,EAAc,GAAA,KAAwB;AACjD,IAAA,MAAM,GAAA,GAAM,IAAI,IAAI,CAAA;AACpB,IAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI,OAAO,GAAA;AAC5C,IAAA,MAAM,CAAA,GAAI,OAAO,GAAG,CAAA;AACpB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,CAAC,KAAK,CAAA,GAAI,CAAA;AAC7B,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,EAAG,IAAI,CAAA,oCAAA,EAAuC,GAAG,CAAA,CAAE,CAAA;AACrE,IAAA,OAAO,CAAA;AAAA,EACT,CAAA;AACA,EAAA,MAAM,MAAA,GAAS,CAAC,IAAA,EAAc,GAAA,GAAM,CAAA,KAA0B;AAC5D,IAAA,MAAM,GAAA,GAAM,IAAI,IAAI,CAAA;AACpB,IAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI,OAAO,MAAA;AAC5C,IAAA,MAAM,CAAA,GAAI,OAAO,GAAG,CAAA;AACpB,IAAA,IAAI,CAAC,MAAA,CAAO,SAAA,CAAU,CAAC,KAAK,CAAA,GAAI,GAAA;AAC9B,MAAA,MAAM,IAAI,MAAM,CAAA,EAAG,IAAI,0BAA0B,GAAG,CAAA,MAAA,EAAS,GAAG,CAAA,CAAE,CAAA;AACpE,IAAA,OAAO,CAAA;AAAA,EACT,CAAA;AACA,EAAA,MAAM,KAAA,GAAQ,OAAO,kBAAkB,CAAA;AACvC,EAAA,MAAM,WAAA,GAAc,OAAO,yBAAyB,CAAA;AACpD,EAAA,IAAK,KAAA,KAAU,MAAA,MAAgB,WAAA,KAAgB,MAAA,CAAA,EAAY;AACzD,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AACA,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,IAAA;AAAA,IACA,UAAA,EAAY,GAAA,CAAI,wBAAA,EAA0B,GAAM,CAAA;AAAA,IAChD,WAAW,GAAA,CAAI,oBAAA;AAAA,IACf,SAAA,EAAW,GAAA,CAAI,sBAAA,EAAwB,CAAC,CAAA;AAAA,IACxC,IAAA,EAAM,GAAA,CAAI,iBAAA,EAAmB,CAAC,CAAA;AAAA,IAC9B,KAAA,EAAO,IAAI,gBAAA,IAAoB,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,QAAQ,GAAG,CAAA,CAAA;AAAA,IAC3D,KAAA;AAAA,IACA,WAAA;AAAA,IACA,WAAA,EAAa,MAAA,CAAO,yBAAA,EAA2B,CAAC;AAAA,GAClD;AACF;AAGO,SAAS,WAAA,CAAY,MAAc,OAAA,EAAwC;AAChF,EAAA,OAAO;AAAA,IACL,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,IAAA,CAAK,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC9C,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,IAAA,CAAK,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC9C,UAAU,IAAI,WAAA,CAAA,qBAAA,CAAsB,IAAA,CAAK,IAAA,EAAM,UAAU,CAAC,CAAA;AAAA;AAAA;AAAA,IAG1D,KAAA,EAAO,IAAI,WAAA,EAAY;AAAA,IACvB,GAAI,OAAA,GAAU,EAAE,OAAA,KAAY;AAAC,GAC/B;AACF;AAGA,eAAsB,OAAA,CACpB,MACA,MAAA,EACgC;AAChC,EAAA,OAAO,mBAAmB,IAAA,EAAM;AAAA,IAC9B,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,MAAM,MAAA,CAAO,IAAA;AAAA,IACb,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,aAAa,MAAA,CAAO,WAAA;AAAA,IACpB,aAAa,MAAA,CAAO;AAAA,GACrB,CAAA;AACH;AAEA,IAAM,KAAA,GAAQ,CAAC,EAAA,KAA8B,IAAI,OAAA,CAAQ,CAAC,CAAA,KAAM,UAAA,CAAW,CAAA,EAAG,EAAE,CAAC,CAAA;AACjF,IAAM,GAAA,GAAM,CAAC,GAAA,KAAuB;AAClC,EAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,SAAA,CAAU,GAAG,CAAC;AAAA,CAAI,CAAA;AACjD,CAAA;AAGA,SAAS,iBAAA,GAAkC;AACzC,EAAA,OAAO;AAAA,IACL,OAAA,EAAS,CAAC,KAAA,KAAU,GAAA,CAAI,EAAE,KAAA,EAAO,QAAA,EAAU,GAAG,KAAA,EAAO;AAAA,GACvD;AACF;AAGA,eAAsB,IAAA,CAAK,GAAA,GAA0C,OAAA,CAAQ,GAAA,EAAoB;AAC/F,EAAA,MAAM,MAAA,GAAS,YAAY,GAAG,CAAA;AAC9B,EAAA,MAAM,IAAA,GAAO,WAAA;AAAA,IACX,MAAA,CAAO,IAAA;AAAA,IACP,GAAA,CAAI,kBAAA,KAAuB,GAAA,GAAM,iBAAA,EAAkB,GAAI;AAAA,GACzD;AAEA,EAAA,IAAI,MAAA,CAAO,SAAS,MAAA,EAAQ;AAC1B,IAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,EAAM,MAAM,CAAA;AACxC,IAAA,GAAA,CAAI;AAAA,MACF,KAAA,EAAO,OAAA;AAAA,MACP,IAAA,EAAM,MAAA;AAAA,MACN,YAAY,KAAA,CAAM,UAAA;AAAA,MAClB,WAAW,KAAA,CAAM,SAAA;AAAA,MACjB,UAAU,KAAA,CAAM,QAAA;AAAA,MAChB,SAAS,KAAA,CAAM;AAAA,KAChB,CAAA;AACD,IAAA;AAAA,EACF;AAGA,EAAA,IAAI,IAAA,GAAO,KAAA;AACX,EAAA,MAAM,WAAW,MAAY;AAC3B,IAAA,IAAA,GAAO,IAAA;AAAA,EACT,CAAA;AACA,EAAA,OAAA,CAAQ,EAAA,CAAG,UAAU,QAAQ,CAAA;AAC7B,EAAA,OAAA,CAAQ,EAAA,CAAG,WAAW,QAAQ,CAAA;AAC9B,EAAA,GAAA,CAAI,EAAE,KAAA,EAAO,OAAA,EAAS,IAAA,EAAM,MAAA,EAAQ,UAAA,EAAY,MAAA,CAAO,UAAA,EAAY,KAAA,EAAO,MAAA,CAAO,KAAA,EAAO,CAAA;AACxF,EAAA,OAAO,CAAC,IAAA,EAAM;AACZ,IAAA,IAAI;AACF,MAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,EAAM,MAAM,CAAA;AACxC,MAAA,GAAA,CAAI;AAAA,QACF,KAAA,EAAO,OAAA;AAAA,QACP,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,WAAW,KAAA,CAAM,SAAA;AAAA,QACjB,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,SAAS,KAAA,CAAM;AAAA,OAChB,CAAA;AAAA,IACH,SAAS,GAAA,EAAK;AACZ,MAAA,GAAA,CAAI,EAAE,KAAA,EAAO,OAAA,EAAS,OAAA,EAAU,GAAA,CAAc,SAAS,CAAA;AAAA,IACzD;AACA,IAAA,IAAI,CAAC,IAAA,EAAM,MAAM,KAAA,CAAM,OAAO,UAAU,CAAA;AAAA,EAC1C;AACA,EAAA,GAAA,CAAI,EAAE,KAAA,EAAO,SAAA,EAAW,CAAA;AAC1B;AAIA,IAAI,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,KAAM,MAAA,IAAa,MAAA,CAAA,IAAA,CAAY,GAAA,KAAQ,aAAA,CAAc,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAC,EAAE,IAAA,EAAM;AAC5F,EAAA,IAAA,EAAK,CAAE,KAAA,CAAM,CAAC,GAAA,KAAiB;AAC7B,IAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAI,GAAA,CAAc,OAAO;AAAA,CAAI,CAAA;AAClD,IAAA,OAAA,CAAQ,KAAK,CAAC,CAAA;AAAA,EAChB,CAAC,CAAA;AACH","file":"compact-segments.js","sourcesContent":["#!/usr/bin/env node\n/**\n * `compact-segments` — the standalone compaction daemon CLI (Phase 4d).\n *\n * A thin wrapper over {@link runCompactionCycle}: discover dirty segments → 2-phase-commit compact → GC. It\n * ships the **local-filesystem** backend (zero-dependency, the dev/reference target) and two **deploy modes**:\n *\n * - `once` — run a single cycle and exit (the **Lambda / one-shot / cron** shape).\n * - `loop` — run a cycle every `intervalMs`, until SIGINT/SIGTERM (the **K8s Deployment / ECS service** shape).\n *\n * **Cloud backends:** rather than have the CLI guess your AWS config, production wires its own ~10-line\n * handler that constructs an `S3ColdDriver` + `DynamoDb{Warm,Registry}Driver` and calls `runCompactionCycle`\n * directly (same two modes — call once, or on a `setInterval`). The library exports everything needed; this\n * binary stays SDK-free (so the core install never pulls a cloud SDK). See the getting-started guide.\n *\n * Config is read from the environment (so it's identical across `once`/`loop` and 12-factor-friendly):\n * CR_COMPACT_ROOT (required) — the local-filesystem root holding cold/ warm/ registry/\n * CR_COMPACT_MODE once | loop (default: once)\n * CR_COMPACT_INTERVAL_MS (default: 30000, loop mode)\n * CR_COMPACT_NAMESPACE scope discovery to one namespace\n * CR_COMPACT_THRESHOLD min dirty chunks to compact (default: 1)\n * CR_COMPACT_KEEP superseded generations to retain as a reader grace window (default: 1)\n * CR_COMPACT_OWNER lease owner id (default: `<hostname>:<pid>`)\n * CR_COMPACT_SHARD / CR_COMPACT_TOTAL_SHARDS run this worker as shard i of N (set both) — partition the\n * fleet across N daemons so each drains only ~1/N of Warm per cycle (gap #3)\n * CR_COMPACT_MAX_SEGMENTS cap segments compacted per cycle (a time budget for a Lambda window); the rest\n * are deferred to the next cycle, most-dirty first\n * CR_COMPACT_METRICS 1 to emit each per-attempt compaction `MetricEvent` (retries, bytes, ms, faults) as a\n * JSON line on stdout — pipe to your metrics pipeline. Off by default (cycle summaries\n * still print regardless); the library's `IMetricsSink` seam is what this wires.\n */\nimport { SystemClock } from '../system-clock';\nimport { hostname } from 'node:os';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nimport {\n LocalFsColdDriver,\n LocalFsRegistryDriver,\n LocalFsWarmDriver,\n runCompactionCycle,\n} from '../index';\nimport type { CompactionCycleResult, CompactionDeps, IMetricsSink } from '../index';\n\nexport interface CompactConfig {\n readonly root: string;\n readonly mode: 'once' | 'loop';\n readonly intervalMs: number;\n readonly namespace?: string;\n readonly threshold: number;\n readonly keep: number;\n readonly owner: string;\n readonly shard?: number;\n readonly totalShards?: number;\n readonly maxSegments?: number;\n}\n\n/** Parse + validate config from an environment map. Throws a clear `Error` on misconfiguration. */\nexport function parseConfig(env: Record<string, string | undefined>): CompactConfig {\n const root = env.CR_COMPACT_ROOT;\n if (root === undefined || root === '') {\n throw new Error('CR_COMPACT_ROOT is required (the local-filesystem storage root)');\n }\n const mode = env.CR_COMPACT_MODE ?? 'once';\n if (mode !== 'once' && mode !== 'loop') {\n throw new Error(`CR_COMPACT_MODE must be \"once\" or \"loop\"; got ${mode}`);\n }\n const num = (name: string, def: number): number => {\n const raw = env[name];\n if (raw === undefined || raw === '') return def; // unset OR empty (a common unset-var expansion) → default\n const v = Number(raw);\n if (!Number.isFinite(v) || v < 0)\n throw new Error(`${name} must be a non-negative number; got ${raw}`);\n return v;\n };\n const intOpt = (name: string, min = 0): number | undefined => {\n const raw = env[name];\n if (raw === undefined || raw === '') return undefined;\n const v = Number(raw);\n if (!Number.isInteger(v) || v < min)\n throw new Error(`${name} must be an integer >= ${min}; got ${raw}`);\n return v;\n };\n const shard = intOpt('CR_COMPACT_SHARD');\n const totalShards = intOpt('CR_COMPACT_TOTAL_SHARDS');\n if ((shard === undefined) !== (totalShards === undefined)) {\n throw new Error('set both CR_COMPACT_SHARD and CR_COMPACT_TOTAL_SHARDS, or neither');\n }\n return {\n root,\n mode,\n intervalMs: num('CR_COMPACT_INTERVAL_MS', 30_000),\n namespace: env.CR_COMPACT_NAMESPACE,\n threshold: num('CR_COMPACT_THRESHOLD', 1),\n keep: num('CR_COMPACT_KEEP', 1),\n owner: env.CR_COMPACT_OWNER ?? `${hostname()}:${process.pid}`,\n shard,\n totalShards,\n maxSegments: intOpt('CR_COMPACT_MAX_SEGMENTS', 1),\n };\n}\n\n/** Build LocalFs compaction deps rooted at `<root>/{cold,warm,registry}`, optionally with a metrics sink. */\nexport function localFsDeps(root: string, metrics?: IMetricsSink): CompactionDeps {\n return {\n cold: new LocalFsColdDriver(join(root, 'cold')),\n warm: new LocalFsWarmDriver(join(root, 'warm')),\n registry: new LocalFsRegistryDriver(join(root, 'registry')),\n // A real clock, not `{ now }`: outside `core/` ambient time is allowed, and a full clock is what lets\n // the generation write hand the event loop back instead of blocking the daemon for its whole duration.\n clock: new SystemClock(),\n ...(metrics ? { metrics } : {}),\n };\n}\n\n/** Run a single compaction cycle and return the full result (the unit of work for both deploy modes). */\nexport async function runOnce(\n deps: CompactionDeps,\n config: CompactConfig,\n): Promise<CompactionCycleResult> {\n return runCompactionCycle(deps, {\n owner: config.owner,\n namespace: config.namespace,\n threshold: config.threshold,\n keep: config.keep,\n shard: config.shard,\n totalShards: config.totalShards,\n maxSegments: config.maxSegments,\n });\n}\n\nconst sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));\nconst log = (obj: unknown): void => {\n process.stdout.write(`${JSON.stringify(obj)}\\n`);\n};\n\n/** A stdout `IMetricsSink` (opt-in via CR_COMPACT_METRICS) — one JSON line per compaction `MetricEvent`. */\nfunction stdoutMetricsSink(): IMetricsSink {\n return {\n onEvent: (event) => log({ event: 'metric', ...event }),\n };\n}\n\n/** CLI entry: parse env, then run one cycle (`once`) or loop until a termination signal (`loop`). */\nexport async function main(env: Record<string, string | undefined> = process.env): Promise<void> {\n const config = parseConfig(env);\n const deps = localFsDeps(\n config.root,\n env.CR_COMPACT_METRICS === '1' ? stdoutMetricsSink() : undefined,\n );\n\n if (config.mode === 'once') {\n const cycle = await runOnce(deps, config);\n log({\n event: 'cycle',\n mode: 'once',\n candidates: cycle.candidates,\n compacted: cycle.compacted,\n deferred: cycle.deferred,\n results: cycle.results,\n });\n return;\n }\n\n // loop mode: run forever, one cycle per interval, until SIGINT/SIGTERM.\n let stop = false;\n const onSignal = (): void => {\n stop = true;\n };\n process.on('SIGINT', onSignal);\n process.on('SIGTERM', onSignal);\n log({ event: 'start', mode: 'loop', intervalMs: config.intervalMs, owner: config.owner });\n while (!stop) {\n try {\n const cycle = await runOnce(deps, config);\n log({\n event: 'cycle',\n candidates: cycle.candidates,\n compacted: cycle.compacted,\n deferred: cycle.deferred,\n results: cycle.results,\n });\n } catch (err) {\n log({ event: 'error', message: (err as Error).message }); // a cycle error must not kill the daemon\n }\n if (!stop) await sleep(config.intervalMs);\n }\n log({ event: 'stopped' });\n}\n\n// Run only when invoked directly (not when imported by tests). `process.argv[1]` is this file when run as a bin;\n// `pathToFileURL` builds a spec-correct file:// URL (drive letters, spaces, UNC paths) for a robust compare.\nif (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {\n main().catch((err: unknown) => {\n process.stderr.write(`${(err as Error).message}\\n`);\n process.exit(1);\n });\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../../src/bin/compact-segments.ts"],"names":[],"mappings":";;;;;;AA6FO,SAAS,YAAY,GAAA,EAAwD;AAClF,EAAA,MAAM,OAAO,GAAA,CAAI,eAAA;AACjB,EAAA,IAAI,IAAA,KAAS,MAAA,IAAa,IAAA,KAAS,EAAA,EAAI;AACrC,IAAA,MAAM,IAAI,MAAM,iEAAiE,CAAA;AAAA,EACnF;AACA,EAAA,MAAM,IAAA,GAAO,IAAI,eAAA,IAAmB,MAAA;AACpC,EAAA,IAAI,IAAA,KAAS,MAAA,IAAU,IAAA,KAAS,MAAA,EAAQ;AACtC,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,8CAAA,EAAiD,IAAI,CAAA,CAAE,CAAA;AAAA,EACzE;AACA,EAAA,MAAM,GAAA,GAAM,CAAC,IAAA,EAAc,GAAA,KAAwB;AACjD,IAAA,MAAM,GAAA,GAAM,IAAI,IAAI,CAAA;AACpB,IAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI,OAAO,GAAA;AAC5C,IAAA,MAAM,CAAA,GAAI,OAAO,GAAG,CAAA;AACpB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,CAAC,KAAK,CAAA,GAAI,CAAA;AAC7B,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,EAAG,IAAI,CAAA,oCAAA,EAAuC,GAAG,CAAA,CAAE,CAAA;AACrE,IAAA,OAAO,CAAA;AAAA,EACT,CAAA;AACA,EAAA,MAAM,MAAA,GAAS,CAAC,IAAA,EAAc,GAAA,GAAM,CAAA,KAA0B;AAC5D,IAAA,MAAM,GAAA,GAAM,IAAI,IAAI,CAAA;AACpB,IAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI,OAAO,MAAA;AAC5C,IAAA,MAAM,CAAA,GAAI,OAAO,GAAG,CAAA;AACpB,IAAA,IAAI,CAAC,MAAA,CAAO,SAAA,CAAU,CAAC,KAAK,CAAA,GAAI,GAAA;AAC9B,MAAA,MAAM,IAAI,MAAM,CAAA,EAAG,IAAI,0BAA0B,GAAG,CAAA,MAAA,EAAS,GAAG,CAAA,CAAE,CAAA;AACpE,IAAA,OAAO,CAAA;AAAA,EACT,CAAA;AACA,EAAA,MAAM,KAAA,GAAQ,OAAO,kBAAkB,CAAA;AACvC,EAAA,MAAM,WAAA,GAAc,OAAO,yBAAyB,CAAA;AACpD,EAAA,IAAK,KAAA,KAAU,MAAA,MAAgB,WAAA,KAAgB,MAAA,CAAA,EAAY;AACzD,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AACA,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,IAAA;AAAA,IACA,UAAA,EAAY,GAAA,CAAI,wBAAA,EAA0B,GAAM,CAAA;AAAA,IAChD,WAAW,GAAA,CAAI,oBAAA;AAAA,IACf,SAAA,EAAW,GAAA,CAAI,sBAAA,EAAwB,CAAC,CAAA;AAAA,IACxC,IAAA,EAAM,GAAA,CAAI,iBAAA,EAAmB,CAAC,CAAA;AAAA,IAC9B,KAAA,EAAO,IAAI,gBAAA,IAAoB,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,QAAQ,GAAG,CAAA,CAAA;AAAA,IAC3D,KAAA;AAAA,IACA,WAAA;AAAA,IACA,WAAA,EAAa,MAAA,CAAO,yBAAA,EAA2B,CAAC,CAAA;AAAA,IAChD,MAAA,EAAQ,IAAI,SAAA,KAAc,GAAA,GAAM,kBAAkB,GAAA,EAAK,GAAA,EAAK,MAAM,CAAA,GAAI;AAAA,GACxE;AACF;AAEA,SAAS,iBAAA,CACP,GAAA,EACA,GAAA,EACA,MAAA,EACc;AACd,EAAA,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAML,KAAA,EAAO,MAAA,CAAO,iBAAA,EAAmB,CAAC,CAAA,IAAK,WAAA,CAAA,oBAAA;AAAA,IACvC,MAAA,EAAQ,IAAI,iBAAA,KAAsB,GAAA;AAAA,IAClC,UAAA,EAAY,GAAA,CAAI,uBAAA,EAAyB,WAAA,CAAA,0BAA0B,CAAA;AAAA,IACnE,eAAA,EAAiB,IAAI,yBAAA,KAA8B,GAAA;AAAA,IACnD,gBAAA,EAAkB,GAAA,CAAI,8BAAA,EAAgC,WAAA,CAAA,0BAA0B;AAAA,GAClF;AACF;AAGO,SAAS,WAAA,CAAY,MAAc,OAAA,EAAwC;AAChF,EAAA,OAAO;AAAA,IACL,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,IAAA,CAAK,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC9C,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,IAAA,CAAK,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC9C,UAAU,IAAI,WAAA,CAAA,qBAAA,CAAsB,IAAA,CAAK,IAAA,EAAM,UAAU,CAAC,CAAA;AAAA;AAAA;AAAA,IAG1D,KAAA,EAAO,IAAI,WAAA,EAAY;AAAA,IACvB,GAAI,OAAA,GAAU,EAAE,OAAA,KAAY;AAAC,GAC/B;AACF;AAGA,eAAsB,OAAA,CACpB,MACA,MAAA,EACgC;AAChC,EAAA,OAAO,mBAAmB,IAAA,EAAM;AAAA,IAC9B,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,MAAM,MAAA,CAAO,IAAA;AAAA,IACb,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,aAAa,MAAA,CAAO,WAAA;AAAA,IACpB,aAAa,MAAA,CAAO;AAAA,GACrB,CAAA;AACH;AAQA,eAAsB,UAAA,CACpB,IAAA,EACA,MAAA,EAMA,WAAA,EAC0C;AAC1C,EAAA,MAAM,SAAS,MAAA,CAAO,MAAA;AACtB,EAAA,IAAI,MAAA,KAAW,QAAW,OAAO,MAAA;AACjC,EAAA,IAAI,WAAA,KAAgB,UAAa,IAAA,CAAK,KAAA,CAAM,KAAI,GAAI,WAAA,GAAc,OAAO,UAAA,EAAY;AACnF,IAAA,OAAO,MAAA;AAAA,EACT;AACA,EAAA,OAAA,IAAO,WAAA,CAAA,aAAA;AAAA,IACL,EAAE,UAAU,IAAA,CAAK,QAAA,EAAU,MAAM,IAAA,CAAK,IAAA,EAAM,IAAA,EAAM,IAAA,CAAK,IAAA,EAAK;AAAA,IAC5D;AAAA,MACE,WAAW,MAAA,CAAO,SAAA;AAAA,MAClB,GAAA,EAAK,IAAA,CAAK,KAAA,CAAM,GAAA,EAAI;AAAA,MACpB,OAAO,MAAA,CAAO,KAAA;AAAA,MACd,QAAQ,MAAA,CAAO,MAAA;AAAA,MACf,iBAAiB,MAAA,CAAO,eAAA;AAAA,MACxB,kBAAkB,MAAA,CAAO;AAAA;AAC3B,GACF;AACF;AAGA,IAAM,kBAAA,GAAqB,EAAA;AAS3B,SAAS,UAAU,MAAA,EAAmC;AACpD,EAAA,MAAM,OAAA,GAAU,OAAO,OAAA,CAAQ,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,WAAW,SAAS,CAAA;AACnE,EAAA,MAAM,WAAmC,EAAC;AAC1C,EAAA,KAAA,MAAW,KAAK,OAAA,EAAS;AAEvB,IAAA,MAAM,MAAM,CAAA,CAAE,MAAA,CAAO,WAAW,SAAS,CAAA,GAAI,WAAW,CAAA,CAAE,MAAA;AAC1D,IAAA,QAAA,CAAS,GAAG,CAAA,GAAA,CAAK,QAAA,CAAS,GAAG,KAAK,CAAA,IAAK,CAAA;AAAA,EACzC;AACA,EAAA,MAAM,YAAA,GAAe,OAAO,OAAA,CAAQ,MAAA;AAAA,IAClC,CAAC,MAAM,CAAA,CAAE,MAAA,KAAW,aAAa,CAAA,CAAE,MAAA,CAAO,qBAAqB,MAAA,GAAS;AAAA,GAC1E;AACA,EAAA,GAAA,CAAI;AAAA,IACF,KAAA,EAAO,QAAA;AAAA,IACP,QAAQ,MAAA,CAAO,MAAA;AAAA,IACf,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,UAAU,MAAA,CAAO,QAAA;AAAA,IACjB,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,aAAa,MAAA,CAAO,WAAA;AAAA,IACpB,kBAAkB,MAAA,CAAO,gBAAA;AAAA,IACzB,SAAS,MAAA,CAAO,OAAA;AAAA,IAChB,eAAA,EAAiB,QAAA;AAAA,IACjB,eAAe,OAAA,CACZ,KAAA,CAAM,GAAG,kBAAkB,CAAA,CAC3B,IAAI,CAAC,CAAA,MAAO,EAAE,OAAA,EAAS,CAAA,CAAE,SAAS,SAAA,EAAW,CAAA,CAAE,WAAW,MAAA,EAAQ,CAAA,CAAE,QAAO,CAAE,CAAA;AAAA,IAChF,iBAAA,EAAmB,YAAA,CAAa,KAAA,CAAM,CAAA,EAAG,kBAAkB,EAAE,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,OAAO,CAAA;AAAA,IACjF,wBAAwB,YAAA,CAAa;AAAA,GACtC,CAAA;AACH;AAEA,IAAM,KAAA,GAAQ,CAAC,EAAA,KAA8B,IAAI,OAAA,CAAQ,CAAC,CAAA,KAAM,UAAA,CAAW,CAAA,EAAG,EAAE,CAAC,CAAA;AACjF,IAAM,GAAA,GAAM,CAAC,GAAA,KAAuB;AAClC,EAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,SAAA,CAAU,GAAG,CAAC;AAAA,CAAI,CAAA;AACjD,CAAA;AAGA,SAAS,iBAAA,GAAkC;AACzC,EAAA,OAAO;AAAA,IACL,OAAA,EAAS,CAAC,KAAA,KAAU,GAAA,CAAI,EAAE,KAAA,EAAO,QAAA,EAAU,GAAG,KAAA,EAAO;AAAA,GACvD;AACF;AAGA,eAAsB,IAAA,CAAK,GAAA,GAA0C,OAAA,CAAQ,GAAA,EAAoB;AAC/F,EAAA,MAAM,MAAA,GAAS,YAAY,GAAG,CAAA;AAC9B,EAAA,MAAM,IAAA,GAAO,WAAA;AAAA,IACX,MAAA,CAAO,IAAA;AAAA,IACP,GAAA,CAAI,kBAAA,KAAuB,GAAA,GAAM,iBAAA,EAAkB,GAAI;AAAA,GACzD;AAEA,EAAA,IAAI,MAAA,CAAO,SAAS,MAAA,EAAQ;AAC1B,IAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,EAAM,MAAM,CAAA;AACxC,IAAA,GAAA,CAAI;AAAA,MACF,KAAA,EAAO,OAAA;AAAA,MACP,IAAA,EAAM,MAAA;AAAA,MACN,YAAY,KAAA,CAAM,UAAA;AAAA,MAClB,WAAW,KAAA,CAAM,SAAA;AAAA,MACjB,UAAU,KAAA,CAAM,QAAA;AAAA,MAChB,SAAS,KAAA,CAAM;AAAA,KAChB,CAAA;AACD,IAAA,MAAM,KAAA,GAAQ,MAAM,UAAA,CAAW,IAAA,EAAM,MAAM,CAAA;AAC3C,IAAA,IAAI,KAAA,KAAU,MAAA,EAAW,SAAA,CAAU,KAAK,CAAA;AACxC,IAAA;AAAA,EACF;AAGA,EAAA,IAAI,IAAA,GAAO,KAAA;AACX,EAAA,MAAM,WAAW,MAAY;AAC3B,IAAA,IAAA,GAAO,IAAA;AAAA,EACT,CAAA;AACA,EAAA,OAAA,CAAQ,EAAA,CAAG,UAAU,QAAQ,CAAA;AAC7B,EAAA,OAAA,CAAQ,EAAA,CAAG,WAAW,QAAQ,CAAA;AAC9B,EAAA,GAAA,CAAI,EAAE,KAAA,EAAO,OAAA,EAAS,IAAA,EAAM,MAAA,EAAQ,UAAA,EAAY,MAAA,CAAO,UAAA,EAAY,KAAA,EAAO,MAAA,CAAO,KAAA,EAAO,CAAA;AAExF,EAAA,IAAI,WAAA;AACJ,EAAA,OAAO,CAAC,IAAA,EAAM;AACZ,IAAA,IAAI;AACF,MAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,EAAM,MAAM,CAAA;AACxC,MAAA,GAAA,CAAI;AAAA,QACF,KAAA,EAAO,OAAA;AAAA,QACP,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,WAAW,KAAA,CAAM,SAAA;AAAA,QACjB,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,SAAS,KAAA,CAAM;AAAA,OAChB,CAAA;AACD,MAAA,MAAM,KAAA,GAAQ,MAAM,UAAA,CAAW,IAAA,EAAM,QAAQ,WAAW,CAAA;AACxD,MAAA,IAAI,UAAU,KAAA,CAAA,EAAW;AACvB,QAAA,WAAA,GAAc,IAAA,CAAK,MAAM,GAAA,EAAI;AAC7B,QAAA,SAAA,CAAU,KAAK,CAAA;AAAA,MACjB;AAAA,IACF,SAAS,GAAA,EAAK;AACZ,MAAA,GAAA,CAAI,EAAE,KAAA,EAAO,OAAA,EAAS,OAAA,EAAU,GAAA,CAAc,SAAS,CAAA;AAAA,IACzD;AACA,IAAA,IAAI,CAAC,IAAA,EAAM,MAAM,KAAA,CAAM,OAAO,UAAU,CAAA;AAAA,EAC1C;AACA,EAAA,GAAA,CAAI,EAAE,KAAA,EAAO,SAAA,EAAW,CAAA;AAC1B;AAIA,IAAI,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,KAAM,MAAA,IAAa,MAAA,CAAA,IAAA,CAAY,GAAA,KAAQ,aAAA,CAAc,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAC,EAAE,IAAA,EAAM;AAC5F,EAAA,IAAA,EAAK,CAAE,KAAA,CAAM,CAAC,GAAA,KAAiB;AAC7B,IAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAI,GAAA,CAAc,OAAO;AAAA,CAAI,CAAA;AAClD,IAAA,OAAA,CAAQ,KAAK,CAAC,CAAA;AAAA,EAChB,CAAC,CAAA;AACH","file":"compact-segments.js","sourcesContent":["#!/usr/bin/env node\n/**\n * `compact-segments` — the standalone compaction daemon CLI (Phase 4d).\n *\n * A thin wrapper over {@link runCompactionCycle}: discover dirty segments → 2-phase-commit compact → GC. It\n * ships the **local-filesystem** backend (zero-dependency, the dev/reference target) and two **deploy modes**:\n *\n * - `once` — run a single cycle and exit (the **Lambda / one-shot / cron** shape).\n * - `loop` — run a cycle every `intervalMs`, until SIGINT/SIGTERM (the **K8s Deployment / ECS service** shape).\n *\n * **Cloud backends:** rather than have the CLI guess your AWS config, production wires its own ~10-line\n * handler that constructs an `S3ColdDriver` + `DynamoDb{Warm,Registry}Driver` and calls `runCompactionCycle`\n * directly (same two modes — call once, or on a `setInterval`). The library exports everything needed; this\n * binary stays SDK-free (so the core install never pulls a cloud SDK). See the getting-started guide.\n *\n * Config is read from the environment (so it's identical across `once`/`loop` and 12-factor-friendly):\n * CR_COMPACT_ROOT (required) — the local-filesystem root holding cold/ warm/ registry/\n * CR_COMPACT_MODE once | loop (default: once)\n * CR_COMPACT_INTERVAL_MS (default: 30000, loop mode)\n * CR_COMPACT_NAMESPACE scope discovery to one namespace\n * CR_COMPACT_THRESHOLD min dirty chunks to compact (default: 1)\n * CR_COMPACT_KEEP superseded generations to retain as a reader grace window (default: 1)\n * CR_COMPACT_OWNER lease owner id (default: `<hostname>:<pid>`)\n * CR_COMPACT_SHARD / CR_COMPACT_TOTAL_SHARDS run this worker as shard i of N (set both) — partition the\n * fleet across N daemons so each drains only ~1/N of Warm per cycle (gap #3)\n * CR_COMPACT_MAX_SEGMENTS cap segments compacted per cycle (a time budget for a Lambda window); the rest\n * are deferred to the next cycle, most-dirty first\n * CR_COMPACT_METRICS 1 to emit each per-attempt compaction `MetricEvent` (retries, bytes, ms, faults) as a\n * JSON line on stdout — pipe to your metrics pipeline. Off by default (cycle summaries\n * still print regardless); the library's `IMetricsSink` seam is what this wires.\n *\n * **Retention (opt-in).** With `CR_RETIRE=1` each cycle also runs {@link retireExpired}, retiring the segments\n * whose `expiresAt` has passed. It is a *separate phase after* compaction, not part of `runCompactionCycle`:\n * compaction's job is to make a segment cheap, retirement's is to delete it, and a destructive step that runs\n * implicitly inside a maintenance cycle is the wrong default for anyone who just wanted their Warm tier drained.\n * Off unless you ask for it, and a custom worker composes the same two calls in the same order.\n * CR_RETIRE 1 to enable the retention sweep (default: off)\n * CR_RETIRE_INTERVAL_MS how often to sweep, independent of the compaction interval (default: 86400000, i.e.\n * daily). Retention windows are measured in days, so sweeping at the 30 s compaction\n * cadence would re-scan the whole registry 2,880 times a day — on DynamoDB that is a\n * full-table Scan each time, billed, competing with the hot path for read capacity.\n * CR_RETIRE_LIMIT max segments retired per sweep (default: 100)\n * CR_RETIRE_DRY_RUN 1 to report what it would retire and delete nothing — start here\n * CR_RETIRE_TOMBSTONE_GRACE_MS ms a retirement's tombstone row must age before the row is deleted\n * (default: 86400000)\n * CR_RETIRE_KEEP_TOMBSTONES 1 to never delete a tombstone row\n * Scope follows CR_COMPACT_NAMESPACE. **Run the sweep from ONE process** — it has no shard option, so N\n * replicas each sweep the whole registry and contend over the same segments.\n */\nimport { SystemClock } from '../system-clock';\nimport { hostname } from 'node:os';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nimport {\n DEFAULT_RETIRE_LIMIT,\n DEFAULT_TOMBSTONE_GRACE_MS,\n LocalFsColdDriver,\n LocalFsRegistryDriver,\n LocalFsWarmDriver,\n retireExpired,\n runCompactionCycle,\n} from '../index';\nimport type {\n CompactionCycleResult,\n CompactionDeps,\n IMetricsSink,\n RetireExpiredResult,\n} from '../index';\n\nexport interface CompactConfig {\n readonly root: string;\n readonly mode: 'once' | 'loop';\n readonly intervalMs: number;\n readonly namespace?: string;\n readonly threshold: number;\n readonly keep: number;\n readonly owner: string;\n readonly shard?: number;\n readonly totalShards?: number;\n readonly maxSegments?: number;\n /** Retention sweep (opt-in) — absent ⇒ the daemon never retires anything. */\n readonly retire?: RetireConfig;\n}\n\nexport interface RetireConfig {\n readonly limit: number;\n readonly dryRun: boolean;\n readonly intervalMs: number;\n readonly purgeTombstones: boolean;\n readonly tombstoneGraceMs: number;\n}\n\n/** Parse + validate config from an environment map. Throws a clear `Error` on misconfiguration. */\nexport function parseConfig(env: Record<string, string | undefined>): CompactConfig {\n const root = env.CR_COMPACT_ROOT;\n if (root === undefined || root === '') {\n throw new Error('CR_COMPACT_ROOT is required (the local-filesystem storage root)');\n }\n const mode = env.CR_COMPACT_MODE ?? 'once';\n if (mode !== 'once' && mode !== 'loop') {\n throw new Error(`CR_COMPACT_MODE must be \"once\" or \"loop\"; got ${mode}`);\n }\n const num = (name: string, def: number): number => {\n const raw = env[name];\n if (raw === undefined || raw === '') return def; // unset OR empty (a common unset-var expansion) → default\n const v = Number(raw);\n if (!Number.isFinite(v) || v < 0)\n throw new Error(`${name} must be a non-negative number; got ${raw}`);\n return v;\n };\n const intOpt = (name: string, min = 0): number | undefined => {\n const raw = env[name];\n if (raw === undefined || raw === '') return undefined;\n const v = Number(raw);\n if (!Number.isInteger(v) || v < min)\n throw new Error(`${name} must be an integer >= ${min}; got ${raw}`);\n return v;\n };\n const shard = intOpt('CR_COMPACT_SHARD');\n const totalShards = intOpt('CR_COMPACT_TOTAL_SHARDS');\n if ((shard === undefined) !== (totalShards === undefined)) {\n throw new Error('set both CR_COMPACT_SHARD and CR_COMPACT_TOTAL_SHARDS, or neither');\n }\n return {\n root,\n mode,\n intervalMs: num('CR_COMPACT_INTERVAL_MS', 30_000),\n namespace: env.CR_COMPACT_NAMESPACE,\n threshold: num('CR_COMPACT_THRESHOLD', 1),\n keep: num('CR_COMPACT_KEEP', 1),\n owner: env.CR_COMPACT_OWNER ?? `${hostname()}:${process.pid}`,\n shard,\n totalShards,\n maxSegments: intOpt('CR_COMPACT_MAX_SEGMENTS', 1),\n retire: env.CR_RETIRE === '1' ? parseRetireConfig(env, num, intOpt) : undefined,\n };\n}\n\nfunction parseRetireConfig(\n env: Record<string, string | undefined>,\n num: (name: string, def: number) => number,\n intOpt: (name: string, min?: number) => number | undefined,\n): RetireConfig {\n return {\n // `intOpt(name, 1)`, not `num`: `num` accepts 0 and 1.5, and `retireExpired` then throws a ValidationError\n // from *inside* the cycle — swallowed into a generic `{\"event\":\"error\"}` line every 30 s in loop mode, so\n // compaction keeps reporting healthy cycles while retention silently never runs. `parseConfig` promises to\n // throw a clear error on misconfiguration; this is what keeps that promise. (CR_COMPACT_MAX_SEGMENTS, the\n // identical concept, already used `intOpt`.)\n limit: intOpt('CR_RETIRE_LIMIT', 1) ?? DEFAULT_RETIRE_LIMIT,\n dryRun: env.CR_RETIRE_DRY_RUN === '1',\n intervalMs: num('CR_RETIRE_INTERVAL_MS', DEFAULT_TOMBSTONE_GRACE_MS),\n purgeTombstones: env.CR_RETIRE_KEEP_TOMBSTONES !== '1',\n tombstoneGraceMs: num('CR_RETIRE_TOMBSTONE_GRACE_MS', DEFAULT_TOMBSTONE_GRACE_MS),\n };\n}\n\n/** Build LocalFs compaction deps rooted at `<root>/{cold,warm,registry}`, optionally with a metrics sink. */\nexport function localFsDeps(root: string, metrics?: IMetricsSink): CompactionDeps {\n return {\n cold: new LocalFsColdDriver(join(root, 'cold')),\n warm: new LocalFsWarmDriver(join(root, 'warm')),\n registry: new LocalFsRegistryDriver(join(root, 'registry')),\n // A real clock, not `{ now }`: outside `core/` ambient time is allowed, and a full clock is what lets\n // the generation write hand the event loop back instead of blocking the daemon for its whole duration.\n clock: new SystemClock(),\n ...(metrics ? { metrics } : {}),\n };\n}\n\n/** Run a single compaction cycle and return the full result (the unit of work for both deploy modes). */\nexport async function runOnce(\n deps: CompactionDeps,\n config: CompactConfig,\n): Promise<CompactionCycleResult> {\n return runCompactionCycle(deps, {\n owner: config.owner,\n namespace: config.namespace,\n threshold: config.threshold,\n keep: config.keep,\n shard: config.shard,\n totalShards: config.totalShards,\n maxSegments: config.maxSegments,\n });\n}\n\n/**\n * Run the retention sweep for one cycle, if configured. Returns `undefined` when retention is off.\n *\n * `now` comes from the deps clock — the same one compaction uses — so a daemon wired with a test clock sweeps\n * against that clock too.\n */\nexport async function retireOnce(\n deps: CompactionDeps,\n config: CompactConfig,\n /**\n * When the sweep last ran (epoch-ms), or `undefined` for \"never\". A sweep is skipped — returning `undefined` —\n * until `CR_RETIRE_INTERVAL_MS` has elapsed, so the retention cadence is independent of the compaction one.\n * `once` mode passes nothing and therefore always sweeps, which is the cron/Lambda shape.\n */\n lastSweptAt?: number,\n): Promise<RetireExpiredResult | undefined> {\n const retire = config.retire;\n if (retire === undefined) return undefined;\n if (lastSweptAt !== undefined && deps.clock.now() - lastSweptAt < retire.intervalMs) {\n return undefined;\n }\n return retireExpired(\n { registry: deps.registry, warm: deps.warm, cold: deps.cold },\n {\n namespace: config.namespace,\n now: deps.clock.now(),\n limit: retire.limit,\n dryRun: retire.dryRun,\n purgeTombstones: retire.purgeTombstones,\n tombstoneGraceMs: retire.tombstoneGraceMs,\n },\n );\n}\n\n/** How many skipped segments one log line names before switching to per-reason counts. */\nconst LOG_SKIPPED_SAMPLE = 20;\n\n/**\n * One JSON line summarising a sweep. Skips are counted per reason AND a bounded sample is named — a silent skip is\n * a lost signal, but the whole list is not loggable: a bad backfill over a 250,000-row fleet would otherwise\n * serialise megabytes into a single line, which is ~78× CloudWatch's 256 KB event limit, so the line that mattered\n * would be truncated or dropped. `retired` and `wouldRetire` are separate fields for the same reason — a counter\n * that means \"deleted\" in one mode and \"would delete\" in another puts phantom deletions on a graph.\n */\nfunction logRetire(result: RetireExpiredResult): void {\n const skipped = result.entries.filter((e) => e.action === 'skipped');\n const byReason: Record<string, number> = {};\n for (const e of skipped) {\n // Collapse `failed: <driver message>` into one bucket; the sample below carries the actual text.\n const key = e.reason.startsWith('failed:') ? 'failed' : e.reason;\n byReason[key] = (byReason[key] ?? 0) + 1;\n }\n const notReclaimed = result.entries.filter(\n (e) => e.action === 'retired' && e.result.generationsRemaining.length > 0,\n );\n log({\n event: 'retire',\n dryRun: result.dryRun,\n scanned: result.scanned,\n eligible: result.eligible,\n retired: result.retired,\n wouldRetire: result.wouldRetire,\n tombstonesPurged: result.tombstonesPurged,\n limited: result.limited,\n skippedByReason: byReason,\n skippedSample: skipped\n .slice(0, LOG_SKIPPED_SAMPLE)\n .map((e) => ({ segment: e.segment, namespace: e.namespace, reason: e.reason })),\n notFullyReclaimed: notReclaimed.slice(0, LOG_SKIPPED_SAMPLE).map((e) => e.segment),\n notFullyReclaimedCount: notReclaimed.length,\n });\n}\n\nconst sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));\nconst log = (obj: unknown): void => {\n process.stdout.write(`${JSON.stringify(obj)}\\n`);\n};\n\n/** A stdout `IMetricsSink` (opt-in via CR_COMPACT_METRICS) — one JSON line per compaction `MetricEvent`. */\nfunction stdoutMetricsSink(): IMetricsSink {\n return {\n onEvent: (event) => log({ event: 'metric', ...event }),\n };\n}\n\n/** CLI entry: parse env, then run one cycle (`once`) or loop until a termination signal (`loop`). */\nexport async function main(env: Record<string, string | undefined> = process.env): Promise<void> {\n const config = parseConfig(env);\n const deps = localFsDeps(\n config.root,\n env.CR_COMPACT_METRICS === '1' ? stdoutMetricsSink() : undefined,\n );\n\n if (config.mode === 'once') {\n const cycle = await runOnce(deps, config);\n log({\n event: 'cycle',\n mode: 'once',\n candidates: cycle.candidates,\n compacted: cycle.compacted,\n deferred: cycle.deferred,\n results: cycle.results,\n });\n const swept = await retireOnce(deps, config);\n if (swept !== undefined) logRetire(swept);\n return;\n }\n\n // loop mode: run forever, one cycle per interval, until SIGINT/SIGTERM.\n let stop = false;\n const onSignal = (): void => {\n stop = true;\n };\n process.on('SIGINT', onSignal);\n process.on('SIGTERM', onSignal);\n log({ event: 'start', mode: 'loop', intervalMs: config.intervalMs, owner: config.owner });\n // `undefined` so the first cycle sweeps; thereafter the retention interval gates it independently of this loop's.\n let lastSweptAt: number | undefined;\n while (!stop) {\n try {\n const cycle = await runOnce(deps, config);\n log({\n event: 'cycle',\n candidates: cycle.candidates,\n compacted: cycle.compacted,\n deferred: cycle.deferred,\n results: cycle.results,\n });\n const swept = await retireOnce(deps, config, lastSweptAt);\n if (swept !== undefined) {\n lastSweptAt = deps.clock.now();\n logRetire(swept);\n }\n } catch (err) {\n log({ event: 'error', message: (err as Error).message }); // a cycle error must not kill the daemon\n }\n if (!stop) await sleep(config.intervalMs);\n }\n log({ event: 'stopped' });\n}\n\n// Run only when invoked directly (not when imported by tests). `process.argv[1]` is this file when run as a bin;\n// `pathToFileURL` builds a spec-correct file:// URL (drive letters, spaces, UNC paths) for a robust compare.\nif (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {\n main().catch((err: unknown) => {\n process.stderr.write(`${(err as Error).message}\\n`);\n process.exit(1);\n });\n}\n"]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { src_exports, CloudRoaring } from '../chunk-
|
|
2
|
+
import { src_exports, CloudRoaring } from '../chunk-D6LIJSGF.js';
|
|
3
3
|
import { randomUUID } from 'crypto';
|
|
4
4
|
import { mkdir, writeFile, rename, rm, open } from 'fs/promises';
|
|
5
5
|
import { join } from 'path';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as core_star from '@cloudbitmaps/core';
|
|
2
|
-
import { runCompactionCycle as runCompactionCycle$1, safeMetrics, NOOP_METRICS, BoundedLru, RetryingWarmDriver, RetryingColdChunkSource, ValidationError, resolveBudget, DEFAULT_BUDGET, SegmentEngine, UnsupportedError, validateSegmentRef, resolvePerOpBudget, splitId, collectWithinBudget, mapWithConcurrency, validateCompactionOptions, compactSegment as compactSegment$1, gcOrphanGenerations, dropSegment, runConsistencyCheck, runExport as runExport$1, estimateCost, groundedReport, bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, IntegrityError, CrbmColdChunkSource } from '@cloudbitmaps/core';
|
|
2
|
+
import { runCompactionCycle as runCompactionCycle$1, safeMetrics, NOOP_METRICS, BoundedLru, RetryingWarmDriver, RetryingColdChunkSource, ValidationError, resolveBudget, DEFAULT_BUDGET, SegmentEngine, UnsupportedError, validateSegmentRef, resolvePerOpBudget, splitId, collectWithinBudget, mapWithConcurrency, validateCompactionOptions, compactSegment as compactSegment$1, gcOrphanGenerations, dropSegment, setSegmentRetention, getSegmentRetention, clearSegmentRetention, retireExpired, runConsistencyCheck, runExport as runExport$1, estimateCost, groundedReport, bulkLoadCrbmGeneration as bulkLoadCrbmGeneration$1, IntegrityError, CrbmColdChunkSource } from '@cloudbitmaps/core';
|
|
3
3
|
import roaring from 'roaring';
|
|
4
4
|
|
|
5
5
|
var __defProp = Object.defineProperty;
|
|
@@ -336,12 +336,12 @@ var CloudRoaring = class {
|
|
|
336
336
|
compactionDeps() {
|
|
337
337
|
if (this.coldDriver === void 0) {
|
|
338
338
|
throw new UnsupportedError(
|
|
339
|
-
"compact/dropSegment/eraseSubject/checkConsistency need the store built with a raw cold driver (IColdDriver), not a pre-built ColdChunkSource \u2014 or call the compactSegment/runConsistencyCheck free functions with explicit deps"
|
|
339
|
+
"compact/dropSegment/retireExpired/eraseSubject/checkConsistency need the store built with a raw cold driver (IColdDriver), not a pre-built ColdChunkSource \u2014 or call the compactSegment/runConsistencyCheck free functions with explicit deps"
|
|
340
340
|
);
|
|
341
341
|
}
|
|
342
342
|
if (this.registry === void 0) {
|
|
343
343
|
throw new UnsupportedError(
|
|
344
|
-
"compact/eraseSubject/checkConsistency need a `registry` in the store config"
|
|
344
|
+
"compact/dropSegment/retireExpired/eraseSubject/checkConsistency need a `registry` in the store config"
|
|
345
345
|
);
|
|
346
346
|
}
|
|
347
347
|
return {
|
|
@@ -374,10 +374,7 @@ var CloudRoaring = class {
|
|
|
374
374
|
* (throws {@link UnsupportedError} otherwise).
|
|
375
375
|
*/
|
|
376
376
|
async subjectReport(id, options = {}) {
|
|
377
|
-
|
|
378
|
-
throw new UnsupportedError("subjectReport needs a `registry` in the store config");
|
|
379
|
-
}
|
|
380
|
-
const registry = this.registry;
|
|
377
|
+
const registry = this.requireRegistry("subjectReport");
|
|
381
378
|
requireScope(options, "subjectReport");
|
|
382
379
|
validateConcurrency(options.concurrency);
|
|
383
380
|
const budget = resolvePerOpBudget(options.budget, this.budget);
|
|
@@ -527,7 +524,12 @@ var CloudRoaring = class {
|
|
|
527
524
|
*
|
|
528
525
|
* **Omitting the namespace addresses a different segment and is a silent no-op** — you get
|
|
529
526
|
* `{ dropped: false, reason: 'absent' }`, not a throw, so a retention loop with that mistake deletes nothing
|
|
530
|
-
* forever and quietly.
|
|
527
|
+
* forever and quietly. Branch on `dropped`, and treat `reason: 'absent'` as the alert.
|
|
528
|
+
*
|
|
529
|
+
* **Works on an accumulator too.** A segment you created by writing to it — never bulk-loaded, never compacted,
|
|
530
|
+
* so it has no registry row and no Cold objects — is retired by deleting its Warm rows alone. That reports
|
|
531
|
+
* `{ dropped: true, reason: 'warm-only' }`: there is no tombstone to write, and deliberately none is written,
|
|
532
|
+
* because a `destroyed` row per retired daily bucket would be registry litter. The data really is gone.
|
|
531
533
|
*
|
|
532
534
|
* **Inspect `generationsRemaining`.** Empty is the normal outcome; non-empty means the storage was NOT fully
|
|
533
535
|
* reclaimed and the drop should be re-run. A compaction that was already in flight when the tombstone landed
|
|
@@ -548,6 +550,116 @@ var CloudRoaring = class {
|
|
|
548
550
|
const deps = this.compactionDeps();
|
|
549
551
|
return dropSegment(ref, { registry: deps.registry, warm: deps.warm, cold: deps.cold }, options);
|
|
550
552
|
}
|
|
553
|
+
/**
|
|
554
|
+
* **Record when this segment becomes eligible for retirement.** One registry write; nothing is deleted here,
|
|
555
|
+
* and nothing starts running. `retireExpired` is what acts on the policy, and **you** decide when that runs —
|
|
556
|
+
* an EventBridge rule, a CronJob, a queue consumer, whatever your deployment already has. This library starts
|
|
557
|
+
* no background thread (it has to work identically in a Lambda, an edge isolate and a long-lived server).
|
|
558
|
+
*
|
|
559
|
+
* `expiresAt` is an **absolute epoch-ms you compute**, not a duration the library derives. A relative TTL would
|
|
560
|
+
* have to be anchored to something the library knows — `updatedAt`, or the current generation — and compaction
|
|
561
|
+
* rewrites both, so "expire 30 days after the last write" would keep a busy daily bucket alive forever
|
|
562
|
+
* precisely because the daemon is working.
|
|
563
|
+
*
|
|
564
|
+
* ```ts
|
|
565
|
+
* const DAY = 86_400_000;
|
|
566
|
+
* const ref = { namespace: 'active-daily', segment: '2026-08-05' };
|
|
567
|
+
* await store.setRetention(ref, { expiresAt: Date.now() + 30 * DAY });
|
|
568
|
+
* ```
|
|
569
|
+
*
|
|
570
|
+
* **Works on an accumulator**, which is the whole point: a segment you created by writing to it has no
|
|
571
|
+
* registry row, so it is invisible to `registry.list()` and therefore to every fleet-wide operation, including
|
|
572
|
+
* the sweep. This call mints the row (`createdRow: true` in the result) with **no Cold generation**, so it
|
|
573
|
+
* becomes enumerable while every read still resolves exactly as before.
|
|
574
|
+
*
|
|
575
|
+
* A value in the past is legal and means "eligible on the next sweep" — backfilling a policy onto existing
|
|
576
|
+
* buckets is normal. A value below `MIN_EXPIRES_AT_MS` (2001-09-09) is rejected: it is almost certainly epoch
|
|
577
|
+
* **seconds**, which would read as long-expired and retire the segment on the next pass. Needs a `registry`
|
|
578
|
+
* in the store config (throws {@link UnsupportedError} otherwise), and refuses a crypto-shredded segment.
|
|
579
|
+
*/
|
|
580
|
+
async setRetention(ref, policy) {
|
|
581
|
+
validateSegmentRef(ref);
|
|
582
|
+
return setSegmentRetention(ref, { registry: this.requireRegistry("setRetention") }, policy);
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* **The stored retention policy**, or `null` if the segment has none (or has no registry row, or is a
|
|
586
|
+
* tombstone). Returns the string `'invalid'` for a row whose `expiresAt` is present but unusable — a
|
|
587
|
+
* hand-edited row, or one from a restore — so a malformed policy is visible rather than silently reading as
|
|
588
|
+
* "never expires" on a segment someone believes is expiring.
|
|
589
|
+
*/
|
|
590
|
+
async getRetention(ref) {
|
|
591
|
+
validateSegmentRef(ref);
|
|
592
|
+
return getSegmentRetention(ref, { registry: this.requireRegistry("getRetention") });
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* **Cancel a segment's expiry** so no sweep retires it. Returns whether a policy was actually removed (`false`
|
|
596
|
+
* when there was none). A separate verb from `setRetention` on purpose: "never expire" as a magic value passed
|
|
597
|
+
* to the setter is how a typo becomes a deletion.
|
|
598
|
+
*/
|
|
599
|
+
async clearRetention(ref) {
|
|
600
|
+
validateSegmentRef(ref);
|
|
601
|
+
return clearSegmentRetention(ref, { registry: this.requireRegistry("clearRetention") });
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* **Run the retention sweep: retire every segment whose `expiresAt` has passed.** This is the call that acts on
|
|
605
|
+
* the policies `setRetention` records — and it is a **call, not a daemon**. Nothing schedules it; you run it from
|
|
606
|
+
* whatever heartbeat your deployment already has (an EventBridge rule, a Kubernetes CronJob, a queue consumer,
|
|
607
|
+
* your compaction worker's loop). A library that started a timer would behave differently in a Lambda, an edge
|
|
608
|
+
* isolate and a long-lived server, which is worse than not having one.
|
|
609
|
+
*
|
|
610
|
+
* Each retirement goes through `dropSegment`, so the Warm → registry → Cold ordering, the re-sweep for a
|
|
611
|
+
* generation staged by an in-flight compaction, and the `generationsRemaining` report all come from one
|
|
612
|
+
* implementation rather than two.
|
|
613
|
+
*
|
|
614
|
+
* ```ts
|
|
615
|
+
* // In your scheduled handler. Start with a preview in a new deployment.
|
|
616
|
+
* const preview = await store.retireExpired({ namespace: 'active-daily', dryRun: true });
|
|
617
|
+
* console.log(`would retire ${preview.retired} of ${preview.scanned} (limited: ${preview.limited})`);
|
|
618
|
+
*
|
|
619
|
+
* const swept = await store.retireExpired({ namespace: 'active-daily' });
|
|
620
|
+
* for (const e of swept.entries) {
|
|
621
|
+
* if (e.action === 'skipped') console.warn(`${e.segment}: ${e.reason}`); // invalid-policy / limit / failed: …
|
|
622
|
+
* if (e.action === 'retired' && e.result.generationsRemaining.length > 0) {
|
|
623
|
+
* console.warn(`${e.segment}: storage not fully reclaimed — re-run`);
|
|
624
|
+
* }
|
|
625
|
+
* }
|
|
626
|
+
* if (swept.limited) scheduleAnotherPassSoon(); // more are still eligible
|
|
627
|
+
* ```
|
|
628
|
+
*
|
|
629
|
+
* **Read the ledger.** A per-segment *fault* is an entry, not an exception — a throw from the middle of a fleet
|
|
630
|
+
* sweep would leave the caller unable to say which segments were retired, having already retired some. (A bad
|
|
631
|
+
* argument does throw, and so does a fleet larger than `maxScanSegments`.) `limited: true` means the per-cycle
|
|
632
|
+
* `limit` (default 100) cut the pass short and more are eligible. That cap is charged on **attempts**, not
|
|
633
|
+
* successes, which is what makes it a real bound: `dropSegment` deletes Warm and writes the tombstone before
|
|
634
|
+
* sweeping Cold, so a fault in the Cold phase is a segment that is already retired. Retirements are sequential
|
|
635
|
+
* (~8 round trips each), so `limit` is a wall-clock knob too, and `retired` counts deletions only — a dry run
|
|
636
|
+
* reports `wouldRetire` instead, so a dashboard summing `retired` can never show a phantom deletion.
|
|
637
|
+
*
|
|
638
|
+
* It also **deletes the tombstone rows its own past retirements left**, after `tombstoneGraceMs` (default 24 h)
|
|
639
|
+
* and only once that segment's Warm rows and Cold generations are provably gone — collecting a straggler
|
|
640
|
+
* generation itself first, since nothing else ever would for a tombstoned segment. Attribution is a **positive
|
|
641
|
+
* marker the sweep stamps on its own retirements**, not an inference from "destroyed + an expired policy": a
|
|
642
|
+
* crypto-shred leaves `retention` untouched, so setting a policy and then honouring a right-to-erasure request
|
|
643
|
+
* mid-window produces exactly that row, and deleting it would destroy the Art. 17 attestation and un-fence the
|
|
644
|
+
* name. Pass `purgeTombstones: false` to keep every tombstone.
|
|
645
|
+
*
|
|
646
|
+
* Needs the store built with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise),
|
|
647
|
+
* because retiring a segment deletes its Cold objects. `now` defaults to the store's clock.
|
|
648
|
+
*/
|
|
649
|
+
async retireExpired(options = {}) {
|
|
650
|
+
const deps = this.compactionDeps();
|
|
651
|
+
return retireExpired(
|
|
652
|
+
{ registry: deps.registry, warm: deps.warm, cold: deps.cold },
|
|
653
|
+
{ ...options, now: options.now ?? this.clock.now() }
|
|
654
|
+
);
|
|
655
|
+
}
|
|
656
|
+
/** The store's registry, or a typed error naming the operation that needs one. */
|
|
657
|
+
requireRegistry(op) {
|
|
658
|
+
if (this.registry === void 0) {
|
|
659
|
+
throw new UnsupportedError(`${op} needs a \`registry\` in the store config`);
|
|
660
|
+
}
|
|
661
|
+
return this.registry;
|
|
662
|
+
}
|
|
551
663
|
/**
|
|
552
664
|
* **Cross-tier DR consistency check (audit gap #11).** After a restore/failover, verify every registered
|
|
553
665
|
* segment's `currentGen` actually has its `.crbm` present in Cold — catching a **torn restore** where the
|
|
@@ -575,10 +687,8 @@ var CloudRoaring = class {
|
|
|
575
687
|
* exists" means the run finished — check `failed`. The `export-segments` CLI wraps this with a filesystem sink.
|
|
576
688
|
*/
|
|
577
689
|
async exportSegments(sink, options = {}) {
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
}
|
|
581
|
-
return runExport$1(this, this.registry, sink, {
|
|
690
|
+
const registry = this.requireRegistry("exportSegments");
|
|
691
|
+
return runExport$1(this, registry, sink, {
|
|
582
692
|
...options,
|
|
583
693
|
codec: options.codec ?? roaringCodec
|
|
584
694
|
});
|
|
@@ -789,8 +899,8 @@ var Segment = class {
|
|
|
789
899
|
});
|
|
790
900
|
}
|
|
791
901
|
};
|
|
792
|
-
var VERSION = "0.
|
|
902
|
+
var VERSION = "0.9.0";
|
|
793
903
|
|
|
794
904
|
export { CloudRoaring, SystemClock, runCompactionCycle, src_exports };
|
|
795
|
-
//# sourceMappingURL=chunk-
|
|
796
|
-
//# sourceMappingURL=chunk-
|
|
905
|
+
//# sourceMappingURL=chunk-D6LIJSGF.js.map
|
|
906
|
+
//# sourceMappingURL=chunk-D6LIJSGF.js.map
|