@cloudbitmaps/roaring 0.1.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/PRIVACY.md +185 -0
- package/README.md +40 -0
- package/dist/azure/index.cjs +14 -0
- package/dist/azure/index.cjs.map +1 -0
- package/dist/azure/index.d.cts +9 -0
- package/dist/azure/index.d.ts +9 -0
- package/dist/azure/index.js +3 -0
- package/dist/azure/index.js.map +1 -0
- package/dist/bin/compact-segments.js +133 -0
- package/dist/bin/compact-segments.js.map +1 -0
- package/dist/bin/export-segments.js +131 -0
- package/dist/bin/export-segments.js.map +1 -0
- package/dist/cassandra/index.cjs +14 -0
- package/dist/cassandra/index.cjs.map +1 -0
- package/dist/cassandra/index.d.cts +1 -0
- package/dist/cassandra/index.d.ts +1 -0
- package/dist/cassandra/index.js +3 -0
- package/dist/cassandra/index.js.map +1 -0
- package/dist/chunk-JWCQLJUG.js +573 -0
- package/dist/chunk-JWCQLJUG.js.map +1 -0
- package/dist/dynamodb/index.cjs +14 -0
- package/dist/dynamodb/index.cjs.map +1 -0
- package/dist/dynamodb/index.d.cts +1 -0
- package/dist/dynamodb/index.d.ts +1 -0
- package/dist/dynamodb/index.js +3 -0
- package/dist/dynamodb/index.js.map +1 -0
- package/dist/gcs/index.cjs +14 -0
- package/dist/gcs/index.cjs.map +1 -0
- package/dist/gcs/index.d.cts +1 -0
- package/dist/gcs/index.d.ts +1 -0
- package/dist/gcs/index.js +3 -0
- package/dist/gcs/index.js.map +1 -0
- package/dist/index.cjs +561 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +446 -0
- package/dist/index.d.ts +446 -0
- package/dist/index.js +542 -0
- package/dist/index.js.map +1 -0
- package/dist/mongodb/index.cjs +14 -0
- package/dist/mongodb/index.cjs.map +1 -0
- package/dist/mongodb/index.d.cts +1 -0
- package/dist/mongodb/index.d.ts +1 -0
- package/dist/mongodb/index.js +3 -0
- package/dist/mongodb/index.js.map +1 -0
- package/dist/mysql/index.cjs +14 -0
- package/dist/mysql/index.cjs.map +1 -0
- package/dist/mysql/index.d.cts +1 -0
- package/dist/mysql/index.d.ts +1 -0
- package/dist/mysql/index.js +3 -0
- package/dist/mysql/index.js.map +1 -0
- package/dist/postgres/index.cjs +14 -0
- package/dist/postgres/index.cjs.map +1 -0
- package/dist/postgres/index.d.cts +1 -0
- package/dist/postgres/index.d.ts +1 -0
- package/dist/postgres/index.js +3 -0
- package/dist/postgres/index.js.map +1 -0
- package/dist/redis/index.cjs +14 -0
- package/dist/redis/index.cjs.map +1 -0
- package/dist/redis/index.d.cts +1 -0
- package/dist/redis/index.d.ts +1 -0
- package/dist/redis/index.js +3 -0
- package/dist/redis/index.js.map +1 -0
- package/dist/s3/index.cjs +14 -0
- package/dist/s3/index.cjs.map +1 -0
- package/dist/s3/index.d.cts +1 -0
- package/dist/s3/index.d.ts +1 -0
- package/dist/s3/index.js +3 -0
- package/dist/s3/index.js.map +1 -0
- package/package.json +183 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var core = require('@cloudbitmaps/core');
|
|
4
|
+
var roaring = require('roaring');
|
|
5
|
+
|
|
6
|
+
function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
7
|
+
|
|
8
|
+
var roaring__default = /*#__PURE__*/_interopDefault(roaring);
|
|
9
|
+
|
|
10
|
+
// src/index.ts
|
|
11
|
+
var { RoaringBitmap32, SerializationFormat, DeserializationFormat } = roaring__default.default;
|
|
12
|
+
var SafeBitmap = class _SafeBitmap {
|
|
13
|
+
bitmap;
|
|
14
|
+
constructor(bitmap) {
|
|
15
|
+
this.bitmap = bitmap;
|
|
16
|
+
}
|
|
17
|
+
static empty() {
|
|
18
|
+
return new _SafeBitmap(new RoaringBitmap32());
|
|
19
|
+
}
|
|
20
|
+
static fromValues(values) {
|
|
21
|
+
return new _SafeBitmap(new RoaringBitmap32(values));
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* S1: validate size, then deserialize with the **portable** (validated) format.
|
|
25
|
+
* Throws `IntegrityError` if the input exceeds `maxBytes` or fails to decode — the
|
|
26
|
+
* native addon is never handed unbounded or unsafe-format input.
|
|
27
|
+
*/
|
|
28
|
+
static safeDeserialize(bytes, maxBytes) {
|
|
29
|
+
if (bytes.length > maxBytes) {
|
|
30
|
+
throw new core.IntegrityError(`serialized bitmap is ${bytes.length}B, exceeds cap ${maxBytes}B`);
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
return new _SafeBitmap(RoaringBitmap32.deserialize(bytes, DeserializationFormat.portable));
|
|
34
|
+
} catch (err) {
|
|
35
|
+
throw new core.IntegrityError(`failed to deserialize bitmap: ${err.message}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
serialize() {
|
|
39
|
+
return this.bitmap.serialize(SerializationFormat.portable);
|
|
40
|
+
}
|
|
41
|
+
add(value) {
|
|
42
|
+
this.bitmap.add(value);
|
|
43
|
+
}
|
|
44
|
+
addMany(values) {
|
|
45
|
+
this.bitmap.addMany(values);
|
|
46
|
+
}
|
|
47
|
+
remove(value) {
|
|
48
|
+
this.bitmap.remove(value);
|
|
49
|
+
}
|
|
50
|
+
removeMany(values) {
|
|
51
|
+
this.bitmap.removeMany(values);
|
|
52
|
+
}
|
|
53
|
+
has(value) {
|
|
54
|
+
return this.bitmap.has(value);
|
|
55
|
+
}
|
|
56
|
+
get size() {
|
|
57
|
+
return this.bitmap.size;
|
|
58
|
+
}
|
|
59
|
+
get isEmpty() {
|
|
60
|
+
return this.bitmap.isEmpty;
|
|
61
|
+
}
|
|
62
|
+
clone() {
|
|
63
|
+
return new _SafeBitmap(this.bitmap.clone());
|
|
64
|
+
}
|
|
65
|
+
// The binary set ops take the `CodecBitmap` interface type (per the seam), but a single store is
|
|
66
|
+
// single-codec (homogeneity contract), so `other` is always a `SafeBitmap` here — reach its private
|
|
67
|
+
// `bitmap` (accessible on same-class instances) for the native op.
|
|
68
|
+
/** In-place union: `this = this ∪ other`. */
|
|
69
|
+
orInPlace(other) {
|
|
70
|
+
this.bitmap.orInPlace(other.bitmap);
|
|
71
|
+
}
|
|
72
|
+
/** In-place difference: `this = this \ other`. */
|
|
73
|
+
andNotInPlace(other) {
|
|
74
|
+
this.bitmap.andNotInPlace(other.bitmap);
|
|
75
|
+
}
|
|
76
|
+
/** In-place intersection: `this = this ∩ other`. */
|
|
77
|
+
andInPlace(other) {
|
|
78
|
+
this.bitmap.andInPlace(other.bitmap);
|
|
79
|
+
}
|
|
80
|
+
/** Ascending iterator over the set values. */
|
|
81
|
+
[Symbol.iterator]() {
|
|
82
|
+
return this.bitmap[Symbol.iterator]();
|
|
83
|
+
}
|
|
84
|
+
toArray() {
|
|
85
|
+
return this.bitmap.toArray();
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
var roaringCodec = {
|
|
89
|
+
empty: () => SafeBitmap.empty(),
|
|
90
|
+
fromValues: (values) => SafeBitmap.fromValues(values),
|
|
91
|
+
safeDeserialize: (bytes, maxBytes) => SafeBitmap.safeDeserialize(bytes, maxBytes)
|
|
92
|
+
};
|
|
93
|
+
var bulkLoadCrbmGeneration = (driver, key, ids, options = {}) => core.bulkLoadCrbmGeneration(driver, key, ids, { ...options, codec: options.codec ?? roaringCodec });
|
|
94
|
+
var compactSegment = (ref, deps, options) => core.compactSegment(ref, { ...deps, codec: deps.codec ?? roaringCodec }, options);
|
|
95
|
+
var runCompactionCycle = (deps, options) => core.runCompactionCycle({ ...deps, codec: deps.codec ?? roaringCodec }, options);
|
|
96
|
+
var runExport = (reader, registry, sink, options = {}) => core.runExport(reader, registry, sink, { ...options, codec: options.codec ?? roaringCodec });
|
|
97
|
+
|
|
98
|
+
// src/index.ts
|
|
99
|
+
var SystemClock = class {
|
|
100
|
+
now() {
|
|
101
|
+
return Date.now();
|
|
102
|
+
}
|
|
103
|
+
sleep(ms) {
|
|
104
|
+
if (ms <= 0) return Promise.resolve();
|
|
105
|
+
return new Promise((resolve) => {
|
|
106
|
+
setTimeout(resolve, ms);
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
var SystemRng = class {
|
|
111
|
+
next() {
|
|
112
|
+
return Math.random();
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
116
|
+
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
117
|
+
function validateConcurrency(concurrency) {
|
|
118
|
+
if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
119
|
+
throw new core.ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
function requireScope(options, op) {
|
|
123
|
+
if (options.namespace === void 0 && options.allNamespaces !== true) {
|
|
124
|
+
throw new core.ValidationError(
|
|
125
|
+
`${op} scans the global id space across all namespaces \u2014 pass an explicit \`namespace\`, or \`{ allNamespaces: true }\` to intentionally sweep the whole fleet`
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function resolveColdSource(options, clock) {
|
|
130
|
+
const cold = options.cold;
|
|
131
|
+
if (cold === null || typeof cold !== "object") {
|
|
132
|
+
throw new core.ValidationError("`cold` must be an IColdDriver or a ColdChunkSource");
|
|
133
|
+
}
|
|
134
|
+
const hasGetChunk = typeof cold.getChunk === "function";
|
|
135
|
+
const hasPutImmutable = typeof cold.putImmutable === "function";
|
|
136
|
+
if (hasGetChunk && hasPutImmutable) {
|
|
137
|
+
throw new core.ValidationError(
|
|
138
|
+
"`cold` exposes both `getChunk` and `putImmutable` \u2014 ambiguous; pass an IColdDriver or a ColdChunkSource, not a hybrid"
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
if (!hasGetChunk && !hasPutImmutable) {
|
|
142
|
+
throw new core.ValidationError("`cold` must be an IColdDriver or a ColdChunkSource");
|
|
143
|
+
}
|
|
144
|
+
if (hasGetChunk) {
|
|
145
|
+
if (options.registry !== void 0 || options.keystore !== void 0 || options.requireEncryption === true) {
|
|
146
|
+
throw new core.ValidationError(
|
|
147
|
+
"registry/keystore/requireEncryption apply only when `cold` is a raw IColdDriver; configure them on the ColdChunkSource you passed instead"
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
return { source: cold, driver: void 0 };
|
|
151
|
+
}
|
|
152
|
+
const driver = cold;
|
|
153
|
+
return {
|
|
154
|
+
source: new core.CrbmColdChunkSource(driver, {
|
|
155
|
+
registry: options.registry,
|
|
156
|
+
keystore: options.keystore,
|
|
157
|
+
requireEncryption: options.requireEncryption,
|
|
158
|
+
clock,
|
|
159
|
+
currentGenTtlMs: options.coldGenTtlMs,
|
|
160
|
+
maxOpenSegments: options.coldReaderCacheMax,
|
|
161
|
+
maxOpenIndexBytes: options.coldReaderCacheMaxBytes
|
|
162
|
+
}),
|
|
163
|
+
driver
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
var CloudRoaring = class {
|
|
167
|
+
engine;
|
|
168
|
+
clock;
|
|
169
|
+
metrics;
|
|
170
|
+
// The store's own drivers, kept so the in-process lifecycle helpers (`compact`/`eraseSubject`/`subjectReport`)
|
|
171
|
+
// reuse them instead of making you re-pass a CompactionDeps. `coldDriver` is set only when `cold` was a raw
|
|
172
|
+
// IColdDriver (a pre-built ColdChunkSource has no underlying driver to compact through).
|
|
173
|
+
coldDriver;
|
|
174
|
+
warmDriver;
|
|
175
|
+
registry;
|
|
176
|
+
keystore;
|
|
177
|
+
requireEncryption;
|
|
178
|
+
/** Resolved store-level per-op budget (null = disabled); the admin scans use it, with a per-op override. */
|
|
179
|
+
budget;
|
|
180
|
+
constructor(options) {
|
|
181
|
+
const clock = options.clock ?? new SystemClock();
|
|
182
|
+
const rng = options.rng ?? new SystemRng();
|
|
183
|
+
const metrics = core.safeMetrics(options.metrics ?? core.NOOP_METRICS);
|
|
184
|
+
const cache = new core.BoundedLru({
|
|
185
|
+
maxEntries: options.cacheMaxChunks ?? DEFAULT_CACHE_MAX_CHUNKS,
|
|
186
|
+
ttlMs: options.cacheTtlMs,
|
|
187
|
+
clock
|
|
188
|
+
});
|
|
189
|
+
const resolved = resolveColdSource(options, clock);
|
|
190
|
+
let cold = resolved.source;
|
|
191
|
+
let warm = options.warm;
|
|
192
|
+
if (options.retry !== false) {
|
|
193
|
+
const retryOpts = {
|
|
194
|
+
clock,
|
|
195
|
+
rng,
|
|
196
|
+
policy: options.retry,
|
|
197
|
+
// Bridge transient-fault retries into the metrics stream, then call the user's own hook.
|
|
198
|
+
onRetry: (info) => {
|
|
199
|
+
metrics.onEvent({
|
|
200
|
+
kind: "retry",
|
|
201
|
+
reason: "transient",
|
|
202
|
+
attempt: info.attempt,
|
|
203
|
+
delayMs: info.delayMs
|
|
204
|
+
});
|
|
205
|
+
options.onRetry?.(info);
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
warm = new core.RetryingWarmDriver(options.warm, retryOpts);
|
|
209
|
+
cold = new core.RetryingColdChunkSource(cold, retryOpts);
|
|
210
|
+
}
|
|
211
|
+
if (options.writeConcurrency !== void 0 && (!Number.isInteger(options.writeConcurrency) || options.writeConcurrency < 1)) {
|
|
212
|
+
throw new core.ValidationError(
|
|
213
|
+
`writeConcurrency must be a positive integer; got ${options.writeConcurrency}`
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
this.budget = core.resolveBudget(options.budget, core.DEFAULT_BUDGET);
|
|
217
|
+
const deps = {
|
|
218
|
+
warm,
|
|
219
|
+
cold,
|
|
220
|
+
cache,
|
|
221
|
+
codec: roaringCodec,
|
|
222
|
+
// the facade injects the flagship codec ([DECISIONS #58]); core stays codec-agnostic
|
|
223
|
+
clock,
|
|
224
|
+
rng,
|
|
225
|
+
occBackoff: options.occBackoff,
|
|
226
|
+
metrics,
|
|
227
|
+
warmReadConsistency: options.warmReadConsistency,
|
|
228
|
+
writeConcurrency: options.writeConcurrency,
|
|
229
|
+
budget: this.budget
|
|
230
|
+
};
|
|
231
|
+
this.engine = new core.SegmentEngine(deps);
|
|
232
|
+
this.clock = clock;
|
|
233
|
+
this.metrics = metrics;
|
|
234
|
+
this.coldDriver = resolved.driver;
|
|
235
|
+
this.warmDriver = options.warm;
|
|
236
|
+
this.registry = options.registry;
|
|
237
|
+
this.keystore = options.keystore;
|
|
238
|
+
this.requireEncryption = options.requireEncryption ?? false;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Build {@link CompactionDeps} from the store's own drivers, for the in-process lifecycle helpers
|
|
242
|
+
* ({@link compact} / {@link eraseSubject}). Requires the store to have been constructed with a **raw cold
|
|
243
|
+
* driver** (a pre-built `ColdChunkSource` has no underlying `IColdDriver` to compact through) and a `registry`
|
|
244
|
+
* (the authoritative `currentGen` pointer compaction swaps). Out-of-process callers use the `compactSegment`
|
|
245
|
+
* free function with explicit deps.
|
|
246
|
+
*/
|
|
247
|
+
compactionDeps() {
|
|
248
|
+
if (this.coldDriver === void 0) {
|
|
249
|
+
throw new core.UnsupportedError(
|
|
250
|
+
"compact/eraseSubject/checkConsistency need the store built with a raw cold driver (IColdDriver), not a pre-built ColdChunkSource \u2014 or call the compactSegment/runConsistencyCheck free functions with explicit deps"
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
if (this.registry === void 0) {
|
|
254
|
+
throw new core.UnsupportedError(
|
|
255
|
+
"compact/eraseSubject/checkConsistency need a `registry` in the store config"
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
return {
|
|
259
|
+
cold: this.coldDriver,
|
|
260
|
+
warm: this.warmDriver,
|
|
261
|
+
registry: this.registry,
|
|
262
|
+
clock: this.clock,
|
|
263
|
+
codec: roaringCodec,
|
|
264
|
+
// facade injects the flagship codec ([DECISIONS #58])
|
|
265
|
+
keystore: this.keystore,
|
|
266
|
+
requireEncryption: this.requireEncryption,
|
|
267
|
+
metrics: this.metrics
|
|
268
|
+
// safe-wrapped sink → store.compact/eraseSubject emit `compaction` events (gap #2)
|
|
269
|
+
};
|
|
270
|
+
}
|
|
271
|
+
/** Get a handle to a segment. Validates the name/namespace grammar (finding S2). */
|
|
272
|
+
segment(name, options) {
|
|
273
|
+
const ref = { segment: name, namespace: options?.namespace };
|
|
274
|
+
core.validateSegmentRef(ref);
|
|
275
|
+
return new Segment(this.engine, ref, this.clock, this.metrics);
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* **Subject access (GDPR Art. 15 / CCPA right-to-know): which segments is this id a member of?** (Phase 6b.)
|
|
279
|
+
*
|
|
280
|
+
* Enumerates the **registered** segments (via the store's own `registry`) and does a tier-merging `has(id)` on
|
|
281
|
+
* each — no drivers to re-pass. Complete only over registered segments (register your segments if you claim
|
|
282
|
+
* SAR support). There is deliberately **no `id → segments` reverse index** — that would tax every write for a
|
|
283
|
+
* rare request; this admin scan is `O(registered segments)` and touches no hot path
|
|
284
|
+
* (phases/06). Requires a `registry` in the store config
|
|
285
|
+
* (throws {@link UnsupportedError} otherwise).
|
|
286
|
+
*/
|
|
287
|
+
async subjectReport(id, options = {}) {
|
|
288
|
+
if (this.registry === void 0) {
|
|
289
|
+
throw new core.UnsupportedError("subjectReport needs a `registry` in the store config");
|
|
290
|
+
}
|
|
291
|
+
const registry = this.registry;
|
|
292
|
+
requireScope(options, "subjectReport");
|
|
293
|
+
validateConcurrency(options.concurrency);
|
|
294
|
+
const budget = core.resolvePerOpBudget(options.budget, this.budget);
|
|
295
|
+
core.splitId(id);
|
|
296
|
+
const recs = [];
|
|
297
|
+
for await (const rec of registry.list(options.namespace)) recs.push(rec);
|
|
298
|
+
core.checkBudget(budget, recs.length, "subjectReport");
|
|
299
|
+
const membership = await core.mapWithConcurrency(
|
|
300
|
+
recs,
|
|
301
|
+
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
302
|
+
async (rec) => {
|
|
303
|
+
if (rec.status === "destroyed") return null;
|
|
304
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
305
|
+
return await this.engine.has(ref, id, { consistent: true }) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
306
|
+
}
|
|
307
|
+
);
|
|
308
|
+
const segments = membership.filter((m) => m !== null);
|
|
309
|
+
return { id, segments, scannedSegments: recs.length };
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* **Subject erasure (GDPR Art. 17): remove an id from every segment it's in, with a physical-deletion
|
|
313
|
+
* guarantee.** (Phase 6b.)
|
|
314
|
+
*
|
|
315
|
+
* For each **registered** segment the id is a member of: writes a logical `remove` tombstone (immediate) and
|
|
316
|
+
* then **force-compacts that segment now** — folding the tombstone into a fresh immutable generation so the
|
|
317
|
+
* bit is *physically* gone from Cold on return, even for an otherwise-idle/archival segment that organic
|
|
318
|
+
* compaction would never touch (the P13 fix). The returned per-segment record is your **erasure ledger** —
|
|
319
|
+
* persist it / route it to your audit sink as the proof of deletion; the physical-purge proof is
|
|
320
|
+
* compaction's own VERIFY step (a re-`has()` here would be unsound — a pinned cold source can still read the
|
|
321
|
+
* old generation; take a fresh generation view after erasing, exactly as with the compaction daemon).
|
|
322
|
+
*
|
|
323
|
+
* Uses the store's **own** drivers (raw cold + warm + registry), so the old integrator obligation to wire a
|
|
324
|
+
* matching `CompactionDeps` is gone — the `remove()` and the force-compaction provably run over the same tiers.
|
|
325
|
+
* Requires the store built with a **raw cold driver + registry** (throws {@link UnsupportedError} otherwise;
|
|
326
|
+
* a pre-built `ColdChunkSource` store has no `IColdDriver` to compact through — use the `compactSegment` free
|
|
327
|
+
* function there). **One contract remains** (an integrator obligation the library cannot check): **do not
|
|
328
|
+
* concurrently re-add the same id while erasing it.** A normal `add(id)` landing between the `remove` and the
|
|
329
|
+
* compaction clears the tombstone and folds the id back into the new generation
|
|
330
|
+
* (`effective = (cold ∪ adds) \ removes`); `physicallyPurged` attests only that compaction committed a
|
|
331
|
+
* generation with the tombstones applied at pin time — quiesce writes for the subject during erasure.
|
|
332
|
+
*
|
|
333
|
+
* **A `physicallyPurged:false` entry means the logical removal holds but the physical purge didn't run this
|
|
334
|
+
* call** — either a daemon held a live lease (`note:'leased-by-other'`; the daemon finishes it), or the
|
|
335
|
+
* force-compaction hit an isolated fault (`note:'error: …'`; per-segment faults are caught so one segment
|
|
336
|
+
* can't discard the whole ledger). `physicallyPurged:false` is **conservative** (a concurrent daemon may
|
|
337
|
+
* already have purged the bit — `'clean'`). **Recovery:** follow up any `physicallyPurged:false` entry with
|
|
338
|
+
* `store.compact(ref)` (or let the daemon do it) — **re-running `eraseSubject` will NOT re-purge it**, because
|
|
339
|
+
* `has()` now reads false (the tombstone) so the segment is skipped as a non-member. The physical-purge proof
|
|
340
|
+
* is compaction's own VERIFY step (a re-`has()` here would be unsound: a pinned cold source can still read the
|
|
341
|
+
* old generation — take a fresh generation view after erasing, as with the daemon). Admin-only path;
|
|
342
|
+
* `O(registered segments)`, no hot-path cost. Per-subject crypto-shred is infeasible (a subject's bit is
|
|
343
|
+
* co-mingled in a shared container), so this is the single-subject erasure route; whole-segment/tenant erasure
|
|
344
|
+
* is the `destroySegment`/`eraseNamespace` free functions.
|
|
345
|
+
*/
|
|
346
|
+
async eraseSubject(id, options) {
|
|
347
|
+
const compaction = this.compactionDeps();
|
|
348
|
+
core.validateCompactionOptions({ owner: options.owner });
|
|
349
|
+
requireScope(options, "eraseSubject");
|
|
350
|
+
validateConcurrency(options.concurrency);
|
|
351
|
+
const budget = core.resolvePerOpBudget(options.budget, this.budget);
|
|
352
|
+
core.splitId(id);
|
|
353
|
+
const recs = [];
|
|
354
|
+
for await (const rec of compaction.registry.list(options.namespace)) recs.push(rec);
|
|
355
|
+
core.checkBudget(budget, recs.length, "eraseSubject");
|
|
356
|
+
const entries = await core.mapWithConcurrency(
|
|
357
|
+
recs,
|
|
358
|
+
options.concurrency ?? DEFAULT_ADMIN_CONCURRENCY,
|
|
359
|
+
async (rec) => {
|
|
360
|
+
if (rec.status === "destroyed") return null;
|
|
361
|
+
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
362
|
+
let removed = false;
|
|
363
|
+
try {
|
|
364
|
+
if (!await this.engine.has(ref, id, { consistent: true })) return null;
|
|
365
|
+
await this.engine.remove(ref, id);
|
|
366
|
+
removed = true;
|
|
367
|
+
const result = await core.compactSegment(ref, compaction, {
|
|
368
|
+
owner: options.owner,
|
|
369
|
+
audit: options.audit
|
|
370
|
+
});
|
|
371
|
+
return {
|
|
372
|
+
segment: rec.segment,
|
|
373
|
+
namespace: rec.namespace,
|
|
374
|
+
removed: true,
|
|
375
|
+
physicallyPurged: result.compacted,
|
|
376
|
+
toGen: result.toGen,
|
|
377
|
+
note: result.compacted ? void 0 : result.reason
|
|
378
|
+
};
|
|
379
|
+
} catch (err) {
|
|
380
|
+
return {
|
|
381
|
+
segment: rec.segment,
|
|
382
|
+
namespace: rec.namespace,
|
|
383
|
+
removed,
|
|
384
|
+
physicallyPurged: false,
|
|
385
|
+
note: `error: ${err instanceof Error ? err.message : String(err)}`
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
);
|
|
390
|
+
const erasedFrom = entries.filter((e) => e !== null);
|
|
391
|
+
return { id, erasedFrom, scannedSegments: recs.length };
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* **Compact one segment now**, in-process, using the store's own drivers — a convenience over the
|
|
395
|
+
* {@link compactSegment} free function. Merges `(cold ∪ adds) \ removes` into a fresh immutable generation,
|
|
396
|
+
* swaps `currentGen`, then version-fenced-purges the archived Warm rows. A no-op (`compacted:false`) when the
|
|
397
|
+
* segment has no dirty rows or a daemon holds a live lease — never throws on the normal contention/lease paths.
|
|
398
|
+
* Requires the store built with a **raw cold driver + registry** (throws {@link UnsupportedError} otherwise);
|
|
399
|
+
* like the daemon, it uses the raw drivers directly (a transient fault surfaces to you rather than retrying
|
|
400
|
+
* under the hood). **Not for the request/hot path** — it reads and rewrites a whole Cold generation (streaming,
|
|
401
|
+
* constant-memory, but full-segment I/O). Use the `compact-segments` daemon (`runCompactionCycle`) for routine
|
|
402
|
+
* background compaction; reach for `store.compact()` for occasional/manual one-shots — a maintenance endpoint,
|
|
403
|
+
* a small daemon-less deployment, or right after a targeted `remove`.
|
|
404
|
+
*/
|
|
405
|
+
async compact(ref, options) {
|
|
406
|
+
core.validateSegmentRef(ref);
|
|
407
|
+
return core.compactSegment(ref, this.compactionDeps(), options);
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* **Cross-tier DR consistency check (audit gap #11).** After a restore/failover, verify every registered
|
|
411
|
+
* segment's `currentGen` actually has its `.crbm` present in Cold — catching a **torn restore** where the
|
|
412
|
+
* registry (`currentGen`) came back ahead of the object store, so a pointer references a generation that
|
|
413
|
+
* isn't there (reads would then throw). Read-only, bounded fan-out; run it at startup after a restore.
|
|
414
|
+
* Returns `{ checked, inconsistent }` — `inconsistent` empty ⇒ coherent; otherwise it names the segments to
|
|
415
|
+
* recover (restore the object store, or roll the registry back to a coherent point). Needs the store built
|
|
416
|
+
* with a **raw cold driver + a registry** (throws {@link UnsupportedError} otherwise). `destroyed`
|
|
417
|
+
* (crypto-shredded) segments are skipped. Pair it with the DR runbook (docs/guide/disaster-recovery.md).
|
|
418
|
+
*/
|
|
419
|
+
async checkConsistency(options = {}) {
|
|
420
|
+
const deps = this.compactionDeps();
|
|
421
|
+
return core.runConsistencyCheck({ cold: deps.cold, registry: deps.registry }, options);
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* **Export ("eject") every registered segment's current effective set** through the injected `sink`, using
|
|
425
|
+
* only public read APIs — so your data is readable **without CloudRoaring** (the exit path; see the README's
|
|
426
|
+
* "Your data stays yours"). `format: 'roaring'` (default) writes one **portable RoaringBitmap32** per segment
|
|
427
|
+
* (loadable by any roaring library); `'ndjson'` writes newline-delimited ids (zero-dependency, streaming).
|
|
428
|
+
* Enumerates via the store's **own** registry (needs one — throws {@link UnsupportedError} otherwise) so the
|
|
429
|
+
* enumeration and the read path provably share one registry; an all-warm segment not yet in the registry can be
|
|
430
|
+
* named via `options.candidates`. Encrypted segments are decrypted transparently **iff** this store was wired
|
|
431
|
+
* with their keystore — the export is therefore **cleartext**; protect it. Crypto-shredded segments are skipped.
|
|
432
|
+
* A segment that can't be read is isolated into the manifest's `failed[]` (the run continues), so "a manifest
|
|
433
|
+
* exists" means the run finished — check `failed`. The `export-segments` CLI wraps this with a filesystem sink.
|
|
434
|
+
*/
|
|
435
|
+
async exportSegments(sink, options = {}) {
|
|
436
|
+
if (this.registry === void 0) {
|
|
437
|
+
throw new core.UnsupportedError("exportSegments needs a `registry` in the store config");
|
|
438
|
+
}
|
|
439
|
+
return core.runExport(this, this.registry, sink, {
|
|
440
|
+
...options,
|
|
441
|
+
codec: options.codec ?? roaringCodec
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Planning cost estimate (Phase 5b) — pure, no instance/data needed: sizing, sales, what-if. For a real,
|
|
446
|
+
* grounded report from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
447
|
+
*/
|
|
448
|
+
static estimateCost(input) {
|
|
449
|
+
return core.estimateCost(input);
|
|
450
|
+
}
|
|
451
|
+
};
|
|
452
|
+
var Segment = class {
|
|
453
|
+
constructor(engine, ref, clock, metrics) {
|
|
454
|
+
this.engine = engine;
|
|
455
|
+
this.ref = ref;
|
|
456
|
+
this.clock = clock;
|
|
457
|
+
this.metrics = metrics;
|
|
458
|
+
this.metricsOn = metrics !== core.NOOP_METRICS;
|
|
459
|
+
}
|
|
460
|
+
engine;
|
|
461
|
+
ref;
|
|
462
|
+
clock;
|
|
463
|
+
metrics;
|
|
464
|
+
metricsOn;
|
|
465
|
+
/**
|
|
466
|
+
* Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
|
|
467
|
+
* entirely when no sink is wired, so the default path pays nothing.
|
|
468
|
+
*/
|
|
469
|
+
async timed(name, fn) {
|
|
470
|
+
if (!this.metricsOn) return fn();
|
|
471
|
+
const startedAt = this.clock.now();
|
|
472
|
+
try {
|
|
473
|
+
return await fn();
|
|
474
|
+
} finally {
|
|
475
|
+
this.metrics.onEvent({ kind: "op", name, ms: Math.max(0, this.clock.now() - startedAt) });
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
/** Add an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
|
|
479
|
+
add(id) {
|
|
480
|
+
return this.timed("add", () => this.engine.add(this.ref, id));
|
|
481
|
+
}
|
|
482
|
+
/** Add many ids. Not atomic across chunks — see the class note. */
|
|
483
|
+
addMany(ids) {
|
|
484
|
+
return this.timed("addMany", () => this.engine.addMany(this.ref, ids));
|
|
485
|
+
}
|
|
486
|
+
/** Remove an id (integer in `[0, 2^32)`). Throws {@link ValidationError} on a bad id. */
|
|
487
|
+
remove(id) {
|
|
488
|
+
return this.timed("remove", () => this.engine.remove(this.ref, id));
|
|
489
|
+
}
|
|
490
|
+
/** Remove many ids. Not atomic across chunks — see the class note. */
|
|
491
|
+
removeMany(ids) {
|
|
492
|
+
return this.timed("removeMany", () => this.engine.removeMany(this.ref, ids));
|
|
493
|
+
}
|
|
494
|
+
has(id) {
|
|
495
|
+
return this.timed("has", () => this.engine.has(this.ref, id));
|
|
496
|
+
}
|
|
497
|
+
count() {
|
|
498
|
+
return this.timed("count", () => this.engine.count(this.ref));
|
|
499
|
+
}
|
|
500
|
+
iterate() {
|
|
501
|
+
return this.engine.iterate(this.ref);
|
|
502
|
+
}
|
|
503
|
+
/**
|
|
504
|
+
* Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
|
|
505
|
+
* Fetches only the Cold chunks present in *all* operands (a key absent from any operand contributes nothing
|
|
506
|
+
* and is never downloaded), streaming under a bounded in-flight window — so the Cold footprint stays small
|
|
507
|
+
* (Lambda-friendly) regardless of segment size. Pass `concurrency` to tune that window (a positive integer).
|
|
508
|
+
* AND is commutative, so `a.intersect([b])` and `b.intersect([a])` yield the same ids. (Warm state is read
|
|
509
|
+
* up front, so total memory also carries each operand's Warm size — negligible under Topology-A.) Pass
|
|
510
|
+
* `budget` to override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
|
|
511
|
+
*/
|
|
512
|
+
intersect(others, options) {
|
|
513
|
+
return this.engine.intersect([this.ref, ...others.map((o) => o.ref)], options);
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* Materialize `this ∩ others…` **into** `dest` (added, not replaced). Streaming + bounded-memory, but
|
|
517
|
+
* **not atomic** across chunks — see {@link addMany}.
|
|
518
|
+
*/
|
|
519
|
+
intersectInto(dest, others, options) {
|
|
520
|
+
return this.timed(
|
|
521
|
+
"intersectInto",
|
|
522
|
+
() => this.engine.intersectInto(dest.ref, [this.ref, ...others.map((o) => o.ref)], options)
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Grounded cost report for this segment (Phase 5b): storage cost from its **real** `.crbm` size (exact,
|
|
527
|
+
* no payload reads); request cost from the supplied `workload` rates. A segment with no Cold generation
|
|
528
|
+
* reports zero storage. See {@link CostReport} — it always includes a verdict (incl. the lose-zone).
|
|
529
|
+
*/
|
|
530
|
+
async costReport(options) {
|
|
531
|
+
const canMeasure = this.engine.supportsColdSize;
|
|
532
|
+
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
533
|
+
return core.groundedReport({
|
|
534
|
+
coldBytes: size?.sizeBytes ?? 0,
|
|
535
|
+
grounded: canMeasure,
|
|
536
|
+
workload: options?.workload,
|
|
537
|
+
topology: options?.topology,
|
|
538
|
+
pricing: options?.pricing,
|
|
539
|
+
extraNotes: canMeasure ? void 0 : ["cold source has no sizeOf() \u2014 storage not measured, reported as $0."]
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
};
|
|
543
|
+
var VERSION = "0.1.0-rc.0";
|
|
544
|
+
|
|
545
|
+
exports.CloudRoaring = CloudRoaring;
|
|
546
|
+
exports.SafeBitmap = SafeBitmap;
|
|
547
|
+
exports.Segment = Segment;
|
|
548
|
+
exports.VERSION = VERSION;
|
|
549
|
+
exports.bulkLoadCrbmGeneration = bulkLoadCrbmGeneration;
|
|
550
|
+
exports.compactSegment = compactSegment;
|
|
551
|
+
exports.roaringCodec = roaringCodec;
|
|
552
|
+
exports.runCompactionCycle = runCompactionCycle;
|
|
553
|
+
exports.runExport = runExport;
|
|
554
|
+
Object.keys(core).forEach(function (k) {
|
|
555
|
+
if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
|
|
556
|
+
enumerable: true,
|
|
557
|
+
get: function () { return core[k]; }
|
|
558
|
+
});
|
|
559
|
+
});
|
|
560
|
+
//# sourceMappingURL=index.cjs.map
|
|
561
|
+
//# sourceMappingURL=index.cjs.map
|