@cloudbitmaps/roaring 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -3
- package/PRIVACY.md +198 -72
- package/README.md +121 -57
- package/dist/bin/export-segments.d.ts +21 -0
- package/dist/bin/export-segments.js +1508 -30
- package/dist/bin/export-segments.js.map +7 -1
- package/dist/codec-bound.d.ts +38 -0
- package/dist/index.d.ts +805 -447
- package/dist/index.js +911 -325
- package/dist/index.js.map +7 -1
- package/dist/moved-options.d.ts +14 -0
- package/dist/portable/decode.d.ts +54 -0
- package/dist/roaring-codec.d.ts +61 -0
- package/dist/system-clock.d.ts +40 -0
- package/dist/testing/conformance.d.ts +34 -0
- package/dist/testing/fuzz-codec.d.ts +9 -0
- package/package.json +11 -116
- package/dist/azure/index.cjs +0 -14
- package/dist/azure/index.cjs.map +0 -1
- package/dist/azure/index.d.cts +0 -9
- package/dist/azure/index.d.ts +0 -9
- package/dist/azure/index.js +0 -3
- package/dist/azure/index.js.map +0 -1
- package/dist/bin/compact-segments.js +0 -201
- package/dist/bin/compact-segments.js.map +0 -1
- package/dist/cassandra/index.cjs +0 -14
- package/dist/cassandra/index.cjs.map +0 -1
- package/dist/cassandra/index.d.cts +0 -1
- package/dist/cassandra/index.d.ts +0 -1
- package/dist/cassandra/index.js +0 -3
- package/dist/cassandra/index.js.map +0 -1
- package/dist/chunk-D6LIJSGF.js +0 -906
- package/dist/chunk-D6LIJSGF.js.map +0 -1
- package/dist/dynamodb/index.cjs +0 -14
- package/dist/dynamodb/index.cjs.map +0 -1
- package/dist/dynamodb/index.d.cts +0 -1
- package/dist/dynamodb/index.d.ts +0 -1
- package/dist/dynamodb/index.js +0 -3
- package/dist/dynamodb/index.js.map +0 -1
- package/dist/gcs/index.cjs +0 -14
- package/dist/gcs/index.cjs.map +0 -1
- package/dist/gcs/index.d.cts +0 -1
- package/dist/gcs/index.d.ts +0 -1
- package/dist/gcs/index.js +0 -3
- package/dist/gcs/index.js.map +0 -1
- package/dist/index.cjs +0 -894
- package/dist/index.cjs.map +0 -1
- package/dist/index.d.cts +0 -734
- package/dist/mongodb/index.cjs +0 -14
- package/dist/mongodb/index.cjs.map +0 -1
- package/dist/mongodb/index.d.cts +0 -1
- package/dist/mongodb/index.d.ts +0 -1
- package/dist/mongodb/index.js +0 -3
- package/dist/mongodb/index.js.map +0 -1
- package/dist/mysql/index.cjs +0 -14
- package/dist/mysql/index.cjs.map +0 -1
- package/dist/mysql/index.d.cts +0 -1
- package/dist/mysql/index.d.ts +0 -1
- package/dist/mysql/index.js +0 -3
- package/dist/mysql/index.js.map +0 -1
- package/dist/postgres/index.cjs +0 -14
- package/dist/postgres/index.cjs.map +0 -1
- package/dist/postgres/index.d.cts +0 -1
- package/dist/postgres/index.d.ts +0 -1
- package/dist/postgres/index.js +0 -3
- package/dist/postgres/index.js.map +0 -1
- package/dist/redis/index.cjs +0 -14
- package/dist/redis/index.cjs.map +0 -1
- package/dist/redis/index.d.cts +0 -1
- package/dist/redis/index.d.ts +0 -1
- package/dist/redis/index.js +0 -3
- package/dist/redis/index.js.map +0 -1
- package/dist/s3/index.cjs +0 -14
- package/dist/s3/index.cjs.map +0 -1
- package/dist/s3/index.d.cts +0 -1
- package/dist/s3/index.d.ts +0 -1
- package/dist/s3/index.js +0 -3
- package/dist/s3/index.js.map +0 -1
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The 0.9.x option spellings this release no longer accepts, and what to tell someone who passes one.
|
|
3
|
+
*
|
|
4
|
+
* NOT part of the public API. It lives in its own module — rather than as an `export` from the barrel — so
|
|
5
|
+
* that the gates which must agree with it (the doc-fence scan, the harness scan, and the test that diffs this
|
|
6
|
+
* table against `CloudRoaringOptions` at the previous tag) can all read ONE copy, without the migration
|
|
7
|
+
* table becoming a permanent public export in the same release whose point was to shrink that surface.
|
|
8
|
+
*
|
|
9
|
+
* Retyping it was tried and is what this replaces: the hand-maintained copies had drifted to naming three
|
|
10
|
+
* spellings no release ever shipped while omitting the five that actually went away with the live tier.
|
|
11
|
+
*/
|
|
12
|
+
export type MovedOptionKind = 'group' | 'renamed' | 'gone';
|
|
13
|
+
/** `[the 0.9.x key, what to do instead, which kind of answer that is]`. */
|
|
14
|
+
export declare const MOVED_OPTIONS: ReadonlyArray<readonly [string, string, MovedOptionKind]>;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A decoded portable-roaring bitmap that can be queried but not modified.
|
|
3
|
+
*
|
|
4
|
+
* Construction parses only the headers — container payloads are read on demand, so a membership test against a
|
|
5
|
+
* large bitmap touches one container rather than materializing the set.
|
|
6
|
+
*/
|
|
7
|
+
export declare class PortableRoaringReader {
|
|
8
|
+
private readonly bytes;
|
|
9
|
+
private readonly view;
|
|
10
|
+
private readonly containers;
|
|
11
|
+
private constructor();
|
|
12
|
+
/**
|
|
13
|
+
* Parse the headers of a portable-format bitmap.
|
|
14
|
+
*
|
|
15
|
+
* @throws {IntegrityError} if the cookie is unrecognized, or any header field runs past the buffer.
|
|
16
|
+
*/
|
|
17
|
+
static decode(bytes: Uint8Array): PortableRoaringReader;
|
|
18
|
+
/**
|
|
19
|
+
* Total cardinality.
|
|
20
|
+
*
|
|
21
|
+
* Served entirely from the descriptive header — no container payload is read, so this is O(containers) and
|
|
22
|
+
* independent of how many values the bitmap holds.
|
|
23
|
+
*/
|
|
24
|
+
count(): number;
|
|
25
|
+
/** Is `value` (a `u32`) in the set? */
|
|
26
|
+
has(value: number): boolean;
|
|
27
|
+
/** Binary search the descriptive header for the container owning `key`. */
|
|
28
|
+
private findContainer;
|
|
29
|
+
/**
|
|
30
|
+
* 1,024 little-endian `u64` words. Read as `u32` at `(low >> 5) * 4`: a LE `u64` is two LE `u32`s with the
|
|
31
|
+
* low word first, so the 32-bit view lands on the right half without any 64-bit arithmetic.
|
|
32
|
+
*/
|
|
33
|
+
private bitmapHas;
|
|
34
|
+
/**
|
|
35
|
+
* Sorted `u16` values — binary search.
|
|
36
|
+
*
|
|
37
|
+
* The `at === low` early return means the `at < low` below is only ever evaluated when `at !== low`, so
|
|
38
|
+
* writing it `at <= low` would behave identically. That is recorded because mutation testing surfaced it as
|
|
39
|
+
* a surviving mutant: it is an *equivalent* one, unkillable by any test rather than a coverage gap — three
|
|
40
|
+
* non-equivalent mutations of these same lines (inverting the branches, never reporting a hit, dropping the
|
|
41
|
+
* last element) are all caught. Do not "fix" the survivor, and do not add a test chasing it.
|
|
42
|
+
*/
|
|
43
|
+
private arrayHas;
|
|
44
|
+
/**
|
|
45
|
+
* `u16` run count, then that many `(start, length)` `u16` pairs sorted by start.
|
|
46
|
+
*
|
|
47
|
+
* `length` is the number of values **after** `start`, so a run covers the inclusive range
|
|
48
|
+
* `[start, start + length]` and a single-value run is stored with length 0. Getting this off by one is the
|
|
49
|
+
* easiest mistake in the format and is why the differential test drives runs specifically.
|
|
50
|
+
*/
|
|
51
|
+
private runHas;
|
|
52
|
+
}
|
|
53
|
+
/** Convenience wrapper over {@link PortableRoaringReader.decode}. */
|
|
54
|
+
export declare function decodePortableRoaring(bytes: Uint8Array): PortableRoaringReader;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { CodecBitmap, CodecInterface } from '@cloudbitmaps/core';
|
|
2
|
+
export declare class SafeBitmap implements CodecBitmap {
|
|
3
|
+
private readonly bitmap;
|
|
4
|
+
private constructor();
|
|
5
|
+
static empty(): SafeBitmap;
|
|
6
|
+
static fromValues(values: Iterable<number>): SafeBitmap;
|
|
7
|
+
/**
|
|
8
|
+
* Validate size, then deserialize with the **portable** (validated) format.
|
|
9
|
+
* Throws `IntegrityError` if the input exceeds `maxBytes` or fails to decode — the
|
|
10
|
+
* native addon is never handed unbounded or unsafe-format input.
|
|
11
|
+
*/
|
|
12
|
+
static safeDeserialize(bytes: Uint8Array, maxBytes: number): SafeBitmap;
|
|
13
|
+
serialize(): Uint8Array;
|
|
14
|
+
/**
|
|
15
|
+
* Convert containers to run-encoding where that is smaller. Representation only — membership is untouched, and
|
|
16
|
+
* the portable format still round-trips through {@link RoaringCodec.safeDeserialize}.
|
|
17
|
+
*
|
|
18
|
+
* Roaring's third container type is a RUN, and no implementation selects it on its own: it takes an explicit
|
|
19
|
+
* `runOptimize()` pass. Nothing here made that call, so run-shaped ids paid array or bitset prices — measured
|
|
20
|
+
* at **570×** for a contiguous 1M-id range (128.1 KiB → 0.2 KiB) and **63×** for a 2,000-run shape
|
|
21
|
+
* (536.5 KiB → 8.5 KiB). Sparse ids come out byte-identical, so this is never a losing trade; roaring keeps
|
|
22
|
+
* whichever encoding is smaller per container.
|
|
23
|
+
*
|
|
24
|
+
* Those two factors are THIS CALL's own before/after on the same ids. They are not the ones the site quotes
|
|
25
|
+
* (543× and 63×) — those compare the optimized encoding against the best *fixed* representation at its floor,
|
|
26
|
+
* a different baseline that happens to land near 63× on the run shape too. `pnpm bench:encoding` prints both
|
|
27
|
+
* columns; don't carry a figure between them without checking which denominator it had.
|
|
28
|
+
*
|
|
29
|
+
* Called only when writing an immutable objects generation — see {@link CodecBitmap.optimize} for why the read
|
|
30
|
+
* path deliberately does not.
|
|
31
|
+
*/
|
|
32
|
+
optimize(): void;
|
|
33
|
+
/**
|
|
34
|
+
* Largest value, or `undefined` when empty. O(1) — roaring reads it off the container index, so the engine's
|
|
35
|
+
* per-chunk range assertion costs one call per chunk rather than a walk per id.
|
|
36
|
+
*/
|
|
37
|
+
maximum(): number | undefined;
|
|
38
|
+
add(value: number): void;
|
|
39
|
+
addMany(values: Iterable<number>): void;
|
|
40
|
+
remove(value: number): void;
|
|
41
|
+
removeMany(values: Iterable<number>): void;
|
|
42
|
+
has(value: number): boolean;
|
|
43
|
+
get size(): number;
|
|
44
|
+
get isEmpty(): boolean;
|
|
45
|
+
clone(): SafeBitmap;
|
|
46
|
+
/** In-place union: `this = this ∪ other`. */
|
|
47
|
+
orInPlace(other: CodecBitmap): void;
|
|
48
|
+
/** In-place difference: `this = this \ other`. */
|
|
49
|
+
andNotInPlace(other: CodecBitmap): void;
|
|
50
|
+
/** In-place intersection: `this = this ∩ other`. */
|
|
51
|
+
andInPlace(other: CodecBitmap): void;
|
|
52
|
+
/** Ascending iterator over the set values. */
|
|
53
|
+
[Symbol.iterator](): IterableIterator<number>;
|
|
54
|
+
toArray(): number[];
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The roaring {@link CodecInterface} — the flagship codec, delegating to {@link SafeBitmap}'s statics. This is
|
|
58
|
+
* the default the `CloudRoaring` facade injects into the engine; `core/` itself never hard-references it once a
|
|
59
|
+
* caller supplies a codec. Moves to `@cloudbitmaps/roaring` when the package split lands.
|
|
60
|
+
*/
|
|
61
|
+
export declare const roaringCodec: CodecInterface;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The default real-time {@link Clock} — the production half of the determinism seam.
|
|
3
|
+
*
|
|
4
|
+
* It lives outside `core/`, so `Date.now()`, `setTimeout` and `setImmediate` are allowed here; that separation is
|
|
5
|
+
* the whole point of the seam, and it is lint-enforced on the other side.
|
|
6
|
+
*
|
|
7
|
+
* It sits in its own module rather than in `index.ts` because two places need it — the facade, and
|
|
8
|
+
* `codec-bound.ts`, which pre-binds it into `bulkLoadCrbmGeneration`. Importing it from `index.ts` would put a
|
|
9
|
+
* cycle between the barrel and a module the barrel re-exports.
|
|
10
|
+
*/
|
|
11
|
+
import type { Clock } from '@cloudbitmaps/core';
|
|
12
|
+
/** Default real-time clock — lives outside `core/`, so `Date.now()` + timers are allowed here. */
|
|
13
|
+
export declare class SystemClock implements Clock {
|
|
14
|
+
now(): number;
|
|
15
|
+
sleep(ms: number): Promise<void>;
|
|
16
|
+
/**
|
|
17
|
+
* `setImmediate`, and it has to be — this is the one place where the choice of primitive is the whole feature.
|
|
18
|
+
*
|
|
19
|
+
* A long CPU-bound loop in `core/` (bulk-load is the one that matters) periodically hands the loop back so a
|
|
20
|
+
* co-resident HTTP server keeps answering. Measured on the **synthetic yield-primitive benchmark** — a
|
|
21
|
+
* 61,035-iteration loop yielding every 1,024, isolating the primitive rather than timing a real load (the
|
|
22
|
+
* end-to-end figures are 450 ms → 19 ms; see `cooperative.ts`, which lists all three experiments):
|
|
23
|
+
*
|
|
24
|
+
* ```text
|
|
25
|
+
* no yield 568 ms wall 568.0 ms worst event-loop gap
|
|
26
|
+
* sleep(0) 555 ms wall 555.2 ms ← a microtask. Yields NOTHING.
|
|
27
|
+
* setImmediate 569 ms wall 13.8 ms ← free, and 41x less starvation
|
|
28
|
+
* setTimeout(1) 625 ms wall 15.0 ms ← +10% wall for no extra relief
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* `sleep(0)` resolving on a microtask is correct for its own contract and useless for this: microtasks drain
|
|
32
|
+
* before the loop ever advances a phase, so `await sleep(0)` in a tight loop never lets a single pending I/O
|
|
33
|
+
* callback run. `setImmediate` fires in the check phase *after* pending I/O, which is exactly the ordering a
|
|
34
|
+
* server wants — inbound requests are serviced before the loop resumes chewing.
|
|
35
|
+
*
|
|
36
|
+
* A timer is not ref'd here (unlike {@link sleep}): a yield expresses "someone else may go first", never
|
|
37
|
+
* "there is outstanding work" — and `setImmediate` cannot outlive the turn that scheduled it anyway.
|
|
38
|
+
*/
|
|
39
|
+
yieldNow(): Promise<void>;
|
|
40
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { SafeBitmap } from '../roaring-codec.js';
|
|
2
|
+
import type { StorageChunkSource, IRegistryDriver } from '@cloudbitmaps/core';
|
|
3
|
+
/**
|
|
4
|
+
* The segment every conformance seeder must write. Exported so a driver's own test file derives the name
|
|
5
|
+
* instead of restating it — a second copy that drifts makes the suite query a segment nobody seeded, which
|
|
6
|
+
* fails as an empty result rather than as the mismatch it actually is.
|
|
7
|
+
*/
|
|
8
|
+
export declare const CONFORMANCE_SEGMENT = "s:v1";
|
|
9
|
+
/**
|
|
10
|
+
* Contract tests for a {@link StorageChunkSource}. `makeSource` MUST build a fresh source seeded with the
|
|
11
|
+
* given chunks (each an immutable Storage bitmap) and nothing else.
|
|
12
|
+
*/
|
|
13
|
+
export declare function storageChunkSourceConformance(label: string, makeSource: (chunks: Array<{
|
|
14
|
+
chunkKey: number;
|
|
15
|
+
bitmap: SafeBitmap;
|
|
16
|
+
}>) => Promise<StorageChunkSource>): void;
|
|
17
|
+
/**
|
|
18
|
+
* Contract tests for an {@link IRegistryDriver}. `makeDriver` MUST return a fresh, empty, isolated driver on
|
|
19
|
+
* each call. The OCC contract (create / token-fenced CAS / ABA) plus the registry's record semantics (forward
|
|
20
|
+
* currentGen, status, clearable keyId, discovery).
|
|
21
|
+
*/
|
|
22
|
+
export declare function registryConformance(label: string, makeDriver: () => IRegistryDriver): void;
|
|
23
|
+
/**
|
|
24
|
+
* Contract tests for an {@link IRegistryDriver} under **concurrent writers** — `makeDrivers` MUST return two
|
|
25
|
+
* independent driver instances over the **same** backing store, as two processes would see it.
|
|
26
|
+
*
|
|
27
|
+
* This exists because {@link registryConformance} drives one driver sequentially, and `ObjectStoreRegistry`
|
|
28
|
+
* checks the OCC token in memory before it ever issues a conditional write. That in-process check answers
|
|
29
|
+
* every sequential case, so the store-level precondition — the only thing that fences writers *across*
|
|
30
|
+
* processes, and the entire reason these drivers are built on conditional writes — is never load-bearing
|
|
31
|
+
* there. A backend that accepted `If-Match` and ignored it passed the full suite; so did a GCS driver whose
|
|
32
|
+
* writes were not fenced at all. These cases are what make the fence testable.
|
|
33
|
+
*/
|
|
34
|
+
export declare function registryConcurrency(label: string, makeDrivers: () => readonly [IRegistryDriver, IRegistryDriver]): void;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal surface for the coverage-guided fuzz harness, CODEC half.
|
|
3
|
+
*
|
|
4
|
+
* Only the concrete bitmap codec lives here — the core half (`CrbmReader`, the non-public `parseIndex`,
|
|
5
|
+
* `BufferReader`, `CloudRoaringError`, `DEFAULT_MAX_PAYLOAD_BYTES`) is built from `@cloudbitmaps/core`'s own
|
|
6
|
+
* `src/testing/fuzz-core.ts` to `fuzz/build/fuzz-core.js`. Splitting it this way keeps the published entries
|
|
7
|
+
* of both packages free of test-only exports: neither widened its public API to feed the fuzzer.
|
|
8
|
+
*/
|
|
9
|
+
export { SafeBitmap } from '../roaring-codec.js';
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cloudbitmaps/roaring",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Distributed, cloud-native Roaring Bitmaps — query and intersect billion-scale integer sets
|
|
3
|
+
"version": "0.10.0",
|
|
4
|
+
"description": "Distributed, cloud-native Roaring Bitmaps — query and intersect billion-scale integer sets straight out of object storage, from a bounded RAM cache over immutable objects in your own bucket, at a fraction of an always-on cache.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"roaring",
|
|
7
7
|
"bitmap",
|
|
@@ -16,17 +16,10 @@
|
|
|
16
16
|
"set",
|
|
17
17
|
"set-intersection",
|
|
18
18
|
"s3",
|
|
19
|
-
"dynamodb",
|
|
20
19
|
"gcs",
|
|
21
20
|
"azure",
|
|
22
|
-
"postgres",
|
|
23
|
-
"redis",
|
|
24
|
-
"mongodb",
|
|
25
|
-
"cassandra",
|
|
26
|
-
"mysql",
|
|
27
|
-
"mariadb",
|
|
28
21
|
"serverless",
|
|
29
|
-
"
|
|
22
|
+
"object-storage",
|
|
30
23
|
"scale-to-zero",
|
|
31
24
|
"segmentation",
|
|
32
25
|
"audience",
|
|
@@ -38,7 +31,8 @@
|
|
|
38
31
|
"author": "Sharvil Kadam",
|
|
39
32
|
"repository": {
|
|
40
33
|
"type": "git",
|
|
41
|
-
"url": "git+https://github.com/cloudbitmaps/cloudbitmaps.git"
|
|
34
|
+
"url": "git+https://github.com/cloudbitmaps/cloudbitmaps.git",
|
|
35
|
+
"directory": "packages/roaring"
|
|
42
36
|
},
|
|
43
37
|
"homepage": "https://github.com/cloudbitmaps/cloudbitmaps#readme",
|
|
44
38
|
"bugs": {
|
|
@@ -46,62 +40,15 @@
|
|
|
46
40
|
},
|
|
47
41
|
"type": "module",
|
|
48
42
|
"engines": {
|
|
49
|
-
"node": ">=22"
|
|
43
|
+
"node": ">=22.12"
|
|
50
44
|
},
|
|
51
45
|
"sideEffects": false,
|
|
52
|
-
"main": "./dist/index.
|
|
53
|
-
"module": "./dist/index.js",
|
|
46
|
+
"main": "./dist/index.js",
|
|
54
47
|
"types": "./dist/index.d.ts",
|
|
55
48
|
"exports": {
|
|
56
49
|
".": {
|
|
57
50
|
"types": "./dist/index.d.ts",
|
|
58
|
-
"
|
|
59
|
-
"require": "./dist/index.cjs"
|
|
60
|
-
},
|
|
61
|
-
"./s3": {
|
|
62
|
-
"types": "./dist/s3/index.d.ts",
|
|
63
|
-
"import": "./dist/s3/index.js",
|
|
64
|
-
"require": "./dist/s3/index.cjs"
|
|
65
|
-
},
|
|
66
|
-
"./dynamodb": {
|
|
67
|
-
"types": "./dist/dynamodb/index.d.ts",
|
|
68
|
-
"import": "./dist/dynamodb/index.js",
|
|
69
|
-
"require": "./dist/dynamodb/index.cjs"
|
|
70
|
-
},
|
|
71
|
-
"./gcs": {
|
|
72
|
-
"types": "./dist/gcs/index.d.ts",
|
|
73
|
-
"import": "./dist/gcs/index.js",
|
|
74
|
-
"require": "./dist/gcs/index.cjs"
|
|
75
|
-
},
|
|
76
|
-
"./azure": {
|
|
77
|
-
"types": "./dist/azure/index.d.ts",
|
|
78
|
-
"import": "./dist/azure/index.js",
|
|
79
|
-
"require": "./dist/azure/index.cjs"
|
|
80
|
-
},
|
|
81
|
-
"./postgres": {
|
|
82
|
-
"types": "./dist/postgres/index.d.ts",
|
|
83
|
-
"import": "./dist/postgres/index.js",
|
|
84
|
-
"require": "./dist/postgres/index.cjs"
|
|
85
|
-
},
|
|
86
|
-
"./redis": {
|
|
87
|
-
"types": "./dist/redis/index.d.ts",
|
|
88
|
-
"import": "./dist/redis/index.js",
|
|
89
|
-
"require": "./dist/redis/index.cjs"
|
|
90
|
-
},
|
|
91
|
-
"./mongodb": {
|
|
92
|
-
"types": "./dist/mongodb/index.d.ts",
|
|
93
|
-
"import": "./dist/mongodb/index.js",
|
|
94
|
-
"require": "./dist/mongodb/index.cjs"
|
|
95
|
-
},
|
|
96
|
-
"./cassandra": {
|
|
97
|
-
"types": "./dist/cassandra/index.d.ts",
|
|
98
|
-
"import": "./dist/cassandra/index.js",
|
|
99
|
-
"require": "./dist/cassandra/index.cjs"
|
|
100
|
-
},
|
|
101
|
-
"./mysql": {
|
|
102
|
-
"types": "./dist/mysql/index.d.ts",
|
|
103
|
-
"import": "./dist/mysql/index.js",
|
|
104
|
-
"require": "./dist/mysql/index.cjs"
|
|
51
|
+
"default": "./dist/index.js"
|
|
105
52
|
}
|
|
106
53
|
},
|
|
107
54
|
"files": [
|
|
@@ -115,69 +62,17 @@
|
|
|
115
62
|
"access": "public"
|
|
116
63
|
},
|
|
117
64
|
"bin": {
|
|
118
|
-
"compact-segments": "./dist/bin/compact-segments.js",
|
|
119
65
|
"export-segments": "./dist/bin/export-segments.js"
|
|
120
66
|
},
|
|
121
67
|
"dependencies": {
|
|
122
|
-
"roaring": "
|
|
123
|
-
"@cloudbitmaps/core": "0.
|
|
124
|
-
},
|
|
125
|
-
"peerDependencies": {
|
|
126
|
-
"@aws-sdk/client-dynamodb": "^3",
|
|
127
|
-
"@aws-sdk/client-s3": "^3",
|
|
128
|
-
"@azure/storage-blob": "^12",
|
|
129
|
-
"@google-cloud/storage": "^7",
|
|
130
|
-
"cassandra-driver": "^4",
|
|
131
|
-
"ioredis": "^5",
|
|
132
|
-
"mongodb": "^6",
|
|
133
|
-
"mysql2": "^3",
|
|
134
|
-
"pg": "^8"
|
|
135
|
-
},
|
|
136
|
-
"peerDependenciesMeta": {
|
|
137
|
-
"@aws-sdk/client-dynamodb": {
|
|
138
|
-
"optional": true
|
|
139
|
-
},
|
|
140
|
-
"@aws-sdk/client-s3": {
|
|
141
|
-
"optional": true
|
|
142
|
-
},
|
|
143
|
-
"@azure/storage-blob": {
|
|
144
|
-
"optional": true
|
|
145
|
-
},
|
|
146
|
-
"@google-cloud/storage": {
|
|
147
|
-
"optional": true
|
|
148
|
-
},
|
|
149
|
-
"cassandra-driver": {
|
|
150
|
-
"optional": true
|
|
151
|
-
},
|
|
152
|
-
"ioredis": {
|
|
153
|
-
"optional": true
|
|
154
|
-
},
|
|
155
|
-
"mongodb": {
|
|
156
|
-
"optional": true
|
|
157
|
-
},
|
|
158
|
-
"mysql2": {
|
|
159
|
-
"optional": true
|
|
160
|
-
},
|
|
161
|
-
"pg": {
|
|
162
|
-
"optional": true
|
|
163
|
-
}
|
|
68
|
+
"roaring": "2.7.0",
|
|
69
|
+
"@cloudbitmaps/core": "^0.10.0"
|
|
164
70
|
},
|
|
165
71
|
"devDependencies": {
|
|
166
|
-
"@aws-sdk/client-dynamodb": "^3.1075.0",
|
|
167
|
-
"@aws-sdk/client-s3": "^3.1075.0",
|
|
168
|
-
"@azure/storage-blob": "^12",
|
|
169
|
-
"@google-cloud/storage": "^7.0.0",
|
|
170
|
-
"cassandra-driver": "^4",
|
|
171
|
-
"ioredis": "^5",
|
|
172
|
-
"mongodb": "^6",
|
|
173
|
-
"mysql2": "^3.23.1",
|
|
174
|
-
"pg": "^8",
|
|
175
|
-
"@types/pg": "^8",
|
|
176
72
|
"@types/node": "^22.0.0",
|
|
177
|
-
"tsup": "^8.0.0",
|
|
178
73
|
"typescript": "^5.5.0"
|
|
179
74
|
},
|
|
180
75
|
"scripts": {
|
|
181
|
-
"build": "
|
|
76
|
+
"build": "node ../../scripts/build.mjs"
|
|
182
77
|
}
|
|
183
78
|
}
|
package/dist/azure/index.cjs
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var azure = require('@cloudbitmaps/core/azure');
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
Object.keys(azure).forEach(function (k) {
|
|
8
|
-
if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
|
|
9
|
-
enumerable: true,
|
|
10
|
-
get: function () { return azure[k]; }
|
|
11
|
-
});
|
|
12
|
-
});
|
|
13
|
-
//# sourceMappingURL=index.cjs.map
|
|
14
|
-
//# sourceMappingURL=index.cjs.map
|
package/dist/azure/index.cjs.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":[],"names":[],"mappings":"","file":"index.cjs","sourcesContent":[]}
|
package/dist/azure/index.d.cts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export * from '@cloudbitmaps/core/azure';
|
|
2
|
-
import '@cloudbitmaps/core/cassandra';
|
|
3
|
-
import '@cloudbitmaps/core/dynamodb';
|
|
4
|
-
import '@cloudbitmaps/core/gcs';
|
|
5
|
-
import '@cloudbitmaps/core/mongodb';
|
|
6
|
-
import '@cloudbitmaps/core/mysql';
|
|
7
|
-
import '@cloudbitmaps/core/postgres';
|
|
8
|
-
import '@cloudbitmaps/core/redis';
|
|
9
|
-
import '@cloudbitmaps/core/s3';
|
package/dist/azure/index.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
export * from '@cloudbitmaps/core/azure';
|
|
2
|
-
import '@cloudbitmaps/core/cassandra';
|
|
3
|
-
import '@cloudbitmaps/core/dynamodb';
|
|
4
|
-
import '@cloudbitmaps/core/gcs';
|
|
5
|
-
import '@cloudbitmaps/core/mongodb';
|
|
6
|
-
import '@cloudbitmaps/core/mysql';
|
|
7
|
-
import '@cloudbitmaps/core/postgres';
|
|
8
|
-
import '@cloudbitmaps/core/redis';
|
|
9
|
-
import '@cloudbitmaps/core/s3';
|
package/dist/azure/index.js
DELETED
package/dist/azure/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":[],"names":[],"mappings":"","file":"index.js","sourcesContent":[]}
|
|
@@ -1,201 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { SystemClock, src_exports, runCompactionCycle } from '../chunk-D6LIJSGF.js';
|
|
3
|
-
import { hostname } from 'os';
|
|
4
|
-
import { join } from 'path';
|
|
5
|
-
import { pathToFileURL } from 'url';
|
|
6
|
-
|
|
7
|
-
function parseConfig(env) {
|
|
8
|
-
const root = env.CR_COMPACT_ROOT;
|
|
9
|
-
if (root === void 0 || root === "") {
|
|
10
|
-
throw new Error("CR_COMPACT_ROOT is required (the local-filesystem storage root)");
|
|
11
|
-
}
|
|
12
|
-
const mode = env.CR_COMPACT_MODE ?? "once";
|
|
13
|
-
if (mode !== "once" && mode !== "loop") {
|
|
14
|
-
throw new Error(`CR_COMPACT_MODE must be "once" or "loop"; got ${mode}`);
|
|
15
|
-
}
|
|
16
|
-
const num = (name, def) => {
|
|
17
|
-
const raw = env[name];
|
|
18
|
-
if (raw === void 0 || raw === "") return def;
|
|
19
|
-
const v = Number(raw);
|
|
20
|
-
if (!Number.isFinite(v) || v < 0)
|
|
21
|
-
throw new Error(`${name} must be a non-negative number; got ${raw}`);
|
|
22
|
-
return v;
|
|
23
|
-
};
|
|
24
|
-
const intOpt = (name, min = 0) => {
|
|
25
|
-
const raw = env[name];
|
|
26
|
-
if (raw === void 0 || raw === "") return void 0;
|
|
27
|
-
const v = Number(raw);
|
|
28
|
-
if (!Number.isInteger(v) || v < min)
|
|
29
|
-
throw new Error(`${name} must be an integer >= ${min}; got ${raw}`);
|
|
30
|
-
return v;
|
|
31
|
-
};
|
|
32
|
-
const shard = intOpt("CR_COMPACT_SHARD");
|
|
33
|
-
const totalShards = intOpt("CR_COMPACT_TOTAL_SHARDS");
|
|
34
|
-
if (shard === void 0 !== (totalShards === void 0)) {
|
|
35
|
-
throw new Error("set both CR_COMPACT_SHARD and CR_COMPACT_TOTAL_SHARDS, or neither");
|
|
36
|
-
}
|
|
37
|
-
return {
|
|
38
|
-
root,
|
|
39
|
-
mode,
|
|
40
|
-
intervalMs: num("CR_COMPACT_INTERVAL_MS", 3e4),
|
|
41
|
-
namespace: env.CR_COMPACT_NAMESPACE,
|
|
42
|
-
threshold: num("CR_COMPACT_THRESHOLD", 1),
|
|
43
|
-
keep: num("CR_COMPACT_KEEP", 1),
|
|
44
|
-
owner: env.CR_COMPACT_OWNER ?? `${hostname()}:${process.pid}`,
|
|
45
|
-
shard,
|
|
46
|
-
totalShards,
|
|
47
|
-
maxSegments: intOpt("CR_COMPACT_MAX_SEGMENTS", 1),
|
|
48
|
-
retire: env.CR_RETIRE === "1" ? parseRetireConfig(env, num, intOpt) : void 0
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
function parseRetireConfig(env, num, intOpt) {
|
|
52
|
-
return {
|
|
53
|
-
// `intOpt(name, 1)`, not `num`: `num` accepts 0 and 1.5, and `retireExpired` then throws a ValidationError
|
|
54
|
-
// from *inside* the cycle — swallowed into a generic `{"event":"error"}` line every 30 s in loop mode, so
|
|
55
|
-
// compaction keeps reporting healthy cycles while retention silently never runs. `parseConfig` promises to
|
|
56
|
-
// throw a clear error on misconfiguration; this is what keeps that promise. (CR_COMPACT_MAX_SEGMENTS, the
|
|
57
|
-
// identical concept, already used `intOpt`.)
|
|
58
|
-
limit: intOpt("CR_RETIRE_LIMIT", 1) ?? src_exports.DEFAULT_RETIRE_LIMIT,
|
|
59
|
-
dryRun: env.CR_RETIRE_DRY_RUN === "1",
|
|
60
|
-
intervalMs: num("CR_RETIRE_INTERVAL_MS", src_exports.DEFAULT_TOMBSTONE_GRACE_MS),
|
|
61
|
-
purgeTombstones: env.CR_RETIRE_KEEP_TOMBSTONES !== "1",
|
|
62
|
-
tombstoneGraceMs: num("CR_RETIRE_TOMBSTONE_GRACE_MS", src_exports.DEFAULT_TOMBSTONE_GRACE_MS)
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
function localFsDeps(root, metrics) {
|
|
66
|
-
return {
|
|
67
|
-
cold: new src_exports.LocalFsColdDriver(join(root, "cold")),
|
|
68
|
-
warm: new src_exports.LocalFsWarmDriver(join(root, "warm")),
|
|
69
|
-
registry: new src_exports.LocalFsRegistryDriver(join(root, "registry")),
|
|
70
|
-
// A real clock, not `{ now }`: outside `core/` ambient time is allowed, and a full clock is what lets
|
|
71
|
-
// the generation write hand the event loop back instead of blocking the daemon for its whole duration.
|
|
72
|
-
clock: new SystemClock(),
|
|
73
|
-
...metrics ? { metrics } : {}
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
async function runOnce(deps, config) {
|
|
77
|
-
return runCompactionCycle(deps, {
|
|
78
|
-
owner: config.owner,
|
|
79
|
-
namespace: config.namespace,
|
|
80
|
-
threshold: config.threshold,
|
|
81
|
-
keep: config.keep,
|
|
82
|
-
shard: config.shard,
|
|
83
|
-
totalShards: config.totalShards,
|
|
84
|
-
maxSegments: config.maxSegments
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
async function retireOnce(deps, config, lastSweptAt) {
|
|
88
|
-
const retire = config.retire;
|
|
89
|
-
if (retire === void 0) return void 0;
|
|
90
|
-
if (lastSweptAt !== void 0 && deps.clock.now() - lastSweptAt < retire.intervalMs) {
|
|
91
|
-
return void 0;
|
|
92
|
-
}
|
|
93
|
-
return (0, src_exports.retireExpired)(
|
|
94
|
-
{ registry: deps.registry, warm: deps.warm, cold: deps.cold },
|
|
95
|
-
{
|
|
96
|
-
namespace: config.namespace,
|
|
97
|
-
now: deps.clock.now(),
|
|
98
|
-
limit: retire.limit,
|
|
99
|
-
dryRun: retire.dryRun,
|
|
100
|
-
purgeTombstones: retire.purgeTombstones,
|
|
101
|
-
tombstoneGraceMs: retire.tombstoneGraceMs
|
|
102
|
-
}
|
|
103
|
-
);
|
|
104
|
-
}
|
|
105
|
-
var LOG_SKIPPED_SAMPLE = 20;
|
|
106
|
-
function logRetire(result) {
|
|
107
|
-
const skipped = result.entries.filter((e) => e.action === "skipped");
|
|
108
|
-
const byReason = {};
|
|
109
|
-
for (const e of skipped) {
|
|
110
|
-
const key = e.reason.startsWith("failed:") ? "failed" : e.reason;
|
|
111
|
-
byReason[key] = (byReason[key] ?? 0) + 1;
|
|
112
|
-
}
|
|
113
|
-
const notReclaimed = result.entries.filter(
|
|
114
|
-
(e) => e.action === "retired" && e.result.generationsRemaining.length > 0
|
|
115
|
-
);
|
|
116
|
-
log({
|
|
117
|
-
event: "retire",
|
|
118
|
-
dryRun: result.dryRun,
|
|
119
|
-
scanned: result.scanned,
|
|
120
|
-
eligible: result.eligible,
|
|
121
|
-
retired: result.retired,
|
|
122
|
-
wouldRetire: result.wouldRetire,
|
|
123
|
-
tombstonesPurged: result.tombstonesPurged,
|
|
124
|
-
limited: result.limited,
|
|
125
|
-
skippedByReason: byReason,
|
|
126
|
-
skippedSample: skipped.slice(0, LOG_SKIPPED_SAMPLE).map((e) => ({ segment: e.segment, namespace: e.namespace, reason: e.reason })),
|
|
127
|
-
notFullyReclaimed: notReclaimed.slice(0, LOG_SKIPPED_SAMPLE).map((e) => e.segment),
|
|
128
|
-
notFullyReclaimedCount: notReclaimed.length
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
132
|
-
var log = (obj) => {
|
|
133
|
-
process.stdout.write(`${JSON.stringify(obj)}
|
|
134
|
-
`);
|
|
135
|
-
};
|
|
136
|
-
function stdoutMetricsSink() {
|
|
137
|
-
return {
|
|
138
|
-
onEvent: (event) => log({ event: "metric", ...event })
|
|
139
|
-
};
|
|
140
|
-
}
|
|
141
|
-
async function main(env = process.env) {
|
|
142
|
-
const config = parseConfig(env);
|
|
143
|
-
const deps = localFsDeps(
|
|
144
|
-
config.root,
|
|
145
|
-
env.CR_COMPACT_METRICS === "1" ? stdoutMetricsSink() : void 0
|
|
146
|
-
);
|
|
147
|
-
if (config.mode === "once") {
|
|
148
|
-
const cycle = await runOnce(deps, config);
|
|
149
|
-
log({
|
|
150
|
-
event: "cycle",
|
|
151
|
-
mode: "once",
|
|
152
|
-
candidates: cycle.candidates,
|
|
153
|
-
compacted: cycle.compacted,
|
|
154
|
-
deferred: cycle.deferred,
|
|
155
|
-
results: cycle.results
|
|
156
|
-
});
|
|
157
|
-
const swept = await retireOnce(deps, config);
|
|
158
|
-
if (swept !== void 0) logRetire(swept);
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
let stop = false;
|
|
162
|
-
const onSignal = () => {
|
|
163
|
-
stop = true;
|
|
164
|
-
};
|
|
165
|
-
process.on("SIGINT", onSignal);
|
|
166
|
-
process.on("SIGTERM", onSignal);
|
|
167
|
-
log({ event: "start", mode: "loop", intervalMs: config.intervalMs, owner: config.owner });
|
|
168
|
-
let lastSweptAt;
|
|
169
|
-
while (!stop) {
|
|
170
|
-
try {
|
|
171
|
-
const cycle = await runOnce(deps, config);
|
|
172
|
-
log({
|
|
173
|
-
event: "cycle",
|
|
174
|
-
candidates: cycle.candidates,
|
|
175
|
-
compacted: cycle.compacted,
|
|
176
|
-
deferred: cycle.deferred,
|
|
177
|
-
results: cycle.results
|
|
178
|
-
});
|
|
179
|
-
const swept = await retireOnce(deps, config, lastSweptAt);
|
|
180
|
-
if (swept !== void 0) {
|
|
181
|
-
lastSweptAt = deps.clock.now();
|
|
182
|
-
logRetire(swept);
|
|
183
|
-
}
|
|
184
|
-
} catch (err) {
|
|
185
|
-
log({ event: "error", message: err.message });
|
|
186
|
-
}
|
|
187
|
-
if (!stop) await sleep(config.intervalMs);
|
|
188
|
-
}
|
|
189
|
-
log({ event: "stopped" });
|
|
190
|
-
}
|
|
191
|
-
if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
192
|
-
main().catch((err) => {
|
|
193
|
-
process.stderr.write(`${err.message}
|
|
194
|
-
`);
|
|
195
|
-
process.exit(1);
|
|
196
|
-
});
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
export { localFsDeps, main, parseConfig, retireOnce, runOnce };
|
|
200
|
-
//# sourceMappingURL=compact-segments.js.map
|
|
201
|
-
//# sourceMappingURL=compact-segments.js.map
|