@cloudbitmaps/roaring 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +59 -34
- package/README.md +3 -1
- package/dist/bin/export-segments.js +105 -119
- package/dist/bin/export-segments.js.map +2 -2
- package/dist/index.d.ts +117 -80
- package/dist/index.js +103 -121
- package/dist/index.js.map +2 -2
- package/dist/open-charging-storage.d.ts +3 -2
- package/dist/testing/conformance.d.ts +15 -10
- package/package.json +2 -2
|
@@ -33,8 +33,6 @@ import {
|
|
|
33
33
|
excludingReservedRows,
|
|
34
34
|
dropSegment,
|
|
35
35
|
eraseIdFromSegment,
|
|
36
|
-
estimateCost,
|
|
37
|
-
groundedReport,
|
|
38
36
|
judgeLoad,
|
|
39
37
|
loadSegment,
|
|
40
38
|
loadSegmentChunks,
|
|
@@ -55,7 +53,7 @@ import {
|
|
|
55
53
|
splitId,
|
|
56
54
|
takeLease
|
|
57
55
|
} from "@cloudbitmaps/core";
|
|
58
|
-
import { validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
56
|
+
import { pointerIdOf, validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
|
|
59
57
|
|
|
60
58
|
// src/open-charging-storage.ts
|
|
61
59
|
var OpenChargingStorage = class {
|
|
@@ -88,8 +86,9 @@ var OpenChargingStorage = class {
|
|
|
88
86
|
if (!this.free.has(key.generation)) this.charge();
|
|
89
87
|
return this.inner.getTail(key, maxBytes);
|
|
90
88
|
}
|
|
91
|
-
delete
|
|
92
|
-
|
|
89
|
+
/** Handed through whole: the erasure's delete of a holder is conditioned on the object it searched. */
|
|
90
|
+
delete(key, options) {
|
|
91
|
+
return this.inner.delete(key, options);
|
|
93
92
|
}
|
|
94
93
|
list(ref) {
|
|
95
94
|
return this.inner.list(ref);
|
|
@@ -655,10 +654,7 @@ import {
|
|
|
655
654
|
BufferReader,
|
|
656
655
|
CountingMetricsSink,
|
|
657
656
|
RecordingAuditSink,
|
|
658
|
-
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2
|
|
659
|
-
AWS_US_EAST_1_ONDEMAND,
|
|
660
|
-
ELASTICACHE_REDIS_US_EAST_1_ONDEMAND,
|
|
661
|
-
ONE_REDIS_HA_CLUSTER
|
|
657
|
+
DEFAULT_RETRY_POLICY as DEFAULT_RETRY_POLICY2
|
|
662
658
|
} from "@cloudbitmaps/core";
|
|
663
659
|
var SystemRng = class {
|
|
664
660
|
/** A float in `[0, 1)`. Jitter only — never key material, never anything a caller can observe. */
|
|
@@ -668,6 +664,11 @@ var SystemRng = class {
|
|
|
668
664
|
};
|
|
669
665
|
var DEFAULT_CACHE_MAX_CHUNKS = 1024;
|
|
670
666
|
var DEFAULT_ADMIN_CONCURRENCY = 8;
|
|
667
|
+
function rowVersionOfLive(version) {
|
|
668
|
+
if (version === null) return null;
|
|
669
|
+
const cut = version.lastIndexOf("#");
|
|
670
|
+
return cut < 0 ? version : version.slice(0, cut);
|
|
671
|
+
}
|
|
671
672
|
function validateConcurrency(concurrency) {
|
|
672
673
|
if (concurrency !== void 0 && (!Number.isInteger(concurrency) || concurrency < 1)) {
|
|
673
674
|
throw new ValidationError2(`concurrency must be a positive integer; got ${concurrency}`);
|
|
@@ -775,23 +776,7 @@ var CALL_KEYS = {
|
|
|
775
776
|
"concurrency",
|
|
776
777
|
"summaries"
|
|
777
778
|
]),
|
|
778
|
-
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"])
|
|
779
|
-
costReport: keysOf()(["pricing", "workload"]),
|
|
780
|
-
workload: keysOf()([
|
|
781
|
-
"readsPerSec",
|
|
782
|
-
"intersectsPerSec",
|
|
783
|
-
"cacheHitRate",
|
|
784
|
-
"chunksPerIntersect",
|
|
785
|
-
"operandsPerIntersect",
|
|
786
|
-
"loadsPerMonth",
|
|
787
|
-
"requestsPerLoad",
|
|
788
|
-
"hotSegments",
|
|
789
|
-
"readerProcesses",
|
|
790
|
-
"genTtlMs",
|
|
791
|
-
"retirementsPerMonth",
|
|
792
|
-
"purgesPerMonth",
|
|
793
|
-
"conditionalDelete"
|
|
794
|
-
])
|
|
779
|
+
exportSegments: keysOf()(["format", "codec", "namespace", "ndjsonBatchBytes"])
|
|
795
780
|
};
|
|
796
781
|
var kindOf = (v) => v === null ? "null" : Array.isArray(v) ? "an array" : `a ${typeof v}`;
|
|
797
782
|
function optionsError(options, op, known, booleans = []) {
|
|
@@ -1143,22 +1128,24 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1143
1128
|
}
|
|
1144
1129
|
/**
|
|
1145
1130
|
* Get a handle to a segment. Validates the name (non-empty, well-formed, within the encoded-length cap). The options
|
|
1146
|
-
* are `{ namespace }` and nothing else:
|
|
1147
|
-
* misspelt `namespace` would otherwise address the default namespace
|
|
1148
|
-
*
|
|
1149
|
-
* `undefined` is read as absent, so a
|
|
1131
|
+
* are a plain object holding `{ namespace }` and nothing else: any other own key, enumerable or not, is refused with
|
|
1132
|
+
* {@link ValidationError}, by name, since a misspelt `namespace` would otherwise address the default namespace, and a
|
|
1133
|
+
* deadline such as `expiresAt` would be left unread. A class instance, or an object built on another, is refused too,
|
|
1134
|
+
* since it can carry an option no scan of its own keys sees. A key whose value is `undefined` is read as absent, so a
|
|
1135
|
+
* spread of options keeps working.
|
|
1150
1136
|
*/
|
|
1151
1137
|
segment(name, options) {
|
|
1152
1138
|
if (options !== void 0 && options !== null) {
|
|
1153
1139
|
if (typeof options !== "object" || Array.isArray(options)) {
|
|
1154
1140
|
throw new ValidationError2("segment: options must be an object such as { namespace }");
|
|
1155
1141
|
}
|
|
1156
|
-
|
|
1142
|
+
const proto = Object.getPrototypeOf(options);
|
|
1143
|
+
if (proto !== null && Object.getPrototypeOf(proto) !== null) {
|
|
1157
1144
|
throw new ValidationError2(
|
|
1158
|
-
"segment:
|
|
1145
|
+
"segment: options must be a plain object such as { namespace }, not a class instance or an object built on another"
|
|
1159
1146
|
);
|
|
1160
1147
|
}
|
|
1161
|
-
const unknown = Object.
|
|
1148
|
+
const unknown = Object.getOwnPropertyNames(options).filter(
|
|
1162
1149
|
(k) => k !== "namespace" && options[k] !== void 0
|
|
1163
1150
|
);
|
|
1164
1151
|
if (unknown.length > 0) {
|
|
@@ -1183,23 +1170,29 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1183
1170
|
/**
|
|
1184
1171
|
* Write `ids` as a **new generation of `dest`** and publish it — the shared body of the `*Into` verbs. An `*Into`
|
|
1185
1172
|
* materialisation is a load, and publishes the same way (hard invariant 1): every load that finds a row fences its
|
|
1186
|
-
* publish on the row's token
|
|
1187
|
-
*
|
|
1188
|
-
*
|
|
1189
|
-
* forward-only. The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
1173
|
+
* publish on the row's token, and every load that finds none on that absence, guarded or not; a guarded load (the
|
|
1174
|
+
* default, since the empty refusal needs the size of the current generation) also fences on the pointer it judged
|
|
1175
|
+
* (`expectFrom`). The destination's previous generation stays readable until the publish lands (readers re-resolve
|
|
1190
1176
|
* within `cache.genTtlMs`).
|
|
1191
1177
|
*
|
|
1192
1178
|
* This routes through `loadSegment` rather than writing the generation itself, and that is the whole point
|
|
1193
1179
|
* of it. A materialisation is a load whose ids happen to come from a combine instead of from upstream, so
|
|
1194
1180
|
* everything `load()` learned the hard way applies unchanged: the generation is written UNPUBLISHED, the
|
|
1195
1181
|
* guard runs while the old generation is still authoritative, the publish is fenced as above, and a refused
|
|
1196
|
-
* object is reclaimed only when a re-read finds the row unchanged (the same token
|
|
1197
|
-
*
|
|
1198
|
-
*
|
|
1199
|
-
*
|
|
1200
|
-
*
|
|
1201
|
-
* the
|
|
1202
|
-
*
|
|
1182
|
+
* object is reclaimed only when a re-read finds the row unchanged (the same token, or one that differs only in
|
|
1183
|
+
* its leases), `destroyed`, or gone, or finds key material on it when the load wrote cleartext; under any other
|
|
1184
|
+
* row it is kept (hard invariant 1: deleting it after a purge-and-recreate would put a live row over a missing
|
|
1185
|
+
* generation).
|
|
1186
|
+
*
|
|
1187
|
+
* Where the row is gone, or keyed over a cleartext write, the object is proved the load's own by its footer, and
|
|
1188
|
+
* the delete passes the version that read reported, so on a storage driver that reports `conditionalDelete` an
|
|
1189
|
+
* object put under the number since is kept. Where the row is unchanged or `destroyed`, the check narrows the
|
|
1190
|
+
* window rather than closing it: the row read and the delete are two round trips, and the delete is by number,
|
|
1191
|
+
* since the write reports no version of the object it made. The other deletes a load leads to are by number too: its
|
|
1192
|
+
* collection deletes what a listing, or the row's list of kept generations, names below the pointer, and a drop's
|
|
1193
|
+
* sweep and a tombstoned segment's collection delete what a listing names under a `destroyed` row. The failure the
|
|
1194
|
+
* checks leave when they refuse is an orphan object, which costs storage until something collects it —
|
|
1195
|
+
* deliberately the cheaper side of the trade.
|
|
1203
1196
|
*
|
|
1204
1197
|
* Written and published in one step, with no guard, an empty combine — a typo'd operand, an `exclude` that
|
|
1205
1198
|
* swallowed everything, an operand that had not loaded yet — would silently replace `dest` with an empty
|
|
@@ -1213,9 +1206,10 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1213
1206
|
* **A `WriteConflictError` does not by itself mean nothing was published**, and that is worth knowing
|
|
1214
1207
|
* before you write the retry. `'superseded'` covers five different causes — the write-once PUT collided,
|
|
1215
1208
|
* the pointer moved, a row appeared where the load found none, the row's token changed, the row was purged —
|
|
1216
|
-
* and only the first
|
|
1217
|
-
* that is not a supersession at all
|
|
1218
|
-
*
|
|
1209
|
+
* and only the first two are always the "somebody beat us" the name suggests. A row can appear, and a token can
|
|
1210
|
+
* change, on a write that is not a supersession at all: a `setRetention` on the destination creates its row when it
|
|
1211
|
+
* has none, and rewrites it when it has one. On top of that, the collection pass that runs AFTER a successful
|
|
1212
|
+
* publish can raise the same error. So: treat it as "re-read the destination and decide",
|
|
1219
1213
|
* never as "the write did not happen".
|
|
1220
1214
|
*
|
|
1221
1215
|
* **And by default it collects nothing**, unlike `load()`. See the `keep` default below.
|
|
@@ -1247,7 +1241,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1247
1241
|
if (reason === "superseded") {
|
|
1248
1242
|
const wrote = result.size > 0 ? `generation ${result.generation} was written and is not current` : `nothing was written \u2014 another writer took generation ${result.generation} first`;
|
|
1249
1243
|
throw new WriteConflictError(
|
|
1250
|
-
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result is not the destination's current generation: ${wrote}. The pointer may have moved, the row may have been rewritten (a retention policy does
|
|
1244
|
+
`${op}: the destination "${dest.segment}" changed while this materialisation was in flight, so its result is not the destination's current generation: ${wrote}. The pointer may have moved, the row may have been created or rewritten (a retention policy does either), dropped or purged. Re-read the destination and re-run.`
|
|
1251
1245
|
);
|
|
1252
1246
|
}
|
|
1253
1247
|
return {
|
|
@@ -1563,7 +1557,7 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1563
1557
|
}
|
|
1564
1558
|
}
|
|
1565
1559
|
const row = await this.withRetries(() => registry.get(ref));
|
|
1566
|
-
return row === null ? null : { generation: row.currentGen, token: row
|
|
1560
|
+
return row === null ? null : { generation: row.currentGen, token: pointerIdOf(row) };
|
|
1567
1561
|
}
|
|
1568
1562
|
/** Throw what a read of, or a publish to, `handle` must throw now: its lease error, if it has one. */
|
|
1569
1563
|
assertLease(handle) {
|
|
@@ -1608,8 +1602,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1608
1602
|
const ref = { segment: rec.segment, namespace: rec.namespace };
|
|
1609
1603
|
if (this.crbmSource !== void 0) {
|
|
1610
1604
|
const held = await this.crbmSource.currentVersion(ref);
|
|
1611
|
-
const listed = rec.currentGen === null ? null : `${rec.currentGen}:${
|
|
1612
|
-
if (held !== listed) this.engine.invalidate(ref);
|
|
1605
|
+
const listed = rec.currentGen === null ? null : `${rec.currentGen}:${pointerIdOf(rec)}`;
|
|
1606
|
+
if (rowVersionOfLive(held) !== listed) this.engine.invalidate(ref);
|
|
1613
1607
|
}
|
|
1614
1608
|
return await this.engine.has(ref, id) ? { segment: rec.segment, namespace: rec.namespace } : null;
|
|
1615
1609
|
}
|
|
@@ -1628,10 +1622,14 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1628
1622
|
* can still hold it, and so can one above the pointer after a {@link CloudRoaring.rollback}, which the rollback
|
|
1629
1623
|
* could make current again. Each holder is deleted — below the pointer by a `keep: 0` collection, above it one
|
|
1630
1624
|
* by one, re-proved against the row first — and the generations above the pointer that never held the id stay
|
|
1631
|
-
* as rollback targets
|
|
1632
|
-
*
|
|
1633
|
-
*
|
|
1634
|
-
*
|
|
1625
|
+
* as rollback targets, but for any object no read of the segment can open, which goes whatever the id. An entry
|
|
1626
|
+
* says `erased: true` only once the call has listed the bucket and read what is left: **no generation of the
|
|
1627
|
+
* segment holds the id**. The returned per-segment record is your **erasure ledger** —
|
|
1628
|
+
* persist it / route it to your audit sink as the proof of deletion (when you pass `audit`, an audit event is also
|
|
1629
|
+
* emitted per segment erased: `segment.rewrite` for a rewrite, `segment.collect` where only other generations held
|
|
1630
|
+
* the id and nothing was rewritten). A segment where the erasure deleted only objects no read of it can open (a first
|
|
1631
|
+
* load's sealed under a key no row holds) found the id in none, so the ledger does not list it; the audit sink gets
|
|
1632
|
+
* `segment.collect` for it with no `fromGeneration`, which is the only record of that deletion.
|
|
1635
1633
|
*
|
|
1636
1634
|
* Uses the backend's **own** two halves, so the membership check and the rewrite provably run
|
|
1637
1635
|
* over the same generation. Requires the store built with a **backend** (throws
|
|
@@ -1653,13 +1651,21 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1653
1651
|
* an error, read off the row: a moved pointer is a `'superseded'` entry, which a re-run settles, and a segment
|
|
1654
1652
|
* that a concurrent `dropSegment` tombstoned or a retention sweep purged is left out of the ledger. A tombstoned
|
|
1655
1653
|
* segment is still searched: a cleartext object left under it that holds the id is deleted, with everything under
|
|
1656
|
-
* the tombstone, and the entry reads `erased: true`.
|
|
1654
|
+
* the tombstone, and the entry reads `erased: true`. So is a segment whose row names no generation yet (a retention
|
|
1655
|
+
* policy set before its first load): a first load's object that holds the id is deleted, once the row's `pointerId`
|
|
1656
|
+
* is renewed so that the load, if still running, is refused at its publish, and the entry reads `erased: true`. An
|
|
1657
|
+
* encrypted first load's object there cannot be searched, so it is deleted whatever the id: on an encrypted store,
|
|
1658
|
+
* an erasure of any id refuses an encrypted first load in flight that has written its object onto a segment made
|
|
1659
|
+
* ahead of its data. Each such delete, and each of a holder above the pointer, names the object the erasure read:
|
|
1660
|
+
* on a storage driver that reports `conditionalDelete`, another erasure of the id that deletes it first, followed by
|
|
1661
|
+
* a load that takes the number freed and publishes, does not lose that load's generation to this erasure's delete,
|
|
1662
|
+
* which is refused, and the entry says what the row says; on one that does not, it can.
|
|
1657
1663
|
*
|
|
1658
1664
|
* **Read `note` on any `erased: false` entry — the two reasons mean different things.** `'superseded'` means
|
|
1659
|
-
* another writer (a load, another erasure, or a rollback) moved the pointer
|
|
1660
|
-
* erase the id. Re-run: it erases the id if it is still there, and lists
|
|
1661
|
-
* erasure of the same id already removed it. Do not read `'superseded'` as
|
|
1662
|
-
* read it as "not done by this call, and the re-run settles it".
|
|
1665
|
+
* another writer (a load, another erasure, or a rollback) moved the pointer, or replaced an object the call meant to
|
|
1666
|
+
* delete, mid-call, so **this call** did not erase the id. Re-run: it erases the id if it is still there, and lists
|
|
1667
|
+
* nothing for the segment if a racing erasure of the same id already removed it. Do not read `'superseded'` as
|
|
1668
|
+
* "the id is still present" — read it as "not done by this call, and the re-run settles it".
|
|
1663
1669
|
* `` `error: …` `` is a per-segment fault (caught so one segment can't discard the whole ledger) and it can land
|
|
1664
1670
|
* on either side of the publish: if the rewrite had not published, the id is still there and a re-run erases
|
|
1665
1671
|
* it; if the publish succeeded and only the **collection** of the old generation failed, the id is already
|
|
@@ -1854,8 +1860,9 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1854
1860
|
* and are then *above* `currentGen`, where collection never looks. They remain until loads pass them (the first
|
|
1855
1861
|
* load whose number one of them holds numbers above them all, and collection then keeps the newest `keep` of what is
|
|
1856
1862
|
* below its pointer), {@link CloudRoaring.dropSegment}
|
|
1857
|
-
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and
|
|
1858
|
-
*
|
|
1863
|
+
* deletes them, or {@link CloudRoaring.eraseSubject} does: all of those present when it rewrites, and, when the
|
|
1864
|
+
* current generation does not hold the id, only those holding it and any no read of the segment can open, whatever
|
|
1865
|
+
* the id. An operator who has just undone a bad load should not have the
|
|
1859
1866
|
* evidence collected out from under them, while a rollback target that still holds erased data would make the
|
|
1860
1867
|
* erasure undoable.
|
|
1861
1868
|
*
|
|
@@ -1924,10 +1931,12 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
1924
1931
|
* published, a retention change, a rollback or an erasure wrote the row, or the row was deleted.
|
|
1925
1932
|
*
|
|
1926
1933
|
* A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
|
|
1927
|
-
* collection deliberately never looks — but only while the segment's registry row is unchanged or
|
|
1928
|
-
*
|
|
1929
|
-
*
|
|
1930
|
-
*
|
|
1934
|
+
* collection deliberately never looks — but only while the segment's registry row is unchanged (or differs only
|
|
1935
|
+
* in its leases), `destroyed` or gone, or has key material when the load wrote cleartext, which has no place in an
|
|
1936
|
+
* encrypted segment's bucket. Under a row that is gone or keyed, a read of the object's footer must prove it the
|
|
1937
|
+
* load's own first. Once another write has otherwise changed the row, the generation number it holds may name
|
|
1938
|
+
* another incarnation's live object, so it leaves the orphan rather than risk deleting live data. No row's list
|
|
1939
|
+
* names it, so it takes no place in a window, and a listing deletes it once a later generation is current above it.
|
|
1931
1940
|
*
|
|
1932
1941
|
* **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
|
|
1933
1942
|
* that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
|
|
@@ -2140,8 +2149,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2140
2149
|
* Pass `purgeTombstones: false` to keep every tombstone, that row included. On a backend whose registry reports
|
|
2141
2150
|
* `conditionalDelete` (AWS S3, Azure Blob, the local filesystem and memory, by default; GCS when you set `conditionalDelete: true`)
|
|
2142
2151
|
* the purge removes the row from the bucket for good, by a delete the store applies only to the version it judged,
|
|
2143
|
-
* so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere
|
|
2144
|
-
*
|
|
2152
|
+
* so a full sweep reads what is live or inside its grace rather than every name a namespace ever held; elsewhere it
|
|
2153
|
+
* leaves a tombstone. Each retirement files a pointer in the due index
|
|
2145
2154
|
* under the day its tombstone's grace ends, so `scan: 'index'` purges as well as retires.
|
|
2146
2155
|
*
|
|
2147
2156
|
* Needs the store built with a **backend** (throws {@link UnsupportedError} otherwise),
|
|
@@ -2365,7 +2374,8 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2365
2374
|
* A segment whose pointer names a missing object is the torn restore this reports; a cold `count()` of it still
|
|
2366
2375
|
* answers the number its row records, while a read of the object throws, so the count alone never shows it. With
|
|
2367
2376
|
* `summaries: true` it also opens each segment's current object (one tail read each) and reports a segment whose
|
|
2368
|
-
* row's summary says another id count or metadata than the object holds
|
|
2377
|
+
* row's summary names another object by its fingerprint, or says another id count or metadata than the object holds,
|
|
2378
|
+
* as `summary-mismatch`, which is what a
|
|
2369
2379
|
* restore of the registry from another point than the bucket leaves a count to answer. A sealed summary is held
|
|
2370
2380
|
* against its object when this store has the key, and counted in `summariesUnchecked` when it does not.
|
|
2371
2381
|
*
|
|
@@ -2401,15 +2411,6 @@ var CloudRoaring = class _CloudRoaring {
|
|
|
2401
2411
|
const registry = this.requireRegistry("exportSegments");
|
|
2402
2412
|
return runExport(this, registry, sink, { ...opts, codec: opts.codec ?? roaringCodec });
|
|
2403
2413
|
}
|
|
2404
|
-
/**
|
|
2405
|
-
* Planning cost estimate — pure, no instance/data needed: sizing, sales, what-if. For a real, grounded report
|
|
2406
|
-
* from live segment sizes, use `store.segment(name).costReport()`. See {@link CostReport}.
|
|
2407
|
-
* @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
|
|
2408
|
-
* needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
|
|
2409
|
-
*/
|
|
2410
|
-
static estimateCost(input) {
|
|
2411
|
-
return estimateCost(input);
|
|
2412
|
-
}
|
|
2413
2414
|
};
|
|
2414
2415
|
var withBatches = (gen, batches) => Object.assign(gen, { batches });
|
|
2415
2416
|
var failing = (err) => {
|
|
@@ -2806,8 +2807,8 @@ var Segment = class _Segment {
|
|
|
2806
2807
|
* Cardinality of the generation this handle reads: **one registry read when cold, none when warm, and no read of the
|
|
2807
2808
|
* object**, with **zero payload reads**. The registry row records the id count of the generation it names, written
|
|
2808
2809
|
* by the write that made it current, and a count answers from that. A row with no summary it can use (one written
|
|
2809
|
-
*
|
|
2810
|
-
* `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
|
|
2810
|
+
* without it, as a registry of your own may, one that names another generation, a sealed one that does not open)
|
|
2811
|
+
* sends the count to the `.crbm` index, summed with a tail read of the object. A segment whose pointer names a missing object (a torn
|
|
2811
2812
|
* restore) still counts the row's number, while a read of the object throws; `checkConsistency` is what finds it.
|
|
2812
2813
|
*
|
|
2813
2814
|
* What this trusts: the row's summary, or the index's sum, and no payload is decoded to confirm it. The summary is
|
|
@@ -2826,17 +2827,30 @@ var Segment = class _Segment {
|
|
|
2826
2827
|
return this.timed("count", () => this.engine.count(this.ref));
|
|
2827
2828
|
}
|
|
2828
2829
|
/**
|
|
2829
|
-
* What the generation this handle reads is, from one resolution: its number, its id count
|
|
2830
|
-
*
|
|
2831
|
-
*
|
|
2832
|
-
*
|
|
2833
|
-
*
|
|
2834
|
-
*
|
|
2835
|
-
*
|
|
2836
|
-
*
|
|
2830
|
+
* What the generation this handle reads is, from one resolution: its number, its id count, the metadata it was
|
|
2831
|
+
* loaded with (absent when it has none), and `sizeBytes`, its object's bytes in storage. All four come from the
|
|
2832
|
+
* registry row when it has a summary this store can use, as `count()`'s do: the summary records the object's
|
|
2833
|
+
* fingerprint, its size with its checksum. That is **one registry read when cold, none when warm, and no read of the
|
|
2834
|
+
* object**. With no summary it can use, they come from the generation's object, opened once: its footer and index,
|
|
2835
|
+
* with no payload read, which is a tail read of the object more when cold (and a range read for an index longer than
|
|
2836
|
+
* the tail) and keeps the generation in the reader cache as a read does. On a pinned handle it answers for the
|
|
2837
|
+
* generation it pinned, from the pinned object. A segment with no generation answers
|
|
2838
|
+
* `{ generation: null, cardinality: 0, sizeBytes: null }`, and a store whose source cannot report a size answers
|
|
2839
|
+
* `sizeBytes: null`. A pointer that names a missing object (a torn restore) still answers from the row's summary, as
|
|
2840
|
+
* `count()` does, while a read of the object throws `NotFoundError`; with no summary to use, `stat()` throws it too.
|
|
2841
|
+
* `checkConsistency` is what finds such a row.
|
|
2842
|
+
*
|
|
2843
|
+
* Trust is as for `count()`: what the row's summary says is believed until the object is next opened. Every open of
|
|
2844
|
+
* the generation holds the object to the fingerprint the summary records, so another object under the number is
|
|
2845
|
+
* refused when it is opened, and holds its count against the summary's; a disagreement in count makes this process
|
|
2846
|
+
* stop using that summary, so `stat()` then answers from the object, whose index is checked for internal consistency
|
|
2847
|
+
* and whose count is the index's sum. A reader already open is not checked again until the row changes, it is
|
|
2848
|
+
* evicted or it is invalidated.
|
|
2849
|
+
*
|
|
2850
|
+
* `sizeBytes` is what `groundedReport` in `@cloudbitmaps/tools` prices storage from.
|
|
2837
2851
|
*
|
|
2838
2852
|
* ```ts
|
|
2839
|
-
* const { generation, cardinality, metadata } = await store.segment('active-30d').stat();
|
|
2853
|
+
* const { generation, cardinality, metadata, sizeBytes } = await store.segment('active-30d').stat();
|
|
2840
2854
|
* ```
|
|
2841
2855
|
*/
|
|
2842
2856
|
async stat() {
|
|
@@ -3126,35 +3140,6 @@ var Segment = class _Segment {
|
|
|
3126
3140
|
)
|
|
3127
3141
|
);
|
|
3128
3142
|
}
|
|
3129
|
-
/**
|
|
3130
|
-
* Grounded cost report for this segment: storage cost from its **real** `.crbm` size (exact, no payload
|
|
3131
|
-
* reads); request cost from the supplied `workload` rates. A segment with no Storage generation reports zero
|
|
3132
|
-
* storage. The pointer refresh is priced at the store's own `cache.genTtlMs`, or at none when the store never
|
|
3133
|
-
* refreshes, unless the workload sets `genTtlMs`. See {@link CostReport} — it always includes a verdict (incl.
|
|
3134
|
-
* the lose-zone).
|
|
3135
|
-
* @deprecated Moves to `@cloudbitmaps/tools`, a package of its own, with a `size` on `stat()` so a grounded report
|
|
3136
|
-
* needs nothing internal. It is a planning tool, and its price list is as old as the release that ships it.
|
|
3137
|
-
*/
|
|
3138
|
-
async costReport(options) {
|
|
3139
|
-
checkOptions(options, "costReport", CALL_KEYS.costReport);
|
|
3140
|
-
const shape = options?.workload;
|
|
3141
|
-
if (shape !== void 0 && shape !== null && (typeof shape !== "object" || Array.isArray(shape))) {
|
|
3142
|
-
throw new ValidationError2("costReport: workload must be an object");
|
|
3143
|
-
}
|
|
3144
|
-
this.assertLeases([this]);
|
|
3145
|
-
const canMeasure = this.engine.supportsStorageSize;
|
|
3146
|
-
const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
|
|
3147
|
-
const refreshMs = this.engine.pointerRefreshMs;
|
|
3148
|
-
const given = options?.workload === void 0 || options.workload === null ? void 0 : picked(options.workload, CALL_KEYS.workload);
|
|
3149
|
-
const workload = refreshMs === void 0 || given?.genTtlMs !== void 0 ? given : { ...given, genTtlMs: refreshMs };
|
|
3150
|
-
return groundedReport({
|
|
3151
|
-
storageBytes: size?.sizeBytes ?? 0,
|
|
3152
|
-
grounded: canMeasure,
|
|
3153
|
-
workload,
|
|
3154
|
-
pricing: options?.pricing,
|
|
3155
|
-
extraNotes: canMeasure ? void 0 : ["storage source has no sizeOf() \u2014 storage not measured, reported as $0."]
|
|
3156
|
-
});
|
|
3157
|
-
}
|
|
3158
3143
|
};
|
|
3159
3144
|
|
|
3160
3145
|
// src/bin/export-segments.ts
|
|
@@ -3277,7 +3262,8 @@ async function main(env = process.env, now = () => Date.now()) {
|
|
|
3277
3262
|
format: manifest.format,
|
|
3278
3263
|
totalSegments: manifest.totalSegments,
|
|
3279
3264
|
totalIds: manifest.totalIds,
|
|
3280
|
-
failed: manifest.failed.length
|
|
3265
|
+
failed: manifest.failed.length,
|
|
3266
|
+
skipped: manifest.skipped.length
|
|
3281
3267
|
});
|
|
3282
3268
|
return manifest;
|
|
3283
3269
|
}
|