@cloudbitmaps/core 0.2.0 → 0.3.0

package/dist/index.cjs

@@ -171,6 +171,18 @@ function checkBudget(budget, projectedRequests, op) {
     );
   }
 }
+async function collectWithinBudget(source, budget, op) {
+  const out = [];
+  for await (const item of source) {
+    out.push(item);
+    if (budget !== null && out.length > budget.maxRequests) {
+      throw new BudgetExceededError(
+        `${op} would fan out to more than ${budget.maxRequests} units, over the per-op budget \u2014 the scan was abandoned at that point rather than completed, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``
+      );
+    }
+  }
+  return out;
+}
 
 // src/core/concurrency.ts
 async function mapWithConcurrency(items, limit, fn) {
@@ -397,6 +409,7 @@ var ZeroRng = class {
     return 0;
   }
 };
+var DEFAULT_MAX_WARM_SCAN_BYTES = 64 * 1024 * 1024;
 var SegmentEngine = class {
   warm;
   cold;
@@ -412,6 +425,7 @@ var SegmentEngine = class {
   /** Consistency for READ-path Warm fetches (precomputed; OCC RMW passes strong explicitly, see `warmGet`). */
   readOpts;
   writeConcurrency;
+  maxWarmScanBytes;
   /** Resolved per-op budget (null = disabled); undefined deps ⇒ the generous default. See {@link checkBudget}. */
   budget;
   constructor(deps) {
@@ -426,6 +440,12 @@ var SegmentEngine = class {
     this.occBackoff = deps.occBackoff ?? DEFAULT_OCC_BACKOFF;
     this.readOpts = { consistent: (deps.warmReadConsistency ?? "strong") !== "eventual" };
     this.writeConcurrency = deps.writeConcurrency ?? 1;
+    this.maxWarmScanBytes = deps.maxWarmScanBytes ?? DEFAULT_MAX_WARM_SCAN_BYTES;
+    if (!Number.isFinite(this.maxWarmScanBytes) || this.maxWarmScanBytes < 1) {
+      throw new ValidationError(
+        `maxWarmScanBytes must be a finite number >= 1 (got ${String(deps.maxWarmScanBytes)})`
+      );
+    }
     this.budget = deps.budget === void 0 ? DEFAULT_BUDGET : deps.budget;
     this.metrics = safeMetrics(deps.metrics ?? NOOP_METRICS);
     this.metricsOn = this.metrics !== NOOP_METRICS;
@@ -457,7 +477,7 @@ var SegmentEngine = class {
     return cold ? cold.has(remainder) : false;
   }
   async count(seg) {
-    const warmRows = await this.collectWarm(seg);
+    const warmRows = await this.collectWarm(seg, "count", this.budget?.maxRequests ?? null);
     const cardinalities = this.cold.cardinalities ? await this.cold.cardinalities(seg) : null;
     if (!cardinalities) {
       const chunkKeys = await this.chunkKeys(seg, warmRows);
@@ -489,7 +509,7 @@ var SegmentEngine = class {
     return this.cold.sizeOf ? this.cold.sizeOf(seg) : Promise.resolve(null);
   }
   async *iterate(seg) {
-    const warmRows = await this.collectWarm(seg);
+    const warmRows = await this.collectWarm(seg, "iterate", this.budget?.maxRequests ?? null);
     const chunkKeys = await this.chunkKeys(seg, warmRows);
     checkBudget(this.budget, chunkKeys.length, "iterate");
     const gen = await this.currentGen(seg);
@@ -536,7 +556,7 @@ var SegmentEngine = class {
     const budget = resolvePerOpBudget(options?.budget, this.budget);
     const operands = await Promise.all(
       segs.map(async (seg) => {
-        const warmRows = await this.collectWarm(seg);
+        const warmRows = await this.collectWarm(seg, "intersect", null);
         const keys = await this.chunkKeys(seg, warmRows);
         const gen = await this.currentGen(seg);
         return { seg, warmRows, keys: new Set(keys), gen };
@@ -677,12 +697,22 @@ var SegmentEngine = class {
     }
     return row;
   }
-  async collectWarm(seg) {
+  async collectWarm(seg, op, rowCeiling) {
     const rows = /* @__PURE__ */ new Map();
     let bytes = 0;
     for await (const row of this.warm.listChunks(seg, this.readOpts)) {
       rows.set(row.chunkKey, row.bytes);
       bytes += row.bytes.length;
+      if (bytes > this.maxWarmScanBytes) {
+        throw new BudgetExceededError(
+          `${op} would hold more than ${this.maxWarmScanBytes} bytes of warm deltas resident for one segment \u2014 the scan was abandoned there rather than completed. Raise \`maxWarmScanBytes\`, narrow the segment, or compact it so fewer chunks carry warm deltas. (This ceiling is independent of \`budget\` and stays in force when \`budget: false\`.)`
+        );
+      }
+      if (rowCeiling !== null && rows.size > rowCeiling) {
+        throw new BudgetExceededError(
+          `${op} would read more than ${rowCeiling} warm chunk rows, over the per-op budget \u2014 the scan was abandoned at that point rather than completed, so no exact total is available. Raise \`budget.maxRequests\`, override it per-op, or set \`budget: false\``
+        );
+      }
     }
     if (this.metricsOn) {
       this.metrics.onEvent({
@@ -1193,22 +1223,44 @@ function validateChunkRef(ref) {
 }
 
 // src/core/crbm/crc32c.ts
-var TABLE = (() => {
+var TABLES = (() => {
   const POLY = 2197175160;
-  const table = new Uint32Array(256);
+  const t0 = new Uint32Array(256);
   for (let n = 0; n < 256; n++) {
     let c = n;
-    for (let k = 0; k < 8; k++) {
-      c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
+    for (let k = 0; k < 8; k++) c = c & 1 ? POLY ^ c >>> 1 : c >>> 1;
+    t0[n] = c >>> 0;
+  }
+  const tables = [t0];
+  for (let k = 1; k < 8; k++) {
+    const prev = tables[k - 1];
+    const next = new Uint32Array(256);
+    for (let n = 0; n < 256; n++) {
+      const v = prev[n];
+      next[n] = (v >>> 8 ^ t0[v & 255]) >>> 0;
     }
-    table[n] = c >>> 0;
+    tables.push(next);
   }
-  return table;
+  return tables;
 })();
+var T0 = TABLES[0];
+var T1 = TABLES[1];
+var T2 = TABLES[2];
+var T3 = TABLES[3];
+var T4 = TABLES[4];
+var T5 = TABLES[5];
+var T6 = TABLES[6];
+var T7 = TABLES[7];
 function crc32c(bytes, seed = 0) {
   let crc = (seed ^ 4294967295) >>> 0;
-  for (let i = 0; i < bytes.length; i++) {
-    crc = (TABLE[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
+  let i = 0;
+  const n8 = bytes.length - bytes.length % 8;
+  for (; i < n8; i += 8) {
+    crc ^= bytes[i] | bytes[i + 1] << 8 | bytes[i + 2] << 16 | bytes[i + 3] << 24;
+    crc = (T7[crc & 255] ^ T6[crc >>> 8 & 255] ^ T5[crc >>> 16 & 255] ^ T4[crc >>> 24 & 255] ^ T3[bytes[i + 4]] ^ T2[bytes[i + 5]] ^ T1[bytes[i + 6]] ^ T0[bytes[i + 7]]) >>> 0;
+  }
+  for (; i < bytes.length; i++) {
+    crc = (T0[(crc ^ bytes[i]) & 255] ^ crc >>> 8) >>> 0;
   }
   return (crc ^ 4294967295) >>> 0;
 }
@@ -1719,6 +1771,7 @@ function coldBlobReader(driver, key) {
   };
 }
 var DEFAULT_CURRENT_GEN_TTL_MS = 2e3;
+var BULK_FLUSH_IDS = 1 << 20;
 var DEFAULT_MAX_OPEN_SEGMENTS = 1024;
 var DEFAULT_MAX_OPEN_INDEX_BYTES = 64 * 1024 * 1024;
 var CrbmColdChunkSource = class {
@@ -1996,15 +2049,29 @@ async function bulkLoadCrbmGeneration(driver, key, ids, options = {}) {
   }
   const codec = requireCodec(options.codec, "bulkLoadCrbmGeneration");
   const byChunk = /* @__PURE__ */ new Map();
+  const pendingByChunk = /* @__PURE__ */ new Map();
+  let pending = 0;
+  const flushPending = () => {
+    for (const [chunkKey, rems] of pendingByChunk) {
+      if (rems.length === 0) continue;
+      const existingBitmap = byChunk.get(chunkKey);
+      if (existingBitmap === void 0) byChunk.set(chunkKey, codec.fromValues(rems));
+      else existingBitmap.addMany(rems);
+      rems.length = 0;
+    }
+    pending = 0;
+  };
   for await (const id of toAsyncIterable(ids)) {
     const { chunkKey, remainder } = splitId(id);
-    let bitmap = byChunk.get(chunkKey);
-    if (bitmap === void 0) {
-      bitmap = codec.empty();
-      byChunk.set(chunkKey, bitmap);
+    let bucket = pendingByChunk.get(chunkKey);
+    if (bucket === void 0) {
+      bucket = [];
+      pendingByChunk.set(chunkKey, bucket);
     }
-    bitmap.add(remainder);
+    bucket.push(remainder);
+    if (++pending >= BULK_FLUSH_IDS) flushPending();
   }
+  flushPending();
   let cardinality = 0;
   const chunks = [];
   for (const [chunkKey, bitmap] of byChunk) {
@@ -3788,6 +3855,7 @@ async function eraseWarm(warm, ref) {
 }
 
 // src/core/consistency.ts
+var DEFAULT_MAX_SCAN_SEGMENTS = 25e4;
 var DEFAULT_CHECK_CONCURRENCY = 8;
 async function listGenerations(cold, ref) {
   const present = /* @__PURE__ */ new Set();
@@ -3799,8 +3867,21 @@ async function runConsistencyCheck(deps, options = {}) {
   if (!Number.isInteger(concurrency) || concurrency < 1) {
     throw new ValidationError(`concurrency must be a positive integer; got ${concurrency}`);
   }
+  const maxScanSegments = options.maxScanSegments ?? DEFAULT_MAX_SCAN_SEGMENTS;
+  if (!Number.isFinite(maxScanSegments) || maxScanSegments < 1) {
+    throw new ValidationError(
+      `maxScanSegments must be a finite number >= 1; got ${String(options.maxScanSegments)}`
+    );
+  }
   const recs = [];
-  for await (const rec of deps.registry.list(options.namespace)) recs.push(rec);
+  for await (const rec of deps.registry.list(options.namespace)) {
+    recs.push(rec);
+    if (recs.length > maxScanSegments) {
+      throw new BudgetExceededError(
+        `checkConsistency would enumerate more than ${maxScanSegments} segments \u2014 the scan was abandoned there rather than completed. Narrow it with \`namespace\`, or raise \`maxScanSegments\` if the fleet really is that large and the memory is available.`
+      );
+    }
+  }
   const results = await mapWithConcurrency(recs, concurrency, async (rec) => {
     if (rec.status === "destroyed") return { kind: "ok" };
     const ref = { segment: rec.segment, namespace: rec.namespace };
@@ -4004,6 +4085,8 @@ exports.CrbmColdChunkSource = CrbmColdChunkSource;
 exports.CrbmReader = CrbmReader;
 exports.CrbmWriter = CrbmWriter;
 exports.DEFAULT_BUDGET = DEFAULT_BUDGET;
+exports.DEFAULT_MAX_SCAN_SEGMENTS = DEFAULT_MAX_SCAN_SEGMENTS;
+exports.DEFAULT_MAX_WARM_SCAN_BYTES = DEFAULT_MAX_WARM_SCAN_BYTES;
 exports.DEFAULT_OCC_BACKOFF = DEFAULT_OCC_BACKOFF;
 exports.DEFAULT_PRICING = DEFAULT_PRICING;
 exports.DEFAULT_RETRY_POLICY = DEFAULT_RETRY_POLICY;
@@ -4037,6 +4120,7 @@ exports.aadFor = aadFor;
 exports.bulkLoadCrbmGeneration = bulkLoadCrbmGeneration;
 exports.checkBudget = checkBudget;
 exports.chunkRefKey = chunkRefKey;
+exports.collectWithinBudget = collectWithinBudget;
 exports.compactSegment = compactSegment;
 exports.destroySegment = destroySegment;
 exports.eraseNamespace = eraseNamespace;