@cloudbitmaps/roaring 0.1.0-rc.0

package/dist/bin/compact-segments.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/bin/compact-segments.ts"],"names":[],"mappings":";;;;;;AAwDO,SAAS,YAAY,GAAA,EAAwD;AAClF,EAAA,MAAM,OAAO,GAAA,CAAI,eAAA;AACjB,EAAA,IAAI,IAAA,KAAS,MAAA,IAAa,IAAA,KAAS,EAAA,EAAI;AACrC,IAAA,MAAM,IAAI,MAAM,iEAAiE,CAAA;AAAA,EACnF;AACA,EAAA,MAAM,IAAA,GAAO,IAAI,eAAA,IAAmB,MAAA;AACpC,EAAA,IAAI,IAAA,KAAS,MAAA,IAAU,IAAA,KAAS,MAAA,EAAQ;AACtC,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,8CAAA,EAAiD,IAAI,CAAA,CAAE,CAAA;AAAA,EACzE;AACA,EAAA,MAAM,GAAA,GAAM,CAAC,IAAA,EAAc,GAAA,KAAwB;AACjD,IAAA,MAAM,GAAA,GAAM,IAAI,IAAI,CAAA;AACpB,IAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI,OAAO,GAAA;AAC5C,IAAA,MAAM,CAAA,GAAI,OAAO,GAAG,CAAA;AACpB,IAAA,IAAI,CAAC,MAAA,CAAO,QAAA,CAAS,CAAC,KAAK,CAAA,GAAI,CAAA;AAC7B,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,EAAG,IAAI,CAAA,oCAAA,EAAuC,GAAG,CAAA,CAAE,CAAA;AACrE,IAAA,OAAO,CAAA;AAAA,EACT,CAAA;AACA,EAAA,MAAM,MAAA,GAAS,CAAC,IAAA,EAAc,GAAA,GAAM,CAAA,KAA0B;AAC5D,IAAA,MAAM,GAAA,GAAM,IAAI,IAAI,CAAA;AACpB,IAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI,OAAO,MAAA;AAC5C,IAAA,MAAM,CAAA,GAAI,OAAO,GAAG,CAAA;AACpB,IAAA,IAAI,CAAC,MAAA,CAAO,SAAA,CAAU,CAAC,KAAK,CAAA,GAAI,GAAA;AAC9B,MAAA,MAAM,IAAI,MAAM,CAAA,EAAG,IAAI,0BAA0B,GAAG,CAAA,MAAA,EAAS,GAAG,CAAA,CAAE,CAAA;AACpE,IAAA,OAAO,CAAA;AAAA,EACT,CAAA;AACA,EAAA,MAAM,KAAA,GAAQ,OAAO,kBAAkB,CAAA;AACvC,EAAA,MAAM,WAAA,GAAc,OAAO,yBAAyB,CAAA;AACpD,EAAA,IAAK,KAAA,KAAU,MAAA,MAAgB,WAAA,KAAgB,MAAA,CAAA,EAAY;AACzD,IAAA,MAAM,IAAI,MAAM,mEAAmE,CAAA;AAAA,EACrF;AACA,EAAA,OAAO;AAAA,IACL,IAAA;AAAA,IACA,IAAA;AAAA,IACA,UAAA,EAAY,GAAA,CAAI,wBAAA,EAA0B,GAAM,CAAA;AAAA,IAChD,WAAW,GAAA,CAAI,oBAAA;AAAA,IACf,SAAA,EAAW,GAAA,CAAI,sBAAA,EAAwB,CAAC,CAAA;AAAA,IACxC,IAAA,EAAM,GAAA,CAAI,iBAAA,EAAmB,CAAC,CAAA;AAAA,IAC9B,KAAA,EAAO,IAAI,gBAAA,IAAoB,CAAA,EAAG,UAAU,CAAA,CAAA,EAAI,QAAQ,GAAG,CAAA,CAAA;AAAA,IAC3D,KAAA;AAAA,IACA,WAAA;AAAA,IACA,WAAA,EAAa,MAAA,CAAO,yBAAA,EAA2B,CAAC;AAAA,GAClD;AACF;AAGO,SAAS,WAAA,CAAY,MAAc,OAAA,EAAwC;AAChF,EAAA,OAAO;AAAA,IACL,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,IAAA,CAAK,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC9C,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,IAAA,CAAK,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IAC9C,UAAU,IAAI,WAAA,CAAA,qBAAA,CAAsB,IAAA,CAAK,IAAA,EAAM,UAAU,CAAC,CAAA;AAAA,IAC1D,OAAO,EAAE,GAAA,EAAK,MAAM,IAAA,CAAK,KAAI,EAAE;AAAA;AAAA,IAC/B,GAAI,OAAA,GAAU,EAAE,OAAA,KAAY;AAAC,GAC/B;AACF;AAGA,eAAsB,OAAA,CACpB,MACA,MAAA,EACgC;AAChC,EAAA,OAAO,mBAAmB,IAAA,EAAM;AAAA,IAC9B,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,MAAM,MAAA,CAAO,IAAA;AAAA,IACb,OAAO,MAAA,CAAO,KAAA;AAAA,IACd,aAAa,MAAA,CAAO,WAAA;AAAA,IACpB,aAAa,MAAA,CAAO;AAAA,GACrB,CAAA;AACH;AAEA,IAAM,KAAA,GAAQ,CAAC,EAAA,KAA8B,IAAI,OAAA,CAAQ,CAAC,CAAA,KAAM,UAAA,CAAW,CAAA,EAAG,EAAE,CAAC,CAAA;AACjF,IAAM,GAAA,GAAM,CAAC,GAAA,KAAuB;AAClC,EAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,SAAA,CAAU,GAAG,CAAC;AAAA,CAAI,CAAA;AACjD,CAAA;AAGA,SAAS,iBAAA,GAAkC;AACzC,EAAA,OAAO;AAAA,IACL,OAAA,EAAS,CAAC,KAAA,KAAU,GAAA,CAAI,EAAE,KAAA,EAAO,QAAA,EAAU,GAAG,KAAA,EAAO;AAAA,GACvD;AACF;AAGA,eAAsB,IAAA,CAAK,GAAA,GAA0C,OAAA,CAAQ,GAAA,EAAoB;AAC/F,EAAA,MAAM,MAAA,GAAS,YAAY,GAAG,CAAA;AAC9B,EAAA,MAAM,IAAA,GAAO,WAAA;AAAA,IACX,MAAA,CAAO,IAAA;AAAA,IACP,GAAA,CAAI,kBAAA,KAAuB,GAAA,GAAM,iBAAA,EAAkB,GAAI;AAAA,GACzD;AAEA,EAAA,IAAI,MAAA,CAAO,SAAS,MAAA,EAAQ;AAC1B,IAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,EAAM,MAAM,CAAA;AACxC,IAAA,GAAA,CAAI;AAAA,MACF,KAAA,EAAO,OAAA;AAAA,MACP,IAAA,EAAM,MAAA;AAAA,MACN,YAAY,KAAA,CAAM,UAAA;AAAA,MAClB,WAAW,KAAA,CAAM,SAAA;AAAA,MACjB,UAAU,KAAA,CAAM,QAAA;AAAA,MAChB,SAAS,KAAA,CAAM;AAAA,KAChB,CAAA;AACD,IAAA;AAAA,EACF;AAGA,EAAA,IAAI,IAAA,GAAO,KAAA;AACX,EAAA,MAAM,WAAW,MAAY;AAC3B,IAAA,IAAA,GAAO,IAAA;AAAA,EACT,CAAA;AACA,EAAA,OAAA,CAAQ,EAAA,CAAG,UAAU,QAAQ,CAAA;AAC7B,EAAA,OAAA,CAAQ,EAAA,CAAG,WAAW,QAAQ,CAAA;AAC9B,EAAA,GAAA,CAAI,EAAE,KAAA,EAAO,OAAA,EAAS,IAAA,EAAM,MAAA,EAAQ,UAAA,EAAY,MAAA,CAAO,UAAA,EAAY,KAAA,EAAO,MAAA,CAAO,KAAA,EAAO,CAAA;AACxF,EAAA,OAAO,CAAC,IAAA,EAAM;AACZ,IAAA,IAAI;AACF,MAAA,MAAM,KAAA,GAAQ,MAAM,OAAA,CAAQ,IAAA,EAAM,MAAM,CAAA;AACxC,MAAA,GAAA,CAAI;AAAA,QACF,KAAA,EAAO,OAAA;AAAA,QACP,YAAY,KAAA,CAAM,UAAA;AAAA,QAClB,WAAW,KAAA,CAAM,SAAA;AAAA,QACjB,UAAU,KAAA,CAAM,QAAA;AAAA,QAChB,SAAS,KAAA,CAAM;AAAA,OAChB,CAAA;AAAA,IACH,SAAS,GAAA,EAAK;AACZ,MAAA,GAAA,CAAI,EAAE,KAAA,EAAO,OAAA,EAAS,OAAA,EAAU,GAAA,CAAc,SAAS,CAAA;AAAA,IACzD;AACA,IAAA,IAAI,CAAC,IAAA,EAAM,MAAM,KAAA,CAAM,OAAO,UAAU,CAAA;AAAA,EAC1C;AACA,EAAA,GAAA,CAAI,EAAE,KAAA,EAAO,SAAA,EAAW,CAAA;AAC1B;AAIA,IAAI,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,KAAM,MAAA,IAAa,MAAA,CAAA,IAAA,CAAY,GAAA,KAAQ,aAAA,CAAc,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAC,EAAE,IAAA,EAAM;AAC5F,EAAA,IAAA,EAAK,CAAE,KAAA,CAAM,CAAC,GAAA,KAAiB;AAC7B,IAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAI,GAAA,CAAc,OAAO;AAAA,CAAI,CAAA;AAClD,IAAA,OAAA,CAAQ,KAAK,CAAC,CAAA;AAAA,EAChB,CAAC,CAAA;AACH","file":"compact-segments.js","sourcesContent":["#!/usr/bin/env node\n/**\n * `compact-segments` — the standalone compaction daemon CLI (Phase 4d).\n *\n * A thin wrapper over {@link runCompactionCycle}: discover dirty segments → 2-phase-commit compact → GC. It\n * ships the **local-filesystem** backend (zero-dependency, the dev/reference target) and two **deploy modes**:\n *\n *   - `once`  — run a single cycle and exit (the **Lambda / one-shot / cron** shape).\n *   - `loop`  — run a cycle every `intervalMs`, until SIGINT/SIGTERM (the **K8s Deployment / ECS service** shape).\n *\n * **Cloud backends:** rather than have the CLI guess your AWS config, production wires its own ~10-line\n * handler that constructs an `S3ColdDriver` + `DynamoDb{Warm,Registry}Driver` and calls `runCompactionCycle`\n * directly (same two modes — call once, or on a `setInterval`). The library exports everything needed; this\n * binary stays SDK-free (so the core install never pulls a cloud SDK). See the getting-started guide.\n *\n * Config is read from the environment (so it's identical across `once`/`loop` and 12-factor-friendly):\n *   CR_COMPACT_ROOT       (required) — the local-filesystem root holding cold/ warm/ registry/\n *   CR_COMPACT_MODE       once | loop                  (default: once)\n *   CR_COMPACT_INTERVAL_MS                              (default: 30000, loop mode)\n *   CR_COMPACT_NAMESPACE  scope discovery to one namespace\n *   CR_COMPACT_THRESHOLD  min dirty chunks to compact   (default: 1)\n *   CR_COMPACT_KEEP       superseded generations to retain as a reader grace window (default: 1)\n *   CR_COMPACT_OWNER      lease owner id                 (default: `<hostname>:<pid>`)\n *   CR_COMPACT_SHARD / CR_COMPACT_TOTAL_SHARDS  run this worker as shard i of N (set both) — partition the\n *                         fleet across N daemons so each drains only ~1/N of Warm per cycle (gap #3)\n *   CR_COMPACT_MAX_SEGMENTS  cap segments compacted per cycle (a time budget for a Lambda window); the rest\n *                         are deferred to the next cycle, most-dirty first\n *   CR_COMPACT_METRICS    1 to emit each per-attempt compaction `MetricEvent` (retries, bytes, ms, faults) as a\n *                         JSON line on stdout — pipe to your metrics pipeline. Off by default (cycle summaries\n *                         still print regardless); the library's `IMetricsSink` seam is what this wires.\n */\nimport { hostname } from 'node:os';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nimport {\n  LocalFsColdDriver,\n  LocalFsRegistryDriver,\n  LocalFsWarmDriver,\n  runCompactionCycle,\n} from '../index';\nimport type { CompactionCycleResult, CompactionDeps, IMetricsSink } from '../index';\n\nexport interface CompactConfig {\n  readonly root: string;\n  readonly mode: 'once' | 'loop';\n  readonly intervalMs: number;\n  readonly namespace?: string;\n  readonly threshold: number;\n  readonly keep: number;\n  readonly owner: string;\n  readonly shard?: number;\n  readonly totalShards?: number;\n  readonly maxSegments?: number;\n}\n\n/** Parse + validate config from an environment map. Throws a clear `Error` on misconfiguration. */\nexport function parseConfig(env: Record<string, string | undefined>): CompactConfig {\n  const root = env.CR_COMPACT_ROOT;\n  if (root === undefined || root === '') {\n    throw new Error('CR_COMPACT_ROOT is required (the local-filesystem storage root)');\n  }\n  const mode = env.CR_COMPACT_MODE ?? 'once';\n  if (mode !== 'once' && mode !== 'loop') {\n    throw new Error(`CR_COMPACT_MODE must be \"once\" or \"loop\"; got ${mode}`);\n  }\n  const num = (name: string, def: number): number => {\n    const raw = env[name];\n    if (raw === undefined || raw === '') return def; // unset OR empty (a common unset-var expansion) → default\n    const v = Number(raw);\n    if (!Number.isFinite(v) || v < 0)\n      throw new Error(`${name} must be a non-negative number; got ${raw}`);\n    return v;\n  };\n  const intOpt = (name: string, min = 0): number | undefined => {\n    const raw = env[name];\n    if (raw === undefined || raw === '') return undefined;\n    const v = Number(raw);\n    if (!Number.isInteger(v) || v < min)\n      throw new Error(`${name} must be an integer >= ${min}; got ${raw}`);\n    return v;\n  };\n  const shard = intOpt('CR_COMPACT_SHARD');\n  const totalShards = intOpt('CR_COMPACT_TOTAL_SHARDS');\n  if ((shard === undefined) !== (totalShards === undefined)) {\n    throw new Error('set both CR_COMPACT_SHARD and CR_COMPACT_TOTAL_SHARDS, or neither');\n  }\n  return {\n    root,\n    mode,\n    intervalMs: num('CR_COMPACT_INTERVAL_MS', 30_000),\n    namespace: env.CR_COMPACT_NAMESPACE,\n    threshold: num('CR_COMPACT_THRESHOLD', 1),\n    keep: num('CR_COMPACT_KEEP', 1),\n    owner: env.CR_COMPACT_OWNER ?? `${hostname()}:${process.pid}`,\n    shard,\n    totalShards,\n    maxSegments: intOpt('CR_COMPACT_MAX_SEGMENTS', 1),\n  };\n}\n\n/** Build LocalFs compaction deps rooted at `<root>/{cold,warm,registry}`, optionally with a metrics sink. */\nexport function localFsDeps(root: string, metrics?: IMetricsSink): CompactionDeps {\n  return {\n    cold: new LocalFsColdDriver(join(root, 'cold')),\n    warm: new LocalFsWarmDriver(join(root, 'warm')),\n    registry: new LocalFsRegistryDriver(join(root, 'registry')),\n    clock: { now: () => Date.now() }, // the bin is outside core/, so ambient time is allowed here\n    ...(metrics ? { metrics } : {}),\n  };\n}\n\n/** Run a single compaction cycle and return the full result (the unit of work for both deploy modes). */\nexport async function runOnce(\n  deps: CompactionDeps,\n  config: CompactConfig,\n): Promise<CompactionCycleResult> {\n  return runCompactionCycle(deps, {\n    owner: config.owner,\n    namespace: config.namespace,\n    threshold: config.threshold,\n    keep: config.keep,\n    shard: config.shard,\n    totalShards: config.totalShards,\n    maxSegments: config.maxSegments,\n  });\n}\n\nconst sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));\nconst log = (obj: unknown): void => {\n  process.stdout.write(`${JSON.stringify(obj)}\\n`);\n};\n\n/** A stdout `IMetricsSink` (opt-in via CR_COMPACT_METRICS) — one JSON line per compaction `MetricEvent`. */\nfunction stdoutMetricsSink(): IMetricsSink {\n  return {\n    onEvent: (event) => log({ event: 'metric', ...event }),\n  };\n}\n\n/** CLI entry: parse env, then run one cycle (`once`) or loop until a termination signal (`loop`). */\nexport async function main(env: Record<string, string | undefined> = process.env): Promise<void> {\n  const config = parseConfig(env);\n  const deps = localFsDeps(\n    config.root,\n    env.CR_COMPACT_METRICS === '1' ? stdoutMetricsSink() : undefined,\n  );\n\n  if (config.mode === 'once') {\n    const cycle = await runOnce(deps, config);\n    log({\n      event: 'cycle',\n      mode: 'once',\n      candidates: cycle.candidates,\n      compacted: cycle.compacted,\n      deferred: cycle.deferred,\n      results: cycle.results,\n    });\n    return;\n  }\n\n  // loop mode: run forever, one cycle per interval, until SIGINT/SIGTERM.\n  let stop = false;\n  const onSignal = (): void => {\n    stop = true;\n  };\n  process.on('SIGINT', onSignal);\n  process.on('SIGTERM', onSignal);\n  log({ event: 'start', mode: 'loop', intervalMs: config.intervalMs, owner: config.owner });\n  while (!stop) {\n    try {\n      const cycle = await runOnce(deps, config);\n      log({\n        event: 'cycle',\n        candidates: cycle.candidates,\n        compacted: cycle.compacted,\n        deferred: cycle.deferred,\n        results: cycle.results,\n      });\n    } catch (err) {\n      log({ event: 'error', message: (err as Error).message }); // a cycle error must not kill the daemon\n    }\n    if (!stop) await sleep(config.intervalMs);\n  }\n  log({ event: 'stopped' });\n}\n\n// Run only when invoked directly (not when imported by tests). `process.argv[1]` is this file when run as a bin;\n// `pathToFileURL` builds a spec-correct file:// URL (drive letters, spaces, UNC paths) for a robust compare.\nif (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {\n  main().catch((err: unknown) => {\n    process.stderr.write(`${(err as Error).message}\\n`);\n    process.exit(1);\n  });\n}\n"]}

package/dist/bin/export-segments.js

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+import { src_exports, CloudRoaring } from '../chunk-JWCQLJUG.js';
+import { randomUUID } from 'crypto';
+import { mkdir, writeFile, rename, rm, open } from 'fs/promises';
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { validateSegmentRef } from '@cloudbitmaps/core';
+
+function parseSegments(raw) {
+  const entries = (raw ?? "").split(",").map((s) => s.trim()).filter((s) => s.length > 0);
+  if (entries.length === 0) return void 0;
+  return entries.map((entry) => {
+    const slash = entry.indexOf("/");
+    const ref = slash === -1 ? { segment: entry } : { namespace: entry.slice(0, slash), segment: entry.slice(slash + 1) };
+    validateSegmentRef(ref);
+    return ref;
+  });
+}
+function parseConfig(env) {
+  const root = env.CR_EXPORT_ROOT;
+  if (root === void 0 || root === "") {
+    throw new Error("CR_EXPORT_ROOT is required (the local-filesystem storage root)");
+  }
+  const out = env.CR_EXPORT_OUT;
+  if (out === void 0 || out === "") {
+    throw new Error("CR_EXPORT_OUT is required (the output directory for the dump)");
+  }
+  const format = env.CR_EXPORT_FORMAT ?? "roaring";
+  if (format !== "roaring" && format !== "ndjson") {
+    throw new Error(`CR_EXPORT_FORMAT must be "roaring" or "ndjson"; got ${format}`);
+  }
+  const namespace = env.CR_EXPORT_NAMESPACE || void 0;
+  return { root, out, format, namespace, segments: parseSegments(env.CR_EXPORT_SEGMENTS) };
+}
+function fsSink(out) {
+  return {
+    async open(ref, ext) {
+      const dir = join(out, ref.namespace ?? "_default");
+      await mkdir(dir, { recursive: true, mode: 448 });
+      const finalPath = join(dir, `${ref.segment}${ext}`);
+      const tmpPath = `${finalPath}.${randomUUID()}.part`;
+      const handle = await open(tmpPath, "wx", 384);
+      return {
+        async write(bytes) {
+          let off = 0;
+          while (off < bytes.length) {
+            const { bytesWritten } = await handle.write(bytes, off, bytes.length - off);
+            if (bytesWritten === 0) throw new Error("export write made no progress");
+            off += bytesWritten;
+          }
+        },
+        async close() {
+          await handle.close();
+          try {
+            await rename(tmpPath, finalPath);
+          } catch (err) {
+            await rm(tmpPath, { force: true });
+            throw err;
+          }
+        },
+        async abort() {
+          await handle.close().catch(() => {
+          });
+          await rm(tmpPath, { force: true });
+        }
+      };
+    }
+  };
+}
+var log = (obj) => {
+  process.stdout.write(`${JSON.stringify(obj)}
+`);
+};
+async function main(env = process.env, now = () => Date.now()) {
+  const config = parseConfig(env);
+  const registry = new src_exports.LocalFsRegistryDriver(join(config.root, "registry"));
+  const store = new CloudRoaring({
+    cold: new src_exports.LocalFsColdDriver(join(config.root, "cold")),
+    warm: new src_exports.LocalFsWarmDriver(join(config.root, "warm")),
+    registry
+  });
+  const manifest = await store.exportSegments(fsSink(config.out), {
+    format: config.format,
+    namespace: config.namespace,
+    candidates: config.segments
+  });
+  await mkdir(config.out, { recursive: true, mode: 448 });
+  const manifestPath = join(config.out, "manifest.json");
+  const manifestTmp = `${manifestPath}.${randomUUID()}.part`;
+  await writeFile(
+    manifestTmp,
+    `${JSON.stringify({ generatedAt: new Date(now()).toISOString(), ...manifest }, null, 2)}
+`,
+    { encoding: "utf8", mode: 384, flag: "wx" }
+  );
+  try {
+    await rename(manifestTmp, manifestPath);
+  } catch (err) {
+    await rm(manifestTmp, { force: true });
+    throw err;
+  }
+  log({
+    event: "export",
+    out: config.out,
+    version: manifest.version,
+    format: manifest.format,
+    totalSegments: manifest.totalSegments,
+    totalIds: manifest.totalIds,
+    failed: manifest.failed.length
+  });
+  return manifest;
+}
+if (process.argv[1] !== void 0 && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main().then((manifest) => {
+    if (manifest.failed.length > 0) {
+      process.stderr.write(
+        `${manifest.failed.length} segment(s) failed to export; see manifest.json "failed"
+`
+      );
+      process.exitCode = 1;
+    }
+  }).catch((err) => {
+    process.stderr.write(`${err.message}
+`);
+    process.exit(1);
+  });
+}
+
+export { fsSink, main, parseConfig };
+//# sourceMappingURL=export-segments.js.map
+//# sourceMappingURL=export-segments.js.map

package/dist/bin/export-segments.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/bin/export-segments.ts"],"names":[],"mappings":";;;;;;;;AAyDA,SAAS,cAAc,GAAA,EAA4D;AACjF,EAAA,MAAM,WAAW,GAAA,IAAO,EAAA,EACrB,MAAM,GAAG,CAAA,CACT,IAAI,CAAC,CAAA,KAAM,CAAA,CAAE,IAAA,EAAM,CAAA,CACnB,MAAA,CAAO,CAAC,CAAA,KAAM,CAAA,CAAE,SAAS,CAAC,CAAA;AAC7B,EAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG,OAAO,MAAA;AACjC,EAAA,OAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,KAAA,KAAU;AAC5B,IAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,OAAA,CAAQ,GAAG,CAAA;AAC/B,IAAA,MAAM,MACJ,KAAA,KAAU,EAAA,GACN,EAAE,OAAA,EAAS,KAAA,KACX,EAAE,SAAA,EAAW,MAAM,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA,EAAG,OAAA,EAAS,MAAM,KAAA,CAAM,KAAA,GAAQ,CAAC,CAAA,EAAE;AAC1E,IAAA,kBAAA,CAAmB,GAAG,CAAA;AACtB,IAAA,OAAO,GAAA;AAAA,EACT,CAAC,CAAA;AACH;AAGO,SAAS,YAAY,GAAA,EAAuD;AACjF,EAAA,MAAM,OAAO,GAAA,CAAI,cAAA;AACjB,EAAA,IAAI,IAAA,KAAS,MAAA,IAAa,IAAA,KAAS,EAAA,EAAI;AACrC,IAAA,MAAM,IAAI,MAAM,gEAAgE,CAAA;AAAA,EAClF;AACA,EAAA,MAAM,MAAM,GAAA,CAAI,aAAA;AAChB,EAAA,IAAI,GAAA,KAAQ,MAAA,IAAa,GAAA,KAAQ,EAAA,EAAI;AACnC,IAAA,MAAM,IAAI,MAAM,+DAA+D,CAAA;AAAA,EACjF;AACA,EAAA,MAAM,MAAA,GAAS,IAAI,gBAAA,IAAoB,SAAA;AACvC,EAAA,IAAI,MAAA,KAAW,SAAA,IAAa,MAAA,KAAW,QAAA,EAAU;AAC/C,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,oDAAA,EAAuD,MAAM,CAAA,CAAE,CAAA;AAAA,EACjF;AACA,EAAA,MAAM,SAAA,GAAY,IAAI,mBAAA,IAAuB,MAAA;AAC7C,EAAA,OAAO,EAAE,MAAM,GAAA,EAAK,MAAA,EAAQ,WAAW,QAAA,EAAU,aAAA,CAAc,GAAA,CAAI,kBAAkB,CAAA,EAAE;AACzF;AAUO,SAAS,OAAO,GAAA,EAAyB;AAC9C,EAAA,OAAO;AAAA,IACL,MAAM,IAAA,CAAK,GAAA,EAAiB,GAAA,EAAa;AACvC,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,GAAA,EAAK,GAAA,CAAI,aAAa,UAAU,CAAA;AACjD,MAAA,MAAM,MAAM,GAAA,EAAK,EAAE,WAAW,IAAA,EAAM,IAAA,EAAM,KAAO,CAAA;AACjD,MAAA,MAAM,SAAA,GAAY,KAAK,GAAA,EAAK,CAAA,EAAG,IAAI,OAAO,CAAA,EAAG,GAAG,CAAA,CAAE,CAAA;AAClD,MAAA,MAAM,OAAA,GAAU,CAAA,EAAG,SAAS,CAAA,CAAA,EAAI,YAAY,CAAA,KAAA,CAAA;AAC5C,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,OAAA,EAAS,MAAM,GAAK,CAAA;AAC9C,MAAA,OAAO;AAAA,QACL,MAAM,MAAM,KAAA,EAAkC;AAG5C,UAAA,IAAI,GAAA,GAAM,CAAA;AACV,UAAA,OAAO,GAAA,GAAM,MAAM,MAAA,EAAQ;AACzB,YAAA,MAAM,EAAE,YAAA,EAAa,GAAI,MAAM,MAAA,CAAO,MAAM,KAAA,EAAO,GAAA,EAAK,KAAA,CAAM,MAAA,GAAS,GAAG,CAAA;AAC1E,YAAA,IAAI,YAAA,KAAiB,CAAA,EAAG,MAAM,IAAI,MAAM,+BAA+B,CAAA;AACvE,YAAA,GAAA,IAAO,YAAA;AAAA,UACT;AAAA,QACF,CAAA;AAAA,QACA,MAAM,KAAA,GAAuB;AAC3B,UAAA,MAAM,OAAO,KAAA,EAAM;AACnB,UAAA,IAAI;AACF,YAAA,MAAM,MAAA,CAAO,SAAS,SAAS,CAAA;AAAA,UACjC,SAAS,GAAA,EAAK;AACZ,YAAA,MAAM,EAAA,CAAG,OAAA,EAAS,EAAE,KAAA,EAAO,MAAM,CAAA;AACjC,YAAA,MAAM,GAAA;AAAA,UACR;AAAA,QACF,CAAA;AAAA,QACA,MAAM,KAAA,GAAuB;AAE3B,UAAA,MAAM,MAAA,CAAO,KAAA,EAAM,CAAE,KAAA,CAAM,MAAM;AAAA,UAAC,CAAC,CAAA;AACnC,UAAA,MAAM,EAAA,CAAG,OAAA,EAAS,EAAE,KAAA,EAAO,MAAM,CAAA;AAAA,QACnC;AAAA,OACF;AAAA,IACF;AAAA,GACF;AACF;AAEA,IAAM,GAAA,GAAM,CAAC,GAAA,KAAuB;AAClC,EAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,EAAG,IAAA,CAAK,SAAA,CAAU,GAAG,CAAC;AAAA,CAAI,CAAA;AACjD,CAAA;AAGA,eAAsB,IAAA,CACpB,MAA0C,OAAA,CAAQ,GAAA,EAClD,MAAoB,MAAM,IAAA,CAAK,KAAI,EACV;AACzB,EAAA,MAAM,MAAA,GAAS,YAAY,GAAG,CAAA;AAC9B,EAAA,MAAM,WAAW,IAAI,WAAA,CAAA,qBAAA,CAAsB,KAAK,MAAA,CAAO,IAAA,EAAM,UAAU,CAAC,CAAA;AACxE,EAAA,MAAM,KAAA,GAAQ,IAAI,YAAA,CAAa;AAAA,IAC7B,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,KAAK,MAAA,CAAO,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IACrD,MAAM,IAAI,WAAA,CAAA,iBAAA,CAAkB,KAAK,MAAA,CAAO,IAAA,EAAM,MAAM,CAAC,CAAA;AAAA,IACrD;AAAA,GACD,CAAA;AAED,EAAA,MAAM,WAAW,MAAM,KAAA,CAAM,eAAe,MAAA,CAAO,MAAA,CAAO,GAAG,CAAA,EAAG;AAAA,IAC9D,QAAQ,MAAA,CAAO,MAAA;AAAA,IACf,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,YAAY,MAAA,CAAO;AAAA,GACpB,CAAA;AAMD,EAAA,MAAM,KAAA,CAAM,OAAO,GAAA,EAAK,EAAE,WAAW,IAAA,EAAM,IAAA,EAAM,KAAO,CAAA;AACxD,EAAA,MAAM,YAAA,GAAe,IAAA,CAAK,MAAA,CAAO,GAAA,EAAK,eAAe,CAAA;AACrD,EAAA,MAAM,WAAA,GAAc,CAAA,EAAG,YAAY,CAAA,CAAA,EAAI,YAAY,CAAA,KAAA,CAAA;AACnD,EAAA,MAAM,SAAA;AAAA,IACJ,WAAA;AAAA,IACA,GAAG,IAAA,CAAK,SAAA,CAAU,EAAE,WAAA,EAAa,IAAI,IAAA,CAAK,GAAA,EAAK,CAAA,CAAE,aAAY,EAAG,GAAG,UAAS,EAAG,IAAA,EAAM,CAAC,CAAC;AAAA,CAAA;AAAA,IACvF,EAAE,QAAA,EAAU,MAAA,EAAQ,IAAA,EAAM,GAAA,EAAO,MAAM,IAAA;AAAK,GAC9C;AACA,EAAA,IAAI;AACF,IAAA,MAAM,MAAA,CAAO,aAAa,YAAY,CAAA;AAAA,EACxC,SAAS,GAAA,EAAK;AACZ,IAAA,MAAM,EAAA,CAAG,WAAA,EAAa,EAAE,KAAA,EAAO,MAAM,CAAA;AACrC,IAAA,MAAM,GAAA;AAAA,EACR;AAGA,EAAA,GAAA,CAAI;AAAA,IACF,KAAA,EAAO,QAAA;AAAA,IACP,KAAK,MAAA,CAAO,GAAA;AAAA,IACZ,SAAS,QAAA,CAAS,OAAA;AAAA,IAClB,QAAQ,QAAA,CAAS,MAAA;AAAA,IACjB,eAAe,QAAA,CAAS,aAAA;AAAA,IACxB,UAAU,QAAA,CAAS,QAAA;AAAA,IACnB,MAAA,EAAQ,SAAS,MAAA,CAAO;AAAA,GACzB,CAAA;AACD,EAAA,OAAO,QAAA;AACT;AAGA,IAAI,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAA,KAAM,MAAA,IAAa,MAAA,CAAA,IAAA,CAAY,GAAA,KAAQ,aAAA,CAAc,OAAA,CAAQ,IAAA,CAAK,CAAC,CAAC,EAAE,IAAA,EAAM;AAC5F,EAAA,IAAA,EAAK,CACF,IAAA,CAAK,CAAC,QAAA,KAAa;AAClB,IAAA,IAAI,QAAA,CAAS,MAAA,CAAO,MAAA,GAAS,CAAA,EAAG;AAE9B,MAAA,OAAA,CAAQ,MAAA,CAAO,KAAA;AAAA,QACb,CAAA,EAAG,QAAA,CAAS,MAAA,CAAO,MAAM,CAAA;AAAA;AAAA,OAC3B;AACA,MAAA,OAAA,CAAQ,QAAA,GAAW,CAAA;AAAA,IACrB;AAAA,EACF,CAAC,CAAA,CACA,KAAA,CAAM,CAAC,GAAA,KAAiB;AACvB,IAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,EAAI,GAAA,CAAc,OAAO;AAAA,CAAI,CAAA;AAClD,IAAA,OAAA,CAAQ,KAAK,CAAC,CAAA;AAAA,EAChB,CAAC,CAAA;AACL","file":"export-segments.js","sourcesContent":["#!/usr/bin/env node\n/**\n * `export-segments` — the standalone **eject / data-export** CLI (the exit path; see the README's \"Your data\n * stays yours\"). Dumps every registered segment's current *effective* set to a portable file, so your data is\n * readable **without CloudRoaring**. A thin filesystem wrapper over `CloudRoaring.exportSegments`.\n *\n * Two formats:\n *   - `roaring` (default) — one portable RoaringBitmap32 per segment (`<segment>.roaring`), loadable by any\n *     roaring library (Java/Go/Python/Rust/C++/…).\n *   - `ndjson` — newline-delimited ids per segment (`<segment>.ndjson`), zero dependencies to read, streamed.\n *\n * Output layout: `<out>/<namespace|_default>/<segment>.<ext>` + a self-describing `<out>/manifest.json`. Files\n * are written to a unique `.part` temp and atomically renamed on completion; the manifest is written last (also\n * atomically), so a directory with a `manifest.json` means the run **finished** (a crash leaves no manifest →\n * re-run). A segment that couldn't be read is isolated into the manifest's `failed[]` and the CLI exits non-zero\n * — always check it, because \"manifest present\" is *the run finished*, not \"every segment succeeded\". Re-running\n * overwrites the segments it re-exports but does **not** prune files for segments that have since disappeared —\n * **export to a fresh directory** for a clean dump. Artifacts are owner-only (decrypted **cleartext** — protect it).\n *\n * Ships the **local-filesystem** backend (zero-dependency, the dev/reference target). For a cloud store, wire a\n * ~10-line script that builds an `S3ColdDriver` (+ `DynamoDb*`/`S3RegistryDriver`) + a `CloudRoaring`, and calls\n * `store.exportSegments(sink, { format })` with your own sink — the binary stays SDK-free.\n *\n * Config is read from the environment (12-factor-friendly):\n *   CR_EXPORT_ROOT       (required) — the local-filesystem root holding cold/ warm/ registry/\n *   CR_EXPORT_OUT        (required) — the output directory for the dump\n *   CR_EXPORT_FORMAT     roaring | ndjson                (default: roaring)\n *   CR_EXPORT_NAMESPACE  scope the export to one namespace\n *   CR_EXPORT_SEGMENTS   comma-separated extra segments to include (all-warm / not-yet-registered): `seg` or `ns/seg`\n */\nimport { randomUUID } from 'node:crypto';\nimport { mkdir, open, rename, rm, writeFile } from 'node:fs/promises';\nimport { join } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nimport { validateSegmentRef } from '@cloudbitmaps/core';\nimport {\n  CloudRoaring,\n  LocalFsColdDriver,\n  LocalFsRegistryDriver,\n  LocalFsWarmDriver,\n} from '../index';\nimport type { ExportFormat, ExportManifest, ExportSink, SegmentRef } from '../index';\n\nexport interface ExportConfig {\n  readonly root: string;\n  readonly out: string;\n  readonly format: ExportFormat;\n  readonly namespace?: string;\n  /** Extra segments to include beyond the registry (all-warm / not-yet-registered); from `CR_EXPORT_SEGMENTS`. */\n  readonly segments?: readonly SegmentRef[];\n}\n\n/**\n * Parse `CR_EXPORT_SEGMENTS` — comma-separated `segment` or `namespace/segment` entries — into refs, or\n * `undefined` when unset/empty. Names are validated up front so a typo fails fast rather than silently landing in\n * the manifest's `failed[]`.\n */\nfunction parseSegments(raw: string | undefined): readonly SegmentRef[] | undefined {\n  const entries = (raw ?? '')\n    .split(',')\n    .map((s) => s.trim())\n    .filter((s) => s.length > 0);\n  if (entries.length === 0) return undefined;\n  return entries.map((entry) => {\n    const slash = entry.indexOf('/'); // segment/namespace names can't contain '/', so the first one splits cleanly\n    const ref: SegmentRef =\n      slash === -1\n        ? { segment: entry }\n        : { namespace: entry.slice(0, slash), segment: entry.slice(slash + 1) };\n    validateSegmentRef(ref); // fail fast on a bad name (empty part, illegal chars)\n    return ref;\n  });\n}\n\n/** Parse + validate config from an environment map. Throws a clear `Error` on misconfiguration. */\nexport function parseConfig(env: Record<string, string | undefined>): ExportConfig {\n  const root = env.CR_EXPORT_ROOT;\n  if (root === undefined || root === '') {\n    throw new Error('CR_EXPORT_ROOT is required (the local-filesystem storage root)');\n  }\n  const out = env.CR_EXPORT_OUT;\n  if (out === undefined || out === '') {\n    throw new Error('CR_EXPORT_OUT is required (the output directory for the dump)');\n  }\n  const format = env.CR_EXPORT_FORMAT ?? 'roaring';\n  if (format !== 'roaring' && format !== 'ndjson') {\n    throw new Error(`CR_EXPORT_FORMAT must be \"roaring\" or \"ndjson\"; got ${format}`);\n  }\n  const namespace = env.CR_EXPORT_NAMESPACE || undefined; // treat '' (an unset shell var) as \"no filter\"\n  return { root, out, format, namespace, segments: parseSegments(env.CR_EXPORT_SEGMENTS) };\n}\n\n/**\n * A filesystem {@link ExportSink}. Writes each segment to a **unique** `<segment><ext>.<uuid>.part` temp\n * (`O_EXCL` create, mode `0o600`) and atomically renames it into `<out>/<namespace>/<segment><ext>` on `close()`,\n * so a half-written file never masquerades as complete and concurrent exports to the same dir can't clobber one\n * another. Uses a `FileHandle` (not `createWriteStream`) so an I/O fault **rejects the write** rather than\n * emitting an unhandled `'error'` event that would crash the process, and loops on short writes so a partial\n * `write()` never truncates the output. Artifacts are **owner-only** (dir `0o700`, files `0o600`) — cleartext.\n */\nexport function fsSink(out: string): ExportSink {\n  return {\n    async open(ref: SegmentRef, ext: string) {\n      const dir = join(out, ref.namespace ?? '_default');\n      await mkdir(dir, { recursive: true, mode: 0o700 });\n      const finalPath = join(dir, `${ref.segment}${ext}`);\n      const tmpPath = `${finalPath}.${randomUUID()}.part`;\n      const handle = await open(tmpPath, 'wx', 0o600); // wx = create-exclusive; unique name ⇒ no collision\n      return {\n        async write(bytes: Uint8Array): Promise<void> {\n          // Loop until every byte lands — FileHandle.write may do a short write; it rejects on I/O error (no\n          // stray 'error' event → no process crash).\n          let off = 0;\n          while (off < bytes.length) {\n            const { bytesWritten } = await handle.write(bytes, off, bytes.length - off);\n            if (bytesWritten === 0) throw new Error('export write made no progress');\n            off += bytesWritten;\n          }\n        },\n        async close(): Promise<void> {\n          await handle.close();\n          try {\n            await rename(tmpPath, finalPath);\n          } catch (err) {\n            await rm(tmpPath, { force: true }); // don't strand the temp if the rename fails\n            throw err;\n          }\n        },\n        async abort(): Promise<void> {\n          // Discard the partial; best-effort so a failing cleanup never masks the caller's original fault.\n          await handle.close().catch(() => {});\n          await rm(tmpPath, { force: true });\n        },\n      };\n    },\n  };\n}\n\nconst log = (obj: unknown): void => {\n  process.stdout.write(`${JSON.stringify(obj)}\\n`);\n};\n\n/** CLI entry: build a LocalFs store, export every enumerated segment, then write the manifest. */\nexport async function main(\n  env: Record<string, string | undefined> = process.env,\n  now: () => number = () => Date.now(),\n): Promise<ExportManifest> {\n  const config = parseConfig(env);\n  const registry = new LocalFsRegistryDriver(join(config.root, 'registry'));\n  const store = new CloudRoaring({\n    cold: new LocalFsColdDriver(join(config.root, 'cold')),\n    warm: new LocalFsWarmDriver(join(config.root, 'warm')),\n    registry,\n  });\n\n  const manifest = await store.exportSegments(fsSink(config.out), {\n    format: config.format,\n    namespace: config.namespace,\n    candidates: config.segments,\n  });\n\n  // The manifest is written LAST (after every segment file is committed), so its presence marks a finished run —\n  // and it's written the same atomic (temp→rename) way, so the marker itself can't be torn by a crash mid-write.\n  // Owner-only (0o600) — the dump is cleartext. (A finished run may still have skipped unreadable segments — see\n  // `manifest.failed`.)\n  await mkdir(config.out, { recursive: true, mode: 0o700 });\n  const manifestPath = join(config.out, 'manifest.json');\n  const manifestTmp = `${manifestPath}.${randomUUID()}.part`;\n  await writeFile(\n    manifestTmp,\n    `${JSON.stringify({ generatedAt: new Date(now()).toISOString(), ...manifest }, null, 2)}\\n`,\n    { encoding: 'utf8', mode: 0o600, flag: 'wx' },\n  );\n  try {\n    await rename(manifestTmp, manifestPath);\n  } catch (err) {\n    await rm(manifestTmp, { force: true });\n    throw err;\n  }\n  // Log AGGREGATES only — never per-segment names/counts (a segment name can encode sensitive purpose, and stdout\n  // is captured by CI/journald/docker, which don't honor the 0o600 boundary). The detail lives in the manifest.\n  log({\n    event: 'export',\n    out: config.out,\n    version: manifest.version,\n    format: manifest.format,\n    totalSegments: manifest.totalSegments,\n    totalIds: manifest.totalIds,\n    failed: manifest.failed.length,\n  });\n  return manifest;\n}\n\n// Run only when invoked directly (not when imported by tests). See compact-segments for the URL-compare rationale.\nif (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {\n  main()\n    .then((manifest) => {\n      if (manifest.failed.length > 0) {\n        // The run finished and the manifest is written, but some segments couldn't be read — signal it non-zero.\n        process.stderr.write(\n          `${manifest.failed.length} segment(s) failed to export; see manifest.json \"failed\"\\n`,\n        );\n        process.exitCode = 1;\n      }\n    })\n    .catch((err: unknown) => {\n      process.stderr.write(`${(err as Error).message}\\n`);\n      process.exit(1);\n    });\n}\n"]}

package/dist/cassandra/index.cjs

@@ -0,0 +1,14 @@
+'use strict';
+
+var cassandra = require('@cloudbitmaps/core/cassandra');
+
+
+
+Object.keys(cassandra).forEach(function (k) {
+	if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
+		enumerable: true,
+		get: function () { return cassandra[k]; }
+	});
+});
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/cassandra/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"index.cjs","sourcesContent":[]}

package/dist/cassandra/index.d.cts

@@ -0,0 +1 @@
+export * from '@cloudbitmaps/core/cassandra';

package/dist/cassandra/index.d.ts

@@ -0,0 +1 @@
+export * from '@cloudbitmaps/core/cassandra';

package/dist/cassandra/index.js

@@ -0,0 +1,3 @@
+export * from '@cloudbitmaps/core/cassandra';
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/cassandra/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"index.js","sourcesContent":[]}