@cloudbitmaps/roaring 0.15.0 → 0.17.0

package/dist/bin/export-segments.js

@@ -21,6 +21,10 @@ import {
   NOOP_METRICS,
   RetryingStorageChunkSource,
   SegmentEngine,
+  LEASE_SKEW_MS,
+  LeaseExpiredError,
+  MAX_LEASE_MS,
+  NotFoundError,
   UnsupportedError,
   ValidationError as ValidationError2,
   WriteConflictError,
@@ -33,17 +37,21 @@ import {
   estimateCost,
   groundedReport,
   loadSegment,
+  loadSegmentChunks,
   mapWithConcurrency,
   resolveBudget,
   resolvePerOpBudget,
+  reapRegistryTombstones,
   retireExpired,
   runConsistencyCheck,
   runExport,
   setSegmentRetention,
   clearSegmentRetention,
   getSegmentRetention,
+  releaseLease,
   safeMetrics,
-  splitId
+  splitId,
+  takeLease
 } from "@cloudbitmaps/core";
 import { validateSegmentRef } from "@cloudbitmaps/core/driver-kit";
 
@@ -103,7 +111,7 @@ function refuseReservedNamespace(namespace) {
 
 // src/roaring-codec.ts
 import roaring from "roaring";
-import { IntegrityError as IntegrityError2 } from "@cloudbitmaps/core";
+import { IntegrityError as IntegrityError2, decodeSerialized } from "@cloudbitmaps/core";
 
 // src/portable/layout.ts
 import { IntegrityError } from "@cloudbitmaps/core";
@@ -351,6 +359,10 @@ var SafeBitmap = class _SafeBitmap {
       throw new IntegrityError2(`failed to deserialize bitmap: ${err.message}`);
     }
   }
+  /** The `roaring` bitmap this wraps, for a caller that was handed it once and keeps no wrapper. */
+  unwrap() {
+    return this.bitmap;
+  }
   serialize() {
     return this.bitmap.serialize(SerializationFormat.portable);
   }
@@ -460,7 +472,8 @@ function bitmapAsLoadInput(input) {
 var roaringCodec = {
   empty: () => SafeBitmap.empty(),
   fromValues: (values) => SafeBitmap.fromValues(values),
-  safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options)
+  safeDeserialize: (bytes, maxBytes, options) => SafeBitmap.safeDeserialize(bytes, maxBytes, options),
+  owns: (bitmap) => bitmap instanceof SafeBitmap
 };
 
 // src/system-clock.ts
@@ -514,6 +527,86 @@ var OPTION_KEYS = {
   seams: ["clock", "rng"]
 };
 
+// src/lease-guards.ts
+var guardIdIterable = (inner, check) => ({
+  [Symbol.asyncIterator]: () => {
+    const it = inner[Symbol.asyncIterator]();
+    let chunk = -1;
+    let first = true;
+    const stop = async (err) => {
+      await it.return?.();
+      throw err;
+    };
+    return {
+      async next() {
+        if (first) {
+          first = false;
+          try {
+            check();
+          } catch (err) {
+            return stop(err);
+          }
+        }
+        const r = await it.next();
+        if (r.done === true) return r;
+        const key = r.value >>> 16;
+        if (key !== chunk) {
+          try {
+            check();
+          } catch (err) {
+            return stop(err);
+          }
+          chunk = key;
+        }
+        return r;
+      },
+      async return() {
+        await it.return?.();
+        return { done: true, value: void 0 };
+      }
+    };
+  }
+});
+var guardIds = (inner, check) => ({
+  [Symbol.asyncIterator]: () => guardIdIterable(inner, check)[Symbol.asyncIterator](),
+  batches: () => guardItems(inner.batches(), check)
+});
+var guardItems = (inner, check) => ({
+  [Symbol.asyncIterator]: () => {
+    const it = inner[Symbol.asyncIterator]();
+    let first = true;
+    const stop = async (err) => {
+      await it.return?.();
+      throw err;
+    };
+    return {
+      async next() {
+        if (first) {
+          first = false;
+          try {
+            check();
+          } catch (err) {
+            return stop(err);
+          }
+        }
+        const r = await it.next();
+        if (r.done === true) return r;
+        try {
+          check();
+        } catch (err) {
+          return stop(err);
+        }
+        return r;
+      },
+      async return() {
+        await it.return?.();
+        return { done: true, value: void 0 };
+      }
+    };
+  }
+});
+var guardChunks = (inner, check) => guardItems(inner, check);
+
 // src/index.ts
 import {
   MemoryStorage,
@@ -524,6 +617,9 @@ import {
   excludingReservedRows as excludingReservedRows2,
   readRetentionPolicy,
   MIN_EXPIRES_AT_MS as MIN_EXPIRES_AT_MS2,
+  LEASE_SKEW_MS as LEASE_SKEW_MS2,
+  MAX_LEASE_MS as MAX_LEASE_MS2,
+  MAX_LEASES_PER_SEGMENT,
   InProcessKeystore,
   NodeAead,
   aadFor,
@@ -531,18 +627,22 @@ import {
   ValidationError as ValidationError3,
   WriteConflictError as WriteConflictError2,
   IntegrityError as IntegrityError3,
-  NotFoundError,
+  NotFoundError as NotFoundError2,
   UnsupportedError as UnsupportedError2,
   CapabilityError,
   TransientError,
   KeyUnavailableError,
   BudgetExceededError,
+  LeaseExpiredError as LeaseExpiredError2,
+  LeaseLimitError,
   isCloudRoaringError,
   isWriteConflictError,
   isTransientError,
   isNotFoundError,
   isIntegrityError,
   isValidationError,
+  isLeaseExpiredError,
+  isLeaseLimitError,
   CrbmReader,
   BufferReader,
   CountingMetricsSink,
@@ -738,6 +838,9 @@ var CloudRoaring = class _CloudRoaring {
       clock
     });
     const resolved = resolveStorageSource(options, clock);
+    if (metrics !== NOOP_METRICS && isStorageBackend(options.storage)) {
+      options.storage.attachMetrics?.(metrics);
+    }
     let storage = resolved.source;
     if (options.retry !== false) {
       const { onRetry: userOnRetry, ...ov } = options.retry ?? {};
@@ -835,7 +938,7 @@ var CloudRoaring = class _CloudRoaring {
       clock: this.clock,
       metrics: this.metrics,
       materialize: (dest, ids, op, options2) => this.materialize(dest, ids, op, options2),
-      pinned: (r, e) => this.pinSegment(r, e),
+      pinned: (r, e, l, at) => this.pinSegment(r, e, l, at),
       combineEngine: (handles) => this.engineForCombine(handles),
       expiresAt
     });
@@ -885,7 +988,7 @@ var CloudRoaring = class _CloudRoaring {
     const metadata = options?.metadata;
     let result;
     try {
-      result = await loadSegment(dest, ids, deps, {
+      const loadOptions = {
         ...options?.allowEmpty === void 0 ? {} : { allowEmpty: options.allowEmpty },
         ...options?.guard === void 0 ? {} : { guard: options.guard },
         ...metadata === void 0 ? {} : { metadata },
@@ -898,7 +1001,8 @@ var CloudRoaring = class _CloudRoaring {
         // as a side effect of adding a guard whose entire purpose is preventing data loss. Opt in with `keep`.
         keep: options?.keep ?? KEEP_EVERY_GENERATION,
         ...options?.audit === void 0 ? {} : { audit: options.audit }
-      });
+      };
+      result = ids instanceof CombineChunks ? await loadSegmentChunks(dest, ids.chunks, deps, loadOptions) : await loadSegment(dest, ids, deps, loadOptions);
     } finally {
       this.engine.invalidate(dest);
     }
@@ -1259,24 +1363,26 @@ var CloudRoaring = class _CloudRoaring {
    * A refused load deletes the object it wrote before returning — it sits above `currentGen`, where generation
    * collection deliberately never looks — but only while the segment's registry row is unchanged or gone. Once
    * another write has changed the row, the generation number it holds may name another incarnation's live object,
-   * so it leaves the orphan rather than risk deleting live data. The orphan is an ordinary generation once a later
-   * one is current above it, and collection counts it within `keep`.
+   * so it leaves the orphan rather than risk deleting live data. No row's list names it, so it takes no place in a
+   * window, and a listing deletes it once a later generation is current above it.
    *
-   * **Collection is by name for the default `keep`.** With `keep` of 0 or 1, a load that found nothing above the
-   * pointer deletes the one generation its publish pushed out of the window and lists nothing; it lists the segment's
-   * objects on every sixteenth generation, and on any load that met an object above the pointer or whose check found
-   * the current generation's object gone, to take what the name-only passes leave, such as the generations an
-   * earlier, wider `keep` held. `keep` of 2 or more lists on every load. {@link LoadResult.collected} then names what
-   * the pass deleted by name, and that generation may have been gone already.
+   * **Collection is by name at any `keep` up to 64.** The segment's row records the generations a load keeps, so a load
+   * that found nothing above the pointer deletes the generations its publish pushed out of the window and lists
+   * nothing. It lists the segment's objects on every sixteenth generation, and on any load that met an object above the
+   * pointer or whose check found the current generation's object gone, to take what the name-only passes leave, such as
+   * an object a crashed load left below the pointer. A `keep` above 64 records no list and lists on every load, and so
+   * does the first load of a row that records none, which then records it. {@link LoadResult.collected} then names
+   * what the pass deleted by name, and one name may have been gone already.
    *
    * What it **throws** for is a fault rather than an outcome: invalid options or ids, and a crypto-shredded
    * segment (`ValidationError`); a key the keystore cannot provide (`KeyUnavailableError`); a current generation
    * that will not open when a guard has to read its size, which it does only when the row has no usable summary of it
-   * (`IntegrityError`); a driver failure; and a collection pass
-   * by listing that could not prove the segment was still the same one (`WriteConflictError`). That one, and a
-   * failure in the collection's own reads or deletes, can be raised **after** the publish already landed, so a throw
-   * does not by itself mean the load did not take effect — re-read the pointer rather than assuming. A collection by
-   * name that finds the segment changed returns an empty `collected` instead.
+   * (`IntegrityError`); a driver failure; and a fault in the collection's own reads, listing or deletes, or in the write
+   * that records the window (anything but a lost race). A collection fault can be raised **after** the publish already
+   * landed, so a throw does not by itself mean the load did not take effect — re-read the pointer rather than assuming.
+   * A collection, by name or by listing, that finds another writer has moved the row since the publish (a load, a
+   * rollback, a purge) stops and returns what it deleted in `collected`: a load that took effect does not throw for a
+   * race it won.
    *
    * **A `TransientError` from the registry write can leave the publish unsettled, and deletes nothing.** The
    * generation's object is sent again after a throttle where the backend allows it (a write id tells a first send
@@ -1348,7 +1454,11 @@ var CloudRoaring = class _CloudRoaring {
     validateSegmentRef(ref);
     const deps = this.lifecycleDeps("dropSegment");
     try {
-      return await dropSegment(ref, { registry: deps.registry, storage: deps.storage }, options);
+      return await dropSegment(
+        ref,
+        { registry: deps.registry, storage: deps.storage, clock: deps.clock, rng: deps.rng },
+        options
+      );
     } finally {
       if (options.dryRun !== true) this.engine.invalidate(ref);
     }
@@ -1381,7 +1491,11 @@ var CloudRoaring = class _CloudRoaring {
    */
   async setRetention(ref, policy) {
     validateSegmentRef(ref);
-    return setSegmentRetention(ref, { registry: this.requireRegistry("setRetention") }, policy);
+    return setSegmentRetention(
+      ref,
+      { registry: this.requireRegistry("setRetention"), clock: this.clock, rng: this.rng },
+      policy
+    );
   }
   /**
    * **The stored retention policy**, or `null` if the segment has none (or has no registry row, or is a
@@ -1400,7 +1514,11 @@ var CloudRoaring = class _CloudRoaring {
    */
   async clearRetention(ref) {
     validateSegmentRef(ref);
-    return clearSegmentRetention(ref, { registry: this.requireRegistry("clearRetention") });
+    return clearSegmentRetention(ref, {
+      registry: this.requireRegistry("clearRetention"),
+      clock: this.clock,
+      rng: this.rng
+    });
   }
   /**
    * **Run the retention sweep: retire every segment whose `expiresAt` has passed.** This is the call that acts on
@@ -1463,7 +1581,7 @@ var CloudRoaring = class _CloudRoaring {
   async retireExpired(options = {}) {
     const deps = this.lifecycleDeps("retireExpired");
     const result = await retireExpired(
-      { registry: deps.registry, storage: deps.storage },
+      { registry: deps.registry, storage: deps.storage, clock: this.clock, rng: this.rng },
       { ...options, now: options.now ?? this.clock.now() }
     );
     if (options.dryRun !== true) {
@@ -1473,6 +1591,57 @@ var CloudRoaring = class _CloudRoaring {
     }
     return result;
   }
+  /**
+   * **Remove the registry rows that releases before 0.12.0 left as `deleted: true` envelopes.** Those releases never
+   * removed a row they deleted, and no later call removes it either: it is invisible to every read, and every full
+   * listing of the registry still reads it (one GET each). This is the one call that removes them. It is an admin call,
+   * not a sweep, and nothing schedules it: run it once after the last process on a release before 0.12.0 is gone.
+   *
+   * ```ts
+   * const preview = await store.reapRegistryTombstones({ dryRun: true });
+   * console.log(`would remove ${preview.wouldReap} of ${preview.examined} rows read`);
+   *
+   * const done = await store.reapRegistryTombstones({ confirmNoLegacyWriters: true });
+   * if (done.limited) runAgain(); // `limit` (default 1,000) stopped the run with rows unread
+   * ```
+   *
+   * **What it removes.** Only a row that is `deleted: true` and whose token carries no incarnation id, which is what a
+   * release before 0.12.0 left, or a 0.12.0 or later release left by deleting a row born before 0.12.0 (both are safe to
+   * remove), whatever its `status`. It never touches a live row, a `destroyed` row that is not `deleted` (a crypto-shred's
+   * or a `dropSegment`'s tombstone: the attestation of an erasure, which it cannot tell from another), or a deleted row
+   * with an incarnation id, and it never touches a generation. **So it does not clean a bucket completely**: the
+   * `destroyed` tombstones `dropSegment` leaves stay, so do live rows written before 0.12.0, and so does a tombstone a
+   * 0.12.0 or later release wrote while `conditionalDelete` was off (GCS by default, S3 on a custom endpoint): every full
+   * listing still reads it, and this call neither removes it nor runs on such a store.
+   * It also covers the library's own bookkeeping rows when `namespace` is not set, by the same rule. The default
+   * namespace cannot be scoped to: leave `namespace` out to cover it.
+   *
+   * **The two things it asks.** `confirmNoLegacyWriters: true` is required for a real run (without it, a
+   * `ValidationError`, and nothing is requested): your statement that no process on a release before 0.12.0 writes
+   * this registry, since one that did would, re-creating a removed name, issue the removed row's tokens again. And the
+   * registry must apply a delete under a precondition: where its `conditionalDelete` capability is off (GCS by default,
+   * S3 on a custom endpoint) it throws `CapabilityError` before any request, dry run included. A registry that keeps
+   * no such rows (the in-memory and local-filesystem ones) throws `UnsupportedError`.
+   *
+   * **Each removal is fenced** on the version it read, so a `create` that lands over the envelope first wins, the
+   * delete is refused, and the row is counted as `skipped.raced` (which also counts a row already gone). That holds where
+   * the backend applies the precondition, which auto-detection checks (S3 only on an AWS host). On an endpoint that
+   * ignores `If-Match` (MinIO, fake-gcs-server), a `conditionalDelete: true` you set makes this an unfenced delete: run
+   * it with every writer stopped. On a versioned bucket a delete leaves a delete marker and the earlier version, record
+   * included, so the name is not erased from the bucket. A `create` that had already read the envelope and
+   * whose write meets the removal throws `WriteConflictError`, which this library does not retry: retry the create, and
+   * it finds no row and writes one.
+   *
+   * **Cost** over R rows read and E removed: `ceil(R / 1000)` LIST requests, R GETs and E DELETEs (a dry run, no
+   * DELETEs). `limit` bounds the removals, and the run stops listing once it is spent. **A run is not resumable**: each
+   * lists and reads from the start (R GETs whatever the `limit`), a dry run with a `limit` reports the same first rows
+   * every time, so `wouldReap` is a total only when `limited` is `false`, and `limited` is `true` whenever the limit was
+   * spent with keys left, removable or not. An unreadable object stops the run with an error naming its key, and
+   * removes nothing more: it may be a newer release's row. Scope the run with `namespace` to get past one.
+   */
+  async reapRegistryTombstones(options = {}) {
+    return reapRegistryTombstones(this.requireRegistry("reapRegistryTombstones"), options);
+  }
   /**
    * **Forget everything this store has cached about a segment.** Use it after destroying or retiring a segment
    * through a path this store did not perform itself.
@@ -1519,14 +1688,54 @@ var CloudRoaring = class _CloudRoaring {
    * pinned view reports the version captured at pin time, marked as a pin's, so its decoded chunks are never
    * those of a live read that fetched across a publish (see {@link PinnedStorageChunkSource.currentVersion}).
    */
-  async pinSegment(ref, expiresAt) {
+  async pinSegment(ref, expiresAt, leaseUntil, named) {
     const crbm = this.crbmSource;
     if (crbm === void 0) {
       throw new UnsupportedError(
         "pin() needs the `.crbm` storage source \u2014 pass a backend or a raw IStorageDriver as `storage` (a pre-built StorageChunkSource that cannot resolve a generation has nothing to pin)"
       );
     }
-    const at = await this.withRetries(() => crbm.pinGeneration(ref));
+    let hold;
+    let take;
+    if (leaseUntil !== void 0) {
+      const registry = this.requireRegistry("pin({ leaseUntil })");
+      if (registry.capabilities().canWrite === false) {
+        throw new UnsupportedError(
+          "pin({ leaseUntil }) needs a registry that can write a row, and this store's cannot in this runtime"
+        );
+      }
+      const holder = this.mintHolder();
+      const deps = { registry, clock: this.clock, rng: this.rng };
+      const generationOf = (row) => named?.generation ?? row.currentGen;
+      hold = {
+        holder,
+        until: leaseUntil,
+        released: false,
+        done: false,
+        release: () => releaseLease(ref, deps, holder)
+      };
+      take = {
+        take: (row) => takeLease(ref, deps, {
+          holder,
+          generation: generationOf(row),
+          until: leaseUntil,
+          row,
+          current: named === void 0
+        })
+      };
+    }
+    let at;
+    try {
+      at = await this.withRetries(
+        () => named === void 0 ? crbm.pinGeneration(ref, take) : crbm.pinGenerationAt(ref, named, take)
+      );
+      if (hold !== void 0 && at === null) {
+        throw new NotFoundError(`segment "${ref.segment}" has no generation to lease`);
+      }
+    } catch (err) {
+      if (hold !== void 0) await hold.release().catch(() => void 0);
+      throw err;
+    }
     const pinnedAt = {
       generation: at?.generation ?? null,
       version: at?.version ?? null,
@@ -1539,12 +1748,20 @@ var CloudRoaring = class _CloudRoaring {
       clock: this.clock,
       metrics: this.metrics,
       materialize: (dest, ids, op, options) => this.materialize(dest, ids, op, options),
-      pinned: (r, e) => this.pinSegment(r, e),
+      pinned: (r, e, l, at2) => this.pinSegment(r, e, l, at2),
       combineEngine: (handles) => this.engineForCombine(handles),
       expiresAt,
-      pinnedAt
+      pinnedAt,
+      lease: hold
     });
   }
+  /** 16 hex digits from the store's `Rng`, which names the holder of one lease: unique, not secret. */
+  mintHolder() {
+    let id = "";
+    for (let i = 0; i < 4; i++)
+      id += Math.floor(this.rng.next() * 65536).toString(16).padStart(4, "0");
+    return id;
+  }
   /**
    * An engine reading every segment in `pins` at its pinned generation and everything else live, with the store's
    * retries, so a transient fault on a pinned read is retried as it would be on a live one.
@@ -1702,6 +1919,34 @@ var EMPTY_IDS = {
   async *batches() {
   }
 };
+var AS_IDS = {
+  none: EMPTY_IDS,
+  failing,
+  guard: guardIds,
+  intersect: (engine, refs, opts) => withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts)),
+  union: (engine, refs, opts) => withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts)),
+  andNot: (engine, base, refs, opts) => withBatches(engine.andNot(base, refs, opts), () => engine.andNotBatches(base, refs, opts))
+};
+var NO_CHUNKS = {
+  async *[Symbol.asyncIterator]() {
+  }
+};
+var CombineChunks = class {
+  constructor(chunks) {
+    this.chunks = chunks;
+  }
+  chunks;
+};
+var AS_CHUNKS = {
+  none: NO_CHUNKS,
+  guard: guardChunks,
+  failing: (err) => ({
+    [Symbol.asyncIterator]: () => ({ next: () => Promise.reject(err) })
+  }),
+  intersect: (engine, refs, opts) => engine.intersectChunks(refs, opts),
+  union: (engine, refs, opts) => engine.unionChunks(refs, opts),
+  andNot: (engine, base, refs, opts) => engine.andNotChunks(base, refs, opts)
+};
 var KEEP_EVERY_GENERATION = Number.MAX_SAFE_INTEGER;
 var makeSegment;
 var minting = false;
@@ -1721,6 +1966,13 @@ var Segment = class _Segment {
    * pinned handle passed as an **operand** is still read at its pin rather than live.
    */
   pinnedAt;
+  /**
+   * The lease this handle holds, when it came from `pin({ leaseUntil })`; `undefined` otherwise. It reports what was
+   * taken and does not change when the lease ends: a read after `lease.until` (or after {@link Segment.release}) throws
+   * {@link LeaseExpiredError}.
+   */
+  lease;
+  leaseHold;
   static {
     makeSegment = (parts) => {
       minting = true;
@@ -1750,6 +2002,10 @@ var Segment = class _Segment {
     this.combineEngine = parts.combineEngine;
     this.expiresAt = parts.expiresAt;
     this.pinnedAt = parts.pinnedAt;
+    this.leaseHold = parts.lease;
+    if (parts.lease !== void 0) {
+      this.lease = Object.freeze({ holder: parts.lease.holder, until: parts.lease.until });
+    }
     this.metricsOn = parts.metrics !== NOOP_METRICS;
   }
   /**
@@ -1778,11 +2034,20 @@ var Segment = class _Segment {
    * 0, so a pin of the old segment never reads the new one: what it has already read still answers, as the
    * instant it pinned, and anything it would have to fetch fails with `NotFoundError`, as a swept pin's does.
    *
-   * **It is a hold, not a lease.** Nothing here stops a collection deleting the generation underneath
-   * you: a pinned read deliberately does **not** heal forward, because silently serving a different generation
-   * is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover your longest pinned
-   * job — see [Generations and `keep`](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/loading.md#generations-and-keep) — or take the pin on a segment
-   * you are not collecting.
+   * **It is a hold, and `pin({ leaseUntil })` makes it a bounded one.** Without a lease nothing stops a collection
+   * deleting the generation underneath you: a pinned read deliberately does **not** heal forward, because silently
+   * serving a different generation is the one thing a pin exists to prevent, so it fails instead. Size `keep` to cover
+   * your longest pinned job — see [Generations and `keep`](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/loading.md#generations-and-keep) — take the pin on a segment
+   * you are not collecting, or lease it.
+   *
+   * **A lease** (`leaseUntil`: epoch **milliseconds**, after now and at most 14 days out) keeps the generation out of
+   * a load's collection until then, from any process, and the pin is taken with it: the lease is written to the segment's
+   * row before the object is opened. A read of the handle after the lease, or after {@link Segment.release}, throws
+   * {@link LeaseExpiredError} at every read site, including when the handle is an operand or an `exclude` of a combine
+   * or the target of an `*Into`, and never reads empty. A stream checks the lease each time it reads a chunk. A segment
+   * holds at most 64 live leases ({@link LeaseLimitError}). A collector holds a lease 60 seconds past its end, which covers
+   * clocks that differ by that much. Erasure, shred, drop and retention expiry ignore a lease. See
+   * [Hold a generation for a job](https://github.com/cloudbitmaps/cloudbitmaps/blob/main/docs/guide/reading.md#hold-a-generation-for-a-job-a-lease).
    *
    * A segment with no current generation pins nothing and reads empty, exactly as it would unpinned. A pinned
    * segment whose row is later dropped or destroyed fails once it must open its object again, rather than go empty
@@ -1822,8 +2087,144 @@ var Segment = class _Segment {
    * Needs a store built on the `.crbm` storage source (the default when you pass a backend or a raw driver). Throws
    * {@link UnsupportedError} on a store wired with a pre-built source that cannot pin.
    */
-  async pin() {
-    return this.pinned(this.ref, this.expiresAt);
+  async pin(options) {
+    this.assertLeases([this]);
+    const leaseUntil = this.leaseUntilOf("pin", options);
+    return this.pinned(this.ref, this.expiresAt, leaseUntil);
+  }
+  /**
+   * Hold what this pin names until `leaseUntil`, or `undefined` for a pin with no lease. Refuses an option it does not
+   * know, a value that is not epoch-milliseconds in the future, one past the longest lease, and one past the handle's own
+   * `expiresAt`, before any request is made.
+   */
+  leaseUntilOf(op, options) {
+    if (options === void 0 || options === null) return void 0;
+    if (typeof options !== "object" || Array.isArray(options)) {
+      throw new ValidationError2(`${op}: options must be an object such as { leaseUntil }`);
+    }
+    const unknown = Object.keys(options).filter((k) => k !== "leaseUntil");
+    if (unknown.length > 0) {
+      throw new ValidationError2(
+        `${op}: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { leaseUntil } only`
+      );
+    }
+    const until = options.leaseUntil;
+    if (until === void 0) return void 0;
+    const now = this.clock.now();
+    if (typeof until !== "number" || !Number.isSafeInteger(until)) {
+      throw new ValidationError2(
+        `${op}: leaseUntil must be an integer epoch-MILLISECONDS; got ${String(until)}`
+      );
+    }
+    if (until <= now) {
+      throw new ValidationError2(
+        `${op}: leaseUntil (${until}) is not after now (${now}); it is epoch MILLISECONDS, so a value in seconds reads as long past. Use \`Date.now() + hours * 3_600_000\``
+      );
+    }
+    if (until > now + MAX_LEASE_MS) {
+      throw new ValidationError2(
+        `${op}: leaseUntil is more than 14 days away; the longest lease is ${MAX_LEASE_MS} ms. Take a new lease when a job outlasts it`
+      );
+    }
+    if (this.expiresAt !== void 0 && until > this.expiresAt) {
+      throw new ValidationError2(
+        `${op}: leaseUntil outlasts this handle's own expiresAt, which would have it read empty while it still holds the generation`
+      );
+    }
+    return until;
+  }
+  /**
+   * End this handle's lease now, so a load's collection may take its generation. Idempotent: a handle with no lease, or
+   * one already released, does nothing and makes no request. Every read of the handle after it throws
+   * {@link LeaseExpiredError} (reason `'released'`), including one already under way at its next chunk. A lease that has
+   * ended by its own time needs no release: this then makes no request either, and the registry row's entry is pruned
+   * by a later write. Otherwise it is one registry read and one write; a failure of either is thrown as the registry's own
+   * typed error, and the entry then holds until its time ends, `release()` being safe to call again.
+   */
+  async release() {
+    const hold = this.leaseHold;
+    if (hold === void 0) return;
+    hold.released = true;
+    if (hold.done) return;
+    if (this.clock.now() >= hold.until + LEASE_SKEW_MS) {
+      hold.done = true;
+      return;
+    }
+    await hold.release();
+    hold.done = true;
+  }
+  /**
+   * The error a read of this handle must throw, or `undefined` when it holds no lease or the lease is live. One property
+   * read on a handle with no lease. It is not {@link Segment.expired}: that rule reads empty, and a lease must never.
+   */
+  leaseError() {
+    const hold = this.leaseHold;
+    if (hold === void 0) return void 0;
+    const name = this.ref.namespace ? `${this.ref.namespace}/${this.ref.segment}` : this.ref.segment;
+    if (hold.released) {
+      return new LeaseExpiredError(
+        `the lease on segment "${name}" was released`,
+        hold.until,
+        "released"
+      );
+    }
+    if (this.clock.now() >= hold.until) {
+      return new LeaseExpiredError(
+        `the lease on segment "${name}" ended; a read of a handle past its lease throws, never reads empty`,
+        hold.until,
+        "expired"
+      );
+    }
+    return void 0;
+  }
+  /**
+   * Throw the first lease error among `handles`: this handle and every operand and exclude of a call. It runs before
+   * any `expiresAt` rule and before the engine, so no operand, exclude or cached reader can answer past a lease.
+   */
+  assertLeases(handles) {
+    for (const h of handles) {
+      const err = h.leaseError();
+      if (err !== void 0) throw err;
+    }
+  }
+  /**
+   * Reopen a generation a pin named earlier, as a pinned handle like the one {@link Segment.pin} returns, held at
+   * `at.generation` instead of the current one. Pass what the earlier pin recorded, `{ generation, fingerprint }`
+   * from its {@link Segment.pinnedAt}. A generation number alone is not an identity, since a purged and re-created
+   * name starts again at 0, so the fingerprint is required: a bare number, a malformed fingerprint or a key other
+   * than `generation` and `fingerprint` is a {@link ValidationError}. Identify a pin by its `generation` and
+   * `fingerprint`: the handle's `pinnedAt.version` can differ from the earlier pin's.
+   *
+   * Throws {@link NotFoundError} when the generation is gone (collected, purged, or on a crypto-shredded segment),
+   * is above the row's pointer, or is another object than the fingerprint names, and never reads empty. A rollback
+   * deletes nothing, so once a later load moves the pointer past a generation it rolled back from, that one can be
+   * reopened while its object is stored. Nothing keeps a generation alive unless you lease it: a later load's
+   * collection can delete it, which is sized by `keep` on every writer, or held by `options.leaseUntil` (see
+   * {@link PinOptions} and {@link Segment.pin}). A leased `pinAt` writes the lease before it verifies the object, and
+   * releases it if the verify fails; for a generation below the pointer, a collector that read the row before the lease
+   * landed can still delete the generation inside its own round trip, and the handle then fails with `NotFoundError`
+   * like any swept pin. A leased handle is checked at every read site as a leased `pin()` is. With a registry it costs one row read and one tail read,
+   * and a leased `pinAt` one conditional write to the row besides; the handle
+   * then reads as one from `pin()` does, including its failure on a chunk it has not cached once the generation is
+   * swept. Works for an encrypted segment. The first argument is an object so that fields can join it later; the lease
+   * is in `options`, the second.
+   */
+  async pinAt(at, options) {
+    this.assertLeases([this]);
+    const { generation, fingerprint } = at ?? {};
+    const unknown = typeof at === "object" && at !== null ? Object.keys(at).filter((k) => k !== "generation" && k !== "fingerprint") : [];
+    if (unknown.length > 0) {
+      throw new ValidationError2(
+        `pinAt: unknown option${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")}; this version takes { generation, fingerprint } only`
+      );
+    }
+    if (typeof generation !== "number" || !Number.isSafeInteger(generation) || generation < 0 || typeof fingerprint !== "string" || !/^\d+:\d+$/.test(fingerprint)) {
+      throw new ValidationError2(
+        "pinAt: pass { generation, fingerprint } as recorded by an earlier pin (its pinnedAt); a generation number without the fingerprint of its object does not identify it"
+      );
+    }
+    const leaseUntil = this.leaseUntilOf("pinAt", options);
+    return this.pinned(this.ref, this.expiresAt, leaseUntil, { generation, fingerprint });
   }
   /**
    * An opaque string that names this handle's segment, namespace included: two handles of one segment have the
@@ -1867,6 +2268,7 @@ var Segment = class _Segment {
    * awaiting.
    */
   refuseIfExpired(op, dest, operands) {
+    this.assertLeases([this, dest, ...operands]);
     const stale = [];
     for (const seg of [this, dest, ...operands]) {
       if (seg.expired())
@@ -1878,6 +2280,10 @@ var Segment = class _Segment {
       );
     }
   }
+  /** `result` as is, or, when a handle of the call holds a lease, checking that lease each time it reads a chunk. */
+  guarded(out, handles, result) {
+    return handles.some((h) => h.leaseHold !== void 0) ? out.guard(result, () => this.assertLeases(handles)) : result;
+  }
   /**
    * Time an op with the injected clock and emit an `op` metric on completion (success or throw). Skipped
    * entirely when no sink is wired, so the default path pays nothing.
@@ -1893,6 +2299,8 @@ var Segment = class _Segment {
   }
   /** Membership: one chunk — the cache, else one ranged GET. Throws {@link ValidationError} on a bad id. */
   has(id) {
+    const lease = this.leaseError();
+    if (lease !== void 0) return Promise.reject(lease);
     if (this.expired()) return Promise.resolve(false);
     return this.timed("has", () => this.engine.has(this.ref, id));
   }
@@ -1915,6 +2323,8 @@ var Segment = class _Segment {
    * structure is checked.
    */
   count() {
+    const lease = this.leaseError();
+    if (lease !== void 0) return Promise.reject(lease);
     if (this.expired()) return Promise.resolve(0);
     return this.timed("count", () => this.engine.count(this.ref));
   }
@@ -1933,6 +2343,7 @@ var Segment = class _Segment {
    * ```
    */
   async stat() {
+    this.assertLeases([this]);
     if (this.expired()) return { generation: null, cardinality: 0 };
     return this.engine.stat(this.ref);
   }
@@ -1952,16 +2363,70 @@ var Segment = class _Segment {
    * without checking its options, as every read of one does.
    */
   iterate(options) {
+    const lease = this.leaseError();
+    if (lease !== void 0) return failing(lease);
     if (this.expired()) return EMPTY_IDS;
     const range = options == null ? void 0 : rangeOf(options);
     const none = range === void 0 || range.after === void 0 && range.through === void 0;
     const { engine, ref } = this;
     const bounds = none ? void 0 : range;
-    return withBatches(engine.iterate(ref, bounds), () => engine.iterateBatches(ref, bounds));
+    const stream = withBatches(
+      engine.iterate(ref, bounds),
+      () => engine.iterateBatches(ref, bounds)
+    );
+    return this.leaseHold === void 0 ? stream : guardIds(stream, () => this.assertLeases([this]));
+  }
+  /**
+   * **Pinned handles only.** The ids at 1-based ranks `n`, `2n`, `3n` …, counted over the ids in `(after, through]`
+   * when a range is given, ascending: `everyNth(1000)` yields the 1,000th id, then the 2,000th, and so on. A last
+   * partial window yields nothing, so a caller cutting a send into windows of `n` appends its own final end.
+   *
+   * It places each boundary from the per-chunk counts the pin already holds and reads only the chunks that hold one,
+   * each once however many boundaries it holds, so it reads at most one chunk per boundary and never more chunks
+   * than the object has. A range with `after` inside a chunk reads that chunk too, to count the ids at or below
+   * `after`, even when fewer than `n` ids remain above it; when the chunks in range hold fewer than `n` ids in all,
+   * nothing is fetched. `through` stops the read and no chunk past it is read, though a `through` inside a chunk may read that last chunk and
+   * yield nothing from it. The reads and the read-ahead window are
+   * those of {@link Segment.iterate}, and the per-op budget is charged before the first fetch, with an upper bound on
+   * the chunks the read can take. A pinned read after the generation is swept throws {@link NotFoundError}.
+   *
+   * The ranks come from the index's counts, which are trusted as {@link Segment.count} trusts them. A chunk that is
+   * read must hold the number of ids its index says, or the read throws {@link IntegrityError}; a chunk that is not
+   * read is not checked, so an index that is wrong about one, and, on an unencrypted object, still adds up to the footer's
+   * total, moves the ids after it to the wrong ranks. An encrypted object's index is authenticated and bound to the
+   * object, so a wrong count there needs the key.
+   *
+   * A live handle is refused with {@link UnsupportedError} at the first read, because its counts and its chunks could
+   * come from two generations and name the wrong id. `n` that is not a positive integer throws
+   * {@link ValidationError} at the first read, as does a bad bound. An expired handle reads empty.
+   *
+   * ```ts
+   * const audience = await store.segment('audience').pin();
+   * const ends: number[] = [];
+   * for await (const id of audience.everyNth(1_000)) ends.push(id); // the end of each window of 1,000
+   * ```
+   */
+  everyNth(n, options) {
+    const lease = this.leaseError();
+    if (lease !== void 0) return failing(lease);
+    if (this.expired()) return EMPTY_IDS;
+    if (this.pinnedAt === void 0) {
+      return failing(
+        new UnsupportedError(
+          "everyNth needs a pinned handle: a live handle can place a rank by one generation and read it from another. Call `pin()` first."
+        )
+      );
+    }
+    const stream = this.engine.everyNth(
+      this.ref,
+      n,
+      options == null ? void 0 : rangeOf(options)
+    );
+    return this.leaseHold === void 0 ? stream : guardIdIterable(stream, () => this.assertLeases([this]));
   }
   /**
    * The options a combine hands the engine, read once, when it is called ({@link readOptions}), with the `exclude`
-   * it was given, already reduced to its live handles ({@link liveExcludes}) and mapped down to the plain refs
+   * it was given ({@link excludesOf}) and mapped down to the plain refs
    * `core` takes. A method rather than a module function because `ref` is class-private — the encapsulation is
    * worth more than the free function.
    *
@@ -1977,18 +2442,35 @@ var Segment = class _Segment {
     };
   }
   /**
-   * The handles in `options.exclude` that have not expired. **An expired exclusion excludes nothing**, in every
-   * combine, so it is dropped here, before the engine is asked for anything: it is never fetched, never
-   * checked for absence, and never counted in the pin-consistency check. `exclude` is read once, so a getter sees
-   * one call.
-   *
-   * With no `exclude` this is one property read and a shared empty list, and with none expired it returns the
-   * caller's own array, so the common case allocates nothing.
+   * The handles in `options.exclude`, read once so a getter sees one call. With none this is one property read and
+   * a shared empty list; otherwise it is the caller's own array, so the common case allocates nothing.
    */
-  liveExcludes(options) {
+  excludesOf(options) {
     const exclude = options?.exclude;
-    if (exclude == null || exclude.length === 0) return NO_SEGMENTS;
-    return exclude.some((e) => e.expired()) ? exclude.filter((e) => !e.expired()) : exclude;
+    return exclude == null || exclude.length === 0 ? NO_SEGMENTS : exclude;
+  }
+  /**
+   * The refusal for a combine whose **exclusion** has expired, or `undefined` when none has. Unlike an expired
+   * operand, which reads empty, an exclusion is a suppression or opt-out list: left out silently, the result
+   * would include the very ids it was passed to remove. So it is refused with a {@link ValidationError} naming each
+   * expired exclusion, as the `*Into` verbs refuse an expired handle.
+   *
+   * It is judged against the injected clock when the combine is called, before the engine is asked for anything:
+   * no request is made, and an expired exclusion that names a segment that does not exist is refused as expired,
+   * not as an absent operand. It runs ahead of the expired-operand shortcuts, so a combine that would read empty
+   * anyway still refuses a lapsed exclusion. A stream already being read is not re-checked when an exclusion
+   * expires part-way, as an operand expiring part-way does not stop it either.
+   */
+  expiredExcludes(op, excludes) {
+    let stale;
+    for (const e of excludes) {
+      if (!e.expired()) continue;
+      (stale ??= []).push(e.ref.namespace ? `${e.ref.namespace}/${e.ref.segment}` : e.ref.segment);
+    }
+    if (stale === void 0) return void 0;
+    return new ValidationError2(
+      `${op}: refusing to read while these exclusions have expired \u2014 ${[...new Set(stale)].join(", ")}. An expired exclusion would subtract nothing, so the result would include the ids it was passed to remove. Renew the exclusion's \`expiresAt\`, open it without one, or leave it out of the call.`
+    );
   }
   /**
    * Chunk-skipping intersection: stream the ids in **this** segment AND every segment in `others`, ascending.
@@ -2000,17 +2482,28 @@ var Segment = class _Segment {
    * override the store's per-op denial-of-wallet budget for this call (or `false` to lift it).
    */
   intersect(others, options) {
-    if (this.expired() || others.some((o) => o.expired())) return EMPTY_IDS;
-    const exclude = this.liveExcludes(options);
+    return this.intersectAs(AS_IDS, others, options);
+  }
+  intersectAs(out, others, options) {
+    const exclude = this.excludesOf(options);
+    const handles = [this, ...others, ...exclude];
+    try {
+      this.assertLeases(handles);
+    } catch (err) {
+      return out.failing(err);
+    }
+    const refused = this.expiredExcludes("intersect", exclude);
+    if (refused) return out.failing(refused);
+    if (this.expired() || others.some((o) => o.expired())) return out.none;
     let engine;
     try {
       engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const refs = [this.ref, ...others.map((o) => o.ref)];
     const opts = this.refsIn(options, exclude);
-    return withBatches(engine.intersect(refs, opts), () => engine.intersectBatches(refs, opts));
+    return this.guarded(out, handles, out.intersect(engine, refs, opts));
   }
   /**
    * Materialize `this ∩ others…` (minus `exclude`) as a **new generation of `dest`** — `dest`'s previous contents
@@ -2037,7 +2530,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...others, ...options?.exclude ?? []]);
     return this.timed(
       "intersectInto",
-      () => this.materialize(dest.ref, this.intersect(others, options), "intersectInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.intersectAs(AS_CHUNKS, others, options)),
+        "intersectInto",
+        options
+      )
     );
   }
   /**
@@ -2051,26 +2549,36 @@ var Segment = class _Segment {
    * segment once (`unionInto`, or a load) is the cheaper shape.
    */
   union(others, options) {
+    return this.unionAs(AS_IDS, others, options);
+  }
+  unionAs(out, others, options) {
+    const exclude = this.excludesOf(options);
+    const handles = [this, ...others, ...exclude];
+    try {
+      this.assertLeases(handles);
+    } catch (err) {
+      return out.failing(err);
+    }
+    const refused = this.expiredExcludes("union", exclude);
+    if (refused) return out.failing(refused);
     const live = others.filter((o) => !o.expired());
     if (this.expired()) {
-      if (live.length === 0) return EMPTY_IDS;
-      return live[0].union(live.slice(1), options);
+      if (live.length === 0) return out.none;
+      return live[0].unionAs(out, live.slice(1), options);
     }
     if (live.length === 0 && others.length > 0) {
-      const exclude2 = this.liveExcludes(options);
-      return exclude2.length > 0 ? this.andNot([...exclude2], options) : this.union([], options);
+      return exclude.length > 0 ? this.andNotAs(out, [...exclude], options) : this.unionAs(out, [], options);
     }
-    if (live.length !== others.length) return this.union(live, options);
-    const exclude = this.liveExcludes(options);
+    if (live.length !== others.length) return this.unionAs(out, live, options);
     let engine;
     try {
       engine = this.combineEngine([this, ...others, ...exclude]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const refs = [this.ref, ...others.map((o) => o.ref)];
     const opts = this.refsIn(options, exclude);
-    return withBatches(engine.union(refs, opts), () => engine.unionBatches(refs, opts));
+    return this.guarded(out, handles, out.union(engine, refs, opts));
   }
   /** Materialize `this ∪ others…` (minus `exclude`) as a **new generation of `dest`** — see {@link intersectInto}. */
   async unionInto(dest, others, options) {
@@ -2078,7 +2586,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...others, ...options?.exclude ?? []]);
     return this.timed(
       "unionInto",
-      () => this.materialize(dest.ref, this.union(others, options), "unionInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.unionAs(AS_CHUNKS, others, options)),
+        "unionInto",
+        options
+      )
     );
   }
   /**
@@ -2089,33 +2602,36 @@ var Segment = class _Segment {
    * suppression list: at most one read per surviving key of `this`, so subtracting a 61,000-chunk global
    * opt-out list from a 40-chunk audience costs at most 40 reads, not 61,000.
    *
-   * An expired handle in `excludes` excludes nothing, and is skipped without being read; if every one has
-   * expired the result is `this` whole. An expired `this` is empty. `exclude` on `intersect` and `union` follows
-   * the same rule.
+   * An expired handle in `excludes` is refused, not skipped: the stream throws {@link ValidationError} naming it,
+   * before any request is made, because a lapsed suppression list would otherwise let through the ids it was
+   * passed to remove. An expired `this` is empty. `exclude` on `intersect` and `union` follows the same rule.
    *
    * To filter the *result of an intersection*, do not chain — pass `exclude` to {@link intersect} instead, so
    * the suppression folds into the same pass rather than materializing an intermediate segment first.
    */
   andNot(excludes, options) {
-    if (this.expired()) return EMPTY_IDS;
-    const liveExcludes = excludes.filter((e) => !e.expired());
-    if (liveExcludes.length === 0 && excludes.length > 0) {
-      return this.union([], options == null ? void 0 : readOptions(options));
+    return this.andNotAs(AS_IDS, excludes, options);
+  }
+  andNotAs(out, excludes, options) {
+    const handles = [this, ...excludes];
+    try {
+      this.assertLeases(handles);
+    } catch (err) {
+      return out.failing(err);
     }
-    if (liveExcludes.length !== excludes.length) return this.andNot(liveExcludes, options);
+    const refused = this.expiredExcludes("andNot", excludes);
+    if (refused) return out.failing(refused);
+    if (this.expired()) return out.none;
     let engine;
     try {
       engine = this.combineEngine([this, ...excludes]) ?? this.engine;
     } catch (err) {
-      return failing(err);
+      return out.failing(err);
     }
     const base = this.ref;
     const refs = excludes.map((o) => o.ref);
     const opts = options == null ? void 0 : readOptions(options);
-    return withBatches(
-      engine.andNot(base, refs, opts),
-      () => engine.andNotBatches(base, refs, opts)
-    );
+    return this.guarded(out, handles, out.andNot(engine, base, refs, opts));
   }
   /** Materialize `this \ (excludes…)` as a **new generation of `dest`** — see {@link intersectInto}. */
   async andNotInto(dest, excludes, options) {
@@ -2123,7 +2639,12 @@ var Segment = class _Segment {
     this.combineEngine([this, ...excludes]);
     return this.timed(
       "andNotInto",
-      () => this.materialize(dest.ref, this.andNot(excludes, options), "andNotInto", options)
+      () => this.materialize(
+        dest.ref,
+        new CombineChunks(this.andNotAs(AS_CHUNKS, excludes, options)),
+        "andNotInto",
+        options
+      )
     );
   }
   /**
@@ -2134,6 +2655,7 @@ var Segment = class _Segment {
    * the lose-zone).
    */
   async costReport(options) {
+    this.assertLeases([this]);
     const canMeasure = this.engine.supportsStorageSize;
     const size = canMeasure ? await this.engine.segmentSize(this.ref) : null;
     const refreshMs = this.engine.pointerRefreshMs;