okengine 0.23.1 → 0.23.2

package/src/elements/store/cache-bus.ts

@@ -0,0 +1,278 @@
+/**
+ * Cross-instance cache invalidation.
+ *
+ * Redis publishes `{ origin, resources }` on `oke:cache:invalidate`.
+ * Postgres, when Redis is not configured, inserts into `oke_cache_invalidations`
+ * and polls on database time. A process applies remote notices with
+ * {@link StoreCache.acceptRemote} and does not publish them again.
+ * Memory and single-instance apps do not broadcast. A missed notice is
+ * bounded by the cache TTL.
+ */
+
+import type { ResourceRef } from "../../manifest/types.ts";
+import type { StoreCache } from "./cache.ts";
+
+/** Redis pub/sub channel for cache invalidation. */
+export const CACHE_INVALIDATE_CHANNEL = "oke:cache:invalidate";
+
+/** Postgres table for cache invalidation when Redis is not configured. */
+export const CACHE_INVALIDATIONS_TABLE = "oke_cache_invalidations";
+
+/** Default overlap when polling the Postgres invalidation table. */
+const DEFAULT_LAG_MS = 30_000;
+
+/** Drop Postgres rows older than this. */
+const RETAIN_MS = 120_000;
+
+/** Redis commands the bus needs. */
+export interface CacheBusRedis {
+  /**
+   * Publish one message.
+   *
+   * @param channel - Channel name
+   * @param message - JSON payload
+   */
+  publish(channel: string, message: string): Promise<unknown>;
+  /**
+   * Subscribe. The returned function removes the listener.
+   *
+   * @param channel - Channel name
+   * @param listener - Message callback
+   */
+  subscribe(channel: string, listener: (message: string) => void): Promise<() => void>;
+}
+
+/** SQL surface the Postgres bus uses. Placeholders are `?`. */
+export interface CacheBusSql {
+  /**
+   * Run a query and return rows.
+   *
+   * @param sql - Statement
+   * @param params - Bound parameters
+   */
+  query(sql: string, params?: readonly unknown[]): Promise<ReadonlyArray<Record<string, unknown>>>;
+  /**
+   * Run a statement.
+   *
+   * @param sql - Statement
+   * @param params - Bound parameters
+   */
+  exec(sql: string, params?: readonly unknown[]): Promise<{ changes: number }>;
+}
+
+/** How {@link openCacheInvalidation} reaches other instances. */
+export type CacheBusOptions =
+  | {
+      readonly kind: "redis";
+      readonly redis: CacheBusRedis;
+      /** Sender id. Remote notices with this origin are ignored. */
+      readonly origin?: string;
+    }
+  | {
+      readonly kind: "postgres";
+      readonly sql: () => Promise<CacheBusSql | undefined>;
+      readonly origin?: string;
+      /** Overlap behind the cursor. Default 30_000. */
+      readonly lagMs?: number;
+    };
+
+/** Handle returned by {@link openCacheInvalidation}. */
+export interface CacheInvalidationBus {
+  /** Tell other instances these resources changed. */
+  publish(resources: readonly ResourceRef[]): void;
+  /** Pull Postgres notices. No-op for Redis. */
+  poll(): Promise<void>;
+  /** Stop the subscription or poll. */
+  stop(): Promise<void>;
+}
+
+/**
+ * Bind `cache` to Redis or Postgres invalidation.
+ *
+ * @param cache - Local cache
+ * @param options - Transport
+ */
+export function openCacheInvalidation(
+  cache: StoreCache,
+  options: CacheBusOptions,
+): CacheInvalidationBus {
+  const origin = options.origin ?? crypto.randomUUID();
+  if (options.kind === "redis") return openRedis(cache, options.redis, origin);
+  return openPostgres(cache, options.sql, origin, options.lagMs ?? DEFAULT_LAG_MS);
+}
+
+/**
+ * Bun Redis client for the invalidation channel.
+ * Subscribe uses its own connection.
+ *
+ * @param url - Redis URL
+ */
+export function bunRedisCacheClient(url: string): CacheBusRedis {
+  const pub = new Bun.RedisClient(url);
+  const sub = new Bun.RedisClient(url);
+  return {
+    publish: (channel, message) => pub.publish(channel, message),
+    async subscribe(channel, listener) {
+      await sub.subscribe(channel, (message) => {
+        listener(message);
+      });
+      return () => {
+        void sub.unsubscribe(channel);
+      };
+    },
+  };
+}
+
+function openRedis(cache: StoreCache, redis: CacheBusRedis, origin: string): CacheInvalidationBus {
+  let unsub: (() => void) | undefined;
+  let stopped = false;
+  const ready = redis
+    .subscribe(CACHE_INVALIDATE_CHANNEL, (message) => {
+      if (stopped) return;
+      const resources = resourcesFromMessage(message, origin);
+      if (resources.length > 0) cache.acceptRemote(resources);
+    })
+    .then((stop) => {
+      unsub = stop;
+    })
+    .catch((err: unknown) => {
+      console.error(err);
+    });
+  return {
+    publish(resources) {
+      if (stopped || resources.length === 0) return;
+      const body = JSON.stringify({ origin, resources });
+      void redis.publish(CACHE_INVALIDATE_CHANNEL, body).catch((err: unknown) => {
+        console.error(err);
+      });
+    },
+    poll: async () => {},
+    async stop() {
+      stopped = true;
+      await ready;
+      unsub?.();
+      unsub = undefined;
+    },
+  };
+}
+
+function openPostgres(
+  cache: StoreCache,
+  connect: () => Promise<CacheBusSql | undefined>,
+  origin: string,
+  lagMs: number,
+): CacheInvalidationBus {
+  let stopped = false;
+  let ready: Promise<void> | undefined;
+  let cursor = 0;
+  let head = true;
+  const seen = new Set<string>();
+
+  async function sql(): Promise<CacheBusSql | undefined> {
+    if (stopped) return undefined;
+    const conn = await connect();
+    if (!conn || stopped) return undefined;
+    if (!ready) ready = ensure(conn);
+    await ready;
+    return conn;
+  }
+
+  return {
+    publish(resources) {
+      if (stopped || resources.length === 0) return;
+      void insert(sql, origin, resources);
+    },
+    async poll() {
+      const conn = await sql();
+      if (!conn || stopped) return;
+      const clock = await conn.query(DB_NOW_SQL);
+      const dbNow = asNumber(clock[0]?.db_now);
+      const since = head ? 0 : cursor - lagMs;
+      const rows = await conn.query(POLL_SQL, [since]);
+      const fresh: ResourceRef[] = [];
+      for (const row of rows) {
+        const id = asString(row.id);
+        if (!id || seen.has(id)) continue;
+        seen.add(id);
+        if (head || asString(row.origin) === origin) continue;
+        const resource = asString(row.resource);
+        if (resource) fresh.push(resource as ResourceRef);
+      }
+      if (!head && fresh.length > 0) cache.acceptRemote(fresh);
+      head = false;
+      cursor = dbNow;
+      await conn.exec(DELETE_SQL, [dbNow - RETAIN_MS]);
+      if (seen.size > 10_000) {
+        seen.clear();
+      }
+    },
+    async stop() {
+      stopped = true;
+    },
+  };
+}
+
+async function ensure(conn: CacheBusSql): Promise<void> {
+  await conn.exec(CREATE_SQL);
+  await conn.exec(INDEX_SQL);
+}
+
+async function insert(
+  connect: () => Promise<CacheBusSql | undefined>,
+  origin: string,
+  resources: readonly ResourceRef[],
+): Promise<void> {
+  const conn = await connect();
+  if (!conn) return;
+  for (const resource of resources) {
+    await conn.exec(INSERT_SQL, [crypto.randomUUID(), resource, origin]);
+  }
+}
+
+function resourcesFromMessage(message: string, origin: string): ResourceRef[] {
+  try {
+    const parsed: unknown = JSON.parse(message);
+    if (parsed === null || typeof parsed !== "object") return [];
+    const rec = parsed as Record<string, unknown>;
+    if (rec.origin === origin) return [];
+    if (!Array.isArray(rec.resources)) return [];
+    const out: ResourceRef[] = [];
+    for (const item of rec.resources) {
+      if (typeof item === "string" && item.length > 0) out.push(item as ResourceRef);
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
+function asString(value: unknown): string {
+  return typeof value === "string" ? value : "";
+}
+
+function asNumber(value: unknown): number {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "bigint") return Number(value);
+  if (typeof value === "string" && value.trim() !== "") {
+    const n = Number(value);
+    if (Number.isFinite(n)) return n;
+  }
+  return 0;
+}
+
+const CREATE_SQL = `CREATE TABLE IF NOT EXISTS ${CACHE_INVALIDATIONS_TABLE} (
+  id TEXT PRIMARY KEY,
+  resource TEXT NOT NULL,
+  origin TEXT NOT NULL,
+  created_at BIGINT NOT NULL DEFAULT ((extract(epoch from clock_timestamp()) * 1000)::bigint)
+)`;
+
+const INDEX_SQL = `CREATE INDEX IF NOT EXISTS ${CACHE_INVALIDATIONS_TABLE}_created_at ON ${CACHE_INVALIDATIONS_TABLE} (created_at)`;
+
+const INSERT_SQL = `INSERT INTO ${CACHE_INVALIDATIONS_TABLE} (id, resource, origin) VALUES (?, ?, ?)`;
+
+const DB_NOW_SQL = `SELECT (extract(epoch from clock_timestamp()) * 1000)::bigint AS db_now`;
+
+const POLL_SQL = `SELECT id, resource, origin, created_at FROM ${CACHE_INVALIDATIONS_TABLE} WHERE created_at >= ?`;
+
+const DELETE_SQL = `DELETE FROM ${CACHE_INVALIDATIONS_TABLE} WHERE created_at < ?`;

package/src/elements/store/cache.test.ts

@@ -258,3 +258,62 @@ describe("autoCacheEligible", () => {
     expect(tier1Lookup((key) => cache.get<number>(key), ["a", "b"])).toBe(1);
   });
 });
+
+describe("tier-1 cache bounds", () => {
+  test("LRU drops the oldest entry past the cap and a remote invalidation is not republished", () => {
+    const seen: string[][] = [];
+    let now = 1_000;
+    const cache = createStoreCache({
+      now: () => now,
+      maxEntries: 2,
+      fanout: (resources) => {
+        seen.push([...resources]);
+      },
+    });
+    cache.set({ tier: 1, key: "a", value: 1, resources: ["sql:a"], expiresAt: now + 60_000 });
+    cache.set({ tier: 1, key: "b", value: 2, resources: ["sql:b"], expiresAt: now + 60_000 });
+    cache.get("a");
+    cache.set({ tier: 1, key: "c", value: 3, resources: ["sql:c"], expiresAt: now + 60_000 });
+    expect(cache.get<number>("b")).toBeUndefined();
+    expect(cache.get<number>("a")).toBe(1);
+    cache.invalidate(["sql:a"]);
+    expect(seen).toEqual([["sql:a"]]);
+    expect(cache.generationOf("sql:a")).toBe(1);
+    cache.acceptRemote(["sql:c"]);
+    expect(seen).toEqual([["sql:a"]]);
+    expect(cache.get("c")).toBeUndefined();
+  });
+
+  test("a flight that is invalidated does not cache, and the waiter refetches", async () => {
+    const cache = createStoreCache({ maxEntries: 10 });
+    let reads = 0;
+    let release!: () => void;
+    const gate = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    const leader = cache.coalesce("k", ["sql:notes"], async () => {
+      reads += 1;
+      await gate;
+      return reads;
+    });
+    const waiter = cache.coalesce("k", ["sql:notes"], async () => {
+      reads += 1;
+      return reads;
+    });
+    cache.invalidate(["sql:notes"]);
+    release();
+    const first = await leader;
+    const second = await waiter;
+    expect(first.cacheable).toBe(false);
+    expect(second.value).not.toBe(first.value);
+    expect(reads).toBe(2);
+  });
+
+  test("verified and operator are part of the key", () => {
+    const anon = tier1FlowDims("read", {}, { verified: false, operatorId: null });
+    const verified = tier1FlowDims("read", {}, { verified: true, operatorId: "op-1" });
+    expect(anon.join("/")).not.toBe(verified.join("/"));
+    expect(verified.join("/")).toContain("v:1");
+    expect(verified.join("/")).toContain("o:op-1");
+  });
+});

package/src/elements/store/cache.ts

@@ -81,6 +81,21 @@ export function isInvalidatedByWrite(key: string, writeEffects: Effects): boolea
   return (writeEffects.writes ?? []).some((w) => w === resource);
 }
 
+/** Options for {@link createStoreCache}. */
+export interface CreateStoreCacheOptions {
+  /** Clock for TTL expiry. */
+  readonly now?: () => number;
+  /** Entry cap. Default 10_000. Oldest entry is dropped. */
+  readonly maxEntries?: number;
+  /** TTL applied when a put does not set one. Default 60_000. */
+  readonly defaultTtlMs?: number;
+  /**
+   * Tell other instances which resources changed. Not called for
+   * {@link StoreCache.acceptRemote}.
+   */
+  readonly fanout?: (resources: readonly ResourceRef[]) => void;
+}
+
 /** In-memory multi-tier cache used by the store runtime and tests. */
 export interface StoreCache {
   /**
@@ -111,6 +126,32 @@ export interface StoreCache {
   keys(): string[];
   /** Clear all tiers. */
   clear(): void;
+  /** Default TTL for puts that omit one. */
+  readonly defaultTtlMs: number;
+  /** Current generation for a resource. Invalidation increments it. */
+  generationOf(resource: string): number;
+  /** True when every resource is still at the captured generation. */
+  sameGeneration(snapshot: Readonly<Record<string, number>>): boolean;
+  /**
+   * Apply an invalidation that arrived from another instance.
+   * Does not call `fanout`.
+   *
+   * @param resources - Resources the other instance wrote
+   */
+  acceptRemote(resources: readonly ResourceRef[]): InvalidationEvent;
+  /**
+   * Run `task` once per key. A caller that joined before an invalidation
+   * refetches once instead of keeping a value the cache will not store.
+   *
+   * @param key - Flight key
+   * @param resources - Resources read by the flight
+   * @param task - Loader
+   */
+  coalesce<T>(
+    key: string,
+    resources: readonly string[],
+    task: () => Promise<T>,
+  ): Promise<{ readonly value: T; readonly cacheable: boolean }>;
 }
 
 /**
@@ -118,37 +159,133 @@ export interface StoreCache {
  *
  * @param now - Clock for TTL expiry
  */
-export function createStoreCache(now: () => number = () => Date.now()): StoreCache {
+export function createStoreCache(
+  nowOrOptions: (() => number) | CreateStoreCacheOptions = {},
+): StoreCache {
+  const options: CreateStoreCacheOptions =
+    typeof nowOrOptions === "function" ? { now: nowOrOptions } : nowOrOptions;
+  const now = options.now ?? (() => Date.now());
+  const maxEntries = options.maxEntries ?? 10_000;
+  const defaultTtlMs = options.defaultTtlMs ?? 60_000;
   const entries = new Map<string, CacheEntry>();
+  const byResource = new Map<string, Set<string>>();
+  const generations = new Map<string, number>();
+  const flights = new Map<
+    string,
+    {
+      readonly promise: Promise<unknown>;
+      readonly resources: readonly string[];
+      readonly generation: Readonly<Record<string, number>>;
+    }
+  >();
 
   function alive(entry: CacheEntry): boolean {
     return entry.expiresAt === null || entry.expiresAt > now();
   }
 
-  return {
+  function forget(key: string): void {
+    const entry = entries.get(key);
+    entries.delete(key);
+    if (!entry) return;
+    for (const resource of entry.resources) byResource.get(resource)?.delete(key);
+  }
+
+  function snapshot(resources: readonly string[]): Record<string, number> {
+    const out: Record<string, number> = {};
+    for (const resource of resources) out[resource] = generations.get(resource) ?? 0;
+    return out;
+  }
+
+  function dropFlights(resources: readonly ResourceRef[]): void {
+    const set = new Set(resources);
+    for (const [key, flight] of flights) {
+      if (flight.resources.some((resource) => set.has(resource as ResourceRef))) {
+        flights.delete(key);
+      }
+    }
+  }
+
+  function invalidateLocal(resources: readonly ResourceRef[]): InvalidationEvent {
+    const keys: string[] = [];
+    for (const resource of resources) {
+      generations.set(resource, (generations.get(resource) ?? 0) + 1);
+      const held = byResource.get(resource);
+      if (!held) continue;
+      for (const key of held) {
+        keys.push(key);
+        forget(key);
+      }
+      held.clear();
+    }
+    dropFlights(resources);
+    return { resources: [...resources], keys };
+  }
+
+  const api: StoreCache = {
+    defaultTtlMs,
     get<T = unknown>(key: string): T | undefined {
       const entry = entries.get(key);
       if (!entry) return undefined;
       if (!alive(entry)) {
-        entries.delete(key);
+        forget(key);
         return undefined;
       }
+      entries.delete(key);
+      entries.set(key, entry);
       return entry.value as T;
     },
     set<T>(entry: CacheEntry<T>): void {
+      forget(entry.key);
       entries.set(entry.key, entry as CacheEntry);
+      for (const resource of entry.resources) {
+        const set = byResource.get(resource) ?? new Set<string>();
+        set.add(entry.key);
+        byResource.set(resource, set);
+      }
+      while (entries.size > maxEntries) {
+        const oldest = entries.keys().next().value;
+        if (oldest === undefined) break;
+        forget(oldest);
+      }
     },
     invalidate(resources: readonly ResourceRef[]): InvalidationEvent {
-      const set = new Set(resources);
-      const keys: string[] = [];
-      for (const [key, entry] of entries) {
-        if (entry.tier !== 1) continue;
-        if (entry.resources.some((r) => set.has(r))) {
-          keys.push(key);
-          entries.delete(key);
-        }
+      const event = invalidateLocal(resources);
+      options.fanout?.(resources);
+      return event;
+    },
+    generationOf(resource: string): number {
+      return generations.get(resource) ?? 0;
+    },
+    sameGeneration(shot: Readonly<Record<string, number>>): boolean {
+      for (const [resource, generation] of Object.entries(shot)) {
+        if ((generations.get(resource) ?? 0) !== generation) return false;
+      }
+      return true;
+    },
+    acceptRemote(resources: readonly ResourceRef[]): InvalidationEvent {
+      return invalidateLocal(resources);
+    },
+    async coalesce<T>(
+      key: string,
+      resources: readonly string[],
+      task: () => Promise<T>,
+    ): Promise<{ readonly value: T; readonly cacheable: boolean }> {
+      const existing = flights.get(key);
+      if (existing) {
+        const value = (await existing.promise) as T;
+        if (api.sameGeneration(existing.generation)) return { value, cacheable: true };
+        const again = await task();
+        return { value: again, cacheable: api.sameGeneration(snapshot(resources)) };
+      }
+      const generation = snapshot(resources);
+      const promise = task();
+      flights.set(key, { promise, resources, generation });
+      try {
+        const value = await promise;
+        return { value, cacheable: api.sameGeneration(generation) };
+      } finally {
+        if (flights.get(key)?.promise === promise) flights.delete(key);
       }
-      return { resources: [...resources], keys };
     },
     invalidateFromEffects(writeEffects: Effects): InvalidationEvent {
       return this.invalidate(resourcesTouchedByWrites(writeEffects));
@@ -158,8 +295,10 @@ export function createStoreCache(now: () => number = () => Date.now()): StoreCac
     },
     clear(): void {
       entries.clear();
+      byResource.clear();
     },
   };
+  return api;
 }
 
 /** Store resource refs that participate in the automatic cache cycle. */
@@ -400,6 +539,10 @@ export interface Tier1Caller {
    * Membership role names. Cache identity only — gates do not read this.
    */
   readonly roles?: Iterable<string>;
+  /** Email / session verified bit. `v:0` and `v:1` must not share an entry. */
+  readonly verified?: boolean;
+  /** Operator id when the caller is on the operator plane. */
+  readonly operatorId?: string | null;
 }
 
 /**
@@ -457,6 +600,8 @@ export function tier1FlowDims(
     `l:${identity.locale ?? ""}`,
     `s:${scopes.join(",")}`,
     `r:${roles.join(",")}`,
+    `v:${identity.verified === true ? "1" : "0"}`,
+    `o:${identity.operatorId ?? ""}`,
   ];
 }
 

package/src/elements/store/runtime.ts

@@ -33,6 +33,7 @@ import {
   tier1KeysForReads,
   type StoreCache,
 } from "./cache.ts";
+import { openCacheInvalidation, type CacheBusOptions } from "./cache-bus.ts";
 import { resolveSqlTarget, type SqlBindingConfig } from "./replica.ts";
 import { createSqlStoreHandle, type SqlStoreHandle } from "./sql-session.ts";
 import { classificationsFromTable, type TableHandle } from "./table.ts";
@@ -96,6 +97,15 @@ export interface CreateStoreRuntimeOptions {
   >;
   /** Clock for cache TTLs. */
   readonly now?: () => number;
+  /** Auto-cache entry cap. Default 10_000. */
+  readonly cacheMaxEntries?: number;
+  /** Auto-cache TTL when a flow sets none. Default 60_000. */
+  readonly cacheDefaultTtlMs?: number;
+  /**
+   * Cross-instance invalidation. Omit for a single process.
+   * Redis is push. Postgres is polled from the scheduler.
+   */
+  readonly cacheBus?: CacheBusOptions;
   /**
    * Domain DDL policy for SQL handles. Default `ensure` (test-friendly).
    * Boot sets `off` for docker/prod and local+autoPush.
@@ -277,6 +287,13 @@ export interface StoreRuntime {
   ): string[];
   /** Close all open connections. */
   close(): Promise<void>;
+  /**
+   * Pull Postgres cache invalidations. No-op unless {@link CreateStoreRuntimeOptions.cacheBus}
+   * is `postgres`.
+   */
+  pollCacheInvalidations(): Promise<void>;
+  /** Which invalidation transport this runtime opened. */
+  readonly cacheInvalidation: "off" | "redis" | "postgres";
   /**
    * Capability-gated files handle for `fx.store` (CRUD + image pipeline).
    *
@@ -300,7 +317,26 @@ export interface StoreRuntime {
  */
 export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRuntime {
   const now = options.now ?? (() => Date.now());
-  const cache = createStoreCache(now);
+  let publishInvalidation: ((resources: readonly ResourceRef[]) => void) | undefined;
+  const cache = createStoreCache({
+    now,
+    fanout: (resources) => {
+      publishInvalidation?.(resources);
+    },
+    ...(options.cacheMaxEntries !== undefined ? { maxEntries: options.cacheMaxEntries } : {}),
+    ...(options.cacheDefaultTtlMs !== undefined ? { defaultTtlMs: options.cacheDefaultTtlMs } : {}),
+  });
+  const cacheInvalidation = options.cacheBus?.kind ?? "off";
+  let pollInvalidation: (() => Promise<void>) | undefined;
+  let stopInvalidation: (() => Promise<void>) | undefined;
+  if (options.cacheBus) {
+    const bus = openCacheInvalidation(cache, options.cacheBus);
+    publishInvalidation = (resources) => {
+      bus.publish(resources);
+    };
+    pollInvalidation = () => bus.poll();
+    stopInvalidation = () => bus.stop();
+  }
   const declarations = new Map<string, StoreDecl>();
   const sqlConns = new Map<string, SqlConnection>();
   const kvNs = new Map<string, KvNamespace>();
@@ -570,7 +606,8 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
     putTier1(effects, value, dimsByResource, ttlMs) {
       const keys = tier1KeysForReads(effects, dimsByResource);
       const resources = (effects.reads ?? []).filter((r): r is ResourceRef => r !== "runs");
-      const expiresAt = ttlMs !== undefined && ttlMs !== null && ttlMs > 0 ? now() + ttlMs : null;
+      const ttl = ttlMs !== undefined && ttlMs !== null && ttlMs > 0 ? ttlMs : cache.defaultTtlMs;
+      const expiresAt = now() + ttl;
       for (const key of keys) {
         cache.set({
           tier: 1,
@@ -582,7 +619,12 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
       }
       return keys;
     },
+    cacheInvalidation,
+    async pollCacheInvalidations() {
+      if (pollInvalidation) await pollInvalidation();
+    },
     async close() {
+      if (stopInvalidation) await stopInvalidation();
       for (const c of sqlConns.values()) await c.close();
       for (const n of kvNs.values()) await n.close();
       for (const b of fileBuckets.values()) await b.close();