@tinycloud/replica 0.3.0-beta.6 → 0.3.0-beta.8

package/dist/sqlite.cjs

@@ -726,6 +726,76 @@ var isHash = (name) => /^[0-9a-f]{64}$/.test(name);
 function errnoOf(error) {
   return typeof error === "object" && error !== null && "code" in error ? error.code : void 0;
 }
+var isShard = (name) => /^[0-9a-f]{2}$/.test(name);
+var isTempName = (name) => /^[0-9a-f]{64}\.[0-9a-f]{12}$/.test(name);
+async function writerBlobFiles(root) {
+  const out = [];
+  let shards;
+  try {
+    shards = await (0, import_promises2.readdir)(root, { withFileTypes: true });
+  } catch (error) {
+    if (errnoOf(error) === "ENOENT" || errnoOf(error) === "ENOTDIR") return out;
+    throw error;
+  }
+  for (const shard of shards) {
+    const temp = shard.name === ".tmp";
+    if (!shard.isDirectory() || !(temp || isShard(shard.name))) continue;
+    const shardPath = (0, import_node_path2.join)(root, shard.name);
+    let files;
+    try {
+      files = await (0, import_promises2.readdir)(shardPath, { withFileTypes: true });
+    } catch (error) {
+      if (errnoOf(error) === "ENOENT" || errnoOf(error) === "ENOTDIR") continue;
+      throw error;
+    }
+    for (const file of files) {
+      if (!file.isFile()) continue;
+      const ok = temp ? isTempName(file.name) : isHash(file.name) && file.name.startsWith(shard.name);
+      if (!ok) continue;
+      const path = (0, import_node_path2.join)(shardPath, file.name);
+      const info = await (0, import_promises2.lstat)(path).catch(() => void 0);
+      if (!info?.isFile()) continue;
+      out.push({ path, hash: temp ? null : file.name, size: info.size });
+    }
+  }
+  return out;
+}
+async function unlinkWriterOrphans(root, keep) {
+  let removed = 0;
+  const touched = /* @__PURE__ */ new Set();
+  for (const file of await writerBlobFiles(root)) {
+    if (file.hash !== null && keep.has(file.hash)) continue;
+    await (0, import_promises2.rm)(file.path, { force: true });
+    touched.add((0, import_node_path2.dirname)(file.path));
+    removed += 1;
+  }
+  for (const dir of touched) if (!dir.endsWith(".tmp")) await (0, import_promises2.rmdir)(dir).catch(() => void 0);
+  return removed;
+}
+async function unlinkBlobsIn(root, keep) {
+  let removed = 0;
+  for (const shard of await entriesOf(root)) {
+    const shardPath = (0, import_node_path2.join)(root, shard);
+    if (shard === ".tmp") {
+      for (const name of await entriesOf(shardPath)) {
+        await (0, import_promises2.rm)((0, import_node_path2.join)(shardPath, name), { force: true });
+        removed += 1;
+      }
+      continue;
+    }
+    let kept = 0;
+    for (const name of await entriesOf(shardPath)) {
+      if (isHash(name) && keep.has(name)) {
+        kept += 1;
+        continue;
+      }
+      await (0, import_promises2.rm)((0, import_node_path2.join)(shardPath, name), { force: true });
+      removed += 1;
+    }
+    if (kept === 0) await (0, import_promises2.rmdir)(shardPath).catch(() => void 0);
+  }
+  return removed;
+}
 async function entriesOf(path) {
   try {
     return await (0, import_promises2.readdir)(path);
@@ -789,6 +859,8 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
   #guard;
   /** Identity of the database file this store opened: a deleted-and-recreated replica is a different file. */
   #ino;
+  #admit;
+  #partitionId;
   #dev;
   #namedAncestors;
   constructor(dir, db, identity, options, directoryFd, namedAncestors = [], shared, readonlySnapshot = false) {
@@ -808,6 +880,8 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
     this.#dev = identity.dev;
     this.#now = options.now ?? Date.now;
     this.#faults = options[FAULTS] ?? {};
+    this.#admit = options.admit;
+    this.#partitionId = options.partitionId ?? "";
     this.#guard = options.guard ?? ((section) => section());
     this.#namedAncestors = namedAncestors;
   }
@@ -877,6 +951,55 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
       }
     }
   }
+  /**
+   * Blob content bytes on disk in the replica at `dir` (0 when absent): the writer's blob files under `blobs/`,
+   * including writer temp files and blobs no entry references any more, so garbage a failed sync
+   * left behind still counts. Counts exactly what `collectOrphans` may delete: foreign entries
+   * (stray files, symlinks, unknown names) are neither counted nor touched. Only blob content is counted: not the database, its WAL, rows or the
+   * pending store. Filesystem-only: it opens no
+   * database, so it can sum a profile's replicas under the caller's own mutation hold.
+   */
+  static async contentBytes(dir) {
+    try {
+      let bytes = 0;
+      for (const file of await writerBlobFiles((0, import_node_path2.join)(dir, "blobs"))) bytes += file.size;
+      return bytes;
+    } catch (error) {
+      throw storageError(error, "Measuring replica content");
+    }
+  }
+  /**
+   * Remove blobs and temp files in the replica at `dir` that no committed row references: what a
+   * crash between writing blobs and committing (or between committing and collecting) leaves.
+   * Returns the number of files removed; 0 when the replica is absent or unreadable.
+   *
+   * The caller MUST hold the profile mutation guard, which every writer of this replica holds
+   * across its blob write and commit; that is what makes an unreferenced blob safe to delete
+   * (it cannot belong to an in-flight write). Never call it without that hold. The database is
+   * read only, so nothing here can take a lease or change committed state.
+   */
+  static async collectOrphans(dir) {
+    const dbPath = (0, import_node_path2.join)(dir, "replica.db");
+    const identity = await (0, import_promises2.lstat)(dbPath).catch(() => void 0);
+    if (!identity?.isFile()) return 0;
+    const opener = await loadSqlite();
+    let keep;
+    let db;
+    try {
+      db = opener(dbPath, { readonly: true });
+      db.exec("PRAGMA busy_timeout = 0");
+      keep = new Set(db.all("SELECT DISTINCT hash FROM entry WHERE content = 1").map((row) => row.hash));
+    } catch {
+      return 0;
+    } finally {
+      db?.close();
+    }
+    try {
+      return await unlinkWriterOrphans((0, import_node_path2.join)(dir, "blobs"), keep);
+    } catch (error) {
+      throw storageError(error, "Collecting orphaned content");
+    }
+  }
   /**
    * Open the replica store in `dir`. With `create: false` a missing replica is
    * REPLICA_NOT_FOUND and nothing is written to disk.
@@ -1366,12 +1489,15 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
   }
   #storeGrantPolicy(g, policy, existing) {
     const installedAt = existing?.installedAt ?? policy.at;
+    const keepExisting = existing !== null && existing.consentDeadline < policy.consentDeadline;
+    const consentDeadline = keepExisting ? existing.consentDeadline : policy.consentDeadline;
+    const consentAnchor = keepExisting ? existing.consentAnchor : policy.consentAnchor;
     const effectiveExpiresAt = Math.min(
       g.expiresAt === null ? Infinity : g.expiresAt * 1e3,
       installedAt + policy.ttlMs,
-      policy.consentDeadline
+      consentDeadline
     );
-    if (!Number.isSafeInteger(installedAt) || installedAt < 0 || !Number.isSafeInteger(effectiveExpiresAt) || effectiveExpiresAt < 0 || !Number.isSafeInteger(policy.consentDeadline) || policy.consentDeadline < 0 || !Number.isSafeInteger(policy.consentAnchor.at) || policy.consentAnchor.at < 0 || !Number.isSafeInteger(policy.ttlMs) || policy.ttlMs <= 0) {
+    if (!Number.isSafeInteger(installedAt) || installedAt < 0 || !Number.isSafeInteger(effectiveExpiresAt) || effectiveExpiresAt < 0 || !Number.isSafeInteger(consentDeadline) || consentDeadline < 0 || !Number.isSafeInteger(consentAnchor.at) || consentAnchor.at < 0 || !Number.isSafeInteger(policy.ttlMs) || policy.ttlMs <= 0) {
       throw new ReplicaError(ReplicaErrorCode.GRANT_EXPIRED, "The replica grant has no finite effective expiry.");
     }
     this.#db.run(
@@ -1383,11 +1509,11 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
       g.cid,
       installedAt,
       effectiveExpiresAt,
-      policy.consentDeadline,
-      policy.consentAnchor.at,
-      policy.consentAnchor.source
+      consentDeadline,
+      consentAnchor.at,
+      consentAnchor.source
     );
-    return { installedAt, effectiveExpiresAt, consentDeadline: policy.consentDeadline, consentAnchor: policy.consentAnchor };
+    return { installedAt, effectiveExpiresAt, consentDeadline, consentAnchor };
   }
   async installGatedGrant(g, policy) {
     return this.#guardedWrite("Installing the gated grant", () => {
@@ -1583,6 +1709,41 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
         }
         reused.set(change.hash, size);
       }
+      if (this.#admit) {
+        await this.#collectLocked(t);
+        const changed = new Set(p.changes.map((change) => change.key));
+        const kept = /* @__PURE__ */ new Set();
+        for (const change of p.changes) if (!change.deleted && change.content) kept.add(change.hash);
+        let blobBytes = 0;
+        for (const [hash, bytes] of p.blobs) {
+          const stored = this.#db.get(
+            "SELECT 1 AS hit FROM entry WHERE hash = ? AND content = 1 AND deleted = 0 LIMIT 1",
+            hash
+          );
+          if (stored === void 0) blobBytes += bytes.byteLength;
+        }
+        const released = /* @__PURE__ */ new Map();
+        for (const key of changed) {
+          const old = this.#db.get(
+            "SELECT hash, size FROM entry WHERE key = ? AND deleted = 0 AND content = 1 AND hash IS NOT NULL AND size IS NOT NULL",
+            key
+          );
+          if (old === void 0 || kept.has(old.hash) || released.has(old.hash)) continue;
+          const holders = this.#db.all(
+            "SELECT key FROM entry WHERE hash = ? AND content = 1 AND deleted = 0",
+            old.hash
+          );
+          if (holders.every((holder) => changed.has(holder.key))) released.set(old.hash, old.size);
+        }
+        let freed = 0;
+        for (const size of released.values()) freed += size;
+        await this.#admit({
+          partitionId: this.#partitionId,
+          blobBytes: blobBytes - freed,
+          entries: p.changes.length,
+          freeBytes: null
+        });
+      }
       try {
         await this.#writeBlobs(p.blobs);
       } catch (error) {
@@ -1658,36 +1819,16 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
         });
       } catch (error) {
         if (isReplicaError(error, ReplicaErrorCode.GRANT_REVOKED)) await this.#purgeContent().catch(() => void 0);
+        else if (this.#admit) await this.#collectLocked(t).catch(() => void 0);
         throw error;
       }
       this.#faults.afterCommit?.();
+      if (this.#admit) await this.#collectLocked(t);
     });
   }
   /** Unlink every blob not in `keep`, and every temp file. Only a missing directory counts as empty. */
-  async #unlinkBlobs(keep) {
-    let removed = 0;
-    const root = (0, import_node_path2.join)(this.#pinnedDir, "blobs");
-    for (const shard of await entriesOf(root)) {
-      const shardPath = (0, import_node_path2.join)(root, shard);
-      if (shard === ".tmp") {
-        for (const name of await entriesOf(shardPath)) {
-          await (0, import_promises2.rm)((0, import_node_path2.join)(shardPath, name), { force: true });
-          removed += 1;
-        }
-        continue;
-      }
-      let kept = 0;
-      for (const name of await entriesOf(shardPath)) {
-        if (isHash(name) && keep.has(name)) {
-          kept += 1;
-          continue;
-        }
-        await (0, import_promises2.rm)((0, import_node_path2.join)(shardPath, name), { force: true });
-        removed += 1;
-      }
-      if (kept === 0) await (0, import_promises2.rmdir)(shardPath).catch(() => void 0);
-    }
-    return removed;
+  #unlinkBlobs(keep) {
+    return unlinkBlobsIn((0, import_node_path2.join)(this.#pinnedDir, "blobs"), keep);
   }
   /** Whether any content file (a blob or a temp file) is on disk. */
   async #holdsContentFiles() {
@@ -1698,33 +1839,35 @@ var SqliteReplicaStore = class _SqliteReplicaStore {
     return false;
   }
   async collectGarbage(t) {
-    return this.#mutate(async () => {
-      const release = this.#shared ? await acquireWritableMutation(this.#shared) : void 0;
-      try {
-        this.#db.exec("BEGIN IMMEDIATE");
-      } catch (error) {
-        release?.();
-        throw storageError(error, "Collecting unused content");
-      }
+    return this.#mutate(() => this.#collectLocked(t));
+  }
+  /** `collectGarbage` for callers already inside the mutation hold (it cannot re-enter `#mutate`). */
+  async #collectLocked(t) {
+    const release = this.#shared ? await acquireWritableMutation(this.#shared) : void 0;
+    try {
+      this.#db.exec("BEGIN IMMEDIATE");
+    } catch (error) {
+      release?.();
+      throw storageError(error, "Collecting unused content");
+    }
+    try {
+      this.#checkLease(t);
+      await this.#faults.afterGarbageCollectionBegin?.();
+      const keep = new Set(
+        this.#db.all("SELECT DISTINCT hash FROM entry WHERE content = 1").map((row) => row.hash)
+      );
+      const removed = await this.#unlinkBlobs(keep);
+      this.#db.exec("COMMIT");
+      return removed;
+    } catch (error) {
       try {
-        this.#checkLease(t);
-        await this.#faults.afterGarbageCollectionBegin?.();
-        const keep = new Set(
-          this.#db.all("SELECT DISTINCT hash FROM entry WHERE content = 1").map((row) => row.hash)
-        );
-        const removed = await this.#unlinkBlobs(keep);
-        this.#db.exec("COMMIT");
-        return removed;
-      } catch (error) {
-        try {
-          this.#db.exec("ROLLBACK");
-        } catch {
-        }
-        throw storageError(error, "Collecting unused content");
-      } finally {
-        release?.();
+        this.#db.exec("ROLLBACK");
+      } catch {
       }
-    });
+      throw storageError(error, "Collecting unused content");
+    } finally {
+      release?.();
+    }
   }
   async reset(t, reason, options = {}) {
     await this.#mutate(async () => {