@evolu/web 3.4.0 → 3.5.0

package/src/Sqlite.ts

@@ -1,172 +1,155 @@
-import type { CreateSqliteDriver, SqliteRow } from "@evolu/common";
+/**
+ * The SQLite driver of Evolu for the web, on `@evolu/sqlite-wasm`.
+ *
+ * @module
+ */
+
 import {
-  bytesToHex,
   createPreparedStatementsCache,
+  err,
   exhaustiveCheck,
   ok,
-  performanceDurationBetween,
-  PositiveMillis,
-  sleep,
-  tryAsync,
+  type CreateSqliteDriver,
+  type DatabaseHeldError,
+  type Name,
+  type Result,
+  type SqliteDriver,
+  type Task,
+  type Typed,
+  type TypeName,
 } from "@evolu/common";
-import sqlite3InitModule, {
-  type Database,
-  type PreparedStatement,
-  type SAHPoolUtil,
+import type { WaitForDatabaseRelease } from "@evolu/common/local-first";
+import {
+  createEncryptedSqliteDatabase,
+  createSqliteDatabase,
+  createSqliteWasm,
+  OpfsName,
+  openSahPool,
+  sqliteWasmUrl,
+  SqliteVfsPath,
+  type deriveLegacySqliteKey,
+  type OpfsRootDep,
+  type SahPool,
+  type SahPoolError,
+  type SahPoolOptions,
+  type SqliteDatabase,
+  type SqliteError,
+  type SqliteStatement,
+  type SqliteWasm,
+  type SqliteWasmDep,
+  type SqliteWasmError,
+  type SubtleCryptoDep,
 } from "@evolu/sqlite-wasm";
 
-// @ts-expect-error Missing types.
-globalThis.sqlite3ApiConfig = {
-  warn: (arg: unknown) => {
-    // Ignore irrelevant warning.
-    // https://github.com/sqlite/sqlite-wasm/issues/62
-    if (
-      typeof arg === "string" &&
-      arg.startsWith("Ignoring inability to install OPFS sqlite3_vfs")
-    )
-      return;
-    // oxlint-disable-next-line eslint/no-console
-    console.warn(arg);
-  },
-};
-
-// Init ASAP.
-const sqlite3Promise = sqlite3InitModule();
-
-const fileName = "evolu1.db";
-
-export const createWasmSqliteDriver: CreateSqliteDriver =
-  (name, options) => async (run) => {
-    const sqlite3 = await sqlite3Promise;
-
-    using disposer = new DisposableStack();
-    const useDatabase = (database: Database): Database =>
-      disposer.adopt(database, (database) => {
-        database.close();
-      });
+/**
+ * Loads SQLite from the {@link sqliteWasmUrl} of `@evolu/sqlite-wasm` for
+ * {@link createWasmSqliteDriver}. It passes the fetch to
+ * {@link createSqliteWasm}, which compiles the binary while it downloads when
+ * the server sends it as `application/wasm`, and from its bytes otherwise.
+ *
+ * Start it in the worker's composition root, so SQLite loads while the worker
+ * waits for its database.
+ */
+export const loadSqliteWasm: Task<SqliteWasm, SqliteWasmError> = (run) =>
+  run(createSqliteWasm(run.deps.nativeFetch(sqliteWasmUrl)), run.deps);
+
+/**
+ * Creates the {@link CreateSqliteDriver} of Evolu for the web.
+ *
+ * A database is the file `/evolu1.db` in a pool of OPFS sync access handles in
+ * the directory `.<name>`, in the format of SQLite's opfs-sahpool, and the pool
+ * encrypts it in the `encrypted` mode. So the databases `@evolu/web` 3 created
+ * with `@evolu/sqlite-wasm` 2.2.4 open unchanged, an encrypted one with the key
+ * 2.2.4 derived from the encryption key, which is never rekeyed. 2.2.4 derived
+ * it because of a bug in SQLite3 Multiple Ciphers: it took the encryption key,
+ * passed in SQLCipher's notation for a raw key, as a passphrase
+ * (https://github.com/utelle/SQLite3MultipleCiphers/issues/218), as
+ * {@link deriveLegacySqliteKey} of `@evolu/sqlite-wasm` describes. A new
+ * encrypted database is encrypted with the encryption key itself, so
+ * `@evolu/web` 3.4.1 and earlier cannot open it. A database in the `memory`
+ * mode uses no OPFS.
+ *
+ * In WebKit on macOS, such as Safari, whose file system ignores case by
+ * default, names that differ only in case share the directory, and both can
+ * hold it at once, so make database names differ in more than case.
+ *
+ * The pool is opened once, so the driver throws while another context holds a
+ * file of it. A DbWorker waits for the files with
+ * {@link createWaitForDatabaseRelease} first.
+ *
+ * The {@link SqliteDriver} contract has no error channel, so every error, such
+ * as a query's {@link SqliteError}, is thrown as the cause of an `Error` whose
+ * message is SQLite's message, or the error's type for an error without one.
+ */
+export const createWasmSqliteDriver =
+  (deps: WasmSqliteDriverDeps): CreateSqliteDriver =>
+  (name, options) =>
+  async (run) => {
+    const sqliteWasm = getOrThrowWithMessage(await deps.sqliteWasmLoad);
+    const databaseDeps = { ...deps, sqliteWasm };
 
     let deleteDatabaseFile = false;
-    const createOpfsSAHPoolVfs = async (
-      options: Parameters<typeof sqlite3.installOpfsSAHPoolVfs>[0],
-    ): Promise<SAHPoolUtil> => {
-      // Evolu opens a database only while it holds the database lock, but a
-      // DbWorker that ended without closing it, because its tab closed,
-      // crashed or navigated away, can hold pool files after the lock has
-      // passed on: WebKit releases a terminated worker's locks and its files
-      // separately, in no set order. sqlite-wasm cannot set up a pool with a
-      // held file, and it then deletes the pool directory, which the held file
-      // usually but not always prevents. Only a worker that has ended can hold
-      // the files, and it can only release them, so the pool is set up once
-      // every file opens.
-      const canOpenPoolFiles = async (): Promise<boolean> => {
-        const root = await navigator.storage.getDirectory();
-        // sqlite-wasm keeps the pool in `.${name}/.opaque` in both OPFS modes.
-        const poolDirectory = await tryAsync(() =>
-          root.getDirectoryHandle(`.${name}`),
-        );
-        if (!poolDirectory.ok) return true;
-        const opaqueDirectory = await tryAsync(() =>
-          poolDirectory.value.getDirectoryHandle(".opaque"),
-        );
-        if (!opaqueDirectory.ok) return true;
-        for await (const handle of opaqueDirectory.value.values()) {
-          if (handle.kind !== "file") continue;
-          const accessHandle = await tryAsync(() =>
-            handle.createSyncAccessHandle(),
-          );
-          if (!accessHandle.ok) {
-            // WebKit rejects a held file with InvalidStateError, other
-            // engines with NoModificationAllowedError, as the spec says.
-            // WebKit also uses InvalidStateError for a closed or invalid
-            // handle and a stopped context. A retry opens fresh handles from
-            // a new listing, and a stopped context ends the loop with its
-            // worker, so retrying those is harmless.
-            if (
-              accessHandle.error instanceof DOMException &&
-              (accessHandle.error.name === "InvalidStateError" ||
-                accessHandle.error.name === "NoModificationAllowedError")
-            )
-              return false;
-            throw accessHandle.error;
-          }
-          accessHandle.value.close();
-        }
-        return true;
-      };
-
-      const waitStart = run.deps.time.performance.now();
-      let retryDelay = 50;
-      let isWaitReported = false;
-      while (!(await canOpenPoolFiles())) {
-        const waited = performanceDurationBetween(
-          waitStart,
-          run.deps.time.performance.now(),
-        );
-        if (!isWaitReported && waited >= 5000) {
-          isWaitReported = true;
-          run.deps.console.warn(
-            `Waiting for an ended DbWorker to release the files of database ${name}.`,
-          );
-        }
-        await run.ok(sleep(PositiveMillis.orThrow(retryDelay)));
-        retryDelay = Math.min(retryDelay * 2, 1000);
-      }
+    using disposer = new DisposableStack();
 
-      const pool = await sqlite3.installOpfsSAHPoolVfs(options);
-      if (pool.isPaused()) await pool.unpauseVfs();
+    const openPool = async (): Promise<SahPool> => {
+      const pool = disposer.use(
+        getOrThrowWithMessage(await run(openDatabasePool(name), databaseDeps)),
+      );
       disposer.defer(() => {
-        if (deleteDatabaseFile) pool.unlink(`/${fileName}`);
-        pool.pauseVfs();
+        if (deleteDatabaseFile)
+          getOrThrowWithMessage(pool.unlink(databasePath));
       });
       return pool;
     };
 
-    let db: Database;
+    let database: SqliteDatabase;
 
     switch (options?.mode) {
       case "memory":
-        // oxlint-disable-next-line react/rules-of-hooks -- useDatabase registers disposal and is not a React Hook.
-        db = useDatabase(new sqlite3.oo1.DB(":memory:"));
+        database = disposer.use(
+          getOrThrowWithMessage(
+            createSqliteDatabase(databaseDeps)({ type: "Memory" }),
+          ),
+        );
         break;
 
       case "encrypted": {
-        // MultipleCiphers encryption requires its VFS wrapper for OPFS SAH-pool.
-        // @ts-expect-error Missing types (update @evolu/sqlite-wasm types)
-        // oxlint-disable-next-line typescript/no-unsafe-call
-        sqlite3.capi.sqlite3mc_vfs_create("opfs", 1);
-        const pool = await createOpfsSAHPoolVfs({
-          directory: `.${name}`,
-        });
-        // oxlint-disable-next-line react/rules-of-hooks -- useDatabase registers disposal and is not a React Hook.
-        db = useDatabase(
-          new pool.OpfsSAHPoolDb(
-            // SQLite normalizes this URI filename to SAH-pool path "/evolu1.db".
-            `file:${fileName}?vfs=multipleciphers-opfs-sahpool`,
+        const encrypted = getOrThrowWithMessage(
+          await run(
+            createEncryptedSqliteDatabase({
+              type: "EncryptedFile",
+              vfs: await openPool(),
+              path: databasePath,
+              key: options.encryptionKey,
+            }),
+            databaseDeps,
           ),
         );
-        db.exec(`
-          PRAGMA cipher = 'sqlcipher';
-          PRAGMA key = "x'${bytesToHex(options.encryptionKey)}'";
-        `);
+        database = disposer.use(encrypted.database);
         break;
       }
 
-      case undefined: {
-        const pool = await createOpfsSAHPoolVfs({ name });
-        // oxlint-disable-next-line react/rules-of-hooks -- useDatabase registers disposal and is not a React Hook.
-        db = useDatabase(new pool.OpfsSAHPoolDb(`file:${fileName}`));
+      case undefined:
+        database = disposer.use(
+          getOrThrowWithMessage(
+            createSqliteDatabase(databaseDeps)({
+              type: "File",
+              vfs: await openPool(),
+              path: databasePath,
+            }),
+          ),
+        );
         break;
-      }
 
       default:
         exhaustiveCheck(options);
     }
 
     const cache = disposer.use(
-      createPreparedStatementsCache<PreparedStatement>(
-        (sql) => db.prepare(sql),
+      createPreparedStatementsCache<SqliteStatement>(
+        (sql) => getOrThrowWithMessage(database.prepare(sql)),
         (statement) => {
-          statement.finalize();
+          statement[Symbol.dispose]();
         },
       ),
     );
@@ -175,41 +158,15 @@ export const createWasmSqliteDriver: CreateSqliteDriver =
 
     return ok({
       exec: (query) => {
-        const prepared = cache.get(query);
-
-        if (prepared) {
-          try {
-            if (query.parameters.length > 0) prepared.bind(query.parameters);
-
-            const rows = [];
-            while (prepared.step()) {
-              rows.push(prepared.get({}));
-            }
-
-            return {
-              rows: rows as ReadonlyArray<SqliteRow>,
-              changes: db.changes(),
-            };
-          } finally {
-            // SQLite refuses to bind a statement whose step failed until it is
-            // reset. That reset returns the step's error again, which
-            // PreparedStatement.reset would throw over the original one.
-            sqlite3.capi.sqlite3_reset(prepared);
-          }
-        }
-
-        const rows = db.exec(query.sql, {
-          returnValue: "resultRows",
-          rowMode: "object",
-          bind: query.parameters,
-        }) as ReadonlyArray<SqliteRow>;
-
-        const changes = db.changes();
-
-        return { rows, changes };
+        const statement = cache.get(query);
+        return getOrThrowWithMessage(
+          statement
+            ? statement.run(query.parameters)
+            : database.run(query.sql, query.parameters),
+        );
       },
 
-      export: () => sqlite3.capi.sqlite3_js_db_export(db),
+      export: () => getOrThrowWithMessage(database.export()),
 
       deleteDatabase: () => {
         deleteDatabaseFile = true;
@@ -221,3 +178,73 @@ export const createWasmSqliteDriver: CreateSqliteDriver =
       },
     });
   };
+
+/** Dependencies of {@link createWasmSqliteDriver}. */
+export type WasmSqliteDriverDeps = OpfsRootDep &
+  SqliteWasmLoadDep &
+  SubtleCryptoDep;
+
+/** Dependency wrapper for SQLite as {@link loadSqliteWasm} loads it. */
+export interface SqliteWasmLoadDep {
+  /**
+   * SQLite, as {@link loadSqliteWasm} loads it, still loading or loaded. A
+   * failed load throws when a database is opened.
+   */
+  readonly sqliteWasmLoad: PromiseLike<Result<SqliteWasm, SqliteWasmError>>;
+}
+
+/**
+ * Creates the {@link WaitForDatabaseRelease} of Evolu for the web. It opens the
+ * pool {@link createWasmSqliteDriver} opens for the database, and disposes it
+ * once it opens.
+ *
+ * When another context holds a file of the pool, the pool is opened again after
+ * 50 ms, then after twice the previous delay, up to a second, with the
+ * `heldTimeout` option of {@link openSahPool}, and the wait fails with
+ * {@link DatabaseHeldError} when it is still held 10 seconds after the first
+ * attempt started. Evolu then refuses to start the database, and the app
+ * receives the error as its `evoluError`. Every other error is thrown, as the
+ * driver throws it.
+ */
+export const createWaitForDatabaseRelease =
+  (deps: OpfsRootDep & SqliteWasmLoadDep): WaitForDatabaseRelease =>
+  (name) =>
+  async (run) => {
+    const sqliteWasm = getOrThrowWithMessage(await deps.sqliteWasmLoad);
+    const opened = await run(
+      // Longer than the two seconds Chromium gives a terminated worker.
+      openDatabasePool(name, { heldTimeout: "10s" }),
+      { ...deps, sqliteWasm },
+    );
+    if (!opened.ok && opened.error.type === "SahPoolHeldError")
+      return err({ type: "DatabaseHeldError", name });
+    using _pool = getOrThrowWithMessage(opened);
+    return ok();
+  };
+
+// The pool of a database's file is in the directory `.<name>`, as in
+// @evolu/web 3.
+const openDatabasePool = (
+  name: Name,
+  options?: Pick<SahPoolOptions, "heldTimeout">,
+): Task<SahPool, SahPoolError, OpfsRootDep & SqliteWasmDep> =>
+  // A Name is non-empty and URL-safe, so `.<name>` always passes.
+  openSahPool({ ...options, directory: [OpfsName.orThrow(`.${name}`)] });
+
+// The path SQLite's opfs-sahpool gave `file:evolu1.db`, which @evolu/web 3
+// opened.
+const databasePath = /*#__PURE__*/ SqliteVfsPath.orThrow("/evolu1.db");
+
+// The SqliteDriver contract has no error channel, and the WaitForDatabaseRelease
+// contract only DatabaseHeldError, so any other error is thrown as the cause of
+// an Error, with SQLite's message when it has one.
+const getOrThrowWithMessage = <T>(result: Result<T, Typed<TypeName>>): T => {
+  if (result.ok) return result.value;
+  const { error } = result;
+  throw new Error(
+    "message" in error && typeof error.message === "string"
+      ? error.message
+      : error.type,
+    { cause: error },
+  );
+};

package/src/local-first/Db.worker.ts

@@ -6,15 +6,29 @@ installPolyfills();
 
 import { createRandomBytes } from "@evolu/common";
 import { startDbWorker } from "@evolu/common/local-first";
-import { createWasmSqliteDriver } from "../Sqlite.ts";
+import {
+  createWaitForDatabaseRelease,
+  createWasmSqliteDriver,
+  loadSqliteWasm,
+} from "../Sqlite.ts";
 import { createRun } from "../Task.ts";
 import { createWorkerDeps, createWorkerSelf } from "../Worker.ts";
 
 const run = createRun({
   ...createWorkerDeps(),
-  createSqliteDriver: createWasmSqliteDriver,
   lockManager: navigator.locks,
   randomBytes: createRandomBytes(),
 });
 
-void run(startDbWorker(createWorkerSelf(self)));
+const sqliteDeps = {
+  opfsRoot: navigator.storage,
+  // SQLite loads while the DbWorker waits for its database.
+  sqliteWasmLoad: run(loadSqliteWasm),
+  subtleCrypto: crypto.subtle,
+};
+
+void run(startDbWorker(createWorkerSelf(self)), {
+  ...run.deps,
+  createSqliteDriver: createWasmSqliteDriver(sqliteDeps),
+  waitForDatabaseRelease: createWaitForDatabaseRelease(sqliteDeps),
+});

package/src/local-first/Evolu.test.ts

@@ -1,4 +1,5 @@
 import {
+  acquireLeaderLock,
   assertEqual,
   assertFalse,
   assertNonNullable,
@@ -7,6 +8,7 @@ import {
   createConsole,
   createIdFromString,
   PositiveInt,
+  testCreateRun,
   testName,
   testStubGlobal,
   type NativeMessagePort,
@@ -222,6 +224,39 @@ describe("createEvoluDeps", () => {
       assertSame(setup.reloadApp.mock.callCount(), 0);
     });
 
+    it("releases the locks of its Evolu instances when the page enters the cache", async () => {
+      using setup = setupWebEvoluDeps();
+      await using run = testCreateRun({
+        lockManager: setup.deps.lockManager,
+      });
+      await using _instanceLock = await run.ok(
+        acquireLeaderLock("cached-instance"),
+      );
+      assertFalse(await isLockAvailable("evolu-leaderlock-cached-instance"));
+
+      setup.dispatchPageTransition("pagehide", true);
+      await waitForMacrotask();
+
+      assertTrue(await isLockAvailable("evolu-leaderlock-cached-instance"));
+    });
+
+    it("releases at once a lock granted after the page entered the cache", async () => {
+      using setup = setupWebEvoluDeps();
+      const name = "evolu-granted-after-cache";
+      const held = Promise.withResolvers<void>();
+      const holding = navigator.locks.request(name, () => held.promise);
+      const callback = mock.fn<LockGrantedCallback<void>>();
+      const request = setup.deps.lockManager.request(name, callback);
+
+      setup.dispatchPageTransition("pagehide", true);
+      held.resolve();
+      await holding;
+      await request;
+
+      assertSame(callback.mock.callCount(), 0);
+      assertTrue(await isLockAvailable(name));
+    });
+
     it("reloads once when the page is restored from the cache", () => {
       using setup = setupWebEvoluDeps();
       setup.dispatchPageTransition("pageshow", false);

package/src/local-first/Evolu.ts

@@ -57,8 +57,9 @@ export interface SharedWorkerUnsupportedDep {
  *
  * When the page enters the browser's back-forward cache, as Safari does on
  * every navigation away, this tab ends its part as if it closed: it stops the
- * database workers it hosts, so another tab takes them over, and it reloads if
- * the user comes back to it.
+ * database workers it hosts, so another tab takes them over, the shared worker
+ * ends this tab's Evolu instances, and the tab reloads if the user comes back
+ * to it.
  *
  * Where the browser offers no persistent storage, as in Safari's Private
  * Browsing or a Firefox private window, the database is kept in memory, and
@@ -293,6 +294,44 @@ export const createEvoluDeps = (
     });
   };
 
+  // Disposing the deps releases the locks this page holds through them, as
+  // closing the page would, and a lock granted afterwards is released at once.
+  // The SharedWorker learns that an Evolu instance ended when it gets the lock
+  // the instance holds. Chrome keeps the locks of a page in its back-forward
+  // cache, and a request made before the page entered the cache waits until
+  // Chrome drops the page (https://issues.chromium.org/issues/567630881), so
+  // the SharedWorker would keep syncing the owners of a cached tab's instances
+  // and rerunning their queries.
+  let areLocksReleased = false;
+  const locksReleased = Promise.withResolvers<void>();
+  disposer.defer(() => {
+    areLocksReleased = true;
+    locksReleased.resolve();
+  });
+
+  function requestLock<T>(
+    name: string,
+    callback: LockGrantedCallback<T>,
+  ): Promise<Awaited<T>>;
+  function requestLock<T>(
+    name: string,
+    options: LockOptions,
+    callback: LockGrantedCallback<T>,
+  ): Promise<Awaited<T>>;
+  function requestLock(
+    name: string,
+    ...args:
+      | [LockGrantedCallback<unknown>]
+      | [LockOptions, LockGrantedCallback<unknown>]
+  ): Promise<unknown> {
+    const [options, callback] = args.length === 1 ? [{}, args[0]] : args;
+    return navigator.locks.request(name, options, (lock) =>
+      areLocksReleased
+        ? undefined
+        : Promise.race([callback(lock), locksReleased.promise]),
+    );
+  }
+
   const evoluDeps = disposer.use(
     createCommonEvoluDeps({
       requestPersistentStorage,
@@ -300,17 +339,21 @@ export const createEvoluDeps = (
       createDbWorker,
       createBroadcastChannel,
       createMessageChannel,
-      lockManager: navigator.locks,
+      lockManager: {
+        query: () => navigator.locks.query(),
+        request: requestLock,
+      },
       reloadApp: reloadThisApp,
       sharedWorker,
     }),
   );
 
-  // A page entering the back-forward cache is frozen with its DbWorkers, which
-  // keep their database locks, so the other tabs would stall. WebKit also
-  // releases the page's own locks, so a restored page would work with state
-  // the other tabs gave up on. The page therefore ends its part as if it
-  // closed, and reloads when it is shown again.
+  // A page entering the back-forward cache is frozen with its DbWorkers, and
+  // WebKit keeps their database locks while the page is cached
+  // (https://bugs.webkit.org/show_bug.cgi?id=316904), so the other tabs would
+  // stall. WebKit also releases the page's own locks, so a restored page would
+  // work with state the other tabs gave up on. The page therefore ends its part
+  // as if it closed, and reloads when it is shown again.
   const reloadRestoredPage = (event: PageTransitionEvent): void => {
     if (event.persisted) reloadThisApp();
   };