@livestore/wa-sqlite 1.0.1-dev.0

package/test/sql_0005.js

@@ -0,0 +1,76 @@
+import * as Comlink from 'comlink';
+
+export function sql_0005(context) {
+  describe('sql_0005', function() {
+    beforeAll(async function() {
+      // Clear persistent storage.
+      const proxy = await context.create();
+      await context.destroy(proxy);
+    });
+  
+    const cleanup = [];
+    beforeEach(async function() {
+      cleanup.splice(0);
+    });
+  
+    afterEach(async function() {
+      for (const fn of cleanup) {
+        await fn();
+      }
+    });
+  
+    it('should transact atomically', async function() {
+      const instances = [];
+      for (let i = 0; i < 8; ++i) {
+        const proxy = await context.create({ reset: false });
+        const sqlite3 = proxy.sqlite3;
+        const db = await sqlite3.open_v2('demo');
+        instances.push({ sqlite3, db });
+        cleanup.push(async () => {
+          await sqlite3.close(db);
+          await context.destroy(proxy);
+        });
+
+        if (i === 0) {
+          await sqlite3.exec(db, `
+            BEGIN IMMEDIATE;
+            CREATE TABLE IF NOT EXISTS t(key PRIMARY KEY, value);
+            INSERT OR IGNORE INTO t VALUES ('foo', 0);
+            COMMIT;
+          `);
+        }
+      }
+
+      const iterations = 32;
+      const values = new Set();
+      await Promise.all(instances.map(async instance => {
+        for (let i = 0; i < iterations; ++i) {
+          const rows = await transact(instance, `
+            BEGIN IMMEDIATE;
+            UPDATE t SET value = value + 1 WHERE key = 'foo';
+            SELECT value FROM t WHERE key = 'foo';
+            COMMIT;
+          `);
+          values.add(rows[0][0]);
+        }
+      }));
+
+      expect(values.size).toBe(instances.length * iterations);
+      expect(Array.from(values).sort((a, b) => b - a).at(0)).toBe(values.size);
+    });
+  });
+}
+
+async function transact({ sqlite3, db }, sql) {
+  while (true) {
+    try {
+      const rows = [];
+      await sqlite3.exec(db, sql, Comlink.proxy(row => rows.push(row)));
+      return rows;
+    } catch (e) {
+      if (e.message !== 'database is locked') {
+        throw e;
+      }
+    }
+  }
+}

package/test/test-worker.js

@@ -0,0 +1,204 @@
+// Copyright 2024 Roy T. Hashimoto. All Rights Reserved.
+
+import * as Comlink from 'comlink';
+import * as SQLite from '../src/sqlite-api.js';
+
+const BUILDS = new Map([
+  ['default', '../dist/wa-sqlite.mjs'],
+  ['asyncify', '../dist/wa-sqlite-async.mjs'],
+  ['jspi', '../dist/wa-sqlite-jspi.mjs'],
+]);
+
+const MODULE = Symbol('module');
+const VFS_CONFIGS = new Map([
+  {
+    name: 'default',
+    vfsModule: null
+  },
+  {
+    name: 'AccessHandlePoolVFS',
+    vfsModule: '../src/examples/AccessHandlePoolVFS.js',
+  },
+  {
+    name: 'OPFSCoopSyncVFS',
+    vfsModule: '../src/examples/OPFSCoopSyncVFS.js',
+  },
+  {
+    name: 'FLOOR',
+    vfsModule: '../src/examples/FLOOR.js',
+  },
+  {
+    name: 'MemoryVFS',
+    vfsModule: '../src/examples/MemoryVFS.js',
+  },
+  {
+    name: 'MemoryAsyncVFS',
+    vfsModule: '../src/examples/MemoryAsyncVFS.js',
+  },
+  {
+    name: 'IDBBatchAtomicVFS',
+    vfsModule: '../src/examples/IDBBatchAtomicVFS.js',
+  },
+  {
+    name: 'IDBMirrorVFS',
+    vfsModule: '../src/examples/IDBMirrorVFS.js',
+  },
+  {
+    name: 'OPFSAdaptiveVFS',
+    vfsModule: '../src/examples/OPFSAdaptiveVFS.js',
+  },
+  {
+    name: 'OPFSAnyContextVFS',
+    vfsModule: '../src/examples/OPFSAnyContextVFS.js',
+  },
+  {
+    name: 'OPFSPermutedVFS',
+    vfsModule: '../src/examples/OPFSPermutedVFS.js',
+  },
+].map(config => [config.name, config]));
+
+const INDEXEDDB_DBNAMES = ['demo'];
+
+const searchParams = new URLSearchParams(location.search);
+
+maybeReset().then(async () => {
+  const buildName = searchParams.get('build') || BUILDS.keys().next().value;
+  const configName = searchParams.get('config') || VFS_CONFIGS.keys().next().value;
+  const config = VFS_CONFIGS.get(configName);
+
+  // Instantiate SQLite.
+  const { default: moduleFactory } = await import(BUILDS.get(buildName));
+  const module = await moduleFactory();
+  const sqlite3 = SQLite.Factory(module);
+
+  const vfs = await (async function() {
+    if (config.vfsModule) {
+      // Create the VFS and register it as the default file system.
+      const namespace = await import(config.vfsModule);
+      const className = config.vfsClass ?? config.vfsModule.match(/([^/]+)\.js$/)[1];
+      const vfsArgs = (config.vfsArgs ?? ['demo', MODULE])
+        .map(arg => arg === MODULE ? module : arg);
+      const vfs = await namespace[className].create(...vfsArgs);
+      sqlite3.vfs_register(vfs, true);
+      return vfs;
+    }
+    return {};
+  })();
+
+  const sqlite3Proxy = new Proxy(sqlite3, {
+    get(target, p, receiver) {
+      // Comlink intercepts some function property names, e.g. "bind",
+      // so allow aliases to avoid the problem.
+      if (typeof p === 'string') p = p.replaceAll('$', '');
+
+      const value = Reflect.get(target, p, receiver);
+      if (typeof value === 'function') {
+        return async (...args) => {
+          const result = await value.apply(target, args);
+          if (p === 'statements') {
+            return Comlink.proxy(result);
+          }
+          return result;
+        };
+      }
+    }
+  });
+
+  const vfsProxy = new Proxy(vfs, {
+    get(target, p, receiver) {
+      const value = Reflect.get(target, p, receiver);
+      if (typeof value === 'function') {
+        return async (...args) => {
+          if (p === 'jRead') {
+            // The read buffer Uint8Array will be passed by proxy so all
+            // access is asynchronous. Pass a local buffer to the VFS
+            // and copy the local buffer to the proxy on completion.
+            const proxyBuffer = args[1];
+            args[1] = new Uint8Array(await proxyBuffer.length);
+            const result = await value.apply(target, args);
+            await proxyBuffer.set(args[1]);
+            return result;
+          }
+          return value.apply(target, args);
+        };
+      }
+    }
+  });
+
+  const { port1, port2 } = new MessageChannel();
+  Comlink.expose({
+    module,
+    sqlite3: sqlite3Proxy,
+    vfs: vfsProxy,
+  }, port1);
+  postMessage(null, [port2]);
+}).catch(e => {
+  console.error(e);
+  postMessage(cvtErrorToCloneable(e));
+});
+
+async function maybeReset() {
+  if (searchParams.get('reset') !== 'true') {
+    return;
+  }  
+  
+  // Limit the amount of time in this function.
+  const abortController = new AbortController();
+  setTimeout(() => abortController.abort(), 10_000);
+
+  // Clear OPFS.
+  const root = await navigator.storage?.getDirectory();
+  if (root) {
+    let opfsDeleted = false;
+    while (!opfsDeleted) {
+      abortController.signal.throwIfAborted();
+      try {
+        // @ts-ignore
+        for await (const name of root.keys()) {
+          await root.removeEntry(name, { recursive: true });
+        }
+        opfsDeleted = true;
+      } catch (e) {
+        // A NoModificationAllowedError is thrown if an entry can't be
+        // deleted because it isn't closed. Just try again.
+        if (e.name === 'NoModificationAllowedError') {
+          await new Promise(resolve => setTimeout(resolve));
+          continue;
+        }
+        throw e;
+      }
+    }
+  }
+
+  // Clear IndexedDB.
+  const dbList = indexedDB.databases ?
+    await indexedDB.databases() :
+    INDEXEDDB_DBNAMES.map(name => ({ name }));
+  await Promise.all(dbList.map(({name}) => {
+    return new Promise((resolve, reject) => {
+      const request = indexedDB.deleteDatabase(name);
+      request.onsuccess = resolve;
+      request.onerror = reject;
+    });
+  }));
+}
+
+function cvtErrorToCloneable(e) {
+  if (e instanceof Error) {
+    const props = new Set([
+      ...['name', 'message', 'stack'].filter(k => e[k] !== undefined),
+      ...Object.getOwnPropertyNames(e)
+    ]);
+    return Object.fromEntries(Array.from(props, k =>  [k, e[k]])
+      .filter(([_, v]) => {
+        // Skip any non-cloneable properties.
+        try {
+          structuredClone(v);
+          return true;
+        } catch (e) {
+          return false;
+        }
+      }));
+  }
+  return e;
+}

package/test/vfs_xAccess.js

@@ -0,0 +1,2 @@
+export function vfs_xAccess(context) {
+}

package/test/vfs_xClose.js

@@ -0,0 +1,52 @@
+import * as Comlink from 'comlink';
+import * as VFS from '../src/VFS.js';
+
+const FILEID = 1;
+
+export function vfs_xClose(context) {
+  describe('vfs_xClose', function() {
+    let proxy, vfs;
+    beforeEach(async function() {
+      proxy = await context.create();
+      vfs = proxy.vfs;
+    });
+
+    afterEach(async function() {
+      await context.destroy(proxy);
+    });
+
+    it('should leave an accessible file', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen('test', FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      await vfs.jClose(FILEID);
+
+      const pAccessOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      rc = await vfs.jAccess('test', VFS.SQLITE_ACCESS_READWRITE, pAccessOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pAccessOutput.getInt32(0, true)).not.toEqual(0);
+    });
+
+    it('should delete on close', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE | VFS.SQLITE_OPEN_DELETEONCLOSE;
+      rc = await vfs.jOpen('test', FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      const pAccessOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      rc = await vfs.jAccess('test', VFS.SQLITE_ACCESS_READWRITE, pAccessOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pAccessOutput.getInt32(0, true)).toEqual(1);
+
+      await vfs.jClose(FILEID);
+
+      rc = await vfs.jAccess('test', VFS.SQLITE_ACCESS_READWRITE, pAccessOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pAccessOutput.getInt32(0, true)).toEqual(0);
+    });
+  });
+}

package/test/vfs_xOpen.js

@@ -0,0 +1,91 @@
+import * as Comlink from 'comlink';
+import * as VFS from '../src/VFS.js';
+
+const FILEID = 1;
+
+export function vfs_xOpen(context) {
+  describe('vfs_xOpen', function() {
+    let proxy, vfs;
+    beforeEach(async function() {
+      proxy = await context.create();
+      vfs = proxy.vfs;
+    });
+
+    afterEach(async function() {
+      await context.destroy(proxy);
+    });
+
+    it('should create a file', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen('test', FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pOpenOutput.getInt32(0, true)).toEqual(openFlags);
+
+      const pAccessOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      rc = await vfs.jAccess('test', VFS.SQLITE_ACCESS_READWRITE, pAccessOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pAccessOutput.getInt32(0, true)).not.toEqual(0);
+    });
+
+    it('should create a database file', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE | VFS.SQLITE_OPEN_MAIN_DB;
+      
+      do {
+        const nRetryOps = await proxy.module.retryOps.length;
+        for (let i = 0; i < nRetryOps; i++) {
+          await proxy.module.retryOps[i];
+        }
+        rc = await vfs.jOpen('test',  1, openFlags, pOpenOutput);
+      } while (rc === VFS.SQLITE_BUSY);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pOpenOutput.getInt32(0, true)).toEqual(openFlags);
+
+      const pAccessOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      rc = await vfs.jAccess('test', VFS.SQLITE_ACCESS_READWRITE, pAccessOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pAccessOutput.getInt32(0, true)).not.toEqual(0);
+    });
+
+    it('should not create a file', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen('test',  1, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_CANTOPEN);
+
+      const pAccessOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      rc = await vfs.jAccess('test', VFS.SQLITE_ACCESS_READWRITE, pAccessOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pAccessOutput.getInt32(0, true)).toEqual(0);
+    });
+
+    it('should open an existing file', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen('test', FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      // Close the file because some VFS implementations don't allow
+      // multiple open handles.
+      await vfs.jClose(FILEID);
+
+      rc = await vfs.jOpen('test', FILEID, VFS.SQLITE_OPEN_READWRITE, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pOpenOutput.getInt32(0, true)).toEqual(VFS.SQLITE_OPEN_READWRITE);
+    });
+
+    it('should create an anonymous file', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen(null, FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect(pOpenOutput.getInt32(0, true)).toEqual(openFlags);
+    });
+  });
+}

package/test/vfs_xRead.js

@@ -0,0 +1,38 @@
+import * as Comlink from 'comlink';
+import * as VFS from '../src/VFS.js';
+
+const FILEID = 1;
+
+export function vfs_xRead(context) {
+  describe('vfs_xRead', function() {
+    let proxy, vfs;
+    beforeEach(async function() {
+      proxy = await context.create();
+      vfs = proxy.vfs;
+    });
+
+    afterEach(async function() {
+      await context.destroy(proxy);
+    });
+
+    it('should signal short read', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen('test', FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      const pData = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+      const iOffset = 0;
+      rc = await vfs.jWrite(FILEID, pData, iOffset);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      const pReadData = Comlink.proxy(new Uint8Array(pData.length * 2).fill(0xfb));
+      rc = await vfs.jRead(FILEID, pReadData, iOffset);
+      expect(rc).toEqual(VFS.SQLITE_IOERR_SHORT_READ);
+      expect(pReadData.subarray(0, pData.length)).toEqual(pData);
+      expect(pReadData.subarray(pData.length))
+        .toEqual(new Uint8Array(pReadData.length - pData.length));
+    });
+  });
+}

package/test/vfs_xWrite.js

@@ -0,0 +1,36 @@
+import * as Comlink from 'comlink';
+import * as VFS from '../src/VFS.js';
+
+const FILEID = 1;
+
+export function vfs_xWrite(context) {
+  describe('vfs_xWrite', function() {
+    let proxy, vfs;
+    beforeEach(async function() {
+      proxy = await context.create();
+      vfs = proxy.vfs;
+    });
+
+    afterEach(async function() {
+      await context.destroy(proxy);
+    });
+
+    it('should round-trip data', async function() {
+      let rc;
+      const pOpenOutput = Comlink.proxy(new DataView(new ArrayBuffer(4)));
+      const openFlags = VFS.SQLITE_OPEN_CREATE | VFS.SQLITE_OPEN_READWRITE;
+      rc = await vfs.jOpen('test', FILEID, openFlags, pOpenOutput);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      const pData = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+      const iOffset = 0;
+      rc = await vfs.jWrite(FILEID, pData, iOffset);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+
+      const pReadData = Comlink.proxy(new Uint8Array(pData.length));
+      rc = await vfs.jRead(FILEID, pReadData, iOffset);
+      expect(rc).toEqual(VFS.SQLITE_OK);
+      expect([...pReadData]).toEqual([...pData]);
+    });
+  });
+}