@livestore/livestore 0.0.39 → 0.0.40

package/src/storage/web-worker/index.ts

@@ -1,11 +1,13 @@
-import { casesHandled } from '@livestore/utils'
+import { casesHandled, notYetImplemented } from '@livestore/utils'
 import type * as otel from '@opentelemetry/api'
 import * as Comlink from 'comlink'
 
+import type { MutationEvent } from '../../index.js'
 import type { PreparedBindValues } from '../../utils/util.js'
 import type { Storage, StorageOtelProps } from '../index.js'
 import { IDB } from '../utils/idb.js'
-import type { WrappedWorker } from './worker.js'
+import type { ExecutionBacklogItem } from './common.js'
+import type { WrappedWorker } from './make-worker.js'
 
 export type StorageType = 'opfs' | 'indexeddb'
 
@@ -13,28 +15,33 @@ export type StorageOptionsWeb = {
   /** Specifies where to persist data for this storage */
   type: StorageType
   fileName: string
+  worker: Worker | (new (options?: { name: string }) => Worker)
 }
 
 export class WebWorkerStorage implements Storage {
-  worker: Comlink.Remote<WrappedWorker>
+  worker: Worker
+  wrappedWorker: Comlink.Remote<WrappedWorker>
   options: StorageOptionsWeb
   otelTracer: otel.Tracer
 
-  executionBacklog: { query: string; bindValues?: PreparedBindValues }[] = []
+  executionBacklog: ExecutionBacklogItem[] = []
   executionPromise: Promise<void> | undefined
 
   private constructor({
     worker,
+    wrappedWorker,
     options,
     otelTracer,
     executionPromise,
   }: {
-    worker: Comlink.Remote<WrappedWorker>
+    worker: Worker
+    wrappedWorker: Comlink.Remote<WrappedWorker>
     options: StorageOptionsWeb
     otelTracer: otel.Tracer
     executionPromise: Promise<void>
   }) {
     this.worker = worker
+    this.wrappedWorker = wrappedWorker
     this.options = options
     this.otelTracer = otelTracer
     this.executionPromise = executionPromise
@@ -43,26 +50,31 @@ export class WebWorkerStorage implements Storage {
   }
 
   static load = (options: StorageOptionsWeb) => {
-    // TODO: Importing the worker like this only works with Vite;
-    // should this really be inside the LiveStore library?
-    // Doesn't work with Firefox right now during dev https://bugzilla.mozilla.org/show_bug.cgi?id=1247687
-    const worker = new Worker(new URL('./worker.js', import.meta.url), {
-      type: 'module',
-    })
+    const worker = options.worker instanceof Worker ? options.worker : new options.worker({ name: 'livestore-worker' })
+    // TODO replace Comlink with Effect worker
     const wrappedWorker = Comlink.wrap<WrappedWorker>(worker)
 
     return ({ otelTracer }: StorageOtelProps) =>
       new WebWorkerStorage({
-        worker: wrappedWorker,
+        worker,
+        wrappedWorker,
         options,
         otelTracer,
-        executionPromise: wrappedWorker.initialize(options),
+        executionPromise: wrappedWorker.initialize({ fileName: options.fileName, type: options.type }),
       })
   }
 
-  execute = (query: string, bindValues?: PreparedBindValues) => {
-    this.executionBacklog.push({ query, bindValues })
+  execute = (query: string, bindValues?: PreparedBindValues, _parentSpan?: otel.Span | undefined) => {
+    this.executionBacklog.push({ _tag: 'execute', query, bindValues })
+    this.scheduleExecution()
+  }
+
+  mutate = (mutationEventEncoded: MutationEvent.Any, _parentSpan?: otel.Span | undefined) => {
+    this.executionBacklog.push({ _tag: 'mutate', mutationEventEncoded })
+    this.scheduleExecution()
+  }
 
+  private scheduleExecution = () => {
     // Instead of sending the queries to the worker immediately, we wait a bit and batch them up (which reduces the number of messages sent to the worker)
     if (this.executionPromise === undefined) {
       this.executionPromise = new Promise((resolve) => {
@@ -76,12 +88,20 @@ export class WebWorkerStorage implements Storage {
   }
 
   private executeBacklog = () => {
-    void this.worker.executeBulk(this.executionBacklog)
+    void this.wrappedWorker.executeBulk(this.executionBacklog)
     this.executionBacklog = []
     this.executionPromise = undefined
   }
 
   getPersistedData = async (_parentSpan?: otel.Span): Promise<Uint8Array> => getPersistedData(this.options)
+
+  getMutationLogData = async (_parentSpan?: otel.Span): Promise<Uint8Array> => getMutationLogData(this.options)
+
+  dangerouslyReset = async () => {
+    // TODO implement graceful shutdown
+    this.worker.terminate()
+    await resetPersistedData(this.options)
+  }
 }
 
 const getPersistedData = async (options: StorageOptionsWeb): Promise<Uint8Array> => {
@@ -89,7 +109,7 @@ const getPersistedData = async (options: StorageOptionsWeb): Promise<Uint8Array>
     case 'opfs': {
       try {
         const rootHandle = await navigator.storage.getDirectory()
-        const fileHandle = await rootHandle.getFileHandle(options.fileName + '.db')
+        const fileHandle = await rootHandle.getFileHandle(options.fileName)
         const file = await fileHandle.getFile()
         const buffer = await file.arrayBuffer()
         const data = new Uint8Array(buffer)
@@ -114,3 +134,52 @@ const getPersistedData = async (options: StorageOptionsWeb): Promise<Uint8Array>
     }
   }
 }
+
+const getMutationLogData = async (options: StorageOptionsWeb): Promise<Uint8Array> => {
+  switch (options.type) {
+    case 'opfs': {
+      try {
+        const rootHandle = await navigator.storage.getDirectory()
+        const fileHandle = await rootHandle.getFileHandle(`${options.fileName}-log.db`)
+        const file = await fileHandle.getFile()
+        const buffer = await file.arrayBuffer()
+        const data = new Uint8Array(buffer)
+
+        return data
+      } catch (error: any) {
+        if (error instanceof DOMException && error.name === 'NotFoundError') {
+          return new Uint8Array()
+        }
+
+        throw error
+      }
+    }
+
+    case 'indexeddb': {
+      return notYetImplemented()
+    }
+    default: {
+      casesHandled(options.type)
+    }
+  }
+}
+
+const resetPersistedData = async (options: StorageOptionsWeb) => {
+  switch (options.type) {
+    case 'opfs': {
+      const rootHandle = await navigator.storage.getDirectory()
+      await rootHandle.removeEntry(options.fileName)
+      await rootHandle.removeEntry(`${options.fileName}-log.db`)
+      break
+    }
+
+    case 'indexeddb': {
+      const idb = new IDB(options.fileName)
+      await idb.deleteDb()
+      break
+    }
+    default: {
+      casesHandled(options.type)
+    }
+  }
+}

package/src/storage/web-worker/make-worker.ts

@@ -0,0 +1,214 @@
+// TODO: create types for these libraries? SQL.js already should have types;
+// we just need the types to apply to the fork.
+import { memoize, shouldNeverHappen } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+import * as Comlink from 'comlink'
+import type * as SqliteWasm from 'sqlite-esm'
+import sqlite3InitModule from 'sqlite-esm'
+
+import type { LiveStoreSchema } from '../../index.js'
+import { makeMutationEventSchema } from '../../schema/mutations.js'
+import { casesHandled, prepareBindValues, sql } from '../../utils/util.js'
+import { IDB } from '../utils/idb.js'
+import type { ExecutionBacklogItem } from './common.js'
+import type { StorageOptionsWeb } from './index.js'
+
+export type WorkerOptions<TSchema extends LiveStoreSchema = LiveStoreSchema> = {
+  schema: TSchema
+  mutationLog?: {
+    /**
+     * Mutations to exclude in the mutation log
+     *
+     * @default new Set(['livestore.RawSql'])
+     */
+    exclude?: ReadonlySet<keyof TSchema['_MutationDefMapType']>
+  }
+}
+
+export const makeWorker = <TSchema extends LiveStoreSchema = LiveStoreSchema>({
+  schema,
+  mutationLog,
+}: WorkerOptions<TSchema>) => {
+  // A global variable to hold the database connection.
+  let db: SqliteWasm.DatabaseApi
+
+  let dbLog: SqliteWasm.DatabaseApi
+
+  let sqlite3: SqliteWasm.Sqlite3Static
+
+  const mutationLogExclude = mutationLog?.exclude ?? new Set(['livestore.RawSql'])
+
+  // TODO refactor
+  const mutationArgsSchema = makeMutationEventSchema(Object.fromEntries(schema.mutations.entries()) as any)
+  const schemaHashMap = new Map([...schema.mutations.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const))
+
+  // TODO get rid of this in favour of a "proper" IDB SQLite storage
+  let idb: IDB | undefined
+
+  /** The location where this database storage persists its data */
+  let options_: Omit<StorageOptionsWeb, 'worker'>
+
+  const configureConnection = () =>
+    db.exec(sql`
+    PRAGMA page_size=8192;
+    PRAGMA journal_mode=MEMORY;
+    PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+  `)
+
+  /** A full virtual filename in the IDB FS */
+
+  const initialize = async (options: Omit<StorageOptionsWeb, 'worker'>) => {
+    options_ = options
+
+    sqlite3 = await sqlite3InitModule({
+      print: (message) => console.log(`[sql-client] ${message}`),
+      printErr: (message) => console.error(`[sql-client] ${message}`),
+    })
+
+    switch (options.type) {
+      case 'opfs': {
+        try {
+          db = new sqlite3.oo1.OpfsDb(options.fileName) // , 'c'
+
+          dbLog = new sqlite3.oo1.OpfsDb(options.fileName + '-log.db') // , 'c'
+        } catch (e) {
+          debugger
+        }
+        break
+      }
+      case 'indexeddb': {
+        try {
+          db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' })
+          idb = new IDB(options.fileName)
+
+          const bytes = await idb.get('db')
+
+          if (bytes !== undefined) {
+            // Based on https://sqlite.org/forum/forumpost/2119230da8ac5357a13b731f462dc76e08621a4a29724f7906d5f35bb8508465
+            // TODO find cleaner way to do this once possible in sqlite3-wasm
+            const p = sqlite3.wasm.allocFromTypedArray(bytes)
+            const _rc = sqlite3.capi.sqlite3_deserialize(db.pointer, 'main', p, bytes.length, bytes.length, 0)
+          }
+        } catch (e) {
+          debugger
+        }
+        break
+      }
+      default: {
+        casesHandled(options.type)
+      }
+    }
+
+    // Creates `mutation_log` table if it doesn't exist
+    dbLog.exec(sql`
+      CREATE TABLE IF NOT EXISTS mutation_log (
+        id TEXT PRIMARY KEY NOT NULL,
+        mutation TEXT NOT NULL,
+        args_json TEXT NOT NULL,
+        schema_hash INTEGER NOT NULL,
+        created_at TEXT NOT NULL
+      );
+    `)
+
+    configureConnection()
+  }
+
+  // TODO get rid of this in favour of a "proper" IDB SQLite storage
+  let idbPersistTimeout: NodeJS.Timeout | undefined
+
+  const executeBulk = (executionItems: ExecutionBacklogItem[]): void => {
+    let batchItems: ExecutionBacklogItem[] = []
+
+    const createdAtMemo = memoize(() => new Date().toISOString())
+
+    while (executionItems.length > 0) {
+      try {
+        db.exec('BEGIN TRANSACTION') // Start the transaction
+        dbLog.exec('BEGIN TRANSACTION') // Start the transaction
+
+        batchItems = executionItems.splice(0, 50)
+
+        // console.debug('livestore-webworker: executing batch', batchItems)
+
+        for (const item of batchItems) {
+          if (item._tag === 'execute') {
+            const { query, bindValues } = item
+            db.exec({ sql: query, bind: bindValues as TODO })
+
+            // NOTE we're not writing `execute` events to the mutation_log
+          } else {
+            const { mutation, args } = Schema.decodeUnknownSync(mutationArgsSchema)(item.mutationEventEncoded)
+
+            const mutationDef = schema.mutations.get(mutation) ?? shouldNeverHappen(`Unknown mutation: ${mutation}`)
+
+            const statementRes = typeof mutationDef.sql === 'function' ? mutationDef.sql(args) : mutationDef.sql
+            const statementSql = typeof statementRes === 'string' ? statementRes : statementRes.sql
+
+            const bindValues =
+              typeof statementRes === 'string' ? item.mutationEventEncoded.args : statementRes.bindValues
+
+            db.exec({ sql: statementSql, bind: prepareBindValues(bindValues ?? {}, statementSql) as TODO })
+
+            // write to mutation_log
+            if (options_.type === 'opfs' && mutationLogExclude.has(mutation) === false) {
+              const schemaHash = schemaHashMap.get(mutation) ?? shouldNeverHappen(`Unknown mutation: ${mutation}`)
+
+              const argsJson = JSON.stringify(item.mutationEventEncoded.args ?? {})
+
+              dbLog.exec({
+                sql: `INSERT INTO mutation_log (id, mutation, args_json, schema_hash, created_at) VALUES (?, ?, ?, ?, ?)`,
+                bind: [
+                  item.mutationEventEncoded.id,
+                  item.mutationEventEncoded.mutation,
+                  argsJson,
+                  schemaHash,
+                  createdAtMemo(),
+                ],
+              })
+            }
+          }
+        }
+
+        db.exec('COMMIT') // Commit the transaction
+        dbLog.exec('COMMIT') // Commit the transaction
+      } catch (error) {
+        try {
+          db.exec('ROLLBACK') // Rollback in case of an error
+          dbLog.exec('ROLLBACK') // Rollback in case of an error
+        } catch (e) {
+          console.error('Error rolling back transaction', e)
+        }
+
+        shouldNeverHappen(`Error executing query: ${error} \n ${JSON.stringify(batchItems)}`)
+      }
+    }
+
+    // TODO get rid of this in favour of a "proper" IDB SQLite storage
+    if (options_.type === 'indexeddb') {
+      if (idbPersistTimeout !== undefined) {
+        clearTimeout(idbPersistTimeout)
+      }
+
+      idbPersistTimeout = setTimeout(() => {
+        const data = sqlite3.capi.sqlite3_js_db_export(db.pointer) as Uint8Array
+
+        void idb!.put('db', data)
+      }, 1000)
+    }
+  }
+
+  const wrappedWorker = { initialize, executeBulk }
+
+  Comlink.expose(wrappedWorker)
+
+  // NOTE keep this around for debugging
+  // db.exec({
+  //   sql: `select * from sqlite_master where name = 'library_tracks'`,
+  //   callback: (_: TODO) => console.log(_),
+  //   rowMode: 'object',
+  // } as TODO)
+
+  return wrappedWorker
+}
+
+export type WrappedWorker = ReturnType<typeof makeWorker>

package/src/storage/web-worker/vite-dev-polyfill.ts

@@ -0,0 +1,33 @@
+// @ts-expect-error TODO remove when Vite does proper treeshaking during dev
+globalThis.$RefreshReg$ = () => {}
+// @ts-expect-error TODO remove when Vite does proper treeshaking during dev
+globalThis.$RefreshSig$ = () => (type: any) => type
+
+globalThis.document = (globalThis as any)?.document ?? {
+  querySelectorAll: () => [],
+  addEventListener: () => {},
+  createElement: () => ({
+    setAttribute: () => {},
+    pathname: '',
+    style: {},
+  }),
+  body: {
+    addEventListener: () => {},
+  },
+  head: {
+    appendChild: () => {},
+  },
+}
+
+globalThis.window = globalThis?.window ?? {
+  AnimationEvent: class AnimationEvent {},
+  TransitionEvent: class TransitionEvent {},
+  addEventListener: () => {},
+  location: {
+    href: '',
+    pathname: '',
+  },
+  document: globalThis.document,
+}
+
+globalThis.HTMLElement = globalThis?.HTMLElement ?? class HTMLElement {}