@livestore/livestore 0.0.0

package/src/backends/base.ts

@@ -0,0 +1,67 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { errorToString } from '@livestore/utils'
+import { identity } from '@livestore/utils/effect'
+import * as otel from '@opentelemetry/api'
+
+import type { LiveStoreEvent } from '../events.js'
+// import { EVENTS_TABLE_NAME } from '../events.js'
+import type { ActionDefinition } from '../schema.js'
+import type { ParamsObject } from '../util.js'
+import type { Backend, SelectResponse } from './index.js'
+
+export abstract class BaseBackend implements Backend {
+  abstract otelTracer: otel.Tracer
+
+  select = async <T = any>(query: string, bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
+    throw new Error('Method not implemented.')
+  }
+
+  execute = (query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): void => {
+    throw new Error('Method not implemented.')
+  }
+
+  getPersistedData = async (parentSpan?: otel.Span): Promise<Uint8Array> => {
+    throw new Error('Method not implemented.')
+  }
+
+  // TODO move `applyEvent` logic to Store and only call `execute` here
+  applyEvent = (event: LiveStoreEvent, eventDefinition: ActionDefinition, parentSpan?: otel.Span): void => {
+    const ctx = parentSpan ? otel.trace.setSpan(otel.context.active(), parentSpan) : otel.context.active()
+    this.otelTracer.startActiveSpan('LiveStore:backend:applyEvent', {}, ctx, (span) => {
+      try {
+        // Careful: this SQL statement is duplicated in the backend.
+        // Remember to update it in src-tauri/src/store.rs:apply_event as well.
+        // await this.execute(sql`insert into ${EVENTS_TABLE_NAME} (id, type, args) values ($id, $type, $args)`, {
+        //   id: event.id,
+        //   type: event.type,
+        //   args: JSON.stringify(event.args ?? {}),
+        // })
+
+        const statement =
+          typeof eventDefinition.statement === 'function'
+            ? eventDefinition.statement(event.args)
+            : eventDefinition.statement
+
+        const prepareBindValues = eventDefinition.prepareBindValues ?? identity
+
+        const bindValues =
+          typeof eventDefinition.statement === 'function' && statement.argsAlreadyBound
+            ? {}
+            : prepareBindValues(event.args)
+
+        span.setAttributes({
+          'livestore.statement.sql': statement.sql,
+          'livestore.statement.writeTables': statement.writeTables,
+          'livestore.statement.bindVales': JSON.stringify(bindValues),
+        })
+
+        this.execute(statement.sql, bindValues, span)
+      } catch (e: any) {
+        span.setStatus({ code: otel.SpanStatusCode.ERROR, message: errorToString(e) })
+        throw e
+      } finally {
+        span.end()
+      }
+    })
+  }
+}

package/src/backends/index.ts

@@ -0,0 +1,94 @@
+// A backend represents a raw SQLite database.
+// Examples include:
+// - A native SQLite process running in a Tauri Rust process
+// - A SQL.js WASM version of SQLite running in a web worker
+//
+// We can send commands to execute various kinds of queries,
+// and respond to various events from the database.
+
+import type * as otel from '@opentelemetry/api'
+
+import type { LiveStoreEvent } from '../events.js'
+import type { ActionDefinition } from '../schema.js'
+import type { ParamsObject } from '../util.js'
+import { casesHandled } from '../util.js'
+import type { BackendOptionsTauri } from './tauri.js'
+import type { BackendOptionsWeb } from './web.js'
+import { WebWorkerBackend } from './web.js'
+import type { BackendOptionsWebInMemory } from './web-in-memory.js'
+import { WebInMemoryBackend } from './web-in-memory.js'
+
+/* A location of a persistent writable SQLite file */
+export type WritableDatabaseLocation =
+  | {
+      type: 'opfs'
+      virtualFilename: string
+    }
+  | {
+      type: 'indexeddb'
+      virtualFilename: string
+    }
+  | {
+      type: 'filesystem'
+      directory: string
+      filename: string
+    }
+  | {
+      type: 'volatile-in-memory'
+    }
+
+export interface Backend {
+  // Select some data from the DB.
+  // This should only do reads, not writes, but we don't strongly enforce that.
+  select<T = any>(query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): Promise<SelectResponse<T>>
+
+  // Execute a query where you don't care about the result.
+  // Used for writes and configuration changes.
+  execute(query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): void
+
+  /** Apply an event to the backend */
+  applyEvent(event: LiveStoreEvent, eventDefiniton: ActionDefinition, parentSpan?: otel.Span): void
+
+  /** Return a snapshot of persisted data from the backend */
+  getPersistedData(parentSpan?: otel.Span): Promise<Uint8Array>
+}
+
+export type BackendType = 'tauri' | 'web' | 'web-in-memory'
+
+export const isBackendType = (type: string): type is BackendType => {
+  return type === 'tauri' || type === 'web' || type === 'web-in-memory'
+}
+
+export type SelectResponse<T = any> = {
+  results: T[]
+
+  // other perf stats metadata about how long the query took
+  [key: string]: any
+}
+
+export enum IndexType {
+  Basic = 'Basic',
+  FullText = 'FullText',
+}
+
+export type BackendOptions = BackendOptionsTauri | BackendOptionsWeb | BackendOptionsWebInMemory
+
+export const createBackend = async (options: BackendOptions): Promise<Backend> => {
+  switch (options.type) {
+    case 'tauri': {
+      // NOTE Dynamic import is needed to avoid Tauri is a dependency of LiveStore (e.g. when used in the web)
+      const { TauriBackend } = await import('./tauri.js')
+      return await TauriBackend.load(options)
+    }
+    case 'web': {
+      return WebWorkerBackend.load(options)
+    }
+    // NOTE currently only needed for testing
+    case 'web-in-memory': {
+      return WebInMemoryBackend.load(options)
+    }
+    default: {
+      casesHandled(options)
+    }
+  }
+}

package/src/backends/noop.ts

@@ -0,0 +1,32 @@
+import { makeNoopTracer } from '@livestore/utils'
+import type * as otel from '@opentelemetry/api'
+
+import type { ParamsObject } from '../util.js'
+import { BaseBackend } from './base.js'
+import type { SelectResponse } from './index.js'
+
+export type BackendOptionsNoop = {
+  type: 'noop'
+  /** Specifies where to persist data for this backend */
+  otelTracer?: otel.Tracer
+}
+
+export class NoopBackend extends BaseBackend {
+  constructor(readonly otelTracer: otel.Tracer) {
+    super()
+  }
+
+  static load = async (options: BackendOptionsNoop): Promise<NoopBackend> => {
+    return new NoopBackend(options.otelTracer ?? makeNoopTracer())
+  }
+
+  execute = (_query: string, _bindValues?: ParamsObject): void => {}
+
+  select = async <T>(_query: string, _bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
+    return { results: [] }
+  }
+
+  getPersistedData = async (): Promise<Uint8Array> => {
+    return new Uint8Array()
+  }
+}

package/src/backends/tauri.ts

@@ -0,0 +1,74 @@
+import { getTraceParentHeader, makeNoopTracer } from '@livestore/utils'
+import type * as otel from '@opentelemetry/api'
+import { invoke } from '@tauri-apps/api/tauri'
+
+import type { ParamsObject } from '../util.js'
+import { prepareBindValues } from '../util.js'
+import { BaseBackend } from './base.js'
+import type { SelectResponse } from './index.js'
+
+export type BackendOptionsTauri = {
+  type: 'tauri'
+  dbDirPath: string
+  appDbFileName: string
+  otelTracer?: otel.Tracer
+  parentSpan: otel.Span
+}
+
+export class TauriBackend extends BaseBackend {
+  constructor(
+    readonly dbFilePath: string,
+    readonly dbDirPath: string,
+    readonly otelTracer: otel.Tracer,
+    readonly parentSpan: otel.Span,
+  ) {
+    super()
+  }
+
+  static load = async ({
+    dbDirPath,
+    appDbFileName,
+    otelTracer,
+    parentSpan,
+  }: BackendOptionsTauri): Promise<TauriBackend> => {
+    const dbFilePath = `${dbDirPath}/${appDbFileName}`
+    await invoke('initialize_connection', { dbName: dbFilePath, otelData: getOtelData_(parentSpan) })
+
+    return new TauriBackend(dbFilePath, dbDirPath, otelTracer ?? makeNoopTracer(), parentSpan)
+  }
+
+  execute = (query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): void => {
+    // console.log({ query, bindValues, prepared: prepareBindValues(bindValues ?? {}, query) })
+    void invoke('execute', {
+      dbName: this.dbFilePath,
+      query,
+      values: prepareBindValues(bindValues ?? {}, query),
+      otelData: this.getOtelData(parentSpan),
+    })
+  }
+
+  select = async <T>(query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): Promise<SelectResponse<T>> => {
+    return invoke('select', {
+      db: this.dbFilePath,
+      query,
+      values: bindValues ?? {},
+      otelData: this.getOtelData(parentSpan),
+    })
+  }
+
+  getPersistedData = async (parentSpan?: otel.Span): Promise<Uint8Array> => {
+    const headers = new Headers()
+    headers.set('traceparent', getTraceParentHeader(parentSpan ?? this.parentSpan))
+
+    return fetch(`http://localhost:38787/get-persisted-data?file_path=${this.dbFilePath}`, { headers }).then(
+      (response) => response.arrayBuffer().then((buffer) => new Uint8Array(buffer)),
+    )
+  }
+
+  private getOtelData = (parentSpan?: otel.Span) => getOtelData_(parentSpan ?? this.parentSpan)!
+}
+
+const getOtelData_ = (parentSpan: otel.Span | undefined) => {
+  const spanContext = parentSpan?.spanContext()
+  return spanContext ? { trace_id: spanContext.traceId, span_id: spanContext.spanId } : undefined
+}

package/src/backends/utils/idb.ts

@@ -0,0 +1,71 @@
+/* eslint-disable unicorn/prefer-add-event-listener */
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+
+export class IDB {
+  private db: IDBDatabase | null = null
+
+  constructor(
+    private dbName: string,
+    private storeName: string = 'binary_store',
+  ) {}
+
+  private async open(): Promise<IDBDatabase> {
+    if (this.db) return this.db
+
+    return new Promise((resolve, reject) => {
+      const openRequest = indexedDB.open(this.dbName, 1)
+
+      openRequest.onupgradeneeded = () => {
+        const db = openRequest.result
+        if (!db.objectStoreNames.contains(this.storeName)) {
+          db.createObjectStore(this.storeName)
+        }
+      }
+
+      openRequest.onsuccess = () => {
+        this.db = openRequest.result
+        resolve(this.db)
+      }
+
+      openRequest.onerror = () => {
+        reject(new Error('Failed to open database.'))
+      }
+    })
+  }
+
+  public async get(key: string): Promise<Uint8Array | undefined> {
+    const db = await this.open()
+
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.storeName, 'readonly')
+      const store = transaction.objectStore(this.storeName)
+      const getRequest = store.get(key)
+
+      getRequest.onsuccess = () => {
+        resolve(getRequest.result)
+      }
+
+      getRequest.onerror = () => {
+        reject(new Error('Failed to get data.'))
+      }
+    })
+  }
+
+  public async put(key: string, value: Uint8Array): Promise<void> {
+    const db = await this.open()
+
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.storeName, 'readwrite')
+      const store = transaction.objectStore(this.storeName)
+      const putRequest = store.put(value, key)
+
+      putRequest.onsuccess = () => {
+        resolve()
+      }
+
+      putRequest.onerror = () => {
+        reject(new Error('Failed to write data.'))
+      }
+    })
+  }
+}

package/src/backends/web-in-memory.ts

@@ -0,0 +1,65 @@
+import { makeNoopTracer } from '@livestore/utils'
+import type * as otel from '@opentelemetry/api'
+import type * as SqliteWasm from 'sqlite-esm'
+import sqlite3InitModule from 'sqlite-esm'
+
+import type { ParamsObject } from '../util.js'
+import { prepareBindValues } from '../util.js'
+import { BaseBackend } from './base.js'
+import type { SelectResponse } from './index.js'
+
+export type BackendOptionsWebInMemory = {
+  type: 'web-in-memory'
+  /** Specifies where to persist data for this backend */
+  otelTracer?: otel.Tracer
+}
+
+declare type DatabaseWithCAPI = SqliteWasm.Database & { capi: SqliteWasm.CAPI }
+
+// NOTE: This backend is currently only used for testing
+export class WebInMemoryBackend extends BaseBackend {
+  constructor(
+    readonly otelTracer: otel.Tracer,
+    readonly db: DatabaseWithCAPI,
+  ) {
+    super()
+  }
+
+  static load = async (options: BackendOptionsWebInMemory): Promise<WebInMemoryBackend> => {
+    const sqlite3 = await sqlite3InitModule({
+      print: (message) => console.log(`[sql-client] ${message}`),
+      printErr: (message) => console.error(`[sql-client] ${message}`),
+    })
+    const db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' }) as DatabaseWithCAPI
+    db.capi = sqlite3.capi
+
+    return new WebInMemoryBackend(options.otelTracer ?? makeNoopTracer(), db)
+  }
+
+  execute = (query: string, bindValues?: ParamsObject): void => {
+    this.db.exec({
+      sql: query,
+      bind: prepareBindValues(bindValues ?? {}, query) as TODO,
+      returnValue: 'resultRows',
+      rowMode: 'object',
+    })
+  }
+
+  select = async <T>(query: string, bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
+    const resultRows: T[] = []
+
+    this.db.exec({
+      sql: query,
+      bind: prepareBindValues(bindValues ?? {}, query) as TODO,
+      rowMode: 'object',
+      resultRows,
+      // callback: (row: any) => console.log('select result', db.filename, query, row),
+    } as TODO)
+
+    return { results: resultRows }
+  }
+
+  getPersistedData = async (): Promise<Uint8Array> => {
+    return this.db.capi.sqlite3_js_db_export(this.db.pointer)
+  }
+}

package/src/backends/web-worker.ts

@@ -0,0 +1,176 @@
+// Web Worker file for running SQLite in a web worker.
+
+// TODO: create types for these libraries? SQL.js already should have types;
+// we just need the types to apply to the fork.
+import { shouldNeverHappen } from '@livestore/utils'
+import * as Comlink from 'comlink'
+import type * as SqliteWasm from 'sqlite-esm'
+import sqlite3InitModule from 'sqlite-esm'
+
+// import { v4 as uuid } from 'uuid'
+import type { Bindable } from '../util.js'
+import { casesHandled, sql } from '../util.js'
+import type { SelectResponse, WritableDatabaseLocation } from './index.js'
+import { IDB } from './utils/idb.js'
+
+// A global variable to hold the database connection.
+// let db: SqliteWasm.Database
+let db: SqliteWasm.DatabaseApi
+
+let sqlite3: SqliteWasm.Sqlite3Static
+
+// TODO get rid of this in favour of a "proper" IDB SQLite backend
+let idb: IDB | undefined
+
+/** The location where this database backend persists its data */
+let persistentDatabaseLocation_: WritableDatabaseLocation
+
+const configureConnection = () =>
+  db.exec(sql`
+    PRAGMA page_size=8192;
+    PRAGMA journal_mode=MEMORY;
+    PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+  `)
+
+/** A full virtual filename in the IDB FS */
+const fullyQualifiedFilename = (name: string) => `${name}.db`
+
+const initialize = async ({ persistentDatabaseLocation }: { persistentDatabaseLocation: WritableDatabaseLocation }) => {
+  persistentDatabaseLocation_ = persistentDatabaseLocation
+
+  sqlite3 = await sqlite3InitModule({
+    print: (message) => console.log(`[sql-client] ${message}`),
+    printErr: (message) => console.error(`[sql-client] ${message}`),
+  })
+
+  switch (persistentDatabaseLocation.type) {
+    case 'opfs': {
+      try {
+        db = new sqlite3.oo1.OpfsDb(fullyQualifiedFilename(persistentDatabaseLocation.virtualFilename)) // , 'c'
+      } catch (e) {
+        debugger
+      }
+      break
+    }
+    case 'indexeddb': {
+      try {
+        db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' })
+        idb = new IDB(persistentDatabaseLocation.virtualFilename)
+
+        const bytes = await idb.get('db')
+
+        if (bytes !== undefined) {
+          // Based on https://sqlite.org/forum/forumpost/2119230da8ac5357a13b731f462dc76e08621a4a29724f7906d5f35bb8508465
+          // TODO find cleaner way to do this once possible in sqlite3-wasm
+          const p = sqlite3.wasm.allocFromTypedArray(bytes)
+          const _rc = sqlite3.capi.sqlite3_deserialize(db.pointer, 'main', p, bytes.length, bytes.length, 0)
+        }
+      } catch (e) {
+        debugger
+      }
+      break
+    }
+    case 'filesystem': {
+      throw new Error('Persisting to native FS is not supported in the web worker backend')
+    }
+    case 'volatile-in-memory': {
+      break
+    }
+    default: {
+      casesHandled(persistentDatabaseLocation)
+    }
+  }
+
+  configureConnection()
+}
+
+// TODO get rid of this in favour of a "proper" IDB SQLite backend
+let idbPersistTimeout: NodeJS.Timeout | undefined
+
+type ExecutionQueueItem = { query: string; bindValues?: Bindable }
+
+const executeBulk = (executionItems: ExecutionQueueItem[]): void => {
+  let batchItems: ExecutionQueueItem[] = []
+
+  while (executionItems.length > 0) {
+    try {
+      db.exec('BEGIN TRANSACTION') // Start the transaction
+
+      batchItems = executionItems.splice(0, 50)
+
+      for (const { query, bindValues } of batchItems) {
+        db.exec({
+          sql: query,
+          bind: bindValues as TODO,
+          returnValue: 'resultRows',
+          rowMode: 'object',
+        })
+      }
+
+      db.exec('COMMIT') // Commit the transaction
+    } catch (error) {
+      try {
+        db.exec('ROLLBACK') // Rollback in case of an error
+      } catch (e) {
+        console.error('Error rolling back transaction', e)
+      }
+
+      shouldNeverHappen(`Error executing query: ${error} \n ${JSON.stringify(batchItems)}`)
+    }
+  }
+
+  // TODO get rid of this in favour of a "proper" IDB SQLite backend
+  if (persistentDatabaseLocation_.type === 'indexeddb') {
+    if (idbPersistTimeout !== undefined) {
+      clearTimeout(idbPersistTimeout)
+    }
+
+    idbPersistTimeout = setTimeout(() => {
+      const data = sqlite3.capi.sqlite3_js_db_export(db.pointer) as Uint8Array
+
+      void idb!.put('db', data)
+    }, 1000)
+  }
+}
+
+const select = <T = any>(query: string, bindValues?: Bindable): SelectResponse<T> => {
+  const resultRows: T[] = []
+
+  db.exec({
+    sql: query,
+    bind: bindValues,
+    rowMode: 'object',
+    resultRows,
+  } as TODO)
+
+  return { results: resultRows }
+}
+
+const getPersistedData = async (): Promise<Uint8Array> => {
+  // TODO get rid of this in favour of a "proper" IDB SQLite backend
+  if (persistentDatabaseLocation_.type === 'indexeddb') {
+    const data = sqlite3.capi.sqlite3_js_db_export(db.pointer)
+    return Comlink.transfer(data, [data.buffer])
+  }
+
+  const rootHandle = await navigator.storage.getDirectory()
+  const fileHandle = await rootHandle.getFileHandle(db.filename)
+  const file = await fileHandle.getFile()
+  const buffer = await file.arrayBuffer()
+  const data = new Uint8Array(buffer)
+
+  return Comlink.transfer(data, [data.buffer])
+}
+
+const wrappedWorker = { initialize, executeBulk, select, getPersistedData }
+
+export type WrappedWorker = typeof wrappedWorker
+
+Comlink.expose(wrappedWorker)
+
+// NOTE keep this around for debugging
+// db.exec({
+//   sql: `select * from sqlite_master where name = 'library_tracks'`,
+//   callback: (_: TODO) => console.log(_),
+//   rowMode: 'object',
+// } as TODO)

package/src/backends/web.ts

@@ -0,0 +1,96 @@
+import { makeNoopTracer } from '@livestore/utils'
+import type * as otel from '@opentelemetry/api'
+import * as Comlink from 'comlink'
+
+import type { ParamsObject } from '../util.js'
+import { prepareBindValues } from '../util.js'
+import { BaseBackend } from './base.js'
+import type { SelectResponse, WritableDatabaseLocation } from './index.js'
+import type { WrappedWorker } from './web-worker.js'
+
+export type BackendOptionsWeb = {
+  type: 'web'
+  /** Specifies where to persist data for this backend */
+  persistentDatabaseLocation: WritableDatabaseLocation
+  otelTracer?: otel.Tracer
+}
+
+export class WebWorkerBackend extends BaseBackend {
+  worker: Comlink.Remote<WrappedWorker>
+  persistentDatabaseLocation: WritableDatabaseLocation
+  otelTracer: otel.Tracer
+
+  executionBacklog: { query: string; bindValues?: ParamsObject }[] = []
+  executionPromise: Promise<void> | undefined = undefined
+
+  private constructor({
+    worker,
+    persistentDatabaseLocation,
+    otelTracer,
+  }: {
+    worker: Comlink.Remote<WrappedWorker>
+    persistentDatabaseLocation: WritableDatabaseLocation
+    otelTracer: otel.Tracer
+  }) {
+    super()
+    this.worker = worker
+    this.persistentDatabaseLocation = persistentDatabaseLocation
+    this.otelTracer = otelTracer
+  }
+
+  static load = async ({ persistentDatabaseLocation, otelTracer }: BackendOptionsWeb): Promise<WebWorkerBackend> => {
+    // TODO: Importing the worker like this only works with Vite;
+    // should this really be inside the LiveStore library?
+    // Doesn't work with Firefox right now during dev https://bugzilla.mozilla.org/show_bug.cgi?id=1247687
+    const worker = new Worker(new URL('./web-worker.js', import.meta.url), {
+      type: 'module',
+    })
+    const wrappedWorker = Comlink.wrap<WrappedWorker>(worker)
+
+    await wrappedWorker.initialize({ persistentDatabaseLocation })
+
+    return new WebWorkerBackend({
+      worker: wrappedWorker,
+      persistentDatabaseLocation,
+      otelTracer: otelTracer ?? makeNoopTracer(),
+    })
+  }
+
+  execute = (query: string, bindValues_?: ParamsObject) => {
+    const bindValues = prepareBindValues(bindValues_ ?? {}, query)
+    this.executionBacklog.push({ query, bindValues })
+
+    // Instead of sending the queries to the worker immediately, we wait a bit and batch them up (which reduces the number of messages sent to the worker)
+    if (this.executionPromise === undefined) {
+      this.executionPromise = new Promise((resolve) => {
+        setTimeout(() => {
+          void this.worker.executeBulk(this.executionBacklog)
+          this.executionBacklog = []
+          this.executionPromise = undefined
+
+          resolve()
+        }, 10)
+      })
+    }
+  }
+
+  select = async <T>(query: string, bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
+    // NOTE we need to wait for the executionBacklog to be worked off, before we run the select query (as it might depend on the previous execution queries)
+    await this.executionPromise
+
+    try {
+      const response = (await this.worker.select(query, bindValues)) as SelectResponse<T>
+      return response
+    } catch (e) {
+      console.error(`Error while executing query via "select": ${query}`)
+      throw e
+    }
+  }
+
+  getPersistedData = async (_parentSpan?: otel.Span): Promise<Uint8Array> => {
+    // NOTE we need to wait for the executionBacklog to be worked off
+    await this.executionPromise
+
+    return this.worker.getPersistedData()
+  }
+}