@livestore/livestore 0.0.12 → 0.0.13

package/src/store.ts

@@ -1,28 +1,30 @@
 import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core'
 import { assertNever, makeNoopSpan, makeNoopTracer, shouldNeverHappen } from '@livestore/utils'
+import { identity } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 import type { GraphQLSchema } from 'graphql'
 import * as graphql from 'graphql'
 import { uniqueId } from 'lodash-es'
 import * as ReactDOM from 'react-dom'
+import initSqlite3Wasm from 'sqlite-esm'
 import { v4 as uuid } from 'uuid'
 
-import type { Backend, BackendOptions } from './backends/index.js'
-import { createBackend } from './backends/index.js'
 import type { ComponentKey } from './componentKey.js'
 import { tableNameForComponentKey } from './componentKey.js'
 import type { LiveStoreEvent } from './events.js'
 import { InMemoryDatabase } from './inMemoryDatabase.js'
+import { migrateDb } from './migrations.js'
 import { getDurationMsFromSpan } from './otel.js'
 import type { GetAtom, Ref } from './reactive.js'
 import { ReactiveGraph } from './reactive.js'
 import { LiveStoreGraphQLQuery } from './reactiveQueries/graphql.js'
 import { LiveStoreJSQuery } from './reactiveQueries/js.js'
 import { LiveStoreSQLQuery } from './reactiveQueries/sql.js'
-import type { ActionDefinition, GetActionArgs, Schema } from './schema.js'
-import { componentStateTables, loadSchema } from './schema.js'
+import type { ActionDefinition, GetActionArgs, Schema, SQLWriteStatement } from './schema.js'
+import { componentStateTables } from './schema.js'
+import type { Storage, StorageInit } from './storage/index.js'
 import type { Bindable, ParamsObject } from './util.js'
-import { sql } from './util.js'
+import { isPromise, sql } from './util.js'
 
 export type LiveStoreQuery<TResult extends Record<string, any> = any> =
   | LiveStoreSQLQuery<TResult>
@@ -55,7 +57,7 @@ export type GraphQLOptions<TContext> = {
 export type StoreOptions<TGraphQLContext extends BaseGraphQLContext> = {
   db: InMemoryDatabase
   schema: Schema
-  backend?: Backend
+  storage?: Storage
   graphQLOptions?: GraphQLOptions<TGraphQLContext>
   otelTracer: otel.Tracer
   otelRootSpanContext: otel.Context
@@ -111,13 +113,13 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
    */
   tableRefs: { [key: string]: Ref<null> }
   activeQueries: Set<LiveStoreQuery>
-  backend?: Backend
+  storage?: Storage
   temporaryQueries: Set<LiveStoreQuery> | undefined
 
   private constructor({
     db,
     schema,
-    backend,
+    storage,
     graphQLOptions,
     otelTracer,
     otelRootSpanContext,
@@ -133,7 +135,7 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
     // TODO generalize the `tableRefs` concept to allow finer-grained refs
     this.tableRefs = {}
     this.activeQueries = new Set()
-    this.backend = backend
+    this.storage = storage
 
     const applyEventsSpan = otelTracer.startSpan('LiveStore:applyEvents', {}, otelRootSpanContext)
     const otelApplyEventsSpanContext = otel.trace.setSpan(otel.context.active(), applyEventsSpan)
@@ -642,7 +644,7 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
               try {
                 const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
-                // TODO: what to do about backend transaction here?
+                // TODO: what to do about storage transaction here?
                 this.inMemoryDB.txn(() => {
                   for (const event of events) {
                     try {
@@ -757,6 +759,15 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
               }
             },
           },
+
+          RawSql: {
+            statement: ({ sql, writeTables }: { sql: string; writeTables: string[] }) => ({
+              sql,
+              writeTables,
+              argsAlreadyBound: false,
+            }),
+            prepareBindValues: ({ bindValues }) => bindValues,
+          },
         }
 
         const actionDefinition = actionDefinitions[eventType] ?? shouldNeverHappen(`Unknown event type: ${eventType}`)
@@ -765,20 +776,25 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
         const eventWithId: LiveStoreEvent = { id: uuid(), type: eventType, args }
 
         // Synchronously apply the event to the in-memory database
-        const { durationMs } = this.inMemoryDB.applyEvent(eventWithId, actionDefinition, otelContext)
+        // const { durationMs } = this.inMemoryDB.applyEvent(eventWithId, actionDefinition, otelContext)
+        const { statement, bindValues } = eventToSql(eventWithId, actionDefinition)
+        const { durationMs } = this.inMemoryDB.execute(statement.sql, bindValues, statement.writeTables, {
+          otelContext,
+        })
 
-        // Asynchronously apply the event to a persistent backend (we're not awaiting this promise here)
-        if (this.backend !== undefined) {
-          this.backend.applyEvent(eventWithId, actionDefinition, span)
+        // Asynchronously apply the event to a persistent storage (we're not awaiting this promise here)
+        if (this.storage !== undefined) {
+          // this.storage.applyEvent(eventWithId, actionDefinition, span)
+          this.storage.execute(statement.sql, bindValues, span)
         }
 
         // Uncomment to print a list of queries currently registered on the store
         // console.log(JSON.parse(JSON.stringify([...this.queries].map((q) => `${labelForKey(q.componentKey)}/${q.label}`))))
 
-        const statement =
-          typeof actionDefinition.statement === 'function'
-            ? actionDefinition.statement(args)
-            : actionDefinition.statement
+        // const statement =
+        //   typeof actionDefinition.statement === 'function'
+        //     ? actionDefinition.statement(args)
+        //     : actionDefinition.statement
 
         span.end()
 
@@ -795,9 +811,9 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
   execute = async (query: string, params: ParamsObject = {}, writeTables?: string[]) => {
     this.inMemoryDB.execute(query, params, writeTables)
 
-    if (this.backend !== undefined) {
+    if (this.storage !== undefined) {
       const parentSpan = otel.trace.getSpan(otel.context.active())
-      this.backend.execute(query, params, parentSpan)
+      this.storage.execute(query, params, parentSpan)
     }
   }
 }
@@ -805,83 +821,100 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
 /** Create a new LiveStore Store */
 export const createStore = async <TGraphQLContext extends BaseGraphQLContext>({
   schema,
-  backendOptions,
+  loadStorage,
   graphQLOptions,
   otelTracer = makeNoopTracer(),
   otelRootSpanContext = otel.context.active(),
   boot,
 }: {
   schema: Schema
-  backendOptions: BackendOptions
+  loadStorage: () => StorageInit | Promise<StorageInit>
   graphQLOptions?: GraphQLOptions<TGraphQLContext>
   otelTracer?: otel.Tracer
   otelRootSpanContext?: otel.Context
-  boot?: (backend: Backend, parentSpan: otel.Span) => Promise<void>
+  boot?: (db: InMemoryDatabase, parentSpan: otel.Span) => unknown | Promise<unknown>
 }): Promise<Store<TGraphQLContext>> => {
   return otelTracer.startActiveSpan('createStore', {}, otelRootSpanContext, async (span) => {
     try {
-      let persistedData: Uint8Array | undefined
-      const backend = await createBackend(backendOptions, {
-        otelTracer: otelTracer ?? makeNoopTracer(),
-        parentSpan: otel.trace.getSpan(otelRootSpanContext ?? otel.context.active()) ?? makeNoopSpan(),
-      })
-      // if we're resetting the database, run boot here.
+      const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
-      let shouldResetDB = false
-      // Uncomment this line if you want to reset the database contents.
-      // let shouldResetDB = true
+      const loadStorageAndPersistedData = async () => {
+        const storage = await otelTracer.startActiveSpan('storage:load', {}, otelContext, async (span) => {
+          try {
+            const init = await loadStorage()
+            const parentSpan = otel.trace.getSpan(otel.context.active()) ?? makeNoopSpan()
+            return init({ otelTracer, parentSpan })
+          } finally {
+            span.end()
+          }
+        })
 
-      const existingTablesRaw = await backend.select(
-        sql`SELECT * FROM sqlite_master WHERE type='table';`,
-        undefined,
-        span,
-      )
-      const existingTables = existingTablesRaw.results.map((t: { name: string }) => t.name)
-      const missingTables = Object.keys(schema.tables).filter((tableName) => !existingTables.includes(tableName))
-      if (existingTables.length === 0) {
-        console.log('No existing tables found, loading from schema')
-        shouldResetDB = true
-      } else if (
-        missingTables.length > 0 &&
-        window.confirm(
-          `Existing DB is missing ${missingTables.length} tables: ${missingTables.join(
-            ', ',
-          )}\n\nReset DB? This will reset all of the following tables to empty: ${Object.keys(schema).join(', ')}`,
+        const persistedData = await otelTracer.startActiveSpan(
+          'storage:getPersistedData',
+          {},
+          otelContext,
+          async (span) => {
+            try {
+              return await storage.getPersistedData(span)
+            } finally {
+              span.end()
+            }
+          },
         )
-      ) {
-        shouldResetDB = true
-      }
 
-      if (localStorage.getItem(RESET_DB_LOCAL_STORAGE_KEY) !== null) {
-        shouldResetDB = true
+        return { storage, persistedData }
       }
 
-      if (shouldResetDB) {
-        await loadSchema(backend, schema)
-        localStorage.removeItem(RESET_DB_LOCAL_STORAGE_KEY)
-      }
+      const loadSqlite3 = () =>
+        initSqlite3Wasm({
+          // Required to load the wasm binary asynchronously. Of course, you can host it wherever you want
+          // You can omit locateFile completely when running in node
+          // locateFile: () => `/sql-wasm.wasm`,
+          print: (message) => console.log(`[livestore sqlite] ${message}`),
+          printErr: (message) => console.error(`[livestore sqlite] ${message}`),
+        })
 
-      if (boot) {
-        await boot(backend!, span)
-      }
+      const [{ storage, persistedData }, sqlite3] = await Promise.all([loadStorageAndPersistedData(), loadSqlite3()])
 
-      const otelContext = otel.trace.setSpan(otel.context.active(), span)
-      await otelTracer.startActiveSpan('backend-getPersistedData', {}, otelContext, async (span) => {
+      const db = InMemoryDatabase.load(persistedData, otelTracer, otelRootSpanContext, sqlite3)
+
+      // Proxy to `db` that also mirrors `execute` calls to `storage`
+      const dbProxy = new Proxy(db, {
+        get: (db, prop, receiver) => {
+          if (prop === 'execute') {
+            const execute: InMemoryDatabase['execute'] = (query, bindValues, writeTables, options) => {
+              storage.execute(query, bindValues, span)
+              return db.execute(query, bindValues, writeTables, options)
+            }
+            return execute
+          } else {
+            return Reflect.get(db, prop, receiver)
+          }
+        },
+      })
+
+      otelTracer.startActiveSpan('migrateDb', {}, otelContext, async (span) => {
         try {
-          persistedData = await backend!.getPersistedData(span)
+          const otelContext = otel.trace.setSpan(otel.context.active(), span)
+          migrateDb({ db: dbProxy, schema, otelContext })
         } finally {
           span.end()
         }
       })
 
-      const db: InMemoryDatabase = await InMemoryDatabase.load(persistedData, otelTracer, otelRootSpanContext)
-      configureSQLite(db)
+      if (boot !== undefined) {
+        const booting = boot(dbProxy, span)
+        // NOTE only awaiting if it's actually a promise to avoid unnecessary async/await
+        if (isPromise(booting)) {
+          await booting
+        }
+      }
 
       // TODO: we can't apply the schema at this point, we've already loaded persisted data!
       // Think about what to do about this case.
       // await applySchema(db, schema)
       return Store.createStore<TGraphQLContext>(
-        { db, schema, backend, graphQLOptions, otelTracer, otelRootSpanContext },
+        { db, schema, storage, graphQLOptions, otelTracer, otelRootSpanContext },
         span,
       )
     } finally {
@@ -890,17 +923,17 @@ export const createStore = async <TGraphQLContext extends BaseGraphQLContext>({
   })
 }
 
-/** Set up SQLite performance; hasn't been super carefully optimized yet. */
-const configureSQLite = (db: InMemoryDatabase) => {
-  db.execute(
-    // TODO: revisit these tuning parameters for max performance
-    sql`
-      PRAGMA page_size=32768;
-      PRAGMA cache_size=10000;
-      PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
-      PRAGMA synchronous='OFF';
-      PRAGMA temp_store='MEMORY';
-      PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
-    `,
-  )
+const eventToSql = (
+  event: LiveStoreEvent,
+  eventDefinition: ActionDefinition,
+): { statement: SQLWriteStatement; bindValues: ParamsObject } => {
+  const statement =
+    typeof eventDefinition.statement === 'function' ? eventDefinition.statement(event.args) : eventDefinition.statement
+
+  const prepareBindValues = eventDefinition.prepareBindValues ?? identity
+
+  const bindValues =
+    typeof eventDefinition.statement === 'function' && statement.argsAlreadyBound ? {} : prepareBindValues(event.args)
+
+  return { statement, bindValues }
 }

package/src/util.ts

@@ -1,5 +1,7 @@
+/// <reference lib="es2022" />
+
 export type ParamsObject = Record<string, SqlValue>
-export type SqlValue = string | number | Uint8Array
+export type SqlValue = string | number | Uint8Array | null
 
 export type Bindable = SqlValue[] | ParamsObject
 
@@ -57,3 +59,5 @@ export const objectToString = (error: any): string => {
     return 'Error while printing error: ' + e
   }
 }
+
+export const isPromise = (value: any): value is Promise<unknown> => typeof value?.then === 'function'

package/tsconfig.json

@@ -9,7 +9,5 @@
     "tsBuildInfoFile": "./dist/.tsbuildinfo"
   },
   "include": ["./src"],
-  "references": [
-    { "path": "../utils" },
-  ]
+  "references": [{ "path": "../../effect-db-schema" }, { "path": "../utils" }]
 }

package/src/backends/base.ts

@@ -1,67 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unused-vars */
-import { errorToString } from '@livestore/utils'
-import { identity } from '@livestore/utils/effect'
-import * as otel from '@opentelemetry/api'
-
-import type { LiveStoreEvent } from '../events.js'
-// import { EVENTS_TABLE_NAME } from '../events.js'
-import type { ActionDefinition } from '../schema.js'
-import type { ParamsObject } from '../util.js'
-import type { Backend, SelectResponse } from './index.js'
-
-export abstract class BaseBackend implements Backend {
-  abstract otelTracer: otel.Tracer
-
-  select = async <T = any>(query: string, bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
-    throw new Error('Method not implemented.')
-  }
-
-  execute = (query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): void => {
-    throw new Error('Method not implemented.')
-  }
-
-  getPersistedData = async (parentSpan?: otel.Span): Promise<Uint8Array> => {
-    throw new Error('Method not implemented.')
-  }
-
-  // TODO move `applyEvent` logic to Store and only call `execute` here
-  applyEvent = (event: LiveStoreEvent, eventDefinition: ActionDefinition, parentSpan?: otel.Span): void => {
-    const ctx = parentSpan ? otel.trace.setSpan(otel.context.active(), parentSpan) : otel.context.active()
-    this.otelTracer.startActiveSpan('LiveStore:backend:applyEvent', {}, ctx, (span) => {
-      try {
-        // Careful: this SQL statement is duplicated in the backend.
-        // Remember to update it in src-tauri/src/store.rs:apply_event as well.
-        // await this.execute(sql`insert into ${EVENTS_TABLE_NAME} (id, type, args) values ($id, $type, $args)`, {
-        //   id: event.id,
-        //   type: event.type,
-        //   args: JSON.stringify(event.args ?? {}),
-        // })
-
-        const statement =
-          typeof eventDefinition.statement === 'function'
-            ? eventDefinition.statement(event.args)
-            : eventDefinition.statement
-
-        const prepareBindValues = eventDefinition.prepareBindValues ?? identity
-
-        const bindValues =
-          typeof eventDefinition.statement === 'function' && statement.argsAlreadyBound
-            ? {}
-            : prepareBindValues(event.args)
-
-        span.setAttributes({
-          'livestore.statement.sql': statement.sql,
-          'livestore.statement.writeTables': statement.writeTables,
-          'livestore.statement.bindVales': JSON.stringify(bindValues),
-        })
-
-        this.execute(statement.sql, bindValues, span)
-      } catch (e: any) {
-        span.setStatus({ code: otel.SpanStatusCode.ERROR, message: errorToString(e) })
-        throw e
-      } finally {
-        span.end()
-      }
-    })
-  }
-}

package/src/backends/index.ts

@@ -1,98 +0,0 @@
-// A backend represents a raw SQLite database.
-// Examples include:
-// - A native SQLite process running in a Tauri Rust process
-// - A SQL.js WASM version of SQLite running in a web worker
-//
-// We can send commands to execute various kinds of queries,
-// and respond to various events from the database.
-
-import type * as otel from '@opentelemetry/api'
-
-import type { LiveStoreEvent } from '../events.js'
-import type { ActionDefinition } from '../schema.js'
-import type { ParamsObject } from '../util.js'
-import { casesHandled } from '../util.js'
-import type { BackendOptionsTauri } from './tauri.js'
-import type { BackendOptionsWeb } from './web.js'
-import { WebWorkerBackend } from './web.js'
-import type { BackendOptionsWebInMemory } from './web-in-memory.js'
-import { WebInMemoryBackend } from './web-in-memory.js'
-
-/* A location of a persistent writable SQLite file */
-export type WritableDatabaseLocation =
-  | {
-      type: 'opfs'
-      virtualFilename: string
-    }
-  | {
-      type: 'indexeddb'
-      virtualFilename: string
-    }
-  | {
-      type: 'filesystem'
-      directory: string
-      filename: string
-    }
-  | {
-      type: 'volatile-in-memory'
-    }
-
-export interface Backend {
-  // Select some data from the DB.
-  // This should only do reads, not writes, but we don't strongly enforce that.
-  select<T = any>(query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): Promise<SelectResponse<T>>
-
-  // Execute a query where you don't care about the result.
-  // Used for writes and configuration changes.
-  execute(query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): void
-
-  /** Apply an event to the backend */
-  applyEvent(event: LiveStoreEvent, eventDefiniton: ActionDefinition, parentSpan?: otel.Span): void
-
-  /** Return a snapshot of persisted data from the backend */
-  getPersistedData(parentSpan?: otel.Span): Promise<Uint8Array>
-}
-
-export type BackendType = 'tauri' | 'web' | 'web-in-memory'
-
-export const isBackendType = (type: string): type is BackendType => {
-  return type === 'tauri' || type === 'web' || type === 'web-in-memory'
-}
-
-export type SelectResponse<T = any> = {
-  results: T[]
-
-  // other perf stats metadata about how long the query took
-  [key: string]: any
-}
-
-export enum IndexType {
-  Basic = 'Basic',
-  FullText = 'FullText',
-}
-
-export type BackendOptions = BackendOptionsTauri | BackendOptionsWeb | BackendOptionsWebInMemory
-export type BackendOtelProps = {
-  otelTracer: otel.Tracer
-  parentSpan: otel.Span
-}
-
-export const createBackend = async (options: BackendOptions, otelProps: BackendOtelProps): Promise<Backend> => {
-  switch (options.type) {
-    case 'tauri': {
-      // NOTE Dynamic import is needed to avoid Tauri is a dependency of LiveStore (e.g. when used in the web)
-      const { TauriBackend } = await import('./tauri.js')
-      return await TauriBackend.load(options, otelProps)
-    }
-    case 'web': {
-      return WebWorkerBackend.load(options, otelProps)
-    }
-    // NOTE currently only needed for testing
-    case 'web-in-memory': {
-      return WebInMemoryBackend.load(options, otelProps)
-    }
-    default: {
-      casesHandled(options)
-    }
-  }
-}

package/src/backends/noop.ts

@@ -1,32 +0,0 @@
-import { makeNoopTracer } from '@livestore/utils'
-import type * as otel from '@opentelemetry/api'
-
-import type { ParamsObject } from '../util.js'
-import { BaseBackend } from './base.js'
-import type { SelectResponse } from './index.js'
-
-export type BackendOptionsNoop = {
-  type: 'noop'
-  /** Specifies where to persist data for this backend */
-  otelTracer?: otel.Tracer
-}
-
-export class NoopBackend extends BaseBackend {
-  constructor(readonly otelTracer: otel.Tracer) {
-    super()
-  }
-
-  static load = async (options: BackendOptionsNoop): Promise<NoopBackend> => {
-    return new NoopBackend(options.otelTracer ?? makeNoopTracer())
-  }
-
-  execute = (_query: string, _bindValues?: ParamsObject): void => {}
-
-  select = async <T>(_query: string, _bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
-    return { results: [] }
-  }
-
-  getPersistedData = async (): Promise<Uint8Array> => {
-    return new Uint8Array()
-  }
-}

package/src/backends/web-in-memory.ts

@@ -1,65 +0,0 @@
-import type * as otel from '@opentelemetry/api'
-import type * as SqliteWasm from 'sqlite-esm'
-import sqlite3InitModule from 'sqlite-esm'
-
-import type { ParamsObject } from '../util.js'
-import { prepareBindValues } from '../util.js'
-import { BaseBackend } from './base.js'
-import type { BackendOtelProps, SelectResponse } from './index.js'
-
-export type BackendOptionsWebInMemory = {
-  type: 'web-in-memory'
-}
-
-declare type DatabaseWithCAPI = SqliteWasm.Database & { capi: SqliteWasm.CAPI }
-
-// NOTE: This backend is currently only used for testing
-export class WebInMemoryBackend extends BaseBackend {
-  constructor(
-    readonly otelTracer: otel.Tracer,
-    readonly db: DatabaseWithCAPI,
-  ) {
-    super()
-  }
-
-  static load = async (
-    _options: BackendOptionsWebInMemory,
-    { otelTracer }: BackendOtelProps,
-  ): Promise<WebInMemoryBackend> => {
-    const sqlite3 = await sqlite3InitModule({
-      print: (message) => console.log(`[sql-client] ${message}`),
-      printErr: (message) => console.error(`[sql-client] ${message}`),
-    })
-    const db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' }) as DatabaseWithCAPI
-    db.capi = sqlite3.capi
-
-    return new WebInMemoryBackend(otelTracer, db)
-  }
-
-  execute = (query: string, bindValues?: ParamsObject): void => {
-    this.db.exec({
-      sql: query,
-      bind: prepareBindValues(bindValues ?? {}, query) as TODO,
-      returnValue: 'resultRows',
-      rowMode: 'object',
-    })
-  }
-
-  select = async <T>(query: string, bindValues?: ParamsObject): Promise<SelectResponse<T>> => {
-    const resultRows: T[] = []
-
-    this.db.exec({
-      sql: query,
-      bind: prepareBindValues(bindValues ?? {}, query) as TODO,
-      rowMode: 'object',
-      resultRows,
-      // callback: (row: any) => console.log('select result', db.filename, query, row),
-    } as TODO)
-
-    return { results: resultRows }
-  }
-
-  getPersistedData = async (): Promise<Uint8Array> => {
-    return this.db.capi.sqlite3_js_db_export(this.db.pointer)
-  }
-}