@livestore/livestore 0.0.12 → 0.0.14

package/src/schema.ts

@@ -1,45 +1,10 @@
-import type { Backend } from './backends/index.js'
-import { EVENTS_TABLE_NAME } from './events.js'
-import type { InMemoryDatabase } from './inMemoryDatabase.js'
-import { sql } from './util.js'
-
-export type ColumnDefinition = {
-  nullable?: boolean
-  primaryKey?: boolean
-} & (
-  | { type: 'text'; default?: string }
-  | { type: 'json'; default?: string }
-  | { type: 'integer'; default?: number }
-  | { type: 'boolean'; default?: boolean }
-  | { type: 'real'; default?: number }
-  | { type: 'blob'; default?: any }
-) // sqlite uses numbers for booleans but we fake it
-
-// TODO: defaults should be nullable for nullable columns
-type ColumnDefinitionWithDefault = {
-  primaryKey?: boolean
-} & (
-  | { type: 'text'; nullable?: true; default: string }
-  | { type: 'json'; nullable?: true; default: string }
-  | { type: 'integer'; nullable?: true; default: number }
-  | { type: 'boolean'; nullable?: true; default: boolean }
-  | { type: 'real'; nullable: true; default: number | null }
-  | { type: 'blob'; nullable: true; default: any | null }
-)
-
-export type TableDefinition = {
-  columns: {
-    [key: string]: ColumnDefinition
-  }
-  /**
-   * Can be used for various purposes e.g. to provide a foreign key constraint like below:
-   * ```ts
-   * columnsRaw: (columnsStr) => `${columnsStr}, foreign key (userId) references users(id)`
-   * ```
-   */
-  columnsRaw?: (columnsStr: string) => string
-  indexes?: Index[]
-}
+import type { PrettifyFlat } from '@livestore/utils'
+import { mapObjectValues } from '@livestore/utils'
+import type { Schema } from '@livestore/utils/effect'
+import type { SqliteAst } from 'effect-db-schema'
+import { SqliteDsl } from 'effect-db-schema'
+
+import { DbSchema } from './index.js'
 
 export type Index = {
   name: string
@@ -48,35 +13,61 @@ export type Index = {
   isUnique?: boolean
 }
 
-export type ComponentStateSchema<T> = {
-  componentType: string
-  columns: {
-    [k in keyof T]: ColumnDefinitionWithDefault
+// A global variable representing component state tables we should create in the database
+export const componentStateTables: { [key: string]: SqliteAst.Table } = {}
+
+export type InputSchema = {
+  tables: {
+    [tableName: string]: SqliteDsl.TableDefinition<any, any>
   }
+  materializedViews?: MaterializedViewDefinitions
+  actions: ActionDefinitions<any>
 }
 
-// A global variable representing component state tables we should create in the database
-export const componentStateTables: { [key: string]: TableDefinition } = {}
+export const makeSchema = <TSchema extends InputSchema>(schema: TSchema): Schema =>
+  ({
+    tables: { ...mapObjectValues(schema.tables, (_tableName, table) => table.ast), ...systemTables },
+    materializedViews: schema.materializedViews ?? {},
+    actions: schema.actions,
+  }) satisfies Schema
+
+export type ComponentStateSchema = SqliteDsl.TableDefinition<any, any> & {
+  // TODO
+  register: () => void
+}
 
-export const defineComponentStateSchema = <T>(
-  schema: ComponentStateSchema<T>,
-): ComponentStateSchema<T & { id: string }> => {
-  const tablePath = `components__${schema.componentType}`
+// TODO get rid of "side effect" in this function (via explicit register fn)
+export const defineComponentStateSchema = <TName extends string, TColumns extends SqliteDsl.Columns>(
+  // TODO get rid of the `name` param here and use the `componentKey` name instead
+  name: TName,
+  columns: TColumns,
+): SqliteDsl.TableDefinition<
+  `components__${TName}`,
+  PrettifyFlat<TColumns & { id: SqliteDsl.ColumnDefinition<SqliteDsl.FieldType.FieldTypeText<string, string>, false> }>
+> => {
+  const tablePath = `components__${name}` as const
   if (Object.keys(componentStateTables).includes(tablePath)) {
     // throw new Error(`Can't register duplicate component: ${name}`)
     console.error(`Can't register duplicate component: ${tablePath}`)
   }
 
-  const schemaWithId = schema as ComponentStateSchema<T & { id: string }>
+  const schemaWithId = columns as unknown as PrettifyFlat<
+    TColumns & {
+      id: SqliteDsl.ColumnDefinition<SqliteDsl.FieldType.FieldTypeText<string, string>, false>
+    }
+  >
+
+  schemaWithId.id = DbSchema.text({ primaryKey: true })
 
-  schemaWithId.columns.id = { type: 'text', primaryKey: true } as any
+  const tableDef = SqliteDsl.table(tablePath, schemaWithId, [])
 
-  componentStateTables[tablePath] = schemaWithId as any
+  // TODO move into register fn
+  componentStateTables[tablePath] = tableDef.ast
 
-  return schemaWithId
+  return tableDef
 }
 
-type SQLWriteStatement = {
+export type SQLWriteStatement = {
   sql: string
 
   /** Tables written by the statement */
@@ -96,31 +87,31 @@ export type Schema = {
   actions: ActionDefinitions<any>
 }
 
-export type TableDefinitions = { [key: string]: TableDefinition }
+export type TableDefinitions = { [key: string]: SqliteAst.Table }
 export type MaterializedViewDefinitions = { [key: string]: {} }
 export type ActionDefinitions<TArgsMap extends Record<string, any>> = {
   [key in keyof TArgsMap]: ActionDefinition<TArgsMap[key]>
 }
 
-export const EVENT_CURSOR_TABLE = 'livestore__event_cursor'
-
-const systemTables = {
-  [EVENTS_TABLE_NAME]: {
-    columns: {
-      id: { type: 'text', primaryKey: true },
-      type: { type: 'text', nullable: false },
-      args: { type: 'text', nullable: false },
-    },
-  },
-  [EVENT_CURSOR_TABLE]: {
-    columns: {
-      id: { type: 'text', primaryKey: true },
-      cursor: { type: 'text', nullable: false },
-    },
-  },
-} as const
-
-export const defineSchema = <S extends Schema>(schema: S) => mergeSystemSchema(schema)
+export const SCHEMA_META_TABLE = '__livestore_schema'
+
+const schemaMetaTable = SqliteDsl.table(SCHEMA_META_TABLE, {
+  tableName: SqliteDsl.text({ primaryKey: true }),
+  schemaHash: SqliteDsl.integer({ nullable: false }),
+  /** ISO date format */
+  updatedAt: SqliteDsl.text({ nullable: false }),
+})
+
+export type SchemaMetaRow = SqliteDsl.FromTable.RowDecoded<typeof schemaMetaTable>
+
+export const systemTables = {
+  // [EVENTS_TABLE_NAME]: SqliteDsl.table(EVENTS_TABLE_NAME, {
+  //   id: SqliteDsl.text({ primaryKey: true }),
+  //   type: SqliteDsl.text({ nullable: false }),
+  //   args: SqliteDsl.text({ nullable: false }),
+  // }).ast,
+  [SCHEMA_META_TABLE]: schemaMetaTable.ast,
+} satisfies TableDefinitions
 
 export const defineTables = <T extends TableDefinitions>(tables: T) => tables
 
@@ -143,77 +134,10 @@ type RecordValues<T> = T extends Record<string, infer V> ? V : never
 
 export type GetActionArgs<A> = A extends ActionDefinition<infer TArgs> ? TArgs : never
 
+// TODO get rid of this
 declare global {
   // NOTE Can be extended
   interface LiveStoreActionDefinitionsTypes {
     [key: string]: ActionDefinition
   }
 }
-
-const mergeSystemSchema = <S extends Schema>(schema: S) => {
-  return {
-    ...schema,
-    tables: {
-      ...schema.tables,
-      ...systemTables,
-    },
-  }
-}
-
-/**
- * Destructively load a schema into a database,
- * dropping any existing tables and creating new ones.
- */
-export const loadSchema = async (backend: InMemoryDatabase | Backend, schema: Schema) => {
-  const fullSchemaWithComponents = { ...schema, tables: { ...schema.tables, ...componentStateTables } }
-
-  // Loop through all the tables and create them in the SQLite database
-  for (const [tableName, tableDefinition] of Object.entries(fullSchemaWithComponents.tables)) {
-    const primaryKeys = Object.entries(tableDefinition.columns)
-      .filter(([_, columnDef]) => columnDef.primaryKey)
-      .map(([columnName, _]) => columnName)
-    const columnDefStrs = Object.entries(tableDefinition.columns).map(([columnName, column]) =>
-      toSqliteColumnSpec(columnName, column),
-    )
-    if (primaryKeys.length > 0) {
-      columnDefStrs.push(`PRIMARY KEY (${primaryKeys.join(', ')})`)
-    }
-    const mapColumns = tableDefinition.columnsRaw ?? ((_) => _)
-    const columnSpec = mapColumns(columnDefStrs.join(', '))
-
-    backend.execute(sql`drop table if exists ${tableName}`)
-
-    backend.execute(sql`create table if not exists ${tableName} (${columnSpec});`)
-  }
-
-  await createIndexes(backend, schema)
-}
-
-const toSqliteColumnSpec = (columnName: string, column: ColumnDefinition) => {
-  const columnType = column.type === 'boolean' ? 'integer' : column.type
-  // const primaryKey = column.primaryKey ? 'primary key' : ''
-  const nullable = column.nullable === false ? 'not null' : ''
-  const defaultValue =
-    column.default === undefined
-      ? ''
-      : column.type === 'text'
-      ? `default '${column.default}'`
-      : `default ${column.default}`
-
-  return `${columnName} ${columnType} ${nullable} ${defaultValue}`
-}
-
-const createIndexFromDefinition = (tableName: string, index: Index) => {
-  const uniqueStr = index.isUnique ? 'UNIQUE' : ''
-  return sql`create ${uniqueStr} index ${index.name} on ${tableName} (${index.columns.join(', ')})`
-}
-
-const createIndexes = async (db: Backend | InMemoryDatabase, schema: Schema) => {
-  for (const [tableName, tableDefinition] of Object.entries(schema.tables)) {
-    if (tableDefinition.indexes !== undefined) {
-      for (const index of tableDefinition.indexes) {
-        db.execute(createIndexFromDefinition(tableName, index))
-      }
-    }
-  }
-}

package/src/storage/in-memory/index.ts

@@ -0,0 +1,21 @@
+import type * as otel from '@opentelemetry/api'
+
+import type { PreparedBindValues } from '../../util.js'
+import type { Storage, StorageOtelProps } from '../index.js'
+
+export type StorageOptionsWebInMemory = {
+  type: 'web-in-memory'
+}
+
+/** NOTE: This storage is currently only used for testing */
+export class InMemoryStorage implements Storage {
+  constructor(readonly otelTracer: otel.Tracer) {}
+
+  static load = async (_options?: StorageOptionsWebInMemory) => {
+    return ({ otelTracer }: StorageOtelProps) => new InMemoryStorage(otelTracer)
+  }
+
+  execute = (_query: string, _bindValues?: PreparedBindValues): void => {}
+
+  getPersistedData = async (): Promise<Uint8Array> => new Uint8Array()
+}

package/src/storage/index.ts

@@ -0,0 +1,27 @@
+// A storage represents a raw SQLite database.
+// Examples include:
+// - A native SQLite process running in a Tauri Rust process
+// - A SQL.js WASM version of SQLite running in a web worker
+//
+// We can send commands to execute various kinds of queries,
+// and respond to various events from the database.
+
+import type * as otel from '@opentelemetry/api'
+
+import type { PreparedBindValues } from '../util.js'
+
+export type StorageInit = (otelProps: StorageOtelProps) => Promise<Storage> | Storage
+
+export interface Storage {
+  execute(query: string, bindValues?: PreparedBindValues, parentSpan?: otel.Span): void
+
+  /** Return a snapshot of persisted data from the storage */
+  getPersistedData(parentSpan?: otel.Span): Promise<Uint8Array>
+}
+
+export type StorageType = 'tauri' | 'web' | 'web-in-memory'
+
+export type StorageOtelProps = {
+  otelTracer: otel.Tracer
+  parentSpan: otel.Span
+}

package/src/{backends/tauri.ts → storage/tauri/index.ts}

@@ -2,38 +2,33 @@ import { getTraceParentHeader } from '@livestore/utils'
 import type * as otel from '@opentelemetry/api'
 import { invoke } from '@tauri-apps/api'
 
-import type { ParamsObject } from '../util.js'
-import { prepareBindValues } from '../util.js'
-import { BaseBackend } from './base.js'
-import type { BackendOtelProps, SelectResponse } from './index.js'
+import type { PreparedBindValues } from '../../util.js'
+import { prepareBindValues } from '../../util.js'
+import type { Storage, StorageOtelProps } from '../index.js'
 
-export type BackendOptionsTauri = {
-  type: 'tauri'
+export type StorageOptionsTauri = {
   dbDirPath: string
   appDbFileName: string
 }
 
-export class TauriBackend extends BaseBackend {
+export class TauriStorage implements Storage {
   constructor(
     readonly dbFilePath: string,
     readonly dbDirPath: string,
     readonly otelTracer: otel.Tracer,
     readonly parentSpan: otel.Span,
-  ) {
-    super()
-  }
+  ) {}
 
-  static load = async (
-    { dbDirPath, appDbFileName }: BackendOptionsTauri,
-    { otelTracer, parentSpan }: BackendOtelProps,
-  ): Promise<TauriBackend> => {
-    const dbFilePath = `${dbDirPath}/${appDbFileName}`
-    await invoke('initialize_connection', { dbName: dbFilePath, otelData: getOtelData_(parentSpan) })
+  static load =
+    ({ dbDirPath, appDbFileName }: StorageOptionsTauri) =>
+    async ({ otelTracer, parentSpan }: StorageOtelProps) => {
+      const dbFilePath = `${dbDirPath}/${appDbFileName}`
+      await invoke('initialize_connection', { dbName: dbFilePath, otelData: getOtelData_(parentSpan) })
 
-    return new TauriBackend(dbFilePath, dbDirPath, otelTracer, parentSpan)
-  }
+      return new TauriStorage(dbFilePath, dbDirPath, otelTracer, parentSpan)
+    }
 
-  execute = (query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): void => {
+  execute = (query: string, bindValues?: PreparedBindValues, parentSpan?: otel.Span): void => {
     // console.log({ query, bindValues, prepared: prepareBindValues(bindValues ?? {}, query) })
     void invoke('execute', {
       dbName: this.dbFilePath,
@@ -43,15 +38,6 @@ export class TauriBackend extends BaseBackend {
     })
   }
 
-  select = async <T>(query: string, bindValues?: ParamsObject, parentSpan?: otel.Span): Promise<SelectResponse<T>> => {
-    return invoke('select', {
-      db: this.dbFilePath,
-      query,
-      values: bindValues ?? {},
-      otelData: this.getOtelData(parentSpan),
-    })
-  }
-
   getPersistedData = async (parentSpan?: otel.Span): Promise<Uint8Array> => {
     const headers = new Headers()
     headers.set('traceparent', getTraceParentHeader(parentSpan ?? this.parentSpan))

package/src/storage/web-worker/index.ts

@@ -0,0 +1,116 @@
+import { casesHandled } from '@livestore/utils'
+import type * as otel from '@opentelemetry/api'
+import * as Comlink from 'comlink'
+
+import type { PreparedBindValues } from '../../util.js'
+import type { Storage, StorageOtelProps } from '../index.js'
+import { IDB } from '../utils/idb.js'
+import type { WrappedWorker } from './worker.js'
+
+export type StorageType = 'opfs' | 'indexeddb'
+
+export type StorageOptionsWeb = {
+  /** Specifies where to persist data for this storage */
+  type: StorageType
+  fileName: string
+}
+
+export class WebWorkerStorage implements Storage {
+  worker: Comlink.Remote<WrappedWorker>
+  options: StorageOptionsWeb
+  otelTracer: otel.Tracer
+
+  executionBacklog: { query: string; bindValues?: PreparedBindValues }[] = []
+  executionPromise: Promise<void> | undefined
+
+  private constructor({
+    worker,
+    options,
+    otelTracer,
+    executionPromise,
+  }: {
+    worker: Comlink.Remote<WrappedWorker>
+    options: StorageOptionsWeb
+    otelTracer: otel.Tracer
+    executionPromise: Promise<void>
+  }) {
+    this.worker = worker
+    this.options = options
+    this.otelTracer = otelTracer
+    this.executionPromise = executionPromise
+
+    executionPromise.then(() => this.executeBacklog())
+  }
+
+  static load = (options: StorageOptionsWeb) => {
+    // TODO: Importing the worker like this only works with Vite;
+    // should this really be inside the LiveStore library?
+    // Doesn't work with Firefox right now during dev https://bugzilla.mozilla.org/show_bug.cgi?id=1247687
+    const worker = new Worker(new URL('./worker.js', import.meta.url), {
+      type: 'module',
+    })
+    const wrappedWorker = Comlink.wrap<WrappedWorker>(worker)
+
+    return ({ otelTracer }: StorageOtelProps) =>
+      new WebWorkerStorage({
+        worker: wrappedWorker,
+        options,
+        otelTracer,
+        executionPromise: wrappedWorker.initialize(options),
+      })
+  }
+
+  execute = (query: string, bindValues?: PreparedBindValues) => {
+    this.executionBacklog.push({ query, bindValues })
+
+    // Instead of sending the queries to the worker immediately, we wait a bit and batch them up (which reduces the number of messages sent to the worker)
+    if (this.executionPromise === undefined) {
+      this.executionPromise = new Promise((resolve) => {
+        setTimeout(() => {
+          this.executeBacklog()
+
+          resolve()
+        }, 10)
+      })
+    }
+  }
+
+  private executeBacklog = () => {
+    void this.worker.executeBulk(this.executionBacklog)
+    this.executionBacklog = []
+    this.executionPromise = undefined
+  }
+
+  getPersistedData = async (_parentSpan?: otel.Span): Promise<Uint8Array> => getPersistedData(this.options)
+}
+
+const getPersistedData = async (options: StorageOptionsWeb): Promise<Uint8Array> => {
+  switch (options.type) {
+    case 'opfs': {
+      try {
+        const rootHandle = await navigator.storage.getDirectory()
+        const fileHandle = await rootHandle.getFileHandle(options.fileName + '.db')
+        const file = await fileHandle.getFile()
+        const buffer = await file.arrayBuffer()
+        const data = new Uint8Array(buffer)
+
+        return data
+      } catch (error: any) {
+        if (error instanceof DOMException && error.name === 'NotFoundError') {
+          return new Uint8Array()
+        }
+
+        throw error
+      }
+    }
+
+    case 'indexeddb': {
+      const idb = new IDB(options.fileName)
+
+      return (await idb.get('db')) ?? new Uint8Array()
+    }
+    default: {
+      casesHandled(options.type)
+    }
+  }
+}

package/src/{backends/web-worker.ts → storage/web-worker/worker.ts}

@@ -8,10 +8,10 @@ import type * as SqliteWasm from 'sqlite-esm'
 import sqlite3InitModule from 'sqlite-esm'
 
 // import { v4 as uuid } from 'uuid'
-import type { Bindable } from '../util.js'
-import { casesHandled, sql } from '../util.js'
-import type { SelectResponse, WritableDatabaseLocation } from './index.js'
-import { IDB } from './utils/idb.js'
+import type { Bindable } from '../../util.js'
+import { casesHandled, sql } from '../../util.js'
+import { IDB } from '../utils/idb.js'
+import type { StorageOptionsWeb } from './index.js'
 
 // A global variable to hold the database connection.
 // let db: SqliteWasm.Database
@@ -19,11 +19,11 @@ let db: SqliteWasm.DatabaseApi
 
 let sqlite3: SqliteWasm.Sqlite3Static
 
-// TODO get rid of this in favour of a "proper" IDB SQLite backend
+// TODO get rid of this in favour of a "proper" IDB SQLite storage
 let idb: IDB | undefined
 
-/** The location where this database backend persists its data */
-let persistentDatabaseLocation_: WritableDatabaseLocation
+/** The location where this database storage persists its data */
+let options_: StorageOptionsWeb
 
 const configureConnection = () =>
   db.exec(sql`
@@ -35,18 +35,18 @@ const configureConnection = () =>
 /** A full virtual filename in the IDB FS */
 const fullyQualifiedFilename = (name: string) => `${name}.db`
 
-const initialize = async ({ persistentDatabaseLocation }: { persistentDatabaseLocation: WritableDatabaseLocation }) => {
-  persistentDatabaseLocation_ = persistentDatabaseLocation
+const initialize = async (options: StorageOptionsWeb) => {
+  options_ = options
 
   sqlite3 = await sqlite3InitModule({
     print: (message) => console.log(`[sql-client] ${message}`),
     printErr: (message) => console.error(`[sql-client] ${message}`),
   })
 
-  switch (persistentDatabaseLocation.type) {
+  switch (options.type) {
     case 'opfs': {
       try {
-        db = new sqlite3.oo1.OpfsDb(fullyQualifiedFilename(persistentDatabaseLocation.virtualFilename)) // , 'c'
+        db = new sqlite3.oo1.OpfsDb(fullyQualifiedFilename(options.fileName)) // , 'c'
       } catch (e) {
         debugger
       }
@@ -55,7 +55,7 @@ const initialize = async ({ persistentDatabaseLocation }: { persistentDatabaseLo
     case 'indexeddb': {
       try {
         db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' })
-        idb = new IDB(persistentDatabaseLocation.virtualFilename)
+        idb = new IDB(options.fileName)
 
         const bytes = await idb.get('db')
 
@@ -70,21 +70,15 @@ const initialize = async ({ persistentDatabaseLocation }: { persistentDatabaseLo
       }
       break
     }
-    case 'filesystem': {
-      throw new Error('Persisting to native FS is not supported in the web worker backend')
-    }
-    case 'volatile-in-memory': {
-      break
-    }
     default: {
-      casesHandled(persistentDatabaseLocation)
+      casesHandled(options.type)
     }
   }
 
   configureConnection()
 }
 
-// TODO get rid of this in favour of a "proper" IDB SQLite backend
+// TODO get rid of this in favour of a "proper" IDB SQLite storage
 let idbPersistTimeout: NodeJS.Timeout | undefined
 
 type ExecutionQueueItem = { query: string; bindValues?: Bindable }
@@ -119,8 +113,8 @@ const executeBulk = (executionItems: ExecutionQueueItem[]): void => {
     }
   }
 
-  // TODO get rid of this in favour of a "proper" IDB SQLite backend
-  if (persistentDatabaseLocation_.type === 'indexeddb') {
+  // TODO get rid of this in favour of a "proper" IDB SQLite storage
+  if (options_.type === 'indexeddb') {
     if (idbPersistTimeout !== undefined) {
       clearTimeout(idbPersistTimeout)
     }
@@ -133,36 +127,7 @@ const executeBulk = (executionItems: ExecutionQueueItem[]): void => {
   }
 }
 
-const select = <T = any>(query: string, bindValues?: Bindable): SelectResponse<T> => {
-  const resultRows: T[] = []
-
-  db.exec({
-    sql: query,
-    bind: bindValues,
-    rowMode: 'object',
-    resultRows,
-  } as TODO)
-
-  return { results: resultRows }
-}
-
-const getPersistedData = async (): Promise<Uint8Array> => {
-  // TODO get rid of this in favour of a "proper" IDB SQLite backend
-  if (persistentDatabaseLocation_.type === 'indexeddb') {
-    const data = sqlite3.capi.sqlite3_js_db_export(db.pointer)
-    return Comlink.transfer(data, [data.buffer])
-  }
-
-  const rootHandle = await navigator.storage.getDirectory()
-  const fileHandle = await rootHandle.getFileHandle(db.filename)
-  const file = await fileHandle.getFile()
-  const buffer = await file.arrayBuffer()
-  const data = new Uint8Array(buffer)
-
-  return Comlink.transfer(data, [data.buffer])
-}
-
-const wrappedWorker = { initialize, executeBulk, select, getPersistedData }
+const wrappedWorker = { initialize, executeBulk }
 
 export type WrappedWorker = typeof wrappedWorker