@livestore/livestore 0.0.10 → 0.0.13

package/src/inMemoryDatabase.ts

@@ -1,18 +1,14 @@
 /* eslint-disable prefer-arrow/prefer-arrow-functions */
 
 import { shouldNeverHappen } from '@livestore/utils'
-import { identity } from '@livestore/utils/effect'
 import type * as otel from '@opentelemetry/api'
 import type * as SqliteWasm from 'sqlite-esm'
-import initSqlJs from 'sqlite-esm'
 
 import BoundMap, { BoundArray } from './bounded-collections.js'
-import type { LiveStoreEvent } from './events.js'
 // import { EVENTS_TABLE_NAME } from './events.js'
 import { sql } from './index.js'
 import { getDurationMsFromSpan, getStartTimeHighResFromSpan } from './otel.js'
 import QueryCache from './QueryCache.js'
-import type { ActionDefinition } from './schema.js'
 import type { Bindable, ParamsObject } from './util.js'
 import { prepareBindValues } from './util.js'
 
@@ -66,18 +62,13 @@ export class InMemoryDatabase {
     public SQL: SqliteWasm.Sqlite3Static,
   ) {}
 
-  static async load(
+  static load(
     data: Uint8Array | undefined,
     otelTracer: otel.Tracer,
     otelRootSpanContext: otel.Context,
-  ): Promise<InMemoryDatabase> {
-    const sqlite3 = await initSqlJs({
-      // Required to load the wasm binary asynchronously. Of course, you can host it wherever you want
-      // You can omit locateFile completely when running in node
-      // locateFile: () => `/sql-wasm.wasm`,
-      print: (message) => console.log(`[sql-client] ${message}`),
-      printErr: (message) => console.error(`[sql-client] ${message}`),
-    })
+    sqlite3: SqliteWasm.Sqlite3Static,
+  ): InMemoryDatabase {
+    // TODO move WASM init higher up in the init process (to do some other work while it's loading)
 
     const db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' }) as DatabaseWithCAPI
     db.capi = sqlite3.capi
@@ -97,7 +88,11 @@ export class InMemoryDatabase {
       )
     }
 
-    return new InMemoryDatabase(db, otelTracer, otelRootSpanContext, sqlite3)
+    const inMemoryDatabase = new InMemoryDatabase(db, otelTracer, otelRootSpanContext, sqlite3)
+
+    configureSQLite(inMemoryDatabase)
+
+    return inMemoryDatabase
   }
 
   txn<TRes>(callback: () => TRes): TRes {
@@ -141,49 +136,75 @@ export class InMemoryDatabase {
     return tablesUsed as string[]
   }
 
-  /**
-   * NOTE `execute` is untraced since it's usually called from `applyEvent` which is traced
-   */
   execute(
     query: string,
     bindValues?: ParamsObject,
     writeTables?: string[],
-    options?: { hasNoEffects?: boolean },
-  ): void {
-    try {
-      let stmt = this.cachedStmts.get(query)
-      if (stmt === undefined) {
-        stmt = this.db.prepare(query)
-        this.cachedStmts.set(query, stmt)
-      }
+    options?: { hasNoEffects?: boolean; otelContext: otel.Context },
+  ): { durationMs: number } {
+    return this.otelTracer.startActiveSpan(
+      'livestore.in-memory-db:execute',
+      // TODO truncate query string
+      { attributes: { 'sql.query': query } },
+      options?.otelContext ?? this.otelRootSpanContext,
+      (span) => {
+        try {
+          let stmt = this.cachedStmts.get(query)
+          if (stmt === undefined) {
+            stmt = this.db.prepare(query)
+            this.cachedStmts.set(query, stmt)
+          }
 
-      if (bindValues !== undefined && Object.keys(bindValues).length > 0) {
-        stmt.bind(prepareBindValues(bindValues, query))
-      }
+          // TODO check whether we can remove the extra `prepareBindValues` call here (e.g. enforce proper type in API)
+          if (bindValues !== undefined && Object.keys(bindValues).length > 0) {
+            stmt.bind(prepareBindValues(bindValues, query))
+          }
 
-      try {
-        stmt.step()
-      } finally {
-        stmt.reset() // Reset is needed for next execution
-      }
-    } catch (error) {
-      shouldNeverHappen(
-        `Error executing query: ${error} \n ${JSON.stringify({
-          query,
-          bindValues,
-        })}`,
-      )
-    }
+          if (import.meta.env.DEV) {
+            this.debugInfo.events.push([query, bindValues])
+          }
 
-    if (options?.hasNoEffects !== true && !this.resultCache.ignoreQuery(query)) {
-      // TODO use write tables instead
-      // check what queries actually end up here.
-      this.resultCache.invalidate(writeTables ?? this.getTablesUsed(query))
-    }
+          try {
+            stmt.step()
+          } finally {
+            stmt.reset() // Reset is needed for next execution
+          }
+        } catch (error) {
+          shouldNeverHappen(
+            `Error executing query: ${error} \n ${JSON.stringify({
+              query,
+              bindValues,
+            })}`,
+          )
+        }
 
-    if (options?.hasNoEffects === true) {
-      return
-    }
+        if (options?.hasNoEffects !== true && !this.resultCache.ignoreQuery(query)) {
+          // TODO use write tables instead
+          // check what queries actually end up here.
+          this.resultCache.invalidate(writeTables ?? this.getTablesUsed(query))
+        }
+
+        span.end()
+
+        const durationMs = getDurationMsFromSpan(span)
+
+        this.debugInfo.queryFrameDuration += durationMs
+        this.debugInfo.queryFrameCount++
+
+        if (durationMs > 5 && import.meta.env.DEV) {
+          this.debugInfo.slowQueries.push([
+            query,
+            bindValues,
+            durationMs,
+            undefined,
+            [],
+            getStartTimeHighResFromSpan(span),
+          ])
+        }
+
+        return { durationMs }
+      },
+    )
   }
 
   select<T = any>(
@@ -218,19 +239,27 @@ export class InMemoryDatabase {
             stmt = this.db.prepare(query)
             this.cachedStmts.set(query, stmt)
           }
-          if (bindValues) {
-            stmt.bind(bindValues ?? {})
+          if (bindValues !== undefined && Object.keys(bindValues).length > 0) {
+            stmt.bind(bindValues)
           }
 
           const result: T[] = []
+
           try {
-            const columns = stmt.getColumnNames()
+            // NOTE `getColumnNames` only works for `SELECT` statements, ignoring other statements for now
+            let columns = undefined
+            try {
+              columns = stmt.getColumnNames()
+            } catch (_e) {}
+
             while (stmt.step()) {
-              const obj: { [key: string]: any } = {}
-              for (const [i, c] of columns.entries()) {
-                obj[c] = stmt.get(i)
+              if (columns !== undefined) {
+                const obj: { [key: string]: any } = {}
+                for (const [i, c] of columns.entries()) {
+                  obj[c] = stmt.get(i)
+                }
+                result.push(obj as unknown as T)
               }
-              result.push(obj as unknown as T)
             }
           } finally {
             // we're caching statements in this iteration. do not free.
@@ -275,67 +304,6 @@ export class InMemoryDatabase {
     )
   }
 
-  // TODO move `applyEvent` logic to Store and only call `execute` here
-  applyEvent(
-    event: LiveStoreEvent,
-    eventDefinition: ActionDefinition,
-    otelContext: otel.Context,
-  ): { durationMs: number } {
-    return this.otelTracer.startActiveSpan('livestore.in-memory-db:applyEvent', {}, otelContext, (span) => {
-      // TODO: in the future, we'll do more CRDT-style stuff here to decide whether to run effects of the event
-
-      // NOTE: These two updates should happen transactionally;
-      // we don't create a transaction here because that's handled in the caller.
-      // The reason for this is that sometimes we want to apply multiple events in a larger transaction.
-
-      // Insert into the events table
-      // this.execute(sql`insert into ${EVENTS_TABLE_NAME} (id, type, args) values ($id, $type, $args)`, {
-      //   id: event.id,
-      //   type: event.type,
-      //   args: JSON.stringify(event.args ?? {}),
-      // })
-
-      const statement =
-        typeof eventDefinition.statement === 'function'
-          ? eventDefinition.statement(event.args)
-          : eventDefinition.statement
-
-      const prepareBindValues = eventDefinition.prepareBindValues ?? identity
-
-      const bindValues =
-        typeof eventDefinition.statement === 'function' && statement.argsAlreadyBound
-          ? {}
-          : prepareBindValues(event.args)
-
-      if (import.meta.env.DEV) {
-        this.debugInfo.events.push([statement.sql, bindValues])
-      }
-
-      // Run the effects of the event
-      this.execute(statement.sql, bindValues, statement.writeTables)
-
-      span.end()
-
-      const durationMs = getDurationMsFromSpan(span)
-
-      this.debugInfo.queryFrameDuration += durationMs
-      this.debugInfo.queryFrameCount++
-
-      if (durationMs > 5 && import.meta.env.DEV) {
-        this.debugInfo.slowQueries.push([
-          statement.sql,
-          bindValues,
-          durationMs,
-          undefined,
-          [],
-          getStartTimeHighResFromSpan(span),
-        ])
-      }
-
-      return { durationMs }
-    })
-  }
-
   export() {
     // Clear statement cache because exporting frees statements
     for (const key of this.cachedStmts.keys()) {
@@ -345,3 +313,18 @@ export class InMemoryDatabase {
     return this.db.capi.sqlite3_js_db_export(this.db.pointer)
   }
 }
+
+/** Set up SQLite performance; hasn't been super carefully optimized yet. */
+const configureSQLite = (db: InMemoryDatabase) => {
+  db.execute(
+    // TODO: revisit these tuning parameters for max performance
+    sql`
+      PRAGMA page_size=32768;
+      PRAGMA cache_size=10000;
+      PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
+      PRAGMA synchronous='OFF';
+      PRAGMA temp_store='MEMORY';
+      PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+    `,
+  )
+}

package/src/index.ts

@@ -6,18 +6,14 @@ export type { QueryDefinition, LiveStoreCreateStoreOptions, LiveStoreContext } f
 export {
   defineComponentStateSchema,
   EVENT_CURSOR_TABLE,
-  defineSchema,
   defineAction,
   defineActions,
   defineTables,
   defineMaterializedViews,
+  makeSchema,
 } from './schema.js'
 export { InMemoryDatabase, type DebugInfo, emptyDebugInfo } from './inMemoryDatabase.js'
-export { createBackend, IndexType } from './backends/index.js'
-export type { BackendOptions, Backend, BackendType } from './backends/index.js'
-export { isBackendType } from './backends/index.js'
-export type { SelectResponse } from './backends/index.js'
-export { WebWorkerBackend } from './backends/web.js'
+export type { Storage, StorageType, StorageInit } from './storage/index.js'
 export type {
   GetAtom,
   AtomDebugInfo,
@@ -32,16 +28,14 @@ export type { LiveStoreGraphQLQuery } from './reactiveQueries/graphql.js'
 
 export { labelForKey } from './componentKey.js'
 export type { ComponentKey } from './componentKey.js'
-export type {
-  Schema,
-  TableDefinition,
-  GetActionArgs,
-  GetApplyEventArgs,
-  ColumnDefinition,
-  Index,
-  ActionDefinition,
-  ActionDefinitions,
-} from './schema.js'
+export type { Schema, GetActionArgs, GetApplyEventArgs, Index, ActionDefinition, ActionDefinitions } from './schema.js'
+
+export { SqliteAst, SqliteDsl } from 'effect-db-schema'
+
+import type { SqliteAst } from 'effect-db-schema'
+export type TableDefinition = SqliteAst.Table
+
+export { SqliteDsl as DbSchema } from 'effect-db-schema'
 
 export { sql, type Bindable } from './util.js'
 export { isEqual } from 'lodash-es'

package/src/migrations.ts

@@ -0,0 +1,101 @@
+import type * as otel from '@opentelemetry/api'
+import { SqliteAst } from 'effect-db-schema'
+import { memoize, omit } from 'lodash-es'
+
+import type { InMemoryDatabase } from './index.js'
+import type { Schema, SchemaMetaRow } from './schema.js'
+import { componentStateTables, SCHEMA_META_TABLE, systemTables } from './schema.js'
+import { sql } from './util.js'
+
+// TODO more graceful DB migration (e.g. backup DB before destructive migrations)
+export const migrateDb = ({
+  db,
+  otelContext,
+  schema,
+}: {
+  db: InMemoryDatabase
+  otelContext: otel.Context
+  schema: Schema
+}) => {
+  db.execute(
+    // TODO use schema migration definition from schema.ts instead
+    sql`create table if not exists ${SCHEMA_META_TABLE} (tableName text primary key, schemaHash text, updatedAt text);`,
+    undefined,
+    [],
+    { otelContext },
+  )
+
+  const schemaMetaRows = db.select<SchemaMetaRow>(sql`SELECT * FROM ${SCHEMA_META_TABLE}`)
+
+  const dbSchemaHashByTable = Object.fromEntries(
+    schemaMetaRows.map(({ tableName, schemaHash }) => [tableName, schemaHash]),
+  )
+
+  const getMemoizedTimestamp = memoize(() => new Date().toISOString())
+  const tableDefs = {
+    // NOTE it's important the `SCHEMA_META_TABLE` comes first since we're writing to it below
+    [SCHEMA_META_TABLE]: systemTables[SCHEMA_META_TABLE],
+    ...omit(schema.tables, [SCHEMA_META_TABLE]),
+    ...componentStateTables,
+  }
+
+  for (const [tableName, tableDef] of Object.entries(tableDefs)) {
+    const dbSchemaHash = dbSchemaHashByTable[tableName]
+    const schemaHash = SqliteAst.hash(tableDef)
+    if (schemaHash !== dbSchemaHash) {
+      console.log(
+        `Schema hash mismatch for table '${tableName}' (DB: ${dbSchemaHash}, expected: ${schemaHash}), migrating table...`,
+      )
+
+      const columnSpec = makeColumnSpec(tableDef)
+
+      // TODO need to possibly handle cascading deletes due to foreign keys
+      db.execute(sql`drop table if exists ${tableName}`, undefined, [], { otelContext })
+      db.execute(sql`create table if not exists ${tableName} (${columnSpec});`, undefined, [], { otelContext })
+
+      for (const index of tableDef.indexes) {
+        db.execute(createIndexFromDefinition(tableName, index), undefined, [], { otelContext })
+      }
+
+      const updatedAt = getMemoizedTimestamp()
+      db.execute(
+        sql`
+          INSERT INTO ${SCHEMA_META_TABLE} (tableName, schemaHash, updatedAt) VALUES ($tableName, $schemaHash, $updatedAt)
+            ON CONFLICT (tableName) DO UPDATE SET schemaHash = $schemaHash, updatedAt = $updatedAt;
+        `,
+        { tableName, schemaHash, updatedAt },
+        [],
+        { otelContext },
+      )
+    }
+  }
+}
+
+const createIndexFromDefinition = (tableName: string, index: SqliteAst.Index) => {
+  const uniqueStr = index.unique ? 'UNIQUE' : ''
+  return sql`create ${uniqueStr} index ${index.name} on ${tableName} (${index.columns.join(', ')})`
+}
+
+const makeColumnSpec = (tableDef: SqliteAst.Table) => {
+  const primaryKeys = tableDef.columns.filter((_) => _.primaryKey).map((_) => _.name)
+  const columnDefStrs = tableDef.columns.map(toSqliteColumnSpec)
+  if (primaryKeys.length > 0) {
+    columnDefStrs.push(`PRIMARY KEY (${primaryKeys.join(', ')})`)
+  }
+
+  return columnDefStrs.join(', ')
+}
+
+const toSqliteColumnSpec = (column: SqliteAst.Column) => {
+  const columnType = column.type._tag
+  // const primaryKey = column.primaryKey ? 'primary key' : ''
+  const nullable = column.nullable === false ? 'not null' : ''
+  const defaultValue =
+    column.default === undefined
+      ? ''
+      : columnType === 'text'
+      ? `default '${column.default}'`
+      : `default ${column.default}`
+
+  return `${column.name} ${columnType} ${nullable} ${defaultValue}`
+}

package/src/otel.ts

@@ -1,16 +1,5 @@
 import type * as otel from '@opentelemetry/api'
 
-// TODO improve - see https://www.notion.so/schickling/Better-solution-for-globalThis-inProgressSpans-503cd7a5f4fc4fb8bdec2e60bde1be1f
-export const TODO_REMOVE_trackLongRunningSpan = (span: otel.Span): void => {
-  // @ts-expect-error TODO get rid of this coupling
-  if (window.inProgressSpans !== undefined && window.inProgressSpans instanceof Set) {
-    // @ts-expect-error TODO get rid of this coupling
-    window.inProgressSpans.add(span)
-  } else {
-    // debugger
-  }
-}
-
 export const getDurationMsFromSpan = (span: otel.Span): number => {
   const durationHr: [seconds: number, nanos: number] = (span as any)._duration
   return durationHr[0] * 1000 + durationHr[1] / 1_000_000

package/src/react/LiveStoreProvider.tsx

@@ -3,21 +3,23 @@ import { mapValues } from 'lodash-es'
 import type { ReactElement, ReactNode } from 'react'
 import React from 'react'
 
-import type { Backend, BackendOptions } from '../backends/index.js'
+// TODO refactor so the `react` module doesn't depend on `effect` module
 import type {
   GlobalQueryDefs,
   LiveStoreContext as StoreContext_,
   LiveStoreCreateStoreOptions,
 } from '../effect/LiveStore.js'
+import type { InMemoryDatabase } from '../inMemoryDatabase.js'
 import type { Schema } from '../schema.js'
+import type { StorageInit } from '../storage/index.js'
 import type { BaseGraphQLContext, GraphQLOptions } from '../store.js'
 import { createStore } from '../store.js'
 import { LiveStoreContext } from './LiveStoreContext.js'
 
 interface LiveStoreProviderProps<GraphQLContext> {
   schema: Schema
-  backendOptions: BackendOptions
-  boot?: (backend: Backend, parentSpan: otel.Span) => Promise<void>
+  loadStorage: () => StorageInit | Promise<StorageInit>
+  boot?: (db: InMemoryDatabase, parentSpan: otel.Span) => unknown | Promise<unknown>
   globalQueryDefs: GlobalQueryDefs
   graphQLOptions?: GraphQLOptions<GraphQLContext>
   otelTracer?: otel.Tracer
@@ -28,7 +30,7 @@ interface LiveStoreProviderProps<GraphQLContext> {
 export const LiveStoreProvider = <GraphQLContext extends BaseGraphQLContext>({
   fallback,
   globalQueryDefs,
-  backendOptions,
+  loadStorage,
   graphQLOptions,
   otelTracer,
   otelRootSpanContext,
@@ -39,7 +41,7 @@ export const LiveStoreProvider = <GraphQLContext extends BaseGraphQLContext>({
   const store = useCreateStore({
     schema,
     globalQueryDefs,
-    backendOptions,
+    loadStorage,
     graphQLOptions,
     otelTracer,
     otelRootSpanContext,
@@ -50,13 +52,15 @@ export const LiveStoreProvider = <GraphQLContext extends BaseGraphQLContext>({
     return fallback
   }
 
+  window.__debugLiveStore = store.store
+
   return <LiveStoreContext.Provider value={store}>{children}</LiveStoreContext.Provider>
 }
 
 const useCreateStore = <GraphQLContext extends BaseGraphQLContext>({
   schema,
   globalQueryDefs,
-  backendOptions,
+  loadStorage,
   graphQLOptions,
   otelTracer,
   otelRootSpanContext,
@@ -69,7 +73,7 @@ const useCreateStore = <GraphQLContext extends BaseGraphQLContext>({
       try {
         const store = await createStore({
           schema,
-          backendOptions,
+          loadStorage,
           graphQLOptions,
           otelTracer,
           otelRootSpanContext,
@@ -87,7 +91,7 @@ const useCreateStore = <GraphQLContext extends BaseGraphQLContext>({
     })()
 
     // TODO: do we need to return any cleanup function here?
-  }, [schema, backendOptions, globalQueryDefs, graphQLOptions, otelTracer, otelRootSpanContext, boot])
+  }, [schema, loadStorage, globalQueryDefs, graphQLOptions, otelTracer, otelRootSpanContext, boot])
 
   return ctxValue
 }

package/src/react/index.ts

@@ -3,9 +3,18 @@ export type {
   ReactiveGraphQL,
   ReactiveSQL,
   Setters,
+  ComponentKeyConfig,
+  QueryResults,
+  QueryDefinitions,
+  ComponentColumns,
+  GetStateType,
+  GetStateTypeEncoded,
 } from './useLiveStoreComponent.js'
 export { LiveStoreContext, useStore } from './LiveStoreContext.js'
 export { LiveStoreProvider } from './LiveStoreProvider.js'
 export { useLiveStoreComponent } from './useLiveStoreComponent.js'
 export { useGraphQL } from './useGraphQL.js'
 export { useGlobalQuery } from './useGlobalQuery.js'
+
+// Needed to make TS happy
+export type { TypedDocumentNode } from '@graphql-typed-document-node/core'

package/src/react/useGlobalQuery.ts

@@ -1,7 +1,6 @@
 import { useEffect, useState } from 'react'
 
 import { labelForKey } from '../componentKey.js'
-import { TODO_REMOVE_trackLongRunningSpan } from '../otel.js'
 import type { LiveStoreQuery, QueryResult } from '../store.js'
 
 export const useGlobalQuery = <Q extends LiveStoreQuery>(query: Q): QueryResult<Q> => {
@@ -15,8 +14,6 @@ export const useGlobalQuery = <Q extends LiveStoreQuery>(query: Q): QueryResult<
       {},
       query.store.otel.queriesSpanContext,
       (span) => {
-        TODO_REMOVE_trackLongRunningSpan(span)
-
         const cancel = query.store.subscribe(
           query,
           (v) => {