@livestore/livestore 0.0.12 → 0.0.14

package/src/store.ts

@@ -1,28 +1,27 @@
-import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core'
 import { assertNever, makeNoopSpan, makeNoopTracer, shouldNeverHappen } from '@livestore/utils'
+import { identity } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 import type { GraphQLSchema } from 'graphql'
-import * as graphql from 'graphql'
-import { uniqueId } from 'lodash-es'
-import * as ReactDOM from 'react-dom'
+import type * as Sqlite from 'sqlite-esm'
 import { v4 as uuid } from 'uuid'
 
-import type { Backend, BackendOptions } from './backends/index.js'
-import { createBackend } from './backends/index.js'
 import type { ComponentKey } from './componentKey.js'
 import { tableNameForComponentKey } from './componentKey.js'
 import type { LiveStoreEvent } from './events.js'
 import { InMemoryDatabase } from './inMemoryDatabase.js'
+import { migrateDb } from './migrations.js'
 import { getDurationMsFromSpan } from './otel.js'
-import type { GetAtom, Ref } from './reactive.js'
-import { ReactiveGraph } from './reactive.js'
-import { LiveStoreGraphQLQuery } from './reactiveQueries/graphql.js'
-import { LiveStoreJSQuery } from './reactiveQueries/js.js'
-import { LiveStoreSQLQuery } from './reactiveQueries/sql.js'
-import type { ActionDefinition, GetActionArgs, Schema } from './schema.js'
-import { componentStateTables, loadSchema } from './schema.js'
-import type { Bindable, ParamsObject } from './util.js'
-import { sql } from './util.js'
+import type { ReactiveGraph, Ref } from './reactive.js'
+import type { ILiveStoreQuery } from './reactiveQueries/base-class.js'
+import { type DbContext, dbGraph } from './reactiveQueries/graph.js'
+import type { LiveStoreGraphQLQuery } from './reactiveQueries/graphql.js'
+import type { LiveStoreJSQuery } from './reactiveQueries/js.js'
+import type { LiveStoreSQLQuery } from './reactiveQueries/sql.js'
+import type { ActionDefinition, GetActionArgs, Schema, SQLWriteStatement } from './schema.js'
+import { componentStateTables } from './schema.js'
+import type { Storage, StorageInit } from './storage/index.js'
+import type { ParamsObject } from './util.js'
+import { isPromise, prepareBindValues, sql } from './util.js'
 
 export type LiveStoreQuery<TResult extends Record<string, any> = any> =
   | LiveStoreSQLQuery<TResult>
@@ -35,8 +34,6 @@ export type BaseGraphQLContext = {
   otelContext?: otel.Context
 }
 
-export const RESET_DB_LOCAL_STORAGE_KEY = 'livestore-reset'
-
 export type QueryResult<TQuery> = TQuery extends LiveStoreSQLQuery<infer R>
   ? ReadonlyArray<Readonly<R>>
   : TQuery extends LiveStoreJSQuery<infer S>
@@ -45,7 +42,7 @@ export type QueryResult<TQuery> = TQuery extends LiveStoreSQLQuery<infer R>
   ? Readonly<Result>
   : never
 
-const globalComponentKey: ComponentKey = { _tag: 'singleton', componentName: '__global', id: 'singleton' }
+export const globalComponentKey: ComponentKey = { _tag: 'singleton', componentName: '__global', id: 'singleton' }
 
 export type GraphQLOptions<TContext> = {
   schema: GraphQLSchema
@@ -54,8 +51,10 @@ export type GraphQLOptions<TContext> = {
 
 export type StoreOptions<TGraphQLContext extends BaseGraphQLContext> = {
   db: InMemoryDatabase
+  /** A `Proxy`d version of `db` except that it also mirrors `execute` calls to the storage */
+  dbProxy: InMemoryDatabase
   schema: Schema
-  backend?: Backend
+  storage?: Storage
   graphQLOptions?: GraphQLOptions<TGraphQLContext>
   otelTracer: otel.Tracer
   otelRootSpanContext: otel.Context
@@ -98,9 +97,11 @@ export type StoreOtel = {
   queriesSpanContext: otel.Context
 }
 
-export class Store<TGraphQLContext extends BaseGraphQLContext> {
-  graph: ReactiveGraph<RefreshReason, QueryDebugInfo>
+export class Store<TGraphQLContext extends BaseGraphQLContext = BaseGraphQLContext> {
+  graph: ReactiveGraph<RefreshReason, QueryDebugInfo, DbContext>
   inMemoryDB: InMemoryDatabase
+  // TODO refactor
+  _proxyDb: InMemoryDatabase
   schema: Schema
   graphQLSchema?: GraphQLSchema
   graphQLContext?: TGraphQLContext
@@ -111,29 +112,31 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
    */
   tableRefs: { [key: string]: Ref<null> }
   activeQueries: Set<LiveStoreQuery>
-  backend?: Backend
+  storage?: Storage
   temporaryQueries: Set<LiveStoreQuery> | undefined
 
   private constructor({
     db,
+    dbProxy,
     schema,
-    backend,
+    storage,
     graphQLOptions,
     otelTracer,
     otelRootSpanContext,
   }: StoreOptions<TGraphQLContext>) {
     this.inMemoryDB = db
-    this.graph = new ReactiveGraph({
-      // TODO move this into React module
-      // Do all our updates inside a single React setState batch to avoid multiple UI re-renders
-      effectsWrapper: (run) => ReactDOM.unstable_batchedUpdates(() => run()),
-      otelTracer,
-    })
+    this._proxyDb = dbProxy
+    // this.graph = new ReactiveGraph({
+    //   // TODO move this into React module
+    //   // Do all our updates inside a single React setState batch to avoid multiple UI re-renders
+    //   effectsWrapper: (run) => ReactDOM.unstable_batchedUpdates(() => run()),
+    //   otelTracer,
+    // })
     this.schema = schema
     // TODO generalize the `tableRefs` concept to allow finer-grained refs
     this.tableRefs = {}
     this.activeQueries = new Set()
-    this.backend = backend
+    this.storage = storage
 
     const applyEventsSpan = otelTracer.startSpan('LiveStore:applyEvents', {}, otelRootSpanContext)
     const otelApplyEventsSpanContext = otel.trace.setSpan(otel.context.active(), applyEventsSpan)
@@ -141,6 +144,9 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
     const queriesSpan = otelTracer.startSpan('LiveStore:queries', {}, otelRootSpanContext)
     const otelQueriesSpanContext = otel.trace.setSpan(otel.context.active(), queriesSpan)
 
+    this.graph = dbGraph
+    this.graph.context = { store: this, otelTracer, rootOtelContext: otelQueriesSpanContext }
+
     this.otel = {
       tracer: otelTracer,
       applyEventsSpanContext: otelApplyEventsSpanContext,
@@ -185,372 +191,340 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
    *
    * NOTE The query is actually running (even if no one has subscribed to it yet) and will be kept up to date.
    */
-  querySQL = <TResult>(
-    genQueryString: (get: GetAtom) => string,
-    {
-      queriedTables,
-      bindValues,
-      componentKey,
-      label,
-      otelContext = otel.context.active(),
-    }: {
-      /**
-       * List of tables that are queried in this query;
-       * used to determine reactive dependencies.
-       *
-       * NOTE In the future we want to auto-generate this via parsing the query
-       */
-      queriedTables: string[]
-      bindValues?: Bindable | undefined
-      componentKey?: ComponentKey | undefined
-      label?: string | undefined
-      otelContext?: otel.Context
-    },
-  ): LiveStoreSQLQuery<TResult> =>
-    this.otel.tracer.startActiveSpan(
-      'querySQL', // NOTE span name will be overridden further down
-      { attributes: { label } },
-      otelContext,
-      (span) => {
-        const otelContext = otel.trace.setSpan(otel.context.active(), span)
-
-        const queryString$ = this.graph.makeThunk(
-          (get, addDebugInfo) => {
-            const queryString = genQueryString(get)
-            addDebugInfo({ _tag: 'js', label: `${label}:queryString`, query: queryString })
-            return queryString
-          },
-          { label: `${label}:queryString`, meta: { liveStoreThunkType: 'sqlQueryString' } },
-          otelContext,
-        )
-
-        label = label ?? queryString$.result
-        span.updateName(`querySQL:${label}`)
-
-        const queryLabel = `${label}:results` + (this.temporaryQueries ? ':temp' : '')
-
-        const results$ = this.graph.makeThunk<TResult[]>(
-          (get, addDebugInfo) =>
-            this.otel.tracer.startActiveSpan(
-              'sql', // NOTE span name will be overridden further down
-              {},
-              otelContext,
-              (span) => {
-                try {
-                  const otelContext = otel.trace.setSpan(otel.context.active(), span)
-
-                  // Establish a reactive dependency on the tables used in the query
-                  for (const tableName of queriedTables) {
-                    const tableRef =
-                      this.tableRefs[tableName] ?? shouldNeverHappen(`No table ref found for ${tableName}`)
-                    get(tableRef)
-                  }
-                  const sqlString = get(queryString$)
-
-                  span.setAttribute('sql.query', sqlString)
-                  span.updateName(`sql:${sqlString.slice(0, 50)}`)
-
-                  const results = this.inMemoryDB.select(sqlString, { queriedTables, bindValues, otelContext })
-
-                  span.setAttribute('sql.rowsCount', results.length)
-                  addDebugInfo({ _tag: 'sql', label: label ?? '', query: sqlString })
-
-                  return results as unknown as TResult[]
-                } finally {
-                  span.end()
-                }
-              },
-            ),
-          { label: queryLabel },
-          otelContext,
-        )
-
-        const query = new LiveStoreSQLQuery<TResult>({
-          label,
-          queryString$,
-          results$,
-          componentKey: componentKey ?? globalComponentKey,
-          store: this,
-          otelContext,
-        })
-
-        this.activeQueries.add(query)
-
-        // TODO get rid of temporary query workaround
-        if (this.temporaryQueries !== undefined) {
-          this.temporaryQueries.add(query)
-        }
-
-        // NOTE we are not ending the span here but in the query `destroy` method
-        return query
-      },
-    )
-
-  queryJS = <TResult>(
-    genResults: (get: GetAtom) => TResult,
-    {
-      componentKey = globalComponentKey,
-      label = `js${uniqueId()}`,
-      otelContext = otel.context.active(),
-    }: { componentKey?: ComponentKey; label?: string; otelContext?: otel.Context },
-  ): LiveStoreJSQuery<TResult> =>
-    this.otel.tracer.startActiveSpan(`queryJS:${label}`, { attributes: { label } }, otelContext, (span) => {
-      const otelContext = otel.trace.setSpan(otel.context.active(), span)
-      const queryLabel = `${label}:results` + (this.temporaryQueries ? ':temp' : '')
-      const results$ = this.graph.makeThunk(
-        (get, addDebugInfo) => {
-          addDebugInfo({ _tag: 'js', label, query: genResults.toString() })
-          return genResults(get)
-        },
-        { label: queryLabel, meta: { liveStoreThunkType: 'jsResults' } },
-        otelContext,
-      )
-
-      const query = new LiveStoreJSQuery<TResult>({
-        label,
-        results$,
-        componentKey,
-        store: this,
-        otelContext,
-      })
-
-      this.activeQueries.add(query)
-
-      // TODO get rid of temporary query workaround
-      if (this.temporaryQueries !== undefined) {
-        this.temporaryQueries.add(query)
-      }
-
-      // NOTE we are not ending the span here but in the query `destroy` method
-      return query
-    })
-
-  queryGraphQL = <TResult extends Record<string, any>, TVariableValues extends Record<string, any>>(
-    document: DocumentNode<TResult, TVariableValues>,
-    genVariableValues: (get: GetAtom) => TVariableValues,
-    {
-      componentKey,
-      label,
-      otelContext = otel.context.active(),
-    }: {
-      componentKey: ComponentKey
-      label?: string
-      otelContext?: otel.Context
-    },
-  ): LiveStoreGraphQLQuery<TResult, TVariableValues, TGraphQLContext> =>
-    this.otel.tracer.startActiveSpan(
-      `queryGraphQL:`, // NOTE span name will be overridden further down
-      {},
-      otelContext,
-      (span) => {
-        const otelContext = otel.trace.setSpan(otel.context.active(), span)
-
-        if (this.graphQLContext === undefined) {
-          return shouldNeverHappen("Can't run a GraphQL query on a store without GraphQL context")
-        }
-
-        const labelWithDefault = label ?? graphql.getOperationAST(document)?.name?.value ?? 'graphql'
-
-        span.updateName(`queryGraphQL:${labelWithDefault}`)
-
-        const variableValues$ = this.graph.makeThunk(
-          genVariableValues,
-          { label: `${labelWithDefault}:variableValues`, meta: { liveStoreThunkType: 'graphqlVariableValues' } },
-          otelContext,
-        )
-
-        const resultsLabel = `${labelWithDefault}:results` + (this.temporaryQueries ? ':temp' : '')
-        const results$ = this.graph.makeThunk<TResult>(
-          (get, addDebugInfo) => {
-            const variableValues = get(variableValues$)
-            const { result, queriedTables } = this.queryGraphQLOnce(document, variableValues, otelContext)
-
-            // Add dependencies on any tables that were used
-            for (const tableName of queriedTables) {
-              const tableRef = this.tableRefs[tableName]
-              assertNever(tableRef !== undefined, `No table ref found for ${tableName}`)
-              get(tableRef!)
-            }
-
-            addDebugInfo({ _tag: 'graphql', label: resultsLabel, query: graphql.print(document) })
-
-            return result
-          },
-          { label: resultsLabel, meta: { liveStoreThunkType: 'graphqlResults' } },
-          otelContext,
-        )
-
-        const query = new LiveStoreGraphQLQuery({
-          document,
-          context: this.graphQLContext,
-          results$,
-          componentKey,
-          label: labelWithDefault,
-          store: this,
-          otelContext,
-        })
-
-        this.activeQueries.add(query)
-
-        // TODO get rid of temporary query workaround
-        if (this.temporaryQueries !== undefined) {
-          this.temporaryQueries.add(query)
-        }
-
-        // NOTE we are not ending the span here but in the query `destroy` method
-        return query
-      },
-    )
-
-  queryGraphQLOnce = <TResult extends Record<string, any>, TVariableValues extends Record<string, any>>(
-    document: DocumentNode<TResult, TVariableValues>,
-    variableValues: TVariableValues,
-    otelContext: otel.Context = this.otel.queriesSpanContext,
-  ): { result: TResult; queriedTables: string[] } => {
-    const schema =
-      this.graphQLSchema ?? shouldNeverHappen("Can't run a GraphQL query on a store without GraphQL schema")
-    const context =
-      this.graphQLContext ?? shouldNeverHappen("Can't run a GraphQL query on a store without GraphQL context")
-    const tracer = this.otel.tracer
-
-    const operationName = graphql.getOperationAST(document)?.name?.value
-
-    return tracer.startActiveSpan(`executeGraphQLQuery: ${operationName}`, {}, otelContext, (span) => {
-      try {
-        span.setAttribute('graphql.variables', JSON.stringify(variableValues))
-        span.setAttribute('graphql.query', graphql.print(document))
-
-        context.queriedTables.clear()
-
-        context.otelContext = otel.trace.setSpan(otel.context.active(), span)
-
-        const res = graphql.executeSync({
-          document,
-          contextValue: context,
-          schema: schema,
-          variableValues,
-        })
-
-        // TODO track number of nested SQL queries via Otel + debug info
-
-        if (res.errors) {
-          span.setStatus({ code: otel.SpanStatusCode.ERROR, message: 'GraphQL error' })
-          span.setAttribute('graphql.error', res.errors.join('\n'))
-          span.setAttribute('graphql.error-detail', JSON.stringify(res.errors))
-          console.error(`graphql error (${operationName})`, res.errors)
-        }
-
-        return { result: res.data as unknown as TResult, queriedTables: Array.from(context.queriedTables.values()) }
-      } finally {
-        span.end()
-      }
-    })
-  }
+  // querySQL = <TResult>(
+  //   genQueryString: string | ((get: GetAtomResult) => string),
+  //   {
+  //     queriedTables,
+  //     bindValues,
+  //     componentKey,
+  //     label,
+  //     otelContext = otel.context.active(),
+  //   }: {
+  //     /**
+  //      * List of tables that are queried in this query;
+  //      * used to determine reactive dependencies.
+  //      *
+  //      * NOTE In the future we want to auto-generate this via parsing the query
+  //      */
+  //     queriedTables: string[]
+  //     bindValues?: Bindable | undefined
+  //     componentKey?: ComponentKey | undefined
+  //     label?: string | undefined
+  //     otelContext?: otel.Context
+  //   },
+  // ): LiveStoreSQLQuery<TResult> =>
+  //   this.otel.tracer.startActiveSpan(
+  //     'querySQL', // NOTE span name will be overridden further down
+  //     { attributes: { label } },
+  //     otelContext,
+  //     (span) => {
+  //       const otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+  //       const queryString$ = this.graph.makeThunk(
+  //         (get, addDebugInfo) => {
+  //           if (typeof genQueryString === 'function') {
+  //             const queryString = genQueryString(makeGetAtomResult(get))
+  //             addDebugInfo({ _tag: 'js', label: `${label}:queryString`, query: queryString })
+  //             return queryString
+  //           } else {
+  //             return genQueryString
+  //           }
+  //         },
+  //         { label: `${label}:queryString`, meta: { liveStoreThunkType: 'sqlQueryString' } },
+  //         otelContext,
+  //       )
+
+  //       label = label ?? queryString$.result
+  //       span.updateName(`querySQL:${label}`)
+
+  //       const queryLabel = `${label}:results` + (this.temporaryQueries ? ':temp' : '')
+
+  //       const results$ = this.graph.makeThunk<ReadonlyArray<TResult>>(
+  //         (get, addDebugInfo) =>
+  //           this.otel.tracer.startActiveSpan(
+  //             'sql:', // NOTE span name will be overridden further down
+  //             {},
+  //             otelContext,
+  //             (span) => {
+  //               try {
+  //                 const otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+  //                 // Establish a reactive dependency on the tables used in the query
+  //                 for (const tableName of queriedTables) {
+  //                   const tableRef =
+  //                     this.tableRefs[tableName] ?? shouldNeverHappen(`No table ref found for ${tableName}`)
+  //                   get(tableRef)
+  //                 }
+  //                 const sqlString = get(queryString$)
+
+  //                 span.setAttribute('sql.query', sqlString)
+  //                 span.updateName(`sql:${sqlString.slice(0, 50)}`)
+
+  //                 const results = this.inMemoryDB.select<TResult>(sqlString, {
+  //                   queriedTables,
+  //                   bindValues: bindValues ? prepareBindValues(bindValues, sqlString) : undefined,
+  //                   otelContext,
+  //                 })
+
+  //                 span.setAttribute('sql.rowsCount', results.length)
+  //                 addDebugInfo({ _tag: 'sql', label: label ?? '', query: sqlString })
+
+  //                 return results
+  //               } finally {
+  //                 span.end()
+  //               }
+  //             },
+  //           ),
+  //         { label: queryLabel },
+  //         otelContext,
+  //       )
+
+  //       const query = new LiveStoreSQLQuery<TResult>({
+  //         label,
+  //         queryString$,
+  //         results$,
+  //         componentKey: componentKey ?? globalComponentKey,
+  //         store: this,
+  //         otelContext,
+  //       })
+
+  //       this.activeQueries.add(query)
+
+  //       // TODO get rid of temporary query workaround
+  //       if (this.temporaryQueries !== undefined) {
+  //         this.temporaryQueries.add(query)
+  //       }
+
+  //       // NOTE we are not ending the span here but in the query `destroy` method
+  //       return query
+  //     },
+  //   )
+
+  // queryJS = <TResult>(
+  //   genResults: (get: GetAtomResult) => TResult,
+  //   {
+  //     componentKey = globalComponentKey,
+  //     label = `js${uniqueId()}`,
+  //     otelContext = otel.context.active(),
+  //   }: { componentKey?: ComponentKey; label?: string; otelContext?: otel.Context },
+  // ): LiveStoreJSQuery<TResult> =>
+  //   this.otel.tracer.startActiveSpan(`queryJS:${label}`, { attributes: { label } }, otelContext, (span) => {
+  //     const otelContext = otel.trace.setSpan(otel.context.active(), span)
+  //     const queryLabel = `${label}:results` + (this.temporaryQueries ? ':temp' : '')
+  //     const results$ = this.graph.makeThunk(
+  //       (get, addDebugInfo) => {
+  //         addDebugInfo({ _tag: 'js', label, query: genResults.toString() })
+  //         return genResults(makeGetAtomResult(get))
+  //       },
+  //       { label: queryLabel, meta: { liveStoreThunkType: 'jsResults' } },
+  //       otelContext,
+  //     )
+
+  //     // const query = new LiveStoreJSQuery<TResult>({
+  //     //   label,
+  //     //   results$,
+  //     //   componentKey,
+  //     //   store: this,
+  //     //   otelContext,
+  //     // })
+
+  //     this.activeQueries.add(query)
+
+  //     // TODO get rid of temporary query workaround
+  //     if (this.temporaryQueries !== undefined) {
+  //       this.temporaryQueries.add(query)
+  //     }
+
+  //     // NOTE we are not ending the span here but in the query `destroy` method
+  //     return query
+  //   })
+
+  // queryGraphQL = <TResult extends Record<string, any>, TVariableValues extends Record<string, any>>(
+  //   document: DocumentNode<TResult, TVariableValues>,
+  //   genVariableValues: TVariableValues | ((get: GetAtomResult) => TVariableValues),
+  //   {
+  //     componentKey,
+  //     label,
+  //     otelContext = otel.context.active(),
+  //   }: {
+  //     componentKey: ComponentKey
+  //     label?: string
+  //     otelContext?: otel.Context
+  //   },
+  // ): LiveStoreGraphQLQuery<TResult, TVariableValues, TGraphQLContext> =>
+  //   this.otel.tracer.startActiveSpan(
+  //     `queryGraphQL:`, // NOTE span name will be overridden further down
+  //     {},
+  //     otelContext,
+  //     (span) => {
+  //       const otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+  //       if (this.graphQLContext === undefined) {
+  //         return shouldNeverHappen("Can't run a GraphQL query on a store without GraphQL context")
+  //       }
+
+  //       const labelWithDefault = label ?? graphql.getOperationAST(document)?.name?.value ?? 'graphql'
+
+  //       span.updateName(`queryGraphQL:${labelWithDefault}`)
+
+  //       const variableValues$ = this.graph.makeThunk(
+  //         (get) => {
+  //           if (typeof genVariableValues === 'function') {
+  //             return genVariableValues(makeGetAtomResult(get))
+  //           } else {
+  //             return genVariableValues
+  //           }
+  //         },
+  //         { label: `${labelWithDefault}:variableValues`, meta: { liveStoreThunkType: 'graphqlVariableValues' } },
+  //         // otelContext,
+  //       )
+
+  //       const resultsLabel = `${labelWithDefault}:results` + (this.temporaryQueries ? ':temp' : '')
+  //       const results$ = this.graph.makeThunk<TResult>(
+  //         (get, addDebugInfo) => {
+  //           const variableValues = get(variableValues$)
+  //           const { result, queriedTables } = this.queryGraphQLOnce(document, variableValues, otelContext)
+
+  //           // Add dependencies on any tables that were used
+  //           for (const tableName of queriedTables) {
+  //             const tableRef = this.tableRefs[tableName]
+  //             assertNever(tableRef !== undefined, `No table ref found for ${tableName}`)
+  //             get(tableRef!)
+  //           }
+
+  //           addDebugInfo({ _tag: 'graphql', label: resultsLabel, query: graphql.print(document) })
+
+  //           return result
+  //         },
+  //         { label: resultsLabel, meta: { liveStoreThunkType: 'graphqlResults' } },
+  //         // otelContext,
+  //       )
+
+  //       const query = new LiveStoreGraphQLQuery({
+  //         document,
+  //         context: this.graphQLContext,
+  //         results$,
+  //         componentKey,
+  //         label: labelWithDefault,
+  //         store: this,
+  //         otelContext,
+  //       })
+
+  //       this.activeQueries.add(query)
+
+  //       // TODO get rid of temporary query workaround
+  //       if (this.temporaryQueries !== undefined) {
+  //         this.temporaryQueries.add(query)
+  //       }
+
+  //       // NOTE we are not ending the span here but in the query `destroy` method
+  //       return query
+  //     },
+  //   )
+
+  // queryGraphQLOnce = <TResult extends Record<string, any>, TVariableValues extends Record<string, any>>(
+  //   document: DocumentNode<TResult, TVariableValues>,
+  //   variableValues: TVariableValues,
+  //   otelContext: otel.Context = this.otel.queriesSpanContext,
+  // ): { result: TResult; queriedTables: string[] } => {
+  //   const schema =
+  //     this.graphQLSchema ?? shouldNeverHappen("Can't run a GraphQL query on a store without GraphQL schema")
+  //   const context =
+  //     this.graphQLContext ?? shouldNeverHappen("Can't run a GraphQL query on a store without GraphQL context")
+  //   const tracer = this.otel.tracer
+
+  //   const operationName = graphql.getOperationAST(document)?.name?.value
+
+  //   return tracer.startActiveSpan(`executeGraphQLQuery: ${operationName}`, {}, otelContext, (span) => {
+  //     try {
+  //       span.setAttribute('graphql.variables', JSON.stringify(variableValues))
+  //       span.setAttribute('graphql.query', graphql.print(document))
+
+  //       context.queriedTables.clear()
+
+  //       context.otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+  //       const res = graphql.executeSync({
+  //         document,
+  //         contextValue: context,
+  //         schema: schema,
+  //         variableValues,
+  //       })
+
+  //       // TODO track number of nested SQL queries via Otel + debug info
+
+  //       if (res.errors) {
+  //         span.setStatus({ code: otel.SpanStatusCode.ERROR, message: 'GraphQL error' })
+  //         span.setAttribute('graphql.error', res.errors.join('\n'))
+  //         span.setAttribute('graphql.error-detail', JSON.stringify(res.errors))
+  //         console.error(`graphql error (${operationName})`, res.errors)
+  //       }
+
+  //       return { result: res.data as unknown as TResult, queriedTables: Array.from(context.queriedTables.values()) }
+  //     } finally {
+  //       span.end()
+  //     }
+  //   })
+  // }
 
   /**
    * Subscribe to the results of a query
    * Returns a function to cancel the subscription.
    */
-  subscribe = <TQuery extends LiveStoreQuery>(
-    query: TQuery,
-    onNewValue: (value: QueryResult<TQuery>) => void,
+  subscribe = <TResult>(
+    query: ILiveStoreQuery<TResult>,
+    onNewValue: (value: TResult) => void,
     onSubsubscribe?: () => void,
-    options?: { label?: string } | undefined,
+    options?: { label?: string; otelContext?: otel.Context } | undefined,
   ): (() => void) =>
     this.otel.tracer.startActiveSpan(
       `LiveStore.subscribe`,
       { attributes: { label: options?.label } },
-      query.otelContext,
+      options?.otelContext ?? this.otel.queriesSpanContext,
       (span) => {
         const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
-        const effect = this.graph.makeEffect(
-          (get) => {
-            const result = get(query.results$) as QueryResult<TQuery>
-            onNewValue(result)
-          },
-          { label: `subscribe:${options?.label}` },
-          otelContext,
-        )
+        const effect = this.graph.makeEffect((get) => onNewValue(get(query.results$)), {
+          label: `subscribe:${options?.label}`,
+        })
+
+        effect.doEffect(otelContext)
 
-        const subscriptionKey = uuid()
+        // const subscriptionKey = uuid()
 
         const unsubscribe = () => {
           try {
             this.graph.destroy(effect)
-            query.activeSubscriptions.delete(subscriptionKey)
+            this.activeQueries.delete(query as LiveStoreQuery)
+            // query.activeSubscriptions.delete(subscriptionKey)
             onSubsubscribe?.()
           } finally {
             span.end()
           }
         }
 
-        query.activeSubscriptions.set(subscriptionKey, unsubscribe)
+        this.activeQueries.add(query as LiveStoreQuery)
+
+        // query.activeSubscriptions.set(subscriptionKey, unsubscribe)
 
         return unsubscribe
       },
     )
 
-  /**
-   * Any queries created in the callback will be destroyed when the callback is complete.
-   * Useful for temporarily creating reactive queries, which is an idempotent operation
-   * that can be safely called inside a React useMemo hook.
-   */
-  inTempQueryContext = <TResult>(callback: () => TResult): TResult => {
-    this.temporaryQueries = new Set()
-    // TODO: consider errors / try/finally here?
-    const result = callback()
-    for (const query of this.temporaryQueries) {
-      this.destroyQuery(query)
-    }
-    this.temporaryQueries = undefined
-    return result
-  }
-
   /**
    * Destroys the entire store, including all queries and subscriptions.
    *
    * Currently only used when shutting down the app for debugging purposes (e.g. to close Otel spans).
    */
   destroy = () => {
-    for (const query of this.activeQueries) {
-      this.destroyQuery(query)
-    }
-
     Object.values(this.tableRefs).forEach((tableRef) => this.graph.destroy(tableRef))
 
-    const applyEventsSpan = otel.trace.getSpan(this.otel.applyEventsSpanContext)!
-    applyEventsSpan.end()
-
-    const queriesSpan = otel.trace.getSpan(this.otel.queriesSpanContext)!
-    queriesSpan.end()
+    otel.trace.getSpan(this.otel.applyEventsSpanContext)!.end()
+    otel.trace.getSpan(this.otel.queriesSpanContext)!.end()
 
     // TODO destroy active subscriptions
   }
 
-  private destroyQuery = (query: LiveStoreQuery) => {
-    if (query._tag === 'sql') {
-      // results are downstream of query string, so will automatically be destroyed together
-      this.graph.destroy(query.queryString$)
-    } else {
-      this.graph.destroy(query.results$)
-    }
-    this.activeQueries.delete(query)
-    query.destroy()
-  }
-
-  /**
-   * Clean up queries and downstream subscriptions associated with a component.
-   * This is critical to avoid memory leaks.
-   */
-  unmountComponent = (componentKey: ComponentKey) => {
-    for (const query of this.activeQueries) {
-      if (query.componentKey === componentKey) {
-        this.destroyQuery(query)
-      }
-    }
-  }
-
   /* Apply a single write event to the store, and refresh all queries in response */
   applyEvent = <TEventType extends string & keyof LiveStoreActionDefinitionsTypes>(
     eventType: TEventType,
@@ -579,20 +553,25 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
             tablesToUpdate.push([tableRef!, null])
           }
 
+          const debugRefreshReason = {
+            _tag: 'applyEvent' as const,
+            event: { type: eventType, args },
+            writeTables: [...writeTables],
+          }
+
           // Update all table refs together in a batch, to only trigger one reactive update
-          this.graph.setRefs(
-            tablesToUpdate,
-            {
-              otelHint: 'applyEvents',
-              skipRefresh,
-              debugRefreshReason: {
-                _tag: 'applyEvent',
-                event: { type: eventType, args },
-                writeTables: [...writeTables],
-              },
-            },
-            otelContext,
-          )
+          this.graph.setRefs(tablesToUpdate, { debugRefreshReason, otelContext })
+
+          if (skipRefresh === false) {
+            // TODO update the graph
+            // this.graph.refresh(
+            //   {
+            //     otelHint: 'applyEvents',
+            //     debugRefreshReason,
+            //   },
+            //   otelContext,
+            // )
+          }
         } catch (e: any) {
           span.setStatus({ code: otel.SpanStatusCode.ERROR, message: e.toString() })
 
@@ -642,7 +621,7 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
               try {
                 const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
-                // TODO: what to do about backend transaction here?
+                // TODO: what to do about storage transaction here?
                 this.inMemoryDB.txn(() => {
                   for (const event of events) {
                     try {
@@ -675,20 +654,18 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
             tablesToUpdate.push([tableRef!, null])
           }
 
+          const debugRefreshReason = {
+            _tag: 'applyEvents' as const,
+            events: [...events].map((e) => ({ type: e.eventType, args: e.args })),
+            writeTables: [...writeTables],
+          }
           // Update all table refs together in a batch, to only trigger one reactive update
-          this.graph.setRefs(
-            tablesToUpdate,
-            {
-              otelHint: 'applyEvents',
-              skipRefresh,
-              debugRefreshReason: {
-                _tag: 'applyEvents',
-                events: [...events].map((e) => ({ type: e.eventType, args: e.args })),
-                writeTables: [...writeTables],
-              },
-            },
-            otelContext,
-          )
+          this.graph.setRefs(tablesToUpdate, { debugRefreshReason, otelContext })
+
+          if (skipRefresh === false) {
+            // TODO update the graph
+            // this.graph.refresh({ debugRefreshReason, otelHint: 'applyEvents' }, otelContext)
+          }
         } catch (e: any) {
           span.setStatus({ code: otel.SpanStatusCode.ERROR, message: e.toString() })
         } finally {
@@ -711,8 +688,9 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
       { attributes: { 'livestore.manualRefreshLabel': label } },
       this.otel.applyEventsSpanContext,
       (span) => {
-        const otelContext = otel.trace.setSpan(otel.context.active(), span)
-        this.graph.refresh({ otelHint: 'manualRefresh', debugRefreshReason: { _tag: 'manualRefresh' } }, otelContext)
+        // const otelContext = otel.trace.setSpan(otel.context.active(), span)
+        // TODO update the graph
+        // this.graph.refresh({ otelHint: 'manualRefresh', debugRefreshReason: { _tag: 'manualRefresh' } }, otelContext)
         span.end()
       },
     )
@@ -757,6 +735,15 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
               }
             },
           },
+
+          RawSql: {
+            statement: ({ sql, writeTables }: { sql: string; writeTables: string[] }) => ({
+              sql,
+              writeTables,
+              argsAlreadyBound: false,
+            }),
+            prepareBindValues: ({ bindValues }) => bindValues,
+          },
         }
 
         const actionDefinition = actionDefinitions[eventType] ?? shouldNeverHappen(`Unknown event type: ${eventType}`)
@@ -765,20 +752,30 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
         const eventWithId: LiveStoreEvent = { id: uuid(), type: eventType, args }
 
         // Synchronously apply the event to the in-memory database
-        const { durationMs } = this.inMemoryDB.applyEvent(eventWithId, actionDefinition, otelContext)
+        // const { durationMs } = this.inMemoryDB.applyEvent(eventWithId, actionDefinition, otelContext)
+        const { statement, bindValues } = eventToSql(eventWithId, actionDefinition)
+        const { durationMs } = this.inMemoryDB.execute(
+          statement.sql,
+          prepareBindValues(bindValues, statement.sql),
+          statement.writeTables,
+          {
+            otelContext,
+          },
+        )
 
-        // Asynchronously apply the event to a persistent backend (we're not awaiting this promise here)
-        if (this.backend !== undefined) {
-          this.backend.applyEvent(eventWithId, actionDefinition, span)
+        // Asynchronously apply the event to a persistent storage (we're not awaiting this promise here)
+        if (this.storage !== undefined) {
+          // this.storage.applyEvent(eventWithId, actionDefinition, span)
+          this.storage.execute(statement.sql, prepareBindValues(bindValues, statement.sql), span)
         }
 
         // Uncomment to print a list of queries currently registered on the store
         // console.log(JSON.parse(JSON.stringify([...this.queries].map((q) => `${labelForKey(q.componentKey)}/${q.label}`))))
 
-        const statement =
-          typeof actionDefinition.statement === 'function'
-            ? actionDefinition.statement(args)
-            : actionDefinition.statement
+        // const statement =
+        //   typeof actionDefinition.statement === 'function'
+        //     ? actionDefinition.statement(args)
+        //     : actionDefinition.statement
 
         span.end()
 
@@ -792,12 +789,12 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
    * This should only be used for framework-internal purposes;
    * all app writes should go through applyEvent.
    */
-  execute = async (query: string, params: ParamsObject = {}, writeTables?: string[]) => {
-    this.inMemoryDB.execute(query, params, writeTables)
+  execute = (query: string, params: ParamsObject = {}, writeTables?: string[]) => {
+    this.inMemoryDB.execute(query, prepareBindValues(params, query), writeTables)
 
-    if (this.backend !== undefined) {
+    if (this.storage !== undefined) {
       const parentSpan = otel.trace.getSpan(otel.context.active())
-      this.backend.execute(query, params, parentSpan)
+      this.storage.execute(query, prepareBindValues(params, query), parentSpan)
     }
   }
 }
@@ -805,83 +802,95 @@ export class Store<TGraphQLContext extends BaseGraphQLContext> {
 /** Create a new LiveStore Store */
 export const createStore = async <TGraphQLContext extends BaseGraphQLContext>({
   schema,
-  backendOptions,
+  loadStorage,
   graphQLOptions,
   otelTracer = makeNoopTracer(),
   otelRootSpanContext = otel.context.active(),
   boot,
+  sqlite3,
 }: {
   schema: Schema
-  backendOptions: BackendOptions
+  loadStorage: () => StorageInit | Promise<StorageInit>
   graphQLOptions?: GraphQLOptions<TGraphQLContext>
   otelTracer?: otel.Tracer
   otelRootSpanContext?: otel.Context
-  boot?: (backend: Backend, parentSpan: otel.Span) => Promise<void>
+  boot?: (db: InMemoryDatabase, parentSpan: otel.Span) => unknown | Promise<unknown>
+  sqlite3: Sqlite.Sqlite3Static
 }): Promise<Store<TGraphQLContext>> => {
   return otelTracer.startActiveSpan('createStore', {}, otelRootSpanContext, async (span) => {
     try {
-      let persistedData: Uint8Array | undefined
-      const backend = await createBackend(backendOptions, {
-        otelTracer: otelTracer ?? makeNoopTracer(),
-        parentSpan: otel.trace.getSpan(otelRootSpanContext ?? otel.context.active()) ?? makeNoopSpan(),
-      })
-      // if we're resetting the database, run boot here.
+      const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
-      let shouldResetDB = false
-      // Uncomment this line if you want to reset the database contents.
-      // let shouldResetDB = true
+      const storage = await otelTracer.startActiveSpan('storage:load', {}, otelContext, async (span) => {
+        try {
+          const init = await loadStorage()
+          const parentSpan = otel.trace.getSpan(otel.context.active()) ?? makeNoopSpan()
+          return init({ otelTracer, parentSpan })
+        } finally {
+          span.end()
+        }
+      })
 
-      const existingTablesRaw = await backend.select(
-        sql`SELECT * FROM sqlite_master WHERE type='table';`,
-        undefined,
-        span,
+      const persistedData = await otelTracer.startActiveSpan(
+        'storage:getPersistedData',
+        {},
+        otelContext,
+        async (span) => {
+          try {
+            return await storage.getPersistedData(span)
+          } finally {
+            span.end()
+          }
+        },
       )
-      const existingTables = existingTablesRaw.results.map((t: { name: string }) => t.name)
-      const missingTables = Object.keys(schema.tables).filter((tableName) => !existingTables.includes(tableName))
-      if (existingTables.length === 0) {
-        console.log('No existing tables found, loading from schema')
-        shouldResetDB = true
-      } else if (
-        missingTables.length > 0 &&
-        window.confirm(
-          `Existing DB is missing ${missingTables.length} tables: ${missingTables.join(
-            ', ',
-          )}\n\nReset DB? This will reset all of the following tables to empty: ${Object.keys(schema).join(', ')}`,
-        )
-      ) {
-        shouldResetDB = true
-      }
-
-      if (localStorage.getItem(RESET_DB_LOCAL_STORAGE_KEY) !== null) {
-        shouldResetDB = true
-      }
 
-      if (shouldResetDB) {
-        await loadSchema(backend, schema)
-        localStorage.removeItem(RESET_DB_LOCAL_STORAGE_KEY)
-      }
+      const db = InMemoryDatabase.load({ data: persistedData, otelTracer, otelRootSpanContext, sqlite3 })
 
-      if (boot) {
-        await boot(backend!, span)
-      }
+      // Proxy to `db` that also mirrors `execute` calls to `storage`
+      const dbProxy = new Proxy(db, {
+        get: (db, prop, receiver) => {
+          if (prop === 'execute') {
+            const execute: InMemoryDatabase['execute'] = (query, bindValues, writeTables, options) => {
+              storage.execute(query, bindValues, span)
+              return db.execute(query, bindValues, writeTables, options)
+            }
+            return execute
+          } else if (prop === 'select') {
+            // NOTE we're even proxying `select` calls here as some apps (e.g. Overtone) currently rely on this
+            // TODO remove this once we've migrated all apps to use `execute` instead of `select`
+            const select: InMemoryDatabase['select'] = (query, options = {}) => {
+              storage.execute(query, options.bindValues as any)
+              return db.select(query, options)
+            }
+            return select
+          } else {
+            return Reflect.get(db, prop, receiver)
+          }
+        },
+      })
 
-      const otelContext = otel.trace.setSpan(otel.context.active(), span)
-      await otelTracer.startActiveSpan('backend-getPersistedData', {}, otelContext, async (span) => {
+      otelTracer.startActiveSpan('migrateDb', {}, otelContext, async (span) => {
         try {
-          persistedData = await backend!.getPersistedData(span)
+          const otelContext = otel.trace.setSpan(otel.context.active(), span)
+          migrateDb({ db: dbProxy, schema, otelContext })
         } finally {
           span.end()
         }
       })
 
-      const db: InMemoryDatabase = await InMemoryDatabase.load(persistedData, otelTracer, otelRootSpanContext)
-      configureSQLite(db)
+      if (boot !== undefined) {
+        const booting = boot(dbProxy, span)
+        // NOTE only awaiting if it's actually a promise to avoid unnecessary async/await
+        if (isPromise(booting)) {
+          await booting
+        }
+      }
 
       // TODO: we can't apply the schema at this point, we've already loaded persisted data!
       // Think about what to do about this case.
       // await applySchema(db, schema)
       return Store.createStore<TGraphQLContext>(
-        { db, schema, backend, graphQLOptions, otelTracer, otelRootSpanContext },
+        { db, dbProxy, schema, storage, graphQLOptions, otelTracer, otelRootSpanContext },
         span,
       )
     } finally {
@@ -890,17 +899,17 @@ export const createStore = async <TGraphQLContext extends BaseGraphQLContext>({
   })
 }
 
-/** Set up SQLite performance; hasn't been super carefully optimized yet. */
-const configureSQLite = (db: InMemoryDatabase) => {
-  db.execute(
-    // TODO: revisit these tuning parameters for max performance
-    sql`
-      PRAGMA page_size=32768;
-      PRAGMA cache_size=10000;
-      PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
-      PRAGMA synchronous='OFF';
-      PRAGMA temp_store='MEMORY';
-      PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
-    `,
-  )
+const eventToSql = (
+  event: LiveStoreEvent,
+  eventDefinition: ActionDefinition,
+): { statement: SQLWriteStatement; bindValues: ParamsObject } => {
+  const statement =
+    typeof eventDefinition.statement === 'function' ? eventDefinition.statement(event.args) : eventDefinition.statement
+
+  const prepareBindValues = eventDefinition.prepareBindValues ?? identity
+
+  const bindValues =
+    typeof eventDefinition.statement === 'function' && statement.argsAlreadyBound ? {} : prepareBindValues(event.args)
+
+  return { statement, bindValues }
 }