@livestore/livestore 0.0.54-dev.5 → 0.0.55-dev.0

package/src/store.ts

@@ -1,22 +1,50 @@
-import type { BootDb, PreparedBindValues, ResetMode, StoreAdapter, StoreAdapterFactory } from '@livestore/common'
-import { Devtools, getExecArgsFromMutation } from '@livestore/common'
-import { version as liveStoreVersion } from '@livestore/common/package.json'
-import type { LiveStoreSchema, MutationEvent, MutationEventSchema } from '@livestore/common/schema'
-import { makeMutationEventSchema } from '@livestore/common/schema'
-import { assertNever, isPromise, makeNoopTracer, ref, shouldNeverHappen } from '@livestore/utils'
-import { Effect, Schema, Stream } from '@livestore/utils/effect'
+import type {
+  BootDb,
+  BootStatus,
+  DebugInfo,
+  ParamsObject,
+  PreparedBindValues,
+  ResetMode,
+  StoreAdapter,
+  StoreAdapterFactory,
+} from '@livestore/common'
+import {
+  Devtools,
+  getExecArgsFromMutation,
+  liveStoreVersion,
+  prepareBindValues,
+  UnexpectedError,
+} from '@livestore/common'
+import type { LiveStoreSchema, MutationEvent } from '@livestore/common/schema'
+import { makeMutationEventSchemaMemo } from '@livestore/common/schema'
+import { assertNever, makeNoopTracer, shouldNeverHappen, throttle } from '@livestore/utils'
+import { cuid } from '@livestore/utils/cuid'
+import {
+  Effect,
+  Exit,
+  FiberSet,
+  Inspectable,
+  Layer,
+  Logger,
+  LogLevel,
+  OtelTracer,
+  Queue,
+  Runtime,
+  Schema,
+  Scope,
+  Stream,
+} from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 import type { GraphQLSchema } from 'graphql'
 
-import { globalDbGraph } from './global-state.js'
-import { MainDatabaseWrapper } from './MainDatabaseWrapper.js'
+import { globalReactivityGraph } from './global-state.js'
+import { emptyDebugInfo as makeEmptyDebugInfo, MainDatabaseWrapper } from './MainDatabaseWrapper.js'
 import type { StackInfo } from './react/utils/stack-info.js'
-import type { DebugRefreshReasonBase, ReactiveGraph, Ref } from './reactive.js'
-import type { DbContext, DbGraph, LiveQuery } from './reactiveQueries/base-class.js'
+import type { DebugRefreshReasonBase, Ref } from './reactive.js'
+import { NOT_REFRESHED_YET } from './reactive.js'
+import type { LiveQuery, QueryContext, ReactivityGraph } from './reactiveQueries/base-class.js'
 import { downloadBlob } from './utils/dev.js'
 import { getDurationMsFromSpan } from './utils/otel.js'
-import type { ParamsObject } from './utils/util.js'
-import { prepareBindValues } from './utils/util.js'
 
 export type BaseGraphQLContext = {
   queriedTables: Set<string>
@@ -29,18 +57,25 @@ export type GraphQLOptions<TContext> = {
   makeContext: (db: MainDatabaseWrapper, tracer: otel.Tracer) => TContext
 }
 
+export type OtelOptions = {
+  tracer: otel.Tracer
+  rootSpanContext: otel.Context
+}
+
 export type StoreOptions<
   TGraphQLContext extends BaseGraphQLContext,
   TSchema extends LiveStoreSchema = LiveStoreSchema,
 > = {
   adapter: StoreAdapter
   schema: TSchema
+  // TODO remove graphql-related stuff from store and move to GraphQL query directly
   graphQLOptions?: GraphQLOptions<TGraphQLContext>
-  otelTracer: otel.Tracer
-  otelRootSpanContext: otel.Context
-  dbGraph: DbGraph
-  mutationEventSchema: MutationEventSchema<any>
+  otelOptions: OtelOptions
+  reactivityGraph: ReactivityGraph
   disableDevtools?: boolean
+  fiberSet: FiberSet.FiberSet
+  // TODO remove this temporary solution and find a better way to avoid re-processing the same mutation
+  __processedMutationIds: Set<string>
 }
 
 export type RefreshReason =
@@ -93,13 +128,12 @@ export type StoreMutateOptions = {
 export class Store<
   TGraphQLContext extends BaseGraphQLContext = BaseGraphQLContext,
   TSchema extends LiveStoreSchema = LiveStoreSchema,
-> {
+> extends Inspectable.Class {
   id = uniqueStoreId()
-  graph: ReactiveGraph<RefreshReason, QueryDebugInfo, DbContext>
+  readonly devtoolsConnectionId = cuid()
+  private fiberSet: FiberSet.FiberSet
+  reactivityGraph: ReactivityGraph
   mainDbWrapper: MainDatabaseWrapper
-  // TODO refactor
-  // _proxyDb: InMemoryDatabase
-  // TODO
   adapter: StoreAdapter
   schema: LiveStoreSchema
   graphQLSchema?: GraphQLSchema
@@ -109,11 +143,11 @@ export class Store<
    * Note we're using `Ref<null>` here as we don't care about the value but only about *that* something has changed.
    * This only works in combination with `equal: () => false` which will always trigger a refresh.
    */
-  tableRefs: { [key: string]: Ref<null, DbContext, RefreshReason> }
+  tableRefs: { [key: string]: Ref<null, QueryContext, RefreshReason> }
 
   // TODO remove this temporary solution and find a better way to avoid re-processing the same mutation
-  __processedMutationIds = new Set<string>()
-  __processedMutationWithoutRefreshIds = new Set<string>()
+  private __processedMutationIds
+  private __processedMutationWithoutRefreshIds = new Set<string>()
 
   /** RC-based set to see which queries are currently subscribed to */
   activeQueries: ReferenceCountedSet<LiveQuery<any>>
@@ -124,60 +158,53 @@ export class Store<
     adapter,
     schema,
     graphQLOptions,
-    dbGraph,
-    otelTracer,
-    otelRootSpanContext,
-    mutationEventSchema,
+    reactivityGraph,
+    otelOptions,
     disableDevtools,
+    __processedMutationIds,
+    fiberSet,
   }: StoreOptions<TGraphQLContext, TSchema>) {
-    this.mainDbWrapper = new MainDatabaseWrapper({ otelTracer, otelRootSpanContext, db: adapter.mainDb })
+    super()
+
+    this.mainDbWrapper = new MainDatabaseWrapper({ otel: otelOptions, db: adapter.mainDb })
     this.adapter = adapter
     this.schema = schema
 
+    this.fiberSet = fiberSet
+
     // TODO refactor
-    this.__mutationEventSchema = mutationEventSchema
-    // this.mutationEventSchema = makeMutationEventSchema(Object.fromEntries(schema.mutations.entries()) as any)
+    this.__mutationEventSchema = makeMutationEventSchemaMemo(schema)
+
+    // TODO remove this temporary solution and find a better way to avoid re-processing the same mutation
+    this.__processedMutationIds = __processedMutationIds
 
     // TODO generalize the `tableRefs` concept to allow finer-grained refs
     this.tableRefs = {}
     this.activeQueries = new ReferenceCountedSet()
 
-    const mutationsSpan = otelTracer.startSpan('LiveStore:mutations', {}, otelRootSpanContext)
+    const mutationsSpan = otelOptions.tracer.startSpan('LiveStore:mutations', {}, otelOptions.rootSpanContext)
     const otelMuationsSpanContext = otel.trace.setSpan(otel.context.active(), mutationsSpan)
 
-    const queriesSpan = otelTracer.startSpan('LiveStore:queries', {}, otelRootSpanContext)
+    const queriesSpan = otelOptions.tracer.startSpan('LiveStore:queries', {}, otelOptions.rootSpanContext)
     const otelQueriesSpanContext = otel.trace.setSpan(otel.context.active(), queriesSpan)
 
-    this.graph = dbGraph
-    this.graph.context = { store: this as any, otelTracer, rootOtelContext: otelQueriesSpanContext }
-
-    this.adapter.coordinator.syncMutations.pipe(
-      Stream.tapSync((mutationEventDecoded) => {
-        this.mutate({ wasSyncMessage: true }, mutationEventDecoded)
-      }),
-      Stream.runDrain,
-      Effect.tapCauseLogPretty,
-      Effect.runFork,
-    )
-
-    if (disableDevtools !== true) {
-      this.bootDevtools()
+    this.reactivityGraph = reactivityGraph
+    this.reactivityGraph.context = {
+      store: this as unknown as Store<BaseGraphQLContext, LiveStoreSchema>,
+      otelTracer: otelOptions.tracer,
+      rootOtelContext: otelQueriesSpanContext,
     }
 
     this.otel = {
-      tracer: otelTracer,
+      tracer: otelOptions.tracer,
       mutationsSpanContext: otelMuationsSpanContext,
       queriesSpanContext: otelQueriesSpanContext,
     }
 
     // Need a set here since `schema.tables` might contain duplicates and some componentStateTables
-    const allTableNames = new Set(
-      this.schema.tables.keys(),
-      // TODO activate dynamic tables
-      // ...Array.from(dynamicallyRegisteredTables.values()).map((_) => _.sqliteDef.name),
-    )
+    const allTableNames = new Set(this.schema.tables.keys())
     const existingTableRefs = new Map(
-      Array.from(this.graph.atoms.values())
+      Array.from(this.reactivityGraph.atoms.values())
         .filter((_): _ is Ref<any, any, any> => _._tag === 'ref' && _.label?.startsWith('tableRef:') === true)
         .map((_) => [_.label!.slice('tableRef:'.length), _] as const),
     )
@@ -189,6 +216,36 @@ export class Store<
       this.graphQLSchema = graphQLOptions.schema
       this.graphQLContext = graphQLOptions.makeContext(this.mainDbWrapper, this.otel.tracer)
     }
+
+    Effect.gen(this, function* () {
+      yield* this.adapter.coordinator.syncMutations.pipe(
+        Stream.tapSync((mutationEventDecoded) => {
+          this.mutate({ wasSyncMessage: true }, mutationEventDecoded)
+        }),
+        Stream.runDrain,
+        Effect.withSpan('LiveStore:syncMutations'),
+        Effect.forkScoped,
+      )
+
+      if (disableDevtools !== true) {
+        yield* this.bootDevtools().pipe(Effect.forkScoped)
+      }
+
+      yield* Effect.addFinalizer(() =>
+        Effect.sync(() => {
+          for (const tableRef of Object.values(this.tableRefs)) {
+            for (const superComp of tableRef.super) {
+              this.reactivityGraph.removeEdge(superComp, tableRef)
+            }
+          }
+
+          otel.trace.getSpan(this.otel.mutationsSpanContext)!.end()
+          otel.trace.getSpan(this.otel.queriesSpanContext)!.end()
+        }),
+      )
+
+      yield* Effect.never
+    }).pipe(Effect.scoped, Effect.withSpan('LiveStore:store-constructor'), FiberSet.run(fiberSet), runEffectFork)
   }
 
   static createStore = <TGraphQLContext extends BaseGraphQLContext, TSchema extends LiveStoreSchema = LiveStoreSchema>(
@@ -196,7 +253,7 @@ export class Store<
     parentSpan: otel.Span,
   ): Store<TGraphQLContext, TSchema> => {
     const ctx = otel.trace.setSpan(otel.context.active(), parentSpan)
-    return storeOptions.otelTracer.startActiveSpan('LiveStore:store-constructor', {}, ctx, (span) => {
+    return storeOptions.otelOptions.tracer.startActiveSpan('LiveStore:store-constructor', {}, ctx, (span) => {
       try {
         return new Store(storeOptions)
       } finally {
@@ -224,7 +281,7 @@ export class Store<
         const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
         const label = `subscribe:${options?.label}`
-        const effect = this.graph.makeEffect((get) => onNewValue(get(query$.results$)), { label })
+        const effect = this.reactivityGraph.makeEffect((get) => onNewValue(get(query$.results$)), { label })
 
         this.activeQueries.add(query$ as LiveQuery<TResult>)
 
@@ -236,7 +293,7 @@ export class Store<
         const unsubscribe = () => {
           // console.log('store unsub', query$.label)
           try {
-            this.graph.destroyNode(effect)
+            this.reactivityGraph.destroyNode(effect)
             this.activeQueries.remove(query$ as LiveQuery<TResult>)
             onUnsubsubscribe?.()
           } finally {
@@ -254,16 +311,7 @@ export class Store<
    * Currently only used when shutting down the app for debugging purposes (e.g. to close Otel spans).
    */
   destroy = async () => {
-    for (const tableRef of Object.values(this.tableRefs)) {
-      for (const superComp of tableRef.super) {
-        this.graph.removeEdge(superComp, tableRef)
-      }
-    }
-
-    otel.trace.getSpan(this.otel.mutationsSpanContext)!.end()
-    otel.trace.getSpan(this.otel.queriesSpanContext)!.end()
-
-    await this.adapter.coordinator.shutdown()
+    await FiberSet.clear(this.fiberSet).pipe(Effect.withSpan('Store:destroy'), runEffectPromise)
   }
 
   mutate: {
@@ -323,6 +371,8 @@ export class Store<
     // mutationsEvents.forEach((_) => console.log(_.mutation, _.id, _.args))
     // console.groupEnd()
 
+    let durationMs: number
+
     return this.otel.tracer.startActiveSpan(
       'LiveStore:mutate',
       { attributes: { 'livestore.mutateLabel': label } },
@@ -353,8 +403,8 @@ export class Store<
                         writeTables.add(tableName)
                       }
                     } catch (e: any) {
-                      debugger
                       console.error(e, mutationEvent)
+                      throw e
                     }
                   }
                 }
@@ -368,13 +418,14 @@ export class Store<
               } catch (e: any) {
                 console.error(e)
                 span.setStatus({ code: otel.SpanStatusCode.ERROR, message: e.toString() })
+                throw e
               } finally {
                 span.end()
               }
             },
           )
 
-          const tablesToUpdate = [] as [Ref<null, DbContext, RefreshReason>, null][]
+          const tablesToUpdate = [] as [Ref<null, QueryContext, RefreshReason>, null][]
           for (const tableName of writeTables) {
             const tableRef = this.tableRefs[tableName]
             assertNever(tableRef !== undefined, `No table ref found for ${tableName}`)
@@ -388,14 +439,18 @@ export class Store<
           }
 
           // Update all table refs together in a batch, to only trigger one reactive update
-          this.graph.setRefs(tablesToUpdate, { debugRefreshReason, otelContext, skipRefresh })
+          this.reactivityGraph.setRefs(tablesToUpdate, { debugRefreshReason, otelContext, skipRefresh })
         } catch (e: any) {
+          console.error(e)
           span.setStatus({ code: otel.SpanStatusCode.ERROR, message: e.toString() })
+          throw e
         } finally {
           span.end()
 
-          return { durationMs: getDurationMsFromSpan(span) }
+          durationMs = getDurationMsFromSpan(span)
         }
+
+        return { durationMs }
       },
     )
   }
@@ -412,7 +467,7 @@ export class Store<
       this.otel.mutationsSpanContext,
       (span) => {
         const otelContext = otel.trace.setSpan(otel.context.active(), span)
-        this.graph.runDeferredEffects({ otelContext })
+        this.reactivityGraph.runDeferredEffects({ otelContext })
         span.end()
       },
     )
@@ -480,10 +535,9 @@ export class Store<
 
         if (coordinatorMode !== 'skip-coordinator') {
           // Asynchronously apply mutation to a persistent storage (we're not awaiting this promise here)
-          void this.adapter.coordinator.mutate(mutationEventEncoded, {
-            span,
-            persisted: coordinatorMode !== 'skip-persist',
-          })
+          this.adapter.coordinator
+            .mutate(mutationEventEncoded as MutationEvent.AnyEncoded, { persisted: coordinatorMode !== 'skip-persist' })
+            .pipe(runEffectFork)
         }
 
         // Uncomment to print a list of queries currently registered on the store
@@ -509,8 +563,7 @@ export class Store<
   ) => {
     this.mainDbWrapper.execute(query, prepareBindValues(params, query), writeTables, { otelContext })
 
-    const parentSpan = otel.trace.getSpan(otel.context.active())
-    this.adapter.coordinator.execute(query, prepareBindValues(params, query), parentSpan)
+    this.adapter.coordinator.execute(query, prepareBindValues(params, query)).pipe(runEffectFork)
   }
 
   select = (query: string, params: ParamsObject = {}) => {
@@ -518,117 +571,227 @@ export class Store<
   }
 
   makeTableRef = (tableName: string) =>
-    this.graph.makeRef(null, {
+    this.reactivityGraph.makeRef(null, {
       equal: () => false,
       label: `tableRef:${tableName}`,
       meta: { liveStoreRefType: 'table' },
     })
 
-  private bootDevtools = () => {
-    const devtoolsChannel = Devtools.makeBroadcastChannels()
-
-    const signalsSubcriptionRef = ref<undefined | (() => void)>(undefined)
-    // let alreadySubscribedToLiveQueries = false
-    const liveQueriesSubscriptionRef = ref<undefined | (() => void)>(undefined)
-    devtoolsChannel.toAppHost.addEventListener('message', async (event) => {
-      const decoded = Schema.decodeUnknownOption(Devtools.MessageToAppHost)(event.data)
-      if (
-        decoded._tag === 'None' ||
-        decoded.value._tag === 'LSD.DevtoolsReadyBroadcast' ||
-        decoded.value._tag === 'LSD.DevtoolsConnected' ||
-        decoded.value.channelId !== this.adapter.coordinator.devtools.channelId
-      ) {
-        // console.log(`Unknown message`, event)
-        return
+  // #region devtools
+  // TODO shutdown behaviour
+  private bootDevtools = () =>
+    Effect.gen(this, function* () {
+      const sendToDevtoolsContentscript = (
+        message: typeof Devtools.DevtoolsWindowMessage.MessageForContentscript.Type,
+      ) => {
+        window.postMessage(Schema.encodeSync(Devtools.DevtoolsWindowMessage.MessageForContentscript)(message), '*')
       }
 
-      const requestId = decoded.value.requestId
-      const sendToDevtools = (message: Devtools.MessageFromAppHost) =>
-        devtoolsChannel.fromAppHost.postMessage(Schema.encodeSync(Devtools.MessageFromAppHost)(message))
-
-      switch (decoded.value._tag) {
-        case 'LSD.SignalsSubscribe': {
-          const includeResults = decoded.value.includeResults
-          const send = () =>
-            sendToDevtools(
-              Devtools.SignalsRes.make({
-                signals: this.graph.getSnapshot({ includeResults }),
-                requestId,
-                liveStoreVersion,
-              }),
-            )
+      sendToDevtoolsContentscript(Devtools.DevtoolsWindowMessage.LoadIframe.make({}))
 
-          send()
+      const channelId = this.adapter.coordinator.devtools.channelId
 
-          if (signalsSubcriptionRef.current === undefined) {
-            signalsSubcriptionRef.current = this.graph.subscribeToRefresh(() => send())
-          }
+      const runtime = yield* Effect.runtime()
 
-          break
-        }
-        case 'LSD.DebugInfoReq': {
-          sendToDevtools(
-            Devtools.DebugInfoRes.make({ debugInfo: this.mainDbWrapper.debugInfo, requestId, liveStoreVersion }),
-          )
-          break
-        }
-        case 'LSD.DebugInfoResetReq': {
-          this.mainDbWrapper.debugInfo.slowQueries.clear()
-          sendToDevtools(Devtools.DebugInfoResetRes.make({ requestId, liveStoreVersion }))
-          break
-        }
-        case 'LSD.DebugInfoRerunQueryReq': {
-          const { queryStr, bindValues, queriedTables } = decoded.value
-          this.mainDbWrapper.select(queryStr, { bindValues, queriedTables, skipCache: true })
-          sendToDevtools(Devtools.DebugInfoRerunQueryRes.make({ requestId, liveStoreVersion }))
-          break
-        }
-        case 'LSD.SignalsUnsubscribe': {
-          signalsSubcriptionRef.current!()
-          signalsSubcriptionRef.current = undefined
-          break
+      window.addEventListener('message', (event) => {
+        const decodedMessageRes = Schema.decodeOption(Devtools.DevtoolsWindowMessage.MessageForStore)(event.data)
+        if (decodedMessageRes._tag === 'None') return
+
+        const message = decodedMessageRes.value
+
+        if (message._tag === 'LSD.WindowMessage.ContentscriptListening') {
+          sendToDevtoolsContentscript(Devtools.DevtoolsWindowMessage.StoreReady.make({ channelId }))
+          return
         }
-        case 'LSD.LiveQueriesSubscribe': {
-          const send = () =>
-            sendToDevtools(
-              Devtools.LiveQueriesRes.make({
-                liveQueries: [...this.activeQueries].map((q) => ({
-                  _tag: q._tag,
-                  id: q.id,
-                  label: q.label,
-                  runs: q.runs,
-                  executionTimes: q.executionTimes.map((_) => Number(_.toString().slice(0, 5))),
-                  lastestResult: q.results$.previousResult,
-                  activeSubscriptions: Array.from(q.activeSubscriptions),
-                })),
-                requestId,
-                liveStoreVersion,
-              }),
-            )
 
-          send()
+        if (message.channelId !== channelId) return
 
-          if (liveQueriesSubscriptionRef.current === undefined) {
-            liveQueriesSubscriptionRef.current = this.graph.subscribeToRefresh(() => send())
-          }
+        if (message._tag === 'LSD.WindowMessage.MessagePortForStore') {
+          type Unsub = () => void
+          type RequestId = string
 
-          break
-        }
-        case 'LSD.LiveQueriesUnsubscribe': {
-          liveQueriesSubscriptionRef.current!()
-          liveQueriesSubscriptionRef.current = undefined
-          break
-        }
-        case 'LSD.ResetAllDataReq': {
-          await this.adapter.coordinator.dangerouslyReset(decoded.value.mode)
-          sendToDevtools(Devtools.ResetAllDataRes.make({ requestId, liveStoreVersion }))
+          const reactivityGraphSubcriptions = new Map<RequestId, Unsub>()
+          const liveQueriesSubscriptions = new Map<RequestId, Unsub>()
+          const debugInfoHistorySubscriptions = new Map<RequestId, Unsub>()
+
+          this.adapter.coordinator.devtools
+            .connect({ port: message.port, connectionId: this.devtoolsConnectionId })
+            .pipe(
+              Effect.tapSync(({ storeMessagePort }) => {
+                // console.log('storeMessagePort', storeMessagePort)
+                storeMessagePort.addEventListener('message', (event) => {
+                  const decodedMessage = Schema.decodeUnknownSync(Devtools.MessageToAppHostStore)(event.data)
+                  // console.log('storeMessagePort message', decodedMessage)
+
+                  if (decodedMessage.channelId !== this.adapter.coordinator.devtools.channelId) {
+                    // console.log(`Unknown message`, event)
+                    return
+                  }
+
+                  const requestId = decodedMessage.requestId
+                  const sendToDevtools = (message: Devtools.MessageFromAppHostStore) =>
+                    storeMessagePort.postMessage(Schema.encodeSync(Devtools.MessageFromAppHostStore)(message))
+
+                  const requestIdleCallback = window.requestIdleCallback ?? ((cb: Function) => cb())
+
+                  switch (decodedMessage._tag) {
+                    case 'LSD.ReactivityGraphSubscribe': {
+                      const includeResults = decodedMessage.includeResults
+
+                      const send = () =>
+                        // In order to not add more work to the current tick, we use requestIdleCallback
+                        // to send the reactivity graph updates to the devtools
+                        requestIdleCallback(
+                          () =>
+                            sendToDevtools(
+                              Devtools.ReactivityGraphRes.make({
+                                reactivityGraph: this.reactivityGraph.getSnapshot({ includeResults }),
+                                requestId,
+                                liveStoreVersion,
+                              }),
+                            ),
+                          { timeout: 500 },
+                        )
+
+                      send()
+
+                      // In some cases, there can be A LOT of reactivity graph updates in a short period of time
+                      // so we throttle the updates to avoid sending too much data
+                      // This might need to be tweaked further and possibly be exposed to the user in some way.
+                      const throttledSend = throttle(send, 20)
+
+                      reactivityGraphSubcriptions.set(requestId, this.reactivityGraph.subscribeToRefresh(throttledSend))
+
+                      break
+                    }
+                    case 'LSD.DebugInfoReq': {
+                      sendToDevtools(
+                        Devtools.DebugInfoRes.make({
+                          debugInfo: this.mainDbWrapper.debugInfo,
+                          requestId,
+                          liveStoreVersion,
+                        }),
+                      )
+                      break
+                    }
+                    case 'LSD.DebugInfoHistorySubscribe': {
+                      const buffer: DebugInfo[] = []
+                      let hasStopped = false
+                      let rafHandle: number | undefined
+
+                      const tick = () => {
+                        buffer.push(this.mainDbWrapper.debugInfo)
+
+                        // NOTE this resets the debug info, so all other "readers" e.g. in other `requestAnimationFrame` loops,
+                        // will get the empty debug info
+                        // TODO We need to come up with a more graceful way to do this. Probably via a single global
+                        // `requestAnimationFrame` loop that is passed in somehow.
+                        this.mainDbWrapper.debugInfo = makeEmptyDebugInfo()
+
+                        if (buffer.length > 10) {
+                          sendToDevtools(
+                            Devtools.DebugInfoHistoryRes.make({
+                              debugInfoHistory: buffer,
+                              requestId,
+                              liveStoreVersion,
+                            }),
+                          )
+                          buffer.length = 0
+                        }
+
+                        if (hasStopped === false) {
+                          rafHandle = requestAnimationFrame(tick)
+                        }
+                      }
+
+                      rafHandle = requestAnimationFrame(tick)
+
+                      const unsub = () => {
+                        hasStopped = true
+                        if (rafHandle !== undefined) {
+                          cancelAnimationFrame(rafHandle)
+                        }
+                      }
+
+                      debugInfoHistorySubscriptions.set(requestId, unsub)
+
+                      break
+                    }
+                    case 'LSD.DebugInfoHistoryUnsubscribe': {
+                      debugInfoHistorySubscriptions.get(requestId)!()
+                      debugInfoHistorySubscriptions.delete(requestId)
+                      break
+                    }
+                    case 'LSD.DebugInfoResetReq': {
+                      this.mainDbWrapper.debugInfo.slowQueries.clear()
+                      sendToDevtools(Devtools.DebugInfoResetRes.make({ requestId, liveStoreVersion }))
+                      break
+                    }
+                    case 'LSD.DebugInfoRerunQueryReq': {
+                      const { queryStr, bindValues, queriedTables } = decodedMessage
+                      this.mainDbWrapper.select(queryStr, { bindValues, queriedTables, skipCache: true })
+                      sendToDevtools(Devtools.DebugInfoRerunQueryRes.make({ requestId, liveStoreVersion }))
+                      break
+                    }
+                    case 'LSD.ReactivityGraphUnsubscribe': {
+                      reactivityGraphSubcriptions.get(requestId)!()
+                      break
+                    }
+                    case 'LSD.LiveQueriesSubscribe': {
+                      const send = () =>
+                        requestIdleCallback(
+                          () =>
+                            sendToDevtools(
+                              Devtools.LiveQueriesRes.make({
+                                liveQueries: [...this.activeQueries].map((q) => ({
+                                  _tag: q._tag,
+                                  id: q.id,
+                                  label: q.label,
+                                  runs: q.runs,
+                                  executionTimes: q.executionTimes.map((_) => Number(_.toString().slice(0, 5))),
+                                  lastestResult:
+                                    q.results$.previousResult === NOT_REFRESHED_YET
+                                      ? 'SYMBOL_NOT_REFRESHED_YET'
+                                      : q.results$.previousResult,
+                                  activeSubscriptions: Array.from(q.activeSubscriptions),
+                                })),
+                                requestId,
+                                liveStoreVersion,
+                              }),
+                            ),
+                          { timeout: 500 },
+                        )
+
+                      send()
+
+                      // Same as in the reactivity graph subscription case above, we need to throttle the updates
+                      const throttledSend = throttle(send, 20)
+
+                      liveQueriesSubscriptions.set(requestId, this.reactivityGraph.subscribeToRefresh(throttledSend))
+
+                      break
+                    }
+                    case 'LSD.LiveQueriesUnsubscribe': {
+                      liveQueriesSubscriptions.get(requestId)!()
+                      liveQueriesSubscriptions.delete(requestId)
+                      break
+                    }
+                    // No default
+                  }
+                })
 
-          break
+                storeMessagePort.start()
+              }),
+              Runtime.runFork(runtime),
+            )
+
+          return
         }
-        // No default
-      }
+      })
+
+      sendToDevtoolsContentscript(Devtools.DevtoolsWindowMessage.StoreReady.make({ channelId }))
     })
-  }
+  // #endregion devtools
 
   __devDownloadDb = () => {
     const data = this.mainDbWrapper.export()
@@ -636,146 +799,214 @@ export class Store<
   }
 
   __devDownloadMutationLogDb = async () => {
-    const data = await this.adapter.coordinator.getMutationLogData()
+    const data = await this.adapter.coordinator.getMutationLogData.pipe(runEffectPromise)
     downloadBlob(data, `livestore-mutationlog-${Date.now()}.db`)
   }
 
   // TODO allow for graceful store reset without requiring a full page reload (which should also call .boot)
-  dangerouslyResetStorage = (mode: ResetMode) => this.adapter.coordinator.dangerouslyReset(mode)
+  dangerouslyResetStorage = (mode: ResetMode) => this.adapter.coordinator.dangerouslyReset(mode).pipe(runEffectPromise)
+
+  toJSON = () => {
+    return {
+      _tag: 'Store',
+      reactivityGraph: this.reactivityGraph.getSnapshot({ includeResults: true }),
+    }
+  }
+}
+
+export type CreateStoreOptions<TGraphQLContext extends BaseGraphQLContext, TSchema extends LiveStoreSchema> = {
+  schema: TSchema
+  adapter: StoreAdapterFactory
+  reactivityGraph?: ReactivityGraph
+  graphQLOptions?: GraphQLOptions<TGraphQLContext>
+  otelOptions?: Partial<OtelOptions>
+  boot?: (db: BootDb, parentSpan: otel.Span) => void | Promise<void> | Effect.Effect<void, unknown, otel.Tracer>
+  batchUpdates?: (run: () => void) => void
+  disableDevtools?: boolean
+  onBootStatus?: (status: BootStatus) => void
 }
 
 /** Create a new LiveStore Store */
-export const createStore = async <
+export const createStorePromise = async <
+  TGraphQLContext extends BaseGraphQLContext,
+  TSchema extends LiveStoreSchema = LiveStoreSchema,
+>({
+  signal,
+  ...options
+}: CreateStoreOptions<TGraphQLContext, TSchema> & { signal?: AbortSignal }): Promise<Store<TGraphQLContext, TSchema>> =>
+  Effect.gen(function* () {
+    const scope = yield* Scope.make()
+    const runtime = yield* Effect.runtime()
+
+    if (signal !== undefined) {
+      signal.addEventListener('abort', () => {
+        Scope.close(scope, Exit.void).pipe(Effect.tapCauseLogPretty, Runtime.runFork(runtime))
+      })
+    }
+
+    return yield* FiberSet.make().pipe(
+      Effect.andThen((fiberSet) => createStore({ ...options, fiberSet })),
+      Scope.extend(scope),
+    )
+  }).pipe(Effect.withSpan('createStore'), runEffectPromise)
+
+// #region createStore
+export const createStore = <
   TGraphQLContext extends BaseGraphQLContext,
   TSchema extends LiveStoreSchema = LiveStoreSchema,
 >({
   schema,
   graphQLOptions,
-  otelTracer = makeNoopTracer(),
-  otelRootSpanContext = otel.context.active(),
+  otelOptions,
   adapter: adapterFactory,
   boot,
-  dbGraph = globalDbGraph,
+  reactivityGraph = globalReactivityGraph,
   batchUpdates,
   disableDevtools,
-}: {
-  schema: TSchema
-  graphQLOptions?: GraphQLOptions<TGraphQLContext>
-  otelTracer?: otel.Tracer
-  otelRootSpanContext?: otel.Context
-  adapter: StoreAdapterFactory
-  boot?: (db: BootDb, parentSpan: otel.Span) => unknown | Promise<unknown>
-  dbGraph?: DbGraph
-  batchUpdates?: (run: () => void) => void
-  disableDevtools?: boolean
-}): Promise<Store<TGraphQLContext, TSchema>> => {
-  return otelTracer.startActiveSpan('createStore', {}, otelRootSpanContext, async (span) => {
-    try {
-      performance.mark('livestore:db-creating')
-      const otelContext = otel.trace.setSpan(otel.context.active(), span)
-
-      const adapterPromise = adapterFactory({ otelTracer, otelContext, schema })
-      const adapter = adapterPromise instanceof Promise ? await adapterPromise : adapterPromise
-      performance.mark('livestore:db-created')
-      performance.measure('livestore:db-create', 'livestore:db-creating', 'livestore:db-created')
-
-      if (batchUpdates !== undefined) {
-        dbGraph.effectsWrapper = batchUpdates
-      }
+  onBootStatus,
+  fiberSet,
+}: CreateStoreOptions<TGraphQLContext, TSchema> & { fiberSet: FiberSet.FiberSet }): Effect.Effect<
+  Store<TGraphQLContext, TSchema>,
+  UnexpectedError,
+  Scope.Scope
+> => {
+  const otelTracer = otelOptions?.tracer ?? makeNoopTracer()
+  const otelRootSpanContext = otelOptions?.rootSpanContext ?? otel.context.active()
+
+  const TracingLive = Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
+    Layer.provide(Layer.sync(OtelTracer.Tracer, () => otelTracer)),
+  )
+
+  return Effect.gen(function* () {
+    const span = yield* OtelTracer.currentOtelSpan.pipe(Effect.orDie)
+
+    const bootStatusQueue = yield* Queue.unbounded<BootStatus>().pipe(Effect.acquireRelease(Queue.shutdown))
+
+    yield* Queue.take(bootStatusQueue).pipe(
+      Effect.tapSync((status) => onBootStatus?.(status)),
+      Effect.forever,
+      Effect.tapCauseLogPretty,
+      Effect.forkScoped,
+    )
 
-      const mutationEventSchema = makeMutationEventSchema(Object.fromEntries(schema.mutations.entries()) as any)
+    const adapter: StoreAdapter = yield* adapterFactory({
+      schema,
+      devtoolsEnabled: disableDevtools !== true,
+      bootStatusQueue,
+      shutdown: (cause) =>
+        Effect.gen(function* () {
+          yield* Effect.logWarning(`Shutting down LiveStore`, cause)
+          // TODO close parent scope? (Needs refactor with Mike A)
+          yield* FiberSet.clear(fiberSet)
+        }).pipe(Effect.withSpan('livestore:shutdown')),
+    }).pipe(Effect.withPerformanceMeasure('livestore:makeAdapter'), Effect.withSpan('createStore:makeAdapter'))
+
+    if (batchUpdates !== undefined) {
+      reactivityGraph.effectsWrapper = batchUpdates
+    }
 
-      // TODO consider moving booting into the storage backend
-      if (boot !== undefined) {
-        let isInTxn = false
-        let txnExecuteStmnts: [string, PreparedBindValues | undefined][] = []
+    const mutationEventSchema = makeMutationEventSchemaMemo(schema)
 
-        const bootDbImpl: BootDb = {
-          _tag: 'BootDb',
-          execute: (queryStr, bindValues) => {
-            const stmt = adapter.mainDb.prepare(queryStr)
-            stmt.execute(bindValues)
+    const __processedMutationIds = new Set<string>()
 
-            if (isInTxn === true) {
-              txnExecuteStmnts.push([queryStr, bindValues])
-            } else {
-              void adapter.coordinator.execute(queryStr, bindValues, undefined)
-            }
-          },
-          mutate: (...list) => {
-            for (const mutationEventDecoded of list) {
-              const mutationDef =
-                schema.mutations.get(mutationEventDecoded.mutation) ??
-                shouldNeverHappen(`Unknown mutation type: ${mutationEventDecoded.mutation}`)
-
-              const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
-              // const { bindValues, statementSql } = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
-
-              for (const { statementSql, bindValues } of execArgsArr) {
-                adapter.mainDb.execute(statementSql, bindValues)
-              }
+    // TODO consider moving booting into the storage backend
+    if (boot !== undefined) {
+      let isInTxn = false
+      let txnExecuteStmnts: [string, PreparedBindValues | undefined][] = []
 
-              const mutationEventEncoded = Schema.encodeUnknownSync(mutationEventSchema)(mutationEventDecoded)
-              void adapter.coordinator.mutate(mutationEventEncoded, { span, persisted: true })
-            }
-          },
-          select: (queryStr, bindValues) => {
-            const stmt = adapter.mainDb.prepare(queryStr)
-            return stmt.select(bindValues)
-          },
-          txn: (callback) => {
-            try {
-              isInTxn = true
-              adapter.mainDb.execute('BEGIN', undefined)
-
-              callback()
-
-              adapter.mainDb.execute('COMMIT', undefined)
-
-              // adapter.coordinator.execute('BEGIN', undefined, undefined)
-              for (const [queryStr, bindValues] of txnExecuteStmnts) {
-                adapter.coordinator.execute(queryStr, bindValues, undefined)
-              }
-              // adapter.coordinator.execute('COMMIT', undefined, undefined)
-            } catch (e: any) {
-              adapter.mainDb.execute('ROLLBACK', undefined)
-              throw e
-            } finally {
-              isInTxn = false
-              txnExecuteStmnts = []
+      const bootDbImpl: BootDb = {
+        _tag: 'BootDb',
+        execute: (queryStr, bindValues) => {
+          const stmt = adapter.mainDb.prepare(queryStr)
+          stmt.execute(bindValues)
+
+          if (isInTxn === true) {
+            txnExecuteStmnts.push([queryStr, bindValues])
+          } else {
+            adapter.coordinator.execute(queryStr, bindValues).pipe(runEffectFork)
+          }
+        },
+        mutate: (...list) => {
+          for (const mutationEventDecoded of list) {
+            const mutationDef =
+              schema.mutations.get(mutationEventDecoded.mutation) ??
+              shouldNeverHappen(`Unknown mutation type: ${mutationEventDecoded.mutation}`)
+
+            __processedMutationIds.add(mutationEventDecoded.id)
+
+            const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+            // const { bindValues, statementSql } = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+
+            for (const { statementSql, bindValues } of execArgsArr) {
+              adapter.mainDb.execute(statementSql, bindValues)
             }
-          },
-        }
 
-        const booting = boot(bootDbImpl, span)
-        // NOTE only awaiting if it's actually a promise to avoid unnecessary async/await
-        if (isPromise(booting)) {
-          await booting
-        }
-      }
+            const mutationEventEncoded = Schema.encodeUnknownSync(mutationEventSchema)(mutationEventDecoded)
 
-      // TODO: we can't apply the schema at this point, we've already loaded persisted data!
-      // Think about what to do about this case.
-      // await applySchema(db, schema)
-      return Store.createStore<TGraphQLContext, TSchema>(
-        {
-          adapter: adapter,
-          schema,
-          graphQLOptions,
-          otelTracer,
-          otelRootSpanContext,
-          dbGraph,
-          mutationEventSchema,
-          disableDevtools,
+            adapter.coordinator
+              .mutate(mutationEventEncoded as MutationEvent.AnyEncoded, { persisted: true })
+              .pipe(runEffectFork)
+          }
+        },
+        select: (queryStr, bindValues) => {
+          const stmt = adapter.mainDb.prepare(queryStr)
+          return stmt.select(bindValues)
         },
-        span,
+        txn: (callback) => {
+          try {
+            isInTxn = true
+            adapter.mainDb.execute('BEGIN', undefined)
+
+            callback()
+
+            adapter.mainDb.execute('COMMIT', undefined)
+
+            // adapter.coordinator.execute('BEGIN', undefined, undefined)
+            for (const [queryStr, bindValues] of txnExecuteStmnts) {
+              adapter.coordinator.execute(queryStr, bindValues).pipe(runEffectFork)
+            }
+            // adapter.coordinator.execute('COMMIT', undefined, undefined)
+          } catch (e: any) {
+            adapter.mainDb.execute('ROLLBACK', undefined)
+            throw e
+          } finally {
+            isInTxn = false
+            txnExecuteStmnts = []
+          }
+        },
+      }
+
+      yield* Effect.tryAll(() => boot(bootDbImpl, span)).pipe(
+        UnexpectedError.mapToUnexpectedError,
+        Effect.withSpan('createStore:boot'),
       )
-    } finally {
-      span.end()
     }
-  })
+
+    return Store.createStore<TGraphQLContext, TSchema>(
+      {
+        adapter,
+        schema,
+        graphQLOptions,
+        otelOptions: { tracer: otelTracer, rootSpanContext: otelRootSpanContext },
+        reactivityGraph,
+        disableDevtools,
+        __processedMutationIds,
+        fiberSet,
+      },
+      span,
+    )
+  }).pipe(
+    Effect.withSpan('createStore', {
+      parent: otelOptions?.rootSpanContext
+        ? OtelTracer.makeExternalSpan(otel.trace.getSpanContext(otelOptions.rootSpanContext)!)
+        : undefined,
+    }),
+    Effect.provide(TracingLive),
+  )
 }
+// #endregion createStore
 
+// TODO consider replacing with Effect's RC data structures
 class ReferenceCountedSet<T> {
   private map: Map<T, number>
 
@@ -811,3 +1042,21 @@ class ReferenceCountedSet<T> {
     }
   }
 }
+
+const runEffectFork = <A, E>(effect: Effect.Effect<A, E, never>) =>
+  effect.pipe(
+    Effect.tapCauseLogPretty,
+    Effect.annotateLogs({ thread: 'window' }),
+    Effect.provide(Logger.pretty),
+    Logger.withMinimumLogLevel(LogLevel.Debug),
+    Effect.runFork,
+  )
+
+const runEffectPromise = <A, E>(effect: Effect.Effect<A, E, never>) =>
+  effect.pipe(
+    Effect.tapCauseLogPretty,
+    Effect.annotateLogs({ thread: 'window' }),
+    Effect.provide(Logger.pretty),
+    Logger.withMinimumLogLevel(LogLevel.Debug),
+    Effect.runPromise,
+  )