@livestore/livestore 0.0.0

package/src/bounded-collections.ts

@@ -0,0 +1,112 @@
+/**
+ * Creates a map that has a fixed number of entries.
+ * Once hitting the bound, earliest insertions are removed
+ */
+export default class BoundMap<K, V> {
+  #map = new Map<K, V>()
+  #sizeLimit: number
+
+  constructor(sizeLimit: number) {
+    this.#sizeLimit = sizeLimit
+  }
+
+  onEvict: ((key: K) => void) | undefined
+
+  set = (key: K, value: V) => {
+    this.#map.set(key, value)
+    // console.log(this.#map.size, this.#sizeLimit);
+    if (this.#map.size > this.#sizeLimit) {
+      const firstKey = this.#map.keys().next().value
+      this.#map.delete(firstKey)
+      if (this.onEvict) {
+        this.onEvict(firstKey)
+      }
+    }
+  }
+
+  get = (key: K): V | undefined => {
+    return this.#map.get(key)
+  }
+
+  delete = (key: K) => {
+    this.#map.delete(key)
+  }
+
+  keys = () => {
+    return this.#map.keys()
+  }
+}
+
+export class BoundSet<V> {
+  #map: BoundMap<V, V>
+
+  constructor(sizeLimit: number) {
+    this.#map = new BoundMap(sizeLimit)
+    this.#map.onEvict = this.#onEvict
+  }
+
+  #onEvict = (v: V) => {
+    if (this.onEvict) {
+      this.onEvict(v)
+    }
+  }
+
+  onEvict: ((key: V) => void) | undefined
+
+  add = (v: V) => {
+    this.#map.set(v, v)
+  };
+
+  [Symbol.iterator] = () => {
+    return this.#map.keys()
+  }
+}
+
+export class BoundArray<V> {
+  #array: V[] = []
+  #sizeLimit: number
+
+  constructor(sizeLimit: number) {
+    this.#sizeLimit = sizeLimit
+  }
+
+  onEvict: ((key: V) => void) | undefined
+
+  push = (v: V) => {
+    this.#array.push(v)
+    if (this.#array.length > this.#sizeLimit) {
+      const first = this.#array.shift()
+      if (first && this.onEvict) {
+        this.onEvict(first)
+      }
+    }
+  }
+
+  get = (index: number): V | undefined => {
+    return this.#array[index]
+  }
+
+  delete = (index: number) => {
+    this.#array.splice(index, 1)
+  }
+
+  get length() {
+    return this.#array.length
+  }
+
+  [Symbol.iterator] = () => {
+    return this.#array[Symbol.iterator]()
+  }
+
+  map = <T>(fn: (v: V) => T): T[] => {
+    return this.#array.map(fn)
+  }
+
+  clear = () => {
+    this.#array = []
+  }
+
+  sort = (fn?: (a: V, b: V) => number) => {
+    return this.#array.sort(fn)
+  }
+}

package/src/componentKey.ts

@@ -0,0 +1,9 @@
+type SingletonKey = { _tag: 'singleton'; componentName: string; id: 'singleton' }
+type EphemeralKey = { _tag: 'ephemeral'; componentName: string; id: string }
+type CustomKey = { _tag: 'custom'; componentName: string; id: string }
+
+export type ComponentKey = SingletonKey | EphemeralKey | CustomKey
+
+export const labelForKey = (key: ComponentKey): string => `${key.componentName}/${key.id}`
+
+export const tableNameForComponentKey = (componentKey: ComponentKey) => `components__${componentKey.componentName}`

package/src/effect/LiveStore.ts

@@ -0,0 +1,123 @@
+import type { Scope } from '@livestore/utils/effect'
+import { Context, Deferred, Duration, Effect, Layer, Otel, pipe } from '@livestore/utils/effect'
+import type * as otel from '@opentelemetry/api'
+import type { GraphQLSchema } from 'graphql'
+import { mapValues } from 'lodash-es'
+
+import type { Backend, BackendOptions } from '../backends/index.js'
+import type { InMemoryDatabase } from '../inMemoryDatabase.js'
+import type { Schema } from '../schema.js'
+import type { BaseGraphQLContext, GraphQLOptions, LiveStoreQuery, Store } from '../store.js'
+import { createStore } from '../store.js'
+
+export type LiveStoreContext = {
+  store: Store<any>
+  globalQueries: LiveStoreQueryTypes
+}
+
+export type QueryDefinition = (store: Store<any>) => LiveStoreQuery
+export type GlobalQueryDefs = { [key: string]: QueryDefinition }
+
+export type LiveStoreCreateStoreOptions<GraphQLContext extends BaseGraphQLContext> = {
+  schema: Schema
+  globalQueryDefs: GlobalQueryDefs
+  backendOptions: BackendOptions
+  graphQLOptions?: GraphQLOptions<GraphQLContext>
+  otelTracer?: otel.Tracer
+  otelRootSpanContext?: otel.Context
+  boot?: (backend: Backend, parentSpan: otel.Span) => Promise<void>
+}
+
+export const LiveStoreContext = Context.Tag<LiveStoreContext>('@livestore/livestore/LiveStoreContext')
+
+export type DeferredStoreContext = Deferred.Deferred<never, LiveStoreContext>
+export const DeferredStoreContext = Context.Tag<DeferredStoreContext>(
+  Symbol.for('@livestore/livestore/DeferredStoreContext'),
+)
+
+// export const DeferredStoreContext = Effect.cached(Effect.flatMap(StoreContext, (_) => Effect.succeed(_)))
+
+export type LiveStoreContextProps<GraphQLContext extends BaseGraphQLContext> = {
+  schema: Schema
+  globalQueryDefs?: Effect.Effect<Otel.Tracer | Otel.Span, never, GlobalQueryDefs>
+  backendOptions: Effect.Effect<Otel.Tracer | Otel.Span, never, BackendOptions>
+  graphQLOptions?: {
+    schema: Effect.Effect<Otel.Tracer, never, GraphQLSchema>
+    makeContext: (db: InMemoryDatabase) => GraphQLContext
+  }
+  boot?: (backend: Backend) => Effect.Effect<Otel.Tracer, never, void>
+}
+
+export const LiveStoreContextLayer = <GraphQLContext extends BaseGraphQLContext>(
+  props: LiveStoreContextProps<GraphQLContext>,
+): Layer.Layer<Otel.Tracer, never, LiveStoreContext> =>
+  Layer.provide(
+    LiveStoreContextDeferred,
+    pipe(Layer.scoped(LiveStoreContext, makeLiveStoreContext(props)), Otel.withSpanLayer('LiveStoreContext')),
+  )
+
+export const LiveStoreContextDeferred = Layer.effect(DeferredStoreContext, Deferred.make<never, LiveStoreContext>())
+
+export const makeLiveStoreContext = <GraphQLContext extends BaseGraphQLContext>({
+  globalQueryDefs,
+  schema,
+  backendOptions: backendOptions_,
+  graphQLOptions: graphQLOptions_,
+  boot: boot_,
+}: LiveStoreContextProps<GraphQLContext>): Effect.Effect<
+  Otel.Tracer | Otel.Span | DeferredStoreContext | Scope.Scope,
+  never,
+  LiveStoreContext
+> =>
+  pipe(
+    Effect.gen(function* ($) {
+      const ctx = yield* $(Effect.context<Otel.Tracer>())
+      const otelRootSpanContext = yield* $(Otel.activeContext)
+      const { tracer: otelTracer } = yield* $(Otel.Tracer)
+
+      const graphQLOptions = yield* $(
+        graphQLOptions_
+          ? Effect.all({ schema: graphQLOptions_.schema, makeContext: Effect.succeed(graphQLOptions_.makeContext) })
+          : Effect.succeed(undefined),
+      )
+
+      const backendOptions = yield* $(backendOptions_ ?? Effect.succeed(undefined))
+
+      const boot = boot_
+        ? (db: Backend) => pipe(boot_(db), Effect.provideContext(ctx), Effect.tapCauseLogPretty, Effect.runPromise)
+        : undefined
+
+      const store = yield* $(
+        Effect.tryPromise(() =>
+          createStore({
+            schema,
+            backendOptions,
+            graphQLOptions,
+            otelTracer,
+            otelRootSpanContext,
+            boot,
+          }),
+        ),
+        Effect.acquireRelease((store) => Effect.sync(() => store.destroy())),
+      )
+
+      window.__debugLiveStore = store
+
+      const globalQueries = yield* $(
+        globalQueryDefs ?? Effect.succeed({}),
+        Effect.map((defs) => mapValues(defs, (queryDef) => queryDef(store))),
+        Otel.withSpan('LiveStore:makeGlobalQueries', {}, store.otel.queriesSpanContext),
+      )
+
+      // NOTE give main thread a chance to render
+      yield* $(Effect.yieldNow())
+
+      return { store, globalQueries }
+    }),
+    Effect.tap((storeCtx) => Effect.flatMap(DeferredStoreContext, (def) => Deferred.succeed(def, storeCtx))),
+    Effect.timeoutFail({
+      onTimeout: () => new Error('Timed out while creating LiveStore store after 10sec'),
+      duration: Duration.seconds(10),
+    }),
+    Effect.orDie,
+  )

package/src/effect/index.ts

@@ -0,0 +1,7 @@
+export {
+  LiveStoreContextLayer,
+  LiveStoreContext,
+  LiveStoreContextDeferred,
+  DeferredStoreContext,
+  type LiveStoreContextProps,
+} from './LiveStore.js'

package/src/events.ts

@@ -0,0 +1,8 @@
+// Todo: typesafe events
+export type LiveStoreEvent = {
+  type: string
+  id: string
+  args?: any
+}
+
+export const EVENTS_TABLE_NAME = '__events'

package/src/inMemoryDatabase.ts

@@ -0,0 +1,347 @@
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+
+import { shouldNeverHappen } from '@livestore/utils'
+import { identity } from '@livestore/utils/effect'
+import type * as otel from '@opentelemetry/api'
+import type * as SqliteWasm from 'sqlite-esm'
+import initSqlJs from 'sqlite-esm'
+
+import BoundMap, { BoundArray } from './bounded-collections.js'
+import type { LiveStoreEvent } from './events.js'
+// import { EVENTS_TABLE_NAME } from './events.js'
+import { sql } from './index.js'
+import { getDurationMsFromSpan, getStartTimeHighResFromSpan } from './otel.js'
+import QueryCache from './QueryCache.js'
+import type { ActionDefinition } from './schema.js'
+import type { Bindable, ParamsObject } from './util.js'
+import { prepareBindValues } from './util.js'
+
+export enum IndexType {
+  Basic = 'Basic',
+  FullText = 'FullText',
+}
+
+export interface Index {
+  indexType: IndexType
+  name: string
+  columns: string[]
+}
+
+declare type DatabaseWithCAPI = SqliteWasm.Database & { capi: SqliteWasm.CAPI }
+
+export interface DebugInfo {
+  slowQueries: BoundArray<SlowQueryInfo>
+  queryFrameDuration: number
+  queryFrameCount: number
+  events: BoundArray<[queryStr: string, bindValues: Bindable | undefined]>
+}
+
+export type SlowQueryInfo = [
+  queryStr: string,
+  bindValues: Bindable | undefined,
+  durationMs: number,
+  rowsCount: number | undefined,
+  queriedTables: string[],
+  startTimePerfNow: DOMHighResTimeStamp,
+]
+
+export const emptyDebugInfo = (): DebugInfo => ({
+  slowQueries: new BoundArray(200),
+  queryFrameDuration: 0,
+  queryFrameCount: 0,
+  events: new BoundArray(1000),
+})
+
+export class InMemoryDatabase {
+  // TODO: how many unique active statements are expected?
+  private cachedStmts = new BoundMap<string, SqliteWasm.PreparedStatement>(200)
+  private tablesUsedCache = new BoundMap<string, string[]>(200)
+  private resultCache = new QueryCache()
+  public debugInfo: DebugInfo = emptyDebugInfo()
+
+  constructor(
+    private db: DatabaseWithCAPI,
+    private otelTracer: otel.Tracer,
+    private otelRootSpanContext: otel.Context,
+    public SQL: SqliteWasm.Sqlite3Static,
+  ) {}
+
+  static async load(
+    data: Uint8Array | undefined,
+    otelTracer: otel.Tracer,
+    otelRootSpanContext: otel.Context,
+  ): Promise<InMemoryDatabase> {
+    const sqlite3 = await initSqlJs({
+      // Required to load the wasm binary asynchronously. Of course, you can host it wherever you want
+      // You can omit locateFile completely when running in node
+      // locateFile: () => `/sql-wasm.wasm`,
+      print: (message) => console.log(`[sql-client] ${message}`),
+      printErr: (message) => console.error(`[sql-client] ${message}`),
+    })
+
+    const db = new sqlite3.oo1.DB({ filename: ':memory:', flags: 'c' }) as DatabaseWithCAPI
+    db.capi = sqlite3.capi
+
+    if (data !== undefined) {
+      // Based on https://sqlite.org/forum/forumpost/2119230da8ac5357a13b731f462dc76e08621a4a29724f7906d5f35bb8508465
+      // TODO find cleaner way to do this once possible in sqlite3-wasm
+      const bytes = data
+      const p = sqlite3.wasm.allocFromTypedArray(bytes)
+      const _rc = sqlite3.capi.sqlite3_deserialize(
+        db.pointer,
+        'main',
+        p,
+        bytes.length,
+        bytes.length,
+        sqlite3.capi.SQLITE_DESERIALIZE_FREEONCLOSE && sqlite3.capi.SQLITE_DESERIALIZE_RESIZEABLE,
+      )
+    }
+
+    return new InMemoryDatabase(db, otelTracer, otelRootSpanContext, sqlite3)
+  }
+
+  txn<TRes>(callback: () => TRes): TRes {
+    this.execute(sql`begin transaction;`)
+    let errored = false
+    let result: TRes
+
+    try {
+      result = callback()
+    } catch (e) {
+      errored = true
+      this.execute(sql`rollback;`)
+      throw e
+    }
+
+    if (!errored) {
+      this.execute(sql`commit;`)
+    }
+
+    return result
+  }
+
+  getTablesUsed(query: string) {
+    const cached = this.tablesUsedCache.get(query)
+    if (cached) {
+      return cached
+    }
+    const stmt = this.db.prepare(
+      `SELECT tbl_name FROM tables_used(?) AS u JOIN sqlite_master ON sqlite_master.name = u.name WHERE u.schema = 'main';`,
+    )
+    const tablesUsed = []
+    try {
+      stmt.bind([query])
+      while (stmt.step()) {
+        tablesUsed.push(stmt.get(0))
+      }
+    } finally {
+      stmt.finalize()
+    }
+    this.tablesUsedCache.set(query, tablesUsed as string[])
+    return tablesUsed as string[]
+  }
+
+  /**
+   * NOTE `execute` is untraced since it's usually called from `applyEvent` which is traced
+   */
+  execute(
+    query: string,
+    bindValues?: ParamsObject,
+    writeTables?: string[],
+    options?: { hasNoEffects?: boolean },
+  ): void {
+    try {
+      let stmt = this.cachedStmts.get(query)
+      if (stmt === undefined) {
+        stmt = this.db.prepare(query)
+        this.cachedStmts.set(query, stmt)
+      }
+
+      if (bindValues !== undefined && Object.keys(bindValues).length > 0) {
+        stmt.bind(prepareBindValues(bindValues, query))
+      }
+
+      try {
+        stmt.step()
+      } finally {
+        stmt.reset() // Reset is needed for next execution
+      }
+    } catch (error) {
+      shouldNeverHappen(
+        `Error executing query: ${error} \n ${JSON.stringify({
+          query,
+          bindValues,
+        })}`,
+      )
+    }
+
+    if (options?.hasNoEffects !== true && !this.resultCache.ignoreQuery(query)) {
+      // TODO use write tables instead
+      // check what queries actually end up here.
+      this.resultCache.invalidate(writeTables ?? this.getTablesUsed(query))
+    }
+
+    if (options?.hasNoEffects === true) {
+      return
+    }
+  }
+
+  select<T = any>(
+    query: string,
+    options?: {
+      queriedTables?: string[]
+      bindValues?: Bindable
+      skipCache?: boolean
+      parentSpanContext?: otel.Context
+    },
+  ): ReadonlyArray<T> {
+    const { queriedTables, bindValues, skipCache = false, parentSpanContext } = options ?? {}
+    return this.otelTracer.startActiveSpan(
+      'sql-in-memory-select',
+      {},
+      parentSpanContext ?? this.otelRootSpanContext,
+      (span) => {
+        try {
+          span.setAttribute('sql.query', query)
+
+          const key = this.resultCache.getKey(query, bindValues)
+          const cachedResult = this.resultCache.get(key)
+          if (skipCache === false && cachedResult !== undefined) {
+            span.setAttribute('sql.rowsCount', cachedResult.length)
+            span.setAttribute('sql.cached', true)
+            span.end()
+            return cachedResult
+          }
+
+          let stmt = this.cachedStmts.get(query)
+          if (stmt === undefined) {
+            stmt = this.db.prepare(query)
+            this.cachedStmts.set(query, stmt)
+          }
+          if (bindValues) {
+            stmt.bind(bindValues ?? {})
+          }
+
+          const result: T[] = []
+          try {
+            const columns = stmt.getColumnNames()
+            while (stmt.step()) {
+              const obj: { [key: string]: any } = {}
+              for (const [i, c] of columns.entries()) {
+                obj[c] = stmt.get(i)
+              }
+              result.push(obj as unknown as T)
+            }
+          } finally {
+            // we're caching statements in this iteration. do not free.
+            // stmt.free();
+            // reset the cached statement so we can use it again in the future
+            stmt.reset()
+          }
+
+          span.setAttribute('sql.rowsCount', result.length)
+          span.setAttribute('sql.cached', false)
+
+          const queriedTables_ = queriedTables ?? this.getTablesUsed(query)
+          this.resultCache.set(queriedTables_, key, result)
+
+          span.end()
+
+          const durationMs = getDurationMsFromSpan(span)
+
+          this.debugInfo.queryFrameDuration += durationMs
+          this.debugInfo.queryFrameCount++
+
+          // TODO also enable in non-dev mode
+          if (durationMs > 5 && import.meta.env.DEV) {
+            this.debugInfo.slowQueries.push([
+              query,
+              bindValues,
+              durationMs,
+              result.length,
+              queriedTables_,
+              getStartTimeHighResFromSpan(span),
+            ])
+          }
+
+          return result
+        } catch (e) {
+          span.end()
+          console.error(query)
+          console.error(bindValues)
+          shouldNeverHappen(`Error executing select query: ${e} \n ${JSON.stringify({ query, bindValues })}`)
+        }
+      },
+    )
+  }
+
+  // TODO move `applyEvent` logic to Store and only call `execute` here
+  applyEvent(
+    event: LiveStoreEvent,
+    eventDefinition: ActionDefinition,
+    parentSpanContext: otel.Context,
+  ): { durationMs: number } {
+    return this.otelTracer.startActiveSpan('livestore.in-memory-db:applyEvent', {}, parentSpanContext, (span) => {
+      // TODO: in the future, we'll do more CRDT-style stuff here to decide whether to run effects of the event
+
+      // NOTE: These two updates should happen transactionally;
+      // we don't create a transaction here because that's handled in the caller.
+      // The reason for this is that sometimes we want to apply multiple events in a larger transaction.
+
+      // Insert into the events table
+      // this.execute(sql`insert into ${EVENTS_TABLE_NAME} (id, type, args) values ($id, $type, $args)`, {
+      //   id: event.id,
+      //   type: event.type,
+      //   args: JSON.stringify(event.args ?? {}),
+      // })
+
+      const statement =
+        typeof eventDefinition.statement === 'function'
+          ? eventDefinition.statement(event.args)
+          : eventDefinition.statement
+
+      const prepareBindValues = eventDefinition.prepareBindValues ?? identity
+
+      const bindValues =
+        typeof eventDefinition.statement === 'function' && statement.argsAlreadyBound
+          ? {}
+          : prepareBindValues(event.args)
+
+      if (import.meta.env.DEV) {
+        this.debugInfo.events.push([statement.sql, bindValues])
+      }
+
+      // Run the effects of the event
+      this.execute(statement.sql, bindValues, statement.writeTables)
+
+      span.end()
+
+      const durationMs = getDurationMsFromSpan(span)
+
+      this.debugInfo.queryFrameDuration += durationMs
+      this.debugInfo.queryFrameCount++
+
+      if (durationMs > 5 && import.meta.env.DEV) {
+        this.debugInfo.slowQueries.push([
+          statement.sql,
+          bindValues,
+          durationMs,
+          undefined,
+          [],
+          getStartTimeHighResFromSpan(span),
+        ])
+      }
+
+      return { durationMs }
+    })
+  }
+
+  export() {
+    // Clear statement cache because exporting frees statements
+    for (const key of this.cachedStmts.keys()) {
+      this.cachedStmts.delete(key)
+    }
+
+    return this.db.capi.sqlite3_js_db_export(this.db.pointer)
+  }
+}

package/src/index.ts

@@ -0,0 +1,47 @@
+export { Store, createStore, RESET_DB_LOCAL_STORAGE_KEY } from './store.js'
+export type { LiveStoreQuery, BaseGraphQLContext, QueryResult, QueryDebugInfo, RefreshReason } from './store.js'
+
+export type { QueryDefinition, LiveStoreCreateStoreOptions, LiveStoreContext } from './effect/LiveStore.js'
+
+export {
+  defineComponentStateSchema,
+  EVENT_CURSOR_TABLE,
+  defineSchema,
+  defineAction,
+  defineActions,
+  defineTables,
+  defineMaterializedViews,
+} from './schema.js'
+export { InMemoryDatabase, type DebugInfo, emptyDebugInfo } from './inMemoryDatabase.js'
+export { createBackend, IndexType } from './backends/index.js'
+export type { BackendOptions, Backend, BackendType } from './backends/index.js'
+export { isBackendType } from './backends/index.js'
+export type { SelectResponse } from './backends/index.js'
+export { WebWorkerBackend } from './backends/web.js'
+export type {
+  GetAtom,
+  AtomDebugInfo,
+  RefreshDebugInfo,
+  RefreshReasonWithGenericReasons,
+  SerializedAtom,
+  SerializedEffect,
+} from './reactive.js'
+export type { LiveStoreJSQuery } from './reactiveQueries/js.js'
+export type { LiveStoreSQLQuery } from './reactiveQueries/sql.js'
+export type { LiveStoreGraphQLQuery } from './reactiveQueries/graphql.js'
+
+export { labelForKey } from './componentKey.js'
+export type { ComponentKey } from './componentKey.js'
+export type {
+  Schema,
+  TableDefinition,
+  GetActionArgs,
+  GetApplyEventArgs,
+  ColumnDefinition,
+  Index,
+  ActionDefinition,
+  ActionDefinitions,
+} from './schema.js'
+
+export { sql, type Bindable } from './util.js'
+export { isEqual } from 'lodash-es'

package/src/otel.ts

@@ -0,0 +1,20 @@
+import type * as otel from '@opentelemetry/api'
+
+// TODO improve - see https://www.notion.so/schickling/Better-solution-for-globalThis-inProgressSpans-503cd7a5f4fc4fb8bdec2e60bde1be1f
+export const TODO_REMOVE_trackLongRunningSpan = (span: otel.Span): void => {
+  // @ts-expect-error TODO get rid of this coupling
+  if (window.inProgressSpans !== undefined && window.inProgressSpans instanceof Set) {
+    // @ts-expect-error TODO get rid of this coupling
+    window.inProgressSpans.add(span)
+  } else {
+    debugger
+  }
+}
+
+export const getDurationMsFromSpan = (span: otel.Span): number => {
+  const durationHr: [seconds: number, nanos: number] = (span as any)._duration
+  return durationHr[0] * 1000 + durationHr[1] / 1_000_000
+}
+
+export const getStartTimeHighResFromSpan = (span: otel.Span): DOMHighResTimeStamp =>
+  (span as any)._performanceStartTime as DOMHighResTimeStamp

package/src/react/LiveStoreContext.ts

@@ -0,0 +1,23 @@
+import React, { useContext } from 'react'
+
+import type { LiveStoreContext as LiveStoreContext_ } from '../effect/LiveStore.js'
+import type { LiveStoreQuery } from '../store.js'
+
+declare global {
+  // NOTE Can be extended
+  interface LiveStoreQueryTypes {
+    [key: string]: LiveStoreQuery
+  }
+}
+
+export const LiveStoreContext = React.createContext<LiveStoreContext_ | undefined>(undefined)
+
+export const useStore = (): LiveStoreContext_ => {
+  const storeContext = useContext(LiveStoreContext)
+
+  if (storeContext === undefined) {
+    throw new Error(`useStore can only be used inside StoreContext.Provider`)
+  }
+
+  return storeContext
+}