@livestore/livestore 0.0.0-snapshot-909cdd1ac2fd591945c2be2b0f53e14d87f3c9d4

package/src/QueryCache.ts

@@ -0,0 +1,81 @@
+import type { Bindable } from '@livestore/common'
+import { BoundMap, BoundSet } from '@livestore/common'
+
+type Opaque<BaseType, BrandType = unknown> = BaseType & {
+  readonly [Symbols.base]: BaseType
+  readonly [Symbols.brand]: BrandType
+}
+
+namespace Symbols {
+  export declare const base: unique symbol
+  export declare const brand: unique symbol
+}
+
+export type CacheKey = Opaque<string, string>
+type TableName = string
+
+const ignore = ['begin', 'rollback', 'commit', 'savepoint', 'release']
+
+// TODO: profile to see how big we need this cache to be.
+const cacheSize = 200
+export default class QueryCache {
+  #entries = new BoundMap<CacheKey, any>(cacheSize)
+  #dependencies = new Map<TableName, BoundSet<CacheKey>>()
+
+  getKey = (sql: string, bindValues?: Bindable): CacheKey => {
+    if (bindValues == null) {
+      return sql as CacheKey
+    }
+
+    if (Array.isArray(bindValues)) {
+      return (sql + '\n' + bindValues.join('\n')) as CacheKey
+    }
+
+    return (sql + '\n' + Object.values(bindValues).join('\n')) as CacheKey
+  }
+
+  get = (key: CacheKey) => {
+    return this.#entries.get(key)
+  }
+
+  set = (queriedTables: Iterable<string>, key: CacheKey, results: any) => {
+    this.#entries.set(key, results)
+    for (const table of queriedTables) {
+      let keys = this.#dependencies.get(table)
+      if (keys == null) {
+        keys = new BoundSet(cacheSize)
+        keys.onEvict = this.#dependencyTrackerEvicted
+        this.#dependencies.set(table, keys)
+      }
+      keys.add(key)
+    }
+  }
+
+  #dependencyTrackerEvicted = (key: CacheKey) => {
+    this.#entries.delete(key)
+  }
+
+  ignoreQuery = (query: string) => {
+    return ignore.some((prefix) => query.startsWith(prefix))
+  }
+
+  // The next simplest step is to create a specific implementation for invalidating
+  // the expensive track list queries only when constraints data in a write overlaps with read constraints.
+  //
+  // As well as either:
+  // a. removeing the big view (since we'll have our cache)
+  // b. incrementally updating the view on insert by the EventImporter
+  //
+  // We'll not try to tackle any generalized approach until we have a proof of concept working.
+  invalidate = (queriedTables: Iterable<string>) => {
+    for (const table of queriedTables) {
+      const keys = this.#dependencies.get(table)
+      if (keys == null) {
+        continue
+      }
+      for (const k of keys) {
+        this.#entries.delete(k)
+      }
+    }
+  }
+}

package/src/SynchronousDatabaseWrapper.ts

@@ -0,0 +1,256 @@
+/* eslint-disable prefer-arrow/prefer-arrow-functions */
+
+import type {
+  DebugInfo,
+  MutableDebugInfo,
+  PreparedBindValues,
+  PreparedStatement,
+  SynchronousDatabase,
+} from '@livestore/common'
+import { BoundArray, BoundMap, sql } from '@livestore/common'
+import type * as otel from '@opentelemetry/api'
+
+import QueryCache from './QueryCache.js'
+import { getDurationMsFromSpan, getStartTimeHighResFromSpan } from './utils/otel.js'
+
+export const emptyDebugInfo = (): DebugInfo => ({
+  slowQueries: new BoundArray(200),
+  queryFrameDuration: 0,
+  queryFrameCount: 0,
+  events: new BoundArray(1000),
+})
+
+export class SynchronousDatabaseWrapper {
+  // TODO: how many unique active statements are expected?
+  private cachedStmts = new BoundMap<string, PreparedStatement>(200)
+  private tablesUsedCache = new BoundMap<string, Set<string>>(200)
+  private resultCache = new QueryCache()
+  private db: SynchronousDatabase
+  private otelTracer: otel.Tracer
+  private otelRootSpanContext: otel.Context
+  private tablesUsedStmt
+  public debugInfo: MutableDebugInfo = emptyDebugInfo()
+
+  constructor({
+    db,
+    otel,
+  }: {
+    db: SynchronousDatabase
+    otel: {
+      tracer: otel.Tracer
+      rootSpanContext: otel.Context
+    }
+  }) {
+    this.db = db
+    this.otelTracer = otel.tracer
+    this.otelRootSpanContext = otel.rootSpanContext
+
+    this.tablesUsedStmt = db.prepare(
+      `SELECT tbl_name FROM tables_used(?) AS u JOIN sqlite_master ON sqlite_master.name = u.name WHERE u.schema = 'main';`,
+    )
+
+    this.cachedStmts.onEvict = (_queryStr, stmt) => stmt.finalize()
+
+    configureSQLite(this)
+  }
+
+  txn<TRes>(callback: () => TRes): TRes {
+    this.execute(sql`begin transaction;`)
+
+    let errored = false
+    let result: TRes
+
+    try {
+      result = callback()
+    } catch (e) {
+      errored = true
+      this.execute(sql`rollback;`)
+      throw e
+    }
+
+    if (!errored) {
+      this.execute(sql`commit;`)
+    }
+
+    return result
+  }
+
+  getTablesUsed(query: string) {
+    // It seems that SQLite doesn't properly handle `DELETE FROM SOME_TABLE` queries without a WHERE clause
+    // So we need to handle these queries separately
+    const tableNameFromPlainDeleteQuery = tryGetTableNameFromPlainDeleteQuery(query)
+    if (tableNameFromPlainDeleteQuery !== undefined) {
+      return new Set<string>([tableNameFromPlainDeleteQuery])
+    }
+
+    const cached = this.tablesUsedCache.get(query)
+    if (cached) {
+      return cached
+    }
+    const stmt = this.tablesUsedStmt
+    const tablesUsed = new Set<string>()
+    try {
+      const results = stmt.select<{ tbl_name: string }>([query] as unknown as PreparedBindValues)
+
+      for (const row of results) {
+        tablesUsed.add(row.tbl_name)
+      }
+    } catch (e) {
+      console.error('Error getting tables used', e, 'for query', query)
+      return new Set<string>()
+    }
+    this.tablesUsedCache.set(query, tablesUsed)
+    return tablesUsed
+  }
+
+  execute(
+    queryStr: string,
+    bindValues?: PreparedBindValues,
+    writeTables?: ReadonlySet<string>,
+    options?: { hasNoEffects?: boolean; otelContext?: otel.Context },
+  ): { durationMs: number } {
+    // console.debug('in-memory-db:execute', query, bindValues)
+
+    return this.otelTracer.startActiveSpan(
+      'livestore.in-memory-db:execute',
+      // TODO truncate query string
+      { attributes: { 'sql.query': queryStr } },
+      options?.otelContext ?? this.otelRootSpanContext,
+      (span) => {
+        let stmt = this.cachedStmts.get(queryStr)
+        if (stmt === undefined) {
+          stmt = this.db.prepare(queryStr)
+          this.cachedStmts.set(queryStr, stmt)
+        }
+
+        stmt.execute(bindValues)
+
+        if (options?.hasNoEffects !== true && !this.resultCache.ignoreQuery(queryStr)) {
+          // TODO use write tables instead
+          // check what queries actually end up here.
+          this.resultCache.invalidate(writeTables ?? this.getTablesUsed(queryStr))
+        }
+
+        span.end()
+
+        const durationMs = getDurationMsFromSpan(span)
+
+        this.debugInfo.queryFrameDuration += durationMs
+        this.debugInfo.queryFrameCount++
+
+        if (durationMs > 5 && import.meta.env.DEV) {
+          this.debugInfo.slowQueries.push({
+            queryStr,
+            bindValues,
+            durationMs,
+            rowsCount: undefined,
+            queriedTables: new Set(),
+            startTimePerfNow: getStartTimeHighResFromSpan(span),
+          })
+        }
+
+        return { durationMs }
+      },
+    )
+  }
+
+  select<T = any>(
+    queryStr: string,
+    options?: {
+      queriedTables?: ReadonlySet<string>
+      bindValues?: PreparedBindValues
+      skipCache?: boolean
+      otelContext?: otel.Context
+    },
+  ): ReadonlyArray<T> {
+    const { queriedTables, bindValues, skipCache = false, otelContext } = options ?? {}
+
+    // console.debug('in-memory-db:select', query, bindValues)
+
+    return this.otelTracer.startActiveSpan(
+      'sql-in-memory-select',
+      {},
+      otelContext ?? this.otelRootSpanContext,
+      (span) => {
+        try {
+          span.setAttribute('sql.query', queryStr)
+
+          const key = this.resultCache.getKey(queryStr, bindValues)
+          const cachedResult = this.resultCache.get(key)
+          if (skipCache === false && cachedResult !== undefined) {
+            span.setAttribute('sql.rowsCount', cachedResult.length)
+            span.setAttribute('sql.cached', true)
+            span.end()
+            return cachedResult
+          }
+
+          let stmt = this.cachedStmts.get(queryStr)
+          if (stmt === undefined) {
+            stmt = this.db.prepare(queryStr)
+            this.cachedStmts.set(queryStr, stmt)
+          }
+
+          const result = stmt.select<T>(bindValues)
+
+          span.setAttribute('sql.rowsCount', result.length)
+          span.setAttribute('sql.cached', false)
+
+          const queriedTables_ = queriedTables ?? this.getTablesUsed(queryStr)
+          this.resultCache.set(queriedTables_, key, result)
+
+          span.end()
+
+          const durationMs = getDurationMsFromSpan(span)
+
+          this.debugInfo.queryFrameDuration += durationMs
+          this.debugInfo.queryFrameCount++
+
+          // TODO also enable in non-dev mode
+          if (durationMs > 5 && import.meta.env.DEV) {
+            this.debugInfo.slowQueries.push({
+              queryStr,
+              bindValues,
+              durationMs,
+              rowsCount: result.length,
+              queriedTables: queriedTables_,
+              startTimePerfNow: getStartTimeHighResFromSpan(span),
+            })
+          }
+
+          return result
+        } finally {
+          span.end()
+        }
+      },
+    )
+  }
+
+  export() {
+    // Clear statement cache because exporting frees statements
+    for (const key of this.cachedStmts.keys()) {
+      this.cachedStmts.delete(key)
+    }
+
+    return this.db.export()
+  }
+}
+
+/** Set up SQLite performance; hasn't been super carefully optimized yet. */
+const configureSQLite = (db: SynchronousDatabaseWrapper) => {
+  db.execute(
+    // TODO: revisit these tuning parameters for max performance
+    sql`
+      PRAGMA page_size=32768;
+      PRAGMA cache_size=10000;
+      PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
+      PRAGMA synchronous='OFF';
+      PRAGMA temp_store='MEMORY';
+      PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+    `,
+  )
+}
+
+const tryGetTableNameFromPlainDeleteQuery = (query: string) => {
+  const [_, tableName] = query.trim().match(/^delete\s+from\s+(\w+)$/i) ?? []
+  return tableName
+}

package/src/ambient.d.ts

@@ -0,0 +1,10 @@
+interface Window {
+  [key: `__debug${string}`]: any
+}
+
+// eslint-disable-next-line no-var
+var __debugLiveStore: any
+
+interface ImportMeta {
+  readonly env: ImportMetaEnv
+}

package/src/effect/LiveStore.ts

@@ -0,0 +1,112 @@
+import type { Adapter, BootDb, BootStatus, UnexpectedError } from '@livestore/common'
+import type { LiveStoreSchema } from '@livestore/common/schema'
+import type { Cause, Scope } from '@livestore/utils/effect'
+import { Context, Deferred, Duration, Effect, FiberSet, Layer, OtelTracer, pipe } from '@livestore/utils/effect'
+import * as otel from '@opentelemetry/api'
+import type { GraphQLSchema } from 'graphql'
+
+import type { BaseGraphQLContext } from '../store.js'
+import { createStore } from '../store.js'
+import type { LiveStoreContextRunning as LiveStoreContextRunning_ } from '../store-context.js'
+import type { SynchronousDatabaseWrapper } from '../SynchronousDatabaseWrapper.js'
+
+export type LiveStoreContextRunning = LiveStoreContextRunning_
+export const LiveStoreContextRunning = Context.GenericTag<LiveStoreContextRunning>(
+  '@livestore/livestore/effect/LiveStoreContextRunning',
+)
+
+export type DeferredStoreContext = Deferred.Deferred<LiveStoreContextRunning, UnexpectedError>
+export const DeferredStoreContext = Context.GenericTag<DeferredStoreContext>(
+  '@livestore/livestore/effect/DeferredStoreContext',
+)
+
+export type LiveStoreContextProps<GraphQLContext extends BaseGraphQLContext> = {
+  schema: LiveStoreSchema
+  /**
+   * The `storeId` can be used to isolate multiple stores from each other.
+   * So it can be useful for multi-tenancy scenarios.
+   *
+   * The `storeId` is also used for persistence.
+   *
+   * @default 'default'
+   */
+  storeId?: string
+  graphQLOptions?: {
+    schema: Effect.Effect<GraphQLSchema, never, otel.Tracer>
+    makeContext: (db: SynchronousDatabaseWrapper, tracer: otel.Tracer, sessionId: string) => GraphQLContext
+  }
+  boot?: (db: BootDb) => Effect.Effect<void, unknown, otel.Tracer>
+  adapter: Adapter
+  disableDevtools?: boolean
+  onBootStatus?: (status: BootStatus) => void
+  batchUpdates: (run: () => void) => void
+}
+
+export const LiveStoreContextLayer = <GraphQLContext extends BaseGraphQLContext>(
+  props: LiveStoreContextProps<GraphQLContext>,
+): Layer.Layer<LiveStoreContextRunning, UnexpectedError | Cause.TimeoutException, otel.Tracer> =>
+  Layer.scoped(LiveStoreContextRunning, makeLiveStoreContext(props)).pipe(
+    Layer.withSpan('LiveStore'),
+    Layer.provide(LiveStoreContextDeferred),
+  )
+
+export const LiveStoreContextDeferred = Layer.effect(
+  DeferredStoreContext,
+  Deferred.make<LiveStoreContextRunning, UnexpectedError>(),
+)
+
+export const makeLiveStoreContext = <GraphQLContext extends BaseGraphQLContext>({
+  schema,
+  storeId = 'default',
+  graphQLOptions: graphQLOptions_,
+  boot,
+  adapter,
+  disableDevtools,
+  onBootStatus,
+  batchUpdates,
+}: LiveStoreContextProps<GraphQLContext>): Effect.Effect<
+  LiveStoreContextRunning,
+  UnexpectedError | Cause.TimeoutException,
+  DeferredStoreContext | Scope.Scope | otel.Tracer
+> =>
+  pipe(
+    Effect.gen(function* () {
+      const otelRootSpanContext = otel.context.active()
+
+      const otelTracer = yield* OtelTracer.Tracer
+
+      const graphQLOptions = yield* graphQLOptions_
+        ? Effect.all({ schema: graphQLOptions_.schema, makeContext: Effect.succeed(graphQLOptions_.makeContext) })
+        : Effect.succeed(undefined)
+
+      // TODO join fiber set and close tear down parent scope in case of error (Needs refactor with Mike A)
+      const fiberSet = yield* FiberSet.make()
+
+      const store = yield* createStore({
+        schema,
+        storeId,
+        graphQLOptions,
+        otelOptions: {
+          tracer: otelTracer,
+          rootSpanContext: otelRootSpanContext,
+        },
+        boot,
+        adapter,
+        disableDevtools,
+        fiberSet,
+        onBootStatus,
+        batchUpdates,
+      })
+
+      globalThis.__debugLiveStore ??= {}
+      // window.__debugLiveStore[schema.key] = store
+
+      return { stage: 'running', store } satisfies LiveStoreContextRunning
+    }),
+    Effect.tapErrorCause((cause) => Effect.flatMap(DeferredStoreContext, (def) => Deferred.failCause(def, cause))),
+    Effect.tap((storeCtx) => Effect.flatMap(DeferredStoreContext, (def) => Deferred.succeed(def, storeCtx))),
+    // This can take quite a while.
+    // TODO make this configurable
+    Effect.timeout(Duration.minutes(5)),
+    Effect.withSpan('@livestore/livestore/effect:makeLiveStoreContext'),
+  )

package/src/effect/index.ts

@@ -0,0 +1,8 @@
+export {
+  LiveStoreContextLayer,
+  LiveStoreContextRunning as LiveStoreContext,
+  LiveStoreContextRunning,
+  LiveStoreContextDeferred,
+  DeferredStoreContext,
+  type LiveStoreContextProps,
+} from './LiveStore.js'

package/src/global-state.ts

@@ -0,0 +1,20 @@
+/**
+ *
+ * LiveStore currently relies on some global state in order to simplify the end-user API.
+ * This however also has the downside that LiveStore can't be used in multiple instances in the same app.
+ * It could possibly also lead to some other problems.
+ *
+ * We should find some better way to do this and ideally remove this global state.
+ *
+ * Another approach could be to use the global state by default but provide an additional way to let the user
+ * explicitly pass instances of state below into the LiveStore constructors.
+ *
+ */
+
+import { GlobalValue } from '@livestore/utils/effect'
+
+import { makeReactivityGraph } from './reactiveQueries/base-class.js'
+
+export const globalReactivityGraph = GlobalValue.globalValue('livestore-global-reactivityGraph', () =>
+  makeReactivityGraph(),
+)

package/src/index.ts

@@ -0,0 +1,64 @@
+export { Store, createStorePromise, createStore } from './store.js'
+export type {
+  BaseGraphQLContext,
+  QueryDebugInfo,
+  RefreshReason,
+  CreateStoreOptions,
+  GraphQLOptions,
+  OtelOptions,
+} from './store.js'
+
+export type { LiveStoreContextRunning } from './effect/LiveStore.js'
+export { StoreAbort, StoreInterrupted, type LiveStoreContext } from './store-context.js'
+
+export { SynchronousDatabaseWrapper, emptyDebugInfo } from './SynchronousDatabaseWrapper.js'
+
+export type {
+  GetAtom,
+  AtomDebugInfo,
+  RefreshDebugInfo,
+  ReactiveGraphSnapshot,
+  SerializedAtom,
+  SerializedEffect,
+  Atom,
+  Node,
+  Ref,
+  Effect,
+} from './reactive.js'
+export { LiveStoreJSQuery, computed } from './reactiveQueries/js.js'
+export { LiveStoreSQLQuery, querySQL } from './reactiveQueries/sql.js'
+export { LiveStoreGraphQLQuery, queryGraphQL } from './reactiveQueries/graphql.js'
+export {
+  type GetAtomResult,
+  type ReactivityGraph,
+  makeReactivityGraph,
+  type LiveQuery,
+  type GetResult,
+  type LiveQueryAny,
+} from './reactiveQueries/base-class.js'
+
+export { globalReactivityGraph } from './global-state.js'
+
+export { type RowResult, type RowResultEncoded, rowQuery, deriveColQuery } from './row-query.js'
+
+export * from '@livestore/common/schema'
+export {
+  sql,
+  SessionIdSymbol,
+  type BootDb,
+  type BootStatus,
+  type SynchronousDatabase,
+  type DebugInfo,
+  type MutableDebugInfo,
+  prepareBindValues,
+  type Bindable,
+  type PreparedBindValues,
+} from '@livestore/common'
+
+export { SqliteAst, SqliteDsl } from '@livestore/db-schema'
+
+export { deepEqual } from '@livestore/utils'
+
+export * from './utils/stack-info.js'
+
+export type { ClientSession, Adapter, PreparedStatement } from '@livestore/common'