@livestore/livestore 0.0.0-snapshot-7438d57f493eb9f78c301e784856d60182aeb884 → 0.0.0-snapshot-669b49b56c8abe87f4e11263af7cbf506deab38e

package/src/{live-queries → reactiveQueries}/computed.ts

@@ -1,4 +1,4 @@
-import type { QueryInfo } from '@livestore/common'
+import type { QueryInfo, QueryInfoNone } from '@livestore/common'
 import * as otel from '@opentelemetry/api'
 
 import { globalReactivityGraph } from '../global-state.js'
@@ -8,7 +8,7 @@ import { getDurationMsFromSpan } from '../utils/otel.js'
 import type { GetAtomResult, LiveQuery, QueryContext, ReactivityGraph } from './base-class.js'
 import { LiveStoreQueryBase, makeGetAtomResult } from './base-class.js'
 
-export const computed = <TResult, TQueryInfo extends QueryInfo = QueryInfo.None>(
+export const computed = <TResult, TQueryInfo extends QueryInfo = QueryInfoNone>(
   fn: (get: GetAtomResult) => TResult,
   options?: {
     label: string
@@ -16,14 +16,14 @@ export const computed = <TResult, TQueryInfo extends QueryInfo = QueryInfo.None>
     queryInfo?: TQueryInfo
   },
 ): LiveQuery<TResult, TQueryInfo> =>
-  new LiveStoreComputedQuery<TResult, TQueryInfo>({
+  new LiveStoreJSQuery<TResult, TQueryInfo>({
     fn,
     label: options?.label ?? fn.toString(),
     reactivityGraph: options?.reactivityGraph,
     queryInfo: options?.queryInfo,
   })
 
-export class LiveStoreComputedQuery<TResult, TQueryInfo extends QueryInfo = QueryInfo.None> extends LiveStoreQueryBase<
+export class LiveStoreJSQuery<TResult, TQueryInfo extends QueryInfo = QueryInfoNone> extends LiveStoreQueryBase<
   TResult,
   TQueryInfo
 > {
@@ -38,19 +38,31 @@ export class LiveStoreComputedQuery<TResult, TQueryInfo extends QueryInfo = Quer
 
   queryInfo: TQueryInfo
 
+  /**
+   * Currently only used for "nested destruction" of piped queries
+   *
+   * i.e. when doing something like `const q = querySQL(...).pipe(...)`
+   * we need to also destory the SQL query when the JS query `q` is destroyed
+   */
+  private onDestroy: (() => void) | undefined
+
   constructor({
     fn,
     label,
+    onDestroy,
     reactivityGraph,
     queryInfo,
   }: {
     label: string
     fn: (get: GetAtomResult) => TResult
+    /** Currently only used for "nested destruction" of piped queries */
+    onDestroy?: () => void
     reactivityGraph?: ReactivityGraph
     queryInfo?: TQueryInfo
   }) {
     super()
 
+    this.onDestroy = onDestroy
     this.label = label
 
     this.reactivityGraph = reactivityGraph ?? globalReactivityGraph
@@ -74,11 +86,23 @@ export class LiveStoreComputedQuery<TResult, TQueryInfo extends QueryInfo = Quer
 
           return res
         }),
-      { label: queryLabel, meta: { liveStoreThunkType: 'computed' } },
+      { label: queryLabel, meta: { liveStoreThunkType: 'jsResults' } },
     )
   }
 
+  // pipe = <U>(fn: (result: TResult, get: GetAtomResult) => U): LiveStoreJSQuery<U> =>
+  //   new LiveStoreJSQuery({
+  //     fn: (get) => {
+  //       const results = get(this.results$)
+  //       return fn(results, get)
+  //     },
+  //     label: `${this.label}:js`,
+  //     onDestroy: () => this.destroy(),
+  //     reactivityGraph: this.reactivityGraph,
+  //   })
+
   destroy = () => {
     this.reactivityGraph.destroyNode(this.results$)
+    this.onDestroy?.()
   }
 }

package/src/{live-queries → reactiveQueries}/graphql.ts

@@ -1,5 +1,5 @@
 import type { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/core'
-import type { QueryInfo } from '@livestore/common'
+import type { QueryInfoNone } from '@livestore/common'
 import { shouldNeverHappen } from '@livestore/utils'
 import { Schema, TreeFormatter } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
@@ -31,7 +31,7 @@ export const queryGraphQL = <
     reactivityGraph?: ReactivityGraph
     map?: MapResult<TResultMapped, TResult>
   } = {},
-): LiveQuery<TResultMapped, QueryInfo.None> =>
+): LiveQuery<TResultMapped, QueryInfoNone> =>
   new LiveStoreGraphQLQuery({ document, genVariableValues, label, reactivityGraph, map })
 
 export class LiveStoreGraphQLQuery<
@@ -39,7 +39,7 @@ export class LiveStoreGraphQLQuery<
   TVariableValues extends Record<string, any>,
   TContext extends BaseGraphQLContext,
   TResultMapped extends Record<string, any> = TResult,
-> extends LiveStoreQueryBase<TResultMapped, QueryInfo.None> {
+> extends LiveStoreQueryBase<TResultMapped, QueryInfoNone> {
   _tag: 'graphql' = 'graphql'
 
   /** The abstract GraphQL query */
@@ -54,7 +54,7 @@ export class LiveStoreGraphQLQuery<
 
   protected reactivityGraph: ReactivityGraph
 
-  queryInfo: QueryInfo.None = { _tag: 'None' }
+  queryInfo: QueryInfoNone = { _tag: 'None' }
 
   private mapResult
 
@@ -105,7 +105,7 @@ export class LiveStoreGraphQLQuery<
         (get, _setDebugInfo, { rootOtelContext }, otelContext) => {
           return genVariableValues(makeGetAtomResult(get, otelContext ?? rootOtelContext))
         },
-        { label: `${labelWithDefault}:variableValues`, meta: { liveStoreThunkType: 'graphql.variables' } },
+        { label: `${labelWithDefault}:variableValues`, meta: { liveStoreThunkType: 'graphqlVariableValues' } },
       )
       this.variableValues$ = variableValues$OrvariableValues
     } else {
@@ -137,11 +137,26 @@ export class LiveStoreGraphQLQuery<
 
         return result
       },
-      { label: resultsLabel, meta: { liveStoreThunkType: 'graphql.result' } },
+      { label: resultsLabel, meta: { liveStoreThunkType: 'graphqlResults' } },
       // otelContext,
     )
   }
 
+  /**
+   * Returns a new reactive query that contains the result of
+   * running an arbitrary JS computation on the results of this SQL query.
+   */
+  // pipe = <U>(fn: (result: TResult, get: GetAtomResult) => U): LiveStoreJSQuery<U> =>
+  //   new LiveStoreJSQuery({
+  //     fn: (get) => {
+  //       const results = get(this.results$)
+  //       return fn(results, get)
+  //     },
+  //     label: `${this.label}:js`,
+  //     onDestroy: () => this.destroy(),
+  //     reactivityGraph: this.reactivityGraph,
+  //   })
+
   queryOnce = ({
     document,
     otelContext,

package/src/reactiveQueries/sql.test.ts

@@ -0,0 +1,308 @@
+import { Effect, Schema } from '@livestore/utils/effect'
+import * as otel from '@opentelemetry/api'
+import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { describe, expect, it } from 'vitest'
+
+import { computed, querySQL, rawSqlMutation, sql } from '../index.js'
+import { makeTodoMvc, tables } from '../utils/tests/fixture.js'
+import { getSimplifiedRootSpan } from '../utils/tests/otel.js'
+
+/*
+TODO write tests for:
+
+- sql queries without and with `map` (incl. callback and schemas)
+- optional and explicit `queriedTables` argument
+*/
+
+describe('otel', () => {
+  let cachedProvider: BasicTracerProvider | undefined
+
+  const makeQuery = Effect.gen(function* () {
+    const exporter = new InMemorySpanExporter()
+
+    const provider = cachedProvider ?? new BasicTracerProvider()
+    cachedProvider = provider
+    provider.addSpanProcessor(new SimpleSpanProcessor(exporter))
+    provider.register()
+
+    const otelTracer = otel.trace.getTracer('test')
+
+    const span = otelTracer.startSpan('test')
+    const otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+    const { store } = yield* makeTodoMvc({ otelTracer, otelContext })
+
+    return {
+      store,
+      otelTracer,
+      exporter,
+      span,
+      provider,
+    }
+  })
+
+  it('otel', async () => {
+    const { exporter } = await Effect.gen(function* () {
+      const { store, exporter, span } = yield* makeQuery
+
+      const query = querySQL(`select * from todos`, {
+        schema: Schema.Array(tables.todos.schema),
+        queriedTables: new Set(['todos']),
+      })
+      expect(query.run()).toMatchInlineSnapshot('[]')
+
+      store.mutate(rawSqlMutation({ sql: sql`INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0)` }))
+
+      expect(query.run()).toMatchInlineSnapshot(`
+      [
+        {
+          "completed": false,
+          "id": "t1",
+          "text": "buy milk",
+        },
+      ]
+    `)
+
+      query.destroy()
+      span.end()
+
+      return { exporter }
+    }).pipe(Effect.scoped, Effect.tapCauseLogPretty, Effect.runPromise)
+
+    expect(getSimplifiedRootSpan(exporter)).toMatchInlineSnapshot(`
+      {
+        "_name": "test",
+        "children": [
+          {
+            "_name": "livestore.in-memory-db:execute",
+            "attributes": {
+              "sql.query": "
+            PRAGMA page_size=32768;
+            PRAGMA cache_size=10000;
+            PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
+            PRAGMA synchronous='OFF';
+            PRAGMA temp_store='MEMORY';
+            PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+          ",
+            },
+          },
+          {
+            "_name": "LiveStore:mutations",
+            "children": [
+              {
+                "_name": "LiveStore:mutate",
+                "attributes": {
+                  "livestore.mutateLabel": "mutate",
+                },
+                "children": [
+                  {
+                    "_name": "LiveStore:processWrites",
+                    "attributes": {
+                      "livestore.mutateLabel": "mutate",
+                    },
+                    "children": [
+                      {
+                        "_name": "LiveStore:mutateWithoutRefresh",
+                        "attributes": {
+                          "livestore.args": "{
+        "sql": "INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0)"
+      }",
+                          "livestore.mutation": "livestore.RawSql",
+                        },
+                        "children": [
+                          {
+                            "_name": "livestore.in-memory-db:execute",
+                            "attributes": {
+                              "sql.query": "INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0)",
+                            },
+                          },
+                        ],
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+          {
+            "_name": "LiveStore:queries",
+            "children": [
+              {
+                "_name": "sql:select * from todos",
+                "attributes": {
+                  "sql.query": "select * from todos",
+                  "sql.rowsCount": 0,
+                },
+                "children": [
+                  {
+                    "_name": "sql-in-memory-select",
+                    "attributes": {
+                      "sql.cached": false,
+                      "sql.query": "select * from todos",
+                      "sql.rowsCount": 0,
+                    },
+                  },
+                ],
+              },
+              {
+                "_name": "sql:select * from todos",
+                "attributes": {
+                  "sql.query": "select * from todos",
+                  "sql.rowsCount": 1,
+                },
+                "children": [
+                  {
+                    "_name": "sql-in-memory-select",
+                    "attributes": {
+                      "sql.cached": false,
+                      "sql.query": "select * from todos",
+                      "sql.rowsCount": 1,
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    `)
+  })
+
+  it('with thunks', async () => {
+    const { exporter } = await Effect.gen(function* () {
+      const { store, exporter, span } = yield* makeQuery
+
+      const defaultTodo = { id: '', text: '', completed: false }
+
+      const filter = computed(() => `where completed = 0`, { label: 'where-filter' })
+      const query = querySQL((get) => `select * from todos ${get(filter)}`, {
+        label: 'all todos',
+        schema: Schema.Array(tables.todos.schema).pipe(Schema.headOrElse(() => defaultTodo)),
+      })
+
+      expect(query.run()).toMatchInlineSnapshot(`
+      {
+        "completed": false,
+        "id": "",
+        "text": "",
+      }
+    `)
+
+      store.mutate(rawSqlMutation({ sql: sql`INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0)` }))
+
+      expect(query.run()).toMatchInlineSnapshot(`
+      {
+        "completed": false,
+        "id": "t1",
+        "text": "buy milk",
+      }
+    `)
+
+      query.destroy()
+      span.end()
+
+      return { exporter }
+    }).pipe(Effect.scoped, Effect.tapCauseLogPretty, Effect.runPromise)
+
+    expect(getSimplifiedRootSpan(exporter)).toMatchInlineSnapshot(`
+      {
+        "_name": "test",
+        "children": [
+          {
+            "_name": "livestore.in-memory-db:execute",
+            "attributes": {
+              "sql.query": "
+            PRAGMA page_size=32768;
+            PRAGMA cache_size=10000;
+            PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
+            PRAGMA synchronous='OFF';
+            PRAGMA temp_store='MEMORY';
+            PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+          ",
+            },
+          },
+          {
+            "_name": "LiveStore:mutations",
+            "children": [
+              {
+                "_name": "LiveStore:mutate",
+                "attributes": {
+                  "livestore.mutateLabel": "mutate",
+                },
+                "children": [
+                  {
+                    "_name": "LiveStore:processWrites",
+                    "attributes": {
+                      "livestore.mutateLabel": "mutate",
+                    },
+                    "children": [
+                      {
+                        "_name": "LiveStore:mutateWithoutRefresh",
+                        "attributes": {
+                          "livestore.args": "{
+        "sql": "INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0)"
+      }",
+                          "livestore.mutation": "livestore.RawSql",
+                        },
+                        "children": [
+                          {
+                            "_name": "livestore.in-memory-db:execute",
+                            "attributes": {
+                              "sql.query": "INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0)",
+                            },
+                          },
+                        ],
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+          {
+            "_name": "LiveStore:queries",
+            "children": [
+              {
+                "_name": "sql:select * from todos where completed = 0",
+                "attributes": {
+                  "sql.query": "select * from todos where completed = 0",
+                  "sql.rowsCount": 0,
+                },
+                "children": [
+                  {
+                    "_name": "js:where-filter",
+                  },
+                  {
+                    "_name": "sql-in-memory-select",
+                    "attributes": {
+                      "sql.cached": false,
+                      "sql.query": "select * from todos where completed = 0",
+                      "sql.rowsCount": 0,
+                    },
+                  },
+                ],
+              },
+              {
+                "_name": "sql:select * from todos where completed = 0",
+                "attributes": {
+                  "sql.query": "select * from todos where completed = 0",
+                  "sql.rowsCount": 1,
+                },
+                "children": [
+                  {
+                    "_name": "sql-in-memory-select",
+                    "attributes": {
+                      "sql.cached": false,
+                      "sql.query": "select * from todos where completed = 0",
+                      "sql.rowsCount": 1,
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    `)
+  })
+})

package/src/reactiveQueries/sql.ts

@@ -0,0 +1,226 @@
+import { type Bindable, prepareBindValues, type QueryInfo, type QueryInfoNone } from '@livestore/common'
+import { shouldNeverHappen } from '@livestore/utils'
+import { Schema, TreeFormatter } from '@livestore/utils/effect'
+import * as otel from '@opentelemetry/api'
+
+import { globalReactivityGraph } from '../global-state.js'
+import type { Thunk } from '../reactive.js'
+import { NOT_REFRESHED_YET } from '../reactive.js'
+import type { RefreshReason } from '../store/store-types.js'
+import { getDurationMsFromSpan } from '../utils/otel.js'
+import type { GetAtomResult, LiveQuery, QueryContext, ReactivityGraph } from './base-class.js'
+import { LiveStoreQueryBase, makeGetAtomResult } from './base-class.js'
+
+/**
+ * NOTE `querySQL` is only supposed to read data. Don't use it to insert/update/delete data but use mutations instead.
+ */
+export const querySQL = <TResultSchema, TResult = TResultSchema>(
+  query: string | ((get: GetAtomResult) => string),
+  options: {
+    schema: Schema.Schema<TResultSchema, ReadonlyArray<any>>
+    map?: (rows: TResultSchema) => TResult
+    /**
+     * Can be provided explicitly to slightly speed up initial query performance
+     *
+     * NOTE In the future we want to do this automatically at build time
+     */
+    queriedTables?: Set<string>
+    bindValues?: Bindable
+    label?: string
+    reactivityGraph?: ReactivityGraph
+  },
+): LiveQuery<TResult, QueryInfoNone> =>
+  new LiveStoreSQLQuery<TResultSchema, TResult, QueryInfoNone>({
+    label: options?.label,
+    genQueryString: query,
+    queriedTables: options?.queriedTables,
+    bindValues: options?.bindValues,
+    reactivityGraph: options?.reactivityGraph,
+    map: options?.map,
+    schema: options.schema,
+    queryInfo: { _tag: 'None' },
+  })
+
+/* An object encapsulating a reactive SQL query */
+export class LiveStoreSQLQuery<
+  TResultSchema,
+  TResult = TResultSchema,
+  TQueryInfo extends QueryInfo = QueryInfoNone,
+> extends LiveStoreQueryBase<TResult, TQueryInfo> {
+  _tag: 'sql' = 'sql'
+
+  /** A reactive thunk representing the query text */
+  queryString$: Thunk<string, QueryContext, RefreshReason> | undefined
+
+  /** A reactive thunk representing the query results */
+  results$: Thunk<TResult, QueryContext, RefreshReason>
+
+  label: string
+
+  protected reactivityGraph
+
+  /** Currently only used by `rowQuery` for lazy table migrations and eager default row insertion */
+  private execBeforeFirstRun
+
+  private mapResult: (rows: TResultSchema) => TResult
+  private schema: Schema.Schema<TResultSchema, ReadonlyArray<any>>
+
+  queryInfo: TQueryInfo
+
+  constructor({
+    genQueryString,
+    queriedTables,
+    bindValues,
+    label = genQueryString.toString(),
+    reactivityGraph,
+    schema,
+    map,
+    execBeforeFirstRun,
+    queryInfo,
+  }: {
+    label?: string
+    genQueryString: string | ((get: GetAtomResult, ctx: QueryContext) => string)
+    queriedTables?: Set<string>
+    bindValues?: Bindable
+    reactivityGraph?: ReactivityGraph
+    schema: Schema.Schema<TResultSchema, ReadonlyArray<any>>
+    map?: (rows: TResultSchema) => TResult
+    execBeforeFirstRun?: (ctx: QueryContext) => void
+    queryInfo?: TQueryInfo
+  }) {
+    super()
+
+    this.label = `sql(${label})`
+    this.reactivityGraph = reactivityGraph ?? globalReactivityGraph
+    this.execBeforeFirstRun = execBeforeFirstRun
+    this.queryInfo = queryInfo ?? ({ _tag: 'None' } as TQueryInfo)
+
+    this.schema = schema
+    this.mapResult = map === undefined ? (rows: any) => rows as TResult : map
+
+    let queryString$OrQueryString: string | Thunk<string, QueryContext, RefreshReason>
+    if (typeof genQueryString === 'function') {
+      queryString$OrQueryString = this.reactivityGraph.makeThunk(
+        (get, setDebugInfo, ctx, otelContext) => {
+          const startMs = performance.now()
+          const queryString = genQueryString(makeGetAtomResult(get, otelContext ?? ctx.rootOtelContext), ctx)
+          const durationMs = performance.now() - startMs
+          setDebugInfo({ _tag: 'computed', label: `${label}:queryString`, query: queryString, durationMs })
+          return queryString
+        },
+        {
+          label: `${label}:queryString`,
+          meta: { liveStoreThunkType: 'sqlQueryString' },
+          equal: (a, b) => a === b,
+        },
+      )
+
+      this.queryString$ = queryString$OrQueryString
+    } else {
+      queryString$OrQueryString = genQueryString
+    }
+
+    const queryLabel = `${label}:results`
+
+    const queriedTablesRef = { current: queriedTables }
+
+    const schemaEqual = Schema.equivalence(schema)
+    // TODO also support derived equality for `map` (probably will depend on having an easy way to transform a schema without an `encode` step)
+    // This would mean dropping the `map` option
+    const equal =
+      map === undefined
+        ? (a: TResult, b: TResult) =>
+            a === NOT_REFRESHED_YET || b === NOT_REFRESHED_YET ? false : schemaEqual(a as any, b as any)
+        : undefined
+
+    const results$ = this.reactivityGraph.makeThunk<TResult>(
+      (get, setDebugInfo, { store, otelTracer, rootOtelContext }, otelContext) =>
+        otelTracer.startActiveSpan(
+          'sql:...', // NOTE span name will be overridden further down
+          {},
+          otelContext ?? rootOtelContext,
+          (span) => {
+            const otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+            if (this.execBeforeFirstRun !== undefined) {
+              this.execBeforeFirstRun({ store, otelTracer, rootOtelContext, effectsWrapper: (run) => run() })
+              this.execBeforeFirstRun = undefined
+            }
+
+            const sqlString =
+              typeof queryString$OrQueryString === 'string'
+                ? queryString$OrQueryString
+                : get(queryString$OrQueryString, otelContext)
+
+            if (queriedTablesRef.current === undefined) {
+              queriedTablesRef.current = store.syncDbWrapper.getTablesUsed(sqlString)
+            }
+
+            // Establish a reactive dependency on the tables used in the query
+            for (const tableName of queriedTablesRef.current) {
+              const tableRef = store.tableRefs[tableName] ?? shouldNeverHappen(`No table ref found for ${tableName}`)
+              get(tableRef, otelContext)
+            }
+
+            span.setAttribute('sql.query', sqlString)
+            span.updateName(`sql:${sqlString.slice(0, 50)}`)
+
+            const rawResults = store.syncDbWrapper.select<any>(sqlString, {
+              queriedTables,
+              bindValues: bindValues ? prepareBindValues(bindValues, sqlString) : undefined,
+              otelContext,
+            })
+
+            span.setAttribute('sql.rowsCount', rawResults.length)
+
+            const parsedResult = Schema.decodeEither(this.schema)(rawResults)
+
+            if (parsedResult._tag === 'Left') {
+              const parseErrorStr = TreeFormatter.formatErrorSync(parsedResult.left)
+              const expectedSchemaStr = String(this.schema.ast)
+              const bindValuesStr = bindValues === undefined ? '' : `\nBind values: ${JSON.stringify(bindValues)}`
+
+              console.error(
+                `\
+Error parsing SQL query result.
+
+Query: ${sqlString}\
+${bindValuesStr}
+
+Expected schema: ${expectedSchemaStr}
+
+Error: ${parseErrorStr}
+
+Result:`,
+                rawResults,
+              )
+              return shouldNeverHappen(`Error parsing SQL query result: ${parsedResult.left}`)
+            }
+
+            const result = this.mapResult(parsedResult.right)
+
+            span.end()
+
+            const durationMs = getDurationMsFromSpan(span)
+
+            this.executionTimes.push(durationMs)
+
+            setDebugInfo({ _tag: 'sql', label, query: sqlString, durationMs })
+
+            return result
+          },
+        ),
+      { label: queryLabel, equal },
+    )
+
+    this.results$ = results$
+  }
+
+  destroy = () => {
+    if (this.queryString$ !== undefined) {
+      this.reactivityGraph.destroyNode(this.queryString$)
+    }
+
+    this.reactivityGraph.destroyNode(this.results$)
+  }
+}