@livestore/livestore 0.0.12 → 0.0.14

package/src/__tests__/reactiveQueries/sql.test.ts

@@ -0,0 +1,372 @@
+import * as otel from '@opentelemetry/api'
+import type { ReadableSpan } from '@opentelemetry/sdk-trace-base'
+import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { describe, expect, it } from 'vitest'
+
+import { queryJS, querySQL, sql } from '../../index.js'
+import { makeTodoMvc } from '../react/fixture.js'
+
+describe('otel', () => {
+  let cachedProvider: BasicTracerProvider | undefined
+
+  const makeQuery = async () => {
+    const exporter = new InMemorySpanExporter()
+
+    const provider = cachedProvider ?? new BasicTracerProvider()
+    cachedProvider = provider
+    provider.addSpanProcessor(new SimpleSpanProcessor(exporter))
+    provider.register()
+
+    const tracer = otel.trace.getTracer('test')
+
+    const span = tracer.startSpan('test')
+    const otelContext = otel.trace.setSpan(otel.context.active(), span)
+
+    const { store } = await makeTodoMvc({ otelTracer: tracer, otelContext })
+
+    return { store, tracer, exporter, span, provider }
+  }
+
+  it('otel', async () => {
+    const { store, exporter, span } = await makeQuery()
+
+    const query = querySQL(`select * from todos`, { queriedTables: ['todos'] })
+    expect(query.run()).toMatchInlineSnapshot('[]')
+
+    store.applyEvent('RawSql', {
+      sql: sql`INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0);`,
+      bindValues: {},
+      writeTables: ['todos'],
+    })
+
+    expect(query.run()).toMatchInlineSnapshot(`
+      [
+        {
+          "completed": 0,
+          "id": "t1",
+          "text": "buy milk",
+        },
+      ]
+    `)
+
+    store.destroy()
+    query.destroy()
+    span.end()
+
+    expect(getSimplifiedRootSpan(exporter)).toMatchInlineSnapshot(`
+      {
+        "_name": "test",
+        "children": [
+          {
+            "_name": "livestore.in-memory-db:execute",
+            "attributes": {
+              "sql.query": "
+            PRAGMA page_size=32768;
+            PRAGMA cache_size=10000;
+            PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
+            PRAGMA synchronous='OFF';
+            PRAGMA temp_store='MEMORY';
+            PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+          ",
+            },
+          },
+          {
+            "_name": "sql-in-memory-select",
+            "attributes": {
+              "sql.cached": false,
+              "sql.query": "SELECT * FROM __livestore_schema",
+              "sql.rowsCount": 0,
+            },
+          },
+          {
+            "_name": "livestore.in-memory-db:execute",
+            "attributes": {
+              "sql.query": "INSERT OR IGNORE INTO app (id, newTodoText, filter) VALUES ('static', '', 'all');",
+            },
+          },
+          {
+            "_name": "LiveStore:applyEvents",
+            "children": [
+              {
+                "_name": "LiveStore:applyEvent",
+                "children": [
+                  {
+                    "_name": "LiveStore:applyEventWithoutRefresh",
+                    "attributes": {
+                      "livestore.actionType": "RawSql",
+                      "livestore.args": "{
+        \\"sql\\": \\"INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0);\\",
+        \\"bindValues\\": {},
+        \\"writeTables\\": [
+          \\"todos\\"
+        ]
+      }",
+                    },
+                    "children": [
+                      {
+                        "_name": "livestore.in-memory-db:execute",
+                        "attributes": {
+                          "sql.query": "INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0);",
+                        },
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+          {
+            "_name": "LiveStore:queries",
+            "children": [
+              {
+                "_name": "sql:select * from todos",
+                "attributes": {
+                  "sql.query": "select * from todos",
+                  "sql.rowsCount": 0,
+                },
+                "children": [
+                  {
+                    "_name": "sql-in-memory-select",
+                    "attributes": {
+                      "sql.cached": false,
+                      "sql.query": "select * from todos",
+                      "sql.rowsCount": 0,
+                    },
+                  },
+                ],
+              },
+              {
+                "_name": "sql:select * from todos",
+                "attributes": {
+                  "sql.query": "select * from todos",
+                  "sql.rowsCount": 1,
+                },
+                "children": [
+                  {
+                    "_name": "sql-in-memory-select",
+                    "attributes": {
+                      "sql.cached": false,
+                      "sql.query": "select * from todos",
+                      "sql.rowsCount": 1,
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    `)
+  })
+
+  it('with thunks', async () => {
+    const { store, exporter, span } = await makeQuery()
+
+    const defaultTodo = { id: '', text: '', completed: 0 }
+
+    const filter = queryJS(() => `where completed = 0`, { label: 'where-filter' })
+    const query = querySQL((get) => `select * from todos ${get(filter)}`, {
+      queriedTables: ['todos'],
+      label: 'all todos',
+    }).getFirstRow({
+      defaultValue: defaultTodo,
+    })
+
+    expect(query.run()).toMatchInlineSnapshot(`
+      {
+        "completed": 0,
+        "id": "",
+        "text": "",
+      }
+    `)
+
+    store.applyEvent('RawSql', {
+      sql: sql`INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0);`,
+      bindValues: {},
+      writeTables: ['todos'],
+    })
+
+    expect(query.run()).toMatchInlineSnapshot(`
+      {
+        "completed": 0,
+        "id": "t1",
+        "text": "buy milk",
+      }
+    `)
+
+    store.destroy()
+    query.destroy()
+    span.end()
+
+    expect(getSimplifiedRootSpan(exporter)).toMatchInlineSnapshot(`
+      {
+        "_name": "test",
+        "children": [
+          {
+            "_name": "livestore.in-memory-db:execute",
+            "attributes": {
+              "sql.query": "
+            PRAGMA page_size=32768;
+            PRAGMA cache_size=10000;
+            PRAGMA journal_mode='MEMORY'; -- we don't flush to disk before committing a write
+            PRAGMA synchronous='OFF';
+            PRAGMA temp_store='MEMORY';
+            PRAGMA foreign_keys='ON'; -- we want foreign key constraints to be enforced
+          ",
+            },
+          },
+          {
+            "_name": "sql-in-memory-select",
+            "attributes": {
+              "sql.cached": false,
+              "sql.query": "SELECT * FROM __livestore_schema",
+              "sql.rowsCount": 0,
+            },
+          },
+          {
+            "_name": "livestore.in-memory-db:execute",
+            "attributes": {
+              "sql.query": "INSERT OR IGNORE INTO app (id, newTodoText, filter) VALUES ('static', '', 'all');",
+            },
+          },
+          {
+            "_name": "LiveStore:applyEvents",
+            "children": [
+              {
+                "_name": "LiveStore:applyEvent",
+                "children": [
+                  {
+                    "_name": "LiveStore:applyEventWithoutRefresh",
+                    "attributes": {
+                      "livestore.actionType": "RawSql",
+                      "livestore.args": "{
+        \\"sql\\": \\"INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0);\\",
+        \\"bindValues\\": {},
+        \\"writeTables\\": [
+          \\"todos\\"
+        ]
+      }",
+                    },
+                    "children": [
+                      {
+                        "_name": "livestore.in-memory-db:execute",
+                        "attributes": {
+                          "sql.query": "INSERT INTO todos (id, text, completed) VALUES ('t1', 'buy milk', 0);",
+                        },
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+          {
+            "_name": "LiveStore:queries",
+            "children": [
+              {
+                "_name": "js:all todos:first",
+                "children": [
+                  {
+                    "_name": "sql:select * from todos where completed = 0",
+                    "attributes": {
+                      "sql.query": "select * from todos where completed = 0",
+                      "sql.rowsCount": 0,
+                    },
+                    "children": [
+                      {
+                        "_name": "js:where-filter",
+                      },
+                      {
+                        "_name": "sql-in-memory-select",
+                        "attributes": {
+                          "sql.cached": false,
+                          "sql.query": "select * from todos where completed = 0",
+                          "sql.rowsCount": 0,
+                        },
+                      },
+                    ],
+                  },
+                ],
+              },
+              {
+                "_name": "js:all todos:first",
+                "children": [
+                  {
+                    "_name": "sql:select * from todos where completed = 0",
+                    "attributes": {
+                      "sql.query": "select * from todos where completed = 0",
+                      "sql.rowsCount": 1,
+                    },
+                    "children": [
+                      {
+                        "_name": "sql-in-memory-select",
+                        "attributes": {
+                          "sql.cached": false,
+                          "sql.query": "select * from todos where completed = 0",
+                          "sql.rowsCount": 1,
+                        },
+                      },
+                    ],
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+    `)
+  })
+})
+
+const compareHrTime = (a: [number, number], b: [number, number]) => {
+  if (a[0] !== b[0]) return a[0] - b[0]
+  return a[1] - b[1]
+}
+
+const omitEmpty = (obj: any) => {
+  const result: any = {}
+  for (const key in obj) {
+    if (
+      obj[key] !== undefined &&
+      !(Array.isArray(obj[key]) && obj[key].length === 0) &&
+      Object.keys(obj[key]).length > 0
+    ) {
+      result[key] = obj[key]
+    }
+  }
+  return result
+}
+
+const getSimplifiedRootSpan = (exporter: InMemorySpanExporter) => {
+  const spans = exporter.getFinishedSpans()
+  const spansMap = new Map<string, NestedSpan>(spans.map((span) => [span.spanContext().spanId, { span, children: [] }]))
+
+  spansMap.forEach((nestedSpan) => {
+    const parentSpan = nestedSpan.span.parentSpanId ? spansMap.get(nestedSpan.span.parentSpanId) : undefined
+    if (parentSpan) {
+      parentSpan.children.push(nestedSpan)
+    }
+  })
+
+  type NestedSpan = { span: ReadableSpan; children: NestedSpan[] }
+  const rootSpan = spansMap.get(spans.find((_) => _.name === 'test')!.spanContext().spanId)!
+
+  type SimplifiedNestedSpan = { _name: string; attributes: any; children: SimplifiedNestedSpan[] }
+
+  const simplifySpan = (span: NestedSpan): SimplifiedNestedSpan =>
+    omitEmpty({
+      _name: span.span.name,
+      attributes: span.span.attributes,
+      children: span.children
+        .filter((_) => _.span.name !== 'createStore')
+        .sort((a, b) => compareHrTime(a.span.startTime, b.span.startTime))
+        .map(simplifySpan),
+    })
+
+  // console.dir(
+  //   spans.map((_) => [_.spanContext().spanId, _.name, _.attributes, _.parentSpanId]),
+  //   { depth: 10 },
+  // )
+
+  return simplifySpan(rootSpan)
+}

package/src/effect/LiveStore.ts

@@ -2,31 +2,35 @@ import type { Scope } from '@livestore/utils/effect'
 import { Context, Deferred, Duration, Effect, Layer, OtelTracer, pipe, Runtime } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 import type { GraphQLSchema } from 'graphql'
-import { mapValues } from 'lodash-es'
+import initSqlite3Wasm from 'sqlite-esm'
 
-import type { Backend, BackendOptions } from '../backends/index.js'
 import type { InMemoryDatabase } from '../inMemoryDatabase.js'
 import type { Schema } from '../schema.js'
+import type { StorageInit } from '../storage/index.js'
 import type { BaseGraphQLContext, GraphQLOptions, LiveStoreQuery, Store } from '../store.js'
 import { createStore } from '../store.js'
 
+// NOTE we're starting to initialize the sqlite wasm binary here (already before calling `createStore`),
+// so that it's ready when we need it
+const sqlite3Promise = initSqlite3Wasm({
+  print: (message) => console.log(`[livestore sqlite] ${message}`),
+  printErr: (message) => console.error(`[livestore sqlite] ${message}`),
+})
+
 // TODO get rid of `LiveStoreContext` wrapper and only expose the `Store` directly
 export type LiveStoreContext = {
-  store: Store<any>
-  globalQueries: LiveStoreQueryTypes
+  store: Store
 }
 
-export type QueryDefinition = (store: Store<any>) => LiveStoreQuery
-export type GlobalQueryDefs = { [key: string]: QueryDefinition }
+export type QueryDefinition = (store: Store) => LiveStoreQuery
 
 export type LiveStoreCreateStoreOptions<GraphQLContext extends BaseGraphQLContext> = {
   schema: Schema
-  globalQueryDefs: GlobalQueryDefs
-  backendOptions: BackendOptions
+  loadStorage: () => StorageInit | Promise<StorageInit>
   graphQLOptions?: GraphQLOptions<GraphQLContext>
   otelTracer?: otel.Tracer
   otelRootSpanContext?: otel.Context
-  boot?: (backend: Backend, parentSpan: otel.Span) => Promise<void>
+  boot?: (db: InMemoryDatabase, parentSpan: otel.Span) => unknown | Promise<unknown>
 }
 
 export const LiveStoreContext = Context.Tag<LiveStoreContext>('@livestore/livestore/LiveStoreContext')
@@ -40,13 +44,12 @@ export const DeferredStoreContext = Context.Tag<DeferredStoreContext>(
 
 export type LiveStoreContextProps<GraphQLContext extends BaseGraphQLContext> = {
   schema: Schema
-  globalQueryDefs?: Effect.Effect<never, never, GlobalQueryDefs>
-  backendOptions: Effect.Effect<never, never, BackendOptions>
+  loadStorage: () => StorageInit | Promise<StorageInit>
   graphQLOptions?: {
     schema: Effect.Effect<otel.Tracer, never, GraphQLSchema>
     makeContext: (db: InMemoryDatabase) => GraphQLContext
   }
-  boot?: (backend: Backend) => Effect.Effect<never, never, void>
+  boot?: (db: InMemoryDatabase) => Effect.Effect<never, never, void>
 }
 
 export const LiveStoreContextLayer = <GraphQLContext extends BaseGraphQLContext>(
@@ -60,9 +63,8 @@ export const LiveStoreContextLayer = <GraphQLContext extends BaseGraphQLContext>
 export const LiveStoreContextDeferred = Layer.effect(DeferredStoreContext, Deferred.make<never, LiveStoreContext>())
 
 export const makeLiveStoreContext = <GraphQLContext extends BaseGraphQLContext>({
-  globalQueryDefs,
   schema,
-  backendOptions: backendOptions_,
+  loadStorage,
   graphQLOptions: graphQLOptions_,
   boot: boot_,
 }: LiveStoreContextProps<GraphQLContext>): Effect.Effect<
@@ -84,22 +86,23 @@ export const makeLiveStoreContext = <GraphQLContext extends BaseGraphQLContext>(
           : Effect.succeed(undefined),
       )
 
-      const backendOptions = yield* $(backendOptions_)
-
       const boot = boot_
-        ? (db: Backend) =>
+        ? (db: InMemoryDatabase) =>
             boot_(db).pipe(Effect.withSpan('boot'), Effect.tapCauseLogPretty, Runtime.runPromise(runtime))
         : undefined
 
+      const sqlite3 = yield* $(Effect.promise(() => sqlite3Promise))
+
       const store = yield* $(
         Effect.tryPromise(() =>
           createStore({
             schema,
-            backendOptions,
+            loadStorage,
             graphQLOptions,
             otelTracer,
             otelRootSpanContext,
             boot,
+            sqlite3,
           }),
         ),
         Effect.acquireRelease((store) => Effect.sync(() => store.destroy())),
@@ -107,19 +110,7 @@ export const makeLiveStoreContext = <GraphQLContext extends BaseGraphQLContext>(
 
       window.__debugLiveStore = store
 
-      // TODO remove global queries concept
-      const globalQueries = yield* $(
-        globalQueryDefs ?? Effect.succeed({} as GlobalQueryDefs),
-        Effect.map((defs) => mapValues(defs, (queryDef) => queryDef(store))),
-        Effect.withSpan('LiveStore:makeGlobalQueries', {
-          parent: OtelTracer.makeExternalSpan(otel.trace.getSpanContext(store.otel.queriesSpanContext)!),
-        }),
-      )
-
-      // NOTE give main thread a chance to render
-      yield* $(Effect.yieldNow())
-
-      return { store, globalQueries }
+      return { store }
     }),
     Effect.tap((storeCtx) => Effect.flatMap(DeferredStoreContext, (def) => Deferred.succeed(def, storeCtx))),
     Effect.timeoutFail({

package/src/events.ts

@@ -5,4 +5,4 @@ export type LiveStoreEvent = {
   args?: any
 }
 
-export const EVENTS_TABLE_NAME = '__events'
+export const EVENTS_TABLE_NAME = '__livestore_events'