@livestore/common 0.2.0-dev.2 → 0.3.0-dev.0

package/src/leader-thread/recreate-db.ts

@@ -0,0 +1,109 @@
+import { casesHandled } from '@livestore/utils'
+import type { HttpClient } from '@livestore/utils/effect'
+import { Effect, Queue } from '@livestore/utils/effect'
+
+import type { InvalidPullError, IsOfflineError, MigrationHooks, SqliteError } from '../index.js'
+import { initializeSingletonTables, migrateDb, rehydrateFromMutationLog, UnexpectedError } from '../index.js'
+import { configureConnection } from './connection.js'
+import { LeaderThreadCtx } from './types.js'
+
+export const recreateDb: Effect.Effect<
+  void,
+  UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
+  LeaderThreadCtx | HttpClient.HttpClient
+> = Effect.gen(function* () {
+  const { db, dbLog, makeSyncDb, schema, bootStatusQueue } = yield* LeaderThreadCtx
+
+  const migrationOptions = schema.migrationOptions
+
+  yield* Effect.addFinalizer(
+    Effect.fn('recreateDb:finalizer')(function* (ex) {
+      if (ex._tag === 'Failure') db.destroy()
+    }),
+  )
+
+  // NOTE to speed up the operations below, we're creating a temporary in-memory database
+  // and later we'll overwrite the persisted database with the new data
+  const tmpSyncDb = yield* makeSyncDb({ _tag: 'in-memory' })
+  yield* configureConnection(tmpSyncDb, { fkEnabled: true })
+
+  const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
+    Effect.gen(function* () {
+      yield* Effect.tryAll(() => hooks?.init?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+
+      yield* migrateDb({
+        db: tmpSyncDb,
+        schema,
+        onProgress: ({ done, total }) =>
+          Queue.offer(bootStatusQueue, { stage: 'migrating', progress: { done, total } }),
+      })
+
+      initializeSingletonTables(schema, tmpSyncDb)
+
+      yield* Effect.tryAll(() => hooks?.pre?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+
+      return tmpSyncDb
+    })
+
+  switch (migrationOptions.strategy) {
+    case 'from-mutation-log': {
+      const hooks = migrationOptions.hooks
+      const tmpSyncDb = yield* initDb(hooks)
+
+      yield* rehydrateFromMutationLog({
+        db: tmpSyncDb,
+        logDb: dbLog,
+        schema,
+        migrationOptions,
+        onProgress: ({ done, total }) =>
+          Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
+      })
+
+      yield* Effect.tryAll(() => hooks?.post?.(tmpSyncDb)).pipe(UnexpectedError.mapToUnexpectedError)
+
+      break
+    }
+    case 'hard-reset': {
+      const hooks = migrationOptions.hooks
+      const tmpInMemoryDb = yield* initDb(hooks)
+
+      // The database is migrated but empty now, so nothing else to do
+
+      yield* Effect.tryAll(() => hooks?.post?.(tmpInMemoryDb)).pipe(UnexpectedError.mapToUnexpectedError)
+
+      break
+    }
+    case 'manual': {
+      const oldDbData = db.export()
+
+      const newDbData = yield* Effect.tryAll(() => migrationOptions.migrate(oldDbData)).pipe(
+        UnexpectedError.mapToUnexpectedError,
+      )
+
+      tmpSyncDb.import(newDbData)
+
+      // TODO validate schema
+
+      break
+    }
+    default: {
+      casesHandled(migrationOptions)
+    }
+  }
+
+  // Import the temporary in-memory database into the persistent database
+  yield* Effect.sync(() => db.import(tmpSyncDb)).pipe(
+    Effect.withSpan('@livestore/common:leader-thread:recreateDb:import'),
+  )
+
+  // TODO maybe bring back re-using this initial snapshot to avoid calling `.export()` again
+  // We've disabled this for now as it made the code too complex, as we often run syncing right after
+  // so the snapshot is no longer up to date
+  // const snapshotFromTmpDb = tmpSyncDb.export()
+
+  tmpSyncDb.close()
+}).pipe(
+  Effect.scoped, // NOTE we're closing the scope here so finalizers are called when the effect is done
+  Effect.withSpan('@livestore/common:leader-thread:recreateDb'),
+  Effect.withPerformanceMeasure('@livestore/common:leader-thread:recreateDb'),
+)

package/src/leader-thread/shutdown-channel.ts

@@ -0,0 +1,13 @@
+import type { WebChannel } from '@livestore/utils/effect'
+import { Schema } from '@livestore/utils/effect'
+
+import { IntentionalShutdownCause } from '../index.js'
+
+export class DedicatedWorkerDisconnectBroadcast extends Schema.TaggedStruct('DedicatedWorkerDisconnectBroadcast', {}) {}
+
+export class All extends Schema.Union(IntentionalShutdownCause, DedicatedWorkerDisconnectBroadcast) {}
+
+/**
+ * Used internally by an adapter to shutdown gracefully.
+ */
+export type ShutdownChannel = WebChannel.WebChannel<typeof All.Type, typeof All.Type>

package/src/leader-thread/types.ts

@@ -0,0 +1,129 @@
+import type {
+  Deferred,
+  Effect,
+  Fiber,
+  HttpClient,
+  Option,
+  Queue,
+  Scope,
+  SubscriptionRef,
+  WebChannel,
+} from '@livestore/utils/effect'
+import { Context, Schema } from '@livestore/utils/effect'
+
+import type {
+  BootStatus,
+  Devtools,
+  InvalidPushError,
+  MakeSynchronousDatabase,
+  PersistenceInfo,
+  SyncBackend,
+  SynchronousDatabase,
+  UnexpectedError,
+} from '../index.js'
+import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
+import type { PayloadUpstream, SyncState } from '../sync/syncstate.js'
+import type { ShutdownChannel } from './shutdown-channel.js'
+
+export type ShutdownState = 'running' | 'shutting-down'
+
+export class OuterWorkerCtx extends Context.Tag('OuterWorkerCtx')<
+  OuterWorkerCtx,
+  {
+    innerFiber: Fiber.RuntimeFiber<any, any>
+  }
+>() {}
+
+export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
+export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
+
+export const InitialSyncOptionsBlocking = Schema.TaggedStruct('Blocking', {
+  timeout: Schema.DurationFromMillis,
+})
+
+export type InitialSyncOptionsBlocking = typeof InitialSyncOptionsBlocking.Type
+
+export const InitialSyncOptions = Schema.Union(InitialSyncOptionsSkip, InitialSyncOptionsBlocking)
+export type InitialSyncOptions = typeof InitialSyncOptions.Type
+
+export type InitialSyncInfo = Option.Option<{
+  cursor: EventId.EventId
+  metadata: Option.Option<Schema.JsonValue>
+}>
+
+// export type InitialSetup =
+//   | { _tag: 'Recreate'; snapshotRef: Ref.Ref<Uint8Array | undefined>; syncInfo: InitialSyncInfo }
+//   | { _tag: 'Reuse'; syncInfo: InitialSyncInfo }
+
+export type LeaderDatabase = SynchronousDatabase<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
+export type PersistenceInfoPair = { db: PersistenceInfo; mutationLog: PersistenceInfo }
+
+export type DevtoolsOptions =
+  | {
+      enabled: false
+    }
+  | {
+      enabled: true
+      makeContext: Effect.Effect<
+        {
+          devtoolsWebChannel: WebChannel.WebChannel<Devtools.MessageToAppLeader, Devtools.MessageFromAppLeader>
+          shutdownChannel: ShutdownChannel
+          persistenceInfo: PersistenceInfoPair
+        },
+        UnexpectedError,
+        Scope.Scope
+      >
+    }
+
+export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
+  LeaderThreadCtx,
+  {
+    schema: LiveStoreSchema
+    storeId: string
+    originId: string
+    makeSyncDb: MakeSynchronousDatabase
+    db: LeaderDatabase
+    dbLog: LeaderDatabase
+    bootStatusQueue: Queue.Queue<BootStatus>
+    // TODO we should find a more elegant way to handle cases which need this ref for their implementation
+    shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
+    mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
+    // devtools: DevtoolsContext
+    syncBackend: SyncBackend | undefined
+    syncProcessor: SyncProcessor
+    connectedClientSessionPullQueues: PullQueueSet
+  }
+>() {}
+
+export type InitialBlockingSyncContext = {
+  blockingDeferred: Deferred.Deferred<void> | undefined
+  update: (_: { remaining: number; processed: number }) => Effect.Effect<void>
+}
+
+export type PullQueueItem = {
+  // mutationEvents: ReadonlyArray<MutationEvent.AnyEncoded>
+  // backendHead: number
+  payload: PayloadUpstream
+  // TODO move `remaining` into `PayloadUpstream`
+  remaining: number
+}
+
+export interface SyncProcessor {
+  push: (
+    /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
+    batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
+  ) => Effect.Effect<void, UnexpectedError | InvalidPushError, HttpClient.HttpClient | LeaderThreadCtx>
+
+  pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+  boot: (args: {
+    dbReady: Deferred.Deferred<void>
+  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient>
+  syncState: Effect.Effect<SyncState, UnexpectedError>
+}
+
+export interface PullQueueSet {
+  makeQueue: (
+    since: EventId.EventId,
+  ) => Effect.Effect<Queue.Queue<PullQueueItem>, UnexpectedError, Scope.Scope | LeaderThreadCtx>
+  offer: (item: PullQueueItem) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+}

package/src/mutation.ts

@@ -1,9 +1,8 @@
-import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import { Schema } from '@livestore/utils/effect'
 
 import { SessionIdSymbol } from './adapter-types.js'
-import type { LiveStoreSchema } from './schema/index.js'
-import type { MutationDef, MutationEvent } from './schema/mutations.js'
+import type * as MutationEvent from './schema/MutationEvent.js'
+import type { MutationDef } from './schema/mutations.js'
 import type { PreparedBindValues } from './util.js'
 import { prepareBindValues } from './util.js'
 
@@ -12,7 +11,7 @@ export const getExecArgsFromMutation = ({
   mutationEventDecoded,
 }: {
   mutationDef: MutationDef.Any
-  mutationEventDecoded: MutationEvent.Any
+  mutationEventDecoded: MutationEvent.Any | MutationEvent.PartialAny
 }): ReadonlyArray<{
   statementSql: string
   bindValues: PreparedBindValues
@@ -52,23 +51,6 @@ export const getExecArgsFromMutation = ({
   })
 }
 
-export const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema) => {
-  const migrationOptions = schema.migrationOptions
-  const mutationLogExclude =
-    migrationOptions.strategy === 'from-mutation-log'
-      ? (migrationOptions.excludeMutations ?? new Set(['livestore.RawSql']))
-      : new Set(['livestore.RawSql'])
-
-  return (mutationName: string, mutationEventDecoded: MutationEvent.Any): boolean => {
-    if (mutationLogExclude.has(mutationName)) return true
-
-    const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
-    const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
-
-    return execArgsArr.some((_) => _.statementSql.includes('__livestore'))
-  }
-})
-
 // NOTE we should explore whether there is a more elegant solution
 // e.g. by leveraging the schema to replace the sessionIdSymbol
 export const replaceSessionIdSymbol = (

package/src/otel.ts

@@ -0,0 +1,20 @@
+import { makeNoopTracer } from '@livestore/utils'
+import { Effect, identity, Layer, OtelTracer } from '@livestore/utils/effect'
+import * as otel from '@opentelemetry/api'
+
+export const provideOtel =
+  ({ otelTracer, parentSpanContext }: { otelTracer?: otel.Tracer; parentSpanContext?: otel.Context }) =>
+  <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, Exclude<R, OtelTracer.OtelTracer>> => {
+    const OtelTracerLive = Layer.succeed(OtelTracer.OtelTracer, otelTracer ?? makeNoopTracer())
+
+    const TracingLive = Layer.unwrapEffect(Effect.map(OtelTracer.make, Layer.setTracer)).pipe(
+      Layer.provideMerge(OtelTracerLive),
+    ) as any as Layer.Layer<OtelTracer.OtelTracer>
+
+    return effect.pipe(
+      parentSpanContext
+        ? Effect.withParentSpan(OtelTracer.makeExternalSpan(otel.trace.getSpanContext(parentSpanContext)!))
+        : identity,
+      Effect.provide(TracingLive),
+    )
+  }

package/src/query-builder/api.ts

@@ -3,7 +3,7 @@ import { type Option, Predicate, type Schema } from '@livestore/utils/effect'
 
 import type { SessionIdSymbol } from '../adapter-types.js'
 import type { QueryInfo } from '../query-info.js'
-import type { DbSchema } from '../schema/index.js'
+import type { DbSchema } from '../schema/mod.js'
 import type { SqliteDsl } from '../schema/table-def.js'
 import type { SqlValue } from '../util.js'
 
@@ -243,8 +243,9 @@ export namespace QueryBuilder {
     >
 
     /**
-     *
+     * Gets a single row from the table and will create it if it doesn't exist yet.
      */
+    // TODO maybe call `getsert`?
     readonly row: TTableDef['options']['isSingleton'] extends true
       ? () => QueryBuilder<RowQuery.Result<TTableDef>, TTableDef, QueryBuilder.ApiFeature, QueryInfo.Row>
       : TTableDef['options']['deriveMutations']['enabled'] extends false

package/src/query-builder/impl.test.ts

@@ -1,7 +1,8 @@
 import { Schema } from '@livestore/utils/effect'
 import { describe, expect, it } from 'vitest'
 
-import { DbSchema } from '../schema/index.js'
+import { DbSchema } from '../schema/mod.js'
+import { getResultSchema } from './impl.js'
 
 const todos = DbSchema.table(
   'todos',
@@ -26,6 +27,12 @@ const comments = DbSchema.table('comments', {
 const db = { todos: todos.query, comments: comments.query }
 
 describe('query builder', () => {
+  describe('result schema', () => {
+    it('should print the schema', () => {
+      expect(String(getResultSchema(db.todos))).toMatchInlineSnapshot(`"ReadonlyArray<todos>"`)
+    })
+  })
+
   describe('basic queries', () => {
     it('should handle simple SELECT queries', () => {
       expect(db.todos.asSql()).toMatchInlineSnapshot(`
@@ -87,6 +94,33 @@ describe('query builder', () => {
           "query": "SELECT id, text FROM 'todos' WHERE deletedAt <= ?",
         }
       `)
+      expect(
+        db.todos
+          .select('id', 'text')
+          .where({ status: { op: 'IN', value: ['active'] } })
+          .asSql(),
+      ).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "active",
+          ],
+          "query": "SELECT id, text FROM 'todos' WHERE status IN (?)",
+        }
+      `)
+      expect(
+        db.todos
+          .select('id', 'text')
+          .where({ status: { op: 'NOT IN', value: ['active', 'completed'] } })
+          .asSql(),
+      ).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "active",
+            "completed",
+          ],
+          "query": "SELECT id, text FROM 'todos' WHERE status NOT IN (?, ?)",
+        }
+      `)
     })
 
     it('should handle OFFSET and LIMIT clauses', () => {

package/src/query-builder/impl.ts

@@ -1,7 +1,7 @@
 import { Option, Predicate, Schema } from '@livestore/utils/effect'
 
 import type { QueryInfo } from '../query-info.js'
-import type { DbSchema } from '../schema/index.js'
+import type { DbSchema } from '../schema/mod.js'
 import type { QueryBuilder, QueryBuilderAst } from './api.js'
 import { QueryBuilderAstSymbol, TypeId } from './api.js'
 
@@ -119,7 +119,8 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
       return makeQueryBuilder(tableDef, {
         ...ast,
         limit: Option.some(1),
-        pickFirst: options?.fallback ? { fallback: options.fallback } : false,
+        // TODO improve
+        pickFirst: options?.fallback ? { fallback: options.fallback } : { fallback: () => undefined },
       })
     },
     // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
@@ -154,7 +155,14 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
     [TypeId]: TypeId,
     [QueryBuilderAstSymbol]: ast,
     asSql: () => astToSql(ast),
-    toString: () => astToSql(ast).query,
+    toString: () => {
+      try {
+        return astToSql(ast).query
+      } catch (cause) {
+        console.debug(`QueryBuilder.toString(): Error converting query builder to string`, cause, ast)
+        return `Error converting query builder to string`
+      }
+    },
     ...api,
   } satisfies QueryBuilder<TResult, TTableDef>
 }
@@ -197,9 +205,18 @@ const astToSql = (ast: QueryBuilderAst) => {
               if (colDef === undefined) {
                 throw new Error(`Column ${col} not found`)
               }
-              const encodedValue = Schema.encodeSync(colDef.schema)(value)
-              bindValues.push(encodedValue)
-              return `${col} ${op} ?`
+              const isArray = op === 'IN' || op === 'NOT IN'
+              const colSchema = isArray ? Schema.Array(colDef.schema) : colDef.schema
+              const encodedValue = Schema.encodeSync(colSchema)(value)
+
+              if (isArray) {
+                bindValues.push(...encodedValue)
+                const placeholders = Array.from({ length: encodedValue.length }, () => '?').join(', ')
+                return `${col} ${op} (${placeholders})`
+              } else {
+                bindValues.push(encodedValue)
+                return `${col} ${op} ?`
+              }
             }
           })
           .join(' AND ')}`

package/src/query-info.ts

@@ -1,5 +1,5 @@
 import type { SessionIdSymbol } from './adapter-types.js'
-import type { DbSchema } from './schema/index.js'
+import type { DbSchema } from './schema/mod.js'
 
 /**
  * Semantic information about a query with supported cases being:

package/src/rehydrate-from-mutationlog.ts

@@ -1,10 +1,10 @@
-import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
+import { isDevEnv, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 
 import { type MigrationOptionsFromMutationLog, type SynchronousDatabase, UnexpectedError } from './adapter-types.js'
 import { getExecArgsFromMutation } from './mutation.js'
-import type { LiveStoreSchema, MutationDef, MutationEvent, MutationLogMetaRow } from './schema/index.js'
-import { MUTATION_LOG_META_TABLE } from './schema/index.js'
+import type { LiveStoreSchema, MutationDef, MutationEvent, MutationLogMetaRow } from './schema/mod.js'
+import { EventId, MUTATION_LOG_META_TABLE } from './schema/mod.js'
 import type { PreparedBindValues } from './util.js'
 import { sql } from './util.js'
 
@@ -72,11 +72,7 @@ This likely means the schema has changed in an incompatible way.
 
         for (const { statementSql, bindValues } of execArgsArr) {
           // TODO cache prepared statements for mutations
-          db.execute(
-            statementSql,
-            bindValues,
-            import.meta.env.DEV ? makeExecuteOptions(statementSql, bindValues) : undefined,
-          )
+          db.execute(statementSql, bindValues, isDevEnv() ? makeExecuteOptions(statementSql, bindValues) : undefined)
           // console.log(`Re-executed mutation ${mutationSql}`, bindValues)
         }
       }).pipe(Effect.withSpan(`@livestore/common:rehydrateFromMutationLog:processMutation`))
@@ -85,7 +81,7 @@ This likely means the schema has changed in an incompatible way.
 
     const stmt = logDb.prepare(sql`\
 SELECT * FROM ${MUTATION_LOG_META_TABLE} 
-WHERE idGlobal > COALESCE($idGlobal, '') AND idLocal > COALESCE($idLocal, '')
+WHERE idGlobal > $idGlobal OR (idGlobal = $idGlobal AND idLocal > $idLocal)
 ORDER BY idGlobal ASC, idLocal ASC
 LIMIT ${CHUNK_SIZE}
 `)
@@ -101,9 +97,9 @@ LIMIT ${CHUNK_SIZE}
         const lastId = Chunk.isChunk(item)
           ? Chunk.last(item).pipe(
               Option.map((_) => ({ global: _.idGlobal, local: _.idLocal })),
-              Option.getOrUndefined,
+              Option.getOrElse(() => EventId.ROOT),
             )
-          : undefined
+          : EventId.ROOT
         const nextItem = Chunk.fromIterable(
           stmt.select<MutationLogMetaRow>({
             $idGlobal: lastId?.global,

package/src/schema/EventId.ts

@@ -0,0 +1,46 @@
+import { Schema } from '@livestore/utils/effect'
+
+/**
+ * LiveStore event id value consisting of a globally unique event sequence number
+ * and a local sequence number.
+ *
+ * The local sequence number is only used for localOnly mutations and starts from 0 for each global sequence number.
+ */
+export type EventId = { global: number; local: number }
+
+export const EventId = Schema.Struct({
+  global: Schema.Number,
+  local: Schema.Number,
+}).annotations({ title: 'LiveStore.EventId' })
+
+/**
+ * Compare two event ids i.e. checks if the first event id is less than the second.
+ */
+export const compare = (a: EventId, b: EventId) => {
+  if (a.global !== b.global) {
+    return a.global - b.global
+  }
+  return a.local - b.local
+}
+
+export const isEqual = (a: EventId, b: EventId) => a.global === b.global && a.local === b.local
+
+export type EventIdPair = { id: EventId; parentId: EventId }
+
+export const ROOT = { global: -1, local: 0 } satisfies EventId
+
+export const isGreaterThan = (a: EventId, b: EventId) => {
+  return a.global > b.global || (a.global === b.global && a.local > b.local)
+}
+
+export const nextPair = (id: EventId, isLocal: boolean) => {
+  if (isLocal) {
+    return { id: { global: id.global, local: id.local + 1 }, parentId: id }
+  }
+
+  return {
+    id: { global: id.global + 1, local: 0 },
+    // NOTE we always point to `local: 0` for non-localOnly mutations
+    parentId: { global: id.global, local: 0 },
+  }
+}