@livestore/common 0.4.0-dev.2 → 0.4.0-dev.21

package/src/leader-thread/eventlog.ts

@@ -1,23 +1,22 @@
 import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
-import { Effect, Option, Schema } from '@livestore/utils/effect'
-
+import { Chunk, Effect, Option, Schema } from '@livestore/utils/effect'
 import type { SqliteDb } from '../adapter-types.ts'
-import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
-import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
+import * as EventSequenceNumber from '../schema/EventSequenceNumber/mod.ts'
+import * as LiveStoreEvent from '../schema/LiveStoreEvent/mod.ts'
 import {
   EVENTLOG_META_TABLE,
   eventlogMetaTable,
   eventlogSystemTables,
   SYNC_STATUS_TABLE,
-  sessionChangesetMetaTable,
-} from '../schema/state/sqlite/system-tables.ts'
+} from '../schema/state/sqlite/system-tables/eventlog-tables.ts'
+import { sessionChangesetMetaTable } from '../schema/state/sqlite/system-tables/state-tables.ts'
 import { migrateTable } from '../schema-management/migrations.ts'
 import { insertRow, updateRows } from '../sql-queries/sql-queries.ts'
 import type { PreparedBindValues } from '../util.ts'
 import { sql } from '../util.ts'
 import { execSql } from './connection.ts'
-import type { InitialSyncInfo } from './types.ts'
-import { LeaderThreadCtx } from './types.ts'
+import type { InitialSyncInfo, StreamEventsOptions } from './types.ts'
+import { LeaderThreadCtx, STREAM_EVENTS_BATCH_SIZE_DEFAULT } from './types.ts'
 
 export const initEventlogDb = (dbEventlog: SqliteDb) =>
   Effect.gen(function* () {
@@ -34,7 +33,7 @@ export const initEventlogDb = (dbEventlog: SqliteDb) =>
     yield* execSql(
       dbEventlog,
       sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
-          SELECT ${EventSequenceNumber.ROOT.global}
+          SELECT ${EventSequenceNumber.Client.ROOT.global}
           WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
       {},
     )
@@ -51,8 +50,8 @@ export const getEventsSince = ({
 }: {
   dbEventlog: SqliteDb
   dbState: SqliteDb
-  since: EventSequenceNumber.EventSequenceNumber
-}): ReadonlyArray<LiveStoreEvent.EncodedWithMeta> => {
+  since: EventSequenceNumber.Client.Composite
+}): ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta> => {
   const pendingEvents = dbEventlog.select(eventlogMetaTable.where('seqNumGlobal', '>=', since.global))
 
   const sessionChangesetRowsDecoded = dbState.select(
@@ -66,7 +65,7 @@ export const getEventsSince = ({
           readModelEvent.seqNumGlobal === eventlogEvent.seqNumGlobal &&
           readModelEvent.seqNumClient === eventlogEvent.seqNumClient,
       )
-      return LiveStoreEvent.EncodedWithMeta.make({
+      return LiveStoreEvent.Client.EncodedWithMeta.make({
         name: eventlogEvent.name,
         args: eventlogEvent.argsJson,
         seqNum: {
@@ -96,14 +95,91 @@ export const getEventsSince = ({
         },
       })
     })
-    .filter((_) => EventSequenceNumber.compare(_.seqNum, since) > 0)
-    .sort((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+    .filter((_) => EventSequenceNumber.Client.compare(_.seqNum, since) > 0)
+    .sort((a, b) => EventSequenceNumber.Client.compare(a.seqNum, b.seqNum))
 }
 
-export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.EventSequenceNumber => {
+export const getEventsFromEventlog = ({
+  dbEventlog,
+  options,
+}: {
+  dbEventlog: SqliteDb
+  options: StreamEventsOptions
+}): Effect.Effect<Chunk.Chunk<LiveStoreEvent.Client.Encoded>> =>
+  Effect.gen(function* () {
+    const since = options.since ?? EventSequenceNumber.Client.ROOT
+    const batchSize = options.batchSize ?? STREAM_EVENTS_BATCH_SIZE_DEFAULT
+
+    const makeQuery = () => {
+      let query = eventlogMetaTable.where('seqNumGlobal', '>', since.global)
+
+      if (options.until) {
+        query = query.where('seqNumGlobal', '<=', options.until.global)
+      }
+
+      if (options.filter && options.filter.length > 0) {
+        query = query.where({ name: { op: 'IN', value: options.filter } })
+      }
+
+      if (options.clientIds && options.clientIds.length > 0) {
+        query = query.where({ clientId: { op: 'IN', value: options.clientIds } })
+      }
+
+      if (options.sessionIds && options.sessionIds.length > 0) {
+        query = query.where({ sessionId: { op: 'IN', value: options.sessionIds } })
+      }
+
+      if (options.includeClientOnly !== true) {
+        query = query.where('seqNumClient', '<=', EventSequenceNumber.Client.DEFAULT)
+      }
+
+      return query
+        .orderBy([
+          { col: 'seqNumGlobal', direction: 'asc' },
+          { col: 'seqNumClient', direction: 'asc' },
+        ])
+        .limit(batchSize)
+    }
+
+    const eventlogEvents = yield* Effect.sync(() => dbEventlog.select(makeQuery()))
+
+    if (eventlogEvents.length === 0) {
+      return Chunk.empty<LiveStoreEvent.Client.Encoded>()
+    }
+
+    const spanAttributes = {
+      'livestore.eventLog.since': since.global,
+      'livestore.eventLog.until': options.until?.global,
+    }
+
+    return yield* Effect.sync(() => {
+      const encodedEvents = eventlogEvents.map((eventlogEvent) => {
+        return LiveStoreEvent.Client.Encoded.make({
+          name: eventlogEvent.name,
+          args: eventlogEvent.argsJson,
+          seqNum: {
+            global: eventlogEvent.seqNumGlobal,
+            client: eventlogEvent.seqNumClient,
+            rebaseGeneration: eventlogEvent.seqNumRebaseGeneration,
+          },
+          parentSeqNum: {
+            global: eventlogEvent.parentSeqNumGlobal,
+            client: eventlogEvent.parentSeqNumClient,
+            rebaseGeneration: eventlogEvent.parentSeqNumRebaseGeneration,
+          },
+          clientId: eventlogEvent.clientId,
+          sessionId: eventlogEvent.sessionId,
+        })
+      })
+
+      return Chunk.fromIterable(encodedEvents)
+    }).pipe(Effect.withSpan('@livestore/common:eventlog:getEventsFromEventlog', { attributes: spanAttributes }))
+  })
+
+export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.Client.Composite => {
   const res = dbEventlog.select<{
-    seqNumGlobal: EventSequenceNumber.GlobalEventSequenceNumber
-    seqNumClient: EventSequenceNumber.ClientEventSequenceNumber
+    seqNumGlobal: EventSequenceNumber.Global.Type
+    seqNumClient: EventSequenceNumber.Client.Type
     seqNumRebaseGeneration: number
   }>(
     sql`select seqNumGlobal, seqNumClient, seqNumRebaseGeneration from ${EVENTLOG_META_TABLE} order by seqNumGlobal DESC, seqNumClient DESC limit 1`,
@@ -111,20 +187,27 @@ export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.E
 
   return res
     ? { global: res.seqNumGlobal, client: res.seqNumClient, rebaseGeneration: res.seqNumRebaseGeneration }
-    : EventSequenceNumber.ROOT
+    : EventSequenceNumber.Client.ROOT
 }
 
-export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.GlobalEventSequenceNumber =>
-  dbEventlog.select<{ head: EventSequenceNumber.GlobalEventSequenceNumber }>(
-    sql`select head from ${SYNC_STATUS_TABLE}`,
-  )[0]?.head ?? EventSequenceNumber.ROOT.global
+export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.Global.Type =>
+  dbEventlog.select<{ head: EventSequenceNumber.Global.Type }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ??
+  EventSequenceNumber.Client.ROOT.global
 
 // TODO use prepared statements
-export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.EventSequenceNumber) =>
+export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.Client.Composite) =>
   dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
 
+export const getBackendIdFromDb = (dbEventlog: SqliteDb): Option.Option<string> =>
+  Option.fromNullable(
+    dbEventlog.select<{ backendId: string | null }>(sql`select backendId from ${SYNC_STATUS_TABLE}`)[0]?.backendId,
+  )
+
+export const updateBackendId = (dbEventlog: SqliteDb, backendId: string) =>
+  dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET backendId = '${backendId}'`)
+
 export const insertIntoEventlog = (
-  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  eventEncoded: LiveStoreEvent.Client.EncodedWithMeta,
   dbEventlog: SqliteDb,
   eventDefSchemaHash: number,
   clientId: string,
@@ -132,7 +215,7 @@ export const insertIntoEventlog = (
 ) =>
   Effect.gen(function* () {
     // Check history consistency during LS_DEV
-    if (LS_DEV && eventEncoded.parentSeqNum.global !== EventSequenceNumber.ROOT.global) {
+    if (LS_DEV && eventEncoded.parentSeqNum.global !== EventSequenceNumber.Client.ROOT.global) {
       const parentEventExists =
         dbEventlog.select<{ count: number }>(
           `SELECT COUNT(*) as count FROM ${EVENTLOG_META_TABLE} WHERE seqNumGlobal = ? AND seqNumClient = ?`,
@@ -141,7 +224,7 @@ export const insertIntoEventlog = (
 
       if (parentEventExists === false) {
         shouldNeverHappen(
-          `Parent mutation ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist`,
+          `Parent event ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist in eventlog`,
         )
       }
     }
@@ -172,7 +255,7 @@ export const insertIntoEventlog = (
     dbEventlog.debug.head = eventEncoded.seqNum
   })
 
-export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>) =>
+export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>) =>
   Effect.gen(function* () {
     const { dbEventlog } = yield* LeaderThreadCtx
 
@@ -192,15 +275,11 @@ export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.EncodedWi
     }
   })
 
-export const getSyncBackendCursorInfo = ({
-  remoteHead,
-}: {
-  remoteHead: EventSequenceNumber.GlobalEventSequenceNumber
-}) =>
+export const getSyncBackendCursorInfo = ({ remoteHead }: { remoteHead: EventSequenceNumber.Global.Type }) =>
   Effect.gen(function* () {
     const { dbEventlog } = yield* LeaderThreadCtx
 
-    if (remoteHead === EventSequenceNumber.ROOT.global) return Option.none()
+    if (remoteHead === EventSequenceNumber.Client.ROOT.global) return Option.none()
 
     const EventlogQuerySchema = Schema.Struct({
       syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
@@ -213,11 +292,7 @@ export const getSyncBackendCursorInfo = ({
     ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
-      cursor: {
-        global: remoteHead,
-        client: EventSequenceNumber.clientDefault,
-        rebaseGeneration: EventSequenceNumber.rebaseGenerationDefault,
-      },
+      eventSequenceNumber: remoteHead,
       metadata: syncMetadataOption,
     }) satisfies InitialSyncInfo
   }).pipe(Effect.withSpan('@livestore/common:eventlog:getSyncBackendCursorInfo', { attributes: { remoteHead } }))

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,7 +1,7 @@
 import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
 
-import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnexpectedError } from '../index.ts'
+import { Devtools, IntentionalShutdownCause, liveStoreVersion, UnknownError } from '../index.ts'
 import { SystemTables } from '../schema/mod.ts'
 import type { DevtoolsOptions, PersistenceInfoPair } from './types.ts'
 import { LeaderThreadCtx } from './types.ts'
@@ -94,6 +94,22 @@ const listenToDevtools = ({
     type RequestId = string
     const handledRequestIds = new Set<RequestId>()
 
+    type LoadDatabaseKind = 'state' | 'eventlog'
+    const loadDatabaseBatchTracker = new Map<string, Set<LoadDatabaseKind>>()
+
+    const registerBatchProgress = (batchId: string, kind: LoadDatabaseKind) => {
+      const entry = loadDatabaseBatchTracker.get(batchId) ?? new Set<LoadDatabaseKind>()
+      entry.add(kind)
+      loadDatabaseBatchTracker.set(batchId, entry)
+      const finished = entry.has('state') && entry.has('eventlog')
+
+      if (finished) {
+        loadDatabaseBatchTracker.delete(batchId)
+      }
+
+      return finished
+    }
+
     yield* incomingMessages.pipe(
       Stream.tap((decodedEvent) =>
         Effect.gen(function* () {
@@ -133,74 +149,85 @@ const listenToDevtools = ({
               return
             }
             case 'LSD.Leader.LoadDatabaseFile.Request': {
-              const { data } = decodedEvent
-
-              let tableNames: Set<string>
+              const { data, batchId } = decodedEvent
 
-              try {
-                const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
-                tmpDb.import(data)
-                const tableNameResults = tmpDb.select<{ name: string }>(
-                  `select name from sqlite_master where type = 'table'`,
-                )
-
-                tableNames = new Set(tableNameResults.map((_) => _.name))
-
-                tmpDb.close()
-              } catch (cause) {
-                yield* Effect.logError(`Error importing database file`, cause)
-                yield* sendMessage(
-                  Devtools.Leader.LoadDatabaseFile.Error.make({
-                    ...reqPayload,
-                    cause: { _tag: 'unexpected-error', cause },
-                  }),
+              const handleLoadDb = Effect.gen(function* () {
+                const tableNames = yield* Effect.acquireRelease(makeSqliteDb({ _tag: 'in-memory' }), (db) =>
+                  Effect.sync(() => db.close()),
+                ).pipe(
+                  Effect.flatMap((db) =>
+                    Effect.try(() => {
+                      db.import(data)
+                      const rows = db.select<{ name: string }>(`select name from sqlite_master where type = 'table'`)
+                      return new Set(rows.map((r) => r.name))
+                    }),
+                  ),
                 )
 
-                return
-              }
+                let databaseKind: LoadDatabaseKind | undefined
 
-              try {
                 if (tableNames.has(SystemTables.EVENTLOG_META_TABLE)) {
-                  // Is eventlog db
+                  databaseKind = 'eventlog'
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+                  yield* Effect.try(() => void dbEventlog.import(data))
 
-                  dbEventlog.import(data)
-
-                  dbState.destroy()
+                  if (batchId === undefined) {
+                    yield* Effect.try(() => void dbState.destroy())
+                  }
                 } else if (
                   tableNames.has(SystemTables.SCHEMA_META_TABLE) &&
                   tableNames.has(SystemTables.SCHEMA_EVENT_DEFS_META_TABLE)
                 ) {
-                  // Is state db
+                  databaseKind = 'state'
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
+                  yield* Effect.try(() => void dbState.import(data))
 
-                  dbState.import(data)
-
-                  dbEventlog.destroy()
+                  if (batchId === undefined) {
+                    yield* Effect.try(() => void dbEventlog.destroy())
+                  }
                 } else {
-                  yield* sendMessage(
-                    Devtools.Leader.LoadDatabaseFile.Error.make({
-                      ...reqPayload,
-                      cause: { _tag: 'unsupported-database' },
-                    }),
-                  )
-                  return
+                  return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
+                }
+
+                const resolvedDatabaseKind = databaseKind
+                if (resolvedDatabaseKind === undefined) {
+                  return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
                 }
 
+                const shouldShutdown =
+                  batchId === undefined ? true : registerBatchProgress(batchId, resolvedDatabaseKind)
+
                 yield* sendMessage(Devtools.Leader.LoadDatabaseFile.Success.make({ ...reqPayload }))
-                yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' })) ?? Effect.void
 
-                return
-              } catch (cause) {
-                yield* Effect.logError(`Error importing database file`, cause)
-                yield* sendMessage(
-                  Devtools.Leader.LoadDatabaseFile.Error.make({
-                    ...reqPayload,
-                    cause: { _tag: 'unexpected-error', cause },
-                  }),
-                )
-                return
-              }
+                if (shouldShutdown) {
+                  yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-import' }))
+                }
+              })
+
+              yield* handleLoadDb.pipe(
+                Effect.catchTag('unsupported-database', () =>
+                  sendMessage(
+                    Devtools.Leader.LoadDatabaseFile.Error.make({
+                      ...reqPayload,
+                      cause: { _tag: 'unsupported-database' as const },
+                    }),
+                  ),
+                ),
+                Effect.catchAll((cause) =>
+                  Effect.logWarning('Error importing database file', cause).pipe(
+                    Effect.zipRight(
+                      sendMessage(
+                        Devtools.Leader.LoadDatabaseFile.Error.make({
+                          ...reqPayload,
+                          cause: { _tag: 'unknown-error' as const, cause },
+                        }),
+                      ),
+                    ),
+                  ),
+                ),
+              )
+
+              return
             }
             case 'LSD.Leader.ResetAllData.Request': {
               const { mode } = decodedEvent
@@ -215,7 +242,7 @@ const listenToDevtools = ({
 
               yield* sendMessage(Devtools.Leader.ResetAllData.Success.make({ ...reqPayload }))
 
-              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' })) ?? Effect.void
+              yield* shutdownChannel.send(IntentionalShutdownCause.make({ reason: 'devtools-reset' }))
 
               return
             }
@@ -262,7 +289,7 @@ const listenToDevtools = ({
 
               if (syncBackend !== undefined) {
                 // TODO consider piggybacking on the existing leader-thread sync-pulling
-                yield* syncBackend.pull(Option.none()).pipe(
+                yield* syncBackend.pull(Option.none(), { live: true }).pipe(
                   Stream.map((_) => _.batch),
                   Stream.flattenIterables,
                   Stream.tap(({ eventEncoded, metadata }) =>
@@ -319,7 +346,11 @@ const listenToDevtools = ({
                   Stream.tap(([isConnected, { latchClosed }]) =>
                     sendMessage(
                       Devtools.Leader.NetworkStatusRes.make({
-                        networkStatus: { isConnected, timestampMs: Date.now(), latchClosed },
+                        networkStatus: {
+                          isConnected,
+                          timestampMs: Date.now(),
+                          devtools: { latchClosed },
+                        },
                         subscriptionId,
                         ...reqPayload,
                         requestId: nanoid(10),
@@ -395,7 +426,7 @@ const listenToDevtools = ({
           }
         }).pipe(Effect.withSpan(`@livestore/common:leader-thread:onDevtoolsMessage:${decodedEvent._tag}`)),
       ),
-      UnexpectedError.mapToUnexpectedErrorStream,
+      UnknownError.mapToUnknownErrorStream,
       Stream.runDrain,
     )
   })

package/src/leader-thread/make-leader-thread-layer.test.ts

@@ -0,0 +1,44 @@
+import { Effect, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Vitest } from '@livestore/utils-dev/node-vitest'
+
+import { makeMockSyncBackend } from '../sync/mock-sync-backend.ts'
+import type { SyncBackend } from '../sync/sync.ts'
+import { makeNetworkStatusSubscribable } from './make-leader-thread-layer.ts'
+import type { DevtoolsContext } from './types.ts'
+
+Vitest.describe('makeNetworkStatusSubscribable', () => {
+  Vitest.scopedLive('tracks sync backend connectivity and devtools latch state', () =>
+    Effect.gen(function* () {
+      const mockBackend = yield* makeMockSyncBackend({ startConnected: false })
+      const syncBackend = yield* mockBackend.makeSyncBackend
+      const latchStateRef = yield* SubscriptionRef.make<{ latchClosed: boolean }>({ latchClosed: false })
+
+      const devtoolsContext: DevtoolsContext = {
+        enabled: true,
+        syncBackendLatch: yield* Effect.makeLatch(true),
+        syncBackendLatchState: latchStateRef,
+      }
+
+      const networkStatus = yield* makeNetworkStatusSubscribable({ syncBackend, devtoolsContext })
+
+      const initial = yield* networkStatus
+      Vitest.expect(initial.isConnected).toBe(false)
+      Vitest.expect(initial.devtools.latchClosed).toBe(false)
+
+      const waitFor = (predicate: (status: SyncBackend.NetworkStatus) => boolean) =>
+        networkStatus.changes.pipe(Stream.filter(predicate), Stream.runHead, Effect.flatten)
+
+      const onlineFiber = yield* waitFor((status) => status.isConnected).pipe(Effect.forkScoped)
+      yield* mockBackend.connect
+      const online = yield* onlineFiber
+      Vitest.expect(online.isConnected).toBe(true)
+      Vitest.expect(online.timestampMs).toBeGreaterThan(initial.timestampMs)
+
+      const latchedFiber = yield* waitFor((status) => status.devtools.latchClosed).pipe(Effect.forkScoped)
+      yield* SubscriptionRef.set(latchStateRef, { latchClosed: true })
+      const latched = yield* latchedFiber
+      Vitest.expect(latched.devtools.latchClosed).toBe(true)
+      Vitest.expect(latched.timestampMs).toBeGreaterThanOrEqual(online.timestampMs)
+    }),
+  )
+})