@livestore/common 0.4.0 → 0.5.0-dev.1

package/src/leader-thread/RejectedPushError.ts

@@ -1,7 +1,7 @@
 /**
  * Push validation errors returned by {@link LeaderSyncProcessor.push}.
  *
- * All three errors share a common {@link RejectedPushErrorTypeId} so consumers can catch the
+ * All errors share a common {@link RejectedPushErrorTypeId} so consumers can catch the
  * family as a group via {@link isRejectedPushError} instead of matching individual tags.
  * Recovery is the same in every case: the client should rebase and retry.
  *
@@ -23,19 +23,16 @@ export const RejectedPushErrorTypeId = '~@livestore/common/RejectedPushError' as
  */
 export class NonMonotonicBatchError extends Schema.TaggedError<NonMonotonicBatchError>(
   `${RejectedPushErrorTypeId}/NonMonotonicBatchError`,
-)(
-  'NonMonotonicBatchError',
-  {
-    /** The sequence number that broke the monotonic invariant (i.e. the one that is >= the next). */
-    precedingSeqNum: EventSequenceNumber.Client.Composite,
-    /** The sequence number that was expected to be greater than `precedingSeqNum`. */
-    violatingSeqNum: EventSequenceNumber.Client.Composite,
-    /** The index in the batch where the violation occurred. */
-    violationIndex: Schema.Number,
-    /** The session that produced the malformed batch. */
-    sessionId: Schema.String,
-  },
-) {
+)('NonMonotonicBatchError', {
+  /** The sequence number that broke the monotonic invariant (i.e. the one that is >= the next). */
+  precedingSeqNum: EventSequenceNumber.Client.Composite,
+  /** The sequence number that was expected to be greater than `precedingSeqNum`. */
+  violatingSeqNum: EventSequenceNumber.Client.Composite,
+  /** The index in the batch where the violation occurred. */
+  violationIndex: Schema.Finite,
+  /** The session that produced the malformed batch. */
+  sessionId: Schema.String,
+}) {
   readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
 
   override get message(): string {
@@ -43,6 +40,24 @@ export class NonMonotonicBatchError extends Schema.TaggedError<NonMonotonicBatch
   }
 }
 
+/** A pushed batch skips an event required to connect it to the leader's admitted push prefix. */
+export class NonContiguousBatchError extends Schema.TaggedError<NonContiguousBatchError>(
+  `${RejectedPushErrorTypeId}/NonContiguousBatchError`,
+)('NonContiguousBatchError', {
+  expectedSeqNum: EventSequenceNumber.Client.Composite,
+  providedSeqNum: EventSequenceNumber.Client.Composite,
+  expectedParentSeqNum: EventSequenceNumber.Client.Composite,
+  providedParentSeqNum: EventSequenceNumber.Client.Composite,
+  violationIndex: Schema.Finite,
+  sessionId: Schema.String,
+}) {
+  readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
+
+  override get message(): string {
+    return `Pushed events are not contiguous at index ${this.violationIndex} (session ${this.sessionId}): expected ${EventSequenceNumber.Client.toString(this.expectedSeqNum)} after ${EventSequenceNumber.Client.toString(this.expectedParentSeqNum)}, got ${EventSequenceNumber.Client.toString(this.providedSeqNum)} after ${EventSequenceNumber.Client.toString(this.providedParentSeqNum)}`
+  }
+}
+
 /**
  * A pushed batch of events failed validation because its rebase generation is older than the leader's current rebase generation.
  *
@@ -52,17 +67,14 @@ export class NonMonotonicBatchError extends Schema.TaggedError<NonMonotonicBatch
  */
 export class StaleRebaseGenerationError extends Schema.TaggedError<StaleRebaseGenerationError>(
   `${RejectedPushErrorTypeId}/StaleRebaseGenerationError`,
-)(
-  'StaleRebaseGenerationError',
-  {
-    /** The leader's current rebase generation. */
-    currentRebaseGeneration: Schema.Number,
-    /** The rebase generation carried by the dropped events. */
-    providedRebaseGeneration: Schema.Number,
-    /** The session that produced the stale batch. */
-    sessionId: Schema.String,
-  },
-) {
+)('StaleRebaseGenerationError', {
+  /** The leader's current rebase generation. */
+  currentRebaseGeneration: Schema.Finite,
+  /** The rebase generation carried by the dropped events. */
+  providedRebaseGeneration: Schema.Finite,
+  /** The session that produced the stale batch. */
+  sessionId: Schema.String,
+}) {
   readonly [RejectedPushErrorTypeId] = RejectedPushErrorTypeId
 
   override get message(): string {
@@ -94,11 +106,12 @@ export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>(
   }
 }
 
-export const RejectedPushError = Schema.Union(
+export const RejectedPushError = Schema.Union([
   LeaderAheadError,
+  NonContiguousBatchError,
   NonMonotonicBatchError,
   StaleRebaseGenerationError,
-)
+])
 
 export type RejectedPushError = typeof RejectedPushError.Type
 

package/src/leader-thread/connection.ts

@@ -83,14 +83,18 @@ export const execSql = (sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
   )
 }
 
-// const selectSqlPrepared = <T>(stmt: PreparedStatement, bind: BindValues) => {
-//   const bindValues = prepareBindValues(bind, stmt.sql)
-//   return Effect.try({
-//     try: () => stmt.select<T>(bindValues),
-//     catch: (cause) =>
-//       new SqliteError({ cause, query: { bindValues, sql: stmt.sql }, code: (cause as WaSqlite.SQLiteError).code }),
-//   })
-// }
+export const selectSql = <T>(sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
+  const bindValues = prepareBindValues(bind, sql)
+  return Effect.try({
+    try: () => sqliteDb.select<T>(sql, bindValues),
+    catch: (cause) =>
+      new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
+  }).pipe(
+    Effect.withSpan(`@livestore/common:selectSql`, {
+      attributes: { 'span.label': sql, sql, bindValueKeys: Object.keys(bindValues) },
+    }),
+  )
+}
 
 // TODO actually use prepared statements
 export const execSqlPrepared = (sqliteDb: SqliteDb, sql: string, bindValues: PreparedBindValues) => {

package/src/leader-thread/eventlog.ts

@@ -1,5 +1,5 @@
 import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
-import { Chunk, Effect, Option, Schema } from '@livestore/utils/effect'
+import { Effect, Option, Schema } from '@livestore/utils/effect'
 
 import type { SqliteDb } from '../adapter-types.ts'
 import { migrateTable } from '../schema-management/migrations.ts'
@@ -107,7 +107,7 @@ export const getEventsFromEventlog = ({
 }: {
   dbEventlog: SqliteDb
   options: StreamEventsOptions
-}): Effect.Effect<Chunk.Chunk<LiveStoreEvent.Client.Encoded>> =>
+}): Effect.Effect<ReadonlyArray<LiveStoreEvent.Client.Encoded>> =>
   Effect.gen(function* () {
     const since = options.since ?? EventSequenceNumber.Client.ROOT
     const batchSize = options.batchSize ?? STREAM_EVENTS_BATCH_SIZE_DEFAULT
@@ -146,7 +146,7 @@ export const getEventsFromEventlog = ({
     const eventlogEvents = yield* Effect.sync(() => dbEventlog.select(makeQuery()))
 
     if (eventlogEvents.length === 0) {
-      return Chunk.empty<LiveStoreEvent.Client.Encoded>()
+      return []
     }
 
     const spanAttributes = {
@@ -174,7 +174,7 @@ export const getEventsFromEventlog = ({
         })
       })
 
-      return Chunk.fromIterable(encodedEvents)
+      return encodedEvents
     }).pipe(Effect.withSpan('@livestore/common:eventlog:getEventsFromEventlog', { attributes: spanAttributes }))
   })
 
@@ -201,7 +201,7 @@ export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumbe
   dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
 
 export const getBackendIdFromDb = (dbEventlog: SqliteDb): Option.Option<string> =>
-  Option.fromNullable(
+  Option.fromNullishOr(
     dbEventlog.select<{ backendId: string | null }>(sql`select backendId from ${SYNC_STATUS_TABLE}`)[0]?.backendId,
   )
 
@@ -258,9 +258,13 @@ export const insertIntoEventlog = (
   })
 
 export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>) =>
-  Effect.gen(function* () {
-    const { dbEventlog } = yield* LeaderThreadCtx
+  LeaderThreadCtx.pipe(Effect.flatMap(({ dbEventlog }) => updateSyncMetadataForDb(dbEventlog, items)))
 
+export const updateSyncMetadataForDb = (
+  dbEventlog: SqliteDb,
+  items: ReadonlyArray<LiveStoreEvent.Client.EncodedWithMeta>,
+) =>
+  Effect.gen(function* () {
     // TODO try to do this in a single query
     for (let i = 0; i < items.length; i++) {
       const event = items[i]!
@@ -277,21 +281,25 @@ export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.Client.En
     }
   })
 
-export const getSyncBackendCursorInfo = ({ remoteHead }: { remoteHead: EventSequenceNumber.Global.Type }) =>
-  Effect.gen(function* () {
-    const { dbEventlog } = yield* LeaderThreadCtx
+export const getSyncBackendCursorInfo = (args: { remoteHead: EventSequenceNumber.Global.Type }) =>
+  LeaderThreadCtx.pipe(Effect.flatMap(({ dbEventlog }) => getSyncBackendCursorInfoForDb(dbEventlog, args)))
 
+export const getSyncBackendCursorInfoForDb = (
+  dbEventlog: SqliteDb,
+  { remoteHead }: { remoteHead: EventSequenceNumber.Global.Type },
+) =>
+  Effect.gen(function* () {
     if (remoteHead === EventSequenceNumber.Client.ROOT.global) return Option.none()
 
     const EventlogQuerySchema = Schema.Struct({
-      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
+      syncMetadataJson: Schema.fromJsonString(Schema.toCodecJson(Schema.Option(Schema.Json))),
     }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
 
     const syncMetadataOption = yield* Effect.sync(() =>
       dbEventlog.select<{ syncMetadataJson: string }>(
         sql`SELECT syncMetadataJson FROM ${EVENTLOG_META_TABLE} WHERE seqNumGlobal = ${remoteHead} ORDER BY seqNumClient ASC LIMIT 1`,
       ),
-    ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
+    ).pipe(Effect.andThen(Schema.decodeEffect(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
 
     return Option.some({
       eventSequenceNumber: remoteHead,

package/src/leader-thread/leader-worker-devtools.ts

@@ -1,4 +1,4 @@
-import { Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Cause, Effect, FiberMap, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
 import { nanoid } from '@livestore/utils/nanoid'
 
 import {
@@ -18,7 +18,8 @@ type SendMessageToDevtools = (message: Devtools.Leader.MessageFromApp) => Effect
 
 /**
  * Type guard for DevtoolsViteNotInstalledError.
- * This error is defined in @livestore/adapter-node but we need to handle it here.
+ * Adapter-specific devtools boot implementations may surface this tagged error;
+ * common handles it structurally to keep devtools startup environment-agnostic.
  */
 const isDevtoolsViteNotInstalledError = (
   error: unknown,
@@ -41,13 +42,13 @@ export const bootDevtools = Effect.fn('@livestore/common:leader-thread:devtools:
   }).pipe(Effect.tapCauseLogPretty, Effect.forkScoped)
 
   const bootResult = yield* options.boot.pipe(
-    Effect.map(Option.some),
+    Effect.asSome,
     Effect.catchIf(isDevtoolsViteNotInstalledError, (error) =>
       Effect.logWarning(`[@livestore/devtools] ${error.message} Devtools will be disabled.`).pipe(
         Effect.as(Option.none()),
       ),
     ),
-    Effect.catchAllCause((cause) =>
+    Effect.catchCause((cause) =>
       Effect.logWarning(
         `[@livestore/devtools] Failed to start devtools server. Devtools will be disabled.`,
         cause,
@@ -80,7 +81,7 @@ export const bootDevtools = Effect.fn('@livestore/common:leader-thread:devtools:
             .pipe(
               Effect.withSpan('@livestore/common:leader-thread:devtools:sendToDevtools'),
               Effect.interruptible,
-              Effect.ignoreLogged,
+              Effect.ignore,
             )
 
         const syncState = yield* syncProcessor.syncState
@@ -92,7 +93,7 @@ export const bootDevtools = Effect.fn('@livestore/common:leader-thread:devtools:
         )
 
         yield* listenToDevtools({
-          incomingMessages: channel.listen.pipe(Stream.flatten(), Stream.orDie),
+          incomingMessages: channel.listen.pipe(Stream.mapEffect(Effect.fromResult), Stream.orDie),
           sendMessage,
           persistenceInfo,
         })
@@ -181,7 +182,9 @@ const listenToDevtools = ({
                     appVersion: liveStoreVersion,
                     receivedVersion: decodedEvent.liveStoreVersion,
                     appDevtoolsProtocolVersion: devtoolsProtocolVersion,
-                    receivedDevtoolsProtocolVersion: resolveDevtoolsProtocolVersion(decodedEvent.devtoolsProtocolVersion),
+                    receivedDevtoolsProtocolVersion: resolveDevtoolsProtocolVersion(
+                      decodedEvent.devtoolsProtocolVersion,
+                    ),
                   }),
                 )
                 return
@@ -204,10 +207,13 @@ const listenToDevtools = ({
                   Effect.sync(() => db.close()),
                 ).pipe(
                   Effect.flatMap((db) =>
-                    Effect.try(() => {
-                      db.import(data)
-                      const rows = db.select<{ name: string }>(`select name from sqlite_master where type = 'table'`)
-                      return new Set(rows.map((r) => r.name))
+                    Effect.try({
+                      try: () => {
+                        db.import(data)
+                        const rows = db.select<{ name: string }>(`select name from sqlite_master where type = 'table'`)
+                        return new Set(rows.map((r) => r.name))
+                      },
+                      catch: (cause) => new Cause.UnknownError(cause),
                     }),
                   ),
                 )
@@ -217,10 +223,16 @@ const listenToDevtools = ({
                 if (tableNames.has(SystemTables.EVENTLOG_META_TABLE) === true) {
                   databaseKind = 'eventlog'
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
-                  yield* Effect.try(() => dbEventlog.import(data))
+                  yield* Effect.try({
+                    try: () => dbEventlog.import(data),
+                    catch: (cause) => new Cause.UnknownError(cause),
+                  })
 
                   if (batchId === undefined) {
-                    yield* Effect.try(() => dbState.destroy())
+                    yield* Effect.try({
+                      try: () => dbState.destroy(),
+                      catch: (cause) => new Cause.UnknownError(cause),
+                    })
                   }
                 } else if (
                   tableNames.has(SystemTables.SCHEMA_META_TABLE) === true &&
@@ -228,10 +240,16 @@ const listenToDevtools = ({
                 ) {
                   databaseKind = 'state'
                   yield* SubscriptionRef.set(shutdownStateSubRef, 'shutting-down')
-                  yield* Effect.try(() => dbState.import(data))
+                  yield* Effect.try({
+                    try: () => dbState.import(data),
+                    catch: (cause) => new Cause.UnknownError(cause),
+                  })
 
                   if (batchId === undefined) {
-                    yield* Effect.try(() => dbEventlog.destroy())
+                    yield* Effect.try({
+                      try: () => dbEventlog.destroy(),
+                      catch: (cause) => new Cause.UnknownError(cause),
+                    })
                   }
                 } else {
                   return yield* Effect.fail({ _tag: 'unsupported-database' } as const)
@@ -261,9 +279,9 @@ const listenToDevtools = ({
                     }),
                   ),
                 ),
-                Effect.catchAll((cause) =>
+                Effect.catch((cause) =>
                   Effect.logWarning('Error importing database file', cause).pipe(
-                    Effect.zipRight(
+                    Effect.andThen(
                       sendMessage(
                         Devtools.Leader.LoadDatabaseFile.Error.make({
                           ...reqPayload,
@@ -339,7 +357,7 @@ const listenToDevtools = ({
                 // TODO consider piggybacking on the existing leader-thread sync-pulling
                 yield* syncBackend.pull(Option.none(), { live: true }).pipe(
                   Stream.map((_) => _.batch),
-                  Stream.flattenIterables,
+                  Stream.flattenIterable,
                   Stream.tap(({ eventEncoded, metadata }) =>
                     sendMessage(
                       Devtools.Leader.SyncHistoryRes.make({
@@ -388,9 +406,9 @@ const listenToDevtools = ({
                 yield* Effect.sleep(1000)
 
                 yield* Stream.zipLatest(
-                  syncBackend.isConnected.changes,
+                  SubscriptionRef.changes(syncBackend.isConnected),
                   devtools.enabled === true
-                    ? devtools.syncBackendLatchState.changes
+                    ? SubscriptionRef.changes(devtools.syncBackendLatchState)
                     : Stream.make({ latchClosed: false }),
                 ).pipe(
                   Stream.tap(([isConnected, { latchClosed }]) =>

package/src/leader-thread/make-leader-thread-layer.test.ts

@@ -1,5 +1,5 @@
 import { Vitest } from '@livestore/utils-dev/node-vitest'
-import { Effect, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import { Effect, Fiber, Latch, Stream, SubscriptionRef } from '@livestore/utils/effect'
 
 import { makeMockSyncBackend } from '../sync/mock-sync-backend.ts'
 import type { SyncBackend } from '../sync/sync.ts'
@@ -7,7 +7,7 @@ import { makeNetworkStatusSubscribable } from './make-leader-thread-layer.ts'
 import type { DevtoolsContext } from './types.ts'
 
 Vitest.describe('makeNetworkStatusSubscribable', () => {
-  Vitest.scopedLive('tracks sync backend connectivity and devtools latch state', () =>
+  Vitest.live('tracks sync backend connectivity and devtools latch state', () =>
     Effect.gen(function* () {
       const mockBackend = yield* makeMockSyncBackend({ startConnected: false })
       const syncBackend = yield* mockBackend.makeSyncBackend
@@ -15,7 +15,7 @@ Vitest.describe('makeNetworkStatusSubscribable', () => {
 
       const devtoolsContext: DevtoolsContext = {
         enabled: true,
-        syncBackendLatch: yield* Effect.makeLatch(true),
+        syncBackendLatch: yield* Latch.make(true),
         syncBackendLatchState: latchStateRef,
       }
 
@@ -26,17 +26,17 @@ Vitest.describe('makeNetworkStatusSubscribable', () => {
       Vitest.expect(initial.devtools.latchClosed).toBe(false)
 
       const waitFor = (predicate: (status: SyncBackend.NetworkStatus) => boolean) =>
-        networkStatus.changes.pipe(Stream.filter(predicate), Stream.runHead, Effect.flatten)
+        networkStatus.changes.pipe(Stream.filter(predicate), Stream.runFirstUnsafe)
 
       const onlineFiber = yield* waitFor((status) => status.isConnected).pipe(Effect.forkScoped)
       yield* mockBackend.connect
-      const online = yield* onlineFiber
+      const online = yield* Fiber.join(onlineFiber)
       Vitest.expect(online.isConnected).toBe(true)
-      Vitest.expect(online.timestampMs).toBeGreaterThan(initial.timestampMs)
+      Vitest.expect(online.timestampMs).toBeGreaterThanOrEqual(initial.timestampMs)
 
       const latchedFiber = yield* waitFor((status) => status.devtools.latchClosed).pipe(Effect.forkScoped)
       yield* SubscriptionRef.set(latchStateRef, { latchClosed: true })
-      const latched = yield* latchedFiber
+      const latched = yield* Fiber.join(latchedFiber)
       Vitest.expect(latched.devtools.latchClosed).toBe(true)
       Vitest.expect(latched.timestampMs).toBeGreaterThanOrEqual(online.timestampMs)
     }),