@livestore/common 0.3.0-dev.11 → 0.3.0-dev.13

package/src/adapter-types.ts

@@ -2,8 +2,7 @@ import type { Cause, Queue, Scope, SubscriptionRef, WebChannel } from '@livestor
 import { Effect, Schema, Stream } from '@livestore/utils/effect'
 
 import type * as Devtools from './devtools/index.js'
-import type { EventId } from './schema/EventId.js'
-import type { LiveStoreSchema, MutationEvent } from './schema/mod.js'
+import type { EventId, LiveStoreSchema, MutationEvent } from './schema/mod.js'
 import type { InvalidPushError } from './sync/sync.js'
 import type { PayloadUpstream, SyncState } from './sync/syncstate.js'
 import type { PreparedBindValues } from './util.js'
@@ -26,7 +25,7 @@ export type SqliteDbChangeset = {
   apply: () => void
 }
 
-export type ClientSession = {
+export interface ClientSession {
   /** SQLite database with synchronous API running in the same thread (usually in-memory) */
   sqliteDb: SqliteDb
   devtools: { enabled: false } | { enabled: true; pullLatch: Effect.Latch; pushLatch: Effect.Latch }
@@ -39,18 +38,24 @@ export type ClientSession = {
   leaderThread: ClientSessionLeaderThreadProxy
 }
 
-export type ClientSessionLeaderThreadProxy = {
+export interface ClientSessionLeaderThreadProxy {
   mutations: {
     pull: Stream.Stream<{ payload: PayloadUpstream; remaining: number }, UnexpectedError>
     push(batch: ReadonlyArray<MutationEvent.AnyEncoded>): Effect.Effect<void, UnexpectedError | InvalidPushError>
-    initialMutationEventId: EventId
+  }
+  /** The initial state after the leader thread has booted */
+  readonly initialState: {
+    /** The latest mutation event id during boot. Used for the client session to resume syncing. */
+    readonly leaderHead: EventId.EventId
+    /** The migrations report from the leader thread */
+    readonly migrationsReport: MigrationsReport
   }
   export: Effect.Effect<Uint8Array, UnexpectedError>
   getMutationLogData: Effect.Effect<Uint8Array, UnexpectedError>
   getSyncState: Effect.Effect<SyncState, UnexpectedError>
   networkStatus: SubscriptionRef.SubscriptionRef<NetworkStatus>
   /** For debugging purposes it can be useful to manually trigger devtools messages (e.g. to reset the database) */
-  sendDevtoolsMessage: (message: Devtools.MessageToAppLeader) => Effect.Effect<void, UnexpectedError>
+  sendDevtoolsMessage: (message: Devtools.Leader.MessageToApp) => Effect.Effect<void, UnexpectedError>
 }
 
 /**
@@ -58,7 +63,7 @@ export type ClientSessionLeaderThreadProxy = {
  * Always assumes a synchronous SQLite build with the `bytecode` and `session` extensions enabled.
  * Can be either in-memory or persisted to disk.
  */
-export type SqliteDb<TReq = any, TMetadata extends TReq = TReq> = {
+export interface SqliteDb<TReq = any, TMetadata extends TReq = TReq> {
   _tag: 'SqliteDb'
   metadata: TMetadata
   prepare(queryStr: string): PreparedStatement
@@ -102,12 +107,11 @@ export type ResetMode = 'all-data' | 'only-app-db'
 export const NetworkStatus = Schema.Struct({
   isConnected: Schema.Boolean,
   timestampMs: Schema.Number,
+  /** Whether the network status devtools latch is closed. Used to simulate network disconnection. */
+  latchClosed: Schema.Boolean,
 })
 
-export type NetworkStatus = {
-  isConnected: boolean
-  timestampMs: number
-}
+export type NetworkStatus = typeof NetworkStatus.Type
 
 export const BootStateProgress = Schema.Struct({
   done: Schema.Number,
@@ -199,7 +203,7 @@ export type MigrationHooks = {
 
 export type MigrationHook = (db: SqliteDb) => void | Promise<void> | Effect.Effect<void, unknown>
 
-export type MigrationOptionsFromMutationLog<TSchema extends LiveStoreSchema = LiveStoreSchema> = {
+export interface MigrationOptionsFromMutationLog<TSchema extends LiveStoreSchema = LiveStoreSchema> {
   strategy: 'from-mutation-log'
   /**
    * Mutations to exclude in the mutation log
@@ -213,13 +217,11 @@ export type MigrationOptionsFromMutationLog<TSchema extends LiveStoreSchema = Li
   }
 }
 
-export type StoreDevtoolsChannel = WebChannel.WebChannel<
-  Devtools.MessageToAppClientSession,
-  Devtools.MessageFromAppClientSession
->
+export interface ClientSessionDevtoolsChannel
+  extends WebChannel.WebChannel<Devtools.ClientSession.MessageToApp, Devtools.ClientSession.MessageFromApp> {}
 
 export type ConnectDevtoolsToStore = (
-  storeDevtoolsChannel: StoreDevtoolsChannel,
+  storeDevtoolsChannel: ClientSessionDevtoolsChannel,
 ) => Effect.Effect<void, UnexpectedError, Scope.Scope>
 
 export type Adapter = (opts: {
@@ -231,3 +233,19 @@ export type Adapter = (opts: {
   shutdown: (cause: Cause.Cause<any>) => Effect.Effect<void>
   connectDevtoolsToStore: ConnectDevtoolsToStore
 }) => Effect.Effect<ClientSession, UnexpectedError, Scope.Scope>
+
+export const MigrationsReportEntry = Schema.Struct({
+  tableName: Schema.String,
+  hashes: Schema.Struct({
+    expected: Schema.Number,
+    actual: Schema.optional(Schema.Number),
+  }),
+})
+
+export const MigrationsReport = Schema.Struct({
+  migrations: Schema.Array(MigrationsReportEntry),
+})
+
+export type MigrationsReport = typeof MigrationsReport.Type
+
+export type MigrationsReportEntry = typeof MigrationsReportEntry.Type

package/src/derived-mutations.ts

@@ -1,7 +1,7 @@
-import type { SqliteDsl } from '@livestore/db-schema'
 import type { GetValForKey } from '@livestore/utils'
 import { ReadonlyRecord, Schema } from '@livestore/utils/effect'
 
+import type { SqliteDsl } from './schema/db-schema/mod.js'
 import type * as MutationEvent from './schema/MutationEvent.js'
 import { defineMutation } from './schema/mutations.js'
 import { getDefaultValuesDecoded } from './schema/schema-helpers.js'
@@ -54,7 +54,7 @@ export const deriveCreateMutationDef = <
 
       return { sql, bindValues, writeTables: new Set([tableName]) }
     },
-    { localOnly: table.options.deriveMutations.localOnly },
+    { clientOnly: table.options.deriveMutations.clientOnly },
   )
 }
 
@@ -84,7 +84,7 @@ export const deriveUpdateMutationDef = <
 
       return { sql, bindValues, writeTables: new Set([tableName]) }
     },
-    { localOnly: table.options.deriveMutations.localOnly },
+    { clientOnly: table.options.deriveMutations.clientOnly },
   )
 }
 
@@ -112,7 +112,7 @@ export const deriveDeleteMutationDef = <
 
       return { sql, bindValues, writeTables: new Set([tableName]) }
     },
-    { localOnly: table.options.deriveMutations.localOnly },
+    { clientOnly: table.options.deriveMutations.clientOnly },
   )
 }
 

package/src/devtools/devtools-bridge.ts

@@ -1,14 +1,17 @@
-import type { Effect, PubSub } from '@livestore/utils/effect'
+import type { Effect, WebChannel } from '@livestore/utils/effect'
 
 import type * as Devtools from './devtools-messages.js'
 
 export type PrepareDevtoolsBridge = {
-  /** Messages coming from the app host (usually responses to requests) */
-  responsePubSub: PubSub.PubSub<Devtools.MessageFromAppLeader | Devtools.MessageFromAppClientSession>
-  sendToAppHost: (msg: Devtools.MessageToAppLeader | Devtools.MessageToAppClientSession) => Effect.Effect<void>
-  clientId: string
-  sessionId: string
+  webchannels: {
+    leader: WebChannel.WebChannel<Devtools.Leader.MessageFromApp, Devtools.Leader.MessageToApp>
+    clientSession: WebChannel.WebChannel<Devtools.ClientSession.MessageFromApp, Devtools.ClientSession.MessageToApp>
+  }
+  clientInfo: {
+    clientId: string
+    sessionId: string
+    isLeader: boolean
+  }
   copyToClipboard: (text: string) => Effect.Effect<void>
   sendEscapeKey?: Effect.Effect<void>
-  isLeader: boolean
 }

package/src/devtools/devtools-messages-client-session.ts

@@ -1,8 +1,9 @@
 import { Schema } from '@livestore/utils/effect'
 
 import { DebugInfo } from '../debug-info.js'
+import { EventId } from '../schema/mod.js'
 import { PreparedBindValues } from '../util.js'
-import { Disconnect, LSDClientSessionReqResMessage } from './devtools-messages-common.js'
+import { LSDClientSessionChannelMessage, LSDClientSessionReqResMessage } from './devtools-messages-common.js'
 
 export class DebugInfoReq extends LSDClientSessionReqResMessage('LSD.ClientSession.DebugInfoReq', {}) {}
 
@@ -39,6 +40,13 @@ export class DebugInfoRerunQueryRes extends LSDClientSessionReqResMessage(
   {},
 ) {}
 
+export class SyncHeadSubscribe extends LSDClientSessionReqResMessage('LSD.ClientSession.SyncHeadSubscribe', {}) {}
+export class SyncHeadUnsubscribe extends LSDClientSessionReqResMessage('LSD.ClientSession.SyncHeadUnsubscribe', {}) {}
+export class SyncHeadRes extends LSDClientSessionReqResMessage('LSD.ClientSession.SyncHeadRes', {
+  local: EventId.EventId,
+  upstream: EventId.EventId,
+}) {}
+
 export class ReactivityGraphSubscribe extends LSDClientSessionReqResMessage(
   'LSD.ClientSession.ReactivityGraphSubscribe',
   {
@@ -78,7 +86,13 @@ export class LiveQueriesRes extends LSDClientSessionReqResMessage('LSD.ClientSes
   liveQueries: Schema.Array(SerializedLiveQuery),
 }) {}
 
-export const MessageToAppClientSession = Schema.Union(
+export class Ping extends LSDClientSessionReqResMessage('LSD.ClientSession.Ping', {}) {}
+
+export class Pong extends LSDClientSessionReqResMessage('LSD.ClientSession.Pong', {}) {}
+
+export class Disconnect extends LSDClientSessionChannelMessage('LSD.ClientSession.Disconnect', {}) {}
+
+export const MessageToApp = Schema.Union(
   DebugInfoReq,
   DebugInfoHistorySubscribe,
   DebugInfoHistoryUnsubscribe,
@@ -89,13 +103,14 @@ export const MessageToAppClientSession = Schema.Union(
   LiveQueriesSubscribe,
   LiveQueriesUnsubscribe,
   Disconnect,
-  // TODO also introduce a ping/pong protocol for the client session <> devtools connection
-  // Ping,
-).annotations({ identifier: 'LSD.MessageToAppClientSession' })
+  Ping,
+  SyncHeadSubscribe,
+  SyncHeadUnsubscribe,
+).annotations({ identifier: 'LSD.ClientSession.MessageToApp' })
 
-export type MessageToAppClientSession = typeof MessageToAppClientSession.Type
+export type MessageToApp = typeof MessageToApp.Type
 
-export const MessageFromAppClientSession = Schema.Union(
+export const MessageFromApp = Schema.Union(
   DebugInfoRes,
   DebugInfoHistoryRes,
   DebugInfoResetRes,
@@ -103,7 +118,8 @@ export const MessageFromAppClientSession = Schema.Union(
   ReactivityGraphRes,
   LiveQueriesRes,
   Disconnect,
-  // Pong,
-).annotations({ identifier: 'LSD.MessageFromAppClientSession' })
+  Pong,
+  SyncHeadRes,
+).annotations({ identifier: 'LSD.ClientSession.MessageFromApp' })
 
-export type MessageFromAppClientSession = typeof MessageFromAppClientSession.Type
+export type MessageFromApp = typeof MessageFromApp.Type

package/src/devtools/devtools-messages-common.ts

@@ -2,10 +2,10 @@ import { Schema } from '@livestore/utils/effect'
 
 import { liveStoreVersion as pkgVersion } from '../version.js'
 
-const requestId = Schema.String
-const clientId = Schema.String
-const sessionId = Schema.String
-const liveStoreVersion = Schema.Literal(pkgVersion)
+export const requestId = Schema.String
+export const clientId = Schema.String
+export const sessionId = Schema.String
+export const liveStoreVersion = Schema.Literal(pkgVersion)
 
 export const LSDMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
   Schema.TaggedStruct(tag, {
@@ -15,6 +15,7 @@ export const LSDMessage = <Tag extends string, Fields extends Schema.Struct.Fiel
 
 export const LSDChannelMessage = <Tag extends string, Fields extends Schema.Struct.Fields>(tag: Tag, fields: Fields) =>
   LSDMessage(tag, {
+    clientId,
     ...fields,
   })
 
@@ -45,8 +46,36 @@ export const LSDReqResMessage = <Tag extends string, Fields extends Schema.Struc
     ...fields,
   })
 
-export class Disconnect extends LSDClientSessionChannelMessage('LSD.Disconnect', {}) {}
-
-export class Ping extends LSDReqResMessage('LSD.Ping', {}) {}
+type DefaultFields = {
+  readonly requestId: typeof Schema.String
+  readonly liveStoreVersion: typeof liveStoreVersion
+  readonly clientId: typeof Schema.String
+}
 
-export class Pong extends LSDReqResMessage('LSD.Pong', {}) {}
+export const LeaderReqResMessage = <
+  Tag extends string,
+  ReqFields extends Schema.Struct.Fields,
+  ResFields extends Schema.Struct.Fields,
+>(
+  tag: Tag,
+  fields: {
+    payload: ReqFields
+    success: ResFields
+  },
+): {
+  Request: Schema.TaggedStruct<`${Tag}.Request`, ReqFields & DefaultFields>
+  Response: Schema.TaggedStruct<`${Tag}.Response`, ResFields & DefaultFields>
+} => {
+  return {
+    Request: Schema.TaggedStruct(`${tag}.Request`, {
+      requestId,
+      liveStoreVersion,
+      ...fields.payload,
+    }).annotations({ identifier: `${tag}.Request` }),
+    Response: Schema.TaggedStruct(`${tag}.Response`, {
+      requestId,
+      liveStoreVersion,
+      ...fields.success,
+    }).annotations({ identifier: `${tag}.Response` }),
+  } as any
+}

package/src/devtools/devtools-messages-leader.ts

@@ -1,15 +1,21 @@
 import { Schema, Transferable } from '@livestore/utils/effect'
 
-import { NetworkStatus } from '../adapter-types.js'
+import { NetworkStatus, UnexpectedError } from '../adapter-types.js'
+import { EventId } from '../schema/mod.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
-import { Disconnect, LSDMessage, LSDReqResMessage, Ping, Pong } from './devtools-messages-common.js'
+import * as SyncState from '../sync/syncstate.js'
+import {
+  LeaderReqResMessage,
+  liveStoreVersion,
+  LSDMessage,
+  LSDReqResMessage,
+  requestId,
+} from './devtools-messages-common.js'
 
 export class ResetAllDataReq extends LSDReqResMessage('LSD.Leader.ResetAllDataReq', {
   mode: Schema.Literal('all-data', 'only-app-db'),
 }) {}
 
-export class ResetAllDataRes extends LSDReqResMessage('LSD.Leader.ResetAllDataRes', {}) {}
-
 export class DatabaseFileInfoReq extends LSDReqResMessage('LSD.Leader.DatabaseFileInfoReq', {}) {}
 
 export class DatabaseFileInfo extends Schema.Struct({
@@ -47,6 +53,13 @@ export class SyncHistoryRes extends LSDReqResMessage('LSD.Leader.SyncHistoryRes'
   metadata: Schema.Option(Schema.JsonValue),
 }) {}
 
+export class SyncHeadSubscribe extends LSDReqResMessage('LSD.Leader.SyncHeadSubscribe', {}) {}
+export class SyncHeadUnsubscribe extends LSDReqResMessage('LSD.Leader.SyncHeadUnsubscribe', {}) {}
+export class SyncHeadRes extends LSDReqResMessage('LSD.Leader.SyncHeadRes', {
+  local: EventId.EventId,
+  upstream: EventId.EventId,
+}) {}
+
 export class SnapshotReq extends LSDReqResMessage('LSD.Leader.SnapshotReq', {}) {}
 
 export class SnapshotRes extends LSDReqResMessage('LSD.Leader.SnapshotRes', {
@@ -62,13 +75,13 @@ export class LoadDatabaseFileRes extends LSDReqResMessage('LSD.Leader.LoadDataba
 }) {}
 
 // TODO refactor this to use push/pull semantics
-export class MutationBroadcast extends LSDMessage('LSD.Leader.MutationBroadcast', {
-  mutationEventEncoded: MutationEvent.AnyEncoded,
+export class SyncPull extends LSDMessage('LSD.Leader.SyncPull', {
+  payload: SyncState.PayloadUpstream,
 }) {}
 
 // TODO refactor this to use push/pull semantics
 export class RunMutationReq extends LSDReqResMessage('LSD.Leader.RunMutationReq', {
-  mutationEventEncoded: MutationEvent.AnyEncoded.pipe(Schema.omit('id', 'parentId')),
+  mutationEventEncoded: MutationEvent.PartialAnyEncoded,
 }) {}
 
 export class RunMutationRes extends LSDReqResMessage('LSD.Leader.RunMutationRes', {}) {}
@@ -79,11 +92,55 @@ export class MutationLogRes extends LSDReqResMessage('LSD.Leader.MutationLogRes'
   mutationLog: Transferable.Uint8Array,
 }) {}
 
-export const MessageToAppLeader = Schema.Union(
+export class Ping extends LSDReqResMessage('LSD.Leader.Ping', {}) {}
+
+export class Pong extends LSDReqResMessage('LSD.Leader.Pong', {}) {}
+
+export class Disconnect extends LSDReqResMessage('LSD.Leader.Disconnect', {}) {}
+
+export const SetSyncLatch = LeaderReqResMessage('LSD.Leader.SetSyncLatch', {
+  payload: {
+    closeLatch: Schema.Boolean,
+  },
+  success: {},
+})
+
+export const ResetAllData = LeaderReqResMessage('LSD.Leader.ResetAllData', {
+  payload: {
+    mode: Schema.Literal('all-data', 'only-app-db'),
+  },
+  success: {},
+})
+
+// TODO move to `Schema.TaggedRequest` once new RPC is ready https://github.com/Effect-TS/effect/pull/4362
+export class DatabaseFileInfo_ extends Schema.TaggedRequest<DatabaseFileInfo_>()('LSD.Leader.DatabaseFileInfo', {
+  payload: {
+    requestId,
+    liveStoreVersion,
+  },
+  success: DatabaseFileInfo,
+  failure: UnexpectedError,
+}) {}
+
+export class NetworkStatus_ extends Schema.TaggedRequest<NetworkStatus_>()('LSD.Leader.NetworkStatus', {
+  payload: {
+    requestId,
+    liveStoreVersion,
+  },
+  success: NetworkStatus,
+  failure: UnexpectedError,
+}) {}
+
+export const MessageToApp_ = Schema.Union(DatabaseFileInfo_, NetworkStatus_)
+
+export type MessageToApp_ = typeof MessageToApp_.Type
+//
+
+export const MessageToApp = Schema.Union(
   SnapshotReq,
   LoadDatabaseFileReq,
   MutationLogReq,
-  ResetAllDataReq,
+  ResetAllData.Request,
   NetworkStatusSubscribe,
   NetworkStatusUnsubscribe,
   Disconnect,
@@ -93,23 +150,28 @@ export const MessageToAppLeader = Schema.Union(
   SyncHistorySubscribe,
   SyncHistoryUnsubscribe,
   SyncingInfoReq,
-).annotations({ identifier: 'LSD.MessageToAppLeader' })
+  SyncHeadSubscribe,
+  SyncHeadUnsubscribe,
+  SetSyncLatch.Request,
+).annotations({ identifier: 'LSD.Leader.MessageToApp' })
 
-export type MessageToAppLeader = typeof MessageToAppLeader.Type
+export type MessageToApp = typeof MessageToApp.Type
 
-export const MessageFromAppLeader = Schema.Union(
+export const MessageFromApp = Schema.Union(
   SnapshotRes,
   LoadDatabaseFileRes,
   MutationLogRes,
-  ResetAllDataRes,
   Disconnect,
-  MutationBroadcast,
+  SyncPull,
   NetworkStatusRes,
   RunMutationRes,
   Pong,
   DatabaseFileInfoRes,
   SyncHistoryRes,
   SyncingInfoRes,
-).annotations({ identifier: 'LSD.MessageFromAppLeader' })
+  SyncHeadRes,
+  ResetAllData.Response,
+  SetSyncLatch.Response,
+).annotations({ identifier: 'LSD.Leader.MessageFromApp' })
 
-export type MessageFromAppLeader = typeof MessageFromAppLeader.Type
+export type MessageFromApp = typeof MessageFromApp.Type

package/src/devtools/devtools-messages.ts

@@ -1,3 +1,3 @@
-export * from './devtools-messages-client-session.js'
-export * from './devtools-messages-leader.js'
+export * as ClientSession from './devtools-messages-client-session.js'
+export * as Leader from './devtools-messages-leader.js'
 export * from './devtools-messages-common.js'

package/src/index.ts

@@ -13,5 +13,5 @@ export * from './debug-info.js'
 export * from './bounded-collections.js'
 export * from './version.js'
 export * from './query-builder/mod.js'
-export * from './sync/syncstate.js'
+export * as SyncState from './sync/syncstate.js'
 export * from './otel.js'