@livestore/common 0.3.0-dev.3 → 0.3.0-dev.31

package/src/leader-thread/types.ts

@@ -1,44 +1,40 @@
 import type {
   Deferred,
   Effect,
-  Fiber,
   HttpClient,
   Option,
   Queue,
   Scope,
+  Stream,
+  Subscribable,
   SubscriptionRef,
   WebChannel,
 } from '@livestore/utils/effect'
 import { Context, Schema } from '@livestore/utils/effect'
 
+import type { LeaderPullCursor, SqliteError } from '../adapter-types.js'
 import type {
   BootStatus,
   Devtools,
-  InvalidPushError,
-  MakeSynchronousDatabase,
+  LeaderAheadError,
+  MakeSqliteDb,
+  MigrationsReport,
   PersistenceInfo,
+  SqliteDb,
   SyncBackend,
-  SynchronousDatabase,
   UnexpectedError,
 } from '../index.js'
-import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
-import type { PayloadUpstream, SyncState } from '../sync/syncstate.js'
+import type { EventId, LiveStoreEvent, LiveStoreSchema } from '../schema/mod.js'
+import type * as SyncState from '../sync/syncstate.js'
 import type { ShutdownChannel } from './shutdown-channel.js'
 
 export type ShutdownState = 'running' | 'shutting-down'
 
-export class OuterWorkerCtx extends Context.Tag('OuterWorkerCtx')<
-  OuterWorkerCtx,
-  {
-    innerFiber: Fiber.RuntimeFiber<any, any>
-  }
->() {}
-
 export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
 export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
 
 export const InitialSyncOptionsBlocking = Schema.TaggedStruct('Blocking', {
-  timeout: Schema.DurationFromMillis,
+  timeout: Schema.Union(Schema.DurationFromMillis, Schema.Number),
 })
 
 export type InitialSyncOptionsBlocking = typeof InitialSyncOptionsBlocking.Type
@@ -55,8 +51,8 @@ export type InitialSyncInfo = Option.Option<{
 //   | { _tag: 'Recreate'; snapshotRef: Ref.Ref<Uint8Array | undefined>; syncInfo: InitialSyncInfo }
 //   | { _tag: 'Reuse'; syncInfo: InitialSyncInfo }
 
-export type LeaderDatabase = SynchronousDatabase<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
-export type PersistenceInfoPair = { db: PersistenceInfo; mutationLog: PersistenceInfo }
+export type LeaderSqliteDb = SqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
+export type PersistenceInfoPair = { readModel: PersistenceInfo; eventlog: PersistenceInfo }
 
 export type DevtoolsOptions =
   | {
@@ -64,10 +60,9 @@ export type DevtoolsOptions =
     }
   | {
       enabled: true
-      makeContext: Effect.Effect<
+      makeBootContext: Effect.Effect<
         {
-          devtoolsWebChannel: WebChannel.WebChannel<Devtools.MessageToAppLeader, Devtools.MessageFromAppLeader>
-          shutdownChannel: ShutdownChannel
+          devtoolsWebChannel: WebChannel.WebChannel<Devtools.Leader.MessageToApp, Devtools.Leader.MessageFromApp>
           persistenceInfo: PersistenceInfoPair
         },
         UnexpectedError,
@@ -75,55 +70,104 @@ export type DevtoolsOptions =
       >
     }
 
+export type DevtoolsContext =
+  | {
+      enabled: true
+      // syncBackendPullLatch: Effect.Latch
+      // syncBackendPushLatch: Effect.Latch
+      syncBackendLatch: Effect.Latch
+      syncBackendLatchState: SubscriptionRef.SubscriptionRef<{ latchClosed: boolean }>
+    }
+  | {
+      enabled: false
+    }
+
 export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   LeaderThreadCtx,
   {
     schema: LiveStoreSchema
     storeId: string
-    originId: string
-    makeSyncDb: MakeSynchronousDatabase
-    db: LeaderDatabase
-    dbLog: LeaderDatabase
+    clientId: string
+    makeSqliteDb: MakeSqliteDb
+    dbReadModel: LeaderSqliteDb
+    dbEventlog: LeaderSqliteDb
     bootStatusQueue: Queue.Queue<BootStatus>
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
-    mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
-    // devtools: DevtoolsContext
+    shutdownChannel: ShutdownChannel
+    eventSchema: LiveStoreEvent.ForEventDefRecord<any>
+    devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
-    syncProcessor: SyncProcessor
-    connectedClientSessionPullQueues: PullQueueSet
+    syncProcessor: LeaderSyncProcessor
+    applyEvent: ApplyEvent
+    initialState: {
+      leaderHead: EventId.EventId
+      migrationsReport: MigrationsReport
+    }
+    /**
+     * e.g. used for `store._dev` APIs
+     *
+     * This is currently separated from `.devtools` as it also needs to work when devtools are disabled
+     */
+    extraIncomingMessagesQueue: Queue.Queue<Devtools.Leader.MessageToApp>
   }
 >() {}
 
+export type ApplyEvent = (
+  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  options?: {
+    /** Needed for rehydrateFromEventlog */
+    skipEventlog?: boolean
+  },
+) => Effect.Effect<
+  { sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } },
+  SqliteError | UnexpectedError
+>
+
 export type InitialBlockingSyncContext = {
   blockingDeferred: Deferred.Deferred<void> | undefined
   update: (_: { remaining: number; processed: number }) => Effect.Effect<void>
 }
 
-export type PullQueueItem = {
-  // mutationEvents: ReadonlyArray<MutationEvent.AnyEncoded>
-  // backendHead: number
-  payload: PayloadUpstream
-  // TODO move `remaining` into `PayloadUpstream`
-  remaining: number
-}
-
-export interface SyncProcessor {
+export interface LeaderSyncProcessor {
+  /** Used by client sessions to subscribe to upstream sync state changes */
+  pull: (args: {
+    cursor: LeaderPullCursor
+  }) => Stream.Stream<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }, UnexpectedError>
+  /** The `pullQueue` API can be used instead of `pull` when more convenient */
+  pullQueue: (args: {
+    cursor: LeaderPullCursor
+  }) => Effect.Effect<
+    Queue.Queue<{ payload: typeof SyncState.PayloadUpstream.Type; mergeCounter: number }>,
+    UnexpectedError,
+    Scope.Scope
+  >
+
+  /** Used by client sessions to push events to the leader thread */
   push: (
     /** `batch` needs to follow the same rules as `batch` in `SyncBackend.push` */
-    batch: ReadonlyArray<MutationEvent.EncodedWithMeta>,
-  ) => Effect.Effect<void, UnexpectedError | InvalidPushError, HttpClient.HttpClient | LeaderThreadCtx>
-
-  pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
-  boot: (args: {
-    dbReady: Deferred.Deferred<void>
-  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient>
-  syncState: Effect.Effect<SyncState, UnexpectedError>
-}
-
-export interface PullQueueSet {
-  makeQueue: (
-    since: EventId.EventId,
-  ) => Effect.Effect<Queue.Queue<PullQueueItem>, UnexpectedError, Scope.Scope | LeaderThreadCtx>
-  offer: (item: PullQueueItem) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+    batch: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>,
+    options?: {
+      /**
+       * If true, the effect will only finish when the local push has been processed (i.e. succeeded or was rejected).
+       * @default false
+       */
+      waitForProcessing?: boolean
+    },
+  ) => Effect.Effect<void, LeaderAheadError>
+
+  /** Currently only used by devtools which don't provide their own event numbers */
+  pushPartial: (args: {
+    event: LiveStoreEvent.PartialAnyEncoded
+    clientId: string
+    sessionId: string
+  }) => Effect.Effect<void, UnexpectedError>
+
+  boot: Effect.Effect<
+    { initialLeaderHead: EventId.EventId },
+    UnexpectedError,
+    LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
+  >
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
+  getMergeCounter: () => number
 }

package/src/materializer-helper.ts

@@ -0,0 +1,110 @@
+import { isReadonlyArray } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+
+import { SessionIdSymbol } from './adapter-types.js'
+import { isQueryBuilder } from './query-builder/api.js'
+import type { EventDef, Materializer, MaterializerResult } from './schema/EventDef.js'
+import type * as LiveStoreEvent from './schema/LiveStoreEvent.js'
+import type { BindValues } from './sql-queries/sql-queries.js'
+import type { PreparedBindValues } from './util.js'
+import { prepareBindValues } from './util.js'
+
+export const getExecArgsFromEvent = ({
+  eventDef: { eventDef, materializer },
+  event,
+}: {
+  eventDef: {
+    eventDef: EventDef.AnyWithoutFn
+    materializer: Materializer
+  }
+  /** Both encoded and decoded events are supported to reduce the number of times we need to decode/encode */
+  event:
+    | {
+        decoded: LiveStoreEvent.AnyDecoded | LiveStoreEvent.PartialAnyDecoded
+        encoded: undefined
+      }
+    | {
+        decoded: undefined
+        encoded: LiveStoreEvent.AnyEncoded | LiveStoreEvent.PartialAnyEncoded
+      }
+}): ReadonlyArray<{
+  statementSql: string
+  bindValues: PreparedBindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  const eventArgsDecoded = event.decoded?.args ?? Schema.decodeUnknownSync(eventDef.schema)(event.encoded!.args)
+
+  const res = materializer(eventArgsDecoded, {
+    clientOnly: eventDef.options.clientOnly,
+    // TODO properly implement this
+    currentFacts: new Map(),
+  })
+
+  const statementRes = mapMaterializerResult(res)
+
+  return statementRes.map((statementRes) => {
+    const statementSql = statementRes.sql
+
+    const eventArgsEncoded = event.encoded?.args ?? Schema.encodeUnknownSync(eventDef.schema)(event.decoded!.args)
+    const bindValues = typeof statementRes === 'string' ? eventArgsEncoded : statementRes.bindValues
+
+    const writeTables = typeof statementRes === 'string' ? undefined : statementRes.writeTables
+
+    return { statementSql, bindValues: prepareBindValues(bindValues ?? {}, statementSql), writeTables }
+  })
+}
+
+const mapMaterializerResult = (
+  materializerResult: MaterializerResult | ReadonlyArray<MaterializerResult>,
+): ReadonlyArray<{
+  sql: string
+  bindValues: BindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  if (isReadonlyArray(materializerResult)) {
+    return materializerResult.flatMap(mapMaterializerResult)
+  }
+  if (isQueryBuilder(materializerResult)) {
+    const { query, bindValues } = materializerResult.asSql()
+    return [{ sql: query, bindValues: bindValues as BindValues, writeTables: undefined }]
+  } else if (typeof materializerResult === 'string') {
+    return [{ sql: materializerResult, bindValues: {} as BindValues, writeTables: undefined }]
+  } else {
+    return [
+      {
+        sql: materializerResult.sql,
+        bindValues: materializerResult.bindValues,
+        writeTables: materializerResult.writeTables,
+      },
+    ]
+  }
+}
+
+// NOTE we should explore whether there is a more elegant solution
+// e.g. by leveraging the schema to replace the sessionIdSymbol
+export const replaceSessionIdSymbol = (
+  bindValues: Record<string, unknown> | ReadonlyArray<unknown>,
+  sessionId: string,
+) => {
+  deepReplaceValue(bindValues, SessionIdSymbol, sessionId)
+}
+
+const deepReplaceValue = <S, R>(input: any, searchValue: S, replaceValue: R): void => {
+  if (Array.isArray(input)) {
+    for (const i in input) {
+      if (input[i] === searchValue) {
+        input[i] = replaceValue
+      } else {
+        deepReplaceValue(input[i], searchValue, replaceValue)
+      }
+    }
+  } else if (typeof input === 'object' && input !== null) {
+    for (const key in input) {
+      if (input[key] === searchValue) {
+        input[key] = replaceValue
+      } else {
+        deepReplaceValue(input[key], searchValue, replaceValue)
+      }
+    }
+  }
+}

package/src/otel.ts

@@ -18,3 +18,11 @@ export const provideOtel =
       Effect.provide(TracingLive),
     )
   }
+
+export const getDurationMsFromSpan = (span: otel.Span): number => {
+  const durationHr: [seconds: number, nanos: number] = (span as any)._duration
+  return durationHr[0] * 1000 + durationHr[1] / 1_000_000
+}
+
+export const getStartTimeHighResFromSpan = (span: otel.Span): DOMHighResTimeStamp =>
+  (span as any)._performanceStartTime as DOMHighResTimeStamp