@livestore/common 0.3.0-dev.1 → 0.3.0-dev.11

package/src/schema-management/common.ts

@@ -1,4 +1,4 @@
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import type { ParamsObject } from '../util.js'
 import { prepareBindValues } from '../util.js'
 
@@ -6,7 +6,7 @@ import { prepareBindValues } from '../util.js'
 // will require proper scope-aware cleanup etc (for testing and apps with multiple LiveStore instances)
 // const cachedStmts = new Map<string, PreparedStatement>()
 
-export const dbExecute = (db: SynchronousDatabase, queryStr: string, bindValues?: ParamsObject) => {
+export const dbExecute = (db: SqliteDb, queryStr: string, bindValues?: ParamsObject) => {
   // let stmt = cachedStmts.get(queryStr)
   // if (!stmt) {
   const stmt = db.prepare(queryStr)
@@ -20,7 +20,7 @@ export const dbExecute = (db: SynchronousDatabase, queryStr: string, bindValues?
   stmt.finalize()
 }
 
-export const dbSelect = <T>(db: SynchronousDatabase, queryStr: string, bindValues?: ParamsObject) => {
+export const dbSelect = <T>(db: SqliteDb, queryStr: string, bindValues?: ParamsObject) => {
   // let stmt = cachedStmts.get(queryStr)
   // if (!stmt) {
   const stmt = db.prepare(queryStr)

package/src/schema-management/migrations.ts

@@ -2,7 +2,7 @@ import { SqliteAst, SqliteDsl } from '@livestore/db-schema'
 import { memoizeByStringifyArgs } from '@livestore/utils'
 import { Effect, Schema as EffectSchema } from '@livestore/utils/effect'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import type { SchemaMetaRow, SchemaMutationsMetaRow } from '../schema/system-tables.js'
 import {
@@ -19,7 +19,7 @@ import { validateSchema } from './validate-mutation-defs.js'
 
 const getMemoizedTimestamp = memoizeByStringifyArgs(() => new Date().toISOString())
 
-export const makeSchemaManager = (db: SynchronousDatabase): Effect.Effect<SchemaManager> =>
+export const makeSchemaManager = (db: SqliteDb): Effect.Effect<SchemaManager> =>
   Effect.gen(function* () {
     yield* migrateTable({
       db,
@@ -51,7 +51,7 @@ export const migrateDb = ({
   schema,
   onProgress,
 }: {
-  db: SynchronousDatabase
+  db: SqliteDb
   schema: LiveStoreSchema
   onProgress?: (opts: { done: number; total: number }) => Effect.Effect<void>
 }) =>
@@ -116,7 +116,7 @@ export const migrateTable = ({
   behaviour,
   skipMetaTable = false,
 }: {
-  db: SynchronousDatabase
+  db: SqliteDb
   tableAst: SqliteAst.Table
   schemaHash?: number
   behaviour: 'drop-and-recreate' | 'create-if-not-exists'
@@ -129,10 +129,10 @@ export const migrateTable = ({
 
     if (behaviour === 'drop-and-recreate') {
       // TODO need to possibly handle cascading deletes due to foreign keys
-      dbExecute(db, sql`drop table if exists ${tableName}`)
-      dbExecute(db, sql`create table if not exists ${tableName} (${columnSpec}) strict`)
+      dbExecute(db, sql`drop table if exists '${tableName}'`)
+      dbExecute(db, sql`create table if not exists '${tableName}' (${columnSpec}) strict`)
     } else if (behaviour === 'create-if-not-exists') {
-      dbExecute(db, sql`create table if not exists ${tableName} (${columnSpec}) strict`)
+      dbExecute(db, sql`create table if not exists '${tableName}' (${columnSpec}) strict`)
     }
 
     for (const index of tableAst.indexes) {
@@ -162,11 +162,11 @@ export const migrateTable = ({
 
 const createIndexFromDefinition = (tableName: string, index: SqliteAst.Index) => {
   const uniqueStr = index.unique ? 'UNIQUE' : ''
-  return sql`create ${uniqueStr} index if not exists ${index.name} on ${tableName} (${index.columns.join(', ')})`
+  return sql`create ${uniqueStr} index if not exists '${index.name}' on '${tableName}' (${index.columns.join(', ')})`
 }
 
 export const makeColumnSpec = (tableAst: SqliteAst.Table) => {
-  const primaryKeys = tableAst.columns.filter((_) => _.primaryKey).map((_) => _.name)
+  const primaryKeys = tableAst.columns.filter((_) => _.primaryKey).map((_) => `'${_.name}'`)
   const columnDefStrs = tableAst.columns.map(toSqliteColumnSpec)
   if (primaryKeys.length > 0) {
     columnDefStrs.push(`PRIMARY KEY (${primaryKeys.join(', ')})`)
@@ -191,5 +191,5 @@ const toSqliteColumnSpec = (column: SqliteAst.Column) => {
     return `default ${encodedDefaultValue}`
   })()
 
-  return `${column.name} ${columnTypeStr} ${nullableStr} ${defaultValueStr}`
+  return `'${column.name}' ${columnTypeStr} ${nullableStr} ${defaultValueStr}`
 }

package/src/sync/{client-session-sync-processor.ts → ClientSessionSyncProcessor.ts}

@@ -1,14 +1,13 @@
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { Scope } from '@livestore/utils/effect'
+import type { Runtime, Scope } from '@livestore/utils/effect'
 import { Effect, Schema, Stream } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
-import type { Coordinator, UnexpectedError } from '../adapter-types.js'
+import type { ClientSession, UnexpectedError } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
 import { type LiveStoreSchema } from '../schema/mod.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
-import type { SyncState } from './syncstate.js'
-import { updateSyncState } from './syncstate.js'
+import { SyncState, updateSyncState } from './syncstate.js'
 
 /**
  * Rebase behaviour:
@@ -21,20 +20,18 @@ import { updateSyncState } from './syncstate.js'
  */
 export const makeClientSessionSyncProcessor = ({
   schema,
-  initialLeaderHead,
-  pushToLeader,
-  pullFromLeader,
+  clientSession,
+  runtime,
   applyMutation,
   rollback,
   refreshTables,
   span,
 }: {
   schema: LiveStoreSchema
-  initialLeaderHead: EventId.EventId
-  pushToLeader: (batch: ReadonlyArray<MutationEvent.AnyEncoded>) => void
-  pullFromLeader: Coordinator['mutations']['pull']
+  clientSession: ClientSession
+  runtime: Runtime.Runtime<Scope.Scope>
   applyMutation: (
-    mutationEventDecoded: MutationEvent.PartialAny,
+    mutationEventDecoded: MutationEvent.PartialAnyDecoded,
     options: { otelContext: otel.Context; withChangeset: boolean },
   ) => {
     writeTables: Set<string>
@@ -42,19 +39,18 @@ export const makeClientSessionSyncProcessor = ({
   }
   rollback: (changeset: Uint8Array) => void
   refreshTables: (tables: Set<string>) => void
-  // rebaseBehaviour: 'auto-rebase' | 'manual-rebase'
   span: otel.Span
 }): ClientSessionSyncProcessor => {
   const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
 
   const syncStateRef = {
-    current: {
-      localHead: initialLeaderHead,
-      upstreamHead: initialLeaderHead,
+    current: new SyncState({
+      localHead: clientSession.leaderThread.mutations.initialMutationEventId,
+      upstreamHead: clientSession.leaderThread.mutations.initialMutationEventId,
       pending: [],
       // TODO init rollbackTail from leader to be ready for backend rebasing
       rollbackTail: [],
-    } as SyncState,
+    }),
   }
 
   const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
@@ -104,7 +100,10 @@ export const makeClientSessionSyncProcessor = ({
       mutationEvent.meta.sessionChangeset = res.sessionChangeset
     }
 
-    pushToLeader(encodedMutationEvents)
+    // console.debug('pushToLeader', encodedMutationEvents.length, ...encodedMutationEvents.map((_) => _.toJSON()))
+    clientSession.leaderThread.mutations
+      .push(encodedMutationEvents)
+      .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
 
     return { writeTables }
   }
@@ -112,10 +111,13 @@ export const makeClientSessionSyncProcessor = ({
   const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
-    yield* pullFromLeader.pipe(
+    yield* clientSession.leaderThread.mutations.pull.pipe(
       Stream.tap(({ payload, remaining }) =>
         Effect.gen(function* () {
           // console.log('pulled payload from leader', { payload, remaining })
+          if (clientSession.devtools.enabled) {
+            yield* clientSession.devtools.pullLatch.await
+          }
 
           const updateResult = updateSyncState({
             syncState: syncStateRef.current,
@@ -155,6 +157,10 @@ export const makeClientSessionSyncProcessor = ({
                 event.meta.sessionChangeset = undefined
               }
             }
+
+            clientSession.leaderThread.mutations
+              .push(updateResult.newSyncState.pending)
+              .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
           } else {
             span.addEvent('pull:advance', {
               payloadTag: payload._tag,
@@ -182,6 +188,7 @@ export const makeClientSessionSyncProcessor = ({
         }),
       ),
       Stream.runDrain,
+      Effect.forever, // NOTE Whenever the leader changes, we need to re-start the stream
       Effect.tapCauseLogPretty,
       Effect.forkScoped,
     )
@@ -196,7 +203,7 @@ export const makeClientSessionSyncProcessor = ({
 
 export interface ClientSessionSyncProcessor {
   push: (
-    batch: ReadonlyArray<MutationEvent.PartialAny>,
+    batch: ReadonlyArray<MutationEvent.PartialAnyDecoded>,
     options: { otelContext: otel.Context },
   ) => {
     writeTables: Set<string>

package/src/sync/index.ts

@@ -1,3 +1,3 @@
 export * from './sync.js'
 export * from './validate-push-payload.js'
-export * from './client-session-sync-processor.js'
+export * from './ClientSessionSyncProcessor.js'

package/src/sync/next/history-dag-common.ts

@@ -20,7 +20,7 @@ export const emptyHistoryDag = (): HistoryDag =>
   })
 
 // TODO consider making `ROOT_ID` parent to itself
-export const rootParentId = { global: EventId.ROOT.global - 1, local: 0 } satisfies EventId.EventId
+export const rootParentId = EventId.make({ global: EventId.ROOT.global - 1, local: EventId.localDefault })
 
 export type HistoryDagNode = {
   id: EventId.EventId

package/src/sync/next/rebase-events.ts

@@ -1,4 +1,4 @@
-import type * as EventId from '../../schema/EventId.js'
+import * as EventId from '../../schema/EventId.js'
 import type * as MutationEvent from '../../schema/MutationEvent.js'
 import type { MutationDef, MutationEventFactsSnapshot } from '../../schema/mutations.js'
 import {
@@ -19,13 +19,13 @@ export type RebaseInput = {
   newRemoteEvents: RebaseEventWithConflict[]
   pendingLocalEvents: RebaseEventWithConflict[]
   validate: (args: {
-    rebasedLocalEvents: MutationEvent.PartialAny[]
+    rebasedLocalEvents: MutationEvent.PartialAnyDecoded[]
     mutationDefs: Record<string, MutationDef.Any>
   }) => FactValidationResult
 }
 
 export type RebaseOutput = {
-  rebasedLocalEvents: MutationEvent.PartialAny[]
+  rebasedLocalEvents: MutationEvent.PartialAnyDecoded[]
 }
 
 export type RebaseFn = (input: RebaseInput) => RebaseOutput
@@ -48,7 +48,7 @@ export const rebaseEvents = ({
   newRemoteEvents: HistoryDagNode[]
   rebaseFn: RebaseFn
   currentFactsSnapshot: MutationEventFactsSnapshot
-}): MutationEvent.Any[] => {
+}): ReadonlyArray<MutationEvent.AnyDecoded> => {
   const initialSnapshot = new Map(currentFactsSnapshot)
   applyFactGroups(
     newRemoteEvents.map((event) => event.factsGroup),
@@ -89,10 +89,10 @@ export const rebaseEvents = ({
   return rebasedLocalEvents.map(
     (event, index) =>
       ({
-        id: { global: headGlobalId + index + 1, local: 0 } satisfies EventId.EventId,
-        parentId: { global: headGlobalId + index, local: 0 } satisfies EventId.EventId,
+        id: EventId.make({ global: headGlobalId + index + 1, local: EventId.localDefault }),
+        parentId: EventId.make({ global: headGlobalId + index, local: EventId.localDefault }),
         mutation: event.mutation,
         args: event.args,
-      }) satisfies MutationEvent.Any,
+      }) satisfies MutationEvent.AnyDecoded,
   )
 }

package/src/sync/next/test/mutation-fixtures.ts

@@ -140,16 +140,9 @@ export const toEventNodes = (
 
   let currentEventId: EventId.EventId = EventId.ROOT
 
-  const getNextEventId = (mutationDef: MutationDef.Any): EventId.EventId => {
-    if (mutationDef.options.localOnly) {
-      return { global: currentEventId.global, local: currentEventId.local + 1 }
-    }
-    return { global: currentEventId.global + 1, local: 0 }
-  }
-
   const eventNodes = partialEvents.map((partialEvent) => {
     const mutationDef = mutationDefs[partialEvent.mutation]!
-    const eventId = getNextEventId(mutationDef)
+    const eventId = EventId.nextPair(currentEventId, mutationDef.options.localOnly).id
     currentEventId = eventId
 
     const factsSnapshot = factsSnapshotForDag(historyDagFromNodes(nodesAcc, { skipFactsCheck: true }), undefined)
@@ -224,8 +217,8 @@ const getParentId = (eventId: EventId.EventId): EventId.EventId => {
   const localParentId = eventId.local - 1
 
   if (localParentId < 0) {
-    return { global: globalParentId - 1, local: 0 }
+    return EventId.make({ global: globalParentId - 1, local: EventId.localDefault })
   }
 
-  return { global: globalParentId, local: localParentId }
+  return EventId.make({ global: globalParentId, local: localParentId })
 }

package/src/sync/sync.ts

@@ -1,12 +1,23 @@
-import type { Effect, HttpClient, Option, Stream, SubscriptionRef } from '@livestore/utils/effect'
+import type { Effect, HttpClient, Option, Scope, Stream, SubscriptionRef } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
+import type { UnexpectedError } from '../adapter-types.js'
+import type { InitialSyncOptions } from '../leader-thread/types.js'
 import * as EventId from '../schema/EventId.js'
 import type * as MutationEvent from '../schema/MutationEvent.js'
 
-export interface SyncBackendOptionsBase {
-  type: string
-  [key: string]: Schema.JsonValue
+/**
+ * Those arguments can be used to implement multi-tenancy etc and are passed in from the store.
+ */
+export type MakeBackendArgs = {
+  storeId: string
+  clientId: string
+}
+
+export type SyncOptions = {
+  makeBackend: (args: MakeBackendArgs) => Effect.Effect<SyncBackend<any>, UnexpectedError, Scope.Scope>
+  /** @default { _tag: 'Skip' } */
+  initialSyncOptions?: InitialSyncOptions
 }
 
 export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
@@ -18,7 +29,7 @@ export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
   ) => Stream.Stream<
     {
       batch: ReadonlyArray<{
-        mutationEventEncoded: MutationEvent.AnyEncoded
+        mutationEventEncoded: MutationEvent.AnyEncodedGlobal
         metadata: Option.Option<TSyncMetadata>
       }>
       remaining: number
@@ -33,7 +44,7 @@ export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
      * - Number of events: 1-100
      * - event ids must be in ascending order
      * */
-    batch: ReadonlyArray<MutationEvent.AnyEncoded>,
+    batch: ReadonlyArray<MutationEvent.AnyEncodedGlobal>,
   ) => Effect.Effect<
     {
       /** Indexes are relative to `batch` */
@@ -46,6 +57,7 @@ export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
 }
 
 export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {}) {}
+
 export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
   reason: Schema.Union(
     Schema.TaggedStruct('Unexpected', {
@@ -61,6 +73,7 @@ export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('In
     }),
   ),
 }) {}
+
 export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
   message: Schema.String,
 }) {}