@livestore/common 0.3.0-dev.12 → 0.3.0-dev.13

package/src/leader-thread/make-leader-thread-layer.ts

@@ -225,7 +225,15 @@ const bootLeaderThread = ({
     yield* Deferred.succeed(dbReady, void 0)
 
     if (initialBlockingSyncContext.blockingDeferred !== undefined) {
-      yield* initialBlockingSyncContext.blockingDeferred
+      // Provides a syncing status right away before the first pull response comes in
+      yield* Queue.offer(bootStatusQueue, {
+        stage: 'syncing',
+        progress: { done: 0, total: -1 },
+      })
+
+      yield* initialBlockingSyncContext.blockingDeferred.pipe(
+        Effect.withSpan('@livestore/common:leader-thread:initial-sync-blocking'),
+      )
     }
 
     yield* Queue.offer(bootStatusQueue, { stage: 'done' })

package/src/leader-thread/mutationlog.ts

@@ -23,18 +23,20 @@ export const getMutationEventsSince = (
       .map((_) => ({
         mutation: _.mutation,
         args: _.argsJson,
-        id: { global: _.idGlobal, local: _.idLocal },
-        parentId: { global: _.parentIdGlobal, local: _.parentIdLocal },
+        id: { global: _.idGlobal, client: _.idClient },
+        parentId: { global: _.parentIdGlobal, client: _.parentIdClient },
+        clientId: _.clientId,
+        sessionId: _.sessionId ?? undefined,
       }))
       .filter((_) => EventId.compare(_.id, since) > 0)
   })
 
-export const getLocalHeadFromDb = (dbMutationLog: SqliteDb): EventId.EventId => {
-  const res = dbMutationLog.select<{ idGlobal: EventId.GlobalEventId; idLocal: EventId.LocalEventId }>(
-    sql`select idGlobal, idLocal from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idLocal DESC limit 1`,
+export const getClientHeadFromDb = (dbMutationLog: SqliteDb): EventId.EventId => {
+  const res = dbMutationLog.select<{ idGlobal: EventId.GlobalEventId; idClient: EventId.ClientEventId }>(
+    sql`select idGlobal, idClient from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idClient DESC limit 1`,
   )[0]
 
-  return res ? { global: res.idGlobal, local: res.idLocal } : EventId.ROOT
+  return res ? { global: res.idGlobal, client: res.idClient } : EventId.ROOT
 }
 
 export const getBackendHeadFromDb = (dbMutationLog: SqliteDb): EventId.GlobalEventId =>

package/src/leader-thread/types.ts

@@ -32,7 +32,7 @@ export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
 export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
 
 export const InitialSyncOptionsBlocking = Schema.TaggedStruct('Blocking', {
-  timeout: Schema.DurationFromMillis,
+  timeout: Schema.Union(Schema.DurationFromMillis, Schema.Number),
 })
 
 export type InitialSyncOptionsBlocking = typeof InitialSyncOptionsBlocking.Type
@@ -134,7 +134,11 @@ export interface LeaderSyncProcessor {
     },
   ) => Effect.Effect<void, InvalidPushError>
 
-  pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
+  pushPartial: (args: {
+    mutationEvent: MutationEvent.PartialAnyEncoded
+    clientId: string
+    sessionId: string | undefined
+  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
   boot: (args: {
     dbReady: Deferred.Deferred<void>
   }) => Effect.Effect<

package/src/query-builder/api.ts

@@ -3,8 +3,8 @@ import { type Option, Predicate, type Schema } from '@livestore/utils/effect'
 
 import type { SessionIdSymbol } from '../adapter-types.js'
 import type { QueryInfo } from '../query-info.js'
+import type { SqliteDsl } from '../schema/db-schema/mod.js'
 import type { DbSchema } from '../schema/mod.js'
-import type { SqliteDsl } from '../schema/table-def.js'
 import type { SqlValue } from '../util.js'
 
 export type QueryBuilderAst = QueryBuilderAst.SelectQuery | QueryBuilderAst.CountQuery | QueryBuilderAst.RowQuery

package/src/rehydrate-from-mutationlog.ts

@@ -60,10 +60,12 @@ This likely means the schema has changed in an incompatible way.
         )
 
         const mutationEventEncoded = {
-          id: { global: row.idGlobal, local: row.idLocal },
-          parentId: { global: row.parentIdGlobal, local: row.parentIdLocal },
+          id: { global: row.idGlobal, client: row.idClient },
+          parentId: { global: row.parentIdGlobal, client: row.parentIdClient },
           mutation: row.mutation,
           args,
+          clientId: row.clientId,
+          sessionId: row.sessionId ?? undefined,
         } satisfies MutationEvent.AnyEncoded
 
         yield* applyMutation(mutationEventEncoded, { skipMutationLog: true })
@@ -73,8 +75,8 @@ This likely means the schema has changed in an incompatible way.
 
     const stmt = logDb.prepare(sql`\
 SELECT * FROM ${MUTATION_LOG_META_TABLE} 
-WHERE idGlobal > $idGlobal OR (idGlobal = $idGlobal AND idLocal > $idLocal)
-ORDER BY idGlobal ASC, idLocal ASC
+WHERE idGlobal > $idGlobal OR (idGlobal = $idGlobal AND idClient > $idClient)
+ORDER BY idGlobal ASC, idClient ASC
 LIMIT ${CHUNK_SIZE}
 `)
 
@@ -88,14 +90,14 @@ LIMIT ${CHUNK_SIZE}
 
         const lastId = Chunk.isChunk(item)
           ? Chunk.last(item).pipe(
-              Option.map((_) => ({ global: _.idGlobal, local: _.idLocal })),
+              Option.map((_) => ({ global: _.idGlobal, client: _.idClient })),
               Option.getOrElse(() => EventId.ROOT),
             )
           : EventId.ROOT
         const nextItem = Chunk.fromIterable(
           stmt.select<MutationLogMetaRow>({
             $idGlobal: lastId?.global,
-            $idLocal: lastId?.local,
+            $idClient: lastId?.client,
           } as any as PreparedBindValues),
         )
         const prevItem = Chunk.isChunk(item) ? item : Chunk.empty()

package/src/schema/EventId.test.ts

@@ -5,8 +5,8 @@ import { EventId } from './mod.js'
 
 Vitest.describe('EventId', () => {
   Vitest.test('nextPair', () => {
-    const e_0_0 = EventId.make({ global: 0, local: 0 })
-    expect(EventId.nextPair(e_0_0, false).id).toStrictEqual({ global: 1, local: 0 })
-    expect(EventId.nextPair(e_0_0, true).id).toStrictEqual({ global: 0, local: 1 })
+    const e_0_0 = EventId.make({ global: 0, client: 0 })
+    expect(EventId.nextPair(e_0_0, false).id).toStrictEqual({ global: 1, client: 0 })
+    expect(EventId.nextPair(e_0_0, true).id).toStrictEqual({ global: 0, client: 1 })
   })
 })

package/src/schema/EventId.ts

@@ -1,26 +1,26 @@
 import { Brand, Schema } from '@livestore/utils/effect'
 
-export type LocalEventId = Brand.Branded<number, 'LocalEventId'>
-export const localEventId = Brand.nominal<LocalEventId>()
-export const LocalEventId = Schema.fromBrand(localEventId)(Schema.Int)
+export type ClientEventId = Brand.Branded<number, 'ClientEventId'>
+export const localEventId = Brand.nominal<ClientEventId>()
+export const ClientEventId = Schema.fromBrand(localEventId)(Schema.Int)
 
 export type GlobalEventId = Brand.Branded<number, 'GlobalEventId'>
 export const globalEventId = Brand.nominal<GlobalEventId>()
 export const GlobalEventId = Schema.fromBrand(globalEventId)(Schema.Int)
 
-export const localDefault = 0 as any as LocalEventId
+export const clientDefault = 0 as any as ClientEventId
 
 /**
  * LiveStore event id value consisting of a globally unique event sequence number
- * and a local sequence number.
+ * and a client sequence number.
  *
- * The local sequence number is only used for localOnly mutations and starts from 0 for each global sequence number.
+ * The client sequence number is only used for clientOnly mutations and starts from 0 for each global sequence number.
  */
-export type EventId = { global: GlobalEventId; local: LocalEventId }
+export type EventId = { global: GlobalEventId; client: ClientEventId }
 
 export const EventId = Schema.Struct({
   global: GlobalEventId,
-  local: LocalEventId,
+  client: ClientEventId,
 }).annotations({ title: 'LiveStore.EventId' })
 
 /**
@@ -30,21 +30,21 @@ export const compare = (a: EventId, b: EventId) => {
   if (a.global !== b.global) {
     return a.global - b.global
   }
-  return a.local - b.local
+  return a.client - b.client
 }
 
-export const isEqual = (a: EventId, b: EventId) => a.global === b.global && a.local === b.local
+export const isEqual = (a: EventId, b: EventId) => a.global === b.global && a.client === b.client
 
 export type EventIdPair = { id: EventId; parentId: EventId }
 
-export const ROOT = { global: -1 as any as GlobalEventId, local: localDefault } satisfies EventId
+export const ROOT = { global: -1 as any as GlobalEventId, client: clientDefault } satisfies EventId
 
 export const isGreaterThan = (a: EventId, b: EventId) => {
-  return a.global > b.global || (a.global === b.global && a.local > b.local)
+  return a.global > b.global || (a.global === b.global && a.client > b.client)
 }
 
 export const isGreaterThanOrEqual = (a: EventId, b: EventId) => {
-  return a.global > b.global || (a.global === b.global && a.local >= b.local)
+  return a.global > b.global || (a.global === b.global && a.client >= b.client)
 }
 
 export const make = (id: EventId | typeof EventId.Encoded): EventId => {
@@ -53,12 +53,12 @@ export const make = (id: EventId | typeof EventId.Encoded): EventId => {
 
 export const nextPair = (id: EventId, isLocal: boolean): EventIdPair => {
   if (isLocal) {
-    return { id: { global: id.global, local: (id.local + 1) as any as LocalEventId }, parentId: id }
+    return { id: { global: id.global, client: (id.client + 1) as any as ClientEventId }, parentId: id }
   }
 
   return {
-    id: { global: (id.global + 1) as any as GlobalEventId, local: localDefault },
-    // NOTE we always point to `local: 0` for non-localOnly mutations
-    parentId: { global: id.global, local: localDefault },
+    id: { global: (id.global + 1) as any as GlobalEventId, client: clientDefault },
+    // NOTE we always point to `client: 0` for non-clientOnly mutations
+    parentId: { global: id.global, client: clientDefault },
   }
 }

package/src/schema/MutationEvent.ts

@@ -10,7 +10,7 @@ export type MutationEventPartial<TMutationsDef extends MutationDef.Any> = {
   args: Schema.Schema.Type<TMutationsDef['schema']>
 }
 
-export type MutationEventPartialEncoded<TMutationsDef extends MutationDef.Any> = {
+export type PartialEncoded<TMutationsDef extends MutationDef.Any> = {
   mutation: TMutationsDef['name']
   args: Schema.Schema.Encoded<TMutationsDef['schema']>
 }
@@ -20,6 +20,8 @@ export type MutationEvent<TMutationsDef extends MutationDef.Any> = {
   args: Schema.Schema.Type<TMutationsDef['schema']>
   id: EventId.EventId
   parentId: EventId.EventId
+  clientId: string
+  sessionId: string | undefined
 }
 
 export type MutationEventEncoded<TMutationsDef extends MutationDef.Any> = {
@@ -27,6 +29,8 @@ export type MutationEventEncoded<TMutationsDef extends MutationDef.Any> = {
   args: Schema.Schema.Encoded<TMutationsDef['schema']>
   id: EventId.EventId
   parentId: EventId.EventId
+  clientId: string
+  sessionId: string | undefined
 }
 
 export type AnyDecoded = MutationEvent<MutationDef.Any>
@@ -35,6 +39,8 @@ export const AnyDecoded = Schema.Struct({
   args: Schema.Any,
   id: EventId.EventId,
   parentId: EventId.EventId,
+  clientId: Schema.String,
+  sessionId: Schema.UndefinedOr(Schema.String),
 }).annotations({ title: 'MutationEvent.AnyDecoded' })
 
 export type AnyEncoded = MutationEventEncoded<MutationDef.Any>
@@ -43,6 +49,8 @@ export const AnyEncoded = Schema.Struct({
   args: Schema.Any,
   id: EventId.EventId,
   parentId: EventId.EventId,
+  clientId: Schema.String,
+  sessionId: Schema.UndefinedOr(Schema.String),
 }).annotations({ title: 'MutationEvent.AnyEncoded' })
 
 export const AnyEncodedGlobal = Schema.Struct({
@@ -50,11 +58,17 @@ export const AnyEncodedGlobal = Schema.Struct({
   args: Schema.Any,
   id: EventId.GlobalEventId,
   parentId: EventId.GlobalEventId,
+  clientId: Schema.String,
 }).annotations({ title: 'MutationEvent.AnyEncodedGlobal' })
 export type AnyEncodedGlobal = typeof AnyEncodedGlobal.Type
 
 export type PartialAnyDecoded = MutationEventPartial<MutationDef.Any>
-export type PartialAnyEncoded = MutationEventPartialEncoded<MutationDef.Any>
+export type PartialAnyEncoded = PartialEncoded<MutationDef.Any>
+
+export const PartialAnyEncoded = Schema.Struct({
+  mutation: Schema.String,
+  args: Schema.Any,
+})
 
 export type PartialForSchema<TSchema extends LiveStoreSchema> = {
   [K in keyof TSchema['_MutationDefMapType']]: MutationEventPartial<TSchema['_MutationDefMapType'][K]>
@@ -75,6 +89,8 @@ export type ForMutationDefRecord<TMutationsDefRecord extends MutationDefRecord>
       args: Schema.Schema.Type<TMutationsDefRecord[K]['schema']>
       id: EventId.EventId
       parentId: EventId.EventId
+      clientId: string
+      sessionId: string | undefined
     }
   }[keyof TMutationsDefRecord],
   {
@@ -83,6 +99,8 @@ export type ForMutationDefRecord<TMutationsDefRecord extends MutationDefRecord>
       args: Schema.Schema.Encoded<TMutationsDefRecord[K]['schema']>
       id: EventId.EventId
       parentId: EventId.EventId
+      clientId: string
+      sessionId: string | undefined
     }
   }[keyof TMutationsDefRecord]
 >
@@ -112,6 +130,8 @@ export const makeMutationEventSchema = <TSchema extends LiveStoreSchema>(
         args: def.schema,
         id: EventId.EventId,
         parentId: EventId.EventId,
+        clientId: Schema.String,
+        sessionId: Schema.UndefinedOr(Schema.String),
       }),
     ),
   ).annotations({ title: 'MutationEvent' }) as any
@@ -136,6 +156,8 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
   args: Schema.Any,
   id: EventId.EventId,
   parentId: EventId.EventId,
+  clientId: Schema.String,
+  sessionId: Schema.UndefinedOr(Schema.String),
   // TODO get rid of `meta` again by cleaning up the usage implementations
   meta: Schema.optionalWith(
     Schema.Any as Schema.Schema<{
@@ -149,7 +171,7 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
     // - More readable way to print the id + parentId
     // - not including `meta`
     return {
-      id: `(${this.id.global},${this.id.local}) → (${this.parentId.global},${this.parentId.local})`,
+      id: `(${this.id.global},${this.id.client}) → (${this.parentId.global},${this.parentId.client})`,
       mutation: this.mutation,
       args: this.args,
     }
@@ -164,8 +186,9 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
   static fromGlobal = (mutationEvent: AnyEncodedGlobal) =>
     new EncodedWithMeta({
       ...mutationEvent,
-      id: { global: mutationEvent.id, local: EventId.localDefault },
-      parentId: { global: mutationEvent.parentId, local: EventId.localDefault },
+      id: { global: mutationEvent.id, client: EventId.clientDefault },
+      parentId: { global: mutationEvent.parentId, client: EventId.clientDefault },
+      sessionId: undefined,
     })
 
   toGlobal = (): AnyEncodedGlobal => ({
@@ -177,7 +200,9 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
 
 export const isEqualEncoded = (a: AnyEncoded, b: AnyEncoded) =>
   a.id.global === b.id.global &&
-  a.id.local === b.id.local &&
+  a.id.client === b.id.client &&
   a.mutation === b.mutation &&
+  a.clientId === b.clientId &&
+  // a.sessionId === b.sessionId &&
   // TODO use schema equality here
   JSON.stringify(a.args) === JSON.stringify(b.args)

package/src/schema/db-schema/ast/sqlite.ts

@@ -0,0 +1,142 @@
+import { type Option, Schema } from '@livestore/utils/effect'
+
+import { hashCode } from '../hash.js'
+
+export namespace ColumnType {
+  export type ColumnType = Text | Null | Real | Integer | Blob
+
+  export type Text = { _tag: 'text' }
+
+  export type Null = { _tag: 'null' }
+
+  export type Real = { _tag: 'real' }
+
+  export type Integer = { _tag: 'integer' }
+
+  export type Blob = { _tag: 'blob' }
+}
+
+export type Column = {
+  _tag: 'column'
+  name: string
+  type: ColumnType.ColumnType
+  primaryKey: boolean
+  nullable: boolean
+  default: Option.Option<any>
+  schema: Schema.Schema<any>
+}
+
+export const column = (props: Omit<Column, '_tag'>): Column => ({ _tag: 'column', ...props })
+
+export type Index = {
+  _tag: 'index'
+  columns: ReadonlyArray<string>
+  name?: string
+  unique?: boolean
+  primaryKey?: boolean
+}
+
+export const index = (
+  columns: ReadonlyArray<string>,
+  name?: string,
+  unique?: boolean,
+  primaryKey?: boolean,
+): Index => ({
+  _tag: 'index',
+  columns,
+  name,
+  unique,
+  primaryKey,
+})
+
+export type ForeignKey = {
+  _tag: 'foreignKey'
+  references: {
+    table: string
+    columns: ReadonlyArray<string>
+  }
+  key: {
+    table: string
+    columns: ReadonlyArray<string>
+  }
+  columns: ReadonlyArray<string>
+}
+
+export type Table = {
+  _tag: 'table'
+  name: string
+  columns: ReadonlyArray<Column>
+  indexes: ReadonlyArray<Index>
+}
+
+export const table = (name: string, columns: ReadonlyArray<Column>, indexes: ReadonlyArray<Index>): Table => ({
+  _tag: 'table',
+  name,
+  columns,
+  indexes,
+})
+
+export type DbSchema = {
+  _tag: 'dbSchema'
+  tables: Table[]
+}
+
+export const dbSchema = (tables: Table[]): DbSchema => ({ _tag: 'dbSchema', tables })
+
+/**
+ * NOTE we're only including SQLite-relevant information in the hash (which excludes the schema mapping)
+ */
+export const hash = (obj: Table | Column | Index | ForeignKey | DbSchema): number =>
+  hashCode(JSON.stringify(trimInfoForHasing(obj)))
+
+const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema): Record<string, any> => {
+  switch (obj._tag) {
+    case 'table': {
+      return {
+        _tag: 'table',
+        name: obj.name,
+        columns: obj.columns.map((column) => trimInfoForHasing(column)),
+        indexes: obj.indexes.map((index) => trimInfoForHasing(index)),
+      }
+    }
+    case 'column': {
+      return {
+        _tag: 'column',
+        name: obj.name,
+        type: obj.type._tag,
+        primaryKey: obj.primaryKey,
+        nullable: obj.nullable,
+        default: obj.default,
+      }
+    }
+    case 'index': {
+      return {
+        _tag: 'index',
+        columns: obj.columns,
+        name: obj.name,
+        unique: obj.unique,
+        primaryKey: obj.primaryKey,
+      }
+    }
+    case 'foreignKey': {
+      return {
+        _tag: 'foreignKey',
+        references: obj.references,
+        key: obj.key,
+        columns: obj.columns,
+      }
+    }
+    case 'dbSchema': {
+      return {
+        _tag: 'dbSchema',
+        tables: obj.tables.map(trimInfoForHasing),
+      }
+    }
+    default: {
+      throw new Error(`Unreachable: ${obj}`)
+    }
+  }
+}
+
+export const structSchemaForTable = (tableDef: Table) =>
+  Schema.Struct(Object.fromEntries(tableDef.columns.map((column) => [column.name, column.schema])))

package/src/schema/db-schema/ast/validate.ts

@@ -0,0 +1,13 @@
+import type { DbSchema } from './sqlite.js'
+
+// TODO
+export const validate = (_dbSchema: DbSchema) => {
+  /*
+	TODO:
+
+	- ensure no duplicate table names
+	- ensure no duplicate column names within a table
+	- ensure no duplicate index names globally
+	- ...
+	*/
+}