@livestore/common 0.3.0-dev.10 → 0.3.0-dev.12

package/src/leader-thread/types.ts

@@ -1,7 +1,6 @@
 import type {
   Deferred,
   Effect,
-  Fiber,
   HttpClient,
   Option,
   Queue,
@@ -16,10 +15,11 @@ import type {
   BootStatus,
   Devtools,
   InvalidPushError,
-  MakeSynchronousDatabase,
+  MakeSqliteDb,
+  MigrationsReport,
   PersistenceInfo,
+  SqliteDb,
   SyncBackend,
-  SynchronousDatabase,
   UnexpectedError,
 } from '../index.js'
 import type { EventId, LiveStoreSchema, MutationEvent } from '../schema/mod.js'
@@ -28,13 +28,6 @@ import type { ShutdownChannel } from './shutdown-channel.js'
 
 export type ShutdownState = 'running' | 'shutting-down'
 
-export class OuterWorkerCtx extends Context.Tag('OuterWorkerCtx')<
-  OuterWorkerCtx,
-  {
-    innerFiber: Fiber.RuntimeFiber<any, any>
-  }
->() {}
-
 export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
 export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
 
@@ -56,8 +49,8 @@ export type InitialSyncInfo = Option.Option<{
 //   | { _tag: 'Recreate'; snapshotRef: Ref.Ref<Uint8Array | undefined>; syncInfo: InitialSyncInfo }
 //   | { _tag: 'Reuse'; syncInfo: InitialSyncInfo }
 
-export type LeaderDatabase = SynchronousDatabase<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
-export type PersistenceInfoPair = { db: PersistenceInfo; mutationLog: PersistenceInfo }
+export type LeaderSqliteDb = SqliteDb<{ dbPointer: number; persistenceInfo: PersistenceInfo }>
+export type PersistenceInfoPair = { readModel: PersistenceInfo; mutationLog: PersistenceInfo }
 
 export type DevtoolsOptions =
   | {
@@ -65,9 +58,9 @@ export type DevtoolsOptions =
     }
   | {
       enabled: true
-      makeContext: Effect.Effect<
+      makeBootContext: Effect.Effect<
         {
-          devtoolsWebChannel: WebChannel.WebChannel<Devtools.MessageToAppLeader, Devtools.MessageFromAppLeader>
+          devtoolsWebChannel: WebChannel.WebChannel<Devtools.Leader.MessageToApp, Devtools.Leader.MessageFromApp>
           persistenceInfo: PersistenceInfoPair
         },
         UnexpectedError,
@@ -75,26 +68,46 @@ export type DevtoolsOptions =
       >
     }
 
+export type DevtoolsContext =
+  | {
+      enabled: true
+      // syncBackendPullLatch: Effect.Latch
+      // syncBackendPushLatch: Effect.Latch
+      syncBackendLatch: Effect.Latch
+      syncBackendLatchState: SubscriptionRef.SubscriptionRef<{ latchClosed: boolean }>
+    }
+  | {
+      enabled: false
+    }
+
 export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
   LeaderThreadCtx,
   {
     schema: LiveStoreSchema
     storeId: string
     clientId: string
-    makeSyncDb: MakeSynchronousDatabase
-    db: LeaderDatabase
-    dbLog: LeaderDatabase
+    makeSqliteDb: MakeSqliteDb
+    dbReadModel: LeaderSqliteDb
+    dbMutationLog: LeaderSqliteDb
     bootStatusQueue: Queue.Queue<BootStatus>
     // TODO we should find a more elegant way to handle cases which need this ref for their implementation
     shutdownStateSubRef: SubscriptionRef.SubscriptionRef<ShutdownState>
     shutdownChannel: ShutdownChannel
     mutationEventSchema: MutationEvent.ForMutationDefRecord<any>
-    // devtools: DevtoolsContext
+    devtools: DevtoolsContext
     syncBackend: SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
     connectedClientSessionPullQueues: PullQueueSet
-    /** e.g. used for `store.__dev` APIs */
-    extraIncomingMessagesQueue: Queue.Queue<Devtools.MessageToAppLeader>
+    initialState: {
+      leaderHead: EventId.EventId
+      migrationsReport: MigrationsReport
+    }
+    /**
+     * e.g. used for `store._dev` APIs
+     *
+     * This is currently separated from `.devtools` as it also needs to work when devtools are disabled
+     */
+    extraIncomingMessagesQueue: Queue.Queue<Devtools.Leader.MessageToApp>
   }
 >() {}
 
@@ -124,7 +137,11 @@ export interface LeaderSyncProcessor {
   pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
   boot: (args: {
     dbReady: Deferred.Deferred<void>
-  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient>
+  }) => Effect.Effect<
+    { initialLeaderHead: EventId.EventId },
+    UnexpectedError,
+    LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
+  >
   syncState: Subscribable.Subscribable<SyncState.SyncState>
 }
 

package/src/query-builder/api.ts

@@ -35,7 +35,7 @@ export namespace QueryBuilderAst {
   export type RowQuery = {
     readonly _tag: 'RowQuery'
     readonly tableDef: DbSchema.TableDefBase
-    readonly id: string | SessionIdSymbol
+    readonly id: string | SessionIdSymbol | number
     readonly insertValues: Record<string, unknown>
   }
 
@@ -252,10 +252,10 @@ export namespace QueryBuilder {
         ? (_: 'Error: Need to enable deriveMutations to use row()') => any
         : TTableDef['options']['requiredInsertColumnNames'] extends never
           ? (
-              id: string | SessionIdSymbol,
+              id: string | SessionIdSymbol | number,
             ) => QueryBuilder<RowQuery.Result<TTableDef>, TTableDef, QueryBuilder.ApiFeature, QueryInfo.Row>
           : <TOptions extends RowQuery.RequiredColumnsOptions<TTableDef>>(
-              id: string | SessionIdSymbol,
+              id: string | SessionIdSymbol | number,
               opts: TOptions,
             ) => QueryBuilder<RowQuery.Result<TTableDef>, TTableDef, QueryBuilder.ApiFeature, QueryInfo.Row>
   }

package/src/query-builder/impl.test.ts

@@ -18,13 +18,23 @@ const todos = DbSchema.table(
   { deriveMutations: true },
 )
 
+const todosWithIntId = DbSchema.table(
+  'todos_with_int_id',
+  {
+    id: DbSchema.integer({ primaryKey: true }),
+    text: DbSchema.text({ default: '', nullable: false }),
+    status: DbSchema.text({ schema: Schema.Literal('active', 'completed') }),
+  },
+  { deriveMutations: true },
+)
+
 const comments = DbSchema.table('comments', {
   id: DbSchema.text({ primaryKey: true }),
   text: DbSchema.text({ default: '', nullable: false }),
   todoId: DbSchema.text({}),
 })
 
-const db = { todos: todos.query, comments: comments.query }
+const db = { todos: todos.query, todosWithIntId: todosWithIntId.query, comments: comments.query }
 
 describe('query builder', () => {
   describe('result schema', () => {
@@ -204,6 +214,17 @@ describe('query builder', () => {
         }
       `)
     })
+
+    it('should handle row queries with numbers', () => {
+      expect(db.todosWithIntId.row(123, { insertValues: { status: 'active' } }).asSql()).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            123,
+          ],
+          "query": "SELECT * FROM 'todos_with_int_id' WHERE id = ?",
+        }
+      `)
+    })
   })
 })
 

package/src/query-builder/impl.ts

@@ -128,12 +128,12 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
       // eslint-disable-next-line prefer-rest-params
       const params = [...arguments]
 
-      let id: string
+      let id: string | number
 
       if (tableDef.options.isSingleton) {
         id = tableDef.sqliteDef.columns.id!.default.pipe(Option.getOrThrow)
       } else {
-        id = params[0] as string
+        id = params[0] as string | number
         if (id === undefined) {
           invalidQueryBuilder(`Id missing for row query on non-singleton table ${tableDef.sqliteDef.name}`)
         }

package/src/query-info.ts

@@ -24,20 +24,20 @@ export namespace QueryInfo {
   export type Row = {
     _tag: 'Row'
     table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
+    id: string | SessionIdSymbol | number
   }
 
   export type Col = {
     _tag: 'Col'
     table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
+    id: string | SessionIdSymbol | number
     column: string
   }
 
   export type ColJsonValue = {
     _tag: 'ColJsonValue'
     table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
+    id: string | SessionIdSymbol | number
     column: string
     /**
      * example: `$.tabs[3].items[2]` (`$` referring to the column value)

package/src/rehydrate-from-mutationlog.ts

@@ -1,7 +1,7 @@
 import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 
-import { type MigrationOptionsFromMutationLog, type SynchronousDatabase, UnexpectedError } from './adapter-types.js'
+import { type MigrationOptionsFromMutationLog, type SqliteDb, UnexpectedError } from './adapter-types.js'
 import { makeApplyMutation } from './leader-thread/apply-mutation.js'
 import type { LiveStoreSchema, MutationDef, MutationEvent, MutationLogMetaRow } from './schema/mod.js'
 import { EventId, MUTATION_LOG_META_TABLE } from './schema/mod.js'
@@ -16,8 +16,8 @@ export const rehydrateFromMutationLog = ({
   migrationOptions,
   onProgress,
 }: {
-  logDb: SynchronousDatabase
-  db: SynchronousDatabase
+  logDb: SqliteDb
+  db: SqliteDb
   schema: LiveStoreSchema
   migrationOptions: MigrationOptionsFromMutationLog
   onProgress: (_: { done: number; total: number }) => Effect.Effect<void>

package/src/schema/EventId.ts

@@ -43,6 +43,10 @@ export const isGreaterThan = (a: EventId, b: EventId) => {
   return a.global > b.global || (a.global === b.global && a.local > b.local)
 }
 
+export const isGreaterThanOrEqual = (a: EventId, b: EventId) => {
+  return a.global > b.global || (a.global === b.global && a.local >= b.local)
+}
+
 export const make = (id: EventId | typeof EventId.Encoded): EventId => {
   return Schema.is(EventId)(id) ? id : Schema.decodeSync(EventId)(id)
 }

package/src/schema-management/common.ts

@@ -1,4 +1,4 @@
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import type { ParamsObject } from '../util.js'
 import { prepareBindValues } from '../util.js'
 
@@ -6,7 +6,7 @@ import { prepareBindValues } from '../util.js'
 // will require proper scope-aware cleanup etc (for testing and apps with multiple LiveStore instances)
 // const cachedStmts = new Map<string, PreparedStatement>()
 
-export const dbExecute = (db: SynchronousDatabase, queryStr: string, bindValues?: ParamsObject) => {
+export const dbExecute = (db: SqliteDb, queryStr: string, bindValues?: ParamsObject) => {
   // let stmt = cachedStmts.get(queryStr)
   // if (!stmt) {
   const stmt = db.prepare(queryStr)
@@ -20,7 +20,7 @@ export const dbExecute = (db: SynchronousDatabase, queryStr: string, bindValues?
   stmt.finalize()
 }
 
-export const dbSelect = <T>(db: SynchronousDatabase, queryStr: string, bindValues?: ParamsObject) => {
+export const dbSelect = <T>(db: SqliteDb, queryStr: string, bindValues?: ParamsObject) => {
   // let stmt = cachedStmts.get(queryStr)
   // if (!stmt) {
   const stmt = db.prepare(queryStr)

package/src/schema-management/migrations.ts

@@ -2,7 +2,7 @@ import { SqliteAst, SqliteDsl } from '@livestore/db-schema'
 import { memoizeByStringifyArgs } from '@livestore/utils'
 import { Effect, Schema as EffectSchema } from '@livestore/utils/effect'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { MigrationsReport, MigrationsReportEntry, SqliteDb, UnexpectedError } from '../adapter-types.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import type { SchemaMetaRow, SchemaMutationsMetaRow } from '../schema/system-tables.js'
 import {
@@ -19,7 +19,7 @@ import { validateSchema } from './validate-mutation-defs.js'
 
 const getMemoizedTimestamp = memoizeByStringifyArgs(() => new Date().toISOString())
 
-export const makeSchemaManager = (db: SynchronousDatabase): Effect.Effect<SchemaManager> =>
+export const makeSchemaManager = (db: SqliteDb): Effect.Effect<SchemaManager> =>
   Effect.gen(function* () {
     yield* migrateTable({
       db,
@@ -51,10 +51,10 @@ export const migrateDb = ({
   schema,
   onProgress,
 }: {
-  db: SynchronousDatabase
+  db: SqliteDb
   schema: LiveStoreSchema
   onProgress?: (opts: { done: number; total: number }) => Effect.Effect<void>
-}) =>
+}): Effect.Effect<MigrationsReport, UnexpectedError> =>
   Effect.gen(function* () {
     yield* migrateTable({
       db,
@@ -81,6 +81,7 @@ export const migrateDb = ({
 
     const tablesToMigrate = new Set<{ tableAst: SqliteAst.Table; schemaHash: number }>()
 
+    const migrationsReportEntries: MigrationsReportEntry[] = []
     for (const tableDef of tableDefs) {
       const tableAst = tableDef.sqliteDef.ast
       const tableName = tableAst.name
@@ -90,9 +91,10 @@ export const migrateDb = ({
       if (schemaHash !== dbSchemaHash) {
         tablesToMigrate.add({ tableAst, schemaHash })
 
-        console.log(
-          `Schema hash mismatch for table '${tableName}' (DB: ${dbSchemaHash}, expected: ${schemaHash}), migrating table...`,
-        )
+        migrationsReportEntries.push({
+          tableName,
+          hashes: { expected: schemaHash, actual: dbSchemaHash },
+        })
       }
     }
 
@@ -107,6 +109,8 @@ export const migrateDb = ({
         yield* onProgress({ done: processedTables, total: tablesCount })
       }
     }
+
+    return { migrations: migrationsReportEntries }
   })
 
 export const migrateTable = ({
@@ -116,7 +120,7 @@ export const migrateTable = ({
   behaviour,
   skipMetaTable = false,
 }: {
-  db: SynchronousDatabase
+  db: SqliteDb
   tableAst: SqliteAst.Table
   schemaHash?: number
   behaviour: 'drop-and-recreate' | 'create-if-not-exists'

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,13 +1,13 @@
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { Scope } from '@livestore/utils/effect'
-import { Effect, Schema, Stream } from '@livestore/utils/effect'
+import type { Runtime, Scope } from '@livestore/utils/effect'
+import { Effect, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
-import type { ClientSessionLeaderThreadProxy, UnexpectedError } from '../adapter-types.js'
+import type { ClientSession, UnexpectedError } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
 import { type LiveStoreSchema } from '../schema/mod.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
-import { SyncState, updateSyncState } from './syncstate.js'
+import * as SyncState from './syncstate.js'
 
 /**
  * Rebase behaviour:
@@ -20,18 +20,16 @@ import { SyncState, updateSyncState } from './syncstate.js'
  */
 export const makeClientSessionSyncProcessor = ({
   schema,
-  initialLeaderHead,
-  pushToLeader,
-  pullFromLeader,
+  clientSession,
+  runtime,
   applyMutation,
   rollback,
   refreshTables,
   span,
 }: {
   schema: LiveStoreSchema
-  initialLeaderHead: EventId.EventId
-  pushToLeader: (batch: ReadonlyArray<MutationEvent.AnyEncoded>) => void
-  pullFromLeader: ClientSessionLeaderThreadProxy['mutations']['pull']
+  clientSession: ClientSession
+  runtime: Runtime.Runtime<Scope.Scope>
   applyMutation: (
     mutationEventDecoded: MutationEvent.PartialAnyDecoded,
     options: { otelContext: otel.Context; withChangeset: boolean },
@@ -46,15 +44,17 @@ export const makeClientSessionSyncProcessor = ({
   const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
 
   const syncStateRef = {
-    current: new SyncState({
-      localHead: initialLeaderHead,
-      upstreamHead: initialLeaderHead,
+    current: new SyncState.SyncState({
+      localHead: clientSession.leaderThread.initialState.leaderHead,
+      upstreamHead: clientSession.leaderThread.initialState.leaderHead,
       pending: [],
       // TODO init rollbackTail from leader to be ready for backend rebasing
       rollbackTail: [],
     }),
   }
 
+  const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
+
   const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
     const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
     return mutationDef.options.localOnly
@@ -73,7 +73,7 @@ export const makeClientSessionSyncProcessor = ({
       )
     })
 
-    const updateResult = updateSyncState({
+    const updateResult = SyncState.updateSyncState({
       syncState: syncStateRef.current,
       payload: { _tag: 'local-push', newEvents: encodedMutationEvents },
       isLocalEvent,
@@ -90,6 +90,7 @@ export const makeClientSessionSyncProcessor = ({
     }
 
     syncStateRef.current = updateResult.newSyncState
+    syncStateUpdateQueue.offer(updateResult.newSyncState).pipe(Effect.runSync)
 
     const writeTables = new Set<string>()
     for (const mutationEvent of updateResult.newEvents) {
@@ -103,7 +104,9 @@ export const makeClientSessionSyncProcessor = ({
     }
 
     // console.debug('pushToLeader', encodedMutationEvents.length, ...encodedMutationEvents.map((_) => _.toJSON()))
-    pushToLeader(encodedMutationEvents)
+    clientSession.leaderThread.mutations
+      .push(encodedMutationEvents)
+      .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
 
     return { writeTables }
   }
@@ -111,12 +114,15 @@ export const makeClientSessionSyncProcessor = ({
   const otelContext = otel.trace.setSpan(otel.context.active(), span)
 
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
-    yield* pullFromLeader.pipe(
+    yield* clientSession.leaderThread.mutations.pull.pipe(
       Stream.tap(({ payload, remaining }) =>
         Effect.gen(function* () {
           // console.log('pulled payload from leader', { payload, remaining })
+          if (clientSession.devtools.enabled) {
+            yield* clientSession.devtools.pullLatch.await
+          }
 
-          const updateResult = updateSyncState({
+          const updateResult = SyncState.updateSyncState({
             syncState: syncStateRef.current,
             payload,
             isLocalEvent,
@@ -129,6 +135,7 @@ export const makeClientSessionSyncProcessor = ({
           }
 
           syncStateRef.current = updateResult.newSyncState
+          syncStateUpdateQueue.offer(updateResult.newSyncState).pipe(Effect.runSync)
 
           if (updateResult._tag === 'rebase') {
             span.addEvent('pull:rebase', {
@@ -140,11 +147,11 @@ export const makeClientSessionSyncProcessor = ({
               remaining,
             })
             if (LS_DEV) {
-              console.debug(
+              Effect.logDebug(
                 'pull:rebase: rollback',
                 updateResult.eventsToRollback.length,
                 ...updateResult.eventsToRollback.map((_) => _.toJSON()),
-              )
+              ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
             for (let i = updateResult.eventsToRollback.length - 1; i >= 0; i--) {
@@ -155,7 +162,9 @@ export const makeClientSessionSyncProcessor = ({
               }
             }
 
-            pushToLeader(updateResult.newSyncState.pending)
+            clientSession.leaderThread.mutations
+              .push(updateResult.newSyncState.pending)
+              .pipe(Effect.tapCauseLogPretty, Effect.provide(runtime), Effect.runFork)
           } else {
             span.addEvent('pull:advance', {
               payloadTag: payload._tag,
@@ -192,7 +201,14 @@ export const makeClientSessionSyncProcessor = ({
   return {
     push,
     boot,
-    syncStateRef,
+    syncState: Subscribable.make({
+      get: Effect.gen(function* () {
+        const syncState = syncStateRef.current
+        if (syncStateRef === undefined) return shouldNeverHappen('Not initialized')
+        return syncState
+      }),
+      changes: Stream.fromQueue(syncStateUpdateQueue),
+    }),
   } satisfies ClientSessionSyncProcessor
 }
 
@@ -205,5 +221,5 @@ export interface ClientSessionSyncProcessor {
   }
   boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
 
-  syncStateRef: { current: SyncState }
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
 }

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.3.0-dev.10' as const
+export const liveStoreVersion = '0.3.0-dev.12' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.