@livestore/common 0.3.0-dev.11 → 0.3.0-dev.12

package/src/leader-thread/recreate-db.ts

@@ -2,19 +2,20 @@ import { casesHandled } from '@livestore/utils'
 import type { HttpClient } from '@livestore/utils/effect'
 import { Effect, Queue } from '@livestore/utils/effect'
 
-import type { InvalidPullError, IsOfflineError, MigrationHooks, SqliteError } from '../index.js'
+import type { InvalidPullError, IsOfflineError, MigrationHooks, MigrationsReport, SqliteError } from '../index.js'
 import { initializeSingletonTables, migrateDb, rehydrateFromMutationLog, UnexpectedError } from '../index.js'
 import { configureConnection } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
 export const recreateDb: Effect.Effect<
-  void,
+  { migrationsReport: MigrationsReport },
   UnexpectedError | SqliteError | IsOfflineError | InvalidPullError,
   LeaderThreadCtx | HttpClient.HttpClient
 > = Effect.gen(function* () {
   const { dbReadModel, dbMutationLog, schema, bootStatusQueue } = yield* LeaderThreadCtx
 
   const migrationOptions = schema.migrationOptions
+  let migrationsReport: MigrationsReport
 
   yield* Effect.addFinalizer(
     Effect.fn('recreateDb:finalizer')(function* (ex) {
@@ -27,13 +28,13 @@ export const recreateDb: Effect.Effect<
   // TODO bring back this optimization
   // const tmpDb = yield* makeSqliteDb({ _tag: 'in-memory' })
   const tmpDb = dbReadModel
-  yield* configureConnection(tmpDb, { fkEnabled: true })
+  yield* configureConnection(tmpDb, { foreignKeys: true })
 
   const initDb = (hooks: Partial<MigrationHooks> | undefined) =>
     Effect.gen(function* () {
       yield* Effect.tryAll(() => hooks?.init?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
-      yield* migrateDb({
+      const migrationsReport = yield* migrateDb({
         db: tmpDb,
         schema,
         onProgress: ({ done, total }) =>
@@ -44,16 +45,18 @@ export const recreateDb: Effect.Effect<
 
       yield* Effect.tryAll(() => hooks?.pre?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
-      return tmpDb
+      return { migrationsReport, tmpDb }
     })
 
   switch (migrationOptions.strategy) {
     case 'from-mutation-log': {
       const hooks = migrationOptions.hooks
-      const tmpDb = yield* initDb(hooks)
+      const initResult = yield* initDb(hooks)
+
+      migrationsReport = initResult.migrationsReport
 
       yield* rehydrateFromMutationLog({
-        db: tmpDb,
+        db: initResult.tmpDb,
         logDb: dbMutationLog,
         schema,
         migrationOptions,
@@ -61,23 +64,27 @@ export const recreateDb: Effect.Effect<
           Queue.offer(bootStatusQueue, { stage: 'rehydrating', progress: { done, total } }),
       })
 
-      yield* Effect.tryAll(() => hooks?.post?.(tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       break
     }
     case 'hard-reset': {
       const hooks = migrationOptions.hooks
-      const tmpInMemoryDb = yield* initDb(hooks)
+      const initResult = yield* initDb(hooks)
+
+      migrationsReport = initResult.migrationsReport
 
       // The database is migrated but empty now, so nothing else to do
 
-      yield* Effect.tryAll(() => hooks?.post?.(tmpInMemoryDb)).pipe(UnexpectedError.mapToUnexpectedError)
+      yield* Effect.tryAll(() => hooks?.post?.(initResult.tmpDb)).pipe(UnexpectedError.mapToUnexpectedError)
 
       break
     }
     case 'manual': {
       const oldDbData = dbReadModel.export()
 
+      migrationsReport = { migrations: [] }
+
       const newDbData = yield* Effect.tryAll(() => migrationOptions.migrate(oldDbData)).pipe(
         UnexpectedError.mapToUnexpectedError,
       )
@@ -106,6 +113,8 @@ export const recreateDb: Effect.Effect<
 
   // TODO bring back
   // tmpDb.close()
+
+  return { migrationsReport }
 }).pipe(
   Effect.scoped, // NOTE we're closing the scope here so finalizers are called when the effect is done
   Effect.withSpan('@livestore/common:leader-thread:recreateDb'),

package/src/leader-thread/types.ts

@@ -1,7 +1,6 @@
 import type {
   Deferred,
   Effect,
-  Fiber,
   HttpClient,
   Option,
   Queue,
@@ -17,6 +16,7 @@ import type {
   Devtools,
   InvalidPushError,
   MakeSqliteDb,
+  MigrationsReport,
   PersistenceInfo,
   SqliteDb,
   SyncBackend,
@@ -28,13 +28,6 @@ import type { ShutdownChannel } from './shutdown-channel.js'
 
 export type ShutdownState = 'running' | 'shutting-down'
 
-export class OuterWorkerCtx extends Context.Tag('OuterWorkerCtx')<
-  OuterWorkerCtx,
-  {
-    innerFiber: Fiber.RuntimeFiber<any, any>
-  }
->() {}
-
 export const InitialSyncOptionsSkip = Schema.TaggedStruct('Skip', {})
 export type InitialSyncOptionsSkip = typeof InitialSyncOptionsSkip.Type
 
@@ -67,7 +60,7 @@ export type DevtoolsOptions =
       enabled: true
       makeBootContext: Effect.Effect<
         {
-          devtoolsWebChannel: WebChannel.WebChannel<Devtools.MessageToAppLeader, Devtools.MessageFromAppLeader>
+          devtoolsWebChannel: WebChannel.WebChannel<Devtools.Leader.MessageToApp, Devtools.Leader.MessageFromApp>
           persistenceInfo: PersistenceInfoPair
         },
         UnexpectedError,
@@ -78,8 +71,10 @@ export type DevtoolsOptions =
 export type DevtoolsContext =
   | {
       enabled: true
-      syncBackendPullLatch: Effect.Latch
-      syncBackendPushLatch: Effect.Latch
+      // syncBackendPullLatch: Effect.Latch
+      // syncBackendPushLatch: Effect.Latch
+      syncBackendLatch: Effect.Latch
+      syncBackendLatchState: SubscriptionRef.SubscriptionRef<{ latchClosed: boolean }>
     }
   | {
       enabled: false
@@ -103,12 +98,16 @@ export class LeaderThreadCtx extends Context.Tag('LeaderThreadCtx')<
     syncBackend: SyncBackend | undefined
     syncProcessor: LeaderSyncProcessor
     connectedClientSessionPullQueues: PullQueueSet
+    initialState: {
+      leaderHead: EventId.EventId
+      migrationsReport: MigrationsReport
+    }
     /**
      * e.g. used for `store._dev` APIs
      *
      * This is currently separated from `.devtools` as it also needs to work when devtools are disabled
      */
-    extraIncomingMessagesQueue: Queue.Queue<Devtools.MessageToAppLeader>
+    extraIncomingMessagesQueue: Queue.Queue<Devtools.Leader.MessageToApp>
   }
 >() {}
 
@@ -138,7 +137,11 @@ export interface LeaderSyncProcessor {
   pushPartial: (mutationEvent: MutationEvent.PartialAnyEncoded) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx>
   boot: (args: {
     dbReady: Deferred.Deferred<void>
-  }) => Effect.Effect<void, UnexpectedError, LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient>
+  }) => Effect.Effect<
+    { initialLeaderHead: EventId.EventId },
+    UnexpectedError,
+    LeaderThreadCtx | Scope.Scope | HttpClient.HttpClient
+  >
   syncState: Subscribable.Subscribable<SyncState.SyncState>
 }
 

package/src/query-builder/api.ts

@@ -35,7 +35,7 @@ export namespace QueryBuilderAst {
   export type RowQuery = {
     readonly _tag: 'RowQuery'
     readonly tableDef: DbSchema.TableDefBase
-    readonly id: string | SessionIdSymbol
+    readonly id: string | SessionIdSymbol | number
     readonly insertValues: Record<string, unknown>
   }
 
@@ -252,10 +252,10 @@ export namespace QueryBuilder {
         ? (_: 'Error: Need to enable deriveMutations to use row()') => any
         : TTableDef['options']['requiredInsertColumnNames'] extends never
           ? (
-              id: string | SessionIdSymbol,
+              id: string | SessionIdSymbol | number,
             ) => QueryBuilder<RowQuery.Result<TTableDef>, TTableDef, QueryBuilder.ApiFeature, QueryInfo.Row>
           : <TOptions extends RowQuery.RequiredColumnsOptions<TTableDef>>(
-              id: string | SessionIdSymbol,
+              id: string | SessionIdSymbol | number,
               opts: TOptions,
             ) => QueryBuilder<RowQuery.Result<TTableDef>, TTableDef, QueryBuilder.ApiFeature, QueryInfo.Row>
   }

package/src/query-builder/impl.test.ts

@@ -18,13 +18,23 @@ const todos = DbSchema.table(
   { deriveMutations: true },
 )
 
+const todosWithIntId = DbSchema.table(
+  'todos_with_int_id',
+  {
+    id: DbSchema.integer({ primaryKey: true }),
+    text: DbSchema.text({ default: '', nullable: false }),
+    status: DbSchema.text({ schema: Schema.Literal('active', 'completed') }),
+  },
+  { deriveMutations: true },
+)
+
 const comments = DbSchema.table('comments', {
   id: DbSchema.text({ primaryKey: true }),
   text: DbSchema.text({ default: '', nullable: false }),
   todoId: DbSchema.text({}),
 })
 
-const db = { todos: todos.query, comments: comments.query }
+const db = { todos: todos.query, todosWithIntId: todosWithIntId.query, comments: comments.query }
 
 describe('query builder', () => {
   describe('result schema', () => {
@@ -204,6 +214,17 @@ describe('query builder', () => {
         }
       `)
     })
+
+    it('should handle row queries with numbers', () => {
+      expect(db.todosWithIntId.row(123, { insertValues: { status: 'active' } }).asSql()).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            123,
+          ],
+          "query": "SELECT * FROM 'todos_with_int_id' WHERE id = ?",
+        }
+      `)
+    })
   })
 })
 

package/src/query-builder/impl.ts

@@ -128,12 +128,12 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
       // eslint-disable-next-line prefer-rest-params
       const params = [...arguments]
 
-      let id: string
+      let id: string | number
 
       if (tableDef.options.isSingleton) {
         id = tableDef.sqliteDef.columns.id!.default.pipe(Option.getOrThrow)
       } else {
-        id = params[0] as string
+        id = params[0] as string | number
         if (id === undefined) {
           invalidQueryBuilder(`Id missing for row query on non-singleton table ${tableDef.sqliteDef.name}`)
         }

package/src/query-info.ts

@@ -24,20 +24,20 @@ export namespace QueryInfo {
   export type Row = {
     _tag: 'Row'
     table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
+    id: string | SessionIdSymbol | number
   }
 
   export type Col = {
     _tag: 'Col'
     table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
+    id: string | SessionIdSymbol | number
     column: string
   }
 
   export type ColJsonValue = {
     _tag: 'ColJsonValue'
     table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
+    id: string | SessionIdSymbol | number
     column: string
     /**
      * example: `$.tabs[3].items[2]` (`$` referring to the column value)

package/src/schema/EventId.ts

@@ -43,6 +43,10 @@ export const isGreaterThan = (a: EventId, b: EventId) => {
   return a.global > b.global || (a.global === b.global && a.local > b.local)
 }
 
+export const isGreaterThanOrEqual = (a: EventId, b: EventId) => {
+  return a.global > b.global || (a.global === b.global && a.local >= b.local)
+}
+
 export const make = (id: EventId | typeof EventId.Encoded): EventId => {
   return Schema.is(EventId)(id) ? id : Schema.decodeSync(EventId)(id)
 }

package/src/schema-management/migrations.ts

@@ -2,7 +2,7 @@ import { SqliteAst, SqliteDsl } from '@livestore/db-schema'
 import { memoizeByStringifyArgs } from '@livestore/utils'
 import { Effect, Schema as EffectSchema } from '@livestore/utils/effect'
 
-import type { SqliteDb } from '../adapter-types.js'
+import type { MigrationsReport, MigrationsReportEntry, SqliteDb, UnexpectedError } from '../adapter-types.js'
 import type { LiveStoreSchema } from '../schema/mod.js'
 import type { SchemaMetaRow, SchemaMutationsMetaRow } from '../schema/system-tables.js'
 import {
@@ -54,7 +54,7 @@ export const migrateDb = ({
   db: SqliteDb
   schema: LiveStoreSchema
   onProgress?: (opts: { done: number; total: number }) => Effect.Effect<void>
-}) =>
+}): Effect.Effect<MigrationsReport, UnexpectedError> =>
   Effect.gen(function* () {
     yield* migrateTable({
       db,
@@ -81,6 +81,7 @@ export const migrateDb = ({
 
     const tablesToMigrate = new Set<{ tableAst: SqliteAst.Table; schemaHash: number }>()
 
+    const migrationsReportEntries: MigrationsReportEntry[] = []
     for (const tableDef of tableDefs) {
       const tableAst = tableDef.sqliteDef.ast
       const tableName = tableAst.name
@@ -90,9 +91,10 @@ export const migrateDb = ({
       if (schemaHash !== dbSchemaHash) {
         tablesToMigrate.add({ tableAst, schemaHash })
 
-        console.log(
-          `Schema hash mismatch for table '${tableName}' (DB: ${dbSchemaHash}, expected: ${schemaHash}), migrating table...`,
-        )
+        migrationsReportEntries.push({
+          tableName,
+          hashes: { expected: schemaHash, actual: dbSchemaHash },
+        })
       }
     }
 
@@ -107,6 +109,8 @@ export const migrateDb = ({
         yield* onProgress({ done: processedTables, total: tablesCount })
       }
     }
+
+    return { migrations: migrationsReportEntries }
   })
 
 export const migrateTable = ({

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,13 +1,13 @@
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
 import type { Runtime, Scope } from '@livestore/utils/effect'
-import { Effect, Schema, Stream } from '@livestore/utils/effect'
+import { Effect, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
 import type { ClientSession, UnexpectedError } from '../adapter-types.js'
 import * as EventId from '../schema/EventId.js'
 import { type LiveStoreSchema } from '../schema/mod.js'
 import * as MutationEvent from '../schema/MutationEvent.js'
-import { SyncState, updateSyncState } from './syncstate.js'
+import * as SyncState from './syncstate.js'
 
 /**
  * Rebase behaviour:
@@ -44,15 +44,17 @@ export const makeClientSessionSyncProcessor = ({
   const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
 
   const syncStateRef = {
-    current: new SyncState({
-      localHead: clientSession.leaderThread.mutations.initialMutationEventId,
-      upstreamHead: clientSession.leaderThread.mutations.initialMutationEventId,
+    current: new SyncState.SyncState({
+      localHead: clientSession.leaderThread.initialState.leaderHead,
+      upstreamHead: clientSession.leaderThread.initialState.leaderHead,
       pending: [],
       // TODO init rollbackTail from leader to be ready for backend rebasing
       rollbackTail: [],
     }),
   }
 
+  const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
+
   const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
     const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
     return mutationDef.options.localOnly
@@ -71,7 +73,7 @@ export const makeClientSessionSyncProcessor = ({
       )
     })
 
-    const updateResult = updateSyncState({
+    const updateResult = SyncState.updateSyncState({
       syncState: syncStateRef.current,
       payload: { _tag: 'local-push', newEvents: encodedMutationEvents },
       isLocalEvent,
@@ -88,6 +90,7 @@ export const makeClientSessionSyncProcessor = ({
     }
 
     syncStateRef.current = updateResult.newSyncState
+    syncStateUpdateQueue.offer(updateResult.newSyncState).pipe(Effect.runSync)
 
     const writeTables = new Set<string>()
     for (const mutationEvent of updateResult.newEvents) {
@@ -119,7 +122,7 @@ export const makeClientSessionSyncProcessor = ({
             yield* clientSession.devtools.pullLatch.await
           }
 
-          const updateResult = updateSyncState({
+          const updateResult = SyncState.updateSyncState({
             syncState: syncStateRef.current,
             payload,
             isLocalEvent,
@@ -132,6 +135,7 @@ export const makeClientSessionSyncProcessor = ({
           }
 
           syncStateRef.current = updateResult.newSyncState
+          syncStateUpdateQueue.offer(updateResult.newSyncState).pipe(Effect.runSync)
 
           if (updateResult._tag === 'rebase') {
             span.addEvent('pull:rebase', {
@@ -143,11 +147,11 @@ export const makeClientSessionSyncProcessor = ({
               remaining,
             })
             if (LS_DEV) {
-              console.debug(
+              Effect.logDebug(
                 'pull:rebase: rollback',
                 updateResult.eventsToRollback.length,
                 ...updateResult.eventsToRollback.map((_) => _.toJSON()),
-              )
+              ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
             for (let i = updateResult.eventsToRollback.length - 1; i >= 0; i--) {
@@ -197,7 +201,14 @@ export const makeClientSessionSyncProcessor = ({
   return {
     push,
     boot,
-    syncStateRef,
+    syncState: Subscribable.make({
+      get: Effect.gen(function* () {
+        const syncState = syncStateRef.current
+        if (syncStateRef === undefined) return shouldNeverHappen('Not initialized')
+        return syncState
+      }),
+      changes: Stream.fromQueue(syncStateUpdateQueue),
+    }),
   } satisfies ClientSessionSyncProcessor
 }
 
@@ -210,5 +221,5 @@ export interface ClientSessionSyncProcessor {
   }
   boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
 
-  syncStateRef: { current: SyncState }
+  syncState: Subscribable.Subscribable<SyncState.SyncState>
 }

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.3.0-dev.11' as const
+export const liveStoreVersion = '0.3.0-dev.12' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.