@livestore/common 0.0.54-dev.21 → 0.0.54-dev.23

package/src/adapter-types.ts

@@ -1,6 +1,5 @@
-import type { Stream, SubscriptionRef, TRef } from '@livestore/utils/effect'
+import type { Effect, Queue, Stream, SubscriptionRef } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
-import type * as otel from '@opentelemetry/api'
 
 import type { LiveStoreSchema, MutationEvent } from './schema/index.js'
 import type { PreparedBindValues } from './util.js'
@@ -22,7 +21,6 @@ export type InMemoryDatabase = {
   _tag: 'InMemoryDatabase'
   prepare(queryStr: string): PreparedStatement
   execute(queryStr: string, bindValues: PreparedBindValues | undefined): GetRowsChangedCount
-  dangerouslyReset(): Promise<void>
   export(): Uint8Array
 }
 
@@ -38,27 +36,54 @@ export type NetworkStatus = {
   timestampMs: number
 }
 
+export const BootStateProgress = Schema.Struct({
+  done: Schema.Number,
+  total: Schema.Number,
+})
+
+export const BootStatus = Schema.Union(
+  Schema.Struct({ stage: Schema.Literal('loading') }),
+  Schema.Struct({ stage: Schema.Literal('migrating'), progress: BootStateProgress }),
+  Schema.Struct({ stage: Schema.Literal('rehydrating'), progress: BootStateProgress }),
+  Schema.Struct({ stage: Schema.Literal('syncing'), progress: BootStateProgress }),
+  Schema.Struct({ stage: Schema.Literal('done') }),
+)
+
+export type BootStatus = typeof BootStatus.Type
+
 export type Coordinator = {
+  isShutdownRef: { current: boolean }
   devtools: {
+    enabled: boolean
     channelId: string
+    /**
+     * Returns a dedicated message port for the store which is established over the message port passed in
+     */
+    connect: (options: {
+      port: MessagePort
+      connectionId: string
+    }) => Effect.Effect<{ storeMessagePort: MessagePort }, UnexpectedError>
   }
-  hasLock: TRef.TRef<boolean>
-  syncMutations: Stream.Stream<MutationEvent.AnyEncoded>
-  execute(queryStr: string, bindValues: PreparedBindValues | undefined, span: otel.Span | undefined): Promise<void>
-  mutate(mutationEventEncoded: MutationEvent.Any, options: { span: otel.Span; persisted: boolean }): Promise<void>
-  dangerouslyReset(mode: ResetMode): Promise<void>
-  export(span: otel.Span | undefined): Promise<Uint8Array | undefined>
+  // TODO is exposing the lock status really needed (or only relevant for web adapter?)
+  lockStatus: SubscriptionRef.SubscriptionRef<LockStatus>
+  syncMutations: Stream.Stream<MutationEvent.AnyEncoded, UnexpectedError>
+  execute(queryStr: string, bindValues: PreparedBindValues | undefined): Effect.Effect<void, UnexpectedError>
+  mutate(mutationEventEncoded: MutationEvent.Any, options: { persisted: boolean }): Effect.Effect<void, UnexpectedError>
+  dangerouslyReset(mode: ResetMode): Effect.Effect<void, UnexpectedError>
+  export: Effect.Effect<Uint8Array | undefined, UnexpectedError>
   /**
    * This is different from `export` since in `getInitialSnapshot` is usually the place for migrations etc to happen
    */
-  getInitialSnapshot(): Promise<Uint8Array>
-  getMutationLogData(): Promise<Uint8Array>
-  shutdown(): Promise<void>
+  getInitialSnapshot: Effect.Effect<Uint8Array, UnexpectedError>
+  getMutationLogData: Effect.Effect<Uint8Array, UnexpectedError>
+  shutdown: Effect.Effect<void, UnexpectedError>
   networkStatus: SubscriptionRef.SubscriptionRef<NetworkStatus>
 }
 
 export type GetRowsChangedCount = () => number
 
+export type LockStatus = 'has-lock' | 'no-lock'
+
 export type BootDb = {
   _tag: 'BootDb'
   execute(queryStr: string, bindValues?: PreparedBindValues): void
@@ -67,6 +92,19 @@ export type BootDb = {
   txn(callback: () => void): void
 }
 
+export class UnexpectedError extends Schema.TaggedError<UnexpectedError>()('LiveStore.UnexpectedError', {
+  cause: Schema.AnyError,
+}) {}
+
+export class SqliteError extends Schema.TaggedError<SqliteError>()('LiveStore.SqliteError', {
+  sql: Schema.String,
+  bindValues: Schema.Record(Schema.String, Schema.Any),
+  /** The SQLite result code */
+  code: Schema.Number,
+  /** The original SQLite3 error */
+  cause: Schema.AnyError,
+}) {}
+
 // TODO possibly allow a combination of these options
 // TODO allow a way to stream the migration progress back to the app
 export type MigrationOptions<TSchema extends LiveStoreSchema = LiveStoreSchema> =
@@ -107,7 +145,7 @@ export type MigrationOptionsFromMutationLog<TSchema extends LiveStoreSchema = Li
 }
 
 export type StoreAdapterFactory = (opts: {
-  otelTracer: otel.Tracer
-  otelContext: otel.Context
   schema: LiveStoreSchema
-}) => StoreAdapter | Promise<StoreAdapter>
+  devtoolsEnabled: boolean
+  bootStatusQueue: Queue.Queue<BootStatus>
+}) => Effect.Effect<StoreAdapter, UnexpectedError>

package/src/devtools/devtools-messages.ts

@@ -1,11 +1,10 @@
-import { version as pkgVersion } from '@livestore/common/package.json'
-import { Schema } from '@livestore/utils/effect'
-import { type SqliteDsl as __SqliteDsl } from 'effect-db-schema'
+import { Schema, Transferable } from '@livestore/utils/effect'
 
 import { NetworkStatus } from '../adapter-types.js'
 import { DebugInfo } from '../debug-info.js'
 import { mutationEventSchemaEncodedAny } from '../schema/mutations.js'
 import { PreparedBindValues } from '../util.js'
+import { liveStoreVersion as pkgVersion } from '../version.js'
 
 const requestId = Schema.String
 const channelId = Schema.String
@@ -20,20 +19,21 @@ export class SnapshotReq extends Schema.TaggedStruct('LSD.SnapshotReq', {
 export class SnapshotRes extends Schema.TaggedStruct('LSD.SnapshotRes', {
   liveStoreVersion,
   requestId,
-  snapshot: Schema.Uint8Array,
+  snapshot: Transferable.Uint8Array,
 }).annotations({ identifier: 'LSD.SnapshotRes' }) {}
 
-export class LoadSnapshotReq extends Schema.TaggedStruct('LSD.LoadSnapshotReq', {
+export class LoadDatabaseFileReq extends Schema.TaggedStruct('LSD.LoadDatabaseFileReq', {
   liveStoreVersion,
   requestId,
   channelId,
-  snapshot: Schema.Uint8Array,
-}).annotations({ identifier: 'LSD.LoadSnapshotReq' }) {}
+  data: Transferable.Uint8Array,
+}).annotations({ identifier: 'LSD.LoadDatabaseFileReq' }) {}
 
-export class LoadSnapshotRes extends Schema.TaggedStruct('LSD.LoadSnapshotRes', {
+export class LoadDatabaseFileRes extends Schema.TaggedStruct('LSD.LoadDatabaseFileRes', {
   liveStoreVersion,
   requestId,
-}).annotations({ identifier: 'LSD.LoadSnapshotRes' }) {}
+  status: Schema.Literal('ok', 'unsupported-file', 'unsupported-database'),
+}).annotations({ identifier: 'LSD.LoadDatabaseFileRes' }) {}
 
 export class DebugInfoReq extends Schema.TaggedStruct('LSD.DebugInfoReq', {
   liveStoreVersion,
@@ -102,21 +102,9 @@ export class MutationLogRes extends Schema.TaggedStruct('LSD.MutationLogRes', {
   liveStoreVersion,
   requestId,
   channelId,
-  mutationLog: Schema.Uint8Array,
+  mutationLog: Transferable.Uint8Array,
 }).annotations({ identifier: 'LSD.MutationLogRes' }) {}
 
-export class LoadMutationLogReq extends Schema.TaggedStruct('LSD.LoadMutationLogReq', {
-  liveStoreVersion,
-  requestId,
-  channelId,
-  mutationLog: Schema.Uint8Array,
-}).annotations({ identifier: 'LSD.LoadMutationLogReq' }) {}
-
-export class LoadMutationLogRes extends Schema.TaggedStruct('LSD.LoadMutationLogRes', {
-  liveStoreVersion,
-  requestId,
-}).annotations({ identifier: 'LSD.LoadMutationLogRes' }) {}
-
 export class ReactivityGraphSubscribe extends Schema.TaggedStruct('LSD.ReactivityGraphSubscribe', {
   liveStoreVersion,
   requestId,
@@ -178,6 +166,19 @@ export class ResetAllDataRes extends Schema.TaggedStruct('LSD.ResetAllDataRes',
   requestId,
 }).annotations({ identifier: 'LSD.ResetAllDataRes' }) {}
 
+export class MessagePortForStoreReq extends Schema.TaggedStruct('LSD.MessagePortForStoreReq', {
+  liveStoreVersion,
+  requestId,
+  channelId,
+}).annotations({ identifier: 'LSD.MessagePortForStoreReq' }) {}
+
+export class MessagePortForStoreRes extends Schema.TaggedStruct('LSD.MessagePortForStoreRes', {
+  liveStoreVersion,
+  requestId,
+  channelId,
+  port: Transferable.MessagePort,
+}).annotations({ identifier: 'LSD.MessagePortForStoreRes' }) {}
+
 export class NetworkStatusChanged extends Schema.TaggedStruct('LSD.NetworkStatusChanged', {
   liveStoreVersion,
   channelId,
@@ -204,11 +205,33 @@ export class Disconnect extends Schema.TaggedStruct('LSD.Disconnect', {
   channelId,
 }).annotations({ identifier: 'LSD.Disconnect' }) {}
 
-export const MessageToAppHost = Schema.Union(
+export class Ping extends Schema.TaggedStruct('LSD.Ping', {
+  liveStoreVersion,
+  requestId,
+  channelId,
+}).annotations({ identifier: 'LSD.Ping' }) {}
+
+export class Pong extends Schema.TaggedStruct('LSD.Pong', {
+  liveStoreVersion,
+  requestId,
+}).annotations({ identifier: 'LSD.Pong' }) {}
+
+export const MessageToAppHostCoordinator = Schema.Union(
   SnapshotReq,
-  LoadSnapshotReq,
+  LoadDatabaseFileReq,
   MutationLogReq,
-  LoadMutationLogReq,
+  ResetAllDataReq,
+  MessagePortForStoreRes,
+  DevtoolsReady,
+  Disconnect,
+  DevtoolsConnected,
+  RunMutationReq,
+  Ping,
+).annotations({ identifier: 'LSD.MessageToAppHostCoordinator' })
+
+export type MessageToAppHostCoordinator = typeof MessageToAppHostCoordinator.Type
+
+export const MessageToAppHostStore = Schema.Union(
   DebugInfoReq,
   DebugInfoResetReq,
   DebugInfoRerunQueryReq,
@@ -216,37 +239,34 @@ export const MessageToAppHost = Schema.Union(
   ReactivityGraphUnsubscribe,
   LiveQueriesSubscribe,
   LiveQueriesUnsubscribe,
-  ResetAllDataReq,
-  DevtoolsReady,
-  Disconnect,
-  DevtoolsConnected,
-  RunMutationReq,
-).annotations({ identifier: 'LSD.MessageToAppHost' })
+  // Ping,
+).annotations({ identifier: 'LSD.MessageToAppHostStore' })
 
-export type MessageToAppHost = typeof MessageToAppHost.Type
+export type MessageToAppHostStore = typeof MessageToAppHostStore.Type
 
-export const MessageFromAppHost = Schema.Union(
+export const MessageFromAppHostCoordinator = Schema.Union(
   SnapshotRes,
-  LoadSnapshotRes,
+  LoadDatabaseFileRes,
   MutationLogRes,
-  LoadMutationLogRes,
-  DebugInfoRes,
-  DebugInfoResetRes,
-  DebugInfoRerunQueryRes,
-  ReactivityGraphRes,
-  LiveQueriesRes,
   ResetAllDataRes,
+  MessagePortForStoreReq,
   Disconnect,
   MutationBroadcast,
   AppHostReady,
   NetworkStatusChanged,
   RunMutationRes,
-).annotations({ identifier: 'LSD.MessageFromAppHost' })
+  Pong,
+).annotations({ identifier: 'LSD.MessageFromAppHostCoordinator' })
+
+export type MessageFromAppHostCoordinator = typeof MessageFromAppHostCoordinator.Type
 
-export type MessageFromAppHost = typeof MessageFromAppHost.Type
+export const MessageFromAppHostStore = Schema.Union(
+  DebugInfoRes,
+  DebugInfoResetRes,
+  DebugInfoRerunQueryRes,
+  ReactivityGraphRes,
+  LiveQueriesRes,
+  // Pong,
+).annotations({ identifier: 'LSD.MessageFromAppHostStore' })
 
-// TODO make specific over app key
-export const makeBroadcastChannels = () => ({
-  fromAppHost: new BroadcastChannel(`livestore-devtools-from-app-host`),
-  toAppHost: new BroadcastChannel(`livestore-devtools-to-app-host`),
-})
+export type MessageFromAppHostStore = typeof MessageFromAppHostStore.Type

package/src/devtools/devtools-window-message.ts

@@ -0,0 +1,25 @@
+import { Schema, Transferable } from '@livestore/utils/effect'
+
+const channelId = Schema.String
+
+export namespace DevtoolsWindowMessage {
+  /** Message is being created in contentscript-iframe, sent to contentscript and then sent to Store */
+  export class MessagePortReady extends Schema.TaggedStruct('LSD.WindowMessage.MessagePortForStore', {
+    port: Transferable.MessagePort,
+    channelId,
+  }) {}
+
+  export class ContentscriptListening extends Schema.TaggedStruct('LSD.WindowMessage.ContentscriptListening', {}) {}
+
+  // export class ContentscriptReady extends Schema.TaggedStruct('LSD.WindowMessage.ContentscriptReady', {
+  // 	channelId,
+  // }) {}
+
+  export class StoreReady extends Schema.TaggedStruct('LSD.WindowMessage.StoreReady', {
+    channelId,
+  }) {}
+
+  export class MessageForStore extends Schema.Union(MessagePortReady, ContentscriptListening) {}
+
+  export class MessageForContentscript extends Schema.Union(StoreReady) {}
+}

package/src/devtools/index.ts

@@ -1 +1,2 @@
 export * from './devtools-messages.js'
+export * from './devtools-window-message.js'

package/src/index.ts

@@ -11,3 +11,4 @@ export * from './sync/index.js'
 export * as Devtools from './devtools/index.js'
 export * from './debug-info.js'
 export * from './bounded-collections.js'
+export * from './version.js'

package/src/rehydrate-from-mutationlog.ts

@@ -1,91 +1,127 @@
 import { shouldNeverHappen } from '@livestore/utils'
-import { Schema } from '@livestore/utils/effect'
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
 
-import type { InMemoryDatabase, MigrationOptionsFromMutationLog } from './adapter-types.js'
+import { type InMemoryDatabase, type MigrationOptionsFromMutationLog, SqliteError } from './adapter-types.js'
 import { getExecArgsFromMutation } from './mutation.js'
 import type { LiveStoreSchema, MutationLogMetaRow } from './schema/index.js'
 import { MUTATION_LOG_META_TABLE } from './schema/index.js'
+import type { PreparedBindValues } from './util.js'
+import { sql } from './util.js'
 
-export const rehydrateFromMutationLog = async ({
+export const rehydrateFromMutationLog = ({
   logDb,
   db,
   schema,
   migrationOptions,
+  onProgress,
 }: {
   logDb: InMemoryDatabase
   db: InMemoryDatabase
   schema: LiveStoreSchema
   migrationOptions: MigrationOptionsFromMutationLog
-}) => {
-  try {
-    // TODO possibly implement this in a streaming fashion
-    const stmt = logDb.prepare(`SELECT * FROM ${MUTATION_LOG_META_TABLE} ORDER BY id ASC`)
-    const results = stmt.select<MutationLogMetaRow>(undefined)
+  onProgress: (_: { done: number; total: number }) => Effect.Effect<void>
+}) =>
+  Effect.gen(function* () {
+    const mutationsCount = logDb
+      .prepare(`SELECT COUNT(*) AS count FROM ${MUTATION_LOG_META_TABLE}`)
+      .select<{ count: number }>(undefined)[0]!.count
 
-    performance.mark('livestore:hydrate-from-mutationlog:start')
+    const processMutation = (row: MutationLogMetaRow) =>
+      Effect.gen(function* () {
+        const mutationDef = schema.mutations.get(row.mutation) ?? shouldNeverHappen(`Unknown mutation ${row.mutation}`)
 
-    for (const row of results) {
-      const mutationDef = schema.mutations.get(row.mutation) ?? shouldNeverHappen(`Unknown mutation ${row.mutation}`)
+        if (migrationOptions.excludeMutations?.has(row.mutation) === true) return
 
-      if (migrationOptions.excludeMutations?.has(row.mutation) === true) continue
-
-      if (Schema.hash(mutationDef.schema) !== row.schemaHash) {
-        console.warn(`Schema hash mismatch for mutation ${row.mutation}. Trying to apply mutation anyway.`)
-      }
+        if (Schema.hash(mutationDef.schema) !== row.schemaHash) {
+          console.warn(`Schema hash mismatch for mutation ${row.mutation}. Trying to apply mutation anyway.`)
+        }
 
-      const argsDecodedEither = Schema.decodeUnknownEither(Schema.parseJson(mutationDef.schema))(row.argsJson)
-      if (argsDecodedEither._tag === 'Left') {
-        return shouldNeverHappen(`\
+        const argsDecodedEither = Schema.decodeUnknownEither(Schema.parseJson(mutationDef.schema))(row.argsJson)
+        if (argsDecodedEither._tag === 'Left') {
+          return shouldNeverHappen(`\
 There was an error decoding the persisted mutation event args for mutation "${row.mutation}".
 This likely means the schema has changed in an incompatible way.
 
 Error: ${argsDecodedEither.left}
         `)
-      }
-
-      const mutationEventDecoded = {
-        id: row.id,
-        mutation: row.mutation,
-        args: argsDecodedEither.right,
-      }
-      // const argsEncoded = JSON.parse(row.args_json)
-      // const mutationSqlRes =
-      //   typeof mutation.sql === 'string'
-      //     ? mutation.sql
-      //     : mutation.sql(Schema.decodeUnknownSync(mutation.schema)(argsEncoded))
-      // const mutationSql = typeof mutationSqlRes === 'string' ? mutationSqlRes : mutationSqlRes.sql
-      // const bindValues = typeof mutationSqlRes === 'string' ? argsEncoded : mutationSqlRes.bindValues
-
-      const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
-
-      for (const { statementSql, bindValues } of execArgsArr) {
-        try {
-          const getRowsChanged = db.execute(statementSql, bindValues)
-          if (
-            import.meta.env.DEV &&
-            getRowsChanged() === 0 &&
-            migrationOptions.logging?.excludeAffectedRows?.(statementSql) !== true
-          ) {
-            console.warn(`Mutation "${mutationDef.name}" did not affect any rows:`, statementSql, bindValues)
+        }
+
+        const mutationEventDecoded = {
+          id: row.id,
+          mutation: row.mutation,
+          args: argsDecodedEither.right,
+        }
+        // const argsEncoded = JSON.parse(row.args_json)
+        // const mutationSqlRes =
+        //   typeof mutation.sql === 'string'
+        //     ? mutation.sql
+        //     : mutation.sql(Schema.decodeUnknownSync(mutation.schema)(argsEncoded))
+        // const mutationSql = typeof mutationSqlRes === 'string' ? mutationSqlRes : mutationSqlRes.sql
+        // const bindValues = typeof mutationSqlRes === 'string' ? argsEncoded : mutationSqlRes.bindValues
+
+        const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+
+        for (const { statementSql, bindValues } of execArgsArr) {
+          try {
+            // TODO cache prepared statements for mutations
+            const getRowsChanged = db.execute(statementSql, bindValues)
+            if (
+              import.meta.env.DEV &&
+              getRowsChanged() === 0 &&
+              migrationOptions.logging?.excludeAffectedRows?.(statementSql) !== true
+            ) {
+              console.warn(`Mutation "${mutationDef.name}" did not affect any rows:`, statementSql, bindValues)
+            }
+            // console.log(`Re-executed mutation ${mutationSql}`, bindValues)
+          } catch (e) {
+            yield* new SqliteError({
+              sql: statementSql,
+              bindValues,
+              code: (e as any).resultCode,
+              cause: e,
+            })
           }
-          // console.log(`Re-executed mutation ${mutationSql}`, bindValues)
-        } catch (e) {
-          console.error(`Error executing migration for mutation ${statementSql}`, bindValues, e)
-          debugger
-          throw e
         }
-      }
-    }
-  } catch (e) {
-    console.error('Error while rehydrating database from mutation log', e)
-    debugger
-    throw e
-  } finally {
-    performance.mark('livestore:hydrate-from-mutationlog:end')
-    performance.measure(
-      'livestore:hydrate-from-mutationlog',
-      'livestore:hydrate-from-mutationlog:start',
-      'livestore:hydrate-from-mutationlog:end',
+      }).pipe(Effect.withSpan(`@livestore/common:rehydrateFromMutationLog:processMutation`))
+
+    const CHUNK_SIZE = 100
+
+    const stmt = logDb.prepare(sql`\
+SELECT * FROM ${MUTATION_LOG_META_TABLE} 
+WHERE id > COALESCE($id, '') 
+ORDER BY id ASC
+LIMIT ${CHUNK_SIZE}
+`)
+
+    let processedMutations = 0
+
+    yield* Stream.unfoldChunk<Chunk.Chunk<MutationLogMetaRow> | { _tag: 'Initial ' }, MutationLogMetaRow>(
+      { _tag: 'Initial ' },
+      (item) => {
+        // End stream if no more rows
+        if (Chunk.isChunk(item) && item.length === 0) return Option.none()
+
+        const lastId = Chunk.isChunk(item) ? Chunk.last(item).pipe(Option.getOrUndefined)?.id : undefined
+        const nextItem = Chunk.fromIterable(
+          stmt.select<MutationLogMetaRow>({ $id: lastId } as any as PreparedBindValues),
+        )
+        const prevItem = Chunk.isChunk(item) ? item : Chunk.empty()
+        return Option.some([prevItem, nextItem])
+      },
+    ).pipe(
+      (_) => _,
+      Stream.bufferChunks({ capacity: 2 }),
+      Stream.tap((row) =>
+        Effect.gen(function* () {
+          yield* processMutation(row)
+
+          processedMutations++
+          yield* onProgress({ done: processedMutations, total: mutationsCount })
+        }),
+      ),
+      Stream.runDrain,
     )
-  }
-}
+  }).pipe(
+    Effect.withPerformanceMeasure('@livestore/common:rehydrateFromMutationLog'),
+    Effect.withSpan('@livestore/common:rehydrateFromMutationLog'),
+  )

package/src/schema/table-def.ts

@@ -86,8 +86,6 @@ export type TableOptions = {
    * @default false
    */
   isSingleton: boolean
-  // TODO remove
-  dynamicRegistration: boolean
   disableAutomaticIdColumn: boolean
   /**
    * Setting this to true will automatically derive insert, update and delete mutations for this table. Example:
@@ -139,7 +137,6 @@ export const table = <
 
   const options_: TableOptions = {
     isSingleton: options?.isSingleton ?? false,
-    dynamicRegistration: options?.dynamicRegistration ?? false,
     disableAutomaticIdColumn: options?.disableAutomaticIdColumn ?? false,
     deriveMutations:
       options?.deriveMutations === true
@@ -239,7 +236,6 @@ type WithId<TColumns extends SqliteDsl.Columns, TOptions extends TableOptions> =
 
 type WithDefaults<TOptionsInput extends TableOptionsInput, TIsSingleColumn extends boolean> = {
   isSingleton: TOptionsInput['isSingleton'] extends true ? true : false
-  dynamicRegistration: TOptionsInput['dynamicRegistration'] extends true ? true : false
   disableAutomaticIdColumn: TOptionsInput['disableAutomaticIdColumn'] extends true ? true : false
   deriveMutations: TOptionsInput['deriveMutations'] extends true
     ? { enabled: true; localOnly: boolean }