@livestore/common 0.0.0-snapshot-83c6b3d6e39244b59235009057fd5c48f6c3103f → 0.0.0-snapshot-2b8a9de3ec1a701aca891ebc2c98eb328274ae9e

package/src/query-info.ts

@@ -9,7 +9,7 @@ import type { DbSchema } from './schema/mod.js'
  *
  * This information is currently only used for derived mutations.
  */
-export type QueryInfo = QueryInfo.None | QueryInfo.Row | QueryInfo.Col | QueryInfo.ColJsonValue
+export type QueryInfo = QueryInfo.None | QueryInfo.Row | QueryInfo.Col | QueryInfo.ColJsonValue | QueryInfo.Write
 // export type QueryInfo<TTableDef extends DbSchema.TableDefBase = DbSchema.TableDefBase> =
 // | QueryInfo.None
 // | QueryInfo.Row<TTableDef>
@@ -45,6 +45,11 @@ export namespace QueryInfo {
     jsonPath: string
   }
 
+  // NOTE Not yet used but we might want to use this in order to avoid write queries in read-only situations
+  export type Write = {
+    _tag: 'Write'
+  }
+
   // NOTE maybe we want to bring back type-params back like below
   // export type Row<TTableDef extends DbSchema.TableDefBase> = {
   //   _tag: 'Row'

package/src/schema/MutationEvent.ts

@@ -178,10 +178,26 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEven
     }
   }
 
-  rebase = (parentId: EventId.EventId, isLocal: boolean) =>
+  /**
+   * Example: (global event)
+   * For event id (2,0) → (1,0) which should be rebased on event id (3,1) → (3,0)
+   * the resulting event id will be (4,0) → (3,0)
+   *
+   * Example: (client event)
+   * For event id (2,1) → (2,0) which should be rebased on event id (3,0) → (2,0)
+   * the resulting event id will be (3,1) → (3,0)
+   *
+   * Syntax: (2,1) → (2,0)
+   *          ^ ^     ^ ^
+   *          | |     | +- client parent id
+   *          | |     +--- global parent id
+   *          | +-- client id
+   *          +---- global id
+   */
+  rebase = (parentId: EventId.EventId, isClient: boolean) =>
     new EncodedWithMeta({
       ...this,
-      ...EventId.nextPair(parentId, isLocal),
+      ...EventId.nextPair(parentId, isClient),
     })
 
   static fromGlobal = (mutationEvent: AnyEncodedGlobal) =>

package/src/schema/db-schema/dsl/mod.ts

@@ -52,8 +52,13 @@ export type AnyIfConstained<In, Out> = '__constrained' extends keyof In ? any :
 export type EmptyObjIfConstained<In> = '__constrained' extends keyof In ? {} : In
 
 export type StructSchemaForColumns<TCols extends ConstraintColumns> = Schema.Schema<
-  AnyIfConstained<TCols, { readonly [K in keyof TCols]: TCols[K]['schema']['Type'] }>,
-  AnyIfConstained<TCols, { readonly [K in keyof TCols]: TCols[K]['schema']['Encoded'] }>
+  AnyIfConstained<TCols, FromColumns.RowDecoded<TCols>>,
+  AnyIfConstained<TCols, FromColumns.RowEncoded<TCols>>
+>
+
+export type InsertStructSchemaForColumns<TCols extends ConstraintColumns> = Schema.Schema<
+  AnyIfConstained<TCols, FromColumns.InsertRowDecoded<TCols>>,
+  AnyIfConstained<TCols, FromColumns.InsertRowEncoded<TCols>>
 >
 
 export const structSchemaForTable = <TTableDefinition extends TableDefinition<any, any>>(
@@ -63,6 +68,20 @@ export const structSchemaForTable = <TTableDefinition extends TableDefinition<an
     title: tableDef.name,
   }) as any
 
+export const insertStructSchemaForTable = <TTableDefinition extends TableDefinition<any, any>>(
+  tableDef: TTableDefinition,
+): InsertStructSchemaForColumns<TTableDefinition['columns']> =>
+  Schema.Struct(
+    Object.fromEntries(
+      tableDef.ast.columns.map((column) => [
+        column.name,
+        column.nullable === true || column.default._tag === 'Some' ? Schema.optional(column.schema) : column.schema,
+      ]),
+    ),
+  ).annotations({
+    title: tableDef.name,
+  }) as any
+
 const columsToAst = (columns: Columns): ReadonlyArray<SqliteAst.Column> => {
   return Object.entries(columns).map(([name, column]) => {
     return {
@@ -161,6 +180,10 @@ export namespace FromColumns {
     readonly [K in keyof TColumns]: Schema.Schema.Type<TColumns[K]['schema']>
   }
 
+  export type RowEncodedAll<TColumns extends Columns> = {
+    readonly [K in keyof TColumns]: Schema.Schema.Encoded<TColumns[K]['schema']>
+  }
+
   export type RowEncoded<TColumns extends Columns> = Types.Simplify<
     Nullable<Pick<RowEncodeNonNullable<TColumns>, NullableColumnNames<TColumns>>> &
       Omit<RowEncodeNonNullable<TColumns>, NullableColumnNames<TColumns>>
@@ -192,4 +215,9 @@ export namespace FromColumns {
     Pick<RowDecodedAll<TColumns>, RequiredInsertColumnNames<TColumns>> &
       Partial<Omit<RowDecodedAll<TColumns>, RequiredInsertColumnNames<TColumns>>>
   >
+
+  export type InsertRowEncoded<TColumns extends Columns> = Types.Simplify<
+    Pick<RowEncodedAll<TColumns>, RequiredInsertColumnNames<TColumns>> &
+      Partial<Omit<RowEncodedAll<TColumns>, RequiredInsertColumnNames<TColumns>>>
+  >
 }

package/src/schema/mutations.ts

@@ -1,5 +1,6 @@
 import { Schema } from '@livestore/utils/effect'
 
+import type { QueryBuilder } from '../query-builder/mod.js'
 import type { BindValues } from '../sql-queries/sql-queries.js'
 
 export type MutationDefMap = {
@@ -20,7 +21,10 @@ export type InternalMutationSchema<TRecord extends MutationDefRecord = MutationD
 
 export type MutationDefSqlResult<TTo> =
   | SingleOrReadonlyArray<string>
-  | ((args: TTo) => SingleOrReadonlyArray<
+  | ((
+      args: TTo,
+      context: { currentFacts: MutationEventFacts; clientOnly: boolean },
+    ) => SingleOrReadonlyArray<
       | string
       | {
           sql: string
@@ -28,8 +32,15 @@ export type MutationDefSqlResult<TTo> =
           bindValues: BindValues
           writeTables?: ReadonlySet<string>
         }
+      | QueryBuilder.Any
     >)
 
+export type MutationHandlerResult = {
+  sql: string
+  bindValues: BindValues
+  writeTables?: ReadonlySet<string>
+}
+
 export type SingleOrReadonlyArray<T> = T | ReadonlyArray<T>
 
 export type MutationDef<TName extends string, TFrom, TTo> = {

package/src/schema/table-def.ts

@@ -19,12 +19,12 @@ export type DefaultSqliteTableDefConstrained = SqliteDsl.TableDefinition<string,
 export type TableDefBase<
   TSqliteDef extends DefaultSqliteTableDef = DefaultSqliteTableDefConstrained,
   TOptions extends TableOptions = TableOptions,
-  TSchema = SqliteDsl.StructSchemaForColumns<TSqliteDef['columns']>,
 > = {
   sqliteDef: TSqliteDef
   options: TOptions
   // Derived from `sqliteDef`, so only exposed for convenience
-  schema: TSchema
+  schema: SqliteDsl.StructSchemaForColumns<TSqliteDef['columns']>
+  insertSchema: SqliteDsl.InsertStructSchemaForColumns<TSqliteDef['columns']>
 }
 
 export type TableDef<
@@ -47,7 +47,10 @@ export type TableDef<
   options: TOptions
   // Derived from `sqliteDef`, so only exposed for convenience
   schema: TSchema
-  query: QueryBuilder<ReadonlyArray<Schema.Schema.Type<TSchema>>, TableDef<TSqliteDef & {}, TOptions>>
+  insertSchema: SqliteDsl.InsertStructSchemaForColumns<TSqliteDef['columns']>
+  query: QueryBuilder<ReadonlyArray<Schema.Schema.Type<TSchema>>, TableDefBase<TSqliteDef & {}, TOptions>>
+  readonly Type: Schema.Schema.Type<TSchema>
+  readonly Encoded: Schema.Schema.Encoded<TSchema>
 } & (TOptions['deriveMutations']['enabled'] extends true
   ? DerivedMutationHelperFns<TSqliteDef['columns'], TOptions>
   : {})
@@ -195,7 +198,14 @@ export const table = <
   const isSingleColumn = SqliteDsl.isColumnDefinition(columnOrColumns) === true
 
   const schema = SqliteDsl.structSchemaForTable(sqliteDef)
-  const tableDef = { sqliteDef, options: options_, schema } satisfies TableDefBase
+  const insertSchema = SqliteDsl.insertStructSchemaForTable(sqliteDef)
+  const tableDef = {
+    sqliteDef,
+    options: options_,
+    schema,
+    insertSchema,
+  } satisfies TableDefBase
+
   const query = makeQueryBuilder(tableDef)
   // const tableDef = { ...tableDefBase, query } satisfies TableDef
 

package/src/sync/ClientSessionSyncProcessor.ts

@@ -17,6 +17,8 @@ import * as SyncState from './syncstate.js'
  * - The goal is to never block the UI, so we'll interrupt rebasing if a new mutations is pushed by the client session.
  * - We also want to avoid "backwards-jumping" in the UI, so we'll transactionally apply a read model changes during a rebase.
  * - We might need to make the rebase behaviour configurable e.g. to let users manually trigger a rebase
+ *
+ * Longer term we should evalutate whether we can unify the ClientSessionSyncProcessor with the LeaderSyncProcessor.
  */
 export const makeClientSessionSyncProcessor = ({
   schema,
@@ -64,7 +66,7 @@ export const makeClientSessionSyncProcessor = ({
   }
 
   const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
-  const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) =>
+  const isClientEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) =>
     getMutationDef(schema, mutationEventEncoded.mutation).options.clientOnly
 
   /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
@@ -88,31 +90,31 @@ export const makeClientSessionSyncProcessor = ({
       )
     })
 
-    const updateResult = SyncState.updateSyncState({
+    const mergeResult = SyncState.merge({
       syncState: syncStateRef.current,
       payload: { _tag: 'local-push', newEvents: encodedMutationEvents },
-      isLocalEvent,
+      isClientEvent,
       isEqualEvent: MutationEvent.isEqualEncoded,
     })
 
-    if (updateResult._tag === 'unexpected-error') {
-      return shouldNeverHappen('Unexpected error in client-session-sync-processor', updateResult.cause)
+    if (mergeResult._tag === 'unexpected-error') {
+      return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.cause)
     }
 
     span.addEvent('local-push', {
       batchSize: encodedMutationEvents.length,
-      updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+      mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
     })
 
-    if (updateResult._tag !== 'advance') {
-      return shouldNeverHappen(`Expected advance, got ${updateResult._tag}`)
+    if (mergeResult._tag !== 'advance') {
+      return shouldNeverHappen(`Expected advance, got ${mergeResult._tag}`)
     }
 
-    syncStateRef.current = updateResult.newSyncState
-    syncStateUpdateQueue.offer(updateResult.newSyncState).pipe(Effect.runSync)
+    syncStateRef.current = mergeResult.newSyncState
+    syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
 
     const writeTables = new Set<string>()
-    for (const mutationEvent of updateResult.newEvents) {
+    for (const mutationEvent of mergeResult.newEvents) {
       // TODO avoid encoding and decoding here again
       const decodedMutationEvent = Schema.decodeSync(mutationEventSchema)(mutationEvent)
       const res = applyMutation(decodedMutationEvent, { otelContext, withChangeset: true })
@@ -166,7 +168,10 @@ export const makeClientSessionSyncProcessor = ({
 
     yield* FiberHandle.run(leaderPushingFiberHandle, backgroundLeaderPushing)
 
-    yield* clientSession.leaderThread.mutations.pull.pipe(
+    // NOTE We need to lazily call `.pull` as we want the cursor to be updated
+    yield* Stream.suspend(() =>
+      clientSession.leaderThread.mutations.pull({ cursor: syncStateRef.current.localHead }),
+    ).pipe(
       Stream.tap(({ payload, remaining }) =>
         Effect.gen(function* () {
           // console.log('pulled payload from leader', { payload, remaining })
@@ -174,29 +179,29 @@ export const makeClientSessionSyncProcessor = ({
             yield* clientSession.devtools.pullLatch.await
           }
 
-          const updateResult = SyncState.updateSyncState({
+          const mergeResult = SyncState.merge({
             syncState: syncStateRef.current,
             payload,
-            isLocalEvent,
+            isClientEvent,
             isEqualEvent: MutationEvent.isEqualEncoded,
           })
 
-          if (updateResult._tag === 'unexpected-error') {
-            return yield* Effect.fail(updateResult.cause)
-          } else if (updateResult._tag === 'reject') {
-            return shouldNeverHappen('Unexpected reject in client-session-sync-processor', updateResult)
+          if (mergeResult._tag === 'unexpected-error') {
+            return yield* Effect.fail(mergeResult.cause)
+          } else if (mergeResult._tag === 'reject') {
+            return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
           }
 
-          syncStateRef.current = updateResult.newSyncState
-          syncStateUpdateQueue.offer(updateResult.newSyncState).pipe(Effect.runSync)
+          syncStateRef.current = mergeResult.newSyncState
+          syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
 
-          if (updateResult._tag === 'rebase') {
+          if (mergeResult._tag === 'rebase') {
             span.addEvent('pull:rebase', {
               payloadTag: payload._tag,
               payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
-              newEventsCount: updateResult.newEvents.length,
-              rollbackCount: updateResult.eventsToRollback.length,
-              res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+              newEventsCount: mergeResult.newEvents.length,
+              rollbackCount: mergeResult.eventsToRollback.length,
+              res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
               remaining,
             })
 
@@ -212,36 +217,36 @@ export const makeClientSessionSyncProcessor = ({
             if (LS_DEV) {
               Effect.logDebug(
                 'pull:rebase: rollback',
-                updateResult.eventsToRollback.length,
-                ...updateResult.eventsToRollback.slice(0, 10).map((_) => _.toJSON()),
+                mergeResult.eventsToRollback.length,
+                ...mergeResult.eventsToRollback.slice(0, 10).map((_) => _.toJSON()),
               ).pipe(Effect.provide(runtime), Effect.runSync)
             }
 
-            for (let i = updateResult.eventsToRollback.length - 1; i >= 0; i--) {
-              const event = updateResult.eventsToRollback[i]!
+            for (let i = mergeResult.eventsToRollback.length - 1; i >= 0; i--) {
+              const event = mergeResult.eventsToRollback[i]!
               if (event.meta.sessionChangeset) {
                 rollback(event.meta.sessionChangeset)
                 event.meta.sessionChangeset = undefined
               }
             }
 
-            yield* BucketQueue.offerAll(leaderPushQueue, updateResult.newSyncState.pending)
+            yield* BucketQueue.offerAll(leaderPushQueue, mergeResult.newSyncState.pending)
           } else {
             span.addEvent('pull:advance', {
               payloadTag: payload._tag,
               payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
-              newEventsCount: updateResult.newEvents.length,
-              res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
+              newEventsCount: mergeResult.newEvents.length,
+              res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
               remaining,
             })
 
             debugInfo.advanceCount++
           }
 
-          if (updateResult.newEvents.length === 0) return
+          if (mergeResult.newEvents.length === 0) return
 
           const writeTables = new Set<string>()
-          for (const mutationEvent of updateResult.newEvents) {
+          for (const mutationEvent of mergeResult.newEvents) {
             const decodedMutationEvent = Schema.decodeSync(mutationEventSchema)(mutationEvent)
             const res = applyMutation(decodedMutationEvent, { otelContext, withChangeset: true })
             for (const table of res.writeTables) {
@@ -259,6 +264,7 @@ export const makeClientSessionSyncProcessor = ({
       ),
       Stream.runDrain,
       Effect.forever, // NOTE Whenever the leader changes, we need to re-start the stream
+      Effect.interruptible,
       Effect.withSpan('client-session-sync-processor:pull'),
       Effect.tapCauseLogPretty,
       Effect.forkScoped,

package/src/sync/sync.ts

@@ -19,6 +19,16 @@ export type SyncOptions = {
   backend?: SyncBackendConstructor<any>
   /** @default { _tag: 'Skip' } */
   initialSyncOptions?: InitialSyncOptions
+  /**
+   * What to do if there is an error during sync.
+   *
+   * Options:
+   * `shutdown` will stop the sync processor and cause the app to crash.
+   * `ignore` will log the error and let the app continue running acting as if it was offline.
+   *
+   * @default 'ignore'
+   * */
+  onSyncError?: 'shutdown' | 'ignore'
 }
 
 export type SyncBackendConstructor<TSyncMetadata = Schema.JsonValue> = (
@@ -59,6 +69,10 @@ export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
     HttpClient.HttpClient
   >
   isConnected: SubscriptionRef.SubscriptionRef<boolean>
+  /**
+   * Metadata describing the sync backend.
+   */
+  metadata: { name: string; description: string } & Record<string, Schema.JsonValue>
 }
 
 export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {}) {}