@livestore/common 0.4.0-dev.2 → 0.4.0-dev.5

package/src/schema/LiveStoreEvent.ts

@@ -283,5 +283,4 @@ export const isEqualEncoded = (a: AnyEncoded, b: AnyEncoded) =>
   a.name === b.name &&
   a.clientId === b.clientId &&
   a.sessionId === b.sessionId &&
-  // TODO use schema equality here
-  JSON.stringify(a.args) === JSON.stringify(b.args)
+  JSON.stringify(a.args) === JSON.stringify(b.args) // TODO use schema equality here

package/src/schema/schema.ts

@@ -110,7 +110,7 @@ export const getEventDef = <TSchema extends LiveStoreSchema>(
 } => {
   const eventDef = schema.eventsDefsMap.get(eventName)
   if (eventDef === undefined) {
-    return shouldNeverHappen(`No mutation definition found for \`${eventName}\`.`)
+    return shouldNeverHappen(`No event definition found for \`${eventName}\`.`)
   }
   const materializer = schema.state.materializers.get(eventName)
   if (materializer === undefined) {

package/src/schema/state/sqlite/client-document-def.test.ts

@@ -50,6 +50,7 @@ describe('client document table', () => {
         currentFacts: new Map(),
         query: {} as any, // unused
         eventDef: Doc[ClientDocumentTableDefSymbol].derived.setEventDef,
+        event: {} as any, // unused in this test
       })
     }
 
@@ -230,6 +231,22 @@ describe('client document table', () => {
         }
       `)
     })
+
+    test('any value (Schema.Any) should fully replace', () => {
+      expect(forSchema(Schema.Any, { a: 1 }, 'id1')).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "id1",
+            "{"a":1}",
+            "{"a":1}",
+          ],
+          "sql": "INSERT INTO 'test' (id, value) VALUES (?, ?) ON CONFLICT (id) DO UPDATE SET value = ?",
+          "writeTables": Set {
+            "test",
+          },
+        }
+      `)
+    })
   })
 })
 

package/src/schema/state/sqlite/client-document-def.ts

@@ -62,9 +62,16 @@ export const clientDocument = <
     },
   } satisfies ClientDocumentTableOptions<TType>
 
+  // Column needs optimistic schema to read historical data formats
+  const optimisticColumnSchema = createOptimisticEventSchema({
+    valueSchema,
+    defaultValue: options.default.value,
+    partialSet: false, // Column always stores full documents
+  })
+
   const columns = {
     id: SqliteDsl.text({ primaryKey: true }),
-    value: SqliteDsl.json({ schema: valueSchema }),
+    value: SqliteDsl.json({ schema: optimisticColumnSchema }),
   }
 
   const tableDef = table({ name, columns })
@@ -140,6 +147,105 @@ export const mergeDefaultValues = <T>(defaultValues: T, explicitDefaultValues: T
   }, {} as any)
 }
 
+/**
+ * Creates an optimistic schema that accepts historical event formats
+ * and transforms them to the current schema, preserving data and intent.
+ *
+ * Decision Matrix for Schema Changes:
+ *
+ * | Change Type         | Partial Set         | Full Set                        | Strategy                |
+ * |---------------------|---------------------|----------------------------------|-------------------------|
+ * | **Compatible Changes**                                                                                 |
+ * | Add optional field  | Preserve existing   | Preserve existing, new field undefined | Direct decode or merge   |
+ * | Add required field  | Preserve existing   | Preserve existing, new field from default | Merge with defaults      |
+ * | **Incompatible Changes**                                                                              |
+ * | Remove field        | Drop removed field  | Drop removed field, preserve others     | Filter & decode         |
+ * | Type change         | Use default for field | Use default for changed field         | Selective merge         |
+ * | Rename field        | Use default         | Use default (can't detect rename)       | Fall back to default    |
+ * | **Edge Cases**                                                                                       |
+ * | Empty event         | Return {}           | Return full default                     | Fallback handling       |
+ * | Invalid structure   | Return {}           | Return full default                     | Fallback handling       |
+ */
+export const createOptimisticEventSchema = ({
+  valueSchema,
+  defaultValue,
+  partialSet,
+}: {
+  valueSchema: Schema.Schema<any, any>
+  defaultValue: any
+  partialSet: boolean
+}) => {
+  const targetSchema = partialSet ? Schema.partial(valueSchema) : valueSchema
+
+  return Schema.transform(
+    Schema.Unknown, // Accept any historical event structure
+    targetSchema, // Output current schema
+    {
+      decode: (eventValue) => {
+        // Try direct decode first (for current schema events)
+        try {
+          return Schema.decodeUnknownSync(targetSchema)(eventValue)
+        } catch {
+          // Optimistic decoding for historical events
+
+          // Handle null/undefined/non-object cases
+          if (typeof eventValue !== 'object' || eventValue === null) {
+            console.warn(`Client document: Non-object event value, using ${partialSet ? 'empty partial' : 'defaults'}`)
+            return partialSet ? {} : defaultValue
+          }
+
+          if (partialSet) {
+            // For partial sets: only preserve fields that exist in new schema
+            const partialResult: Record<string, unknown> = {}
+            let hasValidFields = false
+
+            for (const [key, value] of Object.entries(eventValue as Record<string, unknown>)) {
+              if (key in defaultValue) {
+                partialResult[key] = value
+                hasValidFields = true
+              }
+              // Drop fields that don't exist in new schema
+            }
+
+            if (hasValidFields) {
+              try {
+                return Schema.decodeUnknownSync(targetSchema)(partialResult)
+              } catch {
+                // Even filtered fields don't match schema
+                console.warn('Client document: Partial fields incompatible, returning empty partial')
+                return {}
+              }
+            }
+            return {}
+          } else {
+            // Full set: merge old data with new defaults
+            const merged: Record<string, unknown> = { ...defaultValue }
+
+            // Override defaults with valid fields from old event
+            for (const [key, value] of Object.entries(eventValue as Record<string, unknown>)) {
+              if (key in defaultValue) {
+                merged[key] = value
+              }
+              // Drop fields that don't exist in new schema
+            }
+
+            // Try to decode the merged value
+            try {
+              return Schema.decodeUnknownSync(valueSchema)(merged)
+            } catch {
+              // Merged value still doesn't match (e.g., type changes)
+              // Fall back to pure defaults
+              console.warn('Client document: Could not preserve event data, using defaults')
+              return defaultValue
+            }
+          }
+        }
+      },
+      encode: (value) => value, // Pass-through for encoding
+    },
+  )
+}
+
 export const deriveEventAndMaterializer = ({
   name,
   valueSchema,
@@ -155,7 +261,7 @@ export const deriveEventAndMaterializer = ({
     name: `${name}Set`,
     schema: Schema.Struct({
       id: Schema.Union(Schema.String, Schema.UniqueSymbolFromSelf(SessionIdSymbol)),
-      value: partialSet ? Schema.partial(valueSchema) : valueSchema,
+      value: createOptimisticEventSchema({ valueSchema, defaultValue, partialSet }),
     }).annotations({ title: `${name}Set:Args` }),
     clientOnly: true,
     derived: true,
@@ -281,8 +387,14 @@ export namespace ClientDocumentTableOptions {
     }
   }
 
+  type IsStructLike<T> = T extends {} ? true : false
+
   export type WithDefaults<TInput extends Input<any>> = {
-    partialSet: TInput['partialSet'] extends false ? false : true
+    partialSet: TInput['partialSet'] extends false
+      ? false
+      : IsStructLike<TInput['default']['value']> extends true
+        ? true
+        : false
     default: {
       id: TInput['default']['id'] extends string | SessionIdSymbol ? TInput['default']['id'] : undefined
       value: TInput['default']['value']

package/src/schema/state/sqlite/db-schema/ast/sqlite.ts

@@ -1,4 +1,4 @@
-import { type Option, Schema } from '@livestore/utils/effect'
+import { type Option, Schema, SchemaAST } from '@livestore/utils/effect'
 
 import { hashCode } from '../hash.ts'
 
@@ -85,7 +85,19 @@ export type DbSchema = {
 export const dbSchema = (tables: Table[]): DbSchema => ({ _tag: 'dbSchema', tables })
 
 /**
- * NOTE we're only including SQLite-relevant information in the hash (which excludes the schema mapping)
+ * Helper to detect if a column is a JSON column (has parseJson transformation)
+ */
+const isJsonColumn = (column: Column): boolean => {
+  if (column.type._tag !== 'text') return false
+
+  // Check if the schema AST is a parseJson transformation
+  const ast = column.schema.ast
+  return ast._tag === 'Transformation' && ast.annotations.schemaId === SchemaAST.ParseJsonSchemaId
+}
+
+/**
+ * NOTE we're now including JSON schema information for JSON columns
+ * to detect client document schema changes
  */
 export const hash = (obj: Table | Column | Index | ForeignKey | DbSchema): number =>
   hashCode(JSON.stringify(trimInfoForHasing(obj)))
@@ -101,7 +113,7 @@ const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema):
       }
     }
     case 'column': {
-      return {
+      const baseInfo: Record<string, any> = {
         _tag: 'column',
         name: obj.name,
         type: obj.type._tag,
@@ -110,6 +122,15 @@ const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema):
         autoIncrement: obj.autoIncrement,
         default: obj.default,
       }
+
+      // NEW: Include schema hash for JSON columns
+      // This ensures that changes to the JSON schema are detected
+      if (isJsonColumn(obj) && obj.schema) {
+        // Use Effect's Schema.hash for consistent hashing
+        baseInfo.jsonSchemaHash = Schema.hash(obj.schema)
+      }
+
+      return baseInfo
     }
     case 'index': {
       return {

package/src/schema/state/sqlite/mod.ts

@@ -14,6 +14,7 @@ export {
   ClientDocumentTableDefSymbol,
   type ClientDocumentTableOptions,
   clientDocument,
+  createOptimisticEventSchema,
   tableIsClientDocumentTable,
 } from './client-document-def.ts'
 export * from './column-annotations.ts'

package/src/schema/state/sqlite/query-builder/api.ts

@@ -3,7 +3,7 @@ import { type Option, Predicate, type Schema } from '@livestore/utils/effect'
 
 import type { SessionIdSymbol } from '../../../../adapter-types.ts'
 import type { SqlValue } from '../../../../util.ts'
-import type { ClientDocumentTableDef } from '../client-document-def.ts'
+import type { ClientDocumentTableDef, ClientDocumentTableDefSymbol } from '../client-document-def.ts'
 import type { SqliteDsl } from '../db-schema/mod.ts'
 import type { TableDefBase } from '../table-def.ts'
 
@@ -437,7 +437,12 @@ export namespace QueryBuilder {
 
 export namespace RowQuery {
   export type GetOrCreateOptions<TTableDef extends ClientDocumentTableDef.TraitAny> = {
-    default: Partial<TTableDef['Value']>
+    /**
+     * Default value to use instead of the default value from the table definition
+     */
+    default: TTableDef[ClientDocumentTableDefSymbol]['options']['partialSet'] extends false
+      ? TTableDef['Value']
+      : Partial<TTableDef['Value']>
   }
 
   // TODO get rid of this

package/src/schema/state/sqlite/query-builder/impl.test.ts

@@ -622,6 +622,70 @@ describe('query builder', () => {
         }
       `)
     })
+
+    it('should handle where().delete() - preserving where clauses', () => {
+      expect(dump(db.todos.where({ status: 'completed' }).delete())).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+          ],
+          "query": "DELETE FROM 'todos' WHERE status = ?",
+          "schema": "number",
+        }
+      `)
+
+      // Multiple where clauses
+      expect(dump(db.todos.where({ status: 'completed' }).where({ deletedAt: null }).delete())).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+          ],
+          "query": "DELETE FROM 'todos' WHERE status = ? AND deletedAt IS NULL",
+          "schema": "number",
+        }
+      `)
+    })
+
+    it('should handle where().update() - preserving where clauses', () => {
+      expect(dump(db.todos.where({ id: '123' }).update({ status: 'completed' }))).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+            "123",
+          ],
+          "query": "UPDATE 'todos' SET status = ? WHERE id = ?",
+          "schema": "number",
+        }
+      `)
+
+      // Multiple where clauses
+      expect(
+        dump(db.todos.where({ id: '123' }).where({ deletedAt: null }).update({ status: 'completed' })),
+      ).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+            "123",
+          ],
+          "query": "UPDATE 'todos' SET status = ? WHERE id = ? AND deletedAt IS NULL",
+          "schema": "number",
+        }
+      `)
+    })
+
+    it('should have equivalent behavior for both delete patterns', () => {
+      const pattern1 = dump(db.todos.where({ status: 'completed', id: '123' }).delete())
+      const pattern2 = dump(db.todos.delete().where({ status: 'completed', id: '123' }))
+
+      expect(pattern1).toEqual(pattern2)
+    })
+
+    it('should have equivalent behavior for both update patterns', () => {
+      const pattern1 = dump(db.todos.where({ id: '123' }).update({ status: 'completed', text: 'Updated' }))
+      const pattern2 = dump(db.todos.update({ status: 'completed', text: 'Updated' }).where({ id: '123' }))
+
+      expect(pattern1).toEqual(pattern2)
+    })
   })
 })
 

package/src/schema/state/sqlite/query-builder/impl.ts

@@ -219,21 +219,27 @@ export const makeQueryBuilder = <TResult, TTableDef extends TableDefBase>(
     update: (values) => {
       const filteredValues = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== undefined))
 
+      // Preserve where clauses if coming from a SelectQuery
+      const whereClause = ast._tag === 'SelectQuery' ? ast.where : []
+
       return makeQueryBuilder(tableDef, {
         _tag: 'UpdateQuery',
         tableDef,
         values: filteredValues,
-        where: [],
+        where: whereClause,
         returning: undefined,
         resultSchema: Schema.Void,
       }) as any
     },
 
     delete: () => {
+      // Preserve where clauses if coming from a SelectQuery
+      const whereClause = ast._tag === 'SelectQuery' ? ast.where : []
+
       return makeQueryBuilder(tableDef, {
         _tag: 'DeleteQuery',
         tableDef,
-        where: [],
+        where: whereClause,
         returning: undefined,
         resultSchema: Schema.Void,
       }) as any

package/src/schema/state/sqlite/system-tables.ts

@@ -96,6 +96,8 @@ export const syncStatusTable = table({
   name: SYNC_STATUS_TABLE,
   columns: {
     head: SqliteDsl.integer({ primaryKey: true }),
+    // Null means the sync backend is not yet connected and we haven't yet seen a backend ID
+    backendId: SqliteDsl.text({ nullable: true }),
   },
 })
 

package/src/sync/ClientSessionSyncProcessor.ts

@@ -13,9 +13,9 @@ import {
   Stream,
   Subscribable,
 } from '@livestore/utils/effect'
-import * as otel from '@opentelemetry/api'
+import type * as otel from '@opentelemetry/api'
 
-import { type ClientSession, SyncError, type UnexpectedError } from '../adapter-types.ts'
+import { type ClientSession, type MaterializeError, UnexpectedError } from '../adapter-types.ts'
 import * as EventSequenceNumber from '../schema/EventSequenceNumber.ts'
 import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
 import { getEventDef, type LiveStoreSchema } from '../schema/mod.ts'
@@ -52,15 +52,18 @@ export const makeClientSessionSyncProcessor = ({
   runtime: Runtime.Runtime<Scope.Scope>
   materializeEvent: (
     eventDecoded: LiveStoreEvent.AnyDecoded,
-    options: { otelContext: otel.Context; withChangeset: boolean; materializerHashLeader: Option.Option<number> },
-  ) => {
-    writeTables: Set<string>
-    sessionChangeset:
-      | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
-      | { _tag: 'no-op' }
-      | { _tag: 'unset' }
-    materializerHash: Option.Option<number>
-  }
+    options: { withChangeset: boolean; materializerHashLeader: Option.Option<number> },
+  ) => Effect.Effect<
+    {
+      writeTables: Set<string>
+      sessionChangeset:
+        | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
+        | { _tag: 'no-op' }
+        | { _tag: 'unset' }
+      materializerHash: Option.Option<number>
+    },
+    MaterializeError
+  >
   rollback: (changeset: Uint8Array<ArrayBuffer>) => void
   refreshTables: (tables: Set<string>) => void
   span: otel.Span
@@ -99,7 +102,7 @@ export const makeClientSessionSyncProcessor = ({
   /** We're queuing push requests to reduce the number of messages sent to the leader by batching them */
   const leaderPushQueue = BucketQueue.make<LiveStoreEvent.EncodedWithMeta>().pipe(Effect.runSync)
 
-  const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
+  const push: ClientSessionSyncProcessor['push'] = Effect.fn('client-session-sync-processor:push')(function* (batch) {
     // TODO validate batch
 
     let baseEventSequenceNumber = syncStateRef.current.localHead
@@ -128,21 +131,26 @@ export const makeClientSessionSyncProcessor = ({
       isEqualEvent: LiveStoreEvent.isEqualEncoded,
     })
 
+    yield* Effect.annotateCurrentSpan({
+      batchSize: encodedEventDefs.length,
+      mergeResultTag: mergeResult._tag,
+      eventCounts: encodedEventDefs.reduce<Record<string, number>>((acc, event) => {
+        acc[event.name] = (acc[event.name] ?? 0) + 1
+        return acc
+      }, {}),
+      ...(TRACE_VERBOSE && { mergeResult: JSON.stringify(mergeResult) }),
+    })
+
     if (mergeResult._tag === 'unexpected-error') {
       return shouldNeverHappen('Unexpected error in client-session-sync-processor', mergeResult.message)
     }
 
-    span.addEvent('local-push', {
-      batchSize: encodedEventDefs.length,
-      mergeResult: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-    })
-
     if (mergeResult._tag !== 'advance') {
       return shouldNeverHappen(`Expected advance, got ${mergeResult._tag}`)
     }
 
     syncStateRef.current = mergeResult.newSyncState
-    syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
+    yield* syncStateUpdateQueue.offer(mergeResult.newSyncState)
 
     // Materialize events to state
     const writeTables = new Set<string>()
@@ -153,8 +161,7 @@ export const makeClientSessionSyncProcessor = ({
         writeTables: newWriteTables,
         sessionChangeset,
         materializerHash,
-      } = materializeEvent(decodedEventDef, {
-        otelContext,
+      } = yield* materializeEvent(decodedEventDef, {
         withChangeset: true,
         materializerHashLeader: Option.none(),
       })
@@ -167,10 +174,10 @@ export const makeClientSessionSyncProcessor = ({
 
     // Trigger push to leader
     // console.debug('pushToLeader', encodedEventDefs.length, ...encodedEventDefs.map((_) => _.toJSON()))
-    BucketQueue.offerAll(leaderPushQueue, encodedEventDefs).pipe(Effect.runSync)
+    yield* BucketQueue.offerAll(leaderPushQueue, encodedEventDefs)
 
     return { writeTables }
-  }
+  })
 
   const debugInfo = {
     rebaseCount: 0,
@@ -178,8 +185,6 @@ export const makeClientSessionSyncProcessor = ({
     rejectCount: 0,
   }
 
-  const otelContext = otel.trace.setSpan(otel.context.active(), span)
-
   const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
     if (confirmUnsavedChanges && typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
       const onBeforeUnload = (event: BeforeUnloadEvent) => {
@@ -229,7 +234,7 @@ export const makeClientSessionSyncProcessor = ({
           })
 
           if (mergeResult._tag === 'unexpected-error') {
-            return yield* new SyncError({ cause: mergeResult.message })
+            return yield* new UnexpectedError({ cause: mergeResult.message })
           } else if (mergeResult._tag === 'reject') {
             return shouldNeverHappen('Unexpected reject in client-session-sync-processor', mergeResult)
           }
@@ -244,7 +249,6 @@ export const makeClientSessionSyncProcessor = ({
               newEventsCount: mergeResult.newEvents.length,
               rollbackCount: mergeResult.rollbackEvents.length,
               res: TRACE_VERBOSE ? JSON.stringify(mergeResult) : undefined,
-              rebaseGeneration: mergeResult.newSyncState.localHead.rebaseGeneration,
             })
 
             debugInfo.rebaseCount++
@@ -304,8 +308,7 @@ export const makeClientSessionSyncProcessor = ({
               writeTables: newWriteTables,
               sessionChangeset,
               materializerHash,
-            } = materializeEvent(decodedEventDef, {
-              otelContext,
+            } = yield* materializeEvent(decodedEventDef, {
               withChangeset: true,
               materializerHashLeader: event.meta.materializerHashLeader,
             })
@@ -364,10 +367,7 @@ export const makeClientSessionSyncProcessor = ({
 export interface ClientSessionSyncProcessor {
   push: (
     batch: ReadonlyArray<LiveStoreEvent.PartialAnyDecoded>,
-    options: { otelContext: otel.Context },
-  ) => {
-    writeTables: Set<string>
-  }
+  ) => Effect.Effect<{ writeTables: Set<string> }, MaterializeError>
   boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
   /**
    * Only used for debugging / observability.

package/src/sync/errors.ts

@@ -0,0 +1,38 @@
+import { Schema } from '@livestore/utils/effect'
+import { UnexpectedError } from '../errors.ts'
+import { EventSequenceNumber } from '../schema/mod.ts'
+
+export class IsOfflineError extends Schema.TaggedError<IsOfflineError>()('IsOfflineError', {
+  cause: Schema.Defect,
+}) {}
+
+/** Unique ID generated by the backend when its created. Used to check whether the backend identity has changed. */
+export const BackendId = Schema.String.annotations({ title: '@livestore/sync-cf:BackendId' })
+
+export class BackendIdMismatchError extends Schema.TaggedError<BackendIdMismatchError>()('BackendIdMismatchError', {
+  expected: BackendId,
+  received: BackendId,
+}) {}
+
+export class ServerAheadError extends Schema.TaggedError<ServerAheadError>()('ServerAheadError', {
+  minimumExpectedNum: EventSequenceNumber.GlobalEventSequenceNumber,
+  providedNum: EventSequenceNumber.GlobalEventSequenceNumber,
+}) {}
+
+export class InvalidPushError extends Schema.TaggedError<InvalidPushError>()('InvalidPushError', {
+  cause: Schema.Union(UnexpectedError, ServerAheadError, BackendIdMismatchError),
+}) {}
+
+export class InvalidPullError extends Schema.TaggedError<InvalidPullError>()('InvalidPullError', {
+  cause: Schema.Defect,
+}) {}
+
+export class LeaderAheadError extends Schema.TaggedError<LeaderAheadError>()('LeaderAheadError', {
+  minimumExpectedNum: EventSequenceNumber.EventSequenceNumber,
+  providedNum: EventSequenceNumber.EventSequenceNumber,
+  /** Generation number the client session should use for subsequent pushes */
+  // nextGeneration: Schema.Number,
+}) {}
+
+export const SyncError = Schema.Union(InvalidPushError, InvalidPullError)
+export type SyncError = typeof SyncError.Type

package/src/sync/index.ts

@@ -1,3 +1,4 @@
 export * from './ClientSessionSyncProcessor.ts'
+export * from './mock-sync-backend.ts'
 export * from './sync.ts'
 export * from './validate-push-payload.ts'