@livestore/common 0.3.0 → 0.3.1

package/src/materializer-helper.ts

@@ -1,26 +1,27 @@
-import { isNil, isReadonlyArray } from '@livestore/utils'
-import { Schema } from '@livestore/utils/effect'
+import { isDevEnv, isNil, isReadonlyArray } from '@livestore/utils'
+import { Hash, Option, Schema } from '@livestore/utils/effect'
 
 import type { SqliteDb } from './adapter-types.js'
 import { SessionIdSymbol } from './adapter-types.js'
 import type { EventDef, Materializer, MaterializerContextQuery, MaterializerResult } from './schema/EventDef.js'
 import type * as LiveStoreEvent from './schema/LiveStoreEvent.js'
+import { getEventDef, type LiveStoreSchema } from './schema/schema.js'
 import type { QueryBuilder } from './schema/state/sqlite/query-builder/api.js'
 import { isQueryBuilder } from './schema/state/sqlite/query-builder/api.js'
 import { getResultSchema } from './schema/state/sqlite/query-builder/impl.js'
-import { type BindValues } from './sql-queries/sql-queries.js'
+import type { BindValues } from './sql-queries/sql-queries.js'
 import type { ParamsObject, PreparedBindValues } from './util.js'
 import { prepareBindValues } from './util.js'
 
-export const getExecArgsFromEvent = ({
+export const getExecStatementsFromMaterializer = ({
   eventDef,
   materializer,
-  db,
+  dbState,
   event,
 }: {
   eventDef: EventDef.AnyWithoutFn
   materializer: Materializer
-  db: SqliteDb
+  dbState: SqliteDb
   /** Both encoded and decoded events are supported to reduce the number of times we need to decode/encode */
   event:
     | {
@@ -53,25 +54,25 @@ export const getExecArgsFromEvent = ({
   ) => {
     if (isQueryBuilder(rawQueryOrQueryBuilder)) {
       const { query, bindValues } = rawQueryOrQueryBuilder.asSql()
-      const rawResults = db.select(query, prepareBindValues(bindValues, query))
+      const rawResults = dbState.select(query, prepareBindValues(bindValues, query))
       const resultSchema = getResultSchema(rawQueryOrQueryBuilder)
       return Schema.decodeSync(resultSchema)(rawResults)
     } else {
       const { query, bindValues } = rawQueryOrQueryBuilder
-      return db.select(query, prepareBindValues(bindValues, query))
+      return dbState.select(query, prepareBindValues(bindValues, query))
     }
   }
 
-  const res = materializer(eventArgsDecoded, {
-    eventDef,
-    query,
-    // TODO properly implement this
-    currentFacts: new Map(),
-  })
-
-  const statementRes = mapMaterializerResult(res)
+  const statementResults = fromMaterializerResult(
+    materializer(eventArgsDecoded, {
+      eventDef,
+      query,
+      // TODO properly implement this
+      currentFacts: new Map(),
+    }),
+  )
 
-  return statementRes.map((statementRes) => {
+  return statementResults.map((statementRes) => {
     const statementSql = statementRes.sql
 
     const bindValues = typeof statementRes === 'string' ? eventArgsEncoded : statementRes.bindValues
@@ -82,7 +83,32 @@ export const getExecArgsFromEvent = ({
   })
 }
 
-const mapMaterializerResult = (
+export const makeMaterializerHash =
+  ({ schema, dbState }: { schema: LiveStoreSchema; dbState: SqliteDb }) =>
+  (event: LiveStoreEvent.AnyEncodedGlobal): Option.Option<number> => {
+    if (isDevEnv()) {
+      const { eventDef, materializer } = getEventDef(schema, event.name)
+      const materializerResults = getExecStatementsFromMaterializer({
+        eventDef,
+        materializer,
+        dbState,
+        event: { decoded: undefined, encoded: event },
+      })
+      return Option.some(Hash.string(JSON.stringify(materializerResults)))
+    }
+
+    return Option.none()
+  }
+
+export const hashMaterializerResults = (
+  materializerResults: ReadonlyArray<{
+    statementSql: string
+    bindValues: PreparedBindValues
+    writeTables: ReadonlySet<string> | undefined
+  }>,
+) => Hash.string(JSON.stringify(materializerResults))
+
+const fromMaterializerResult = (
   materializerResult: MaterializerResult | ReadonlyArray<MaterializerResult>,
 ): ReadonlyArray<{
   sql: string
@@ -90,7 +116,7 @@ const mapMaterializerResult = (
   writeTables: ReadonlySet<string> | undefined
 }> => {
   if (isReadonlyArray(materializerResult)) {
-    return materializerResult.flatMap(mapMaterializerResult)
+    return materializerResult.flatMap(fromMaterializerResult)
   }
   if (isQueryBuilder(materializerResult)) {
     const { query, bindValues } = materializerResult.asSql()

package/src/rematerialize-from-eventlog.ts

@@ -27,13 +27,13 @@ export const rematerializeFromEventlog = ({
       `SELECT COUNT(*) AS count FROM ${SystemTables.EVENTLOG_META_TABLE}`,
     )[0]!.count
 
-    const hashEvent = memoizeByRef((event: EventDef.AnyWithoutFn) => Schema.hash(event.schema))
+    const hashEventDef = memoizeByRef((event: EventDef.AnyWithoutFn) => Schema.hash(event.schema))
 
     const processEvent = (row: SystemTables.EventlogMetaRow) =>
       Effect.gen(function* () {
         const eventDef = getEventDef(schema, row.name)
 
-        if (hashEvent(eventDef.eventDef) !== row.schemaHash) {
+        if (hashEventDef(eventDef.eventDef) !== row.schemaHash) {
           yield* Effect.logWarning(
             `Schema hash mismatch for event definition ${row.name}. Trying to materialize event anyway.`,
           )

package/src/schema/EventDef.ts

@@ -27,12 +27,18 @@ export type EventDef<TName extends string, TType, TEncoded = TType, TDerived ext
     derived: TDerived
   }
 
-  /** Helper function to construct a partial mutation event */
+  /** Helper function to construct a partial event */
   (args: TType): {
     name: TName
     args: TType
   }
 
+  /** Helper function to construct a partial encoded event */
+  encoded: (args: TEncoded) => {
+    name: TName
+    args: TEncoded
+  }
+
   readonly Event: {
     name: TName
     args: TType
@@ -123,6 +129,10 @@ export const defineEvent = <TName extends string, TType, TEncoded = TType, TDeri
 
   Object.defineProperty(makePartialEvent, 'name', { value: name })
   Object.defineProperty(makePartialEvent, 'schema', { value: schema })
+  Object.defineProperty(makePartialEvent, 'encoded', {
+    value: (args: TEncoded) => ({ name: name, args }),
+  })
+
   Object.defineProperty(makePartialEvent, 'options', {
     value: {
       clientOnly: options?.clientOnly ?? false,

package/src/schema/LiveStoreEvent.ts

@@ -171,12 +171,25 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('LiveStoreEve
       Schema.TaggedStruct('unset', {}),
     ),
     syncMetadata: Schema.Option(Schema.JsonValue),
+    /** Used to detect if the materializer is side effecting (during dev) */
+    materializerHashLeader: Schema.Option(Schema.Number),
+    materializerHashSession: Schema.Option(Schema.Number),
   }).pipe(
     Schema.mutable,
     Schema.optional,
     Schema.withDefaults({
-      constructor: () => ({ sessionChangeset: { _tag: 'unset' as const }, syncMetadata: Option.none() }),
-      decoding: () => ({ sessionChangeset: { _tag: 'unset' as const }, syncMetadata: Option.none() }),
+      constructor: () => ({
+        sessionChangeset: { _tag: 'unset' as const },
+        syncMetadata: Option.none(),
+        materializerHashLeader: Option.none(),
+        materializerHashSession: Option.none(),
+      }),
+      decoding: () => ({
+        sessionChangeset: { _tag: 'unset' as const },
+        syncMetadata: Option.none(),
+        materializerHashLeader: Option.none(),
+        materializerHashSession: Option.none(),
+      }),
     }),
   ),
 }) {
@@ -214,12 +227,24 @@ export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('LiveStoreEve
       ...EventSequenceNumber.nextPair(parentSeqNum, isClient),
     })
 
-  static fromGlobal = (event: AnyEncodedGlobal, syncMetadata: Option.Option<Schema.JsonValue>) =>
+  static fromGlobal = (
+    event: AnyEncodedGlobal,
+    meta: {
+      syncMetadata: Option.Option<Schema.JsonValue>
+      materializerHashLeader: Option.Option<number>
+      materializerHashSession: Option.Option<number>
+    },
+  ) =>
     new EncodedWithMeta({
       ...event,
       seqNum: { global: event.seqNum, client: EventSequenceNumber.clientDefault },
       parentSeqNum: { global: event.parentSeqNum, client: EventSequenceNumber.clientDefault },
-      meta: { sessionChangeset: { _tag: 'unset' as const }, syncMetadata },
+      meta: {
+        sessionChangeset: { _tag: 'unset' as const },
+        syncMetadata: meta.syncMetadata,
+        materializerHashLeader: meta.materializerHashLeader,
+        materializerHashSession: meta.materializerHashSession,
+      },
     })
 
   toGlobal = (): AnyEncodedGlobal => ({

package/src/schema/state/sqlite/query-builder/api.ts

@@ -117,7 +117,7 @@ export type QueryBuilder<
 > = {
   readonly [QueryBuilderTypeId]: QueryBuilderTypeId
   readonly [QueryBuilderAstSymbol]: QueryBuilderAst
-  readonly ['ResultType']: TResult
+  readonly ResultType: TResult
   readonly asSql: () => { query: string; bindValues: SqlValue[] }
   readonly toString: () => string
 } & Omit<QueryBuilder.ApiFull<TResult, TTableDef, TWithout>, TWithout>

package/src/schema/state/sqlite/table-def.ts

@@ -177,7 +177,7 @@ export namespace FromColumns {
   export type InsertRowDecoded<TColumns extends SqliteDsl.Columns> = SqliteDsl.FromColumns.InsertRowDecoded<TColumns>
 }
 
-type SqliteTableDefForInput<
+export type SqliteTableDefForInput<
   TName extends string,
   TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition<any, any>,
 > = SqliteDsl.TableDefinition<TName, PrettifyFlat<ToColumns<TColumns>>>

package/src/sqlite-db-helper.ts

@@ -0,0 +1,41 @@
+import { Schema } from '@livestore/utils/effect'
+
+import type { SqliteDb } from './adapter-types.js'
+import { getResultSchema, isQueryBuilder } from './schema/state/sqlite/query-builder/mod.js'
+import type { PreparedBindValues } from './util.js'
+
+export const makeExecute = (
+  execute: (
+    queryStr: string,
+    bindValues: PreparedBindValues | undefined,
+    options?: { onRowsChanged?: (rowsChanged: number) => void },
+  ) => void,
+): SqliteDb['execute'] => {
+  return (...args: any[]) => {
+    const [queryStrOrQueryBuilder, bindValuesOrOptions, maybeOptions] = args
+
+    if (isQueryBuilder(queryStrOrQueryBuilder)) {
+      const { query, bindValues } = queryStrOrQueryBuilder.asSql()
+      return execute(query, bindValues as unknown as PreparedBindValues, bindValuesOrOptions)
+    } else {
+      return execute(queryStrOrQueryBuilder, bindValuesOrOptions, maybeOptions)
+    }
+  }
+}
+
+export const makeSelect = <T>(
+  select: (queryStr: string, bindValues: PreparedBindValues | undefined) => ReadonlyArray<T>,
+): SqliteDb['select'] => {
+  return (...args: any[]) => {
+    const [queryStrOrQueryBuilder, maybeBindValues] = args
+
+    if (isQueryBuilder(queryStrOrQueryBuilder)) {
+      const { query, bindValues } = queryStrOrQueryBuilder.asSql()
+      const resultSchema = getResultSchema(queryStrOrQueryBuilder)
+      const results = select(query, bindValues as unknown as PreparedBindValues)
+      return Schema.decodeSync(resultSchema)(results)
+    } else {
+      return select(queryStrOrQueryBuilder, maybeBindValues)
+    }
+  }
+}

package/src/sync/ClientSessionSyncProcessor.ts

@@ -1,6 +1,6 @@
 /// <reference lib="dom" />
 import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { Runtime, Scope } from '@livestore/utils/effect'
+import { Option, type Runtime, type Scope } from '@livestore/utils/effect'
 import { BucketQueue, Effect, FiberHandle, Queue, Schema, Stream, Subscribable } from '@livestore/utils/effect'
 import * as otel from '@opentelemetry/api'
 
@@ -38,10 +38,11 @@ export const makeClientSessionSyncProcessor = ({
   runtime: Runtime.Runtime<Scope.Scope>
   materializeEvent: (
     eventDecoded: LiveStoreEvent.PartialAnyDecoded,
-    options: { otelContext: otel.Context; withChangeset: boolean },
+    options: { otelContext: otel.Context; withChangeset: boolean; materializerHashLeader: Option.Option<number> },
   ) => {
     writeTables: Set<string>
     sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } | { _tag: 'unset' }
+    materializerHash: Option.Option<number>
   }
   rollback: (changeset: Uint8Array) => void
   refreshTables: (tables: Set<string>) => void
@@ -67,6 +68,7 @@ export const makeClientSessionSyncProcessor = ({
     }),
   }
 
+  /** Only used for debugging / observability, it's not relied upon for correctness of the sync processor. */
   const syncStateUpdateQueue = Queue.unbounded<SyncState.SyncState>().pipe(Effect.runSync)
   const isClientEvent = (eventEncoded: LiveStoreEvent.EncodedWithMeta) =>
     getEventDef(schema, eventEncoded.name).eventDef.options.clientOnly
@@ -116,17 +118,28 @@ export const makeClientSessionSyncProcessor = ({
     syncStateRef.current = mergeResult.newSyncState
     syncStateUpdateQueue.offer(mergeResult.newSyncState).pipe(Effect.runSync)
 
+    // Materialize events to state
     const writeTables = new Set<string>()
     for (const event of mergeResult.newEvents) {
       // TODO avoid encoding and decoding here again
       const decodedEventDef = Schema.decodeSync(eventSchema)(event)
-      const res = materializeEvent(decodedEventDef, { otelContext, withChangeset: true })
-      for (const table of res.writeTables) {
+      const {
+        writeTables: newWriteTables,
+        sessionChangeset,
+        materializerHash,
+      } = materializeEvent(decodedEventDef, {
+        otelContext,
+        withChangeset: true,
+        materializerHashLeader: Option.none(),
+      })
+      for (const table of newWriteTables) {
         writeTables.add(table)
       }
-      event.meta.sessionChangeset = res.sessionChangeset
+      event.meta.sessionChangeset = sessionChangeset
+      event.meta.materializerHashSession = materializerHash
     }
 
+    // Trigger push to leader
     // console.debug('pushToLeader', encodedEventDefs.length, ...encodedEventDefs.map((_) => _.toJSON()))
     BucketQueue.offerAll(leaderPushQueue, encodedEventDefs).pipe(Effect.runSync)
 
@@ -242,6 +255,7 @@ export const makeClientSessionSyncProcessor = ({
               }
             }
 
+            // Pushing rebased pending events to leader
             yield* BucketQueue.offerAll(leaderPushQueue, mergeResult.newSyncState.pending)
           } else {
             span.addEvent('merge:pull:advance', {
@@ -261,12 +275,21 @@ export const makeClientSessionSyncProcessor = ({
           for (const event of mergeResult.newEvents) {
             // TODO apply changeset if available (will require tracking of write tables as well)
             const decodedEventDef = Schema.decodeSync(eventSchema)(event)
-            const res = materializeEvent(decodedEventDef, { otelContext, withChangeset: true })
-            for (const table of res.writeTables) {
+            const {
+              writeTables: newWriteTables,
+              sessionChangeset,
+              materializerHash,
+            } = materializeEvent(decodedEventDef, {
+              otelContext,
+              withChangeset: true,
+              materializerHashLeader: event.meta.materializerHashLeader,
+            })
+            for (const table of newWriteTables) {
               writeTables.add(table)
             }
 
-            event.meta.sessionChangeset = res.sessionChangeset
+            event.meta.sessionChangeset = sessionChangeset
+            event.meta.materializerHashSession = materializerHash
           }
 
           refreshTables(writeTables)
@@ -321,6 +344,9 @@ export interface ClientSessionSyncProcessor {
     writeTables: Set<string>
   }
   boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
+  /**
+   * Only used for debugging / observability.
+   */
   syncState: Subscribable.Subscribable<SyncState.SyncState>
   debug: {
     print: () => void

package/src/sync/next/test/event-fixtures.ts

@@ -24,6 +24,8 @@ export const facts = defineFacts({
   inputValue: (id: string) => `input-value-${id}`,
 })
 
+// TODO also consider the case of "OR" in `require`
+// TODO compaction strategy: "make more coarse" (one data point per hour)
 export const events = {
   createTodo: defineEvent({
     name: 'createTodo',

package/src/version.ts

@@ -2,13 +2,13 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.3.0' as const
+export const liveStoreVersion = '0.3.1' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.
  * Whenever this version changes, LiveStore will start with fresh database files. Old database files are not deleted.
  *
- * While LiveStore is in alpha, this might happen more frequently.
+ * While LiveStore is in beta, this might happen more frequently.
  * In the future, LiveStore will provide a migration path for older database files to avoid the impression of data loss.
  */
 export const liveStoreStorageFormatVersion = 4