@livestore/common 0.3.0-dev.3 → 0.3.0-dev.30

package/src/schema/mutations.ts

@@ -1,192 +0,0 @@
-import { Schema } from '@livestore/utils/effect'
-
-import type { BindValues } from '../sql-queries/sql-queries.js'
-import type * as EventId from './EventId.js'
-
-export type MutationDefMap = Map<string | 'livestore.RawSql', MutationDef.Any>
-export type MutationDefRecord = {
-  'livestore.RawSql': RawSqlMutation
-  [name: string]: MutationDef.Any
-}
-
-export type InternalMutationSchema<TRecord extends MutationDefRecord = MutationDefRecord> = {
-  _DefRecord: TRecord
-
-  map: Map<keyof TRecord, TRecord[keyof TRecord]>
-  schemaHashMap: Map<keyof TRecord, number>
-}
-
-export type MutationDefSqlResult<TTo> =
-  | SingleOrReadonlyArray<string>
-  | ((args: TTo) => SingleOrReadonlyArray<
-      | string
-      | {
-          sql: string
-          /** Note args need to be manually encoded to `BindValues` when returning this argument */
-          bindValues: BindValues
-          writeTables?: ReadonlySet<string>
-        }
-    >)
-
-export type SingleOrReadonlyArray<T> = T | ReadonlyArray<T>
-
-export type MutationDef<TName extends string, TFrom, TTo> = {
-  name: TName
-  schema: Schema.Schema<TTo, TFrom>
-  sql: MutationDefSqlResult<NoInfer<TTo>>
-  options: {
-    /** Warning: This feature is not fully implemented yet */
-    historyId: string
-    /**
-     * When set to true, the mutation won't be synced over the network
-     */
-    localOnly: boolean
-    /** Warning: This feature is not fully implemented yet */
-    facts: FactsCallback<TTo> | undefined
-  }
-
-  /** Helper function to construct a partial mutation event */
-  (
-    args: TTo,
-    options?: {
-      id?: number
-    },
-  ): {
-    mutation: TName
-    args: TTo
-    // TODO remove/clean up after sync-next is fully implemented
-    id?: EventId.EventId
-  }
-}
-
-export type FactsCallback<TTo> = (
-  args: TTo,
-  currentFacts: MutationEventFacts,
-) => {
-  modify: {
-    set: Iterable<MutationEventFactInput>
-    unset: Iterable<MutationEventFactInput>
-  }
-  require: Iterable<MutationEventFactInput>
-}
-
-export namespace MutationDef {
-  export type Any = MutationDef<string, any, any>
-}
-
-export type MutationEventKey = string
-export type MutationEventFact = string
-export type MutationEventFacts = ReadonlyMap<string, any>
-
-export type MutationEventFactsGroup = {
-  modifySet: MutationEventFacts
-  modifyUnset: MutationEventFacts
-
-  /**
-   * Events on independent "dependency" branches are commutative which can facilitate more prioritized syncing
-   */
-  depRequire: MutationEventFacts
-  depRead: MutationEventFacts
-}
-
-export type MutationEventFactsSnapshot = Map<string, any>
-
-export type MutationEventFactInput = string | readonly [string, any]
-
-export const defineFacts = <
-  TRecord extends Record<string, MutationEventFactInput | ((...args: any[]) => MutationEventFactInput)>,
->(
-  record: TRecord,
-): TRecord => record
-
-export type DefineMutationOptions<TTo> = {
-  // TODO actually implement this
-  onError?: (error: any) => void
-  historyId?: string
-  /** Warning: This feature is not fully implemented yet */
-  facts?: (
-    args: TTo,
-    currentFacts: MutationEventFacts,
-  ) => {
-    modify?: {
-      set?: Iterable<MutationEventFactInput>
-      unset?: Iterable<MutationEventFactInput>
-    }
-    /**
-     * Two purposes: constrain history and constrain compaction
-     */
-    require?: Iterable<MutationEventFactInput>
-  }
-  /**
-   * When set to true, the mutation won't be synced over the network
-   */
-  localOnly?: boolean
-}
-
-// TODO possibly also allow for mutation event subsumption behaviour
-export const defineMutation = <TName extends string, TFrom, TTo>(
-  name: TName,
-  schema: Schema.Schema<TTo, TFrom>,
-  sql: MutationDefSqlResult<NoInfer<TTo>>,
-  options?: DefineMutationOptions<TTo>,
-): MutationDef<TName, TFrom, TTo> => {
-  const makePartialEvent = (
-    args: TTo,
-    options?: {
-      id?: EventId.EventId
-    },
-  ) => ({ mutation: name, args, ...options })
-
-  Object.defineProperty(makePartialEvent, 'name', { value: name })
-  Object.defineProperty(makePartialEvent, 'schema', { value: schema })
-  Object.defineProperty(makePartialEvent, 'sql', { value: sql })
-  Object.defineProperty(makePartialEvent, 'options', {
-    value: {
-      historyId: options?.historyId ?? 'main',
-      localOnly: options?.localOnly ?? false,
-      facts: options?.facts
-        ? (args, currentFacts) => {
-            const res = options.facts!(args, currentFacts)
-            return {
-              modify: {
-                set: res.modify?.set ? new Set(res.modify.set) : new Set(),
-                unset: res.modify?.unset ? new Set(res.modify.unset) : new Set(),
-              },
-              require: res.require ? new Set(res.require) : new Set(),
-            }
-          }
-        : undefined,
-    } satisfies MutationDef.Any['options'],
-  })
-
-  return makePartialEvent as MutationDef<TName, TFrom, TTo>
-}
-
-export const makeMutationDefRecord = <TInputRecord extends Record<string, MutationDef.Any>>(
-  inputRecord: TInputRecord,
-): {
-  [K in TInputRecord[keyof TInputRecord]['name']]: Extract<TInputRecord[keyof TInputRecord], { name: K }>
-} => {
-  const result: any = {}
-
-  for (const [name, def] of Object.entries(inputRecord)) {
-    result[name] = def
-  }
-
-  result['livestore.RawSql'] = rawSqlMutation
-
-  return result
-}
-
-export const rawSqlMutation = defineMutation(
-  'livestore.RawSql',
-  Schema.Struct({
-    sql: Schema.String,
-    bindValues: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.Any })),
-    writeTables: Schema.optional(Schema.ReadonlySet(Schema.String)),
-  }),
-  ({ sql, bindValues, writeTables }) => ({ sql, bindValues: bindValues ?? {}, writeTables }),
-)
-
-export type RawSqlMutation = typeof rawSqlMutation
-export type RawSqlMutationEvent = ReturnType<typeof rawSqlMutation>

package/src/sync/client-session-sync-processor.ts

@@ -1,207 +0,0 @@
-import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
-import type { Scope } from '@livestore/utils/effect'
-import { Effect, Schema, Stream } from '@livestore/utils/effect'
-import * as otel from '@opentelemetry/api'
-
-import type { Coordinator, UnexpectedError } from '../adapter-types.js'
-import * as EventId from '../schema/EventId.js'
-import { type LiveStoreSchema } from '../schema/mod.js'
-import * as MutationEvent from '../schema/MutationEvent.js'
-import type { SyncState } from './syncstate.js'
-import { updateSyncState } from './syncstate.js'
-
-/**
- * Rebase behaviour:
- * - We continously pull mutations from the leader and apply them to the local store.
- * - If there was a race condition (i.e. the leader and client session have both advacned),
- *   we'll need to rebase the local pending mutations on top of the leader's head.
- * - The goal is to never block the UI, so we'll interrupt rebasing if a new mutations is pushed by the client session.
- * - We also want to avoid "backwards-jumping" in the UI, so we'll transactionally apply a read model changes during a rebase.
- * - We might need to make the rebase behaviour configurable e.g. to let users manually trigger a rebase
- */
-export const makeClientSessionSyncProcessor = ({
-  schema,
-  initialLeaderHead,
-  pushToLeader,
-  pullFromLeader,
-  applyMutation,
-  rollback,
-  refreshTables,
-  span,
-}: {
-  schema: LiveStoreSchema
-  initialLeaderHead: EventId.EventId
-  pushToLeader: (batch: ReadonlyArray<MutationEvent.AnyEncoded>) => void
-  pullFromLeader: Coordinator['mutations']['pull']
-  applyMutation: (
-    mutationEventDecoded: MutationEvent.PartialAny,
-    options: { otelContext: otel.Context; withChangeset: boolean },
-  ) => {
-    writeTables: Set<string>
-    sessionChangeset: Uint8Array | undefined
-  }
-  rollback: (changeset: Uint8Array) => void
-  refreshTables: (tables: Set<string>) => void
-  // rebaseBehaviour: 'auto-rebase' | 'manual-rebase'
-  span: otel.Span
-}): ClientSessionSyncProcessor => {
-  const mutationEventSchema = MutationEvent.makeMutationEventSchemaMemo(schema)
-
-  const syncStateRef = {
-    current: {
-      localHead: initialLeaderHead,
-      upstreamHead: initialLeaderHead,
-      pending: [],
-      // TODO init rollbackTail from leader to be ready for backend rebasing
-      rollbackTail: [],
-    } as SyncState,
-  }
-
-  const isLocalEvent = (mutationEventEncoded: MutationEvent.EncodedWithMeta) => {
-    const mutationDef = schema.mutations.get(mutationEventEncoded.mutation)!
-    return mutationDef.options.localOnly
-  }
-
-  const push: ClientSessionSyncProcessor['push'] = (batch, { otelContext }) => {
-    // TODO validate batch
-
-    let baseEventId = syncStateRef.current.localHead
-    const encodedMutationEvents = batch.map((mutationEvent) => {
-      const mutationDef = schema.mutations.get(mutationEvent.mutation)!
-      const nextIdPair = EventId.nextPair(baseEventId, mutationDef.options.localOnly)
-      baseEventId = nextIdPair.id
-      return new MutationEvent.EncodedWithMeta(
-        Schema.encodeUnknownSync(mutationEventSchema)({ ...mutationEvent, ...nextIdPair }),
-      )
-    })
-
-    const updateResult = updateSyncState({
-      syncState: syncStateRef.current,
-      payload: { _tag: 'local-push', newEvents: encodedMutationEvents },
-      isLocalEvent,
-      isEqualEvent: MutationEvent.isEqualEncoded,
-    })
-
-    span.addEvent('local-push', {
-      batchSize: encodedMutationEvents.length,
-      updateResult: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-    })
-
-    if (updateResult._tag !== 'advance') {
-      return shouldNeverHappen(`Expected advance, got ${updateResult._tag}`)
-    }
-
-    syncStateRef.current = updateResult.newSyncState
-
-    const writeTables = new Set<string>()
-    for (const mutationEvent of updateResult.newEvents) {
-      // TODO avoid encoding and decoding here again
-      const decodedMutationEvent = Schema.decodeSync(mutationEventSchema)(mutationEvent)
-      const res = applyMutation(decodedMutationEvent, { otelContext, withChangeset: true })
-      for (const table of res.writeTables) {
-        writeTables.add(table)
-      }
-      mutationEvent.meta.sessionChangeset = res.sessionChangeset
-    }
-
-    pushToLeader(encodedMutationEvents)
-
-    return { writeTables }
-  }
-
-  const otelContext = otel.trace.setSpan(otel.context.active(), span)
-
-  const boot: ClientSessionSyncProcessor['boot'] = Effect.gen(function* () {
-    yield* pullFromLeader.pipe(
-      Stream.tap(({ payload, remaining }) =>
-        Effect.gen(function* () {
-          // console.log('pulled payload from leader', { payload, remaining })
-
-          const updateResult = updateSyncState({
-            syncState: syncStateRef.current,
-            payload,
-            isLocalEvent,
-            isEqualEvent: MutationEvent.isEqualEncoded,
-          })
-
-          if (updateResult._tag === 'reject') {
-            debugger
-            throw new Error('TODO: implement reject in client-session-sync-queue for pull')
-          }
-
-          syncStateRef.current = updateResult.newSyncState
-
-          if (updateResult._tag === 'rebase') {
-            span.addEvent('pull:rebase', {
-              payloadTag: payload._tag,
-              payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
-              newEventsCount: updateResult.newEvents.length,
-              rollbackCount: updateResult.eventsToRollback.length,
-              res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-              remaining,
-            })
-            if (LS_DEV) {
-              console.debug(
-                'pull:rebase: rollback',
-                updateResult.eventsToRollback.length,
-                ...updateResult.eventsToRollback.map((_) => _.toJSON()),
-              )
-            }
-
-            for (let i = updateResult.eventsToRollback.length - 1; i >= 0; i--) {
-              const event = updateResult.eventsToRollback[i]!
-              if (event.meta.sessionChangeset) {
-                rollback(event.meta.sessionChangeset)
-                event.meta.sessionChangeset = undefined
-              }
-            }
-          } else {
-            span.addEvent('pull:advance', {
-              payloadTag: payload._tag,
-              payload: TRACE_VERBOSE ? JSON.stringify(payload) : undefined,
-              newEventsCount: updateResult.newEvents.length,
-              res: TRACE_VERBOSE ? JSON.stringify(updateResult) : undefined,
-              remaining,
-            })
-          }
-
-          if (updateResult.newEvents.length === 0) return
-
-          const writeTables = new Set<string>()
-          for (const mutationEvent of updateResult.newEvents) {
-            const decodedMutationEvent = Schema.decodeSync(mutationEventSchema)(mutationEvent)
-            const res = applyMutation(decodedMutationEvent, { otelContext, withChangeset: true })
-            for (const table of res.writeTables) {
-              writeTables.add(table)
-            }
-
-            mutationEvent.meta.sessionChangeset = res.sessionChangeset
-          }
-
-          refreshTables(writeTables)
-        }),
-      ),
-      Stream.runDrain,
-      Effect.tapCauseLogPretty,
-      Effect.forkScoped,
-    )
-  })
-
-  return {
-    push,
-    boot,
-    syncStateRef,
-  } satisfies ClientSessionSyncProcessor
-}
-
-export interface ClientSessionSyncProcessor {
-  push: (
-    batch: ReadonlyArray<MutationEvent.PartialAny>,
-    options: { otelContext: otel.Context },
-  ) => {
-    writeTables: Set<string>
-  }
-  boot: Effect.Effect<void, UnexpectedError, Scope.Scope>
-
-  syncStateRef: { current: SyncState }
-}

package/src/sync/next/test/mutation-fixtures.ts

@@ -1,231 +0,0 @@
-import { Schema } from '@livestore/utils/effect'
-
-import * as EventId from '../../../schema/EventId.js'
-import type { MutationDef } from '../../../schema/mutations.js'
-import { defineFacts, defineMutation } from '../../../schema/mutations.js'
-import { factsSnapshotForDag, getFactsGroupForMutationArgs } from '../facts.js'
-import { historyDagFromNodes } from '../history-dag.js'
-import type { HistoryDagNode } from '../history-dag-common.js'
-import { rootEventNode } from '../history-dag-common.js'
-
-/** Used for conflict detection and event history compaction */
-export const facts = defineFacts({
-  todoExists: (id: string) => `todo-exists-${id}`,
-  todoIsWriteable: (id: string, writeable: boolean) => [`todo-is-writeable-${id}`, writeable],
-  todoCompleted: (id: string, completed: boolean) => [`todo-completed-${id}`, completed],
-  todoTextUpdated: (id: string) => `todo-text-updated-${id}`,
-  inputValue: (id: string) => `input-value-${id}`,
-})
-
-export const mutations = {
-  createTodo: defineMutation(
-    'createTodo',
-    Schema.Struct({ id: Schema.String, text: Schema.String }),
-    'INSERT INTO todos (id, text) VALUES ($id, $text)',
-    {
-      facts: ({ id }) => ({
-        modify: {
-          set: [facts.todoExists(id), facts.todoIsWriteable(id, true), facts.todoCompleted(id, false)],
-        },
-      }),
-    },
-  ),
-  upsertTodo: defineMutation(
-    'upsertTodo',
-    Schema.Struct({ id: Schema.String, text: Schema.optional(Schema.String) }),
-    'INSERT INTO todos (id, text) VALUES ($id, $text) ON CONFLICT (id) DO UPDATE SET text = $text',
-    {
-      facts: ({ id }, currentFacts) =>
-        // TODO enable an API along the lines of `map.has(key, value)`
-        currentFacts.has(facts.todoExists(id)) && currentFacts.get(facts.todoIsWriteable(id, true)[0]) === false
-          ? { require: [facts.todoExists(id), facts.todoIsWriteable(id, true)] }
-          : { modify: { set: [facts.todoExists(id), facts.todoIsWriteable(id, true), facts.todoTextUpdated(id)] } },
-    },
-  ),
-  completeTodo: defineMutation(
-    'completeTodo',
-    Schema.Struct({ id: Schema.String }),
-    // consider `RETURNING` to validate before applying facts
-    'UPDATE todos SET completed = true WHERE id = $id',
-    {
-      // prewrite assertions from DB
-      // enables more concurrency
-      // turning database inside out
-      // similar to upsert semantics
-      facts: ({ id }) => ({
-        require: [facts.todoExists(id), facts.todoIsWriteable(id, true)],
-        modify: { set: [facts.todoCompleted(id, true)] },
-      }),
-    },
-  ),
-  uncompleteTodo: defineMutation(
-    'uncompleteTodo',
-    Schema.Struct({ id: Schema.String }),
-    'UPDATE todos SET completed = false WHERE id = $id',
-    {
-      facts: ({ id }) => ({
-        require: [facts.todoExists(id), facts.todoIsWriteable(id, true)],
-        modify: { set: [facts.todoCompleted(id, false)] },
-      }),
-    },
-  ),
-  completeTodos: defineMutation(
-    'completeTodos',
-    Schema.Struct({ ids: Schema.Array(Schema.String) }),
-    'UPDATE todos SET completed = true WHERE id IN ($ids:csv)',
-    {
-      facts: ({ ids }) => ({
-        require: ids.flatMap((id) => [facts.todoExists(id), facts.todoIsWriteable(id, true)]),
-        modify: { set: ids.map((id) => facts.todoCompleted(id, true)) },
-      }),
-    },
-  ),
-  toggleTodo: defineMutation(
-    'toggleTodo',
-    Schema.Struct({ id: Schema.String }),
-    'UPDATE todos SET completed = NOT completed WHERE id = $id',
-    {
-      facts: ({ id }, currentFacts) => {
-        const currentIsCompleted = currentFacts.get(facts.todoCompleted(id, true)[0]) === true
-        return {
-          require: [facts.todoExists(id), facts.todoIsWriteable(id, true)],
-          modify: {
-            // remove: [facts.todoCompleted(id, currentIsCompleted)],
-            set: [facts.todoCompleted(id, !currentIsCompleted)],
-          },
-        }
-      },
-    },
-  ),
-  setReadonlyTodo: defineMutation(
-    'setReadonlyTodo',
-    Schema.Struct({ id: Schema.String, readonly: Schema.Boolean }),
-    'UPDATE todos SET readonly = $readonly WHERE id = $id',
-    {
-      facts: ({ id, readonly }) => ({
-        require: [facts.todoExists(id)],
-        modify: { set: [facts.todoIsWriteable(id, !readonly)] },
-      }),
-    },
-  ),
-  setTextTodo: defineMutation(
-    'setTextTodo',
-    Schema.Struct({ id: Schema.String, text: Schema.String }),
-    'UPDATE todos SET text = $text WHERE id = $id',
-    {
-      facts: ({ id }) => ({
-        require: [facts.todoExists(id), facts.todoIsWriteable(id, true)],
-        modify: { set: [facts.todoTextUpdated(id)] },
-      }),
-    },
-  ),
-  setInputValue: defineMutation(
-    'setInputValue',
-    Schema.Struct({ id: Schema.String, text: Schema.String }),
-    'UPDATE todos SET text = $text WHERE id = $id',
-    {
-      localOnly: true,
-      facts: ({ id }) => ({ modify: { set: [facts.inputValue(id)] } }),
-    },
-  ),
-}
-
-export type PartialEvent = { mutation: string; args: any }
-
-export const toEventNodes = (
-  partialEvents: PartialEvent[],
-  mutationDefs: Record<string, MutationDef.Any>,
-): HistoryDagNode[] => {
-  const nodesAcc: HistoryDagNode[] = [rootEventNode]
-
-  let currentEventId: EventId.EventId = EventId.ROOT
-
-  const getNextEventId = (mutationDef: MutationDef.Any): EventId.EventId => {
-    if (mutationDef.options.localOnly) {
-      return { global: currentEventId.global, local: currentEventId.local + 1 }
-    }
-    return { global: currentEventId.global + 1, local: 0 }
-  }
-
-  const eventNodes = partialEvents.map((partialEvent) => {
-    const mutationDef = mutationDefs[partialEvent.mutation]!
-    const eventId = getNextEventId(mutationDef)
-    currentEventId = eventId
-
-    const factsSnapshot = factsSnapshotForDag(historyDagFromNodes(nodesAcc, { skipFactsCheck: true }), undefined)
-    // console.log('factsSnapshot', eventId, factsSnapshot)
-    // const depRead: MutationEventFactsSnapshot = new Map<string, any>()
-    // const factsSnapshotProxy = new Proxy(factsSnapshot, {
-    //   get: (target, prop) => {
-    //     if (prop === 'has') {
-    //       return (key: string) => {
-    //         depRead.set(key, EMPTY_FACT_VALUE)
-    //         return target.has(key)
-    //       }
-    //     } else if (prop === 'get') {
-    //       return (key: string) => {
-    //         depRead.set(key, EMPTY_FACT_VALUE)
-    //         return target.get(key)
-    //       }
-    //     }
-
-    //     notYetImplemented(`toEventNodes: ${prop.toString()} is not yet implemented`)
-    //   },
-    // })
-
-    // const factsRes = mutationDef.options.facts?.(partialEvent.args, factsSnapshotProxy)
-    // console.log('factsRes', factsRes?.modify, factsRes?.require)
-    // const iterableToMap = (iterable: Iterable<MutationEventFactInput>) => {
-    //   const map = new Map()
-    //   for (const item of iterable) {
-    //     if (typeof item === 'string') {
-    //       map.set(item, EMPTY_FACT_VALUE)
-    //     } else {
-    //       map.set(item[0], item[1])
-    //     }
-    //   }
-    //   return map
-    // }
-    // const facts = {
-    //   modifyAdd: factsRes?.modify.add ? iterableToMap(factsRes.modify.add) : new Map(),
-    //   modifyRemove: factsRes?.modify.remove ? iterableToMap(factsRes.modify.remove) : new Map(),
-    //   depRequire: factsRes?.require ? iterableToMap(factsRes.require) : new Map(),
-    //   depRead,
-    // } satisfies MutationEventFactsGroup
-
-    // applyFactGroup(facts, factsSnapshot)
-
-    const facts = getFactsGroupForMutationArgs({
-      factsCallback: mutationDef.options.facts,
-      args: partialEvent.args,
-      currentFacts: factsSnapshot,
-    })
-
-    const node = {
-      id: eventId,
-      parentId: getParentId(eventId),
-      mutation: partialEvent.mutation,
-      args: partialEvent.args,
-      factsGroup: facts,
-    } satisfies HistoryDagNode
-    nodesAcc.push(node)
-    return node
-  })
-
-  eventNodes.unshift(rootEventNode as never)
-
-  // console.log('eventNodes', eventNodes)
-
-  return eventNodes
-}
-
-const getParentId = (eventId: EventId.EventId): EventId.EventId => {
-  const globalParentId = eventId.global
-  const localParentId = eventId.local - 1
-
-  if (localParentId < 0) {
-    return { global: globalParentId - 1, local: 0 }
-  }
-
-  return { global: globalParentId, local: localParentId }
-}