@livestore/common 0.3.0-dev.3 → 0.3.0-dev.31

package/src/leader-thread/mutationlog.ts

@@ -1,42 +0,0 @@
-import { Effect, Schema } from '@livestore/utils/effect'
-
-import type { SynchronousDatabase } from '../adapter-types.js'
-import * as EventId from '../schema/EventId.js'
-import { MUTATION_LOG_META_TABLE, mutationLogMetaTable, SYNC_STATUS_TABLE } from '../schema/system-tables.js'
-import { prepareBindValues, sql } from '../util.js'
-import { LeaderThreadCtx } from './types.js'
-
-export const getMutationEventsSince = (since: EventId.EventId) =>
-  Effect.gen(function* () {
-    const { dbLog } = yield* LeaderThreadCtx
-
-    const query = mutationLogMetaTable.query.where('idGlobal', '>=', since.global).asSql()
-    const pendingMutationEventsRaw = dbLog.select(query.query, prepareBindValues(query.bindValues, query.query))
-    const pendingMutationEvents = Schema.decodeUnknownSync(mutationLogMetaTable.schema.pipe(Schema.Array))(
-      pendingMutationEventsRaw,
-    )
-
-    return pendingMutationEvents
-      .map((_) => ({
-        mutation: _.mutation,
-        args: _.argsJson,
-        id: { global: _.idGlobal, local: _.idLocal },
-        parentId: { global: _.parentIdGlobal, local: _.parentIdLocal },
-      }))
-      .filter((_) => EventId.compare(_.id, since) > 0)
-  })
-
-export const getLocalHeadFromDb = (dbLog: SynchronousDatabase) => {
-  const res = dbLog.select<{ idGlobal: number; idLocal: number }>(
-    sql`select idGlobal, idLocal from ${MUTATION_LOG_META_TABLE} order by idGlobal DESC, idLocal DESC limit 1`,
-  )[0]
-
-  return res ? { global: res.idGlobal, local: res.idLocal } : EventId.ROOT
-}
-
-export const getBackendHeadFromDb = (dbLog: SynchronousDatabase) =>
-  dbLog.select<{ head: number }>(sql`select head from ${SYNC_STATUS_TABLE}`)[0]?.head ?? EventId.ROOT.global
-
-// TODO use prepared statements
-export const updateBackendHead = (dbLog: SynchronousDatabase, head: EventId.EventId) =>
-  dbLog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)

package/src/leader-thread/pull-queue-set.ts

@@ -1,58 +0,0 @@
-import { Effect, Queue } from '@livestore/utils/effect'
-
-import * as MutationEvent from '../schema/MutationEvent.js'
-import { getMutationEventsSince } from './mutationlog.js'
-import { type PullQueueItem, type PullQueueSet } from './types.js'
-
-export const makePullQueueSet = Effect.gen(function* () {
-  const set = new Set<Queue.Queue<PullQueueItem>>()
-
-  yield* Effect.addFinalizer(() =>
-    Effect.gen(function* () {
-      for (const queue of set) {
-        yield* Queue.shutdown(queue)
-      }
-
-      set.clear()
-    }),
-  )
-
-  const makeQueue: PullQueueSet['makeQueue'] = (since) =>
-    Effect.gen(function* () {
-      const queue = yield* Queue.unbounded<PullQueueItem>().pipe(Effect.acquireRelease(Queue.shutdown))
-
-      yield* Effect.addFinalizer(() => Effect.sync(() => set.delete(queue)))
-
-      const mutationEvents = yield* getMutationEventsSince(since)
-
-      if (mutationEvents.length > 0) {
-        const newEvents = mutationEvents.map((mutationEvent) => new MutationEvent.EncodedWithMeta(mutationEvent))
-        yield* queue.offer({ payload: { _tag: 'upstream-advance', newEvents }, remaining: 0 })
-      }
-
-      set.add(queue)
-
-      return queue
-    })
-
-  const offer: PullQueueSet['offer'] = (item) =>
-    Effect.gen(function* () {
-      // Short-circuit if the payload is an empty upstream advance
-      if (
-        item.payload._tag === 'upstream-advance' &&
-        item.payload.newEvents.length === 0 &&
-        item.payload.trimRollbackUntil === undefined
-      ) {
-        return
-      }
-
-      for (const queue of set) {
-        yield* Queue.offer(queue, item)
-      }
-    })
-
-  return {
-    makeQueue,
-    offer,
-  }
-})

package/src/mutation.ts

@@ -1,81 +0,0 @@
-import { Schema } from '@livestore/utils/effect'
-
-import { SessionIdSymbol } from './adapter-types.js'
-import type * as MutationEvent from './schema/MutationEvent.js'
-import type { MutationDef } from './schema/mutations.js'
-import type { PreparedBindValues } from './util.js'
-import { prepareBindValues } from './util.js'
-
-export const getExecArgsFromMutation = ({
-  mutationDef,
-  mutationEventDecoded,
-}: {
-  mutationDef: MutationDef.Any
-  mutationEventDecoded: MutationEvent.Any | MutationEvent.PartialAny
-}): ReadonlyArray<{
-  statementSql: string
-  bindValues: PreparedBindValues
-  writeTables: ReadonlySet<string> | undefined
-}> => {
-  let statementRes: ReadonlyArray<
-    string | { sql: string; bindValues: Record<string, unknown>; writeTables?: ReadonlySet<string> }
-  >
-
-  switch (typeof mutationDef.sql) {
-    case 'function': {
-      const res = mutationDef.sql(mutationEventDecoded.args)
-      statementRes = Array.isArray(res) ? res : [res]
-      break
-    }
-    case 'string': {
-      statementRes = [mutationDef.sql]
-      break
-    }
-    default: {
-      statementRes = mutationDef.sql
-      break
-    }
-  }
-
-  return statementRes.map((statementRes) => {
-    const statementSql = typeof statementRes === 'string' ? statementRes : statementRes.sql
-
-    const bindValues =
-      typeof statementRes === 'string'
-        ? Schema.encodeUnknownSync(mutationDef.schema)(mutationEventDecoded.args)
-        : statementRes.bindValues
-
-    const writeTables = typeof statementRes === 'string' ? undefined : statementRes.writeTables
-
-    return { statementSql, bindValues: prepareBindValues(bindValues ?? {}, statementSql), writeTables }
-  })
-}
-
-// NOTE we should explore whether there is a more elegant solution
-// e.g. by leveraging the schema to replace the sessionIdSymbol
-export const replaceSessionIdSymbol = (
-  bindValues: Record<string, unknown> | ReadonlyArray<unknown>,
-  sessionId: string,
-) => {
-  deepReplaceValue(bindValues, SessionIdSymbol, sessionId)
-}
-
-const deepReplaceValue = <S, R>(input: any, searchValue: S, replaceValue: R): void => {
-  if (Array.isArray(input)) {
-    for (const i in input) {
-      if (input[i] === searchValue) {
-        input[i] = replaceValue
-      } else {
-        deepReplaceValue(input[i], searchValue, replaceValue)
-      }
-    }
-  } else if (typeof input === 'object' && input !== null) {
-    for (const key in input) {
-      if (input[key] === searchValue) {
-        input[key] = replaceValue
-      } else {
-        deepReplaceValue(input[key], searchValue, replaceValue)
-      }
-    }
-  }
-}

package/src/query-info.ts

@@ -1,78 +0,0 @@
-import type { SessionIdSymbol } from './adapter-types.js'
-import type { DbSchema } from './schema/mod.js'
-
-/**
- * Semantic information about a query with supported cases being:
- * - a whole row
- * - a single column value
- * - a sub value in a JSON column
- *
- * This information is currently only used for derived mutations.
- */
-export type QueryInfo = QueryInfo.None | QueryInfo.Row | QueryInfo.Col | QueryInfo.ColJsonValue
-// export type QueryInfo<TTableDef extends DbSchema.TableDefBase = DbSchema.TableDefBase> =
-// | QueryInfo.None
-// | QueryInfo.Row<TTableDef>
-// | QueryInfo.ColJsonValue<TTableDef, GetJsonColumn<TTableDef>>
-// | QueryInfo.Col<TTableDef, keyof TTableDef['sqliteDef']['columns']>
-
-export namespace QueryInfo {
-  export type None = {
-    _tag: 'None'
-  }
-
-  export type Row = {
-    _tag: 'Row'
-    table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
-  }
-
-  export type Col = {
-    _tag: 'Col'
-    table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
-    column: string
-  }
-
-  export type ColJsonValue = {
-    _tag: 'ColJsonValue'
-    table: DbSchema.TableDefBase
-    id: string | SessionIdSymbol
-    column: string
-    /**
-     * example: `$.tabs[3].items[2]` (`$` referring to the column value)
-     */
-    jsonPath: string
-  }
-
-  // NOTE maybe we want to bring back type-params back like below
-  // export type Row<TTableDef extends DbSchema.TableDefBase> = {
-  //   _tag: 'Row'
-  //   table: TTableDef
-  //   id: string | SessionIdSymbol
-  // }
-
-  // export type Col<TTableDef extends DbSchema.TableDefBase, TColName extends keyof TTableDef['sqliteDef']['columns']> = {
-  //   _tag: 'Col'
-  //   table: TTableDef
-  //   id: string | SessionIdSymbol
-  //   column: TColName
-  // }
-
-  // export type ColJsonValue<TTableDef extends DbSchema.TableDefBase, TColName extends GetJsonColumn<TTableDef>> = {
-  //   _tag: 'ColJsonValue'
-  //   table: TTableDef
-  //   id: string | SessionIdSymbol
-  //   column: TColName
-  //   /**
-  //    * example: `$.tabs[3].items[2]` (`$` referring to the column value)
-  //    */
-  //   jsonPath: string
-  // }
-}
-
-// type GetJsonColumn<TTableDef extends DbSchema.TableDefBase> = keyof {
-//   [ColName in keyof TTableDef['sqliteDef']['columns'] as TTableDef['sqliteDef']['columns'][ColName]['columnType'] extends 'text'
-//     ? ColName
-//     : never]: {}
-// }

package/src/rehydrate-from-mutationlog.ts

@@ -1,127 +0,0 @@
-import { isDevEnv, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
-import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
-
-import { type MigrationOptionsFromMutationLog, type SynchronousDatabase, UnexpectedError } from './adapter-types.js'
-import { getExecArgsFromMutation } from './mutation.js'
-import type { LiveStoreSchema, MutationDef, MutationEvent, MutationLogMetaRow } from './schema/mod.js'
-import { EventId, MUTATION_LOG_META_TABLE } from './schema/mod.js'
-import type { PreparedBindValues } from './util.js'
-import { sql } from './util.js'
-
-export const rehydrateFromMutationLog = ({
-  logDb,
-  db,
-  schema,
-  migrationOptions,
-  onProgress,
-}: {
-  logDb: SynchronousDatabase
-  db: SynchronousDatabase
-  schema: LiveStoreSchema
-  migrationOptions: MigrationOptionsFromMutationLog
-  onProgress: (_: { done: number; total: number }) => Effect.Effect<void>
-}) =>
-  Effect.gen(function* () {
-    const mutationsCount = logDb.select<{ count: number }>(
-      `SELECT COUNT(*) AS count FROM ${MUTATION_LOG_META_TABLE}`,
-    )[0]!.count
-
-    const hashMutation = memoizeByRef((mutation: MutationDef.Any) => Schema.hash(mutation.schema))
-
-    const processMutation = (row: MutationLogMetaRow) =>
-      Effect.gen(function* () {
-        const mutationDef = schema.mutations.get(row.mutation) ?? shouldNeverHappen(`Unknown mutation ${row.mutation}`)
-
-        if (migrationOptions.excludeMutations?.has(row.mutation) === true) return
-
-        if (hashMutation(mutationDef) !== row.schemaHash) {
-          yield* Effect.logWarning(
-            `Schema hash mismatch for mutation ${row.mutation}. Trying to apply mutation anyway.`,
-          )
-        }
-
-        const argsDecoded = yield* Schema.decodeUnknown(Schema.parseJson(mutationDef.schema))(row.argsJson).pipe(
-          Effect.mapError((cause) =>
-            UnexpectedError.make({
-              cause,
-              note: `\
-There was an error during rehydrating from the mutation log while decoding
-the persisted mutation event args for mutation "${row.mutation}".
-This likely means the schema has changed in an incompatible way.
-`,
-            }),
-          ),
-        )
-
-        const mutationEventDecoded = {
-          id: { global: row.idGlobal, local: row.idLocal },
-          parentId: { global: row.parentIdGlobal, local: row.parentIdLocal },
-          mutation: row.mutation,
-          args: argsDecoded,
-        } satisfies MutationEvent.Any
-
-        const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
-
-        const makeExecuteOptions = (statementSql: string, bindValues: any) => ({
-          onRowsChanged: (rowsChanged: number) => {
-            if (rowsChanged === 0 && migrationOptions.logging?.excludeAffectedRows?.(statementSql) !== true) {
-              console.warn(`Mutation "${mutationDef.name}" did not affect any rows:`, statementSql, bindValues)
-            }
-          },
-        })
-
-        for (const { statementSql, bindValues } of execArgsArr) {
-          // TODO cache prepared statements for mutations
-          db.execute(statementSql, bindValues, isDevEnv() ? makeExecuteOptions(statementSql, bindValues) : undefined)
-          // console.log(`Re-executed mutation ${mutationSql}`, bindValues)
-        }
-      }).pipe(Effect.withSpan(`@livestore/common:rehydrateFromMutationLog:processMutation`))
-
-    const CHUNK_SIZE = 100
-
-    const stmt = logDb.prepare(sql`\
-SELECT * FROM ${MUTATION_LOG_META_TABLE} 
-WHERE idGlobal > $idGlobal OR (idGlobal = $idGlobal AND idLocal > $idLocal)
-ORDER BY idGlobal ASC, idLocal ASC
-LIMIT ${CHUNK_SIZE}
-`)
-
-    let processedMutations = 0
-
-    yield* Stream.unfoldChunk<Chunk.Chunk<MutationLogMetaRow> | { _tag: 'Initial ' }, MutationLogMetaRow>(
-      { _tag: 'Initial ' },
-      (item) => {
-        // End stream if no more rows
-        if (Chunk.isChunk(item) && item.length === 0) return Option.none()
-
-        const lastId = Chunk.isChunk(item)
-          ? Chunk.last(item).pipe(
-              Option.map((_) => ({ global: _.idGlobal, local: _.idLocal })),
-              Option.getOrElse(() => EventId.ROOT),
-            )
-          : EventId.ROOT
-        const nextItem = Chunk.fromIterable(
-          stmt.select<MutationLogMetaRow>({
-            $idGlobal: lastId?.global,
-            $idLocal: lastId?.local,
-          } as any as PreparedBindValues),
-        )
-        const prevItem = Chunk.isChunk(item) ? item : Chunk.empty()
-        return Option.some([prevItem, nextItem])
-      },
-    ).pipe(
-      Stream.bufferChunks({ capacity: 2 }),
-      Stream.tap((row) =>
-        Effect.gen(function* () {
-          yield* processMutation(row)
-
-          processedMutations++
-          yield* onProgress({ done: processedMutations, total: mutationsCount })
-        }),
-      ),
-      Stream.runDrain,
-    )
-  }).pipe(
-    Effect.withPerformanceMeasure('@livestore/common:rehydrateFromMutationLog'),
-    Effect.withSpan('@livestore/common:rehydrateFromMutationLog'),
-  )

package/src/schema/MutationEvent.ts

@@ -1,161 +0,0 @@
-import { memoizeByRef } from '@livestore/utils'
-import type { Deferred } from '@livestore/utils/effect'
-import { Schema } from '@livestore/utils/effect'
-
-import * as EventId from './EventId.js'
-import type { MutationDef, MutationDefRecord } from './mutations.js'
-import type { LiveStoreSchema } from './schema.js'
-
-export type MutationEventPartial<TMutationsDef extends MutationDef.Any> = {
-  mutation: TMutationsDef['name']
-  args: Schema.Schema.Type<TMutationsDef['schema']>
-}
-
-export type MutationEventPartialEncoded<TMutationsDef extends MutationDef.Any> = {
-  mutation: TMutationsDef['name']
-  args: Schema.Schema.Encoded<TMutationsDef['schema']>
-}
-
-export type MutationEvent<TMutationsDef extends MutationDef.Any> = {
-  mutation: TMutationsDef['name']
-  args: Schema.Schema.Type<TMutationsDef['schema']>
-  id: EventId.EventId
-  parentId: EventId.EventId
-}
-
-export type MutationEventEncoded<TMutationsDef extends MutationDef.Any> = {
-  mutation: TMutationsDef['name']
-  args: Schema.Schema.Encoded<TMutationsDef['schema']>
-  id: EventId.EventId
-  parentId: EventId.EventId
-}
-
-export type Any = MutationEvent<MutationDef.Any>
-export type AnyEncoded = MutationEventEncoded<MutationDef.Any>
-
-export type PartialAny = MutationEventPartial<MutationDef.Any>
-export type PartialAnyEncoded = MutationEventPartialEncoded<MutationDef.Any>
-
-export type PartialForSchema<TSchema extends LiveStoreSchema> = {
-  [K in keyof TSchema['_MutationDefMapType']]: MutationEventPartial<TSchema['_MutationDefMapType'][K]>
-}[keyof TSchema['_MutationDefMapType']]
-
-export type ForSchema<TSchema extends LiveStoreSchema> = {
-  [K in keyof TSchema['_MutationDefMapType']]: MutationEvent<TSchema['_MutationDefMapType'][K]>
-}[keyof TSchema['_MutationDefMapType']]
-
-export const isPartialMutationEvent = (mutationEvent: Any | PartialAny): mutationEvent is PartialAny =>
-  'id' in mutationEvent === false && 'parentId' in mutationEvent === false
-
-export type ForMutationDefRecord<TMutationsDefRecord extends MutationDefRecord> = Schema.Schema<
-  {
-    [K in keyof TMutationsDefRecord]: {
-      mutation: K
-      args: Schema.Schema.Type<TMutationsDefRecord[K]['schema']>
-      id: EventId.EventId
-      parentId: EventId.EventId
-    }
-  }[keyof TMutationsDefRecord],
-  {
-    [K in keyof TMutationsDefRecord]: {
-      mutation: K
-      args: Schema.Schema.Encoded<TMutationsDefRecord[K]['schema']>
-      id: EventId.EventId
-      parentId: EventId.EventId
-    }
-  }[keyof TMutationsDefRecord]
->
-
-export type MutationEventPartialSchema<TMutationsDefRecord extends MutationDefRecord> = Schema.Schema<
-  {
-    [K in keyof TMutationsDefRecord]: {
-      mutation: K
-      args: Schema.Schema.Type<TMutationsDefRecord[K]['schema']>
-    }
-  }[keyof TMutationsDefRecord],
-  {
-    [K in keyof TMutationsDefRecord]: {
-      mutation: K
-      args: Schema.Schema.Encoded<TMutationsDefRecord[K]['schema']>
-    }
-  }[keyof TMutationsDefRecord]
->
-
-export const makeMutationEventSchema = <TSchema extends LiveStoreSchema>(
-  schema: TSchema,
-): ForMutationDefRecord<TSchema['_MutationDefMapType']> =>
-  Schema.Union(
-    ...[...schema.mutations.values()].map((def) =>
-      Schema.Struct({
-        mutation: Schema.Literal(def.name),
-        args: def.schema,
-        id: EventId.EventId,
-        parentId: EventId.EventId,
-      }),
-    ),
-  ).annotations({ title: 'MutationEvent' }) as any
-
-export const makeMutationEventPartialSchema = <TSchema extends LiveStoreSchema>(
-  schema: TSchema,
-): MutationEventPartialSchema<TSchema['_MutationDefMapType']> =>
-  Schema.Union(
-    ...[...schema.mutations.values()].map((def) =>
-      Schema.Struct({
-        mutation: Schema.Literal(def.name),
-        args: def.schema,
-      }),
-    ),
-  ).annotations({ title: 'MutationEventSchemaPartial' }) as any
-
-export const makeMutationEventSchemaMemo = memoizeByRef(makeMutationEventSchema)
-
-export const Any = Schema.Struct({
-  mutation: Schema.String,
-  args: Schema.Any,
-  id: EventId.EventId,
-  parentId: EventId.EventId,
-}).annotations({ title: 'MutationEvent.Any' })
-
-export const DecodedAny = Schema.typeSchema(Any).annotations({
-  title: 'MutationEvent.DecodedAny',
-})
-
-export const EncodedAny = Schema.encodedSchema(Any).annotations({
-  title: 'MutationEvent.EncodedAny',
-})
-
-/** Equivalent to EncodedAny but with a meta field and some convenience methods */
-export class EncodedWithMeta extends Schema.Class<EncodedWithMeta>('MutationEvent.EncodedWithMeta')({
-  mutation: Schema.String,
-  args: Schema.Any,
-  id: EventId.EventId,
-  parentId: EventId.EventId,
-  meta: Schema.optionalWith(
-    Schema.Any as Schema.Schema<{ deferred?: Deferred.Deferred<void>; sessionChangeset?: Uint8Array }>,
-    { default: () => ({}) },
-  ),
-}) {
-  toJSON = (): any => {
-    // Only used for logging/debugging
-    // - More readable way to print the id + parentId
-    // - not including `meta`
-    return {
-      id: `(${this.id.global},${this.id.local}) → (${this.parentId.global},${this.parentId.local})`,
-      mutation: this.mutation,
-      args: this.args,
-    }
-  }
-
-  rebase = (parentId: EventId.EventId, isLocal: boolean) =>
-    new EncodedWithMeta({
-      ...this,
-      ...EventId.nextPair(this.id, isLocal),
-    })
-}
-
-export const isEqualEncoded = (a: AnyEncoded, b: AnyEncoded) =>
-  a.id.global === b.id.global &&
-  a.id.local === b.id.local &&
-  a.mutation === b.mutation &&
-  // TODO use schema equality here
-  JSON.stringify(a.args) === JSON.stringify(b.args)