@livestore/common 0.0.0-snapshot-909cdd1ac2fd591945c2be2b0f53e14d87f3c9d4

package/src/schema/index.ts

@@ -0,0 +1,144 @@
+import type { SqliteDsl } from '@livestore/db-schema'
+import { SqliteAst } from '@livestore/db-schema'
+import { isReadonlyArray, shouldNeverHappen } from '@livestore/utils'
+
+import type { MigrationOptions } from '../adapter-types.js'
+import { makeDerivedMutationDefsForTable } from '../derived-mutations.js'
+import {
+  type MutationDef,
+  type MutationDefMap,
+  type MutationDefRecord,
+  type RawSqlMutation,
+  rawSqlMutation,
+} from './mutations.js'
+import { systemTables } from './system-tables.js'
+import { type TableDef, tableHasDerivedMutations } from './table-def.js'
+
+export * from './system-tables.js'
+export * as DbSchema from './table-def.js'
+export * from './mutations.js'
+export * from './schema-helpers.js'
+
+export const LiveStoreSchemaSymbol = Symbol.for('livestore.LiveStoreSchema')
+export type LiveStoreSchemaSymbol = typeof LiveStoreSchemaSymbol
+
+export type LiveStoreSchema<
+  TDbSchema extends SqliteDsl.DbSchema = SqliteDsl.DbSchema,
+  TMutationsDefRecord extends MutationDefRecord = MutationDefRecord,
+> = {
+  readonly _Type: LiveStoreSchemaSymbol
+  /** Only used on type-level */
+  readonly _DbSchemaType: TDbSchema
+  /** Only used on type-level */
+  readonly _MutationDefMapType: TMutationsDefRecord
+
+  readonly tables: Map<string, TableDef>
+  readonly mutations: MutationDefMap
+  /** Compound hash of all table defs etc */
+  readonly hash: number
+
+  migrationOptions: MigrationOptions
+}
+
+export type InputSchema = {
+  readonly tables: Record<string, TableDef> | ReadonlyArray<TableDef>
+  readonly mutations?: ReadonlyArray<MutationDef.Any> | Record<string, MutationDef.Any>
+  /**
+   * Can be used to isolate multiple LiveStore apps running in the same origin
+   */
+  // TODO remove this in favour of storeId
+  readonly key?: string
+}
+
+export const makeSchema = <TInputSchema extends InputSchema>(
+  /** Note when using the object-notation for tables/mutations, the object keys are ignored and not used as table/mutation names */
+  inputSchema: TInputSchema & {
+    /** "hard-reset" is currently the default strategy */
+    migrations?: MigrationOptions<FromInputSchema.DeriveSchema<TInputSchema>>
+  },
+): FromInputSchema.DeriveSchema<TInputSchema> => {
+  const inputTables: ReadonlyArray<TableDef> = Array.isArray(inputSchema.tables)
+    ? inputSchema.tables
+    : Object.values(inputSchema.tables)
+
+  const tables = new Map<string, TableDef>()
+
+  for (const tableDef of inputTables) {
+    // TODO validate tables (e.g. index names are unique)
+    if (tables.has(tableDef.sqliteDef.ast.name)) {
+      shouldNeverHappen(`Duplicate table name: ${tableDef.sqliteDef.ast.name}. Please use unique names for tables.`)
+    }
+    tables.set(tableDef.sqliteDef.ast.name, tableDef)
+  }
+
+  for (const tableDef of systemTables) {
+    tables.set(tableDef.sqliteDef.name, tableDef)
+  }
+
+  const mutations: MutationDefMap = new Map()
+
+  if (isReadonlyArray(inputSchema.mutations)) {
+    for (const mutation of inputSchema.mutations) {
+      mutations.set(mutation.name, mutation)
+    }
+  } else {
+    for (const mutation of Object.values(inputSchema.mutations ?? {})) {
+      if (mutations.has(mutation.name)) {
+        shouldNeverHappen(`Duplicate mutation name: ${mutation.name}. Please use unique names for mutations.`)
+      }
+      mutations.set(mutation.name, mutation)
+    }
+  }
+
+  mutations.set(rawSqlMutation.name, rawSqlMutation)
+
+  for (const tableDef of tables.values()) {
+    if (tableHasDerivedMutations(tableDef)) {
+      const derivedMutationDefs = makeDerivedMutationDefsForTable(tableDef)
+      mutations.set(derivedMutationDefs.insert.name, derivedMutationDefs.insert)
+      mutations.set(derivedMutationDefs.update.name, derivedMutationDefs.update)
+      mutations.set(derivedMutationDefs.delete.name, derivedMutationDefs.delete)
+    }
+  }
+
+  const hash = SqliteAst.hash({
+    _tag: 'dbSchema',
+    tables: [...tables.values()].map((_) => _.sqliteDef.ast),
+  })
+
+  return {
+    _Type: LiveStoreSchemaSymbol,
+    _DbSchemaType: Symbol.for('livestore.DbSchemaType') as any,
+    _MutationDefMapType: Symbol.for('livestore.MutationDefMapType') as any,
+    tables,
+    mutations,
+    migrationOptions: inputSchema.migrations ?? { strategy: 'hard-reset' },
+    hash,
+  } satisfies LiveStoreSchema
+}
+
+export namespace FromInputSchema {
+  export type DeriveSchema<TInputSchema extends InputSchema> = LiveStoreSchema<
+    DbSchemaFromInputSchemaTables<TInputSchema['tables']>,
+    MutationDefRecordFromInputSchemaMutations<TInputSchema['mutations']>
+  >
+
+  /**
+   * In case of ...
+   * - array: we use the table name of each array item (= table definition) as the object key
+   * - object: we discard the keys of the input object and use the table name of each object value (= table definition) as the new object key
+   */
+  type DbSchemaFromInputSchemaTables<TTables extends InputSchema['tables']> =
+    TTables extends ReadonlyArray<TableDef>
+      ? { [K in TTables[number] as K['sqliteDef']['name']]: K['sqliteDef'] }
+      : TTables extends Record<string, TableDef>
+        ? { [K in keyof TTables as TTables[K]['sqliteDef']['name']]: TTables[K]['sqliteDef'] }
+        : never
+
+  type MutationDefRecordFromInputSchemaMutations<TMutations extends InputSchema['mutations']> =
+    TMutations extends ReadonlyArray<MutationDef.Any>
+      ? { [K in TMutations[number] as K['name']]: K } & { 'livestore.RawSql': RawSqlMutation }
+      : TMutations extends { [name: string]: MutationDef.Any }
+        ? { [K in keyof TMutations as TMutations[K]['name']]: TMutations[K] } & { 'livestore.RawSql': RawSqlMutation }
+        : never
+}

package/src/schema/mutations.ts

@@ -0,0 +1,313 @@
+import { memoizeByRef } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+
+import { EventId } from '../adapter-types.js'
+import type { BindValues } from '../sql-queries/sql-queries.js'
+import type { LiveStoreSchema } from './index.js'
+
+export type MutationDefMap = Map<string | 'livestore.RawSql', MutationDef.Any>
+export type MutationDefRecord = {
+  'livestore.RawSql': RawSqlMutation
+  [name: string]: MutationDef.Any
+}
+
+export type InternalMutationSchema<TRecord extends MutationDefRecord = MutationDefRecord> = {
+  _DefRecord: TRecord
+
+  map: Map<keyof TRecord, TRecord[keyof TRecord]>
+  schemaHashMap: Map<keyof TRecord, number>
+}
+
+export type MutationDefSqlResult<TTo> =
+  | SingleOrReadonlyArray<string>
+  | ((args: TTo) => SingleOrReadonlyArray<
+      | string
+      | {
+          sql: string
+          /** Note args need to be manually encoded to `BindValues` when returning this argument */
+          bindValues: BindValues
+          writeTables?: ReadonlySet<string>
+        }
+    >)
+
+export type SingleOrReadonlyArray<T> = T | ReadonlyArray<T>
+
+export type MutationDef<TName extends string, TFrom, TTo> = {
+  name: TName
+  schema: Schema.Schema<TTo, TFrom>
+  sql: MutationDefSqlResult<NoInfer<TTo>>
+  options: {
+    /** Warning: This feature is not fully implemented yet */
+    historyId: string
+    /**
+     * When set to true, the mutation won't be synced over the network
+     */
+    localOnly: boolean
+    /** Warning: This feature is not fully implemented yet */
+    facts: FactsCallback<TTo> | undefined
+  }
+
+  /** Helper function to construct a partial mutation event */
+  (
+    args: TTo,
+    options?: {
+      id?: number
+    },
+  ): {
+    mutation: TName
+    args: TTo
+    // TODO remove/clean up after sync-next is fully implemented
+    id?: EventId
+  }
+}
+
+export type FactsCallback<TTo> = (
+  args: TTo,
+  currentFacts: MutationEventFacts,
+) => {
+  modify: {
+    set: Iterable<MutationEventFactInput>
+    unset: Iterable<MutationEventFactInput>
+  }
+  require: Iterable<MutationEventFactInput>
+}
+
+export namespace MutationDef {
+  export type Any = MutationDef<string, any, any>
+}
+
+export type MutationEventKey = string
+export type MutationEventFact = string
+export type MutationEventFacts = ReadonlyMap<string, any>
+
+export type MutationEventFactsGroup = {
+  modifySet: MutationEventFacts
+  modifyUnset: MutationEventFacts
+
+  /**
+   * Events on independent "dependency" branches are commutative which can facilitate more prioritized syncing
+   */
+  depRequire: MutationEventFacts
+  depRead: MutationEventFacts
+}
+
+export type MutationEventFactsSnapshot = Map<string, any>
+
+export type MutationEventFactInput = string | readonly [string, any]
+
+export const defineFacts = <
+  TRecord extends Record<string, MutationEventFactInput | ((...args: any[]) => MutationEventFactInput)>,
+>(
+  record: TRecord,
+): TRecord => record
+
+export type DefineMutationOptions<TTo> = {
+  historyId?: string
+  /** Warning: This feature is not fully implemented yet */
+  facts?: (
+    args: TTo,
+    currentFacts: MutationEventFacts,
+  ) => {
+    modify?: {
+      set?: Iterable<MutationEventFactInput>
+      unset?: Iterable<MutationEventFactInput>
+    }
+    /**
+     * Two purposes: constrain history and constrain compaction
+     */
+    require?: Iterable<MutationEventFactInput>
+  }
+  /**
+   * When set to true, the mutation won't be synced over the network
+   */
+  localOnly?: boolean
+}
+
+// TODO possibly also allow for mutation event subsumption behaviour
+export const defineMutation = <TName extends string, TFrom, TTo>(
+  name: TName,
+  schema: Schema.Schema<TTo, TFrom>,
+  sql: MutationDefSqlResult<NoInfer<TTo>>,
+  options?: DefineMutationOptions<TTo>,
+): MutationDef<TName, TFrom, TTo> => {
+  const makePartialEvent = (
+    args: TTo,
+    options?: {
+      id?: EventId
+    },
+  ) => ({ mutation: name, args, ...options })
+
+  Object.defineProperty(makePartialEvent, 'name', { value: name })
+  Object.defineProperty(makePartialEvent, 'schema', { value: schema })
+  Object.defineProperty(makePartialEvent, 'sql', { value: sql })
+  Object.defineProperty(makePartialEvent, 'options', {
+    value: {
+      historyId: options?.historyId ?? 'main',
+      localOnly: options?.localOnly ?? false,
+      facts: options?.facts
+        ? (args, currentFacts) => {
+            const res = options.facts!(args, currentFacts)
+            return {
+              modify: {
+                set: res.modify?.set ? new Set(res.modify.set) : new Set(),
+                unset: res.modify?.unset ? new Set(res.modify.unset) : new Set(),
+              },
+              require: res.require ? new Set(res.require) : new Set(),
+            }
+          }
+        : undefined,
+    } satisfies MutationDef.Any['options'],
+  })
+
+  return makePartialEvent as MutationDef<TName, TFrom, TTo>
+}
+
+export const makeMutationDefRecord = <TInputRecord extends Record<string, MutationDef.Any>>(
+  inputRecord: TInputRecord,
+): {
+  [K in TInputRecord[keyof TInputRecord]['name']]: Extract<TInputRecord[keyof TInputRecord], { name: K }>
+} => {
+  const result: any = {}
+
+  for (const [name, def] of Object.entries(inputRecord)) {
+    result[name] = def
+  }
+
+  result['livestore.RawSql'] = rawSqlMutation
+
+  return result
+}
+
+export const rawSqlMutation = defineMutation(
+  'livestore.RawSql',
+  Schema.Struct({
+    sql: Schema.String,
+    bindValues: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.Any })),
+    writeTables: Schema.optional(Schema.ReadonlySet(Schema.String)),
+  }),
+  ({ sql, bindValues, writeTables }) => ({ sql, bindValues: bindValues ?? {}, writeTables }),
+)
+
+export type RawSqlMutation = typeof rawSqlMutation
+export type RawSqlMutationEvent = ReturnType<typeof rawSqlMutation>
+
+export type MutationEventPartial<TMutationsDef extends MutationDef.Any> = {
+  mutation: TMutationsDef['name']
+  args: Schema.Schema.Type<TMutationsDef['schema']>
+}
+
+export type MutationEventPartialEncoded<TMutationsDef extends MutationDef.Any> = {
+  mutation: TMutationsDef['name']
+  args: Schema.Schema.Encoded<TMutationsDef['schema']>
+}
+
+export type MutationEvent<TMutationsDef extends MutationDef.Any> = {
+  mutation: TMutationsDef['name']
+  args: Schema.Schema.Type<TMutationsDef['schema']>
+  id: EventId
+  parentId: EventId
+}
+
+export type MutationEventEncoded<TMutationsDef extends MutationDef.Any> = {
+  mutation: TMutationsDef['name']
+  args: Schema.Schema.Encoded<TMutationsDef['schema']>
+  id: EventId
+  parentId: EventId
+}
+
+export namespace MutationEvent {
+  export type Any = MutationEvent<MutationDef.Any>
+  export type AnyEncoded = MutationEventEncoded<MutationDef.Any>
+
+  export type PartialAny = MutationEventPartial<MutationDef.Any>
+  export type PartialAnyEncoded = MutationEventPartialEncoded<MutationDef.Any>
+
+  export type PartialForSchema<TSchema extends LiveStoreSchema> = {
+    [K in keyof TSchema['_MutationDefMapType']]: MutationEventPartial<TSchema['_MutationDefMapType'][K]>
+  }[keyof TSchema['_MutationDefMapType']]
+
+  export type ForSchema<TSchema extends LiveStoreSchema> = {
+    [K in keyof TSchema['_MutationDefMapType']]: MutationEvent<TSchema['_MutationDefMapType'][K]>
+  }[keyof TSchema['_MutationDefMapType']]
+}
+
+export const isPartialMutationEvent = (
+  mutationEvent: MutationEvent.Any | MutationEvent.PartialAny,
+): mutationEvent is MutationEvent.PartialAny => 'id' in mutationEvent === false && 'parentId' in mutationEvent === false
+
+export type MutationEventSchema<TMutationsDefRecord extends MutationDefRecord> = Schema.Schema<
+  {
+    [K in keyof TMutationsDefRecord]: {
+      mutation: K
+      args: Schema.Schema.Type<TMutationsDefRecord[K]['schema']>
+      id: EventId
+      parentId: EventId
+    }
+  }[keyof TMutationsDefRecord],
+  {
+    [K in keyof TMutationsDefRecord]: {
+      mutation: K
+      args: Schema.Schema.Encoded<TMutationsDefRecord[K]['schema']>
+      id: EventId
+      parentId: EventId
+    }
+  }[keyof TMutationsDefRecord]
+>
+
+export type MutationEventPartialSchema<TMutationsDefRecord extends MutationDefRecord> = Schema.Schema<
+  {
+    [K in keyof TMutationsDefRecord]: {
+      mutation: K
+      args: Schema.Schema.Type<TMutationsDefRecord[K]['schema']>
+    }
+  }[keyof TMutationsDefRecord],
+  {
+    [K in keyof TMutationsDefRecord]: {
+      mutation: K
+      args: Schema.Schema.Encoded<TMutationsDefRecord[K]['schema']>
+    }
+  }[keyof TMutationsDefRecord]
+>
+
+export const makeMutationEventSchema = <TSchema extends LiveStoreSchema>(
+  schema: TSchema,
+): MutationEventSchema<TSchema['_MutationDefMapType']> =>
+  Schema.Union(
+    ...[...schema.mutations.values()].map((def) =>
+      Schema.Struct({
+        mutation: Schema.Literal(def.name),
+        args: def.schema,
+        id: EventId,
+        parentId: EventId,
+      }),
+    ),
+  ).annotations({ title: 'MutationEventSchema' }) as any
+
+export const makeMutationEventPartialSchema = <TSchema extends LiveStoreSchema>(
+  schema: TSchema,
+): MutationEventPartialSchema<TSchema['_MutationDefMapType']> =>
+  Schema.Union(
+    ...[...schema.mutations.values()].map((def) =>
+      Schema.Struct({
+        mutation: Schema.Literal(def.name),
+        args: def.schema,
+      }),
+    ),
+  ).annotations({ title: 'MutationEventSchemaPartial' }) as any
+
+export const makeMutationEventSchemaMemo = memoizeByRef(makeMutationEventSchema)
+
+export const mutationEventSchemaAny = Schema.Struct({
+  mutation: Schema.String,
+  args: Schema.Any,
+  id: EventId,
+  parentId: EventId,
+}).annotations({ title: 'MutationEventSchema.Any' })
+
+export const mutationEventSchemaDecodedAny = Schema.typeSchema(mutationEventSchemaAny).annotations({
+  title: 'MutationEventSchema.DecodedAny',
+})
+
+export const mutationEventSchemaEncodedAny = Schema.encodedSchema(mutationEventSchemaAny).annotations({
+  title: 'MutationEventSchema.EncodedAny',
+})

package/src/schema/schema-helpers.ts

@@ -0,0 +1,49 @@
+import { SqliteDsl } from '@livestore/db-schema'
+import { shouldNeverHappen } from '@livestore/utils'
+import { pipe, ReadonlyRecord, Schema } from '@livestore/utils/effect'
+
+import type { TableDef } from './table-def.js'
+
+export const getDefaultValuesEncoded = <TTableDef extends TableDef>(
+  tableDef: TTableDef,
+  fallbackValues?: Record<string, any>,
+) =>
+  pipe(
+    tableDef.sqliteDef.columns,
+    ReadonlyRecord.filter((col, key) => {
+      if (fallbackValues?.[key] !== undefined) return true
+      if (key === 'id') return false
+      return col!.default._tag === 'None' || SqliteDsl.isSqlDefaultValue(col!.default.value) === false
+    }),
+    ReadonlyRecord.map((column, columnName) =>
+      fallbackValues?.[columnName] === undefined
+        ? column!.default._tag === 'None'
+          ? column!.nullable === true
+            ? null
+            : shouldNeverHappen(`Column ${columnName} has no default value and is not nullable`)
+          : Schema.encodeSync(column!.schema)(column!.default.value)
+        : fallbackValues[columnName],
+    ),
+  )
+
+export const getDefaultValuesDecoded = <TTableDef extends TableDef>(
+  tableDef: TTableDef,
+  fallbackValues?: Record<string, any>,
+) =>
+  pipe(
+    tableDef.sqliteDef.columns,
+    ReadonlyRecord.filter((col, key) => {
+      if (fallbackValues?.[key] !== undefined) return true
+      if (key === 'id') return false
+      return col!.default._tag === 'None' || SqliteDsl.isSqlDefaultValue(col!.default.value) === false
+    }),
+    ReadonlyRecord.map((column, columnName) =>
+      fallbackValues?.[columnName] === undefined
+        ? column!.default._tag === 'None'
+          ? column!.nullable === true
+            ? null
+            : shouldNeverHappen(`Column ${columnName} has no default value and is not nullable`)
+          : Schema.validateSync(column!.schema)(column!.default.value)
+        : fallbackValues[columnName],
+    ),
+  )

package/src/schema/system-tables.ts

@@ -0,0 +1,84 @@
+import { type SqliteAst as __SqliteAst, SqliteDsl } from '@livestore/db-schema'
+import { Schema } from '@livestore/utils/effect'
+
+import type { FromTable } from './table-def.js'
+import { table } from './table-def.js'
+
+/// App DB
+
+export const SCHEMA_META_TABLE = '__livestore_schema'
+
+export const schemaMetaTable = table(
+  SCHEMA_META_TABLE,
+  {
+    tableName: SqliteDsl.text({ primaryKey: true }),
+    schemaHash: SqliteDsl.integer({ nullable: false }),
+    /** ISO date format */
+    updatedAt: SqliteDsl.text({ nullable: false }),
+  },
+  { disableAutomaticIdColumn: true },
+)
+
+export type SchemaMetaRow = FromTable.RowDecoded<typeof schemaMetaTable>
+
+export const SCHEMA_MUTATIONS_META_TABLE = '__livestore_schema_mutations'
+
+export const schemaMutationsMetaTable = table(
+  SCHEMA_MUTATIONS_META_TABLE,
+  {
+    mutationName: SqliteDsl.text({ primaryKey: true }),
+    schemaHash: SqliteDsl.integer({ nullable: false }),
+    /** ISO date format */
+    updatedAt: SqliteDsl.text({ nullable: false }),
+  },
+  { disableAutomaticIdColumn: true },
+)
+
+export type SchemaMutationsMetaRow = FromTable.RowDecoded<typeof schemaMutationsMetaTable>
+
+/**
+ * Table which stores SQLite changeset blobs which is used for rolling back
+ * read-model state during rebasing.
+ */
+export const SESSION_CHANGESET_META_TABLE = '__livestore_session_changeset'
+
+export const sessionChangesetMetaTable = table(
+  SESSION_CHANGESET_META_TABLE,
+  {
+    idGlobal: SqliteDsl.integer({ primaryKey: true }),
+    idLocal: SqliteDsl.integer({ primaryKey: true }),
+    changeset: SqliteDsl.blob({}),
+  },
+  { disableAutomaticIdColumn: true },
+)
+
+export type SessionChangesetMetaRow = FromTable.RowDecoded<typeof sessionChangesetMetaTable>
+
+export const systemTables = [schemaMetaTable, schemaMutationsMetaTable, sessionChangesetMetaTable]
+
+/// Mutation log DB
+
+export const SyncStatus = Schema.Literal('synced', 'pending', 'error', 'localOnly')
+export type SyncStatus = typeof SyncStatus.Type
+
+export const MUTATION_LOG_META_TABLE = 'mutation_log'
+
+export const mutationLogMetaTable = table(
+  MUTATION_LOG_META_TABLE,
+  {
+    idGlobal: SqliteDsl.integer({ primaryKey: true }),
+    idLocal: SqliteDsl.integer({ primaryKey: true }),
+    parentIdGlobal: SqliteDsl.integer({}),
+    parentIdLocal: SqliteDsl.integer({}),
+    mutation: SqliteDsl.text({}),
+    argsJson: SqliteDsl.text({ schema: Schema.parseJson(Schema.Any) }),
+    schemaHash: SqliteDsl.integer({}),
+    /** ISO date format */
+    createdAt: SqliteDsl.text({}),
+    syncStatus: SqliteDsl.text({ schema: SyncStatus }),
+    syncMetadataJson: SqliteDsl.text({ schema: Schema.parseJson(Schema.Option(Schema.JsonValue)) }),
+  },
+  { disableAutomaticIdColumn: true, indexes: [] },
+)
+
+export type MutationLogMetaRow = FromTable.RowDecoded<typeof mutationLogMetaTable>