@livestore/common 0.0.46-dev.4 → 0.0.47-dev.0

package/src/index.ts

@@ -1,2 +1,6 @@
 export * from './util.js'
 export * from './database.js'
+export * from './migrations.js'
+export * from './mutation.js'
+export * from './init-singleton-tables.js'
+export * from './rehydrate-from-mutationlog.js'

package/src/init-singleton-tables.ts

@@ -0,0 +1,24 @@
+import type { MainDatabase } from './database.js'
+import type { LiveStoreSchema } from './schema/index.js'
+import { DbSchema } from './schema/index.js'
+import { prepareBindValues, sql } from './util.js'
+
+export const initializeSingletonTables = (schema: LiveStoreSchema, db: MainDatabase) => {
+  for (const [, tableDef] of schema.tables) {
+    if (tableDef.options.isSingleton) {
+      const defaultValues = DbSchema.getDefaultValuesEncoded(tableDef, undefined)
+
+      const defaultColumnNames = [...Object.keys(defaultValues), 'id']
+      const columnValues = defaultColumnNames.map((name) => `$${name}`).join(', ')
+
+      const tableName = tableDef.sqliteDef.name
+      const insertQuery = sql`insert into ${tableName} (${defaultColumnNames.join(
+        ', ',
+      )}) select ${columnValues} where not exists(select 1 from ${tableName} where id = 'singleton')`
+
+      const bindValues = prepareBindValues({ ...defaultValues, id: 'singleton' }, insertQuery)
+
+      db.execute(insertQuery, bindValues)
+    }
+  }
+}

package/src/migrations.ts

@@ -0,0 +1,155 @@
+import { memoize } from '@livestore/utils'
+import { Schema as EffectSchema } from '@livestore/utils/effect'
+import type * as otel from '@opentelemetry/api'
+import { SqliteAst, SqliteDsl } from 'effect-db-schema'
+
+import type { MainDatabase } from './database.js'
+import type { LiveStoreSchema } from './schema/index.js'
+import type { SchemaMetaRow } from './schema/system-tables.js'
+import { SCHEMA_META_TABLE, systemTables } from './schema/system-tables.js'
+import type { ParamsObject } from './util.js'
+import { prepareBindValues, sql } from './util.js'
+
+const getMemoizedTimestamp = memoize(() => new Date().toISOString())
+
+// TODO bring back statement caching
+// will require proper scope-aware cleanup etc (for testing and apps with multiple LiveStore instances)
+// const cachedStmts = new Map<string, PreparedStatement>()
+
+const dbExecute = (db: MainDatabase, queryStr: string, bindValues?: ParamsObject) => {
+  // let stmt = cachedStmts.get(queryStr)
+  // if (!stmt) {
+  const stmt = db.prepare(queryStr)
+  // cachedStmts.set(queryStr, stmt)
+  // }
+
+  const preparedBindValues = bindValues ? prepareBindValues(bindValues, queryStr) : undefined
+
+  stmt.execute(preparedBindValues)
+}
+
+const dbSelect = <T>(db: MainDatabase, queryStr: string, bindValues?: ParamsObject) => {
+  // let stmt = cachedStmts.get(queryStr)
+  // if (!stmt) {
+  const stmt = db.prepare(queryStr)
+  // cachedStmts.set(queryStr, stmt)
+  // }
+
+  return stmt.select<T>(bindValues ? prepareBindValues(bindValues, queryStr) : undefined)
+}
+
+// TODO more graceful DB migration (e.g. backup DB before destructive migrations)
+export const migrateDb = ({
+  db,
+  otelContext,
+  schema,
+}: {
+  db: MainDatabase
+  otelContext: otel.Context
+  schema: LiveStoreSchema
+}) => {
+  dbExecute(
+    db,
+    // TODO use schema migration definition from schema.ts instead
+    sql`create table if not exists ${SCHEMA_META_TABLE} (tableName text primary key, schemaHash text, updatedAt text);`,
+  )
+
+  const schemaMetaRows = dbSelect<SchemaMetaRow>(db, sql`SELECT * FROM ${SCHEMA_META_TABLE}`)
+
+  const dbSchemaHashByTable = Object.fromEntries(
+    schemaMetaRows.map(({ tableName, schemaHash }) => [tableName, schemaHash]),
+  )
+
+  const tableDefs = new Set([
+    // NOTE it's important the `SCHEMA_META_TABLE` comes first since we're writing to it below
+    ...systemTables,
+    ...Array.from(schema.tables.values()).filter((_) => _.sqliteDef.name !== SCHEMA_META_TABLE),
+  ])
+
+  for (const tableDef of tableDefs) {
+    const tableAst = tableDef.sqliteDef.ast
+    const tableName = tableAst.name
+    const dbSchemaHash = dbSchemaHashByTable[tableName]
+    const schemaHash = SqliteAst.hash(tableAst)
+
+    // @ts-expect-error TODO fix typing
+    const skipMigrations = import.meta.env.VITE_LIVESTORE_SKIP_MIGRATIONS !== undefined
+
+    if (schemaHash !== dbSchemaHash && skipMigrations === false) {
+      console.log(
+        `Schema hash mismatch for table '${tableName}' (DB: ${dbSchemaHash}, expected: ${schemaHash}), migrating table...`,
+      )
+
+      migrateTable({ db, tableAst, otelContext, schemaHash })
+    }
+  }
+}
+
+export const migrateTable = ({
+  db,
+  tableAst,
+  // otelContext,
+  schemaHash,
+}: {
+  db: MainDatabase
+  tableAst: SqliteAst.Table
+  otelContext: otel.Context
+  schemaHash: number
+}) => {
+  console.log(`Migrating table '${tableAst.name}'...`)
+  const tableName = tableAst.name
+  const columnSpec = makeColumnSpec(tableAst)
+
+  // TODO need to possibly handle cascading deletes due to foreign keys
+  dbExecute(db, sql`drop table if exists ${tableName}`)
+  dbExecute(db, sql`create table if not exists ${tableName} (${columnSpec})`)
+
+  for (const index of tableAst.indexes) {
+    dbExecute(db, createIndexFromDefinition(tableName, index))
+  }
+
+  const updatedAt = getMemoizedTimestamp()
+
+  dbExecute(
+    db,
+    sql`
+      INSERT INTO ${SCHEMA_META_TABLE} (tableName, schemaHash, updatedAt) VALUES ($tableName, $schemaHash, $updatedAt)
+        ON CONFLICT (tableName) DO UPDATE SET schemaHash = $schemaHash, updatedAt = $updatedAt;
+    `,
+    { tableName, schemaHash, updatedAt },
+  )
+}
+
+const createIndexFromDefinition = (tableName: string, index: SqliteAst.Index) => {
+  const uniqueStr = index.unique ? 'UNIQUE' : ''
+  return sql`create ${uniqueStr} index ${index.name} on ${tableName} (${index.columns.join(', ')})`
+}
+
+const makeColumnSpec = (tableAst: SqliteAst.Table) => {
+  const primaryKeys = tableAst.columns.filter((_) => _.primaryKey).map((_) => _.name)
+  const columnDefStrs = tableAst.columns.map(toSqliteColumnSpec)
+  if (primaryKeys.length > 0) {
+    columnDefStrs.push(`PRIMARY KEY (${primaryKeys.join(', ')})`)
+  }
+
+  return columnDefStrs.join(', ')
+}
+
+/** NOTE primary keys are applied on a table level not on a column level to account for multi-column primary keys */
+const toSqliteColumnSpec = (column: SqliteAst.Column) => {
+  const columnTypeStr = column.type._tag
+  const nullableStr = column.nullable === false ? 'not null' : ''
+  const defaultValueStr = (() => {
+    if (column.default._tag === 'None') return ''
+
+    if (SqliteDsl.isSqlDefaultValue(column.default.value)) return `default ${column.default.value.sql}`
+
+    const encodeValue = EffectSchema.encodeSync(column.schema)
+    const encodedDefaultValue = encodeValue(column.default.value)
+
+    if (columnTypeStr === 'text') return `default '${encodedDefaultValue}'`
+    return `default ${encodedDefaultValue}`
+  })()
+
+  return `${column.name} ${columnTypeStr} ${nullableStr} ${defaultValueStr}`
+}

package/src/mutation.ts

@@ -0,0 +1,50 @@
+import { Schema } from '@livestore/utils/effect'
+
+import type { MutationDef, MutationEvent } from './schema/mutations.js'
+import type { PreparedBindValues } from './util.js'
+import { prepareBindValues } from './util.js'
+
+export const getExecArgsFromMutation = ({
+  mutationDef,
+  mutationEventDecoded,
+}: {
+  mutationDef: MutationDef.Any
+  mutationEventDecoded: MutationEvent.Any
+}): ReadonlyArray<{
+  statementSql: string
+  bindValues: PreparedBindValues
+  writeTables: ReadonlySet<string> | undefined
+}> => {
+  let statementRes: ReadonlyArray<
+    string | { sql: string; bindValues: Record<string, unknown>; writeTables?: ReadonlySet<string> }
+  >
+
+  switch (typeof mutationDef.sql) {
+    case 'function': {
+      const res = mutationDef.sql(mutationEventDecoded.args)
+      statementRes = Array.isArray(res) ? res : [res]
+      break
+    }
+    case 'string': {
+      statementRes = [mutationDef.sql]
+      break
+    }
+    default: {
+      statementRes = mutationDef.sql
+      break
+    }
+  }
+
+  return statementRes.map((statementRes) => {
+    const statementSql = typeof statementRes === 'string' ? statementRes : statementRes.sql
+
+    const bindValues =
+      typeof statementRes === 'string'
+        ? Schema.encodeUnknownSync(mutationDef.schema)(mutationEventDecoded.args)
+        : statementRes.bindValues
+
+    const writeTables = typeof statementRes === 'string' ? undefined : statementRes.writeTables
+
+    return { statementSql, bindValues: prepareBindValues(bindValues ?? {}, statementSql), writeTables }
+  })
+}

package/src/rehydrate-from-mutationlog.ts

@@ -0,0 +1,77 @@
+import { shouldNeverHappen } from '@livestore/utils'
+import { Schema } from '@livestore/utils/effect'
+
+import type { MainDatabase } from './database.js'
+import { getExecArgsFromMutation } from './mutation.js'
+import type { LiveStoreSchema } from './schema/index.js'
+
+type MutationLogRow = {
+  id: string
+  mutation: string
+  args_json: string
+  schema_hash: number
+  created_at: string
+}
+
+export const rehydrateFromMutationLog = ({
+  logDb,
+  db,
+  schema,
+}: {
+  logDb: MainDatabase
+  db: MainDatabase
+  schema: LiveStoreSchema
+}) => {
+  try {
+    const stmt = logDb.prepare('SELECT * FROM mutation_log ORDER BY id ASC')
+    const results = stmt.select<MutationLogRow>(undefined)
+
+    performance.mark('livestore:hydrate-from-mutationlog:start')
+
+    for (const row of results) {
+      const mutationDef = schema.mutations.get(row.mutation) ?? shouldNeverHappen(`Unknown mutation ${row.mutation}`)
+
+      if (Schema.hash(mutationDef.schema) !== row.schema_hash) {
+        throw new Error(`Schema hash mismatch for mutation ${row.mutation}`)
+      }
+
+      const argsDecoded = Schema.decodeUnknownSync(Schema.parseJson(mutationDef.schema))(row.args_json)
+      const mutationEventDecoded = {
+        id: row.id,
+        mutation: row.mutation,
+        args: argsDecoded,
+      }
+      // const argsEncoded = JSON.parse(row.args_json)
+      // const mutationSqlRes =
+      //   typeof mutation.sql === 'string'
+      //     ? mutation.sql
+      //     : mutation.sql(Schema.decodeUnknownSync(mutation.schema)(argsEncoded))
+      // const mutationSql = typeof mutationSqlRes === 'string' ? mutationSqlRes : mutationSqlRes.sql
+      // const bindValues = typeof mutationSqlRes === 'string' ? argsEncoded : mutationSqlRes.bindValues
+
+      const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+
+      for (const { statementSql, bindValues } of execArgsArr) {
+        try {
+          db.execute(statementSql, bindValues)
+          // console.log(`Re-executed mutation ${mutationSql}`, bindValues)
+        } catch (e) {
+          console.error(`Error executing migration for mutation ${statementSql}`, bindValues, e)
+          debugger
+          throw e
+        }
+      }
+    }
+  } catch (e) {
+    console.error('Error while rehydrating database from mutation log', e)
+    debugger
+    throw e
+  } finally {
+    performance.mark('livestore:hydrate-from-mutationlog:end')
+    performance.measure(
+      'livestore:hydrate-from-mutationlog',
+      'livestore:hydrate-from-mutationlog:start',
+      'livestore:hydrate-from-mutationlog:end',
+    )
+  }
+}

package/src/schema/index.ts

@@ -1,6 +1,6 @@
 import { isReadonlyArray } from '@livestore/utils'
 import type { ReadonlyArray } from '@livestore/utils/effect'
-import type { SqliteDsl } from 'effect-db-schema'
+import { SqliteAst, type SqliteDsl } from 'effect-db-schema'
 
 import {
   type MutationDef,
@@ -80,6 +80,12 @@ export const makeSchema = <TInputSchema extends InputSchema>(
   } satisfies LiveStoreSchema
 }
 
+export const makeSchemaHash = (schema: LiveStoreSchema) =>
+  SqliteAst.hash({
+    _tag: 'dbSchema',
+    tables: [...schema.tables.values()].map((_) => _.sqliteDef.ast),
+  })
+
 /**
  * In case of ...
  * - array: we use the table name of each array item (= table definition) as the object key

package/src/schema/mutations.ts

@@ -17,19 +17,24 @@ export type InternalMutationSchema<TRecord extends MutationDefRecord = MutationD
   schemaHashMap: Map<keyof TRecord, number>
 }
 
+export type MutationDefSqlResult<TTo> =
+  | SingleOrReadonlyArray<string>
+  | ((args: TTo) => SingleOrReadonlyArray<
+      | string
+      | {
+          sql: string
+          /** Note args need to be manually encoded to `BindValues` when returning this argument */
+          bindValues: BindValues
+          writeTables?: ReadonlySet<string>
+        }
+    >)
+
+export type SingleOrReadonlyArray<T> = T | ReadonlyArray<T>
+
 export type MutationDef<TName extends string, TFrom, TTo> = {
   name: TName
   schema: Schema.Schema<TTo, TFrom>
-  sql:
-    | string
-    | ((args: TTo) =>
-        | string
-        | {
-            sql: string
-            /** Note args need to be manually encoded to `BindValues` when returning this argument */
-            bindValues: BindValues
-            writeTables?: ReadonlySet<string>
-          })
+  sql: MutationDefSqlResult<TTo>
 
   /** Helper function to construct mutation event */
   (args: TTo): { mutation: TName; args: TTo; id: string }
@@ -43,7 +48,7 @@ export namespace MutationDef {
 export const defineMutation = <TName extends string, TFrom, TTo>(
   name: TName,
   schema: Schema.Schema<TTo, TFrom>,
-  sql: string | ((args: TTo) => string | { sql: string; bindValues: BindValues; writeTables?: ReadonlySet<string> }),
+  sql: MutationDefSqlResult<TTo>,
 ): MutationDef<TName, TFrom, TTo> => {
   const makeEvent = (args: TTo) => ({ mutation: name, args, id: cuid() })
 

package/src/schema/table-def.ts

@@ -198,15 +198,17 @@ export const getDefaultValuesDecoded = <TTableDef extends TableDef>(
   )
 
 type WithId<TColumns extends SqliteDsl.Columns, TOptions extends TableOptions> = TColumns &
-  (TOptions['disableAutomaticIdColumn'] extends true
+  ('id' extends keyof TColumns
     ? {}
-    : TOptions['isSingleton'] extends true
-      ? {
-          id: SqliteDsl.ColumnDefinition<'singleton', 'singleton'>
-        }
-      : {
-          id: SqliteDsl.ColumnDefinition<string, string>
-        })
+    : TOptions['disableAutomaticIdColumn'] extends true
+      ? {}
+      : TOptions['isSingleton'] extends true
+        ? {
+            id: SqliteDsl.ColumnDefinition<'singleton', 'singleton'>
+          }
+        : {
+            id: SqliteDsl.ColumnDefinition<string, string>
+          })
 
 type WithDefaults<TOptionsInput extends TableOptionsInput> = {
   isSingleton: TOptionsInput['isSingleton'] extends true ? true : false