@livestore/common 0.3.0-dev.2 → 0.3.0-dev.21

package/src/leader-thread/apply-mutation.ts

@@ -1,22 +1,29 @@
-import { memoizeByRef, shouldNeverHappen } from '@livestore/utils'
+import { LS_DEV, memoizeByRef, shouldNeverHappen } from '@livestore/utils'
 import type { Scope } from '@livestore/utils/effect'
 import { Effect, Option, Schema } from '@livestore/utils/effect'
 
-import type { SqliteError, SynchronousDatabase, UnexpectedError } from '../index.js'
+import type { PreparedBindValues, SqliteDb, SqliteError, UnexpectedError } from '../index.js'
+import { getExecArgsFromMutation } from '../mutation.js'
 import {
-  getExecArgsFromMutation,
+  EventId,
+  getMutationDef,
+  type LiveStoreSchema,
   MUTATION_LOG_META_TABLE,
+  type MutationEvent,
   mutationLogMetaTable,
   SESSION_CHANGESET_META_TABLE,
   sessionChangesetMetaTable,
-} from '../index.js'
-import type { LiveStoreSchema, MutationEvent } from '../schema/mod.js'
+} from '../schema/mod.js'
 import { insertRow } from '../sql-queries/index.js'
 import { execSql, execSqlPrepared } from './connection.js'
 import { LeaderThreadCtx } from './types.js'
 
 export type ApplyMutation = (
   mutationEventEncoded: MutationEvent.AnyEncoded,
+  options?: {
+    /** Needed for rehydrateFromMutationLog */
+    skipMutationLog?: boolean
+  },
 ) => Effect.Effect<void, SqliteError | UnexpectedError>
 
 export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope | LeaderThreadCtx> = Effect.gen(
@@ -28,18 +35,30 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
       // TODO Running `Schema.hash` can be a bottleneck for larger schemas. There is an opportunity to run this
       // at build time and lookup the pre-computed hash at runtime.
       // Also see https://github.com/Effect-TS/effect/issues/2719
-      [...leaderThreadCtx.schema.mutations.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
+      [...leaderThreadCtx.schema.mutations.map.entries()].map(([k, v]) => [k, Schema.hash(v.schema)] as const),
     )
 
-    return (mutationEventEncoded) =>
+    return (mutationEventEncoded, options) =>
       Effect.gen(function* () {
-        const { mutationEventSchema, schema, db, dbLog } = leaderThreadCtx
-        const mutationEventDecoded = Schema.decodeUnknownSync(mutationEventSchema)(mutationEventEncoded)
-
-        const mutationName = mutationEventDecoded.mutation
-        const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
-
-        const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+        const { schema, dbReadModel: db, dbMutationLog } = leaderThreadCtx
+        const skipMutationLog = options?.skipMutationLog ?? false
+
+        const mutationName = mutationEventEncoded.mutation
+        const mutationDef = getMutationDef(schema, mutationName)
+
+        const execArgsArr = getExecArgsFromMutation({
+          mutationDef,
+          mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },
+        })
+
+        // NOTE we might want to bring this back if we want to debug no-op mutations
+        // const makeExecuteOptions = (statementSql: string, bindValues: any) => ({
+        //   onRowsChanged: (rowsChanged: number) => {
+        //     if (rowsChanged === 0) {
+        //       console.warn(`Mutation "${mutationDef.name}" did not affect any rows:`, statementSql, bindValues)
+        //     }
+        //   },
+        // })
 
         // console.group('[@livestore/common:leader-thread:applyMutation]', { mutationName })
 
@@ -53,31 +72,35 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
 
         const changeset = session.changeset()
         session.finish()
-        // NOTE for no-op mutations (e.g. if the state didn't change) the changeset will be empty
-        // TODO possibly write a null value instead of omitting the row
-        if (changeset !== undefined && changeset.length > 0) {
-          // TODO use prepared statements
-          yield* execSql(
-            db,
-            ...insertRow({
-              tableName: SESSION_CHANGESET_META_TABLE,
-              columns: sessionChangesetMetaTable.sqliteDef.columns,
-              values: {
-                idGlobal: mutationEventEncoded.id.global,
-                idLocal: mutationEventEncoded.id.local,
-                changeset,
-                debug: execArgsArr,
-              },
-            }),
-          )
-        }
+
+        // TODO use prepared statements
+        yield* execSql(
+          db,
+          ...insertRow({
+            tableName: SESSION_CHANGESET_META_TABLE,
+            columns: sessionChangesetMetaTable.sqliteDef.columns,
+            values: {
+              idGlobal: mutationEventEncoded.id.global,
+              idClient: mutationEventEncoded.id.client,
+              // NOTE the changeset will be empty (i.e. null) for no-op mutations
+              changeset: changeset ?? null,
+              debug: execArgsArr,
+            },
+          }),
+        )
 
         // console.groupEnd()
 
         // write to mutation_log
-        const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventDecoded)
-        if (excludeFromMutationLog === false) {
-          yield* insertIntoMutationLog(mutationEventEncoded, dbLog, mutationDefSchemaHashMap)
+        const excludeFromMutationLog = shouldExcludeMutationFromLog(mutationName, mutationEventEncoded)
+        if (skipMutationLog === false && excludeFromMutationLog === false) {
+          yield* insertIntoMutationLog(
+            mutationEventEncoded,
+            dbMutationLog,
+            mutationDefSchemaHashMap,
+            mutationEventEncoded.clientId,
+            mutationEventEncoded.sessionId,
+          )
         } else {
           //   console.debug('[@livestore/common:leader-thread] skipping mutation log write', mutation, statementSql, bindValues)
         }
@@ -86,7 +109,7 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
           attributes: {
             mutationName: mutationEventEncoded.mutation,
             mutationId: mutationEventEncoded.id,
-            'span.label': mutationEventEncoded.mutation,
+            'span.label': `(${mutationEventEncoded.id.global},${mutationEventEncoded.id.client}) ${mutationEventEncoded.mutation}`,
           },
         }),
         // Effect.logDuration('@livestore/common:leader-thread:applyMutation'),
@@ -96,27 +119,45 @@ export const makeApplyMutation: Effect.Effect<ApplyMutation, never, Scope.Scope
 
 const insertIntoMutationLog = (
   mutationEventEncoded: MutationEvent.AnyEncoded,
-  dbLog: SynchronousDatabase,
+  dbMutationLog: SqliteDb,
   mutationDefSchemaHashMap: Map<string, number>,
+  clientId: string,
+  sessionId: string | undefined,
 ) =>
   Effect.gen(function* () {
     const mutationName = mutationEventEncoded.mutation
     const mutationDefSchemaHash =
       mutationDefSchemaHashMap.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
 
+    if (LS_DEV && mutationEventEncoded.parentId.global !== EventId.ROOT.global) {
+      const parentMutationExists =
+        dbMutationLog.select<{ count: number }>(
+          `SELECT COUNT(*) as count FROM ${MUTATION_LOG_META_TABLE} WHERE idGlobal = ? AND idClient = ?`,
+          [mutationEventEncoded.parentId.global, mutationEventEncoded.parentId.client] as any as PreparedBindValues,
+        )[0]!.count === 1
+
+      if (parentMutationExists === false) {
+        shouldNeverHappen(
+          `Parent mutation ${mutationEventEncoded.parentId.global},${mutationEventEncoded.parentId.client} does not exist`,
+        )
+      }
+    }
+
     // TODO use prepared statements
     yield* execSql(
-      dbLog,
+      dbMutationLog,
       ...insertRow({
         tableName: MUTATION_LOG_META_TABLE,
         columns: mutationLogMetaTable.sqliteDef.columns,
         values: {
           idGlobal: mutationEventEncoded.id.global,
-          idLocal: mutationEventEncoded.id.local,
+          idClient: mutationEventEncoded.id.client,
           parentIdGlobal: mutationEventEncoded.parentId.global,
-          parentIdLocal: mutationEventEncoded.parentId.local,
+          parentIdClient: mutationEventEncoded.parentId.client,
           mutation: mutationEventEncoded.mutation,
           argsJson: mutationEventEncoded.args ?? {},
+          clientId,
+          sessionId: sessionId ?? null,
           schemaHash: mutationDefSchemaHash,
           syncMetadataJson: Option.none(),
         },
@@ -132,11 +173,14 @@ const makeShouldExcludeMutationFromLog = memoizeByRef((schema: LiveStoreSchema)
       ? (migrationOptions.excludeMutations ?? new Set(['livestore.RawSql']))
       : new Set(['livestore.RawSql'])
 
-  return (mutationName: string, mutationEventDecoded: MutationEvent.Any): boolean => {
+  return (mutationName: string, mutationEventEncoded: MutationEvent.AnyEncoded): boolean => {
     if (mutationLogExclude.has(mutationName)) return true
 
-    const mutationDef = schema.mutations.get(mutationName) ?? shouldNeverHappen(`Unknown mutation: ${mutationName}`)
-    const execArgsArr = getExecArgsFromMutation({ mutationDef, mutationEventDecoded })
+    const mutationDef = getMutationDef(schema, mutationName)
+    const execArgsArr = getExecArgsFromMutation({
+      mutationDef,
+      mutationEvent: { decoded: undefined, encoded: mutationEventEncoded },
+    })
 
     return execArgsArr.some((_) => _.statementSql.includes('__livestore'))
   }

package/src/leader-thread/connection.ts

@@ -1,7 +1,7 @@
 // import type { WaSqlite } from '@livestore/sqlite-wasm'
 import { Effect } from '@livestore/utils/effect'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import { SqliteError } from '../adapter-types.js'
 import type { BindValues } from '../sql-queries/index.js'
 import type { PreparedBindValues } from '../util.js'
@@ -12,21 +12,66 @@ namespace WaSqlite {
   export type SQLiteError = any
 }
 
-export const configureConnection = (syncDb: SynchronousDatabase, { fkEnabled }: { fkEnabled: boolean }) =>
+type ConnectionOptions = {
+  /**
+   * The database connection locking mode.
+   *
+   * @remarks
+   *
+   * This **option is ignored** when used on an **in-memory database** as they can only operate in exclusive locking mode.
+   * In-memory databases can’t share state between connections (unless using a
+   * {@link https://www.sqlite.org/sharedcache.html#shared_cache_and_in_memory_databases|shared cache}),
+   * making concurrent access impossible. This is functionally equivalent to exclusive locking.
+   *
+   * @defaultValue
+   * The default is `"NORMAL"` unless it was unless overridden at compile-time using `SQLITE_DEFAULT_LOCKING_MODE`.
+   *
+   * @see {@link https://www.sqlite.org/pragma.html#pragma_locking_mode|`locking_mode` pragma}
+   */
+  lockingMode?: 'NORMAL' | 'EXCLUSIVE'
+
+  /**
+   * Whether to enforce foreign key constraints.
+   *
+   * @privateRemarks
+   *
+   * We require a value for this option to minimize future problems, as the default value might change in future
+   * versions of SQLite.
+   *
+   * @see {@link https://www.sqlite.org/pragma.html#pragma_foreign_keys|`foreign_keys` pragma}
+   */
+  foreignKeys: boolean
+}
+
+export const configureConnection = (sqliteDb: SqliteDb, { foreignKeys, lockingMode }: ConnectionOptions) =>
   execSql(
-    syncDb,
+    sqliteDb,
+    // We use the WAL journal mode is significantly faster in most scenarios than the traditional rollback journal mode.
+    // It specifically significantly improves write performance. However, when using the WAL journal mode, transactions
+    // that involve changes against multiple ATTACHed databases are atomic for each database but are not atomic
+    // across all databases as a set. Additionally, it is not possible to change the page size after entering WAL mode,
+    // whether on an empty database or by using VACUUM or the backup API. To change the page size, we must switch to the
+    // rollback journal mode.
+    //
+    // When connected to an in-memory database, the WAL journal mode option is ignored because an in-memory database can
+    // only be in either the MEMORY or OFF options. By default, an in-memory database is in the MEMORY option, which
+    // means that it stores the rollback journal in volatile RAM. This saves disk I/O but at the expense of safety and
+    // integrity. If the thread using SQLite crashes in the middle of a transaction, then the database file will very
+    // likely go corrupt.
     sql`
+    -- disable WAL until we have it working properly
+    -- PRAGMA journal_mode=WAL;
     PRAGMA page_size=8192;
-    PRAGMA journal_mode=MEMORY;
-    ${fkEnabled ? sql`PRAGMA foreign_keys='ON';` : sql`PRAGMA foreign_keys='OFF';`}
+    PRAGMA foreign_keys=${foreignKeys ? 'ON' : 'OFF'};
+    ${lockingMode === undefined ? '' : sql`PRAGMA locking_mode=${lockingMode};`}
   `,
     {},
   )
 
-export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValues) => {
+export const execSql = (sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
   const bindValues = prepareBindValues(bind, sql)
   return Effect.try({
-    try: () => syncDb.execute(sql, bindValues),
+    try: () => sqliteDb.execute(sql, bindValues),
     catch: (cause) =>
       new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
   }).pipe(
@@ -48,9 +93,9 @@ export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValu
 // }
 
 // TODO actually use prepared statements
-export const execSqlPrepared = (syncDb: SynchronousDatabase, sql: string, bindValues: PreparedBindValues) => {
+export const execSqlPrepared = (sqliteDb: SqliteDb, sql: string, bindValues: PreparedBindValues) => {
   return Effect.try({
-    try: () => syncDb.execute(sql, bindValues),
+    try: () => sqliteDb.execute(sql, bindValues),
     catch: (cause) =>
       new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
   }).pipe(