@livestore/common 0.3.0-dev.5 → 0.3.0-dev.51

package/src/leader-thread/connection.ts

@@ -1,7 +1,7 @@
 // import type { WaSqlite } from '@livestore/sqlite-wasm'
 import { Effect } from '@livestore/utils/effect'
 
-import type { SynchronousDatabase } from '../adapter-types.js'
+import type { SqliteDb } from '../adapter-types.js'
 import { SqliteError } from '../adapter-types.js'
 import type { BindValues } from '../sql-queries/index.js'
 import type { PreparedBindValues } from '../util.js'
@@ -12,21 +12,66 @@ namespace WaSqlite {
   export type SQLiteError = any
 }
 
-export const configureConnection = (syncDb: SynchronousDatabase, { fkEnabled }: { fkEnabled: boolean }) =>
+type ConnectionOptions = {
+  /**
+   * The database connection locking mode.
+   *
+   * @remarks
+   *
+   * This **option is ignored** when used on an **in-memory database** as they can only operate in exclusive locking mode.
+   * In-memory databases can’t share state between connections (unless using a
+   * {@link https://www.sqlite.org/sharedcache.html#shared_cache_and_in_memory_databases|shared cache}),
+   * making concurrent access impossible. This is functionally equivalent to exclusive locking.
+   *
+   * @defaultValue
+   * The default is `"NORMAL"` unless it was unless overridden at compile-time using `SQLITE_DEFAULT_LOCKING_MODE`.
+   *
+   * @see {@link https://www.sqlite.org/pragma.html#pragma_locking_mode|`locking_mode` pragma}
+   */
+  lockingMode?: 'NORMAL' | 'EXCLUSIVE'
+
+  /**
+   * Whether to enforce foreign key constraints.
+   *
+   * @privateRemarks
+   *
+   * We require a value for this option to minimize future problems, as the default value might change in future
+   * versions of SQLite.
+   *
+   * @see {@link https://www.sqlite.org/pragma.html#pragma_foreign_keys|`foreign_keys` pragma}
+   */
+  foreignKeys: boolean
+}
+
+export const configureConnection = (sqliteDb: SqliteDb, { foreignKeys, lockingMode }: ConnectionOptions) =>
   execSql(
-    syncDb,
+    sqliteDb,
+    // We use the WAL journal mode is significantly faster in most scenarios than the traditional rollback journal mode.
+    // It specifically significantly improves write performance. However, when using the WAL journal mode, transactions
+    // that involve changes against multiple ATTACHed databases are atomic for each database but are not atomic
+    // across all databases as a set. Additionally, it is not possible to change the page size after entering WAL mode,
+    // whether on an empty database or by using VACUUM or the backup API. To change the page size, we must switch to the
+    // rollback journal mode.
+    //
+    // When connected to an in-memory database, the WAL journal mode option is ignored because an in-memory database can
+    // only be in either the MEMORY or OFF options. By default, an in-memory database is in the MEMORY option, which
+    // means that it stores the rollback journal in volatile RAM. This saves disk I/O but at the expense of safety and
+    // integrity. If the thread using SQLite crashes in the middle of a transaction, then the database file will very
+    // likely go corrupt.
     sql`
+    -- disable WAL until we have it working properly
+    -- PRAGMA journal_mode=WAL;
     PRAGMA page_size=8192;
-    PRAGMA journal_mode=MEMORY;
-    ${fkEnabled ? sql`PRAGMA foreign_keys='ON';` : sql`PRAGMA foreign_keys='OFF';`}
+    PRAGMA foreign_keys=${foreignKeys ? 'ON' : 'OFF'};
+    ${lockingMode === undefined ? '' : sql`PRAGMA locking_mode=${lockingMode};`}
   `,
     {},
   )
 
-export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValues) => {
+export const execSql = (sqliteDb: SqliteDb, sql: string, bind: BindValues) => {
   const bindValues = prepareBindValues(bind, sql)
   return Effect.try({
-    try: () => syncDb.execute(sql, bindValues),
+    try: () => sqliteDb.execute(sql, bindValues),
     catch: (cause) =>
       new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
   }).pipe(
@@ -48,9 +93,9 @@ export const execSql = (syncDb: SynchronousDatabase, sql: string, bind: BindValu
 // }
 
 // TODO actually use prepared statements
-export const execSqlPrepared = (syncDb: SynchronousDatabase, sql: string, bindValues: PreparedBindValues) => {
+export const execSqlPrepared = (sqliteDb: SqliteDb, sql: string, bindValues: PreparedBindValues) => {
   return Effect.try({
-    try: () => syncDb.execute(sql, bindValues),
+    try: () => sqliteDb.execute(sql, bindValues),
     catch: (cause) =>
       new SqliteError({ cause, query: { bindValues, sql }, code: (cause as WaSqlite.SQLiteError).code }),
   }).pipe(

package/src/leader-thread/eventlog.ts

@@ -0,0 +1,199 @@
+import { LS_DEV, shouldNeverHappen } from '@livestore/utils'
+import { Effect, Option, Schema } from '@livestore/utils/effect'
+
+import type { SqliteDb } from '../adapter-types.js'
+import * as EventSequenceNumber from '../schema/EventSequenceNumber.js'
+import * as LiveStoreEvent from '../schema/LiveStoreEvent.js'
+import {
+  EVENTLOG_META_TABLE,
+  eventlogMetaTable,
+  eventlogSystemTables,
+  sessionChangesetMetaTable,
+  SYNC_STATUS_TABLE,
+} from '../schema/state/sqlite/system-tables.js'
+import { migrateTable } from '../schema-management/migrations.js'
+import { insertRow, updateRows } from '../sql-queries/sql-queries.js'
+import type { PreparedBindValues } from '../util.js'
+import { prepareBindValues, sql } from '../util.js'
+import { execSql } from './connection.js'
+import type { InitialSyncInfo } from './types.js'
+import { LeaderThreadCtx } from './types.js'
+
+export const initEventlogDb = (dbEventlog: SqliteDb) =>
+  Effect.gen(function* () {
+    for (const tableDef of eventlogSystemTables) {
+      yield* migrateTable({
+        db: dbEventlog,
+        behaviour: 'create-if-not-exists',
+        tableAst: tableDef.sqliteDef.ast,
+        skipMetaTable: true,
+      })
+    }
+
+    // Create sync status row if it doesn't exist
+    yield* execSql(
+      dbEventlog,
+      sql`INSERT INTO ${SYNC_STATUS_TABLE} (head)
+          SELECT ${EventSequenceNumber.ROOT.global}
+          WHERE NOT EXISTS (SELECT 1 FROM ${SYNC_STATUS_TABLE})`,
+      {},
+    )
+  })
+
+/** Exclusive of the "since event" */
+export const getEventsSince = (
+  since: EventSequenceNumber.EventSequenceNumber,
+): Effect.Effect<ReadonlyArray<LiveStoreEvent.EncodedWithMeta>, never, LeaderThreadCtx> =>
+  Effect.gen(function* () {
+    const { dbEventlog, dbState } = yield* LeaderThreadCtx
+
+    const query = eventlogMetaTable.where('seqNumGlobal', '>=', since.global).asSql()
+    const pendingEventsRaw = dbEventlog.select(query.query, prepareBindValues(query.bindValues, query.query))
+    const pendingEvents = Schema.decodeUnknownSync(eventlogMetaTable.rowSchema.pipe(Schema.Array))(pendingEventsRaw)
+
+    const sessionChangesetRows = sessionChangesetMetaTable.where('seqNumGlobal', '>=', since.global).asSql()
+    const sessionChangesetRowsRaw = dbState.select(
+      sessionChangesetRows.query,
+      prepareBindValues(sessionChangesetRows.bindValues, sessionChangesetRows.query),
+    )
+    const sessionChangesetRowsDecoded = Schema.decodeUnknownSync(
+      sessionChangesetMetaTable.rowSchema.pipe(Schema.Array),
+    )(sessionChangesetRowsRaw)
+
+    return pendingEvents
+      .map((eventlogEvent) => {
+        const sessionChangeset = sessionChangesetRowsDecoded.find(
+          (readModelEvent) =>
+            readModelEvent.seqNumGlobal === eventlogEvent.seqNumGlobal &&
+            readModelEvent.seqNumClient === eventlogEvent.seqNumClient,
+        )
+        return LiveStoreEvent.EncodedWithMeta.make({
+          name: eventlogEvent.name,
+          args: eventlogEvent.argsJson,
+          seqNum: { global: eventlogEvent.seqNumGlobal, client: eventlogEvent.seqNumClient },
+          parentSeqNum: { global: eventlogEvent.parentSeqNumGlobal, client: eventlogEvent.parentSeqNumClient },
+          clientId: eventlogEvent.clientId,
+          sessionId: eventlogEvent.sessionId,
+          meta: {
+            sessionChangeset:
+              sessionChangeset && sessionChangeset.changeset !== null
+                ? {
+                    _tag: 'sessionChangeset' as const,
+                    data: sessionChangeset.changeset,
+                    debug: sessionChangeset.debug,
+                  }
+                : { _tag: 'unset' as const },
+            syncMetadata: eventlogEvent.syncMetadataJson,
+          },
+        })
+      })
+      .filter((_) => EventSequenceNumber.compare(_.seqNum, since) > 0)
+      .sort((a, b) => EventSequenceNumber.compare(a.seqNum, b.seqNum))
+  })
+
+export const getClientHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.EventSequenceNumber => {
+  const res = dbEventlog.select<{
+    seqNumGlobal: EventSequenceNumber.GlobalEventSequenceNumber
+    seqNumClient: EventSequenceNumber.ClientEventSequenceNumber
+  }>(
+    sql`select seqNumGlobal, seqNumClient from ${EVENTLOG_META_TABLE} order by seqNumGlobal DESC, seqNumClient DESC limit 1`,
+  )[0]
+
+  return res ? { global: res.seqNumGlobal, client: res.seqNumClient } : EventSequenceNumber.ROOT
+}
+
+export const getBackendHeadFromDb = (dbEventlog: SqliteDb): EventSequenceNumber.GlobalEventSequenceNumber =>
+  dbEventlog.select<{ head: EventSequenceNumber.GlobalEventSequenceNumber }>(
+    sql`select head from ${SYNC_STATUS_TABLE}`,
+  )[0]?.head ?? EventSequenceNumber.ROOT.global
+
+// TODO use prepared statements
+export const updateBackendHead = (dbEventlog: SqliteDb, head: EventSequenceNumber.EventSequenceNumber) =>
+  dbEventlog.execute(sql`UPDATE ${SYNC_STATUS_TABLE} SET head = ${head.global}`)
+
+export const insertIntoEventlog = (
+  eventEncoded: LiveStoreEvent.EncodedWithMeta,
+  dbEventlog: SqliteDb,
+  eventDefSchemaHash: number,
+  clientId: string,
+  sessionId: string,
+) =>
+  Effect.gen(function* () {
+    // Check history consistency during LS_DEV
+    if (LS_DEV && eventEncoded.parentSeqNum.global !== EventSequenceNumber.ROOT.global) {
+      const parentEventExists =
+        dbEventlog.select<{ count: number }>(
+          `SELECT COUNT(*) as count FROM ${EVENTLOG_META_TABLE} WHERE seqNumGlobal = ? AND seqNumClient = ?`,
+          [eventEncoded.parentSeqNum.global, eventEncoded.parentSeqNum.client] as any as PreparedBindValues,
+        )[0]!.count === 1
+
+      if (parentEventExists === false) {
+        shouldNeverHappen(
+          `Parent mutation ${eventEncoded.parentSeqNum.global},${eventEncoded.parentSeqNum.client} does not exist`,
+        )
+      }
+    }
+
+    // TODO use prepared statements
+    yield* execSql(
+      dbEventlog,
+      ...insertRow({
+        tableName: EVENTLOG_META_TABLE,
+        columns: eventlogMetaTable.sqliteDef.columns,
+        values: {
+          seqNumGlobal: eventEncoded.seqNum.global,
+          seqNumClient: eventEncoded.seqNum.client,
+          parentSeqNumGlobal: eventEncoded.parentSeqNum.global,
+          parentSeqNumClient: eventEncoded.parentSeqNum.client,
+          name: eventEncoded.name,
+          argsJson: eventEncoded.args ?? {},
+          clientId,
+          sessionId,
+          schemaHash: eventDefSchemaHash,
+          syncMetadataJson: eventEncoded.meta.syncMetadata,
+        },
+      }),
+    )
+  })
+
+export const updateSyncMetadata = (items: ReadonlyArray<LiveStoreEvent.EncodedWithMeta>) =>
+  Effect.gen(function* () {
+    const { dbEventlog } = yield* LeaderThreadCtx
+
+    // TODO try to do this in a single query
+    for (let i = 0; i < items.length; i++) {
+      const event = items[i]!
+
+      yield* execSql(
+        dbEventlog,
+        ...updateRows({
+          tableName: EVENTLOG_META_TABLE,
+          columns: eventlogMetaTable.sqliteDef.columns,
+          where: { seqNumGlobal: event.seqNum.global, seqNumClient: event.seqNum.client },
+          updateValues: { syncMetadataJson: event.meta.syncMetadata },
+        }),
+      )
+    }
+  })
+
+export const getSyncBackendCursorInfo = (remoteHead: EventSequenceNumber.GlobalEventSequenceNumber) =>
+  Effect.gen(function* () {
+    const { dbEventlog } = yield* LeaderThreadCtx
+
+    if (remoteHead === EventSequenceNumber.ROOT.global) return Option.none()
+
+    const EventlogQuerySchema = Schema.Struct({
+      syncMetadataJson: Schema.parseJson(Schema.Option(Schema.JsonValue)),
+    }).pipe(Schema.pluck('syncMetadataJson'), Schema.Array, Schema.head)
+
+    const syncMetadataOption = yield* Effect.sync(() =>
+      dbEventlog.select<{ syncMetadataJson: string }>(
+        sql`SELECT syncMetadataJson FROM ${EVENTLOG_META_TABLE} WHERE seqNumGlobal = ${remoteHead} ORDER BY seqNumClient ASC LIMIT 1`,
+      ),
+    ).pipe(Effect.andThen(Schema.decode(EventlogQuerySchema)), Effect.map(Option.flatten), Effect.orDie)
+
+    return Option.some({
+      cursor: { global: remoteHead, client: EventSequenceNumber.clientDefault },
+      metadata: syncMetadataOption,
+    }) satisfies InitialSyncInfo
+  }).pipe(Effect.withSpan('@livestore/common:eventlog:getSyncBackendCursorInfo', { attributes: { remoteHead } }))