@livestore/common 0.3.0-dev.3 → 0.3.0-dev.31

package/src/query-builder/impl.ts

@@ -1,27 +1,29 @@
-import { Option, Predicate, Schema } from '@livestore/utils/effect'
+import { casesHandled, shouldNeverHappen } from '@livestore/utils'
+import { Match, Option, Predicate, Schema } from '@livestore/utils/effect'
 
-import type { QueryInfo } from '../query-info.js'
-import type { DbSchema } from '../schema/mod.js'
+import type { State } from '../schema/mod.js'
 import type { QueryBuilder, QueryBuilderAst } from './api.js'
-import { QueryBuilderAstSymbol, TypeId } from './api.js'
+import { QueryBuilderAstSymbol, QueryBuilderTypeId } from './api.js'
+import { astToSql } from './astToSql.js'
 
-export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBase>(
+export const makeQueryBuilder = <TResult, TTableDef extends State.SQLite.TableDefBase>(
   tableDef: TTableDef,
   ast: QueryBuilderAst = emptyAst(tableDef),
-): QueryBuilder<TResult, TTableDef, never, QueryInfo.None> => {
+): QueryBuilder<TResult, TTableDef, never> => {
   const api = {
     // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
     select() {
-      assertQueryBuilderAst(ast)
+      assertSelectQueryBuilderAst(ast)
 
       // eslint-disable-next-line prefer-rest-params
       const params = [...arguments]
 
-      if (params.length === 2 && typeof params[0] === 'string' && typeof params[1] === 'object') {
-        const [col, options] = params as any as [string, { pluck: boolean }]
+      // Pluck if there's only one column selected
+      if (params.length === 1) {
+        const [col] = params as any as [string]
         return makeQueryBuilder(tableDef, {
           ...ast,
-          resultSchemaSingle: options.pluck ? ast.resultSchemaSingle.pipe(Schema.pluck(col)) : ast.resultSchemaSingle,
+          resultSchemaSingle: ast.resultSchemaSingle.pipe(Schema.pluck(col)),
           select: { columns: [col] },
         })
       }
@@ -36,8 +38,9 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
       }) as any
     },
     // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
-    where() {
-      if (isRowQuery(ast)) return invalidQueryBuilder()
+    where: function () {
+      if (ast._tag === 'InsertQuery') return invalidQueryBuilder('Cannot use where with insert')
+      if (ast._tag === 'RowQuery') return invalidQueryBuilder('Cannot use where with row')
 
       if (arguments.length === 1) {
         // eslint-disable-next-line prefer-rest-params
@@ -50,24 +53,45 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
               : { col, op: '=', value },
           )
 
-        return makeQueryBuilder(tableDef, {
-          ...ast,
-          where: [...ast.where, ...newOps],
-        }) as any
+        switch (ast._tag) {
+          case 'CountQuery':
+          case 'SelectQuery':
+          case 'UpdateQuery':
+          case 'DeleteQuery': {
+            return makeQueryBuilder(tableDef, {
+              ...ast,
+              where: [...ast.where, ...newOps],
+            }) as any
+          }
+          default: {
+            return casesHandled(ast)
+          }
+        }
       }
 
       // eslint-disable-next-line prefer-rest-params
       const [col, opOrValue, valueOrUndefined] = arguments
       const op = valueOrUndefined === undefined ? '=' : opOrValue
       const value = valueOrUndefined === undefined ? opOrValue : valueOrUndefined
-      return makeQueryBuilder(tableDef, {
-        ...ast,
-        where: [...ast.where, { col, op, value }],
-      })
+
+      switch (ast._tag) {
+        case 'CountQuery':
+        case 'SelectQuery':
+        case 'UpdateQuery':
+        case 'DeleteQuery': {
+          return makeQueryBuilder(tableDef, {
+            ...ast,
+            where: [...ast.where, { col, op, value }],
+          }) as any
+        }
+        default: {
+          return casesHandled(ast)
+        }
+      }
     },
     // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
     orderBy() {
-      assertQueryBuilderAst(ast)
+      assertSelectQueryBuilderAst(ast)
 
       if (arguments.length === 0 || arguments.length > 2) return invalidQueryBuilder()
 
@@ -89,12 +113,12 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
       }) as any
     },
     limit: (limit) => {
-      assertQueryBuilderAst(ast)
+      assertSelectQueryBuilderAst(ast)
 
       return makeQueryBuilder(tableDef, { ...ast, limit: Option.some(limit) })
     },
     offset: (offset) => {
-      assertQueryBuilderAst(ast)
+      assertSelectQueryBuilderAst(ast)
 
       return makeQueryBuilder(tableDef, { ...ast, offset: Option.some(offset) })
     },
@@ -102,58 +126,129 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
       if (isRowQuery(ast)) return invalidQueryBuilder()
 
       return makeQueryBuilder(tableDef, {
-        ...ast,
+        _tag: 'CountQuery',
+        tableDef,
+        where: [],
         resultSchema: Schema.Struct({ count: Schema.Number }).pipe(
           Schema.pluck('count'),
           Schema.Array,
           Schema.headOrElse(),
         ),
-        _tag: 'CountQuery',
       })
     },
     first: (options) => {
-      assertQueryBuilderAst(ast)
+      assertSelectQueryBuilderAst(ast)
 
       if (ast.limit._tag === 'Some') return invalidQueryBuilder(`.first() can't be called after .limit()`)
 
       return makeQueryBuilder(tableDef, {
         ...ast,
         limit: Option.some(1),
-        // TODO improve
-        pickFirst: options?.fallback ? { fallback: options.fallback } : { fallback: () => undefined },
+        pickFirst: options?.fallback ? { fallback: options.fallback } : 'no-fallback',
       })
     },
-    // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
-    row() {
-      // eslint-disable-next-line prefer-rest-params
-      const params = [...arguments]
+    // // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
+    // getOrCreate() {
+    //   if (tableDef.options.isClientDocumentTable === false) {
+    //     return invalidQueryBuilder(`getOrCreate() is not allowed when table is not a client document table`)
+    //   }
+
+    //   // eslint-disable-next-line prefer-rest-params
+    //   const params = [...arguments]
+
+    //   let id: string | number
+
+    //   // TODO refactor to handle default id
+    //   id = params[0] as string | number
+    //   if (id === undefined) {
+    //     invalidQueryBuilder(`Id missing for row query on non-singleton table ${tableDef.sqliteDef.name}`)
+    //   }
+
+    //   // TODO validate all required columns are present and values are matching the schema
+    //   const insertValues: Record<string, unknown> = params[1]?.insertValues ?? {}
+
+    //   return makeQueryBuilder(tableDef, {
+    //     _tag: 'RowQuery',
+    //     id,
+    //     tableDef,
+    //     insertValues,
+    //   }) as any
+    // },
+    insert: (values) => {
+      const filteredValues = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== undefined))
 
-      let id: string
+      return makeQueryBuilder(tableDef, {
+        _tag: 'InsertQuery',
+        tableDef,
+        values: filteredValues,
+        onConflict: undefined,
+        returning: undefined,
+        resultSchema: Schema.Void,
+      }) as any
+    },
+    onConflict: (
+      targetOrTargets: string | string[],
+      action: 'ignore' | 'replace' | 'update',
+      updateValues?: Record<string, unknown>,
+    ) => {
+      const targets = Array.isArray(targetOrTargets) ? targetOrTargets : [targetOrTargets]
+
+      assertInsertQueryBuilderAst(ast)
+
+      const onConflict = Match.value(action).pipe(
+        Match.when('ignore', () => ({ targets, action: { _tag: 'ignore' } }) satisfies QueryBuilderAst.OnConflict),
+        Match.when('replace', () => ({ targets, action: { _tag: 'replace' } }) satisfies QueryBuilderAst.OnConflict),
+        Match.when(
+          'update',
+          () => ({ targets, action: { _tag: 'update', update: updateValues! } }) satisfies QueryBuilderAst.OnConflict,
+        ),
+        Match.exhaustive,
+      )
 
-      if (tableDef.options.isSingleton) {
-        id = tableDef.sqliteDef.columns.id!.default.pipe(Option.getOrThrow)
-      } else {
-        id = params[0] as string
-        if (id === undefined) {
-          invalidQueryBuilder(`Id missing for row query on non-singleton table ${tableDef.sqliteDef.name}`)
-        }
-      }
+      return makeQueryBuilder(tableDef, {
+        ...ast,
+        onConflict,
+      }) as any
+    },
+
+    returning: (...columns) => {
+      assertWriteQueryBuilderAst(ast)
+
+      return makeQueryBuilder(tableDef, {
+        ...ast,
+        returning: columns,
+        resultSchema: tableDef.rowSchema.pipe(Schema.pick(...columns), Schema.Array),
+      }) as any
+    },
 
-      // TODO validate all required columns are present and values are matching the schema
-      const insertValues: Record<string, unknown> = params[1]?.insertValues ?? {}
+    update: (values) => {
+      const filteredValues = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== undefined))
 
       return makeQueryBuilder(tableDef, {
-        _tag: 'RowQuery',
-        id,
+        _tag: 'UpdateQuery',
         tableDef,
-        insertValues,
+        values: filteredValues,
+        where: [],
+        returning: undefined,
+        resultSchema: Schema.Void,
       }) as any
     },
-  } satisfies QueryBuilder.ApiFull<TResult, TTableDef, never, QueryInfo.None>
+
+    delete: () => {
+      return makeQueryBuilder(tableDef, {
+        _tag: 'DeleteQuery',
+        tableDef,
+        where: [],
+        returning: undefined,
+        resultSchema: Schema.Void,
+      }) as any
+    },
+  } satisfies QueryBuilder.ApiFull<TResult, TTableDef, never>
 
   return {
-    [TypeId]: TypeId,
+    [QueryBuilderTypeId]: QueryBuilderTypeId,
     [QueryBuilderAstSymbol]: ast,
+    ['ResultType']: 'only-for-type-inference' as TResult,
     asSql: () => astToSql(ast),
     toString: () => {
       try {
@@ -167,119 +262,89 @@ export const makeQueryBuilder = <TResult, TTableDef extends DbSchema.TableDefBas
   } satisfies QueryBuilder<TResult, TTableDef>
 }
 
-const emptyAst = (tableDef: DbSchema.TableDefBase) =>
-  ({
-    _tag: 'SelectQuery',
-    columns: [],
-    pickFirst: false,
-    select: { columns: [] },
-    orderBy: [],
-    offset: Option.none(),
-    limit: Option.none(),
-    tableDef,
-    where: [],
-    resultSchemaSingle: tableDef.schema,
-  }) satisfies QueryBuilderAst
-
-const astToSql = (ast: QueryBuilderAst) => {
-  if (isRowQuery(ast)) {
-    // TODO
-    return { query: `SELECT * FROM '${ast.tableDef.sqliteDef.name}' WHERE id = ?`, bindValues: [ast.id as TODO] }
+const emptyAst = (tableDef: State.SQLite.TableDefBase): QueryBuilderAst.SelectQuery => ({
+  _tag: 'SelectQuery',
+  columns: [],
+  pickFirst: false,
+  select: { columns: [] },
+  orderBy: [],
+  offset: Option.none(),
+  limit: Option.none(),
+  tableDef,
+  where: [],
+  resultSchemaSingle: tableDef.rowSchema,
+})
+
+// Helper functions
+// eslint-disable-next-line prefer-arrow/prefer-arrow-functions
+function assertSelectQueryBuilderAst(ast: QueryBuilderAst): asserts ast is QueryBuilderAst.SelectQuery {
+  if (ast._tag !== 'SelectQuery') {
+    return shouldNeverHappen('Expected SelectQuery but got ' + ast._tag)
   }
+}
 
-  const bindValues: unknown[] = []
-
-  // TODO bind values
-  const whereStmt =
-    ast.where.length > 0
-      ? `WHERE ${ast.where
-          .map(({ col, op, value }) => {
-            if (value === null) {
-              if (op !== '=' && op !== '!=') {
-                throw new Error(`Unsupported operator for NULL value: ${op}`)
-              }
-              const opStmt = op === '=' ? 'IS' : 'IS NOT'
-              return `${col} ${opStmt} NULL`
-            } else {
-              const colDef = ast.tableDef.sqliteDef.columns[col]
-              if (colDef === undefined) {
-                throw new Error(`Column ${col} not found`)
-              }
-              const isArray = op === 'IN' || op === 'NOT IN'
-              const colSchema = isArray ? Schema.Array(colDef.schema) : colDef.schema
-              const encodedValue = Schema.encodeSync(colSchema)(value)
-
-              if (isArray) {
-                bindValues.push(...encodedValue)
-                const placeholders = Array.from({ length: encodedValue.length }, () => '?').join(', ')
-                return `${col} ${op} (${placeholders})`
-              } else {
-                bindValues.push(encodedValue)
-                return `${col} ${op} ?`
-              }
-            }
-          })
-          .join(' AND ')}`
-      : ''
-
-  if (ast._tag === 'CountQuery') {
-    const selectFromStmt = `SELECT COUNT(*) as count FROM '${ast.tableDef.sqliteDef.name}'`
-    const query = [selectFromStmt, whereStmt].filter((_) => _.length > 0).join(' ')
-    return { query, bindValues }
+// eslint-disable-next-line prefer-arrow/prefer-arrow-functions
+function assertInsertQueryBuilderAst(ast: QueryBuilderAst): asserts ast is QueryBuilderAst.InsertQuery {
+  if (ast._tag !== 'InsertQuery') {
+    return shouldNeverHappen('Expected InsertQuery but got ' + ast._tag)
   }
-  const columnsStmt = ast.select.columns.length === 0 ? '*' : ast.select.columns.join(', ')
-  const selectStmt = `SELECT ${columnsStmt}`
-  const fromStmt = `FROM '${ast.tableDef.sqliteDef.name}'`
-
-  const orderByStmt =
-    ast.orderBy.length > 0
-      ? `ORDER BY ${ast.orderBy.map(({ col, direction }) => `${col} ${direction}`).join(', ')}`
-      : ''
-
-  const limitStmt = ast.limit._tag === 'Some' ? `LIMIT ?` : ''
-  if (ast.limit._tag === 'Some') bindValues.push(ast.limit.value)
-
-  const offsetStmt = ast.offset._tag === 'Some' ? `OFFSET ?` : ''
-  if (ast.offset._tag === 'Some') bindValues.push(ast.offset.value)
-
-  const query = [selectStmt, fromStmt, whereStmt, orderByStmt, offsetStmt, limitStmt]
-    .map((_) => _.trim())
-    .filter((_) => _.length > 0)
-    .join(' ')
-
-  // TODO bind values
-  return { query, bindValues }
 }
 
 // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
-function assertQueryBuilderAst(ast: QueryBuilderAst): asserts ast is QueryBuilderAst.SelectQuery {
-  if (ast._tag !== 'SelectQuery') {
-    throw new Error('Expected SelectQuery but got ' + ast._tag)
+function assertWriteQueryBuilderAst(ast: QueryBuilderAst): asserts ast is QueryBuilderAst.WriteQuery {
+  if (ast._tag !== 'InsertQuery' && ast._tag !== 'UpdateQuery' && ast._tag !== 'DeleteQuery') {
+    return shouldNeverHappen('Expected WriteQuery but got ' + ast._tag)
   }
 }
 
 const isRowQuery = (ast: QueryBuilderAst): ast is QueryBuilderAst.RowQuery => ast._tag === 'RowQuery'
 
 export const invalidQueryBuilder = (msg?: string) => {
-  throw new Error('Invalid query builder' + (msg ? `: ${msg}` : ''))
+  return shouldNeverHappen('Invalid query builder' + (msg ? `: ${msg}` : ''))
 }
 
-export const getResultSchema = (qb: QueryBuilder<any, any, any>) => {
+export const getResultSchema = (qb: QueryBuilder<any, any, any>): Schema.Schema<any> => {
   const queryAst = qb[QueryBuilderAstSymbol]
-  if (queryAst._tag === 'SelectQuery') {
-    const arraySchema = Schema.Array(queryAst.resultSchemaSingle)
-    if (queryAst.pickFirst !== false) {
-      return arraySchema.pipe(Schema.headOrElse(queryAst.pickFirst.fallback))
+  switch (queryAst._tag) {
+    case 'SelectQuery': {
+      const arraySchema = Schema.Array(queryAst.resultSchemaSingle)
+      if (queryAst.pickFirst === false) {
+        return arraySchema
+      } else if (queryAst.pickFirst === 'no-fallback') {
+        // Will throw if the array is empty
+        return arraySchema.pipe(Schema.headOrElse())
+      } else {
+        const fallbackValue = queryAst.pickFirst.fallback()
+        return Schema.Union(arraySchema, Schema.Tuple(Schema.Literal(fallbackValue))).pipe(
+          Schema.headOrElse(() => fallbackValue),
+        )
+      }
+    }
+    case 'CountQuery': {
+      return Schema.Struct({ count: Schema.Number }).pipe(Schema.pluck('count'), Schema.Array, Schema.headOrElse())
     }
+    case 'InsertQuery':
+    case 'UpdateQuery':
+    case 'DeleteQuery': {
+      // For write operations with RETURNING clause, we need to return the appropriate schema
+      if (queryAst.returning && queryAst.returning.length > 0) {
+        // Create a schema for the returned columns
+        return queryAst.tableDef.rowSchema.pipe(Schema.pick(...queryAst.returning), Schema.Array)
+      }
 
-    return arraySchema
-  } else if (queryAst._tag === 'CountQuery') {
-    return Schema.Struct({ count: Schema.Number }).pipe(Schema.pluck('count'), Schema.Array, Schema.headOrElse())
-  } else {
-    if (queryAst.tableDef.options.isSingleColumn) {
-      return queryAst.tableDef.schema.pipe(Schema.pluck('value'), Schema.Array, Schema.headOrElse())
-    } else {
-      return queryAst.tableDef.schema.pipe(Schema.Array, Schema.headOrElse())
+      // For write operations without RETURNING, the result is the number of affected rows
+      return Schema.Number
+    }
+    case 'RowQuery': {
+      return queryAst.tableDef.rowSchema.pipe(
+        Schema.pluck('value'),
+        Schema.annotations({ title: `${queryAst.tableDef.sqliteDef.name}.value` }),
+        Schema.Array,
+        Schema.headOrElse(),
+      )
+    }
+    default: {
+      casesHandled(queryAst)
     }
   }
 }

package/src/query-builder/mod.ts

@@ -7,4 +7,11 @@ export * from './impl.js'
  * - Close abstraction to SQLite to provide a simple & convenient API with predictable behaviour
  * - Use table schema definitions to parse, map & validate query results
  * - Implementation detail: Separate type-level & AST-based runtime implementation
+ *
+ * Currently not supported (not exhaustive list):
+ * - Assumes a `id` column as primary key
+ * - Composite primary keys
+ *
+ * Other known limitations
+ * - Doesn't exclude all invalid query patterns on type level `e.g. `db.todos.returning('id')`
  */

package/src/rehydrate-from-eventlog.ts

@@ -0,0 +1,114 @@
+import { memoizeByRef } from '@livestore/utils'
+import { Chunk, Effect, Option, Schema, Stream } from '@livestore/utils/effect'
+
+import { type SqliteDb, UnexpectedError } from './adapter-types.js'
+import type { ApplyEvent } from './leader-thread/mod.js'
+import type { EventDef, EventlogMetaRow, LiveStoreSchema } from './schema/mod.js'
+import { EventId, EVENTLOG_META_TABLE, getEventDef, LiveStoreEvent } from './schema/mod.js'
+import type { PreparedBindValues } from './util.js'
+import { sql } from './util.js'
+
+export const rehydrateFromEventlog = ({
+  dbEventlog,
+  // TODO re-use this db when bringing back the boot in-memory db implementation
+  // db,
+  schema,
+  onProgress,
+  applyEvent,
+}: {
+  dbEventlog: SqliteDb
+  // db: SqliteDb
+  schema: LiveStoreSchema
+  onProgress: (_: { done: number; total: number }) => Effect.Effect<void>
+  applyEvent: ApplyEvent
+}) =>
+  Effect.gen(function* () {
+    const eventsCount = dbEventlog.select<{ count: number }>(`SELECT COUNT(*) AS count FROM ${EVENTLOG_META_TABLE}`)[0]!
+      .count
+
+    const hashEvent = memoizeByRef((event: EventDef.AnyWithoutFn) => Schema.hash(event.schema))
+
+    const processEvent = (row: EventlogMetaRow) =>
+      Effect.gen(function* () {
+        const eventDef = getEventDef(schema, row.name)
+
+        if (hashEvent(eventDef.eventDef) !== row.schemaHash) {
+          yield* Effect.logWarning(`Schema hash mismatch for mutation ${row.name}. Trying to apply mutation anyway.`)
+        }
+
+        const args = JSON.parse(row.argsJson)
+
+        // Checking whether the schema has changed in an incompatible way
+        yield* Schema.decodeUnknown(eventDef.eventDef.schema)(args).pipe(
+          Effect.mapError((cause) =>
+            UnexpectedError.make({
+              cause,
+              note: `\
+There was an error during rehydrating from the eventlog while decoding
+the persisted mutation event args for mutation "${row.name}".
+This likely means the schema has changed in an incompatible way.
+`,
+            }),
+          ),
+        )
+
+        const eventEncoded = LiveStoreEvent.EncodedWithMeta.make({
+          id: { global: row.idGlobal, client: row.idClient },
+          parentId: { global: row.parentIdGlobal, client: row.parentIdClient },
+          name: row.name,
+          args,
+          clientId: row.clientId,
+          sessionId: row.sessionId,
+        })
+
+        yield* applyEvent(eventEncoded, { skipEventlog: true })
+      }).pipe(Effect.withSpan(`@livestore/common:rehydrateFromEventlog:processEvent`))
+
+    const CHUNK_SIZE = 100
+
+    const stmt = dbEventlog.prepare(sql`\
+SELECT * FROM ${EVENTLOG_META_TABLE} 
+WHERE idGlobal > $idGlobal OR (idGlobal = $idGlobal AND idClient > $idClient)
+ORDER BY idGlobal ASC, idClient ASC
+LIMIT ${CHUNK_SIZE}
+`)
+
+    let processedEvents = 0
+
+    yield* Stream.unfoldChunk<Chunk.Chunk<EventlogMetaRow> | { _tag: 'Initial ' }, EventlogMetaRow>(
+      { _tag: 'Initial ' },
+      (item) => {
+        // End stream if no more rows
+        if (Chunk.isChunk(item) && item.length === 0) return Option.none()
+
+        const lastId = Chunk.isChunk(item)
+          ? Chunk.last(item).pipe(
+              Option.map((_) => ({ global: _.idGlobal, client: _.idClient })),
+              Option.getOrElse(() => EventId.ROOT),
+            )
+          : EventId.ROOT
+        const nextItem = Chunk.fromIterable(
+          stmt.select<EventlogMetaRow>({
+            $idGlobal: lastId?.global,
+            $idClient: lastId?.client,
+          } as any as PreparedBindValues),
+        )
+        const prevItem = Chunk.isChunk(item) ? item : Chunk.empty()
+        return Option.some([prevItem, nextItem])
+      },
+    ).pipe(
+      Stream.bufferChunks({ capacity: 2 }),
+      Stream.tap((row) =>
+        Effect.gen(function* () {
+          yield* processEvent(row)
+
+          processedEvents++
+          yield* onProgress({ done: processedEvents, total: eventsCount })
+        }),
+      ),
+      Stream.runDrain,
+    )
+  }).pipe(
+    Effect.withPerformanceMeasure('@livestore/common:rehydrateFromEventlog'),
+    Effect.withSpan('@livestore/common:rehydrateFromEventlog'),
+  )