@livestore/common 0.3.2-dev.9 → 0.4.0-dev.1

package/src/schema/state/sqlite/table-def.ts

@@ -1,12 +1,16 @@
-import type { Nullable } from '@livestore/utils'
-import type { Schema, Types } from '@livestore/utils/effect'
+import { type Nullable, shouldNeverHappen } from '@livestore/utils'
+import { Option, type Schema, SchemaAST, type Types } from '@livestore/utils/effect'
 
+import { getColumnDefForSchema, schemaFieldsToColumns } from './column-def.ts'
 import { SqliteDsl } from './db-schema/mod.ts'
 import type { QueryBuilder } from './query-builder/mod.ts'
 import { makeQueryBuilder, QueryBuilderAstSymbol, QueryBuilderTypeId } from './query-builder/mod.ts'
 
 export const { blob, boolean, column, datetime, integer, isColumnDefinition, json, real, text } = SqliteDsl
 
+// Re-export the column definition function
+export { getColumnDefForSchema }
+
 export type StateType = 'singleton' | 'dynamic'
 
 export type DefaultSqliteTableDef = SqliteDsl.TableDefinition<string, SqliteDsl.Columns>
@@ -17,6 +21,7 @@ export const TableDefInternalsSymbol = Symbol('TableDefInternals')
 export type TableDefInternalsSymbol = typeof TableDefInternalsSymbol
 
 export type TableDefBase<
+  // TODO replace SqliteDef type param with Effect Schema (see below)
   TSqliteDef extends DefaultSqliteTableDef = DefaultSqliteTableDefConstrained,
   TOptions extends TableOptions = TableOptions,
 > = {
@@ -28,10 +33,14 @@ export type TableDefBase<
 }
 
 export type TableDef<
+  // TODO replace SqliteDef type param with Effect Schema
+  // We can only do this with Effect Schema v4 once the default values are tracked on the type level
+  // https://github.com/livestorejs/livestore/issues/382
   TSqliteDef extends DefaultSqliteTableDef = DefaultSqliteTableDefConstrained,
   TOptions extends TableOptions = TableOptions,
   // NOTE we're not using `SqliteDsl.StructSchemaForColumns<TSqliteDef['columns']>`
   // as we don't want the alias type for users to show up, so we're redefining it here
+  // TODO adjust this to `TSchema = Schema.TypeLiteral<` but requires some advance type-level work
   TSchema = Schema.Schema<
     SqliteDsl.AnyIfConstained<
       TSqliteDef['columns'],
@@ -66,28 +75,192 @@ export type TableOptions = {
   readonly isClientDocumentTable: boolean
 }
 
-export const table = <
+/**
+ * Creates a SQLite table definition from columns or an Effect Schema.
+ *
+ * This function supports two main ways to define a table:
+ * 1. Using explicit column definitions
+ * 2. Using an Effect Schema (either the `name` property needs to be provided or the schema needs to have a title/identifier)
+ *
+ * ```ts
+ * // Using explicit columns
+ * const usersTable = State.SQLite.table({
+ *   name: 'users',
+ *   columns: {
+ *     id: State.SQLite.text({ primaryKey: true }),
+ *     name: State.SQLite.text({ nullable: false }),
+ *     email: State.SQLite.text({ nullable: false }),
+ *     age: State.SQLite.integer({ nullable: true }),
+ *   },
+ * })
+ * ```
+ *
+ * ```ts
+ * // Using Effect Schema with annotations
+ * import { Schema } from '@livestore/utils/effect'
+ *
+ * const UserSchema = Schema.Struct({
+ *   id: Schema.Int.pipe(State.SQLite.withPrimaryKey).pipe(State.SQLite.withAutoIncrement),
+ *   email: Schema.String.pipe(State.SQLite.withUnique),
+ *   name: Schema.String,
+ *   active: Schema.Boolean.pipe(State.SQLite.withDefault(true)),
+ *   createdAt: Schema.optional(Schema.Date),
+ * })
+ *
+ * // Option 1: With explicit name
+ * const usersTable = State.SQLite.table({
+ *   name: 'users',
+ *   schema: UserSchema,
+ * })
+ *
+ * // Option 2: With name from schema annotation (title or identifier)
+ * const AnnotatedUserSchema = UserSchema.annotations({ title: 'users' })
+ * const usersTable2 = State.SQLite.table({
+ *   schema: AnnotatedUserSchema,
+ * })
+ * ```
+ *
+ * ```ts
+ * // Adding indexes
+ * const PostSchema = Schema.Struct({
+ *   id: Schema.String.pipe(State.SQLite.withPrimaryKey),
+ *   title: Schema.String,
+ *   authorId: Schema.String,
+ *   createdAt: Schema.Date,
+ * }).annotations({ identifier: 'posts' })
+ *
+ * const postsTable = State.SQLite.table({
+ *   schema: PostSchema,
+ *   indexes: [
+ *     { name: 'idx_posts_author', columns: ['authorId'] },
+ *     { name: 'idx_posts_created', columns: ['createdAt'], isUnique: false },
+ *   ],
+ * })
+ * ```
+ *
+ * @remarks
+ * - Primary key columns are automatically non-nullable
+ * - Columns with `State.SQLite.withUnique` annotation automatically get unique indexes
+ * - The `State.SQLite.withAutoIncrement` annotation only works with integer primary keys
+ * - Default values can be literal values or SQL expressions
+ * - When using Effect Schema without explicit name, the schema must have a title or identifier annotation
+ */
+// Overload 1: With columns
+// TODO drop support for `column` when Effect Schema v4 is released
+export function table<
   TName extends string,
-  TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition<any, any>,
+  TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition.Any,
   const TOptionsInput extends TableOptionsInput = TableOptionsInput,
 >(
   args: {
     name: TName
     columns: TColumns
   } & Partial<TOptionsInput>,
-): TableDef<SqliteTableDefForInput<TName, TColumns>, WithDefaults<TColumns>> => {
-  const { name, columns: columnOrColumns, ...options } = args
-  const tablePath = name
+): TableDef<SqliteTableDefForInput<TName, TColumns>, WithDefaults<TColumns>>
+
+// Overload 2: With schema and explicit name
+export function table<
+  TName extends string,
+  TSchema extends Schema.Schema.AnyNoContext,
+  const TOptionsInput extends TableOptionsInput = TableOptionsInput,
+>(
+  args: {
+    name: TName
+    schema: TSchema
+  } & Partial<TOptionsInput>,
+): TableDef<
+  SqliteTableDefForSchemaInput<TName, Schema.Schema.Type<TSchema>, Schema.Schema.Encoded<TSchema>, TSchema>,
+  TableOptions
+>
+
+// Overload 3: With schema and no name (uses schema annotations)
+export function table<
+  TSchema extends Schema.Schema.AnyNoContext,
+  const TOptionsInput extends TableOptionsInput = TableOptionsInput,
+>(
+  args: {
+    schema: TSchema
+  } & Partial<TOptionsInput>,
+): TableDef<
+  SqliteTableDefForSchemaInput<string, Schema.Schema.Type<TSchema>, Schema.Schema.Encoded<TSchema>, TSchema>,
+  TableOptions
+>
+
+// Implementation
+export function table<
+  TName extends string,
+  TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition.Any,
+  const TOptionsInput extends TableOptionsInput = TableOptionsInput,
+>(
+  args: (
+    | {
+        name: TName
+        columns: TColumns
+      }
+    | {
+        name: TName
+        schema: Schema.Schema.AnyNoContext
+      }
+    | {
+        schema: Schema.Schema.AnyNoContext
+      }
+  ) &
+    Partial<TOptionsInput>,
+): TableDef<any, any> {
+  const { ...options } = args
+
+  let tableName: string
+  let columns: SqliteDsl.Columns
+  let additionalIndexes: SqliteDsl.Index[] = []
+
+  if ('columns' in args) {
+    tableName = args.name
+    const columnOrColumns = args.columns
+    columns = (
+      SqliteDsl.isColumnDefinition(columnOrColumns) ? { value: columnOrColumns } : columnOrColumns
+    ) as SqliteDsl.Columns
+    additionalIndexes = []
+  } else if ('schema' in args) {
+    const result = schemaFieldsToColumns(SchemaAST.getPropertySignatures(args.schema.ast))
+    columns = result.columns
+
+    // We'll set tableName first, then use it for index names
+    let tempTableName: string
+
+    // If name is provided, use it; otherwise extract from schema annotations
+    if ('name' in args) {
+      tempTableName = args.name
+    } else {
+      // Use title or identifier, with preference for title
+      tempTableName = SchemaAST.getTitleAnnotation(args.schema.ast).pipe(
+        Option.orElse(() => SchemaAST.getIdentifierAnnotation(args.schema.ast)),
+        Option.getOrElse(() =>
+          shouldNeverHappen(
+            'When using schema without explicit name, the schema must have a title or identifier annotation',
+          ),
+        ),
+      )
+    }
+
+    tableName = tempTableName
+
+    // Create unique indexes for columns with unique annotation
+    additionalIndexes = (result.uniqueColumns || []).map((columnName) => ({
+      name: `idx_${tableName}_${columnName}_unique`,
+      columns: [columnName],
+      isUnique: true,
+    }))
+  } else {
+    return shouldNeverHappen('Either `columns` or `schema` must be provided when calling `table()`')
+  }
 
   const options_: TableOptions = {
     isClientDocumentTable: false,
   }
 
-  const columns = (
-    SqliteDsl.isColumnDefinition(columnOrColumns) ? { value: columnOrColumns } : columnOrColumns
-  ) as SqliteDsl.Columns
-
-  const sqliteDef = SqliteDsl.table(tablePath, columns, options?.indexes ?? [])
+  // Combine user-provided indexes with unique column indexes
+  const allIndexes = [...(options?.indexes ?? []), ...additionalIndexes]
+  const sqliteDef = SqliteDsl.table(tableName, columns, allIndexes)
 
   const rowSchema = SqliteDsl.structSchemaForTable(sqliteDef)
   const insertSchema = SqliteDsl.insertStructSchemaForTable(sqliteDef)
@@ -179,19 +352,52 @@ export namespace FromColumns {
 
 export type SqliteTableDefForInput<
   TName extends string,
-  TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition<any, any>,
+  TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition.Any,
 > = SqliteDsl.TableDefinition<TName, PrettifyFlat<ToColumns<TColumns>>>
 
-type WithDefaults<TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition<any, any>> = {
+export type SqliteTableDefForSchemaInput<
+  TName extends string,
+  TType,
+  TEncoded,
+  _TSchema = any,
+> = TableDefInput.ForSchema<TName, TType, TEncoded, _TSchema>
+
+export type WithDefaults<TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition.Any> = {
   isClientDocumentTable: false
   requiredInsertColumnNames: SqliteDsl.FromColumns.RequiredInsertColumnNames<ToColumns<TColumns>>
 }
 
 export type PrettifyFlat<T> = T extends infer U ? { [K in keyof U]: U[K] } : never
 
-type ToColumns<TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition<any, any>> =
+export type ToColumns<TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition.Any> =
   TColumns extends SqliteDsl.Columns
     ? TColumns
-    : TColumns extends SqliteDsl.ColumnDefinition<any, any>
+    : TColumns extends SqliteDsl.ColumnDefinition.Any
       ? { value: TColumns }
       : never
+
+export declare namespace SchemaToColumns {
+  // Type helper to create column definition with proper schema
+  export type ColumnDefForType<TEncoded, TType> = SqliteDsl.ColumnDefinition<TEncoded, TType>
+
+  // Create columns type from schema Type and Encoded
+  export type FromTypes<TType, TEncoded> = TType extends Record<string, any>
+    ? TEncoded extends Record<string, any>
+      ? {
+          [K in keyof TType & keyof TEncoded]: ColumnDefForType<TEncoded[K], TType[K]>
+        }
+      : SqliteDsl.Columns
+    : SqliteDsl.Columns
+}
+
+export declare namespace TableDefInput {
+  export type ForColumns<
+    TName extends string,
+    TColumns extends SqliteDsl.Columns | SqliteDsl.ColumnDefinition.Any,
+  > = SqliteDsl.TableDefinition<TName, PrettifyFlat<ToColumns<TColumns>>>
+
+  export type ForSchema<TName extends string, TType, TEncoded, _TSchema = any> = SqliteDsl.TableDefinition<
+    TName,
+    SchemaToColumns.FromTypes<TType, TEncoded>
+  >
+}

package/src/schema-management/common.ts

@@ -1,4 +1,4 @@
-import type { SqliteDb } from '../adapter-types.ts'
+import { type SqliteDb, SqliteError } from '../adapter-types.ts'
 import type { ParamsObject } from '../util.ts'
 import { prepareBindValues } from '../util.ts'
 
@@ -15,9 +15,16 @@ export const dbExecute = (db: SqliteDb, queryStr: string, bindValues?: ParamsObj
 
   const preparedBindValues = bindValues ? prepareBindValues(bindValues, queryStr) : undefined
 
-  stmt.execute(preparedBindValues)
+  try {
+    stmt.execute(preparedBindValues)
 
-  stmt.finalize()
+    stmt.finalize()
+  } catch (cause) {
+    throw new SqliteError({
+      cause,
+      query: { sql: queryStr, bindValues: preparedBindValues ?? {} },
+    })
+  }
 }
 
 export const dbSelect = <T>(db: SqliteDb, queryStr: string, bindValues?: ParamsObject) => {

package/src/schema-management/migrations.ts

@@ -1,11 +1,12 @@
 import { memoizeByStringifyArgs } from '@livestore/utils'
-import { Effect, Schema as EffectSchema } from '@livestore/utils/effect'
+import { Effect } from '@livestore/utils/effect'
 
 import type { SqliteDb } from '../adapter-types.ts'
 import type { MigrationsReport, MigrationsReportEntry } from '../defs.ts'
 import type { UnexpectedError } from '../errors.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
-import { SqliteAst, SqliteDsl } from '../schema/state/sqlite/db-schema/mod.ts'
+import { makeColumnSpec } from '../schema/state/sqlite/column-spec.ts'
+import { SqliteAst } from '../schema/state/sqlite/db-schema/mod.ts'
 import type { SchemaEventDefsMetaRow, SchemaMetaRow } from '../schema/state/sqlite/system-tables.ts'
 import {
   isStateSystemTable,
@@ -169,35 +170,5 @@ export const migrateTable = ({
 
 const createIndexFromDefinition = (tableName: string, index: SqliteAst.Index) => {
   const uniqueStr = index.unique ? 'UNIQUE' : ''
-  return sql`create ${uniqueStr} index if not exists '${index.name}' on '${tableName}' (${index.columns.join(', ')})`
-}
-
-export const makeColumnSpec = (tableAst: SqliteAst.Table) => {
-  const primaryKeys = tableAst.columns.filter((_) => _.primaryKey).map((_) => `'${_.name}'`)
-  const columnDefStrs = tableAst.columns.map(toSqliteColumnSpec)
-  if (primaryKeys.length > 0) {
-    columnDefStrs.push(`PRIMARY KEY (${primaryKeys.join(', ')})`)
-  }
-
-  return columnDefStrs.join(', ')
-}
-
-/** NOTE primary keys are applied on a table level not on a column level to account for multi-column primary keys */
-const toSqliteColumnSpec = (column: SqliteAst.Column) => {
-  const columnTypeStr = column.type._tag
-  const nullableStr = column.nullable === false ? 'not null' : ''
-  const defaultValueStr = (() => {
-    if (column.default._tag === 'None') return ''
-
-    if (column.default.value === null) return 'default null'
-    if (SqliteDsl.isSqlDefaultValue(column.default.value)) return `default ${column.default.value.sql}`
-
-    const encodeValue = EffectSchema.encodeSync(column.schema)
-    const encodedDefaultValue = encodeValue(column.default.value)
-
-    if (columnTypeStr === 'text') return `default '${encodedDefaultValue}'`
-    return `default ${encodedDefaultValue}`
-  })()
-
-  return `'${column.name}' ${columnTypeStr} ${nullableStr} ${defaultValueStr}`
+  return sql`create ${uniqueStr} index if not exists '${index.name}' on '${tableName}' (${index.columns.map((col) => `'${col}'`).join(', ')})`
 }

package/src/sqlite-db-helper.ts

@@ -1,6 +1,6 @@
 import { Schema } from '@livestore/utils/effect'
 
-import type { SqliteDb } from './adapter-types.ts'
+import { type SqliteDb, SqliteError } from './adapter-types.ts'
 import { getResultSchema, isQueryBuilder } from './schema/state/sqlite/query-builder/mod.ts'
 import type { PreparedBindValues } from './util.ts'
 
@@ -39,3 +39,21 @@ export const makeSelect = <T>(
     }
   }
 }
+
+export const validateSnapshot = (snapshot: Uint8Array) => {
+  const headerBytes = new TextDecoder().decode(snapshot.slice(0, 16))
+  const hasValidHeader = headerBytes.startsWith('SQLite format 3')
+
+  if (!hasValidHeader) {
+    throw new SqliteError({
+      cause: 'Invalid SQLite header',
+      note: `Expected header to start with 'SQLite format 3', but got: ${headerBytes}`,
+    })
+  }
+}
+
+export const makeExport = (exportFn: () => Uint8Array<ArrayBuffer>) => () => {
+  const snapshot = exportFn()
+  validateSnapshot(snapshot)
+  return snapshot
+}

package/src/sqlite-types.ts

@@ -25,12 +25,12 @@ export interface SqliteDb<TReq = any, TMetadata extends TReq = TReq> {
   select<T>(queryStr: string, bindValues?: PreparedBindValues | undefined): ReadonlyArray<T>
   select<T>(queryBuilder: QueryBuilder<T, any, any>): T
 
-  export(): Uint8Array
-  import: (data: Uint8Array | SqliteDb<TReq>) => void
+  export(): Uint8Array<ArrayBuffer>
+  import: (data: Uint8Array<ArrayBuffer> | SqliteDb<TReq>) => void
   close(): void
   destroy(): void
   session(): SqliteDbSession
-  makeChangeset: (data: Uint8Array) => SqliteDbChangeset
+  makeChangeset: (data: Uint8Array<ArrayBuffer>) => SqliteDbChangeset
 }
 
 export type SqliteDebugInfo = { head: EventSequenceNumber.EventSequenceNumber }
@@ -56,7 +56,7 @@ export interface PreparedStatement {
 }
 
 export type SqliteDbSession = {
-  changeset: () => Uint8Array | undefined
+  changeset: () => Uint8Array<ArrayBuffer> | undefined
   finish: () => void
 }
 

package/src/sync/ClientSessionSyncProcessor.ts

@@ -3,6 +3,7 @@ import { LS_DEV, shouldNeverHappen, TRACE_VERBOSE } from '@livestore/utils'
 import {
   BucketQueue,
   Effect,
+  Exit,
   FiberHandle,
   Option,
   Queue,
@@ -54,10 +55,13 @@ export const makeClientSessionSyncProcessor = ({
     options: { otelContext: otel.Context; withChangeset: boolean; materializerHashLeader: Option.Option<number> },
   ) => {
     writeTables: Set<string>
-    sessionChangeset: { _tag: 'sessionChangeset'; data: Uint8Array; debug: any } | { _tag: 'no-op' } | { _tag: 'unset' }
+    sessionChangeset:
+      | { _tag: 'sessionChangeset'; data: Uint8Array<ArrayBuffer>; debug: any }
+      | { _tag: 'no-op' }
+      | { _tag: 'unset' }
     materializerHash: Option.Option<number>
   }
-  rollback: (changeset: Uint8Array) => void
+  rollback: (changeset: Uint8Array<ArrayBuffer>) => void
   refreshTables: (tables: Set<string>) => void
   span: otel.Span
   params: {
@@ -316,7 +320,7 @@ export const makeClientSessionSyncProcessor = ({
           refreshTables(writeTables)
         }).pipe(
           Effect.tapCauseLogPretty,
-          Effect.catchAllCause((cause) => clientSession.shutdown(cause)),
+          Effect.catchAllCause((cause) => clientSession.shutdown(Exit.failCause(cause))),
         ),
       ),
       Stream.runDrain,
@@ -381,13 +385,14 @@ export interface ClientSessionSyncProcessor {
 // TODO turn this into a build-time "macro" so all simulation snippets are removed for production builds
 const SIMULATION_ENABLED = true
 
+// Warning: High values for the simulation params can lead to very long test runs since those get multiplied with the number of events
 export const ClientSessionSyncProcessorSimulationParams = Schema.Struct({
   pull: Schema.Struct({
-    '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 1000)),
-    '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 1000)),
-    '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 1000)),
-    '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 1000)),
-    '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 1000)),
+    '1_before_leader_push_fiber_interrupt': Schema.Int.pipe(Schema.between(0, 25)),
+    '2_before_leader_push_queue_clear': Schema.Int.pipe(Schema.between(0, 25)),
+    '3_before_rebase_rollback': Schema.Int.pipe(Schema.between(0, 25)),
+    '4_before_leader_push_queue_offer': Schema.Int.pipe(Schema.between(0, 25)),
+    '5_before_leader_push_fiber_run': Schema.Int.pipe(Schema.between(0, 25)),
   }),
 })
 type ClientSessionSyncProcessorSimulationParams = typeof ClientSessionSyncProcessorSimulationParams.Type

package/src/sync/sync.ts

@@ -31,6 +31,7 @@ export type SyncOptions = {
   onSyncError?: 'shutdown' | 'ignore'
 }
 
+// TODO rename to `SyncProviderClientConstructor`
 export type SyncBackendConstructor<TSyncMetadata = Schema.JsonValue> = (
   args: MakeBackendArgs,
 ) => Effect.Effect<SyncBackend<TSyncMetadata>, UnexpectedError, Scope.Scope | HttpClient.HttpClient>
@@ -41,6 +42,7 @@ export type SyncBackendConstructor<TSyncMetadata = Schema.JsonValue> = (
 // - dynamic sync backend data;
 //   - data center location (e.g. colo on CF workers)
 
+// TODO rename to `SyncProviderClient`
 export type SyncBackend<TSyncMetadata = Schema.JsonValue> = {
   /**
    * Can be implemented to prepare a connection to the sync backend to speed up the first pull/push.

package/src/util.ts

@@ -4,11 +4,16 @@ import type { Brand } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
 export type ParamsObject = Record<string, SqlValue>
-export type SqlValue = string | number | Uint8Array | null
+export type SqlValue = string | number | Uint8Array<ArrayBuffer> | null
 
 export type Bindable = ReadonlyArray<SqlValue> | ParamsObject
 
-export const SqlValueSchema = Schema.Union(Schema.String, Schema.Number, Schema.Uint8Array, Schema.Null)
+export const SqlValueSchema = Schema.Union(
+  Schema.String,
+  Schema.Number,
+  Schema.Uint8Array as any as Schema.Schema<Uint8Array<ArrayBuffer>>,
+  Schema.Null,
+)
 
 export const PreparedBindValues = Schema.Union(
   Schema.Array(SqlValueSchema),

package/src/version.ts

@@ -2,7 +2,7 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.3.2-dev.9' as const
+export const liveStoreVersion = '0.4.0-dev.1' as const
 
 /**
  * This version number is incremented whenever the internal storage format changes in a breaking way.