@livestore/common 0.4.0-dev.1 → 0.4.0-dev.10

package/src/schema/state/sqlite/column-def.ts

@@ -1,4 +1,4 @@
-import { shouldNeverHappen, type Writeable } from '@livestore/utils'
+import { shouldNeverHappen } from '@livestore/utils'
 import { Option, Schema, SchemaAST } from '@livestore/utils/effect'
 
 import { AutoIncrement, ColumnType, Default, PrimaryKeyId, Unique } from './column-annotations.ts'
@@ -6,96 +6,61 @@ import { SqliteDsl } from './db-schema/mod.ts'
 
 /**
  * Maps a schema to a SQLite column definition, respecting column annotations.
+ *
+ * Note: When used with schema-based table definitions, optional fields (| undefined)
+ * are transformed to nullable fields (| null) to match SQLite's NULL semantics.
+ * Fields with both null and undefined will emit a warning as this is a lossy conversion.
  */
 export const getColumnDefForSchema = (
   schema: Schema.Schema.AnyNoContext,
   propertySignature?: SchemaAST.PropertySignature,
+  forceNullable = false,
 ): SqliteDsl.ColumnDefinition.Any => {
   const ast = schema.ast
 
-  // 1. Extract annotations
+  // Extract annotations
   const getAnnotation = <T>(annotationId: symbol): Option.Option<T> =>
     propertySignature
       ? hasPropertyAnnotation<T>(propertySignature, annotationId)
       : SchemaAST.getAnnotation<T>(annotationId)(ast)
 
-  const annotations = {
-    primaryKey: getAnnotation<boolean>(PrimaryKeyId).pipe(Option.getOrElse(() => false)),
-    autoIncrement: getAnnotation<boolean>(AutoIncrement).pipe(Option.getOrElse(() => false)),
-    defaultValue: getAnnotation<unknown>(Default),
-    columnType: SchemaAST.getAnnotation<SqliteDsl.FieldColumnType>(ColumnType)(ast),
-  }
+  const columnType = SchemaAST.getAnnotation<SqliteDsl.FieldColumnType>(ColumnType)(ast)
 
-  // 2. Resolve the core type and nullable info
-  const typeInfo = resolveType(ast)
+  // Check if schema has null (e.g., Schema.NullOr) or undefined or if it's forced nullable (optional field)
+  const isNullable = forceNullable || hasNull(ast) || hasUndefined(ast)
 
-  // 3. Create column definition based on resolved type
-  let columnDef: SqliteDsl.ColumnDefinition.Any
+  // Get base column definition with nullable flag
+  const baseColumn = Option.isSome(columnType)
+    ? getColumnForType(columnType.value, isNullable)
+    : getColumnForSchema(schema, isNullable)
 
-  // Custom column type overrides everything
-  if (Option.isSome(annotations.columnType)) {
-    columnDef = createColumnFromType(annotations.columnType.value, typeInfo.coreType)
-  }
-  // Lossy case: both null and undefined need JSON
-  else if (typeInfo.hasNull && typeInfo.hasUndefined) {
-    columnDef = {
-      ...SqliteDsl.text(),
-      nullable: true,
-      schema: Schema.parseJson(schema),
-    }
-  }
-  // Regular nullable/optional case
-  else if (typeInfo.hasNull || typeInfo.hasUndefined) {
-    const baseColumnDef = createColumnFromAST(typeInfo.coreType, Schema.make(typeInfo.coreType))
-    const isComplexOptional = typeInfo.hasUndefined && !isPrimitiveAST(typeInfo.coreType)
-
-    columnDef = {
-      ...baseColumnDef,
-      nullable: true,
-      schema: isComplexOptional ? Schema.parseJson(schema) : schema,
-    }
-  }
-  // Non-nullable type
-  else {
-    columnDef = createColumnFromAST(ast, schema)
-  }
+  // Apply annotations
+  const primaryKey = getAnnotation<boolean>(PrimaryKeyId).pipe(Option.getOrElse(() => false))
+  const autoIncrement = getAnnotation<boolean>(AutoIncrement).pipe(Option.getOrElse(() => false))
+  const defaultValue = getAnnotation<unknown>(Default)
 
-  // 4. Apply annotations
-  const result = { ...columnDef }
-  if (annotations.primaryKey) result.primaryKey = true
-  if (annotations.autoIncrement) result.autoIncrement = true
-  if (Option.isSome(annotations.defaultValue)) {
-    result.default = Option.some(annotations.defaultValue.value)
+  return {
+    ...baseColumn,
+    ...(primaryKey && { primaryKey: true }),
+    ...(autoIncrement && { autoIncrement: true }),
+    ...(Option.isSome(defaultValue) && { default: Option.some(defaultValue.value) }),
   }
-
-  return result
 }
 
-/**
- * Checks if a property signature has a specific annotation, checking both
- * the property signature itself and its type AST.
- */
 const hasPropertyAnnotation = <T>(
   propertySignature: SchemaAST.PropertySignature,
   annotationId: symbol,
 ): Option.Option<T> => {
-  // When using Schema.optional(Schema.String).pipe(withPrimaryKey) in a struct,
-  // the annotation ends up on a PropertySignatureDeclaration, not the Union type
-  // Check if this is a PropertySignatureDeclaration with annotations
   if ('annotations' in propertySignature && propertySignature.annotations) {
     const annotation = SchemaAST.getAnnotation<T>(annotationId)(propertySignature as any)
-    if (Option.isSome(annotation)) {
-      return annotation
-    }
+    if (Option.isSome(annotation)) return annotation
   }
-
-  // Otherwise check the type AST
   return SchemaAST.getAnnotation<T>(annotationId)(propertySignature.type)
 }
 
 /**
  * Maps schema property signatures to SQLite column definitions.
- * Returns both columns and unique column names for index creation.
+ * Optional fields (| undefined) become nullable columns (| null).
  */
 export const schemaFieldsToColumns = (
   propertySignatures: ReadonlyArray<SchemaAST.PropertySignature>,
@@ -104,187 +69,147 @@ export const schemaFieldsToColumns = (
   const uniqueColumns: string[] = []
 
   for (const prop of propertySignatures) {
-    if (typeof prop.name === 'string') {
-      // Create a schema from the AST
-      const fieldSchema = Schema.make(prop.type)
-      // Check if property has primary key annotation
-      const hasPrimaryKey = hasPropertyAnnotation<boolean>(prop, PrimaryKeyId).pipe(Option.getOrElse(() => false))
-      // Check if property has unique annotation
-      const hasUnique = hasPropertyAnnotation<boolean>(prop, Unique).pipe(Option.getOrElse(() => false))
-
-      columns[prop.name] = schemaFieldToColumn(fieldSchema, prop, hasPrimaryKey)
-
-      if (hasUnique) {
-        uniqueColumns.push(prop.name)
+    if (typeof prop.name !== 'string') continue
+
+    const fieldSchema = Schema.make(prop.type)
+
+    // Warn about lossy conversion for fields with both null and undefined
+    if (prop.isOptional) {
+      const { hasNull, hasUndefined } = checkNullUndefined(fieldSchema.ast)
+      if (hasNull && hasUndefined) {
+        console.warn(`Field '${prop.name}' has both null and undefined - treating | undefined as | null`)
       }
     }
-  }
-
-  return { columns, uniqueColumns }
-}
 
-/**
- * Converts a schema field and its property signature to a SQLite column definition.
- */
-const schemaFieldToColumn = (
-  fieldSchema: Schema.Schema.AnyNoContext,
-  propertySignature: SchemaAST.PropertySignature,
-  forceHasPrimaryKey?: boolean,
-): SqliteDsl.ColumnDefinition.Any => {
-  // Determine column type based on schema type
-  const columnDef = getColumnDefForSchema(fieldSchema, propertySignature)
-
-  // Create a new object with appropriate properties
-  const result: Writeable<SqliteDsl.ColumnDefinition.Any> = {
-    columnType: columnDef.columnType,
-    schema: columnDef.schema,
-    default: columnDef.default,
-    nullable: columnDef.nullable,
-    primaryKey: columnDef.primaryKey,
-    autoIncrement: columnDef.autoIncrement,
-  }
+    // Get column definition - pass nullable flag for optional fields
+    const columnDef = getColumnDefForSchema(fieldSchema, prop, prop.isOptional)
 
-  // Set primaryKey property explicitly
-  if (forceHasPrimaryKey || columnDef.primaryKey) {
-    result.primaryKey = true
-  } else {
-    result.primaryKey = false
-  }
+    // Check for primary key and unique annotations
+    const hasPrimaryKey = hasPropertyAnnotation<boolean>(prop, PrimaryKeyId).pipe(Option.getOrElse(() => false))
+    const hasUnique = hasPropertyAnnotation<boolean>(prop, Unique).pipe(Option.getOrElse(() => false))
 
-  // Check for invalid primary key + nullable combination
-  if (result.primaryKey && (propertySignature.isOptional || columnDef.nullable)) {
-    return shouldNeverHappen(
-      `Primary key columns cannot be nullable. Found nullable primary key for column. ` +
-        `Either remove the primary key annotation or use a non-nullable schema.`,
-    )
-  }
+    // Build final column
+    columns[prop.name] = {
+      ...columnDef,
+      ...(hasPrimaryKey && { primaryKey: true }),
+    }
 
-  // Set nullable property explicitly
-  if (propertySignature.isOptional) {
-    result.nullable = true
-  } else if (columnDef.nullable) {
-    result.nullable = true
-  } else {
-    result.nullable = false
-  }
+    // Validate primary key + nullable
+    const column = columns[prop.name]
+    if (column?.primaryKey && column.nullable) {
+      throw new Error('Primary key columns cannot be nullable')
+    }
 
-  // Only add autoIncrement if it's true
-  if (columnDef.autoIncrement) {
-    result.autoIncrement = true
+    if (hasUnique) uniqueColumns.push(prop.name)
   }
 
-  return result as SqliteDsl.ColumnDefinition.Any
+  return { columns, uniqueColumns }
 }
 
-/**
- * Resolves type information from an AST, unwrapping unions and tracking nullability.
- */
-const resolveType = (
-  ast: SchemaAST.AST,
-): {
-  coreType: SchemaAST.AST
-  hasNull: boolean
-  hasUndefined: boolean
-} => {
-  if (!SchemaAST.isUnion(ast)) {
-    return { coreType: ast, hasNull: false, hasUndefined: false }
-  }
-
+const checkNullUndefined = (ast: SchemaAST.AST): { hasNull: boolean; hasUndefined: boolean } => {
   let hasNull = false
   let hasUndefined = false
-  let coreType: SchemaAST.AST | undefined
 
   const visit = (type: SchemaAST.AST): void => {
-    if (SchemaAST.isUndefinedKeyword(type)) {
-      hasUndefined = true
-    } else if (SchemaAST.isLiteral(type) && type.literal === null) {
-      hasNull = true
-    } else if (SchemaAST.isUnion(type)) {
-      type.types.forEach(visit)
-    } else if (!coreType) {
-      coreType = type
-    }
+    if (SchemaAST.isUndefinedKeyword(type)) hasUndefined = true
+    else if (SchemaAST.isLiteral(type) && type.literal === null) hasNull = true
+    else if (SchemaAST.isUnion(type)) type.types.forEach(visit)
   }
 
-  ast.types.forEach(visit)
-  return { coreType: coreType || ast, hasNull, hasUndefined }
+  visit(ast)
+  return { hasNull, hasUndefined }
 }
 
-/**
- * Creates a column definition from an AST node.
- */
-const createColumnFromAST = (
-  ast: SchemaAST.AST,
-  schema: Schema.Schema.AnyNoContext,
-): SqliteDsl.ColumnDefinition.Any => {
-  // Follow refinements and transformations to their core type
-  if (SchemaAST.isRefinement(ast)) {
-    // Special case for Schema.Int
-    const identifier = SchemaAST.getIdentifierAnnotation(ast).pipe(Option.getOrElse(() => ''))
-    if (identifier === 'Int') return SqliteDsl.integer()
-    return createColumnFromAST(ast.from, Schema.make(ast.from))
-  }
-
-  if (SchemaAST.isTransformation(ast)) {
-    return createColumnFromAST(ast.to, Schema.make(ast.to))
+const hasNull = (ast: SchemaAST.AST): boolean => {
+  if (SchemaAST.isLiteral(ast) && ast.literal === null) return true
+  if (SchemaAST.isUnion(ast)) {
+    return ast.types.some((type) => hasNull(type))
   }
+  return false
+}
 
-  // Primitive types
-  if (SchemaAST.isStringKeyword(ast)) return SqliteDsl.text()
-  if (SchemaAST.isNumberKeyword(ast)) return SqliteDsl.real()
-  if (SchemaAST.isBooleanKeyword(ast)) return SqliteDsl.boolean()
-
-  // Literals
-  if (SchemaAST.isLiteral(ast)) {
-    const value = ast.literal
-    if (typeof value === 'string') return SqliteDsl.text()
-    if (typeof value === 'number') return SqliteDsl.real()
-    if (typeof value === 'boolean') return SqliteDsl.boolean()
+const hasUndefined = (ast: SchemaAST.AST): boolean => {
+  if (SchemaAST.isUndefinedKeyword(ast)) return true
+  if (SchemaAST.isUnion(ast)) {
+    return ast.types.some((type) => hasUndefined(type))
   }
-
-  // Everything else is complex
-  return SqliteDsl.json({ schema })
+  return false
 }
 
-/**
- * Creates a column from a specific column type string.
- */
-const createColumnFromType = (columnType: string, ast: SchemaAST.AST): SqliteDsl.ColumnDefinition.Any => {
+const getColumnForType = (columnType: string, nullable = false): SqliteDsl.ColumnDefinition.Any => {
   switch (columnType) {
     case 'text':
-      return SqliteDsl.text()
+      return SqliteDsl.text({ nullable })
     case 'integer':
-      // Preserve boolean transformation
-      return SchemaAST.isBooleanKeyword(ast) ? SqliteDsl.boolean() : SqliteDsl.integer()
+      return SqliteDsl.integer({ nullable })
     case 'real':
-      return SqliteDsl.real()
+      return SqliteDsl.real({ nullable })
     case 'blob':
-      return SqliteDsl.blob()
+      return SqliteDsl.blob({ nullable })
     default:
       return shouldNeverHappen(`Unsupported column type: ${columnType}`)
   }
 }
 
-/**
- * Checks if an AST represents a primitive (non-complex) type.
- */
-const isPrimitiveAST = (ast: SchemaAST.AST): boolean => {
-  if (
-    SchemaAST.isStringKeyword(ast) ||
-    SchemaAST.isNumberKeyword(ast) ||
-    SchemaAST.isBooleanKeyword(ast) ||
-    SchemaAST.isLiteral(ast)
-  ) {
-    return true
+const getColumnForSchema = (schema: Schema.Schema.AnyNoContext, nullable = false): SqliteDsl.ColumnDefinition.Any => {
+  const ast = schema.ast
+  // Strip nullable wrapper to get core type
+  const coreAst = stripNullable(ast)
+  const coreSchema = stripNullable(ast) === ast ? schema : Schema.make(coreAst)
+
+  // Special case: Boolean is transformed to integer in SQLite
+  if (SchemaAST.isBooleanKeyword(coreAst)) {
+    return SqliteDsl.boolean({ nullable })
+  }
+
+  // Get the encoded AST - what actually gets stored in SQLite
+  const encodedAst = Schema.encodedSchema(coreSchema).ast
+
+  // Check if the encoded type matches SQLite native types
+  if (SchemaAST.isStringKeyword(encodedAst)) {
+    return SqliteDsl.text({ schema: coreSchema, nullable })
   }
 
-  if (SchemaAST.isRefinement(ast)) {
-    return isPrimitiveAST(ast.from)
+  if (SchemaAST.isNumberKeyword(encodedAst)) {
+    // Special cases for integer columns
+    const id = SchemaAST.getIdentifierAnnotation(coreAst).pipe(Option.getOrElse(() => ''))
+    if (id === 'Int' || id === 'DateFromNumber') {
+      return SqliteDsl.integer({ schema: coreSchema, nullable })
+    }
+    return SqliteDsl.real({ schema: coreSchema, nullable })
   }
 
-  if (SchemaAST.isTransformation(ast)) {
-    return isPrimitiveAST(ast.to)
+  // Literals based on their type
+  if (SchemaAST.isLiteral(coreAst)) {
+    const value = coreAst.literal
+    if (typeof value === 'boolean') return SqliteDsl.boolean({ nullable })
   }
 
-  return false
+  // Literals based on their encoded type
+  if (SchemaAST.isLiteral(encodedAst)) {
+    const value = encodedAst.literal
+    if (typeof value === 'string') return SqliteDsl.text({ schema: coreSchema, nullable })
+    if (typeof value === 'number') {
+      // Check if the original schema is Int
+      const id = SchemaAST.getIdentifierAnnotation(coreAst).pipe(Option.getOrElse(() => ''))
+      if (id === 'Int') {
+        return SqliteDsl.integer({ schema: coreSchema, nullable })
+      }
+      return SqliteDsl.real({ schema: coreSchema, nullable })
+    }
+  }
+
+  // Everything else needs JSON encoding
+  return SqliteDsl.json({ schema: coreSchema, nullable })
+}
+
+const stripNullable = (ast: SchemaAST.AST): SchemaAST.AST => {
+  if (!SchemaAST.isUnion(ast)) return ast
+
+  // Find non-null/undefined type
+  const core = ast.types.find(
+    (type) => !(SchemaAST.isLiteral(type) && type.literal === null) && !SchemaAST.isUndefinedKeyword(type),
+  )
+
+  return core || ast
 }

package/src/schema/state/sqlite/db-schema/ast/sqlite.ts

@@ -1,4 +1,5 @@
-import { type Option, Schema } from '@livestore/utils/effect'
+import { omitUndefineds } from '@livestore/utils'
+import { type Option, Schema, SchemaAST } from '@livestore/utils/effect'
 
 import { hashCode } from '../hash.ts'
 
@@ -45,9 +46,7 @@ export const index = (
 ): Index => ({
   _tag: 'index',
   columns,
-  name,
-  unique,
-  primaryKey,
+  ...omitUndefineds({ name, unique, primaryKey }),
 })
 
 export type ForeignKey = {
@@ -85,7 +84,19 @@ export type DbSchema = {
 export const dbSchema = (tables: Table[]): DbSchema => ({ _tag: 'dbSchema', tables })
 
 /**
- * NOTE we're only including SQLite-relevant information in the hash (which excludes the schema mapping)
+ * Helper to detect if a column is a JSON column (has parseJson transformation)
+ */
+const isJsonColumn = (column: Column): boolean => {
+  if (column.type._tag !== 'text') return false
+
+  // Check if the schema AST is a parseJson transformation
+  const ast = column.schema.ast
+  return ast._tag === 'Transformation' && ast.annotations.schemaId === SchemaAST.ParseJsonSchemaId
+}
+
+/**
+ * NOTE we're now including JSON schema information for JSON columns
+ * to detect client document schema changes
  */
 export const hash = (obj: Table | Column | Index | ForeignKey | DbSchema): number =>
   hashCode(JSON.stringify(trimInfoForHasing(obj)))
@@ -101,7 +112,7 @@ const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema):
       }
     }
     case 'column': {
-      return {
+      const baseInfo: Record<string, any> = {
         _tag: 'column',
         name: obj.name,
         type: obj.type._tag,
@@ -110,6 +121,15 @@ const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema):
         autoIncrement: obj.autoIncrement,
         default: obj.default,
       }
+
+      // NEW: Include schema hash for JSON columns
+      // This ensures that changes to the JSON schema are detected
+      if (isJsonColumn(obj) && obj.schema) {
+        // Use Effect's Schema.hash for consistent hashing
+        baseInfo.jsonSchemaHash = Schema.hash(obj.schema)
+      }
+
+      return baseInfo
     }
     case 'index': {
       return {

package/src/schema/state/sqlite/db-schema/dsl/mod.ts

@@ -1,4 +1,5 @@
 import type { Nullable } from '@livestore/utils'
+import { omitUndefineds } from '@livestore/utils'
 import type { Option, Types } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
@@ -46,7 +47,7 @@ export const table = <TTableName extends string, TColumns extends Columns, TInde
     indexes: indexesToAst(indexes ?? []),
   }
 
-  return { name, columns, indexes, ast }
+  return { name, columns, ...omitUndefineds({ indexes }), ast }
 }
 
 export type AnyIfConstained<In, Out> = '__constrained' extends keyof In ? any : Out

package/src/schema/state/sqlite/mod.ts

@@ -14,6 +14,7 @@ export {
   ClientDocumentTableDefSymbol,
   type ClientDocumentTableOptions,
   clientDocument,
+  createOptimisticEventSchema,
   tableIsClientDocumentTable,
 } from './client-document-def.ts'
 export * from './column-annotations.ts'

package/src/schema/state/sqlite/query-builder/api.ts

@@ -3,7 +3,7 @@ import { type Option, Predicate, type Schema } from '@livestore/utils/effect'
 
 import type { SessionIdSymbol } from '../../../../adapter-types.ts'
 import type { SqlValue } from '../../../../util.ts'
-import type { ClientDocumentTableDef } from '../client-document-def.ts'
+import type { ClientDocumentTableDef, ClientDocumentTableDefSymbol } from '../client-document-def.ts'
 import type { SqliteDsl } from '../db-schema/mod.ts'
 import type { TableDefBase } from '../table-def.ts'
 
@@ -437,7 +437,12 @@ export namespace QueryBuilder {
 
 export namespace RowQuery {
   export type GetOrCreateOptions<TTableDef extends ClientDocumentTableDef.TraitAny> = {
-    default: Partial<TTableDef['Value']>
+    /**
+     * Default value to use instead of the default value from the table definition
+     */
+    default: TTableDef[ClientDocumentTableDefSymbol]['options']['partialSet'] extends false
+      ? TTableDef['Value']
+      : Partial<TTableDef['Value']>
   }
 
   // TODO get rid of this