@livestore/common 0.4.0-dev.0 → 0.4.0-dev.10

package/src/schema/state/sqlite/column-def.ts

@@ -0,0 +1,215 @@
+import { shouldNeverHappen } from '@livestore/utils'
+import { Option, Schema, SchemaAST } from '@livestore/utils/effect'
+
+import { AutoIncrement, ColumnType, Default, PrimaryKeyId, Unique } from './column-annotations.ts'
+import { SqliteDsl } from './db-schema/mod.ts'
+
+/**
+ * Maps a schema to a SQLite column definition, respecting column annotations.
+ *
+ * Note: When used with schema-based table definitions, optional fields (| undefined)
+ * are transformed to nullable fields (| null) to match SQLite's NULL semantics.
+ * Fields with both null and undefined will emit a warning as this is a lossy conversion.
+ */
+export const getColumnDefForSchema = (
+  schema: Schema.Schema.AnyNoContext,
+  propertySignature?: SchemaAST.PropertySignature,
+  forceNullable = false,
+): SqliteDsl.ColumnDefinition.Any => {
+  const ast = schema.ast
+
+  // Extract annotations
+  const getAnnotation = <T>(annotationId: symbol): Option.Option<T> =>
+    propertySignature
+      ? hasPropertyAnnotation<T>(propertySignature, annotationId)
+      : SchemaAST.getAnnotation<T>(annotationId)(ast)
+
+  const columnType = SchemaAST.getAnnotation<SqliteDsl.FieldColumnType>(ColumnType)(ast)
+
+  // Check if schema has null (e.g., Schema.NullOr) or undefined or if it's forced nullable (optional field)
+  const isNullable = forceNullable || hasNull(ast) || hasUndefined(ast)
+
+  // Get base column definition with nullable flag
+  const baseColumn = Option.isSome(columnType)
+    ? getColumnForType(columnType.value, isNullable)
+    : getColumnForSchema(schema, isNullable)
+
+  // Apply annotations
+  const primaryKey = getAnnotation<boolean>(PrimaryKeyId).pipe(Option.getOrElse(() => false))
+  const autoIncrement = getAnnotation<boolean>(AutoIncrement).pipe(Option.getOrElse(() => false))
+  const defaultValue = getAnnotation<unknown>(Default)
+
+  return {
+    ...baseColumn,
+    ...(primaryKey && { primaryKey: true }),
+    ...(autoIncrement && { autoIncrement: true }),
+    ...(Option.isSome(defaultValue) && { default: Option.some(defaultValue.value) }),
+  }
+}
+
+const hasPropertyAnnotation = <T>(
+  propertySignature: SchemaAST.PropertySignature,
+  annotationId: symbol,
+): Option.Option<T> => {
+  if ('annotations' in propertySignature && propertySignature.annotations) {
+    const annotation = SchemaAST.getAnnotation<T>(annotationId)(propertySignature as any)
+    if (Option.isSome(annotation)) return annotation
+  }
+  return SchemaAST.getAnnotation<T>(annotationId)(propertySignature.type)
+}
+
+/**
+ * Maps schema property signatures to SQLite column definitions.
+ * Optional fields (| undefined) become nullable columns (| null).
+ */
+export const schemaFieldsToColumns = (
+  propertySignatures: ReadonlyArray<SchemaAST.PropertySignature>,
+): { columns: SqliteDsl.Columns; uniqueColumns: string[] } => {
+  const columns: SqliteDsl.Columns = {}
+  const uniqueColumns: string[] = []
+
+  for (const prop of propertySignatures) {
+    if (typeof prop.name !== 'string') continue
+
+    const fieldSchema = Schema.make(prop.type)
+
+    // Warn about lossy conversion for fields with both null and undefined
+    if (prop.isOptional) {
+      const { hasNull, hasUndefined } = checkNullUndefined(fieldSchema.ast)
+      if (hasNull && hasUndefined) {
+        console.warn(`Field '${prop.name}' has both null and undefined - treating | undefined as | null`)
+      }
+    }
+
+    // Get column definition - pass nullable flag for optional fields
+    const columnDef = getColumnDefForSchema(fieldSchema, prop, prop.isOptional)
+
+    // Check for primary key and unique annotations
+    const hasPrimaryKey = hasPropertyAnnotation<boolean>(prop, PrimaryKeyId).pipe(Option.getOrElse(() => false))
+    const hasUnique = hasPropertyAnnotation<boolean>(prop, Unique).pipe(Option.getOrElse(() => false))
+
+    // Build final column
+    columns[prop.name] = {
+      ...columnDef,
+      ...(hasPrimaryKey && { primaryKey: true }),
+    }
+
+    // Validate primary key + nullable
+    const column = columns[prop.name]
+    if (column?.primaryKey && column.nullable) {
+      throw new Error('Primary key columns cannot be nullable')
+    }
+
+    if (hasUnique) uniqueColumns.push(prop.name)
+  }
+
+  return { columns, uniqueColumns }
+}
+
+const checkNullUndefined = (ast: SchemaAST.AST): { hasNull: boolean; hasUndefined: boolean } => {
+  let hasNull = false
+  let hasUndefined = false
+
+  const visit = (type: SchemaAST.AST): void => {
+    if (SchemaAST.isUndefinedKeyword(type)) hasUndefined = true
+    else if (SchemaAST.isLiteral(type) && type.literal === null) hasNull = true
+    else if (SchemaAST.isUnion(type)) type.types.forEach(visit)
+  }
+
+  visit(ast)
+  return { hasNull, hasUndefined }
+}
+
+const hasNull = (ast: SchemaAST.AST): boolean => {
+  if (SchemaAST.isLiteral(ast) && ast.literal === null) return true
+  if (SchemaAST.isUnion(ast)) {
+    return ast.types.some((type) => hasNull(type))
+  }
+  return false
+}
+
+const hasUndefined = (ast: SchemaAST.AST): boolean => {
+  if (SchemaAST.isUndefinedKeyword(ast)) return true
+  if (SchemaAST.isUnion(ast)) {
+    return ast.types.some((type) => hasUndefined(type))
+  }
+  return false
+}
+
+const getColumnForType = (columnType: string, nullable = false): SqliteDsl.ColumnDefinition.Any => {
+  switch (columnType) {
+    case 'text':
+      return SqliteDsl.text({ nullable })
+    case 'integer':
+      return SqliteDsl.integer({ nullable })
+    case 'real':
+      return SqliteDsl.real({ nullable })
+    case 'blob':
+      return SqliteDsl.blob({ nullable })
+    default:
+      return shouldNeverHappen(`Unsupported column type: ${columnType}`)
+  }
+}
+
+const getColumnForSchema = (schema: Schema.Schema.AnyNoContext, nullable = false): SqliteDsl.ColumnDefinition.Any => {
+  const ast = schema.ast
+  // Strip nullable wrapper to get core type
+  const coreAst = stripNullable(ast)
+  const coreSchema = stripNullable(ast) === ast ? schema : Schema.make(coreAst)
+
+  // Special case: Boolean is transformed to integer in SQLite
+  if (SchemaAST.isBooleanKeyword(coreAst)) {
+    return SqliteDsl.boolean({ nullable })
+  }
+
+  // Get the encoded AST - what actually gets stored in SQLite
+  const encodedAst = Schema.encodedSchema(coreSchema).ast
+
+  // Check if the encoded type matches SQLite native types
+  if (SchemaAST.isStringKeyword(encodedAst)) {
+    return SqliteDsl.text({ schema: coreSchema, nullable })
+  }
+
+  if (SchemaAST.isNumberKeyword(encodedAst)) {
+    // Special cases for integer columns
+    const id = SchemaAST.getIdentifierAnnotation(coreAst).pipe(Option.getOrElse(() => ''))
+    if (id === 'Int' || id === 'DateFromNumber') {
+      return SqliteDsl.integer({ schema: coreSchema, nullable })
+    }
+    return SqliteDsl.real({ schema: coreSchema, nullable })
+  }
+
+  // Literals based on their type
+  if (SchemaAST.isLiteral(coreAst)) {
+    const value = coreAst.literal
+    if (typeof value === 'boolean') return SqliteDsl.boolean({ nullable })
+  }
+
+  // Literals based on their encoded type
+  if (SchemaAST.isLiteral(encodedAst)) {
+    const value = encodedAst.literal
+    if (typeof value === 'string') return SqliteDsl.text({ schema: coreSchema, nullable })
+    if (typeof value === 'number') {
+      // Check if the original schema is Int
+      const id = SchemaAST.getIdentifierAnnotation(coreAst).pipe(Option.getOrElse(() => ''))
+      if (id === 'Int') {
+        return SqliteDsl.integer({ schema: coreSchema, nullable })
+      }
+      return SqliteDsl.real({ schema: coreSchema, nullable })
+    }
+  }
+
+  // Everything else needs JSON encoding
+  return SqliteDsl.json({ schema: coreSchema, nullable })
+}
+
+const stripNullable = (ast: SchemaAST.AST): SchemaAST.AST => {
+  if (!SchemaAST.isUnion(ast)) return ast
+
+  // Find non-null/undefined type
+  const core = ast.types.find(
+    (type) => !(SchemaAST.isLiteral(type) && type.literal === null) && !SchemaAST.isUndefinedKeyword(type),
+  )
+
+  return core || ast
+}

package/src/schema/state/sqlite/db-schema/ast/sqlite.ts

@@ -1,4 +1,5 @@
-import { type Option, Schema } from '@livestore/utils/effect'
+import { omitUndefineds } from '@livestore/utils'
+import { type Option, Schema, SchemaAST } from '@livestore/utils/effect'
 
 import { hashCode } from '../hash.ts'
 
@@ -45,9 +46,7 @@ export const index = (
 ): Index => ({
   _tag: 'index',
   columns,
-  name,
-  unique,
-  primaryKey,
+  ...omitUndefineds({ name, unique, primaryKey }),
 })
 
 export type ForeignKey = {
@@ -85,7 +84,19 @@ export type DbSchema = {
 export const dbSchema = (tables: Table[]): DbSchema => ({ _tag: 'dbSchema', tables })
 
 /**
- * NOTE we're only including SQLite-relevant information in the hash (which excludes the schema mapping)
+ * Helper to detect if a column is a JSON column (has parseJson transformation)
+ */
+const isJsonColumn = (column: Column): boolean => {
+  if (column.type._tag !== 'text') return false
+
+  // Check if the schema AST is a parseJson transformation
+  const ast = column.schema.ast
+  return ast._tag === 'Transformation' && ast.annotations.schemaId === SchemaAST.ParseJsonSchemaId
+}
+
+/**
+ * NOTE we're now including JSON schema information for JSON columns
+ * to detect client document schema changes
  */
 export const hash = (obj: Table | Column | Index | ForeignKey | DbSchema): number =>
   hashCode(JSON.stringify(trimInfoForHasing(obj)))
@@ -101,7 +112,7 @@ const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema):
       }
     }
     case 'column': {
-      return {
+      const baseInfo: Record<string, any> = {
         _tag: 'column',
         name: obj.name,
         type: obj.type._tag,
@@ -110,6 +121,15 @@ const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema):
         autoIncrement: obj.autoIncrement,
         default: obj.default,
       }
+
+      // NEW: Include schema hash for JSON columns
+      // This ensures that changes to the JSON schema are detected
+      if (isJsonColumn(obj) && obj.schema) {
+        // Use Effect's Schema.hash for consistent hashing
+        baseInfo.jsonSchemaHash = Schema.hash(obj.schema)
+      }
+
+      return baseInfo
     }
     case 'index': {
       return {

package/src/schema/state/sqlite/db-schema/dsl/mod.ts

@@ -1,4 +1,5 @@
 import type { Nullable } from '@livestore/utils'
+import { omitUndefineds } from '@livestore/utils'
 import type { Option, Types } from '@livestore/utils/effect'
 import { Schema } from '@livestore/utils/effect'
 
@@ -46,7 +47,7 @@ export const table = <TTableName extends string, TColumns extends Columns, TInde
     indexes: indexesToAst(indexes ?? []),
   }
 
-  return { name, columns, indexes, ast }
+  return { name, columns, ...omitUndefineds({ indexes }), ast }
 }
 
 export type AnyIfConstained<In, Out> = '__constrained' extends keyof In ? any : Out

package/src/schema/state/sqlite/mod.ts

@@ -14,6 +14,7 @@ export {
   ClientDocumentTableDefSymbol,
   type ClientDocumentTableOptions,
   clientDocument,
+  createOptimisticEventSchema,
   tableIsClientDocumentTable,
 } from './client-document-def.ts'
 export * from './column-annotations.ts'

package/src/schema/state/sqlite/query-builder/api.ts

@@ -3,7 +3,7 @@ import { type Option, Predicate, type Schema } from '@livestore/utils/effect'
 
 import type { SessionIdSymbol } from '../../../../adapter-types.ts'
 import type { SqlValue } from '../../../../util.ts'
-import type { ClientDocumentTableDef } from '../client-document-def.ts'
+import type { ClientDocumentTableDef, ClientDocumentTableDefSymbol } from '../client-document-def.ts'
 import type { SqliteDsl } from '../db-schema/mod.ts'
 import type { TableDefBase } from '../table-def.ts'
 
@@ -437,7 +437,12 @@ export namespace QueryBuilder {
 
 export namespace RowQuery {
   export type GetOrCreateOptions<TTableDef extends ClientDocumentTableDef.TraitAny> = {
-    default: Partial<TTableDef['Value']>
+    /**
+     * Default value to use instead of the default value from the table definition
+     */
+    default: TTableDef[ClientDocumentTableDefSymbol]['options']['partialSet'] extends false
+      ? TTableDef['Value']
+      : Partial<TTableDef['Value']>
   }
 
   // TODO get rid of this

package/src/schema/state/sqlite/query-builder/impl.test.ts

@@ -409,9 +409,7 @@ describe('query builder', () => {
     })
 
     it('should handle INSERT queries with undefined values', () => {
-      expect(
-        dump(db.todos.insert({ id: '123', text: 'Buy milk', status: 'active', completed: undefined })),
-      ).toMatchInlineSnapshot(`
+      expect(dump(db.todos.insert({ id: '123', text: 'Buy milk', status: 'active' }))).toMatchInlineSnapshot(`
         {
           "bindValues": [
             "123",
@@ -478,9 +476,7 @@ describe('query builder', () => {
     })
 
     it('should handle UPDATE queries with undefined values', () => {
-      expect(
-        dump(db.todos.update({ status: undefined, text: 'some text' }).where({ id: '123' })),
-      ).toMatchInlineSnapshot(`
+      expect(dump(db.todos.update({ text: 'some text' }).where({ id: '123' }))).toMatchInlineSnapshot(`
         {
           "bindValues": [
             "some text",
@@ -622,6 +618,191 @@ describe('query builder', () => {
         }
       `)
     })
+
+    it('should handle where().delete() - preserving where clauses', () => {
+      expect(dump(db.todos.where({ status: 'completed' }).delete())).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+          ],
+          "query": "DELETE FROM 'todos' WHERE status = ?",
+          "schema": "number",
+        }
+      `)
+
+      // Multiple where clauses
+      expect(dump(db.todos.where({ status: 'completed' }).where({ deletedAt: null }).delete())).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+          ],
+          "query": "DELETE FROM 'todos' WHERE status = ? AND deletedAt IS NULL",
+          "schema": "number",
+        }
+      `)
+    })
+
+    it('should handle where().update() - preserving where clauses', () => {
+      expect(dump(db.todos.where({ id: '123' }).update({ status: 'completed' }))).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+            "123",
+          ],
+          "query": "UPDATE 'todos' SET status = ? WHERE id = ?",
+          "schema": "number",
+        }
+      `)
+
+      // Multiple where clauses
+      expect(
+        dump(db.todos.where({ id: '123' }).where({ deletedAt: null }).update({ status: 'completed' })),
+      ).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "completed",
+            "123",
+          ],
+          "query": "UPDATE 'todos' SET status = ? WHERE id = ? AND deletedAt IS NULL",
+          "schema": "number",
+        }
+      `)
+    })
+
+    it('should have equivalent behavior for both delete patterns', () => {
+      const pattern1 = dump(db.todos.where({ status: 'completed', id: '123' }).delete())
+      const pattern2 = dump(db.todos.delete().where({ status: 'completed', id: '123' }))
+
+      expect(pattern1).toEqual(pattern2)
+    })
+
+    it('should have equivalent behavior for both update patterns', () => {
+      const pattern1 = dump(db.todos.where({ id: '123' }).update({ status: 'completed', text: 'Updated' }))
+      const pattern2 = dump(db.todos.update({ status: 'completed', text: 'Updated' }).where({ id: '123' }))
+
+      expect(pattern1).toEqual(pattern2)
+    })
+  })
+
+  describe('schema transforms', () => {
+    const Flat = Schema.Struct({
+      id: Schema.String.pipe(State.SQLite.withPrimaryKey),
+      contactFirstName: Schema.String,
+      contactLastName: Schema.String,
+      contactEmail: Schema.String.pipe(State.SQLite.withUnique),
+    })
+
+    const Nested = Schema.transform(
+      Flat,
+      Schema.Struct({
+        id: Schema.String,
+        contact: Schema.Struct({
+          firstName: Schema.String,
+          lastName: Schema.String,
+          email: Schema.String,
+        }),
+      }),
+      {
+        decode: ({ id, contactFirstName, contactLastName, contactEmail }) => ({
+          id,
+          contact: {
+            firstName: contactFirstName,
+            lastName: contactLastName,
+            email: contactEmail,
+          },
+        }),
+        encode: ({ id, contact }) => ({
+          id,
+          contactFirstName: contact.firstName,
+          contactLastName: contact.lastName,
+          contactEmail: contact.email,
+        }),
+      },
+    )
+
+    const makeContactsTable = () =>
+      State.SQLite.table({
+        name: 'contacts',
+        schema: Nested,
+        // schema: Flat,
+      })
+
+    it('exposes flattened insert type while schema type is nested', () => {
+      const contactsTable = makeContactsTable()
+
+      type InsertInput = Parameters<(typeof contactsTable)['insert']>[0]
+      type NestedType = Schema.Schema.Type<typeof Nested>
+
+      type Assert<T extends true> = T
+
+      type InsertKeys = keyof InsertInput
+      type NestedKeys = keyof NestedType
+
+      type _InsertHasFlattenedColumns = Assert<
+        'contactFirstName' extends InsertKeys
+          ? 'contactLastName' extends InsertKeys
+            ? 'contactEmail' extends InsertKeys
+              ? true
+              : false
+            : false
+          : false
+      >
+
+      type _InsertDoesNotExposeNested = Assert<Extract<'contact', InsertKeys> extends never ? true : false>
+
+      type _SchemaTypeIsNested = Assert<'contact' extends NestedKeys ? true : false>
+
+      void contactsTable
+    })
+
+    it('fails to encode nested inserts because flat columns are required', () => {
+      const contactsTable = makeContactsTable()
+
+      expect(
+        contactsTable
+          // TODO in the future we should use decoded types here instead of encoded
+          .insert({
+            id: 'person-1',
+            contactFirstName: 'Ada',
+            contactLastName: 'Lovelace',
+            contactEmail: 'ada@example.com',
+          })
+          .asSql(),
+      ).toMatchInlineSnapshot(`
+        {
+          "bindValues": [
+            "person-1",
+            "Ada",
+            "Lovelace",
+            "ada@example.com",
+          ],
+          "query": "INSERT INTO 'contacts' (id, contactFirstName, contactLastName, contactEmail) VALUES (?, ?, ?, ?)",
+          "usedTables": Set {
+            "contacts",
+          },
+        }
+      `)
+    })
+
+    it('fails to encode nested inserts because flat columns are required', () => {
+      const contactsTable = makeContactsTable()
+
+      expect(() =>
+        contactsTable
+          .insert({
+            id: 'person-1',
+            // @ts-expect-error
+            contact: {
+              firstName: 'Ada',
+              lastName: 'Lovelace',
+              email: 'ada@example.com',
+            },
+          })
+          .asSql(),
+      ).toThrowErrorMatchingInlineSnapshot(`
+        [ParseError: contacts\n└─ ["contactFirstName"]\n   └─ is missing]
+      `)
+    })
   })
 })
 

package/src/schema/state/sqlite/query-builder/impl.ts

@@ -219,21 +219,27 @@ export const makeQueryBuilder = <TResult, TTableDef extends TableDefBase>(
     update: (values) => {
       const filteredValues = Object.fromEntries(Object.entries(values).filter(([, value]) => value !== undefined))
 
+      // Preserve where clauses if coming from a SelectQuery
+      const whereClause = ast._tag === 'SelectQuery' ? ast.where : []
+
       return makeQueryBuilder(tableDef, {
         _tag: 'UpdateQuery',
         tableDef,
         values: filteredValues,
-        where: [],
+        where: whereClause,
         returning: undefined,
         resultSchema: Schema.Void,
       }) as any
     },
 
     delete: () => {
+      // Preserve where clauses if coming from a SelectQuery
+      const whereClause = ast._tag === 'SelectQuery' ? ast.where : []
+
       return makeQueryBuilder(tableDef, {
         _tag: 'DeleteQuery',
         tableDef,
-        where: [],
+        where: whereClause,
         returning: undefined,
         resultSchema: Schema.Void,
       }) as any

package/src/schema/state/sqlite/system-tables.ts

@@ -96,6 +96,8 @@ export const syncStatusTable = table({
   name: SYNC_STATUS_TABLE,
   columns: {
     head: SqliteDsl.integer({ primaryKey: true }),
+    // Null means the sync backend is not yet connected and we haven't yet seen a backend ID
+    backendId: SqliteDsl.text({ nullable: true }),
   },
 })