@livestore/common 0.4.0-dev.12 → 0.4.0-dev.14

package/src/schema/state/sqlite/column-def.test.ts

@@ -381,6 +381,19 @@ describe('getColumnDefForSchema', () => {
       expect((table.rowSchema as any).fields.count.toString()).toBe('Int | null')
     })
 
+    it('should treat unions of string literals as text columns without JSON parsing', () => {
+      const schema = Schema.Struct({
+        id: Schema.String,
+        status: Schema.Literal('idle', 'running', 'stopped'),
+      })
+
+      const table = State.SQLite.table({ name: 'timers', schema })
+
+      expect(table.sqliteDef.columns.status.columnType).toBe('text')
+      expect(table.sqliteDef.columns.status.schema.toString()).toBe('"idle" | "running" | "stopped"')
+      expect((table.rowSchema as any).fields.status.toString()).toBe('"idle" | "running" | "stopped"')
+    })
+
     it('should handle Schema.NullOr with complex types', () => {
       const schema = Schema.Struct({
         data: Schema.NullOr(Schema.Struct({ value: Schema.Number })),

package/src/schema/state/sqlite/column-def.ts

@@ -185,6 +185,10 @@ const getColumnForSchema = (schema: Schema.Schema.AnyNoContext, nullable = false
     if (typeof value === 'boolean') return SqliteDsl.boolean({ nullable })
   }
 
+  if (isLiteralUnionOf(coreAst, (value): value is string => typeof value === 'string')) {
+    return SqliteDsl.text({ schema: coreSchema, nullable })
+  }
+
   // Literals based on their encoded type
   if (SchemaAST.isLiteral(encodedAst)) {
     const value = encodedAst.literal
@@ -199,6 +203,10 @@ const getColumnForSchema = (schema: Schema.Schema.AnyNoContext, nullable = false
     }
   }
 
+  if (isLiteralUnionOf(encodedAst, (value): value is string => typeof value === 'string')) {
+    return SqliteDsl.text({ schema: coreSchema, nullable })
+  }
+
   // Everything else needs JSON encoding
   return SqliteDsl.json({ schema: coreSchema, nullable })
 }
@@ -221,3 +229,11 @@ const stripNullable = (ast: SchemaAST.AST): SchemaAST.AST => {
 
   return SchemaAST.Union.make(coreTypes, ast.annotations)
 }
+
+const isLiteralUnionOf = <T extends SchemaAST.LiteralValue>(
+  ast: SchemaAST.AST,
+  predicate: (value: SchemaAST.LiteralValue) => value is T,
+): ast is SchemaAST.Union & { types: ReadonlyArray<SchemaAST.Literal & { literal: T }> } =>
+  SchemaAST.isUnion(ast) &&
+  ast.types.length > 0 &&
+  ast.types.every((type) => SchemaAST.isLiteral(type) && predicate(type.literal))

package/src/schema/state/sqlite/mod.ts

@@ -5,7 +5,7 @@ import type { Materializer } from '../../EventDef.ts'
 import type { InternalState } from '../../schema.ts'
 import { ClientDocumentTableDefSymbol, tableIsClientDocumentTable } from './client-document-def.ts'
 import { SqliteAst } from './db-schema/mod.ts'
-import { stateSystemTables } from './system-tables.ts'
+import { stateSystemTables } from './system-tables/state-tables.ts'
 import type { TableDef, TableDefBase } from './table-def.ts'
 
 export * from '../../EventDef.ts'

package/src/schema/state/sqlite/system-tables/eventlog-tables.ts

@@ -0,0 +1,64 @@
+import { Schema } from '@livestore/utils/effect'
+
+import * as EventSequenceNumber from '../../../EventSequenceNumber.ts'
+import { SqliteDsl } from '../db-schema/mod.ts'
+import { table } from '../table-def.ts'
+
+/**
+ * EVENTLOG DATABASE SYSTEM TABLES
+ *
+ * ⚠️  CRITICAL: NEVER modify eventlog schemas without bumping `liveStoreStorageFormatVersion`!
+ * Eventlog is the source of truth - schema changes cause permanent data loss.
+ *
+ * TODO: Implement proper eventlog versioning system to prevent accidental data loss
+ */
+
+export const EVENTLOG_META_TABLE = 'eventlog'
+
+/**
+ * Main client-side event log storing all events (global and local/rebased).
+ */
+export const eventlogMetaTable = table({
+  name: EVENTLOG_META_TABLE,
+  columns: {
+    // TODO Adjust modeling so a global event never needs a client id component
+    seqNumGlobal: SqliteDsl.integer({ primaryKey: true, schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    seqNumClient: SqliteDsl.integer({ primaryKey: true, schema: EventSequenceNumber.ClientEventSequenceNumber }),
+    seqNumRebaseGeneration: SqliteDsl.integer({ primaryKey: true }),
+    parentSeqNumGlobal: SqliteDsl.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    parentSeqNumClient: SqliteDsl.integer({ schema: EventSequenceNumber.ClientEventSequenceNumber }),
+    parentSeqNumRebaseGeneration: SqliteDsl.integer({}),
+    /** Event definition name */
+    name: SqliteDsl.text({}),
+    argsJson: SqliteDsl.text({ schema: Schema.parseJson(Schema.Any) }),
+    clientId: SqliteDsl.text({}),
+    sessionId: SqliteDsl.text({}),
+    schemaHash: SqliteDsl.integer({}),
+    syncMetadataJson: SqliteDsl.text({ schema: Schema.parseJson(Schema.Option(Schema.JsonValue)) }),
+  },
+  indexes: [
+    { columns: ['seqNumGlobal'], name: 'idx_eventlog_seqNumGlobal' },
+    { columns: ['seqNumGlobal', 'seqNumClient', 'seqNumRebaseGeneration'], name: 'idx_eventlog_seqNum' },
+  ],
+})
+
+export type EventlogMetaRow = typeof eventlogMetaTable.Type
+
+export const SYNC_STATUS_TABLE = '__livestore_sync_status'
+
+/**
+ * Tracks sync status including the remote head position and backend identity.
+ */
+// TODO support sync backend identity (to detect if sync backend changes)
+export const syncStatusTable = table({
+  name: SYNC_STATUS_TABLE,
+  columns: {
+    head: SqliteDsl.integer({ primaryKey: true }),
+    // Null means the sync backend is not yet connected and we haven't yet seen a backend ID
+    backendId: SqliteDsl.text({ nullable: true }),
+  },
+})
+
+export type SyncStatusRow = typeof syncStatusTable.Type
+
+export const eventlogSystemTables = [eventlogMetaTable, syncStatusTable] as const

package/src/schema/state/sqlite/system-tables/mod.ts

@@ -0,0 +1,2 @@
+export * from './eventlog-tables.ts'
+export * from './state-tables.ts'

package/src/schema/state/sqlite/system-tables/state-tables.ts

@@ -0,0 +1,69 @@
+import * as EventSequenceNumber from '../../../EventSequenceNumber.ts'
+import { SqliteDsl } from '../db-schema/mod.ts'
+import { table } from '../table-def.ts'
+
+/**
+ * STATE DATABASE SYSTEM TABLES
+ *
+ * ⚠️  SAFE TO CHANGE: State tables are automatically rebuilt from eventlog when schema changes.
+ * No need to bump `liveStoreStorageFormatVersion` (uses hash-based migration via SqliteAst.hash()).
+ */
+
+export const SCHEMA_META_TABLE = '__livestore_schema'
+
+/**
+ * Tracks schema hashes for user-defined tables to detect schema changes.
+ */
+export const schemaMetaTable = table({
+  name: SCHEMA_META_TABLE,
+  columns: {
+    tableName: SqliteDsl.text({ primaryKey: true }),
+    schemaHash: SqliteDsl.integer({ nullable: false }),
+    /** ISO date format */
+    updatedAt: SqliteDsl.text({ nullable: false }),
+  },
+})
+
+export type SchemaMetaRow = typeof schemaMetaTable.Type
+
+export const SCHEMA_EVENT_DEFS_META_TABLE = '__livestore_schema_event_defs'
+
+/**
+ * Tracks schema hashes for event definitions to detect event schema changes.
+ */
+export const schemaEventDefsMetaTable = table({
+  name: SCHEMA_EVENT_DEFS_META_TABLE,
+  columns: {
+    eventName: SqliteDsl.text({ primaryKey: true }),
+    schemaHash: SqliteDsl.integer({ nullable: false }),
+    /** ISO date format */
+    updatedAt: SqliteDsl.text({ nullable: false }),
+  },
+})
+
+export type SchemaEventDefsMetaRow = typeof schemaEventDefsMetaTable.Type
+
+/**
+ * Table which stores SQLite changeset blobs which is used for rolling back
+ * read-model state during rebasing.
+ */
+export const SESSION_CHANGESET_META_TABLE = '__livestore_session_changeset'
+
+export const sessionChangesetMetaTable = table({
+  name: SESSION_CHANGESET_META_TABLE,
+  columns: {
+    // TODO bring back primary key
+    seqNumGlobal: SqliteDsl.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
+    seqNumClient: SqliteDsl.integer({ schema: EventSequenceNumber.ClientEventSequenceNumber }),
+    seqNumRebaseGeneration: SqliteDsl.integer({}),
+    changeset: SqliteDsl.blob({ nullable: true }),
+    debug: SqliteDsl.json({ nullable: true }),
+  },
+  indexes: [{ columns: ['seqNumGlobal', 'seqNumClient'], name: 'idx_session_changeset_id' }],
+})
+
+export type SessionChangesetMetaRow = typeof sessionChangesetMetaTable.Type
+
+export const stateSystemTables = [schemaMetaTable, schemaEventDefsMetaTable, sessionChangesetMetaTable] as const
+
+export const isStateSystemTable = (tableName: string) => stateSystemTables.some((_) => _.sqliteDef.name === tableName)

package/src/schema-management/migrations.ts

@@ -1,3 +1,34 @@
+/**
+ * AUTOMATIC HASH-BASED SCHEMA MIGRATIONS
+ *
+ * This module implements automatic schema versioning using hash-based change detection.
+ *
+ * ⚠️  CRITICAL DISTINCTION:
+ * - STATE TABLES (safe to modify): Changes trigger rematerialization from eventlog
+ * - EVENTLOG TABLES (NEVER modify): Changes cause data loss - need manual versioning!
+ *
+ * How it works:
+ * 1. Each table's schema is hashed using SqliteAst.hash()
+ * 2. Hashes are stored in SCHEMA_META_TABLE after successful migrations
+ * 3. On app start, current schema hashes are compared with stored hashes
+ * 4. Mismatches trigger migrations:
+ *    - State tables: Recreated and repopulated from eventlog (safe, no data loss)
+ *    - Eventlog tables: Uses 'create-if-not-exists' (UNSAFE - causes data loss!)
+ *
+ * State Table Changes (SAFE):
+ * - User-defined tables are rebuilt from eventlog
+ * - System tables (schemaMetaTable, etc.) are recreated
+ * - Data preserved through rematerializeFromEventlog()
+ *
+ * Eventlog Table Changes (UNSAFE):
+ * - eventlogMetaTable, syncStatusTable changes cause "soft reset"
+ * - Old table becomes inaccessible (but remains in DB)
+ * - No automatic migration - effectively data loss
+ * - TODO: Implement proper EVENTLOG_PERSISTENCE_FORMAT_VERSION system
+ *
+ * See system-tables/state-tables.ts and system-tables/eventlog-tables.ts for detailed documentation on each table type.
+ */
+
 import { memoizeByStringifyArgs } from '@livestore/utils'
 import { Effect } from '@livestore/utils/effect'
 
@@ -7,14 +38,14 @@ import type { UnexpectedError } from '../errors.ts'
 import type { LiveStoreSchema } from '../schema/mod.ts'
 import { makeColumnSpec } from '../schema/state/sqlite/column-spec.ts'
 import { SqliteAst } from '../schema/state/sqlite/db-schema/mod.ts'
-import type { SchemaEventDefsMetaRow, SchemaMetaRow } from '../schema/state/sqlite/system-tables.ts'
+import type { SchemaEventDefsMetaRow, SchemaMetaRow } from '../schema/state/sqlite/system-tables/state-tables.ts'
 import {
   isStateSystemTable,
   SCHEMA_EVENT_DEFS_META_TABLE,
   SCHEMA_META_TABLE,
   schemaEventDefsMetaTable,
   stateSystemTables,
-} from '../schema/state/sqlite/system-tables.ts'
+} from '../schema/state/sqlite/system-tables/state-tables.ts'
 import { sql } from '../util.ts'
 import type { SchemaManager } from './common.ts'
 import { dbExecute, dbSelect } from './common.ts'

package/src/sync/syncstate.test.ts

@@ -441,6 +441,48 @@ describe('syncstate', () => {
             expectEventArraysEqual(result.newEvents, [e1_1, e1_2, e1_3])
             expectEventArraysEqual(result.confirmedEvents, [])
           })
+
+          // Leaders can choose to ignore client-only events while still returning them for broadcast.
+          // Ensure pending/local head only reflects events that must be pushed upstream.
+          it('keeps pending empty when pushing only client-only events that are being ignored', () => {
+            const syncState = new SyncState.SyncState({
+              pending: [],
+              upstreamHead: EventSequenceNumber.ROOT,
+              localHead: EventSequenceNumber.ROOT,
+            })
+
+            const result = merge({
+              syncState,
+              payload: SyncState.PayloadLocalPush.make({ newEvents: [e0_1] }),
+              ignoreClientEvents: true,
+            })
+
+            expectAdvance(result)
+            expectEventArraysEqual(result.newSyncState.pending, [])
+            expect(result.newSyncState.upstreamHead).toMatchObject(EventSequenceNumber.ROOT)
+            expect(result.newSyncState.localHead).toMatchObject(EventSequenceNumber.ROOT)
+            expectEventArraysEqual(result.newEvents, [e0_1])
+          })
+
+          it('appends only upstream-bound events to pending when ignoring client-only pushes', () => {
+            const syncState = new SyncState.SyncState({
+              pending: [],
+              upstreamHead: EventSequenceNumber.ROOT,
+              localHead: EventSequenceNumber.ROOT,
+            })
+
+            const result = merge({
+              syncState,
+              payload: SyncState.PayloadLocalPush.make({ newEvents: [e0_1, e1_0] }),
+              ignoreClientEvents: true,
+            })
+
+            expectAdvance(result)
+            expectEventArraysEqual(result.newSyncState.pending, [e1_0])
+            expect(result.newSyncState.upstreamHead).toMatchObject(EventSequenceNumber.ROOT)
+            expect(result.newSyncState.localHead).toMatchObject(e1_0.seqNum)
+            expectEventArraysEqual(result.newEvents, [e0_1, e1_0])
+          })
         })
 
         describe('reject', () => {

package/src/sync/syncstate.ts

@@ -15,7 +15,7 @@ import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
  *                 +------------------------+
  *               ▼                       ▼
  *        Upstream Head             Local Head
- *             (1,0)     (1,1), (1,2), (2,0)
+ *              e1        e1.1, e1.2, e2
  * ```
  *
  * **Pending Events**: Events awaiting acknowledgment from the upstream.
@@ -31,7 +31,7 @@ import * as LiveStoreEvent from '../schema/LiveStoreEvent.ts'
  * Invariants:
  * 1. **Chain Continuity**: Each event must reference its immediate parent.
  * 2. **Head Ordering**: Upstream Head ≤ Local Head.
- * 3. **Event number sequence**: Must follow the pattern (1,0)→(1,1)→(1,2)→(2,0).
+ * 3. **Event number sequence**: Must follow the pattern e1→e1.1→e1.2→e2.
  *
  * A few further notes to help form an intuition:
  * - The goal is to keep the pending events as small as possible (i.e. to have synced with the next upstream node)
@@ -407,13 +407,20 @@ export const merge = ({
           }),
         )
       } else {
+        const nonClientEvents = ignoreClientEvents
+          ? payload.newEvents.filter((event) => !isClientEvent(event))
+          : payload.newEvents
+        const newPending = [...syncState.pending, ...nonClientEvents]
+        const newLocalHead =
+          newPending.at(-1)?.seqNum ?? EventSequenceNumber.max(syncState.localHead, syncState.upstreamHead)
+
         return validateMergeResult(
           MergeResultAdvance.make({
             _tag: 'advance',
             newSyncState: new SyncState({
-              pending: [...syncState.pending, ...payload.newEvents],
+              pending: newPending,
               upstreamHead: syncState.upstreamHead,
-              localHead: payload.newEvents.at(-1)!.seqNum,
+              localHead: newLocalHead,
             }),
             newEvents: payload.newEvents,
             confirmedEvents: [],

package/src/version.ts

@@ -2,13 +2,23 @@
 // import packageJson from '../package.json' with { type: 'json' }
 // export const liveStoreVersion = packageJson.version
 
-export const liveStoreVersion = '0.4.0-dev.12' as const
+export const liveStoreVersion = '0.4.0-dev.14' as const
 
 /**
- * This version number is incremented whenever the internal storage format changes in a breaking way.
- * Whenever this version changes, LiveStore will start with fresh database files. Old database files are not deleted.
+ * CRITICAL: Increment this version whenever you modify client-side EVENTLOG table schemas.
  *
- * While LiveStore is in beta, this might happen more frequently.
- * In the future, LiveStore will provide a migration path for older database files to avoid the impression of data loss.
+ * Used to generate database file names (e.g., `eventlog@6.db`, `state@6.db`) across all client adapters.
+ *
+ * Bump required when:
+ * - Modifying eventlog system tables (eventlogMetaTable, syncStatusTable) in schema/state/sqlite/system-tables/eventlog-tables.ts
+ * - Changing columns, types, constraints, or indexes in eventlog tables
+ *
+ * Bump NOT required when:
+ * - Modifying STATE table schemas (auto-migrated via hash-based detection and rebuilt from eventlog)
+ * - Changing query patterns or client-side implementation details
+ *
+ * ⚠️  CRITICAL: Eventlog changes without bumping this version cause permanent data loss!
+ *
+ * Impact: Version changes trigger a "soft reset" - old data becomes inaccessible but remains on disk.
  */
 export const liveStoreStorageFormatVersion = 6

package/src/schema/state/sqlite/system-tables.ts

@@ -1,106 +0,0 @@
-import { Schema } from '@livestore/utils/effect'
-
-import * as EventSequenceNumber from '../../EventSequenceNumber.ts'
-import { SqliteDsl } from './db-schema/mod.ts'
-import { table } from './table-def.ts'
-
-/// State DB
-
-export const SCHEMA_META_TABLE = '__livestore_schema'
-
-export const schemaMetaTable = table({
-  name: SCHEMA_META_TABLE,
-  columns: {
-    tableName: SqliteDsl.text({ primaryKey: true }),
-    schemaHash: SqliteDsl.integer({ nullable: false }),
-    /** ISO date format */
-    updatedAt: SqliteDsl.text({ nullable: false }),
-  },
-})
-
-export type SchemaMetaRow = typeof schemaMetaTable.Type
-
-export const SCHEMA_EVENT_DEFS_META_TABLE = '__livestore_schema_event_defs'
-
-export const schemaEventDefsMetaTable = table({
-  name: SCHEMA_EVENT_DEFS_META_TABLE,
-  columns: {
-    eventName: SqliteDsl.text({ primaryKey: true }),
-    schemaHash: SqliteDsl.integer({ nullable: false }),
-    /** ISO date format */
-    updatedAt: SqliteDsl.text({ nullable: false }),
-  },
-})
-
-export type SchemaEventDefsMetaRow = typeof schemaEventDefsMetaTable.Type
-
-/**
- * Table which stores SQLite changeset blobs which is used for rolling back
- * read-model state during rebasing.
- */
-export const SESSION_CHANGESET_META_TABLE = '__livestore_session_changeset'
-
-export const sessionChangesetMetaTable = table({
-  name: SESSION_CHANGESET_META_TABLE,
-  columns: {
-    // TODO bring back primary key
-    seqNumGlobal: SqliteDsl.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
-    seqNumClient: SqliteDsl.integer({ schema: EventSequenceNumber.ClientEventSequenceNumber }),
-    seqNumRebaseGeneration: SqliteDsl.integer({}),
-    changeset: SqliteDsl.blob({ nullable: true }),
-    debug: SqliteDsl.json({ nullable: true }),
-  },
-  indexes: [{ columns: ['seqNumGlobal', 'seqNumClient'], name: 'idx_session_changeset_id' }],
-})
-
-export type SessionChangesetMetaRow = typeof sessionChangesetMetaTable.Type
-
-export const stateSystemTables = [schemaMetaTable, schemaEventDefsMetaTable, sessionChangesetMetaTable] as const
-
-export const isStateSystemTable = (tableName: string) => stateSystemTables.some((_) => _.sqliteDef.name === tableName)
-
-/// Eventlog DB
-
-export const EVENTLOG_META_TABLE = 'eventlog'
-
-export const eventlogMetaTable = table({
-  name: EVENTLOG_META_TABLE,
-  columns: {
-    // TODO Adjust modeling so a global event never needs a client id component
-    seqNumGlobal: SqliteDsl.integer({ primaryKey: true, schema: EventSequenceNumber.GlobalEventSequenceNumber }),
-    seqNumClient: SqliteDsl.integer({ primaryKey: true, schema: EventSequenceNumber.ClientEventSequenceNumber }),
-    seqNumRebaseGeneration: SqliteDsl.integer({ primaryKey: true }),
-    parentSeqNumGlobal: SqliteDsl.integer({ schema: EventSequenceNumber.GlobalEventSequenceNumber }),
-    parentSeqNumClient: SqliteDsl.integer({ schema: EventSequenceNumber.ClientEventSequenceNumber }),
-    parentSeqNumRebaseGeneration: SqliteDsl.integer({}),
-    /** Event definition name */
-    name: SqliteDsl.text({}),
-    argsJson: SqliteDsl.text({ schema: Schema.parseJson(Schema.Any) }),
-    clientId: SqliteDsl.text({}),
-    sessionId: SqliteDsl.text({}),
-    schemaHash: SqliteDsl.integer({}),
-    syncMetadataJson: SqliteDsl.text({ schema: Schema.parseJson(Schema.Option(Schema.JsonValue)) }),
-  },
-  indexes: [
-    { columns: ['seqNumGlobal'], name: 'idx_eventlog_seqNumGlobal' },
-    { columns: ['seqNumGlobal', 'seqNumClient', 'seqNumRebaseGeneration'], name: 'idx_eventlog_seqNum' },
-  ],
-})
-
-export type EventlogMetaRow = typeof eventlogMetaTable.Type
-
-export const SYNC_STATUS_TABLE = '__livestore_sync_status'
-
-// TODO support sync backend identity (to detect if sync backend changes)
-export const syncStatusTable = table({
-  name: SYNC_STATUS_TABLE,
-  columns: {
-    head: SqliteDsl.integer({ primaryKey: true }),
-    // Null means the sync backend is not yet connected and we haven't yet seen a backend ID
-    backendId: SqliteDsl.text({ nullable: true }),
-  },
-})
-
-export type SyncStatusRow = typeof syncStatusTable.Type
-
-export const eventlogSystemTables = [eventlogMetaTable, syncStatusTable] as const