@livestore/common 0.5.0-dev.0 → 0.5.0-dev.1

package/src/schema/state/sqlite/db-schema/ast/fingerprint.test.ts

@@ -0,0 +1,209 @@
+import { describe, expect, test } from 'vitest'
+
+import { Schema } from '@livestore/utils/effect'
+
+import { liveStoreStorageFormatVersion } from '../../../../../version.ts'
+import { makeState } from '../../mod.ts'
+import { REBUILD_META_TABLE } from '../../system-tables/state-tables.ts'
+import { SqliteDsl } from '../mod.ts'
+import { digestToFingerprint } from './fingerprint-digest.ts'
+import { fingerprint } from './fingerprint.ts'
+
+describe('SQLite storage fingerprints', () => {
+  test('pins the final fingerprint output', () => {
+    expect(fingerprint(makePhysicalTable().ast)).toBe('UnLwYzVBhwzPCK5q7TrPU3q2N0dTe8m_98bud8HsAs8')
+    expect(fingerprint(makeJsonTable('documents', representativeJsonSchema).ast)).toBe(
+      'kUzaurzV2rcXYLljHOZ64TDR9c_RebqD7y5ZUM_nPBE',
+    )
+    expect(fingerprint(makeJsonTable('documents', normalizedCheckIdsJsonSchema).ast)).toBe(
+      'I0XYuOUSYqWwG5Opxbc1u-TNNE3pFQCHzpBZ_ChYGaA',
+    )
+    expect(makeState({ tables: [], materializers: {} }).sqlite.hash).toBe('o8mmnRDnhXhkgDpBybM3FK7S-PkaLEj_pKcnbzgg3TU')
+  })
+
+  test('isolates pre-completion-marker state databases without changing the hash algorithm', () => {
+    const state = makeState({ tables: [], materializers: {} })
+    const legacyTables = [...state.sqlite.tables.values()]
+      .filter((table) => table.sqliteDef.name !== REBUILD_META_TABLE)
+      .map((table) => table.sqliteDef.ast)
+    const legacyHash = fingerprint({ _tag: 'dbSchema', tables: legacyTables })
+    expect(legacyHash).toBe('H5Uktp6Ffp84WDj_RWPnfDnaQEu_E61AMl7ieZ9Zn8k')
+    expect(state.sqlite.hash).not.toBe(legacyHash)
+  })
+
+  test('matches the standard SHA-256 vector', () => {
+    expect(digestToFingerprint(new TextEncoder().encode('abc'))).toBe('ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0')
+  })
+
+  test('keeps the full digest inside the Cloudflare VFS filename limit', () => {
+    const value = fingerprint(makePhysicalTable().ast)
+
+    expect(value).toHaveLength(43)
+    expect(`state${value}@${liveStoreStorageFormatVersion}.db`.length).toBeLessThanOrEqual(56)
+  })
+
+  test('is independent of table and index declaration order', () => {
+    const first = makeJsonTable('first', Schema.Struct({ value: Schema.String }))
+    const second = makeJsonTable('second', Schema.Struct({ value: Schema.Finite }))
+
+    expect(fingerprint({ _tag: 'dbSchema', tables: [first.ast, second.ast] })).toBe(
+      fingerprint({ _tag: 'dbSchema', tables: [second.ast, first.ast] }),
+    )
+
+    const table = SqliteDsl.table(
+      'indexed',
+      {
+        id: SqliteDsl.text({ primaryKey: true }),
+        age: SqliteDsl.integer(),
+      },
+      [
+        { name: 'indexed_by_age', columns: ['age'], isUnique: false },
+        { name: 'indexed_by_id', columns: ['id'], isUnique: true },
+      ],
+    ).ast
+    expect(fingerprint({ ...table, indexes: table.indexes.toReversed() })).toBe(fingerprint(table))
+  })
+
+  test('ignores Effect metadata that does not affect accepted encoded JSON', () => {
+    const plain = makeJsonTable('documents', Schema.Struct({ value: Schema.String }))
+    const annotated = makeJsonTable(
+      'documents',
+      Schema.Struct({ value: Schema.String }).annotate({ description: 'documentation only', title: 'Document' }),
+    )
+    const readonlyArray = makeJsonTable('documents', Schema.Array(Schema.String))
+    const mutableArray = makeJsonTable('documents', Schema.mutable(Schema.Array(Schema.String)))
+
+    expect(fingerprint(plain.ast)).toBe(fingerprint(annotated.ast))
+    expect(fingerprint(readonlyArray.ast)).toBe(fingerprint(mutableArray.ast))
+  })
+
+  test('changes when represented JSON checks change', () => {
+    const minimumTwo = makeJsonTable('documents', Schema.Struct({ value: Schema.String.check(Schema.isMinLength(2)) }))
+    const minimumThree = makeJsonTable(
+      'documents',
+      Schema.Struct({ value: Schema.String.check(Schema.isMinLength(3)) }),
+    )
+
+    expect(fingerprint(minimumTwo.ast)).not.toBe(fingerprint(minimumThree.ast))
+  })
+
+  test('distinguishes codecs with the same encoded primitive', () => {
+    const date = makeJsonTable('documents', Schema.Struct({ value: Schema.DateFromString }))
+    const string = makeJsonTable('documents', Schema.Struct({ value: Schema.String }))
+
+    expect(fingerprint(date.ast)).not.toBe(fingerprint(string.ast))
+  })
+
+  test('handles recursive JSON schemas deterministically', () => {
+    const first = makeJsonTable('trees', makeTreeSchema())
+    const second = makeJsonTable('trees', makeTreeSchema())
+
+    expect(fingerprint(first.ast)).toBe(fingerprint(second.ast))
+  })
+
+  test('supports opaque Effect schemas without persistence annotations', () => {
+    const OpaqueString = Schema.declare<string>((value): value is string => typeof value === 'string')
+    const table = makeJsonTable('opaque_documents', Schema.Struct({ value: OpaqueString }))
+
+    expect(() => fingerprint(table.ast)).not.toThrow()
+  })
+
+  test('keeps application-owned representation payload fields', () => {
+    const first = Schema.declare<string>((value): value is string => typeof value === 'string', {
+      representation: { id: 'example/opaque', payload: { annotations: 'first', isMutable: true } },
+    })
+    const second = Schema.declare<string>((value): value is string => typeof value === 'string', {
+      representation: { id: 'example/opaque', payload: { annotations: 'second', isMutable: true } },
+    })
+
+    expect(fingerprint(makeJsonTable('documents', first).ast)).not.toBe(
+      fingerprint(makeJsonTable('documents', second).ast),
+    )
+  })
+
+  test('tracks JSON codecs passed through the generic text column API', () => {
+    const first = SqliteDsl.table('documents', {
+      value: SqliteDsl.text({ schema: Schema.fromJsonString(Schema.Struct({ value: Schema.String })) }),
+    })
+    const second = SqliteDsl.table('documents', {
+      value: SqliteDsl.text({ schema: Schema.fromJsonString(Schema.Struct({ value: Schema.Finite })) }),
+    })
+
+    expect(fingerprint(first.ast)).not.toBe(fingerprint(second.ast))
+  })
+
+  test('gives equivalent JSON DSL forms the same fingerprint', () => {
+    const valueSchema = Schema.Struct({ value: Schema.String })
+    const specialized = SqliteDsl.table('documents', {
+      value: SqliteDsl.json({ schema: valueSchema }),
+    })
+    const generic = SqliteDsl.table('documents', {
+      value: SqliteDsl.text({ schema: Schema.fromJsonString(valueSchema) }),
+    })
+
+    expect(fingerprint(specialized.ast)).toBe(fingerprint(generic.ast))
+  })
+
+  test('classifies physical defaults without hashing function source text', () => {
+    const none = SqliteDsl.text()
+    const firstThunk = SqliteDsl.text({ default: () => 'first' })
+    const secondThunk = SqliteDsl.text({ default: () => 'second' })
+    const literal = SqliteDsl.text({ default: 'first' })
+    const toTable = (definition: SqliteDsl.ColumnDefinition.Any) =>
+      SqliteDsl.table('defaults', { value: definition }).ast
+
+    expect(fingerprint(toTable(firstThunk))).toBe(fingerprint(toTable(secondThunk)))
+    expect(fingerprint(toTable(firstThunk))).not.toBe(fingerprint(toTable(none)))
+    expect(fingerprint(toTable(literal))).not.toBe(fingerprint(toTable(firstThunk)))
+  })
+})
+
+const makePhysicalTable = () =>
+  SqliteDsl.table(
+    'users',
+    {
+      id: SqliteDsl.text({ primaryKey: true }),
+      age: SqliteDsl.integer({ nullable: true }),
+    },
+    [{ name: 'users_by_age', columns: ['age'], isUnique: false }],
+  )
+
+const makeJsonTable = (name: string, schema: Schema.Codec<unknown, unknown>) =>
+  SqliteDsl.table(name, {
+    id: SqliteDsl.text({ primaryKey: true }),
+    value: SqliteDsl.json({ schema }),
+  })
+
+const representativeJsonSchema = Schema.Struct({
+  title: Schema.String.check(Schema.isMinLength(1)),
+  tags: Schema.Array(Schema.String),
+  status: Schema.Union([Schema.Literal('draft'), Schema.Literal('published')]),
+  publishedAt: Schema.NullOr(Schema.DateFromString),
+  metadata: Schema.Record(Schema.String, Schema.Json),
+})
+
+/** Uses the JSON-compatible checks whose representation ids the fingerprint normalizes. */
+const normalizedCheckIdsJsonSchema = Schema.Struct({
+  slug: Schema.String.check(
+    Schema.isBetweenLength(1, 64),
+    Schema.isStartingWith('a'),
+    Schema.isEndingWith('z'),
+    Schema.isIncluding('-'),
+  ),
+  tags: Schema.Array(Schema.String).check(Schema.isBetweenLength(0, 10)),
+  initial: Schema.Char,
+  labels: Schema.Record(Schema.String, Schema.String).check(Schema.isBetweenProperties(0, 5)),
+})
+
+interface TreeNode {
+  readonly value: string
+  readonly children: ReadonlyArray<TreeNode>
+}
+
+const makeTreeSchema = (): Schema.Codec<TreeNode> => {
+  const TreeNode: Schema.Codec<TreeNode> = Schema.Struct({
+    value: Schema.String,
+    children: Schema.Array(Schema.suspend(() => TreeNode)),
+  })
+  return TreeNode
+}

package/src/schema/state/sqlite/db-schema/ast/fingerprint.ts

@@ -0,0 +1,225 @@
+import { casesHandled, textEncodeToArrayBuffer } from '@livestore/utils'
+import { Schema, SchemaAST, SchemaRepresentation } from '@livestore/utils/effect'
+
+import { isDefaultThunk, isSqlDefaultValue } from '../dsl/field-defs.ts'
+import { hasJsonStringEncoding } from '../has-json-string-encoding.ts'
+import { digestToFingerprint } from './fingerprint-digest.ts'
+import type { Column, ColumnType, DbSchema, Index, Table } from './sqlite.ts'
+
+type FingerprintInput = Table | DbSchema
+
+export const fingerprint = (input: FingerprintInput): string => digestToFingerprint(toCanonicalBytes(input))
+
+type CanonicalValue = Schema.Json
+
+type SchemaDescriptorCache = WeakMap<Schema.Top, CanonicalValue>
+
+const domain = 'livestore/state-storage-fingerprint'
+
+const toCanonicalBytes = (input: FingerprintInput): Uint8Array<ArrayBuffer> =>
+  textEncodeToArrayBuffer(JSON.stringify(toRootDescriptor(input)))
+
+const toRootDescriptor = (input: FingerprintInput): CanonicalValue => {
+  // Limit schema memoization to one fingerprint operation so descriptors remain collectible.
+  const cache: SchemaDescriptorCache = new WeakMap()
+  return [domain, toDescriptor(input, cache)]
+}
+
+const toDescriptor = (input: FingerprintInput, cache: SchemaDescriptorCache): CanonicalValue => {
+  switch (input._tag) {
+    case 'table':
+      return tableDescriptor(input, cache)
+    case 'dbSchema':
+      return dbSchemaDescriptor(input, cache)
+    default:
+      return casesHandled(input)
+  }
+}
+
+const columnDescriptor = (column: Column, cache: SchemaDescriptorCache): CanonicalValue => {
+  const {
+    _tag: _tagIgnored,
+    name,
+    type,
+    primaryKey,
+    nullable,
+    autoIncrement,
+    default: defaultValue,
+    schema,
+    ...unhandled
+  } = column
+  assertNoUnhandledKeys(unhandled)
+
+  return [
+    'column',
+    name,
+    columnTypeDescriptor(type),
+    primaryKey,
+    nullable,
+    autoIncrement,
+    defaultDescriptor(defaultValue, schema),
+    hasJsonStringEncoding(schema.ast) === true ? schemaDescriptor(schema, cache) : null,
+  ]
+}
+
+const columnTypeDescriptor = (type: ColumnType.ColumnType): string => {
+  const { _tag, ...unhandled } = type
+  assertNoUnhandledKeys(unhandled)
+  switch (_tag) {
+    case 'text':
+    case 'null':
+    case 'real':
+    case 'integer':
+    case 'blob':
+      return _tag
+    default:
+      return casesHandled(_tag)
+  }
+}
+
+const indexDescriptor = (index: Index): CanonicalValue => {
+  const { _tag: _tagIgnored, name, unique, primaryKey, columns, ...unhandled } = index
+  assertNoUnhandledKeys(unhandled)
+  return ['index', name ?? null, unique ?? false, primaryKey ?? false, columns]
+}
+
+const tableDescriptor = (table: Table, cache: SchemaDescriptorCache): CanonicalValue => {
+  const { _tag: _tagIgnored, name, columns, indexes, ...unhandled } = table
+  assertNoUnhandledKeys(unhandled)
+  return [
+    'table',
+    name,
+    columns.map((column) => columnDescriptor(column, cache)),
+    sortCanonicalValues(indexes.map(indexDescriptor)),
+  ]
+}
+
+const dbSchemaDescriptor = (dbSchema: DbSchema, cache: SchemaDescriptorCache): CanonicalValue => {
+  const { _tag: _tagIgnored, tables, ...unhandled } = dbSchema
+  assertNoUnhandledKeys(unhandled)
+  return ['database', sortByCanonicalKey(tables, (table) => table.name).map((table) => tableDescriptor(table, cache))]
+}
+
+const defaultDescriptor = (value: Column['default'], schema: Column['schema']): CanonicalValue => {
+  if (value._tag === 'None') return ['none']
+  if (isDefaultThunk(value.value) === true) return ['runtime-default']
+  if (value.value === null) return ['value', null]
+  if (isSqlDefaultValue(value.value) === true) return ['sql', value.value.sql]
+
+  return ['value', canonicalValue(Schema.encodeSync(schema)(value.value))]
+}
+
+const schemaDescriptor = (schema: Schema.Top, cache: SchemaDescriptorCache): CanonicalValue => {
+  const cached = cache.get(schema)
+  if (cached !== undefined) return cached
+
+  // Both ends matter: the encoded side describes stored JSON while the type side
+  // distinguishes codecs such as DateFromString from an unconstrained string.
+  const document = SchemaRepresentation.toRepresentations([schema.ast, SchemaAST.toType(schema.ast)], {
+    referencePolicy: () => undefined,
+  })
+  const descriptor: CanonicalValue = ['schema', schemaRepresentationDescriptor(document)]
+  cache.set(schema, descriptor)
+  return descriptor
+}
+
+const schemaRepresentationDescriptor = (value: unknown, parentKey?: string): CanonicalValue => {
+  // Representation payloads are application-owned JSON. Keys such as `annotations`
+  // inside them are data, not Effect metadata.
+  if (parentKey === 'payload') return canonicalValue(value)
+
+  if (Array.isArray(value) === true) return value.map((item) => schemaRepresentationDescriptor(item))
+
+  if (value !== null && typeof value === 'object' && value instanceof Uint8Array === false) {
+    const sourceEntries = Object.entries(value).map(([key, entryValue]): [string, unknown] =>
+      parentKey === 'representation' && key === 'id' && typeof entryValue === 'string'
+        ? [key, renamedEffectCheckIds.get(entryValue) ?? entryValue]
+        : [key, entryValue],
+    )
+    const isUnion = sourceEntries.some(([key, entryValue]) => key === '_tag' && entryValue === 'Union')
+    // Effect rc.113 removed this representation field because unions only support `anyOf`.
+    // Restore the old default so dependency upgrades preserve existing state fingerprints.
+    if (isUnion === true && sourceEntries.some(([key]) => key === 'mode') === false) {
+      sourceEntries.push(['mode', 'anyOf'])
+    }
+    const entries = sortByCanonicalKey(
+      sourceEntries.filter(([key]) => effectIgnoredFields.has(key) === false),
+      ([key]) => key,
+    ).map(([key, entryValue]) => [key, schemaRepresentationDescriptor(entryValue, key)] as const)
+    return Object.fromEntries(entries)
+  }
+
+  return canonicalValue(value)
+}
+
+const effectIgnoredFields = new Set(['annotations', 'isMutable'])
+
+/** Maps renamed Effect check ids to the ids existing fingerprints were computed from, so the rename does not rebuild state. */
+const renamedEffectCheckIds = new Map([
+  ['effect/schema/isBetweenLength', 'effect/schema/isLengthBetween'],
+  ['effect/schema/isStartingWith', 'effect/schema/isStartsWith'],
+  ['effect/schema/isEndingWith', 'effect/schema/isEndsWith'],
+  ['effect/schema/isIncluding', 'effect/schema/isIncludes'],
+  ['effect/schema/isBetweenSize', 'effect/schema/isSizeBetween'],
+  ['effect/schema/isBetweenProperties', 'effect/schema/isPropertiesLengthBetween'],
+])
+
+const canonicalValue = (value: unknown): CanonicalValue => {
+  if (Array.isArray(value) === true) return value.map(canonicalValue)
+
+  if (value !== null && typeof value === 'object' && value instanceof Uint8Array === false) {
+    const entries = sortByCanonicalKey(Object.entries(value), ([key]) => key).map(
+      ([key, entryValue]) => [key, canonicalValue(entryValue)] as const,
+    )
+    return Object.fromEntries(entries)
+  }
+
+  if (value === null || typeof value === 'string' || typeof value === 'boolean') return value
+
+  if (typeof value === 'number') {
+    if (Number.isNaN(value) === true) return specialValue('number', 'NaN')
+    if (value === Number.POSITIVE_INFINITY) return specialValue('number', 'Infinity')
+    if (value === Number.NEGATIVE_INFINITY) return specialValue('number', '-Infinity')
+    if (Object.is(value, -0) === true) return specialValue('number', '-0')
+    return value
+  }
+
+  if (typeof value === 'bigint') return specialValue('bigint', value.toString(10))
+  if (value === undefined) return specialValue('undefined')
+
+  if (typeof value === 'symbol') {
+    const symbolKey = Symbol.keyFor(value)
+    if (symbolKey === undefined) throw new Error('Local symbols cannot be represented in a persistent fingerprint')
+    return specialValue('symbol', symbolKey)
+  }
+
+  if (typeof value === 'function') throw new Error('Functions cannot be represented in a persistent fingerprint')
+  if (value instanceof Uint8Array) return specialValue('bytes', bytesToHex(value))
+
+  throw new Error('Unsupported canonical value')
+}
+
+const specialValue = (type: string, value?: string): CanonicalValue => ({
+  $livestore$type: value === undefined ? [type] : [type, value],
+})
+
+const bytesToHex = (bytes: Uint8Array): string => {
+  let output = ''
+  for (const byte of bytes) output += byte.toString(16).padStart(2, '0')
+  return output
+}
+
+/** Avoid repeatedly serializing deep descriptors from inside an O(n log n) sort comparator. */
+const sortCanonicalValues = (values: ReadonlyArray<CanonicalValue>): ReadonlyArray<CanonicalValue> =>
+  values
+    .map((value) => ({ value, key: JSON.stringify(value) }))
+    .toSorted((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0))
+    .map(({ value }) => value)
+
+const sortByCanonicalKey = <A>(values: ReadonlyArray<A>, key: (value: A) => CanonicalValue): ReadonlyArray<A> =>
+  values
+    .map((value) => ({ value, key: JSON.stringify(key(value)) }))
+    .toSorted((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0))
+    .map(({ value }) => value)
+
+const assertNoUnhandledKeys = (_unhandled: Record<PropertyKey, never>): void => {}

package/src/schema/state/sqlite/db-schema/ast/sqlite.ts

@@ -1,7 +1,5 @@
 import { omitUndefineds } from '@livestore/utils'
-import { type Option, Schema, SchemaAST } from '@livestore/utils/effect'
-
-import { hashCode } from '../hash.ts'
+import { type Option, Schema } from '@livestore/utils/effect'
 
 export namespace ColumnType {
   export type ColumnType = Text | Null | Real | Integer | Blob
@@ -83,92 +81,5 @@ export type DbSchema = {
 
 export const dbSchema = (tables: Table[]): DbSchema => ({ _tag: 'dbSchema', tables })
 
-/**
- * Helper to detect if a column is a JSON column (encoded as a JSON string)
- */
-const isJsonColumn = (column: Column): boolean => {
-  if (column.type._tag !== 'text') return false
-
-  return hasJsonStringEncoding(column.schema.ast)
-}
-
-/**
- * `SchemaTransformation.fromJsonString` is a factory since Effect rc.109, so encoding links can no
- * longer be matched by identity. `Schema.fromJsonString` annotates its encoded side as
- * `application/json`, which identifies the link regardless of the reviver/replacer/space options.
- */
-const hasJsonStringEncoding = (ast: SchemaAST.AST): boolean => {
-  if (ast.encoding?.some((link) => link.to.annotations?.contentMediaType === 'application/json') === true) {
-    return true
-  }
-  return SchemaAST.isUnion(ast) === true && ast.types.some(hasJsonStringEncoding)
-}
-
-/**
- * NOTE we're now including JSON schema information for JSON columns
- * to detect client document schema changes
- */
-export const hash = (obj: Table | Column | Index | ForeignKey | DbSchema): number =>
-  hashCode(JSON.stringify(trimInfoForHasing(obj)))
-
-const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema): Record<string, any> => {
-  switch (obj._tag) {
-    case 'table': {
-      return {
-        _tag: 'table',
-        name: obj.name,
-        columns: obj.columns.map((column) => trimInfoForHasing(column)),
-        indexes: obj.indexes.map((index) => trimInfoForHasing(index)),
-      }
-    }
-    case 'column': {
-      const baseInfo: Record<string, any> = {
-        _tag: 'column',
-        name: obj.name,
-        type: obj.type._tag,
-        primaryKey: obj.primaryKey,
-        nullable: obj.nullable,
-        autoIncrement: obj.autoIncrement,
-        default: obj.default,
-      }
-
-      // NEW: Include schema hash for JSON columns
-      // This ensures that changes to the JSON schema are detected
-      if (isJsonColumn(obj) === true && obj.schema !== undefined) {
-        // Use Effect's Schema.hash for consistent hashing
-        baseInfo.jsonSchemaHash = Schema.hash(obj.schema)
-      }
-
-      return baseInfo
-    }
-    case 'index': {
-      return {
-        _tag: 'index',
-        columns: obj.columns,
-        name: obj.name,
-        unique: obj.unique,
-        primaryKey: obj.primaryKey,
-      }
-    }
-    case 'foreignKey': {
-      return {
-        _tag: 'foreignKey',
-        references: obj.references,
-        key: obj.key,
-        columns: obj.columns,
-      }
-    }
-    case 'dbSchema': {
-      return {
-        _tag: 'dbSchema',
-        tables: obj.tables.map(trimInfoForHasing),
-      }
-    }
-    default: {
-      throw new Error(`Unreachable: ${String(obj)}`)
-    }
-  }
-}
-
 export const structSchemaForTable = (tableDef: Table) =>
   Schema.Struct(Object.fromEntries(tableDef.columns.map((column) => [column.name, column.schema])))

package/src/schema/state/sqlite/db-schema/dsl/field-defs.ts

@@ -267,6 +267,7 @@ export const defaultSchemaForColumnType = <TColumnType extends FieldColumnType>(
     }
     case 'real': {
       // oxlint-disable-next-line typescript-eslint(no-unsafe-type-assertion) -- switch-based type narrowing for column type to schema mapping; each case is correct for its branch
+      // @effect-diagnostics-next-line schemaNumber:off -- SQLite REAL permits Infinity/NaN; SqliteReal is the documented domain carve-out
       return SqliteReal as Schema.Codec<T>
     }
     case 'blob': {

package/src/schema/state/sqlite/db-schema/has-json-string-encoding.ts

@@ -0,0 +1,9 @@
+import { SchemaAST } from '@livestore/utils/effect'
+
+const getContentMediaType = SchemaAST.resolveAt<string>('contentMediaType')
+
+export const hasJsonStringEncoding = (ast: SchemaAST.AST): boolean => hasJsonMediaType(SchemaAST.toEncoded(ast))
+
+const hasJsonMediaType = (ast: SchemaAST.AST): boolean =>
+  getContentMediaType(ast) === 'application/json' ||
+  (SchemaAST.isUnion(ast) === true && ast.types.some(hasJsonMediaType))

package/src/schema/state/sqlite/mod.ts

@@ -4,6 +4,7 @@ import type { MigrationOptions } from '../../../adapter-types.ts'
 import type { Materializer } from '../../EventDef/mod.ts'
 import type { InternalState } from '../../schema.ts'
 import { ClientDocumentTableDefSymbol, tableIsClientDocumentTable } from './client-document-def.ts'
+import { fingerprint } from './db-schema/ast/fingerprint.ts'
 import { SqliteAst } from './db-schema/mod.ts'
 import { stateSystemTables } from './system-tables/state-tables.ts'
 import type { TableDef, TableDefBase } from './table-def.ts'
@@ -55,7 +56,7 @@ export const makeState = <TStateInput extends InputState>(inputSchema: TStateInp
     }
   }
 
-  const hash = SqliteAst.hash({
+  const hash = fingerprint({
     _tag: 'dbSchema',
     tables: [...tables.values()].map((_) => _.sqliteDef.ast),
   })

package/src/schema/state/sqlite/query-builder/astToSql.ts

@@ -4,6 +4,7 @@ import { Schema, SchemaAST, Struct } from '@livestore/utils/effect'
 import { SessionIdSymbol } from '../../../../session-id-symbol.ts'
 import type { SqlValue } from '../../../../util.ts'
 import type { State } from '../../../mod.ts'
+import { hasJsonStringEncoding } from '../db-schema/has-json-string-encoding.ts'
 import type { QueryBuilderAst } from './api.ts'
 
 /**
@@ -47,14 +48,6 @@ const getJsonArrayElementSchema = (colSchema: Schema.Top): Schema.Top | undefine
   return undefined
 }
 
-/**
- * `SchemaTransformation.fromJsonString` is a factory since Effect rc.109, so encoding links can no
- * longer be matched by identity. `Schema.fromJsonString` annotates its encoded side as
- * `application/json`, which identifies the link regardless of the reviver/replacer/space options.
- */
-const hasJsonStringEncoding = (ast: SchemaAST.AST): boolean =>
-  ast.encoding?.some((link) => link.to.annotations?.contentMediaType === 'application/json') === true
-
 /**
  * Encodes a JSON array element to the representation returned by SQLite's json_each().
  * Objects/arrays are stringified so they match json_each's TEXT representation.

package/src/schema/state/sqlite/system-tables/state-tables.ts

@@ -6,7 +6,7 @@ import { table } from '../table-def.ts'
  * STATE DATABASE SYSTEM TABLES
  *
  * ⚠️  SAFE TO CHANGE: State tables are automatically rebuilt from eventlog when schema changes.
- * No need to bump `liveStoreStorageFormatVersion` (uses hash-based migration via SqliteAst.hash()).
+ * No need to bump `liveStoreStorageFormatVersion` because state tables use fingerprint-based migration.
  */
 
 export const SCHEMA_META_TABLE = '__livestore_schema'
@@ -18,7 +18,7 @@ export const schemaMetaTable = table({
   name: SCHEMA_META_TABLE,
   columns: {
     tableName: SqliteDsl.text({ primaryKey: true }),
-    schemaHash: SqliteDsl.integer({ nullable: false }),
+    schemaHash: SqliteDsl.text({ nullable: false }),
     /** ISO date format */
     updatedAt: SqliteDsl.text({ nullable: false }),
   },
@@ -64,6 +64,16 @@ export const stateHeadMetaTable = table({
 
 export type StateHeadMetaRow = typeof stateHeadMetaTable.Type
 
+export const REBUILD_META_TABLE = '__livestore_rebuild'
+
+/** Only a completed replay and successful migration hooks may insert the marker row. */
+export const rebuildMetaTable = table({
+  name: REBUILD_META_TABLE,
+  columns: {
+    id: SqliteDsl.integer({ primaryKey: true }),
+  },
+})
+
 /**
  * Table which stores SQLite changeset blobs which is used for rolling back
  * read-model state during rebasing.
@@ -104,6 +114,7 @@ export const stateSystemTables = [
   schemaEventDefsMetaTable,
   stateHeadMetaTable,
   sessionChangesetMetaTable,
+  rebuildMetaTable,
 ] as const
 
 export const isStateSystemTable = (tableName: string) => stateSystemTables.some((_) => _.sqliteDef.name === tableName)