@livestore/common 0.4.0 → 0.5.0-dev.1

package/src/schema/state/sqlite/db-schema/ast/fingerprint.ts

@@ -0,0 +1,225 @@
+import { casesHandled, textEncodeToArrayBuffer } from '@livestore/utils'
+import { Schema, SchemaAST, SchemaRepresentation } from '@livestore/utils/effect'
+
+import { isDefaultThunk, isSqlDefaultValue } from '../dsl/field-defs.ts'
+import { hasJsonStringEncoding } from '../has-json-string-encoding.ts'
+import { digestToFingerprint } from './fingerprint-digest.ts'
+import type { Column, ColumnType, DbSchema, Index, Table } from './sqlite.ts'
+
+type FingerprintInput = Table | DbSchema
+
+export const fingerprint = (input: FingerprintInput): string => digestToFingerprint(toCanonicalBytes(input))
+
+type CanonicalValue = Schema.Json
+
+type SchemaDescriptorCache = WeakMap<Schema.Top, CanonicalValue>
+
+const domain = 'livestore/state-storage-fingerprint'
+
+const toCanonicalBytes = (input: FingerprintInput): Uint8Array<ArrayBuffer> =>
+  textEncodeToArrayBuffer(JSON.stringify(toRootDescriptor(input)))
+
+const toRootDescriptor = (input: FingerprintInput): CanonicalValue => {
+  // Limit schema memoization to one fingerprint operation so descriptors remain collectible.
+  const cache: SchemaDescriptorCache = new WeakMap()
+  return [domain, toDescriptor(input, cache)]
+}
+
+const toDescriptor = (input: FingerprintInput, cache: SchemaDescriptorCache): CanonicalValue => {
+  switch (input._tag) {
+    case 'table':
+      return tableDescriptor(input, cache)
+    case 'dbSchema':
+      return dbSchemaDescriptor(input, cache)
+    default:
+      return casesHandled(input)
+  }
+}
+
+const columnDescriptor = (column: Column, cache: SchemaDescriptorCache): CanonicalValue => {
+  const {
+    _tag: _tagIgnored,
+    name,
+    type,
+    primaryKey,
+    nullable,
+    autoIncrement,
+    default: defaultValue,
+    schema,
+    ...unhandled
+  } = column
+  assertNoUnhandledKeys(unhandled)
+
+  return [
+    'column',
+    name,
+    columnTypeDescriptor(type),
+    primaryKey,
+    nullable,
+    autoIncrement,
+    defaultDescriptor(defaultValue, schema),
+    hasJsonStringEncoding(schema.ast) === true ? schemaDescriptor(schema, cache) : null,
+  ]
+}
+
+const columnTypeDescriptor = (type: ColumnType.ColumnType): string => {
+  const { _tag, ...unhandled } = type
+  assertNoUnhandledKeys(unhandled)
+  switch (_tag) {
+    case 'text':
+    case 'null':
+    case 'real':
+    case 'integer':
+    case 'blob':
+      return _tag
+    default:
+      return casesHandled(_tag)
+  }
+}
+
+const indexDescriptor = (index: Index): CanonicalValue => {
+  const { _tag: _tagIgnored, name, unique, primaryKey, columns, ...unhandled } = index
+  assertNoUnhandledKeys(unhandled)
+  return ['index', name ?? null, unique ?? false, primaryKey ?? false, columns]
+}
+
+const tableDescriptor = (table: Table, cache: SchemaDescriptorCache): CanonicalValue => {
+  const { _tag: _tagIgnored, name, columns, indexes, ...unhandled } = table
+  assertNoUnhandledKeys(unhandled)
+  return [
+    'table',
+    name,
+    columns.map((column) => columnDescriptor(column, cache)),
+    sortCanonicalValues(indexes.map(indexDescriptor)),
+  ]
+}
+
+const dbSchemaDescriptor = (dbSchema: DbSchema, cache: SchemaDescriptorCache): CanonicalValue => {
+  const { _tag: _tagIgnored, tables, ...unhandled } = dbSchema
+  assertNoUnhandledKeys(unhandled)
+  return ['database', sortByCanonicalKey(tables, (table) => table.name).map((table) => tableDescriptor(table, cache))]
+}
+
+const defaultDescriptor = (value: Column['default'], schema: Column['schema']): CanonicalValue => {
+  if (value._tag === 'None') return ['none']
+  if (isDefaultThunk(value.value) === true) return ['runtime-default']
+  if (value.value === null) return ['value', null]
+  if (isSqlDefaultValue(value.value) === true) return ['sql', value.value.sql]
+
+  return ['value', canonicalValue(Schema.encodeSync(schema)(value.value))]
+}
+
+const schemaDescriptor = (schema: Schema.Top, cache: SchemaDescriptorCache): CanonicalValue => {
+  const cached = cache.get(schema)
+  if (cached !== undefined) return cached
+
+  // Both ends matter: the encoded side describes stored JSON while the type side
+  // distinguishes codecs such as DateFromString from an unconstrained string.
+  const document = SchemaRepresentation.toRepresentations([schema.ast, SchemaAST.toType(schema.ast)], {
+    referencePolicy: () => undefined,
+  })
+  const descriptor: CanonicalValue = ['schema', schemaRepresentationDescriptor(document)]
+  cache.set(schema, descriptor)
+  return descriptor
+}
+
+const schemaRepresentationDescriptor = (value: unknown, parentKey?: string): CanonicalValue => {
+  // Representation payloads are application-owned JSON. Keys such as `annotations`
+  // inside them are data, not Effect metadata.
+  if (parentKey === 'payload') return canonicalValue(value)
+
+  if (Array.isArray(value) === true) return value.map((item) => schemaRepresentationDescriptor(item))
+
+  if (value !== null && typeof value === 'object' && value instanceof Uint8Array === false) {
+    const sourceEntries = Object.entries(value).map(([key, entryValue]): [string, unknown] =>
+      parentKey === 'representation' && key === 'id' && typeof entryValue === 'string'
+        ? [key, renamedEffectCheckIds.get(entryValue) ?? entryValue]
+        : [key, entryValue],
+    )
+    const isUnion = sourceEntries.some(([key, entryValue]) => key === '_tag' && entryValue === 'Union')
+    // Effect rc.113 removed this representation field because unions only support `anyOf`.
+    // Restore the old default so dependency upgrades preserve existing state fingerprints.
+    if (isUnion === true && sourceEntries.some(([key]) => key === 'mode') === false) {
+      sourceEntries.push(['mode', 'anyOf'])
+    }
+    const entries = sortByCanonicalKey(
+      sourceEntries.filter(([key]) => effectIgnoredFields.has(key) === false),
+      ([key]) => key,
+    ).map(([key, entryValue]) => [key, schemaRepresentationDescriptor(entryValue, key)] as const)
+    return Object.fromEntries(entries)
+  }
+
+  return canonicalValue(value)
+}
+
+const effectIgnoredFields = new Set(['annotations', 'isMutable'])
+
+/** Maps renamed Effect check ids to the ids existing fingerprints were computed from, so the rename does not rebuild state. */
+const renamedEffectCheckIds = new Map([
+  ['effect/schema/isBetweenLength', 'effect/schema/isLengthBetween'],
+  ['effect/schema/isStartingWith', 'effect/schema/isStartsWith'],
+  ['effect/schema/isEndingWith', 'effect/schema/isEndsWith'],
+  ['effect/schema/isIncluding', 'effect/schema/isIncludes'],
+  ['effect/schema/isBetweenSize', 'effect/schema/isSizeBetween'],
+  ['effect/schema/isBetweenProperties', 'effect/schema/isPropertiesLengthBetween'],
+])
+
+const canonicalValue = (value: unknown): CanonicalValue => {
+  if (Array.isArray(value) === true) return value.map(canonicalValue)
+
+  if (value !== null && typeof value === 'object' && value instanceof Uint8Array === false) {
+    const entries = sortByCanonicalKey(Object.entries(value), ([key]) => key).map(
+      ([key, entryValue]) => [key, canonicalValue(entryValue)] as const,
+    )
+    return Object.fromEntries(entries)
+  }
+
+  if (value === null || typeof value === 'string' || typeof value === 'boolean') return value
+
+  if (typeof value === 'number') {
+    if (Number.isNaN(value) === true) return specialValue('number', 'NaN')
+    if (value === Number.POSITIVE_INFINITY) return specialValue('number', 'Infinity')
+    if (value === Number.NEGATIVE_INFINITY) return specialValue('number', '-Infinity')
+    if (Object.is(value, -0) === true) return specialValue('number', '-0')
+    return value
+  }
+
+  if (typeof value === 'bigint') return specialValue('bigint', value.toString(10))
+  if (value === undefined) return specialValue('undefined')
+
+  if (typeof value === 'symbol') {
+    const symbolKey = Symbol.keyFor(value)
+    if (symbolKey === undefined) throw new Error('Local symbols cannot be represented in a persistent fingerprint')
+    return specialValue('symbol', symbolKey)
+  }
+
+  if (typeof value === 'function') throw new Error('Functions cannot be represented in a persistent fingerprint')
+  if (value instanceof Uint8Array) return specialValue('bytes', bytesToHex(value))
+
+  throw new Error('Unsupported canonical value')
+}
+
+const specialValue = (type: string, value?: string): CanonicalValue => ({
+  $livestore$type: value === undefined ? [type] : [type, value],
+})
+
+const bytesToHex = (bytes: Uint8Array): string => {
+  let output = ''
+  for (const byte of bytes) output += byte.toString(16).padStart(2, '0')
+  return output
+}
+
+/** Avoid repeatedly serializing deep descriptors from inside an O(n log n) sort comparator. */
+const sortCanonicalValues = (values: ReadonlyArray<CanonicalValue>): ReadonlyArray<CanonicalValue> =>
+  values
+    .map((value) => ({ value, key: JSON.stringify(value) }))
+    .toSorted((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0))
+    .map(({ value }) => value)
+
+const sortByCanonicalKey = <A>(values: ReadonlyArray<A>, key: (value: A) => CanonicalValue): ReadonlyArray<A> =>
+  values
+    .map((value) => ({ value, key: JSON.stringify(key(value)) }))
+    .toSorted((left, right) => (left.key < right.key ? -1 : left.key > right.key ? 1 : 0))
+    .map(({ value }) => value)
+
+const assertNoUnhandledKeys = (_unhandled: Record<PropertyKey, never>): void => {}

package/src/schema/state/sqlite/db-schema/ast/sqlite.ts

@@ -1,7 +1,5 @@
 import { omitUndefineds } from '@livestore/utils'
-import { type Option, Schema, SchemaAST } from '@livestore/utils/effect'
-
-import { hashCode } from '../hash.ts'
+import { type Option, Schema } from '@livestore/utils/effect'
 
 export namespace ColumnType {
   export type ColumnType = Text | Null | Real | Integer | Blob
@@ -25,7 +23,7 @@ export type Column = {
   nullable: boolean
   autoIncrement: boolean
   default: Option.Option<any>
-  schema: Schema.Schema<any>
+  schema: Schema.Codec<any, any>
 }
 
 export const column = (props: Omit<Column, '_tag'>): Column => ({ _tag: 'column', ...props })
@@ -83,82 +81,5 @@ export type DbSchema = {
 
 export const dbSchema = (tables: Table[]): DbSchema => ({ _tag: 'dbSchema', tables })
 
-/**
- * Helper to detect if a column is a JSON column (has parseJson transformation)
- */
-const isJsonColumn = (column: Column): boolean => {
-  if (column.type._tag !== 'text') return false
-
-  // Check if the schema AST is a parseJson transformation
-  const ast = column.schema.ast
-  return ast._tag === 'Transformation' && ast.annotations.schemaId === SchemaAST.ParseJsonSchemaId
-}
-
-/**
- * NOTE we're now including JSON schema information for JSON columns
- * to detect client document schema changes
- */
-export const hash = (obj: Table | Column | Index | ForeignKey | DbSchema): number =>
-  hashCode(JSON.stringify(trimInfoForHasing(obj)))
-
-const trimInfoForHasing = (obj: Table | Column | Index | ForeignKey | DbSchema): Record<string, any> => {
-  switch (obj._tag) {
-    case 'table': {
-      return {
-        _tag: 'table',
-        name: obj.name,
-        columns: obj.columns.map((column) => trimInfoForHasing(column)),
-        indexes: obj.indexes.map((index) => trimInfoForHasing(index)),
-      }
-    }
-    case 'column': {
-      const baseInfo: Record<string, any> = {
-        _tag: 'column',
-        name: obj.name,
-        type: obj.type._tag,
-        primaryKey: obj.primaryKey,
-        nullable: obj.nullable,
-        autoIncrement: obj.autoIncrement,
-        default: obj.default,
-      }
-
-      // NEW: Include schema hash for JSON columns
-      // This ensures that changes to the JSON schema are detected
-      if (isJsonColumn(obj) === true && obj.schema !== undefined) {
-        // Use Effect's Schema.hash for consistent hashing
-        baseInfo.jsonSchemaHash = Schema.hash(obj.schema)
-      }
-
-      return baseInfo
-    }
-    case 'index': {
-      return {
-        _tag: 'index',
-        columns: obj.columns,
-        name: obj.name,
-        unique: obj.unique,
-        primaryKey: obj.primaryKey,
-      }
-    }
-    case 'foreignKey': {
-      return {
-        _tag: 'foreignKey',
-        references: obj.references,
-        key: obj.key,
-        columns: obj.columns,
-      }
-    }
-    case 'dbSchema': {
-      return {
-        _tag: 'dbSchema',
-        tables: obj.tables.map(trimInfoForHasing),
-      }
-    }
-    default: {
-      throw new Error(`Unreachable: ${String(obj)}`)
-    }
-  }
-}
-
 export const structSchemaForTable = (tableDef: Table) =>
   Schema.Struct(Object.fromEntries(tableDef.columns.map((column) => [column.name, column.schema])))

package/src/schema/state/sqlite/db-schema/dsl/__snapshots__/field-defs.test.ts.snap

@@ -3,77 +3,77 @@
 exports[`FieldDefs > boolean 1`] = `
 {
   "autoIncrement": false,
-  "columnType": "text",
+  "columnType": "integer",
   "default": {
     "_id": "Option",
     "_tag": "None",
   },
   "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Boolean",
+    "decodedAst": "Boolean",
+    "encodedAst": "Union",
+  },
 }
 `;
 
 exports[`FieldDefs > boolean 2`] = `
 {
   "autoIncrement": false,
-  "columnType": "text",
+  "columnType": "integer",
   "default": {
     "_id": "Option",
     "_tag": "None",
   },
   "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Boolean",
+    "decodedAst": "Boolean",
+    "encodedAst": "Union",
+  },
 }
 `;
 
 exports[`FieldDefs > boolean 3`] = `
 {
   "autoIncrement": false,
-  "columnType": "text",
+  "columnType": "integer",
   "default": {
     "_id": "Option",
     "_tag": "Some",
-    "value": null,
+    "value": false,
   },
-  "nullable": true,
+  "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
-}
-`;
-
-exports[`FieldDefs > boolean 4`] = `
-{
-  "autoIncrement": false,
-  "columnType": "text",
-  "default": {
-    "_id": "Option",
-    "_tag": "Some",
-    "value": "foo",
+  "schema": {
+    "ast": "Boolean",
+    "decodedAst": "Boolean",
+    "encodedAst": "Union",
   },
-  "nullable": true,
-  "primaryKey": false,
-  "schema": [Function],
 }
 `;
 
-exports[`FieldDefs > boolean 5`] = `
+exports[`FieldDefs > datetime 1`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
   "default": {
     "_id": "Option",
-    "_tag": "Some",
-    "value": "foo",
+    "_tag": "None",
   },
-  "nullable": true,
+  "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Declaration",
+    "decodedAst": "Declaration",
+    "encodedAst": "String",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 6`] = `
+exports[`FieldDefs > datetime 2`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
@@ -83,11 +83,15 @@ exports[`FieldDefs > boolean 6`] = `
   },
   "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Declaration",
+    "decodedAst": "Declaration",
+    "encodedAst": "String",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 7`] = `
+exports[`FieldDefs > datetime 3`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
@@ -98,28 +102,34 @@ exports[`FieldDefs > boolean 7`] = `
   },
   "nullable": true,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Union",
+    "decodedAst": "Union",
+    "encodedAst": "Union",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 8`] = `
+exports[`FieldDefs > datetime 4`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
   "default": {
     "_id": "Option",
     "_tag": "Some",
-    "value": {
-      "name": "Bob",
-    },
+    "value": 2022-02-02T00:00:00.000Z,
   },
-  "nullable": true,
+  "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Declaration",
+    "decodedAst": "Declaration",
+    "encodedAst": "String",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 9`] = `
+exports[`FieldDefs > json 1`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
@@ -129,93 +139,143 @@ exports[`FieldDefs > boolean 9`] = `
   },
   "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Any",
+    "decodedAst": "Any",
+    "encodedAst": "String",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 10`] = `
+exports[`FieldDefs > json 2`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
   "default": {
     "_id": "Option",
-    "_tag": "None",
+    "_tag": "Some",
+    "value": null,
   },
-  "nullable": false,
+  "nullable": true,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Union",
+    "decodedAst": "Union",
+    "encodedAst": "Union",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 11`] = `
+exports[`FieldDefs > json 3`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
   "default": {
     "_id": "Option",
     "_tag": "Some",
-    "value": null,
+    "value": {
+      "name": "Bob",
+    },
   },
   "nullable": true,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Union",
+    "decodedAst": "Union",
+    "encodedAst": "Union",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 12`] = `
+exports[`FieldDefs > text 1`] = `
 {
   "autoIncrement": false,
   "columnType": "text",
   "default": {
     "_id": "Option",
-    "_tag": "Some",
-    "value": 2022-02-02T00:00:00.000Z,
+    "_tag": "None",
   },
   "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "String",
+    "decodedAst": "String",
+    "encodedAst": "String",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 13`] = `
+exports[`FieldDefs > text 2`] = `
 {
   "autoIncrement": false,
-  "columnType": "integer",
+  "columnType": "text",
   "default": {
     "_id": "Option",
     "_tag": "None",
   },
   "nullable": false,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "String",
+    "decodedAst": "String",
+    "encodedAst": "String",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 14`] = `
+exports[`FieldDefs > text 3`] = `
 {
   "autoIncrement": false,
-  "columnType": "integer",
+  "columnType": "text",
   "default": {
     "_id": "Option",
-    "_tag": "None",
+    "_tag": "Some",
+    "value": null,
   },
-  "nullable": false,
+  "nullable": true,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Union",
+    "decodedAst": "Union",
+    "encodedAst": "Union",
+  },
 }
 `;
 
-exports[`FieldDefs > boolean 15`] = `
+exports[`FieldDefs > text 4`] = `
 {
   "autoIncrement": false,
-  "columnType": "integer",
+  "columnType": "text",
   "default": {
     "_id": "Option",
     "_tag": "Some",
-    "value": false,
+    "value": "foo",
   },
-  "nullable": false,
+  "nullable": true,
   "primaryKey": false,
-  "schema": [Function],
+  "schema": {
+    "ast": "Union",
+    "decodedAst": "Union",
+    "encodedAst": "Union",
+  },
+}
+`;
+
+exports[`FieldDefs > text 5`] = `
+{
+  "autoIncrement": false,
+  "columnType": "text",
+  "default": {
+    "_id": "Option",
+    "_tag": "Some",
+    "value": "foo",
+  },
+  "nullable": true,
+  "primaryKey": false,
+  "schema": {
+    "ast": "Union",
+    "decodedAst": "Union",
+    "encodedAst": "Union",
+  },
 }
 `;