@michaelstewart/convex-tanstack-db-collection 0.0.1

package/src/expression-parser.ts

@@ -0,0 +1,255 @@
+import type { LoadSubsetOptions } from '@tanstack/db'
+import type { ExtractedFilters, FilterDimension } from './types.js'
+import { toKey } from './serialization.js'
+
+/**
+ * TanStack DB expression types (simplified for our needs)
+ * These mirror the IR types from @tanstack/db
+ */
+interface PropRef {
+  type: `ref`
+  path: Array<string>
+}
+
+interface Value {
+  type: `val`
+  value: unknown
+}
+
+interface Func {
+  type: `func`
+  name: string
+  args: Array<BasicExpression>
+}
+
+type BasicExpression = PropRef | Value | Func
+
+/**
+ * Check if a value is a PropRef expression
+ */
+function isPropRef(expr: unknown): expr is PropRef {
+  return (
+    typeof expr === `object` &&
+    expr !== null &&
+    `type` in expr &&
+    expr.type === `ref` &&
+    `path` in expr &&
+    Array.isArray((expr as PropRef).path)
+  )
+}
+
+/**
+ * Check if a value is a Value expression
+ */
+function isValue(expr: unknown): expr is Value {
+  return (
+    typeof expr === `object` &&
+    expr !== null &&
+    `type` in expr &&
+    expr.type === `val` &&
+    `value` in expr
+  )
+}
+
+/**
+ * Check if a value is a Func expression
+ */
+function isFunc(expr: unknown): expr is Func {
+  return (
+    typeof expr === `object` &&
+    expr !== null &&
+    `type` in expr &&
+    expr.type === `func` &&
+    `name` in expr &&
+    `args` in expr &&
+    Array.isArray((expr as Func).args)
+  )
+}
+
+/**
+ * Check if a PropRef matches our target field.
+ * Handles both aliased (e.g., ['msg', 'pageId']) and direct (e.g., ['pageId']) paths.
+ */
+function propRefMatchesField(propRef: PropRef, fieldName: string): boolean {
+  const { path } = propRef
+  // Direct field reference: ['pageId']
+  if (path.length === 1 && path[0] === fieldName) {
+    return true
+  }
+  // Aliased field reference: ['msg', 'pageId'] or ['m', 'pageId']
+  if (path.length === 2 && path[1] === fieldName) {
+    return true
+  }
+  return false
+}
+
+/**
+ * Extract filter values from an 'eq' function call.
+ * Pattern: eq(ref(filterField), val(x)) or eq(val(x), ref(filterField))
+ */
+function extractFromEq(func: Func, filterField: string): unknown[] {
+  if (func.args.length !== 2) return []
+
+  const [left, right] = func.args
+
+  // eq(ref, val)
+  if (isPropRef(left) && propRefMatchesField(left, filterField) && isValue(right)) {
+    return [right.value]
+  }
+
+  // eq(val, ref) - reversed order
+  if (isValue(left) && isPropRef(right) && propRefMatchesField(right, filterField)) {
+    return [left.value]
+  }
+
+  return []
+}
+
+/**
+ * Extract filter values from an 'in' function call.
+ * Pattern: in(ref(filterField), val([a, b, c]))
+ */
+function extractFromIn(func: Func, filterField: string): unknown[] {
+  if (func.args.length !== 2) return []
+
+  const [left, right] = func.args
+
+  // in(ref, val)
+  if (isPropRef(left) && propRefMatchesField(left, filterField) && isValue(right)) {
+    const val = right.value
+    if (Array.isArray(val)) {
+      return val
+    }
+  }
+
+  return []
+}
+
+/**
+ * Recursively walk an expression tree to find all filter values for the given field.
+ * Handles 'eq', 'in', 'and', and 'or' expressions.
+ */
+function walkExpression(expr: unknown, filterField: string): unknown[] {
+  if (!isFunc(expr)) return []
+
+  const { name, args } = expr
+  const results: unknown[] = []
+
+  switch (name) {
+    case `eq`:
+      results.push(...extractFromEq(expr, filterField))
+      break
+
+    case `in`:
+      results.push(...extractFromIn(expr, filterField))
+      break
+
+    case `and`:
+    case `or`:
+      // Recursively process all arguments
+      for (const arg of args) {
+        results.push(...walkExpression(arg, filterField))
+      }
+      break
+
+    // For other functions, recursively check their arguments
+    // (in case of nested expressions)
+    default:
+      for (const arg of args) {
+        results.push(...walkExpression(arg, filterField))
+      }
+      break
+  }
+
+  return results
+}
+
+/**
+ * Extract filter values from LoadSubsetOptions.
+ *
+ * Parses the `where` expression to find equality (`.eq()`) and set membership (`.in()`)
+ * comparisons for the specified filter field.
+ *
+ * @param options - The LoadSubsetOptions from a live query
+ * @param filterField - The field name to extract values for (e.g., 'pageId')
+ * @returns Array of unique filter values found in the expression
+ *
+ * @example
+ * // For query: where(m => m.pageId.eq('page-1'))
+ * extractFilterValues(options, 'pageId') // returns ['page-1']
+ *
+ * @example
+ * // For query: where(m => inArray(m.pageId, ['page-1', 'page-2']))
+ * extractFilterValues(options, 'pageId') // returns ['page-1', 'page-2']
+ *
+ * @example
+ * // For query: where(m => and(m.pageId.eq('page-1'), m.status.eq('active')))
+ * extractFilterValues(options, 'pageId') // returns ['page-1']
+ */
+export function extractFilterValues(
+  options: LoadSubsetOptions,
+  filterField: string
+): unknown[] {
+  const { where } = options
+
+  if (!where) {
+    return []
+  }
+
+  // Extract from the where expression
+  const values = walkExpression(where, filterField)
+
+  // Return unique values using toKey for stable identity comparison
+  const seen = new Set<string>()
+  const unique: unknown[] = []
+  for (const value of values) {
+    const key = toKey(value)
+    if (!seen.has(key)) {
+      seen.add(key)
+      unique.push(value)
+    }
+  }
+
+  return unique
+}
+
+/**
+ * Check if LoadSubsetOptions contains a filter for the specified field.
+ */
+export function hasFilterField(options: LoadSubsetOptions, filterField: string): boolean {
+  return extractFilterValues(options, filterField).length > 0
+}
+
+/**
+ * Extract filter values for multiple filter dimensions from LoadSubsetOptions.
+ *
+ * Parses the `where` expression to find values for each configured filter dimension.
+ * Results are keyed by convexArg for direct use in Convex query args.
+ *
+ * @param options - The LoadSubsetOptions from a live query
+ * @param filterDimensions - Array of filter dimensions to extract
+ * @returns Object mapping convexArg -> values for dimensions with matches
+ *
+ * @example
+ * // For query: where(m => m.pageId.eq('page-1') && m.authorId.eq('user-1'))
+ * extractMultipleFilterValues(options, [
+ *   { filterField: 'pageId', convexArg: 'pageIds' },
+ *   { filterField: 'authorId', convexArg: 'authorIds' },
+ * ])
+ * // returns { pageIds: ['page-1'], authorIds: ['user-1'] }
+ */
+export function extractMultipleFilterValues(
+  options: LoadSubsetOptions,
+  filterDimensions: FilterDimension[]
+): ExtractedFilters {
+  const result: ExtractedFilters = {}
+
+  for (const dim of filterDimensions) {
+    const values = extractFilterValues(options, dim.filterField)
+    if (values.length > 0) {
+      result[dim.convexArg] = values
+    }
+  }
+
+  return result
+}

package/src/index.ts

@@ -0,0 +1,247 @@
+import type {
+  CollectionConfig,
+  LoadSubsetOptions,
+  SyncConfig,
+  UtilsRecord,
+} from '@tanstack/db'
+import type { StandardSchemaV1 } from '@standard-schema/spec'
+import type { FunctionReference, FunctionReturnType } from 'convex/server'
+import { ConvexSyncManager } from './ConvexSyncManager.js'
+import { extractMultipleFilterValues } from './expression-parser.js'
+import type { ConvexCollectionConfig, FilterConfig, FilterDimension } from './types.js'
+
+// Re-export types
+export type {
+  ConvexCollectionConfig,
+  ConvexUnsubscribe,
+  ExtractedFilters,
+  FilterConfig,
+  FilterDimension,
+} from './types.js'
+export { extractFilterValues, extractMultipleFilterValues, hasFilterField } from './expression-parser.js'
+export { ConvexSyncManager } from './ConvexSyncManager.js'
+export { serializeValue, deserializeValue, toKey, fromKey } from './serialization.js'
+
+// Default configuration values
+const DEFAULT_UPDATED_AT_FIELD = `updatedAt`
+const DEFAULT_DEBOUNCE_MS = 50
+const DEFAULT_TAIL_OVERLAP_MS = 10000
+const DEFAULT_RESUBSCRIBE_THRESHOLD = 10
+
+/**
+ * Normalize filter configuration to an array of FilterDimension.
+ * - undefined = [] (0 filters, global sync)
+ * - single object = [object] (1 filter)
+ * - array = array as-is (N filters)
+ */
+function normalizeFilterConfig(filters: FilterConfig): FilterDimension[] {
+  if (filters === undefined) return []
+  return Array.isArray(filters) ? filters : [filters]
+}
+
+/**
+ * Schema output type inference helper
+ */
+type InferSchemaOutput<T> = T extends StandardSchemaV1
+  ? StandardSchemaV1.InferOutput<T> extends object
+    ? StandardSchemaV1.InferOutput<T>
+    : Record<string, unknown>
+  : Record<string, unknown>
+
+/**
+ * Infer the item type from a Convex query's return type.
+ * Expects the query to return an array of items.
+ */
+type InferQueryItemType<TQuery extends FunctionReference<'query'>> =
+  FunctionReturnType<TQuery> extends Array<infer T>
+    ? T extends object
+      ? T
+      : Record<string, unknown>
+    : Record<string, unknown>
+
+/**
+ * Creates collection options for use with TanStack DB's createCollection.
+ * This integrates Convex real-time subscriptions with TanStack DB collections
+ * using the "backfill + tail" synchronization pattern.
+ *
+ * @example
+ * ```typescript
+ * import { createCollection } from '@tanstack/react-db'
+ * import { convexCollectionOptions } from '@michaelstewart/convex-tanstack-db-collection'
+ * import { api } from '@convex/_generated/api'
+ *
+ * // Single filter dimension
+ * const messagesCollection = createCollection(
+ *   convexCollectionOptions({
+ *     client: convexClient,
+ *     query: api.messages.sync,
+ *     filters: { filterField: 'pageId', convexArg: 'pageIds' },
+ *     getKey: (msg) => msg._id,
+ *
+ *     onInsert: async ({ transaction }) => {
+ *       const newMsg = transaction.mutations[0].modified
+ *       await convexClient.mutation(api.messages.create, newMsg)
+ *     },
+ *   })
+ * )
+ *
+ * // Multiple filter dimensions
+ * const filteredCollection = createCollection(
+ *   convexCollectionOptions({
+ *     client: convexClient,
+ *     query: api.items.syncFiltered,
+ *     filters: [
+ *       { filterField: 'pageId', convexArg: 'pageIds' },
+ *       { filterField: 'authorId', convexArg: 'authorIds' },
+ *     ],
+ *     getKey: (item) => item._id,
+ *   })
+ * )
+ *
+ * // No filters (global sync)
+ * const allItemsCollection = createCollection(
+ *   convexCollectionOptions({
+ *     client: convexClient,
+ *     query: api.items.syncAll,  // Query takes only { after }
+ *     getKey: (item) => item._id,
+ *   })
+ * )
+ *
+ * // In UI:
+ * const { data: messages } = useLiveQuery(q =>
+ *   q.from({ msg: messagesCollection })
+ *    .where(({ msg }) => msg.pageId.eq('page-123'))
+ * )
+ * ```
+ */
+
+// Overload for when schema is provided
+export function convexCollectionOptions<
+  TSchema extends StandardSchemaV1,
+  TKey extends string | number = string | number,
+  TUtils extends UtilsRecord = UtilsRecord,
+>(
+  config: ConvexCollectionConfig<InferSchemaOutput<TSchema>, TKey, TSchema, TUtils> & {
+    schema: TSchema
+  }
+): CollectionConfig<InferSchemaOutput<TSchema>, TKey, TSchema, TUtils>
+
+// Overload for when no schema is provided - T is inferred from query's return type
+export function convexCollectionOptions<
+  TQuery extends FunctionReference<'query'>,
+  T extends InferQueryItemType<TQuery> = InferQueryItemType<TQuery>,
+  TKey extends string | number = string | number,
+  TUtils extends UtilsRecord = UtilsRecord,
+>(
+  config: Omit<ConvexCollectionConfig<T, TKey, never, TUtils>, 'query'> & {
+    schema?: never
+    query: TQuery
+    getKey: (item: T) => TKey
+  }
+): CollectionConfig<T, TKey, never, TUtils>
+
+// Implementation - uses concrete types; overloads provide proper type inference
+export function convexCollectionOptions(
+  config: ConvexCollectionConfig<Record<string, unknown>, string | number, never, UtilsRecord>
+): CollectionConfig<Record<string, unknown>, string | number, never, UtilsRecord> {
+  const {
+    client,
+    query,
+    filters,
+    updatedAtFieldName = DEFAULT_UPDATED_AT_FIELD,
+    debounceMs = DEFAULT_DEBOUNCE_MS,
+    tailOverlapMs = DEFAULT_TAIL_OVERLAP_MS,
+    resubscribeThreshold = DEFAULT_RESUBSCRIBE_THRESHOLD,
+    getKey,
+    onInsert,
+    onUpdate,
+    ...baseConfig
+  } = config
+
+  // Normalize filter configuration
+  const filterDimensions = normalizeFilterConfig(filters)
+
+  // Create the sync manager
+  const syncManager = new ConvexSyncManager<any, any>({
+    client,
+    query,
+    filterDimensions,
+    updatedAtFieldName,
+    debounceMs,
+    tailOverlapMs,
+    resubscribeThreshold,
+    getKey: getKey as (item: any) => string | number,
+  })
+
+  // Create the sync configuration
+  const syncConfig: SyncConfig<any, any> = {
+    sync: (params) => {
+      const { collection, begin, write, commit, markReady } = params
+
+      // Initialize sync manager with callbacks
+      syncManager.setCallbacks({
+        collection: {
+          get: (key) => collection.get(key),
+          has: (key) => collection.has(key),
+        },
+        begin,
+        write,
+        commit,
+        markReady,
+      })
+
+      // Return loadSubset, unloadSubset, and cleanup handlers
+      return {
+        loadSubset: (options: LoadSubsetOptions): Promise<void> => {
+          // 0-filter case: global sync
+          if (filterDimensions.length === 0) {
+            return syncManager.requestFilters({})
+          }
+
+          // Extract filter values from the where expression
+          const extracted = extractMultipleFilterValues(options, filterDimensions)
+
+          // Sync if ANY dimension has values (any-filter matching)
+          // Convex query arg validators enforce which combinations are valid
+          if (Object.keys(extracted).length === 0) {
+            // No filter values found - this is expected for queries that filter
+            // by other fields (e.g., clientId, parentId). These queries read from
+            // the already-synced collection and don't need to trigger a sync.
+            return Promise.resolve()
+          }
+
+          return syncManager.requestFilters(extracted)
+        },
+
+        unloadSubset: (options: LoadSubsetOptions): void => {
+          // 0-filter case: global sync
+          if (filterDimensions.length === 0) {
+            syncManager.releaseFilters({})
+            return
+          }
+
+          // Extract filter values from the where expression
+          const extracted = extractMultipleFilterValues(options, filterDimensions)
+
+          if (Object.keys(extracted).length > 0) {
+            syncManager.releaseFilters(extracted)
+          }
+        },
+
+        cleanup: () => {
+          syncManager.cleanup()
+        },
+      }
+    },
+  }
+
+  // Return the complete collection config
+  return {
+    ...baseConfig,
+    getKey,
+    syncMode: `on-demand`, // Always on-demand since we sync based on query predicates
+    sync: syncConfig,
+    onInsert,
+    onUpdate,
+  }
+}

package/src/serialization.ts

@@ -0,0 +1,125 @@
+/**
+ * Value serialization utilities for stable hashing and round-tripping.
+ *
+ * Adapted from TanStack DB's query-db-collection:
+ * https://github.com/TanStack/db/blob/main/packages/query-db-collection/src/serialization.ts
+ */
+
+interface TypeMarker {
+  __type: string
+  value?: unknown
+  sign?: number
+}
+
+function isTypeMarker(value: unknown): value is TypeMarker {
+  return (
+    typeof value === `object` &&
+    value !== null &&
+    `__type` in value &&
+    typeof (value as TypeMarker).__type === `string`
+  )
+}
+
+/**
+ * Serializes a value into a JSON-safe format that preserves special JS types.
+ * Handles: undefined, NaN, Infinity, -Infinity, Date, arrays, and objects.
+ */
+export function serializeValue(value: unknown): unknown {
+  if (value === undefined) {
+    return { __type: `undefined` }
+  }
+
+  if (typeof value === `number`) {
+    if (Number.isNaN(value)) {
+      return { __type: `nan` }
+    }
+    if (value === Number.POSITIVE_INFINITY) {
+      return { __type: `infinity`, sign: 1 }
+    }
+    if (value === Number.NEGATIVE_INFINITY) {
+      return { __type: `infinity`, sign: -1 }
+    }
+  }
+
+  if (
+    value === null ||
+    typeof value === `string` ||
+    typeof value === `number` ||
+    typeof value === `boolean`
+  ) {
+    return value
+  }
+
+  if (value instanceof Date) {
+    return { __type: `date`, value: value.toJSON() }
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => serializeValue(item))
+  }
+
+  if (typeof value === `object`) {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>).map(([key, val]) => [
+        key,
+        serializeValue(val),
+      ])
+    )
+  }
+
+  return value
+}
+
+/**
+ * Deserializes a value back from its JSON-safe format.
+ * Restores: undefined, NaN, Infinity, -Infinity, Date, arrays, and objects.
+ */
+export function deserializeValue(value: unknown): unknown {
+  if (isTypeMarker(value)) {
+    switch (value.__type) {
+      case `undefined`:
+        return undefined
+      case `nan`:
+        return NaN
+      case `infinity`:
+        return value.sign === 1 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY
+      case `date`:
+        return new Date(value.value as string)
+      default:
+        return value
+    }
+  }
+
+  if (value === null || typeof value === `string` || typeof value === `number` || typeof value === `boolean`) {
+    return value
+  }
+
+  if (Array.isArray(value)) {
+    return value.map((item) => deserializeValue(item))
+  }
+
+  if (typeof value === `object`) {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>).map(([key, val]) => [
+        key,
+        deserializeValue(val),
+      ])
+    )
+  }
+
+  return value
+}
+
+/**
+ * Converts a value to a stable string key for use in Sets/Maps.
+ */
+export function toKey(value: unknown): string {
+  return JSON.stringify(serializeValue(value))
+}
+
+/**
+ * Restores a value from its string key representation.
+ */
+export function fromKey(key: string): unknown {
+  return deserializeValue(JSON.parse(key))
+}