@michaelstewart/convex-tanstack-db-collection 0.0.1

package/dist/esm/index.js

@@ -0,0 +1,679 @@
+function isTypeMarker(value) {
+  return typeof value === `object` && value !== null && `__type` in value && typeof value.__type === `string`;
+}
+function serializeValue(value) {
+  if (value === void 0) {
+    return { __type: `undefined` };
+  }
+  if (typeof value === `number`) {
+    if (Number.isNaN(value)) {
+      return { __type: `nan` };
+    }
+    if (value === Number.POSITIVE_INFINITY) {
+      return { __type: `infinity`, sign: 1 };
+    }
+    if (value === Number.NEGATIVE_INFINITY) {
+      return { __type: `infinity`, sign: -1 };
+    }
+  }
+  if (value === null || typeof value === `string` || typeof value === `number` || typeof value === `boolean`) {
+    return value;
+  }
+  if (value instanceof Date) {
+    return { __type: `date`, value: value.toJSON() };
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => serializeValue(item));
+  }
+  if (typeof value === `object`) {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, val]) => [
+        key,
+        serializeValue(val)
+      ])
+    );
+  }
+  return value;
+}
+function deserializeValue(value) {
+  if (isTypeMarker(value)) {
+    switch (value.__type) {
+      case `undefined`:
+        return void 0;
+      case `nan`:
+        return NaN;
+      case `infinity`:
+        return value.sign === 1 ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
+      case `date`:
+        return new Date(value.value);
+      default:
+        return value;
+    }
+  }
+  if (value === null || typeof value === `string` || typeof value === `number` || typeof value === `boolean`) {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => deserializeValue(item));
+  }
+  if (typeof value === `object`) {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, val]) => [
+        key,
+        deserializeValue(val)
+      ])
+    );
+  }
+  return value;
+}
+function toKey(value) {
+  return JSON.stringify(serializeValue(value));
+}
+function fromKey(key) {
+  return deserializeValue(JSON.parse(key));
+}
+class ConvexSyncManager {
+  constructor(options) {
+    this.activeDimensions = /* @__PURE__ */ new Map();
+    this.refCounts = /* @__PURE__ */ new Map();
+    this.pendingFilters = {};
+    this.globalCursor = 0;
+    this.currentSubscription = null;
+    this.debounceTimer = null;
+    this.isProcessing = false;
+    this.markedReady = false;
+    this.hasRequestedGlobal = false;
+    this.globalRefCount = 0;
+    this.messagesSinceSubscription = 0;
+    this.callbacks = null;
+    this.client = options.client;
+    this.query = options.query;
+    this.filterDimensions = options.filterDimensions;
+    this.updatedAtFieldName = options.updatedAtFieldName;
+    this.debounceMs = options.debounceMs;
+    this.tailOverlapMs = options.tailOverlapMs;
+    this.resubscribeThreshold = options.resubscribeThreshold;
+    this.getKey = options.getKey;
+    for (const dim of this.filterDimensions) {
+      this.activeDimensions.set(dim.convexArg, /* @__PURE__ */ new Set());
+    }
+  }
+  /**
+   * Initialize the sync manager with callbacks from TanStack DB
+   */
+  setCallbacks(callbacks) {
+    this.callbacks = callbacks;
+  }
+  /**
+   * Create a composite key for ref counting multi-filter combinations.
+   * Uses serialized values for deterministic keys.
+   */
+  createCompositeKey(filters) {
+    const sorted = Object.keys(filters).sort().reduce(
+      (acc, key) => {
+        const values = filters[key];
+        if (values) {
+          acc[key] = values.map((v) => toKey(v)).sort();
+        }
+        return acc;
+      },
+      {}
+    );
+    return JSON.stringify(sorted);
+  }
+  /**
+   * Request filters to be synced (called by loadSubset)
+   * Filters are batched via debouncing for efficiency
+   */
+  requestFilters(filters) {
+    var _a;
+    if (this.filterDimensions.length === 0) {
+      this.globalRefCount++;
+      if (!this.hasRequestedGlobal) {
+        this.hasRequestedGlobal = true;
+        return this.scheduleProcessing();
+      }
+      return Promise.resolve();
+    }
+    const compositeKey = this.createCompositeKey(filters);
+    const count = this.refCounts.get(compositeKey) || 0;
+    this.refCounts.set(compositeKey, count + 1);
+    let hasNewValues = false;
+    for (const [convexArg, values] of Object.entries(filters)) {
+      const activeSet = this.activeDimensions.get(convexArg);
+      if (!activeSet) continue;
+      const dim = this.filterDimensions.find((d) => d.convexArg === convexArg);
+      if (dim == null ? void 0 : dim.single) {
+        const existingCount = activeSet.size;
+        const pendingCount = ((_a = this.pendingFilters[convexArg]) == null ? void 0 : _a.length) ?? 0;
+        const newValues = values.filter((v) => {
+          var _a2;
+          const serialized = toKey(v);
+          const alreadyActive = activeSet.has(serialized);
+          const alreadyPending = (_a2 = this.pendingFilters[convexArg]) == null ? void 0 : _a2.some(
+            (pv) => toKey(pv) === serialized
+          );
+          return !alreadyActive && !alreadyPending;
+        });
+        if (existingCount + pendingCount + newValues.length > 1) {
+          throw new Error(
+            `Filter '${dim.filterField}' is configured as single but multiple values were requested. Active: ${existingCount}, Pending: ${pendingCount}, New: ${newValues.length}. Use single: false if you need to sync multiple values.`
+          );
+        }
+      }
+      for (const value of values) {
+        const serialized = toKey(value);
+        if (!activeSet.has(serialized)) {
+          if (!this.pendingFilters[convexArg]) {
+            this.pendingFilters[convexArg] = [];
+          }
+          const alreadyPending = this.pendingFilters[convexArg].some(
+            (v) => toKey(v) === serialized
+          );
+          if (!alreadyPending) {
+            this.pendingFilters[convexArg].push(value);
+            hasNewValues = true;
+          }
+        }
+      }
+    }
+    if (hasNewValues) {
+      return this.scheduleProcessing();
+    }
+    return Promise.resolve();
+  }
+  /**
+   * Release filters when no longer needed (called by unloadSubset)
+   */
+  releaseFilters(filters) {
+    if (this.filterDimensions.length === 0) {
+      this.globalRefCount = Math.max(0, this.globalRefCount - 1);
+      if (this.globalRefCount === 0 && this.hasRequestedGlobal) {
+        this.hasRequestedGlobal = false;
+        this.updateSubscription();
+      }
+      return;
+    }
+    const compositeKey = this.createCompositeKey(filters);
+    const count = (this.refCounts.get(compositeKey) || 0) - 1;
+    if (count <= 0) {
+      this.refCounts.delete(compositeKey);
+    } else {
+      this.refCounts.set(compositeKey, count);
+    }
+    this.cleanupUnreferencedValues();
+  }
+  /**
+   * Remove values from activeDimensions that are no longer referenced
+   * by any composite key in refCounts
+   */
+  cleanupUnreferencedValues() {
+    const referencedValues = /* @__PURE__ */ new Map();
+    for (const dim of this.filterDimensions) {
+      referencedValues.set(dim.convexArg, /* @__PURE__ */ new Set());
+    }
+    for (const compositeKey of this.refCounts.keys()) {
+      try {
+        const filters = JSON.parse(compositeKey);
+        for (const [convexArg, serializedValues] of Object.entries(filters)) {
+          const refSet = referencedValues.get(convexArg);
+          if (refSet) {
+            for (const serialized of serializedValues) {
+              refSet.add(serialized);
+            }
+          }
+        }
+      } catch {
+      }
+    }
+    let needsSubscriptionUpdate = false;
+    for (const [convexArg, activeSet] of this.activeDimensions) {
+      const refSet = referencedValues.get(convexArg);
+      for (const serialized of activeSet) {
+        if (!refSet.has(serialized)) {
+          activeSet.delete(serialized);
+          needsSubscriptionUpdate = true;
+        }
+      }
+    }
+    if (needsSubscriptionUpdate) {
+      this.updateSubscription();
+    }
+  }
+  /**
+   * Schedule debounced processing of pending filters
+   */
+  scheduleProcessing() {
+    return new Promise((resolve, reject) => {
+      if (this.debounceTimer) {
+        clearTimeout(this.debounceTimer);
+      }
+      this.debounceTimer = setTimeout(async () => {
+        try {
+          await this.processFilterBatch();
+          resolve();
+        } catch (error) {
+          reject(error);
+        }
+      }, this.debounceMs);
+    });
+  }
+  /**
+   * Process the current batch of pending filters
+   */
+  async processFilterBatch() {
+    if (this.isProcessing) {
+      return;
+    }
+    const hasPendingFilters = Object.keys(this.pendingFilters).length > 0;
+    const needsGlobalSync = this.filterDimensions.length === 0 && this.hasRequestedGlobal;
+    if (!hasPendingFilters && !needsGlobalSync) {
+      return;
+    }
+    this.isProcessing = true;
+    try {
+      if (this.filterDimensions.length === 0) {
+        await this.runGlobalBackfill();
+      } else {
+        const newFilters = { ...this.pendingFilters };
+        this.pendingFilters = {};
+        for (const [convexArg, values] of Object.entries(newFilters)) {
+          const activeSet = this.activeDimensions.get(convexArg);
+          if (activeSet) {
+            for (const value of values) {
+              activeSet.add(toKey(value));
+            }
+          }
+        }
+        await this.runBackfill(newFilters);
+      }
+      this.updateSubscription();
+      if (!this.markedReady && this.callbacks) {
+        this.callbacks.markReady();
+        this.markedReady = true;
+      }
+    } finally {
+      this.isProcessing = false;
+    }
+  }
+  /**
+   * Run global backfill for 0-filter case
+   */
+  async runGlobalBackfill() {
+    try {
+      const args = { after: 0 };
+      const items = await this.client.query(this.query, args);
+      if (Array.isArray(items)) {
+        this.handleIncomingData(items);
+      }
+    } catch (error) {
+      console.error("[ConvexSyncManager] Global backfill error:", error);
+      throw error;
+    }
+  }
+  /**
+   * Run backfill query for new filter values to get their full history
+   */
+  async runBackfill(newFilters) {
+    if (Object.keys(newFilters).length === 0) return;
+    try {
+      const args = {
+        ...newFilters,
+        after: 0
+      };
+      const items = await this.client.query(this.query, args);
+      if (Array.isArray(items)) {
+        this.handleIncomingData(items);
+      }
+    } catch (error) {
+      console.error("[ConvexSyncManager] Backfill error:", error);
+      throw error;
+    }
+  }
+  /**
+   * Build query args from all active dimensions
+   */
+  buildQueryArgs(after) {
+    const args = { after };
+    for (const [convexArg, serializedValues] of this.activeDimensions) {
+      const values = [...serializedValues].map((s) => fromKey(s));
+      const dim = this.filterDimensions.find((d) => d.convexArg === convexArg);
+      args[convexArg] = (dim == null ? void 0 : dim.single) ? values[0] : values;
+    }
+    return args;
+  }
+  /**
+   * Update the live subscription to cover all active filter values
+   */
+  updateSubscription() {
+    if (this.currentSubscription) {
+      this.currentSubscription();
+      this.currentSubscription = null;
+    }
+    this.messagesSinceSubscription = 0;
+    if (this.filterDimensions.length === 0) {
+      if (!this.hasRequestedGlobal) {
+        return;
+      }
+    } else {
+      let hasActiveValues = false;
+      for (const activeSet of this.activeDimensions.values()) {
+        if (activeSet.size > 0) {
+          hasActiveValues = true;
+          break;
+        }
+      }
+      if (!hasActiveValues) {
+        return;
+      }
+    }
+    const cursor = Math.max(0, this.globalCursor - this.tailOverlapMs);
+    const args = this.buildQueryArgs(cursor);
+    if ("onUpdate" in this.client) {
+      const subscription = this.client.onUpdate(
+        this.query,
+        args,
+        (result) => {
+          if (result !== void 0) {
+            const items = result;
+            if (Array.isArray(items)) {
+              this.handleIncomingData(items);
+            }
+          }
+        },
+        (error) => {
+          console.error(`[ConvexSyncManager] Subscription error:`, error);
+        }
+      );
+      this.currentSubscription = () => subscription.unsubscribe();
+    } else {
+      const watch = this.client.watchQuery(this.query, args);
+      this.currentSubscription = watch.onUpdate(() => {
+        const result = watch.localQueryResult();
+        if (result !== void 0) {
+          const items = result;
+          if (Array.isArray(items)) {
+            this.handleIncomingData(items);
+          }
+        }
+      });
+    }
+  }
+  /**
+   * Handle incoming data from backfill or subscription
+   * Uses LWW (Last-Write-Wins) to resolve conflicts
+   */
+  handleIncomingData(items) {
+    if (!this.callbacks || items.length === 0) return;
+    const { collection, begin, write, commit } = this.callbacks;
+    const previousCursor = this.globalCursor;
+    let newItemCount = 0;
+    begin();
+    for (const item of items) {
+      const key = this.getKey(item);
+      const incomingTs = item[this.updatedAtFieldName];
+      if (incomingTs !== void 0 && incomingTs > this.globalCursor) {
+        this.globalCursor = incomingTs;
+      }
+      const existing = collection.get(key);
+      if (!existing) {
+        write({ type: `insert`, value: item });
+        if (incomingTs !== void 0 && incomingTs > previousCursor) {
+          newItemCount++;
+        }
+      } else {
+        const existingTs = existing[this.updatedAtFieldName];
+        if (incomingTs !== void 0 && existingTs !== void 0) {
+          if (incomingTs > existingTs) {
+            write({ type: `update`, value: item });
+            if (incomingTs > previousCursor) {
+              newItemCount++;
+            }
+          }
+        } else if (incomingTs !== void 0) {
+          write({ type: `update`, value: item });
+        }
+      }
+    }
+    commit();
+    this.messagesSinceSubscription += newItemCount;
+    if (this.resubscribeThreshold > 0 && this.messagesSinceSubscription >= this.resubscribeThreshold && this.currentSubscription !== null) {
+      this.updateSubscription();
+    }
+  }
+  /**
+   * Clean up all resources
+   */
+  cleanup() {
+    if (this.debounceTimer) {
+      clearTimeout(this.debounceTimer);
+      this.debounceTimer = null;
+    }
+    if (this.currentSubscription) {
+      this.currentSubscription();
+      this.currentSubscription = null;
+    }
+    for (const activeSet of this.activeDimensions.values()) {
+      activeSet.clear();
+    }
+    this.refCounts.clear();
+    this.pendingFilters = {};
+    this.globalCursor = 0;
+    this.markedReady = false;
+    this.hasRequestedGlobal = false;
+    this.globalRefCount = 0;
+    this.messagesSinceSubscription = 0;
+    this.callbacks = null;
+  }
+  /**
+   * Get debug info about current state
+   */
+  getDebugInfo() {
+    const activeDimensions = {};
+    for (const [convexArg, serializedValues] of this.activeDimensions) {
+      activeDimensions[convexArg] = [...serializedValues].map((s) => fromKey(s));
+    }
+    return {
+      activeDimensions,
+      globalCursor: this.globalCursor,
+      pendingFilters: { ...this.pendingFilters },
+      hasSubscription: this.currentSubscription !== null,
+      markedReady: this.markedReady,
+      hasRequestedGlobal: this.hasRequestedGlobal,
+      messagesSinceSubscription: this.messagesSinceSubscription
+    };
+  }
+}
+function isPropRef(expr) {
+  return typeof expr === `object` && expr !== null && `type` in expr && expr.type === `ref` && `path` in expr && Array.isArray(expr.path);
+}
+function isValue(expr) {
+  return typeof expr === `object` && expr !== null && `type` in expr && expr.type === `val` && `value` in expr;
+}
+function isFunc(expr) {
+  return typeof expr === `object` && expr !== null && `type` in expr && expr.type === `func` && `name` in expr && `args` in expr && Array.isArray(expr.args);
+}
+function propRefMatchesField(propRef, fieldName) {
+  const { path } = propRef;
+  if (path.length === 1 && path[0] === fieldName) {
+    return true;
+  }
+  if (path.length === 2 && path[1] === fieldName) {
+    return true;
+  }
+  return false;
+}
+function extractFromEq(func, filterField) {
+  if (func.args.length !== 2) return [];
+  const [left, right] = func.args;
+  if (isPropRef(left) && propRefMatchesField(left, filterField) && isValue(right)) {
+    return [right.value];
+  }
+  if (isValue(left) && isPropRef(right) && propRefMatchesField(right, filterField)) {
+    return [left.value];
+  }
+  return [];
+}
+function extractFromIn(func, filterField) {
+  if (func.args.length !== 2) return [];
+  const [left, right] = func.args;
+  if (isPropRef(left) && propRefMatchesField(left, filterField) && isValue(right)) {
+    const val = right.value;
+    if (Array.isArray(val)) {
+      return val;
+    }
+  }
+  return [];
+}
+function walkExpression(expr, filterField) {
+  if (!isFunc(expr)) return [];
+  const { name, args } = expr;
+  const results = [];
+  switch (name) {
+    case `eq`:
+      results.push(...extractFromEq(expr, filterField));
+      break;
+    case `in`:
+      results.push(...extractFromIn(expr, filterField));
+      break;
+    case `and`:
+    case `or`:
+      for (const arg of args) {
+        results.push(...walkExpression(arg, filterField));
+      }
+      break;
+    // For other functions, recursively check their arguments
+    // (in case of nested expressions)
+    default:
+      for (const arg of args) {
+        results.push(...walkExpression(arg, filterField));
+      }
+      break;
+  }
+  return results;
+}
+function extractFilterValues(options, filterField) {
+  const { where } = options;
+  if (!where) {
+    return [];
+  }
+  const values = walkExpression(where, filterField);
+  const seen = /* @__PURE__ */ new Set();
+  const unique = [];
+  for (const value of values) {
+    const key = toKey(value);
+    if (!seen.has(key)) {
+      seen.add(key);
+      unique.push(value);
+    }
+  }
+  return unique;
+}
+function hasFilterField(options, filterField) {
+  return extractFilterValues(options, filterField).length > 0;
+}
+function extractMultipleFilterValues(options, filterDimensions) {
+  const result = {};
+  for (const dim of filterDimensions) {
+    const values = extractFilterValues(options, dim.filterField);
+    if (values.length > 0) {
+      result[dim.convexArg] = values;
+    }
+  }
+  return result;
+}
+const DEFAULT_UPDATED_AT_FIELD = `updatedAt`;
+const DEFAULT_DEBOUNCE_MS = 50;
+const DEFAULT_TAIL_OVERLAP_MS = 1e4;
+const DEFAULT_RESUBSCRIBE_THRESHOLD = 10;
+function normalizeFilterConfig(filters) {
+  if (filters === void 0) return [];
+  return Array.isArray(filters) ? filters : [filters];
+}
+function convexCollectionOptions(config) {
+  const {
+    client,
+    query,
+    filters,
+    updatedAtFieldName = DEFAULT_UPDATED_AT_FIELD,
+    debounceMs = DEFAULT_DEBOUNCE_MS,
+    tailOverlapMs = DEFAULT_TAIL_OVERLAP_MS,
+    resubscribeThreshold = DEFAULT_RESUBSCRIBE_THRESHOLD,
+    getKey,
+    onInsert,
+    onUpdate,
+    ...baseConfig
+  } = config;
+  const filterDimensions = normalizeFilterConfig(filters);
+  const syncManager = new ConvexSyncManager({
+    client,
+    query,
+    filterDimensions,
+    updatedAtFieldName,
+    debounceMs,
+    tailOverlapMs,
+    resubscribeThreshold,
+    getKey
+  });
+  const syncConfig = {
+    sync: (params) => {
+      const { collection, begin, write, commit, markReady } = params;
+      syncManager.setCallbacks({
+        collection: {
+          get: (key) => collection.get(key),
+          has: (key) => collection.has(key)
+        },
+        begin,
+        write,
+        commit,
+        markReady
+      });
+      return {
+        loadSubset: (options) => {
+          if (filterDimensions.length === 0) {
+            return syncManager.requestFilters({});
+          }
+          const extracted = extractMultipleFilterValues(options, filterDimensions);
+          if (Object.keys(extracted).length === 0) {
+            return Promise.resolve();
+          }
+          return syncManager.requestFilters(extracted);
+        },
+        unloadSubset: (options) => {
+          if (filterDimensions.length === 0) {
+            syncManager.releaseFilters({});
+            return;
+          }
+          const extracted = extractMultipleFilterValues(options, filterDimensions);
+          if (Object.keys(extracted).length > 0) {
+            syncManager.releaseFilters(extracted);
+          }
+        },
+        cleanup: () => {
+          syncManager.cleanup();
+        }
+      };
+    }
+  };
+  return {
+    ...baseConfig,
+    getKey,
+    syncMode: `on-demand`,
+    // Always on-demand since we sync based on query predicates
+    sync: syncConfig,
+    onInsert,
+    onUpdate
+  };
+}
+export {
+  ConvexSyncManager,
+  convexCollectionOptions,
+  deserializeValue,
+  extractFilterValues,
+  extractMultipleFilterValues,
+  fromKey,
+  hasFilterField,
+  serializeValue,
+  toKey
+};
+//# sourceMappingURL=index.js.map