@statezero/core 0.2.52 → 0.2.54

package/dist/syncEngine/stores/operationEventHandlers.js

@@ -2,11 +2,7 @@ import { operationEvents, Status, Type, operationRegistry, createMembershipState
 import { modelStoreRegistry } from '../registries/modelStoreRegistry.js';
 import { querysetStoreRegistry } from '../registries/querysetStoreRegistry.js';
 import { metricRegistry } from '../registries/metricRegistry.js';
-import { getFingerprint } from './utils.js';
-import { QuerySet } from '../../flavours/django/querySet.js';
-import { isEqual, isNil } from 'lodash-es';
-import hash from 'object-hash';
-import { filter, applyOrderBy } from '../../filtering/localFiltering.js';
+import { computeMetricFromQueryset } from '../metrics/metricOptCalcs.js';
 import { recordDebugEvent } from '../../debug/statezeroDebug.js';
 /**
  * Check if two model classes represent the same model.
@@ -17,267 +13,8 @@ function isSameModel(modelClassA, modelClassB) {
     return modelClassA.modelName === modelClassB.modelName &&
         modelClassA.configKey === modelClassB.configKey;
 }
-/**
- * Evaluates and routes a CREATE operation to querysets, tracking membership state.
- *
- * Membership state has two flags:
- * - optimistic: Should we show this item immediately in the UI?
- * - verify: Should remote sync check this queryset?
- *
- * For each queryset:
- * - If offset > 0: { optimistic: false, verify: true } - can't determine position
- * - If local filter says no match: { optimistic: false, verify: true } - local filtering unreliable
- * - If item matches filter AND has room: { optimistic: true, verify: false } - show it
- * - If at limit and ordering excludes: { optimistic: false, verify: false } - certain it's out
- *
- * @param {Operation} operation - The CREATE operation to route
- * @param {Function} applyAction - Function to apply the operation to a store
- */
-function routeCreateOperation(operation, applyAction) {
-    const modelClass = operation.queryset.ModelClass;
-    const instances = operation.instances;
-    Array.from(querysetStoreRegistry._stores.entries()).forEach(([semanticKey, store]) => {
-        if (!isSameModel(store.modelClass, modelClass))
-            return;
-        recordDebugEvent({
-            type: "routing",
-            operationId: operation.operationId,
-            operationType: operation.type,
-            semanticKey,
-            modelName: store.modelClass?.modelName,
-            configKey: store.modelClass?.configKey,
-            phase: "start",
-        });
-        const serializerOptions = store.queryset?._serializerOptions || {};
-        const { offset, limit } = serializerOptions;
-        // Offset > 0: can't determine position based on slicing
-        if (offset != null && offset > 0) {
-            operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, true));
-            recordDebugEvent({
-                type: "routing",
-                operationId: operation.operationId,
-                operationType: operation.type,
-                semanticKey,
-                modelName: store.modelClass?.modelName,
-                configKey: store.modelClass?.configKey,
-                decision: "offset>0",
-                optimistic: false,
-                verify: true,
-            });
-            return;
-        }
-        // Evaluate if instances match this queryset's filter
-        // Use fromPk to get live representation with proper FK structure
-        const liveInstances = instances.map(inst => modelClass.fromPk(inst[modelClass.primaryKeyField])).filter(Boolean);
-        const ast = store.queryset.build();
-        const matchingInstances = filter(liveInstances, ast, modelClass, true);
-        if (matchingInstances.length === 0) {
-            // Local filter says no match - trust it (local filtering is robust)
-            operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, false));
-            recordDebugEvent({
-                type: "routing",
-                operationId: operation.operationId,
-                operationType: operation.type,
-                semanticKey,
-                modelName: store.modelClass?.modelName,
-                configKey: store.modelClass?.configKey,
-                decision: "filter:no-match",
-                optimistic: false,
-                verify: false,
-            });
-            return;
-        }
-        // Item matches filter - check limit
-        if (limit != null) {
-            const currentCount = store.groundTruthPks?.length || 0;
-            // At capacity: check if ordering could affect position
-            if (currentCount >= limit) {
-                // Check explicit ordering on queryset OR implicit ordering from Django Meta
-                const hasExplicitOrdering = store.queryset._orderBy && store.queryset._orderBy.length > 0;
-                const hasImplicitOrdering = (modelClass.schema?.default_ordering?.length || 0) > 0;
-                if (hasExplicitOrdering || hasImplicitOrdering) {
-                    // Use local sorting to determine if new items would be in top N
-                    const orderBy = hasExplicitOrdering
-                        ? store.queryset._orderBy
-                        : modelClass.schema.default_ordering;
-                    // Get current items from model store
-                    const modelStore = modelStoreRegistry.getStore(modelClass);
-                    const renderedItems = modelStore.render(store.groundTruthPks, true, false) || [];
-                    const currentItems = renderedItems.filter(Boolean);
-                    // If some ground truth items couldn't be rendered, we can't trust local sort
-                    if (currentItems.length < store.groundTruthPks.length) {
-                        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, true));
-                        recordDebugEvent({
-                            type: "routing",
-                            operationId: operation.operationId,
-                            operationType: operation.type,
-                            semanticKey,
-                            modelName: store.modelClass?.modelName,
-                            configKey: store.modelClass?.configKey,
-                            decision: "limit:ordering:incomplete-render",
-                            optimistic: false,
-                            verify: true,
-                        });
-                        return;
-                    }
-                    const allItems = [...currentItems, ...matchingInstances];
-                    // Sort and take top N
-                    const sorted = applyOrderBy(allItems, orderBy, modelClass);
-                    const topN = sorted.slice(0, limit);
-                    const topNPks = new Set(topN.map(item => item[modelClass.primaryKeyField]));
-                    // Check if any new items made it into the top N
-                    const newItemPks = matchingInstances.map(item => item[modelClass.primaryKeyField]);
-                    const anyInTopN = newItemPks.some(pk => topNPks.has(pk));
-                    if (anyInTopN) {
-                        applyAction(store);
-                        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(true, false));
-                        recordDebugEvent({
-                            type: "routing",
-                            operationId: operation.operationId,
-                            operationType: operation.type,
-                            semanticKey,
-                            modelName: store.modelClass?.modelName,
-                            configKey: store.modelClass?.configKey,
-                            decision: "limit:ordering:in-top",
-                            optimistic: true,
-                            verify: false,
-                        });
-                    }
-                    else {
-                        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, false));
-                        recordDebugEvent({
-                            type: "routing",
-                            operationId: operation.operationId,
-                            operationType: operation.type,
-                            semanticKey,
-                            modelName: store.modelClass?.modelName,
-                            configKey: store.modelClass?.configKey,
-                            decision: "limit:ordering:out",
-                            optimistic: false,
-                            verify: false,
-                        });
-                    }
-                }
-                else {
-                    // No ordering - new items go at end, won't be in first N
-                    operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, false));
-                    recordDebugEvent({
-                        type: "routing",
-                        operationId: operation.operationId,
-                        operationType: operation.type,
-                        semanticKey,
-                        modelName: store.modelClass?.modelName,
-                        configKey: store.modelClass?.configKey,
-                        decision: "limit:no-ordering",
-                        optimistic: false,
-                        verify: false,
-                    });
-                }
-                return;
-            }
-            // Room for some or all items - continue to DEFINITELY_YES below
-        }
-        // Matches filter, has room or no limit - DEFINITELY_YES
-        applyAction(store);
-        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(true, false));
-        recordDebugEvent({
-            type: "routing",
-            operationId: operation.operationId,
-            operationType: operation.type,
-            semanticKey,
-            modelName: store.modelClass?.modelName,
-            configKey: store.modelClass?.configKey,
-            decision: "match:has-room",
-            optimistic: true,
-            verify: false,
-        });
-    });
-}
-/**
- * Evaluates and tracks membership state for UPDATE operations.
- *
- * For UPDATE, we check filter FIRST, then offset:
- * - If local filter says no match: { optimistic: false, verify: true } - local filtering unreliable
- * - If matched AND offset > 0: { optimistic: false, verify: true } - can't determine position
- * - If matched AND offset = 0: { optimistic: true, verify: false } - show it
- *
- * @param {Operation} operation - The UPDATE operation
- */
-function routeUpdateOperation(operation, applyAction) {
-    const modelClass = operation.queryset.ModelClass;
-    // frozenInstances is already serialized with proper FK structure
-    const beforeInstances = operation.frozenInstances;
-    // For after state, use fromPk to get current live representation
-    const afterInstances = operation.instances.map(inst => modelClass.fromPk(inst[modelClass.primaryKeyField])).filter(Boolean);
-    Array.from(querysetStoreRegistry._stores.entries()).forEach(([semanticKey, store]) => {
-        if (!isSameModel(store.modelClass, modelClass))
-            return;
-        // Check filter match FIRST (optimization: skip offset check if no match)
-        const ast = store.queryset.build();
-        const matchedBefore = filter(beforeInstances, ast, modelClass, false).length > 0;
-        const matchesAfter = filter(afterInstances, ast, modelClass, false).length > 0;
-        // If local filter says no match before AND after, trust it
-        if (!matchedBefore && !matchesAfter) {
-            operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, false));
-            return;
-        }
-        // Item matches filter - now check offset
-        const serializerOptions = store.queryset?._serializerOptions || {};
-        const { offset } = serializerOptions;
-        if (offset != null && offset > 0) {
-            // Can't determine position in paginated window
-            operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, true));
-            return;
-        }
-        // Item matched before OR matches after, no offset - definitely affected
-        // Apply the operation so the queryset can add/remove PKs from its candidate set
-        applyAction(store);
-        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(true, false));
-    });
-}
-/**
- * Evaluates and tracks membership state for DELETE operations.
- *
- * For DELETE, we check filter FIRST, then offset:
- * - If local filter says no match: { optimistic: false, verify: true } - local filtering unreliable
- * - If matched AND offset > 0: { optimistic: false, verify: true } - can't determine position
- * - If matched AND offset = 0: { optimistic: true, verify: false } - show deletion
- *
- * @param {Operation} operation - The DELETE operation
- */
-function routeDeleteOperation(operation, applyAction) {
-    const modelClass = operation.queryset.ModelClass;
-    // frozenInstances is already serialized with proper FK structure
-    const beforeInstances = operation.frozenInstances;
-    Array.from(querysetStoreRegistry._stores.entries()).forEach(([semanticKey, store]) => {
-        if (!isSameModel(store.modelClass, modelClass))
-            return;
-        // Check filter match FIRST (optimization: skip offset check if no match)
-        const ast = store.queryset.build();
-        const matchedBefore = filter(beforeInstances, ast, modelClass, false).length > 0;
-        // If local filter says no match before, trust it
-        if (!matchedBefore) {
-            operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, false));
-            return;
-        }
-        // Item matched filter - now check offset
-        const serializerOptions = store.queryset?._serializerOptions || {};
-        const { offset } = serializerOptions;
-        if (offset != null && offset > 0) {
-            // Can't determine position in paginated window
-            operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(false, true));
-            return;
-        }
-        // Item matched before, no offset - definitely affected
-        applyAction(store);
-        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(true, false));
-    });
-}
 /**
  * Process an operation in the model store
- *
- * @param {Operation} operation - The operation to process
- * @param {string} actionType - The action to perform ('add', 'update', 'confirm', 'reject')
  */
 function processModelStore(operation, actionType) {
     const ModelClass = operation.queryset.ModelClass;
@@ -300,115 +37,110 @@ function processModelStore(operation, actionType) {
     }
 }
 /**
- * Process an operation in the queryset stores based on operation type
- * Uses different routing strategies based on the operation type
- *
- * @param {Operation} operation - The operation to process
- * @param {string} actionType - The action to perform ('add', 'update', 'confirm', 'reject')
+ * Determine if a queryset store needs server verification after a local operation.
+ * Offset querysets always need it. No-offset querysets only need it when the page
+ * is full (an item beyond the limit might need to scroll in).
+ */
+function needsVerify(store) {
+    const hasOffset = (store.queryset._serializerOptions?.offset ?? 0) > 0;
+    if (hasOffset)
+        return true;
+    const limit = store.queryset._serializerOptions?.limit;
+    if (!limit)
+        return false;
+    return store.render().length >= limit;
+}
+/**
+ * Track membership state for sync optimization.
+ * No-offset querysets with unfilled pages skip sync — local filtering is perfect.
+ * Offset querysets and full pages need server verification.
  */
 function processQuerysetStores(operation, actionType) {
+    if (operation.type === Type.CHECKPOINT)
+        return;
+    if (actionType !== 'add')
+        return;
     const ModelClass = operation.queryset.ModelClass;
-    const queryset = operation.queryset;
-    // Apply the appropriate action to a single queryset store
-    const applyAction = (store) => {
-        switch (actionType) {
-            case 'add':
-                store.addOperation(operation);
-                break;
-            case 'update':
-                store.updateOperation(operation);
-                break;
-            case 'confirm':
-                store.confirm(operation);
-                break;
-            case 'reject':
-                store.reject(operation);
-                break;
-        }
-    };
-    let querysetStoreMap;
-    // Route to querysets based on operation type
-    // All operation types track membership state for sync optimization
-    // CREATE: adds operation to store + tracks membership
-    // UPDATE/DELETE: only tracks membership (model store + reactivity handles rendering)
-    switch (operation.type) {
-        case Type.CREATE:
-        case Type.BULK_CREATE:
-        case Type.GET_OR_CREATE:
-        case Type.UPDATE_OR_CREATE:
-            // For creates, evaluate each queryset and track membership state
-            // This allows us to skip syncing querysets that we know don't contain the item
-            routeCreateOperation(operation, applyAction);
-            return;
-        case Type.UPDATE:
-        case Type.UPDATE_INSTANCE:
-            // Route update to affected querysets so items can move between filtered sets
-            // (e.g., an item changing name from 'beta' to 'alpha' needs to appear in the alpha queryset)
-            routeUpdateOperation(operation, applyAction);
-            return;
-        case Type.DELETE:
-        case Type.DELETE_INSTANCE:
-            // Add delete to matching queryset stores for optimistic removal
-            routeDeleteOperation(operation, applyAction);
-            return;
-        case Type.CHECKPOINT:
-            // Model store handles the change, querysets re-render via local filtering
-            // No membership tracking needed for checkpoints
-            return;
-        default:
-            // For other operation types, route like creates
-            routeCreateOperation(operation, applyAction);
-            return;
+    for (const [semanticKey, store] of querysetStoreRegistry._stores.entries()) {
+        if (!isSameModel(store.modelClass, ModelClass))
+            continue;
+        operationRegistry.setQuerysetState(operation.operationId, semanticKey, createMembershipState(true, needsVerify(store)));
     }
 }
 /**
- * Process an operation in the metric stores
- *
- * For metrics, we route operations UP the family tree - any metric on an ancestor
- * queryset should receive the operation so it can check if it affects the metric.
- *
- * @param {Operation} operation - The operation to process
- * @param {string} actionType - The action to perform ('add', 'update', 'confirm', 'reject')
+ * Snapshot current metric values BEFORE any changes are applied.
+ * Returns a Map of metricStoreKey → currentValue.
  */
-function processMetricStores(operation, actionType) {
+function snapshotMetrics(operation) {
     const queryset = operation.queryset;
-    const allMetricStores = new Set();
-    // Walk up the queryset family tree and collect all metrics
+    const snapshots = new Map();
     let current = queryset;
     while (current) {
         const stores = metricRegistry.getAllStoresForQueryset(current);
         if (stores && stores.length > 0) {
-            stores.forEach(store => allMetricStores.add(store));
+            for (const store of stores) {
+                const key = store.cacheKey;
+                if (!snapshots.has(key)) {
+                    snapshots.set(key, {
+                        store,
+                        value: store._lastCalculatedValue ?? store.groundTruthValue
+                    });
+                }
+            }
         }
         current = current.__parent;
     }
-    if (allMetricStores.size === 0) {
+    return snapshots;
+}
+/**
+ * Process metric stores using before/after delta approach.
+ *
+ * For 'add'/'update': compute new metric value from queryset store,
+ * calculate delta vs snapshot, store delta.
+ * For 'confirm'/'reject': delegate to metric store.
+ *
+ * CHECKPOINTs are data refreshes, not business operations — they must not
+ * affect aggregation metrics.
+ */
+function processMetricStores(operation, actionType, metricSnapshots) {
+    if (operation.type === Type.CHECKPOINT)
         return;
-    }
-    // Apply the action to each matching metric store
-    allMetricStores.forEach(store => {
+    if (!metricSnapshots || metricSnapshots.size === 0)
+        return;
+    for (const [key, snapshot] of metricSnapshots) {
+        const store = snapshot.store;
         switch (actionType) {
-            case 'add':
-                store.addOperation(operation);
+            case 'add': {
+                const qsStore = querysetStoreRegistry.getStore(store.queryset);
+                const newValue = computeMetricFromQueryset(store.metricType, qsStore, store.field, store.modelClass);
+                const beforeValue = snapshot.value ?? 0;
+                const delta = newValue - beforeValue;
+                if (delta !== 0) {
+                    store.addDelta(operation.operationId, delta);
+                }
                 break;
-            case 'update':
-                store.updateOperation(operation);
+            }
+            case 'update': {
+                // Recompute — the operation may have been mutated
+                const qsStore = querysetStoreRegistry.getStore(store.queryset);
+                const newValue = computeMetricFromQueryset(store.metricType, qsStore, store.field, store.modelClass);
+                const beforeValue = snapshot.value ?? 0;
+                const delta = newValue - beforeValue;
+                store.updateDelta(operation.operationId, delta);
                 break;
+            }
             case 'confirm':
-                store.confirm(operation);
+                store.confirmDelta(operation.operationId);
                 break;
             case 'reject':
-                store.reject(operation);
+                store.rejectDelta(operation.operationId);
                 break;
         }
-    });
+    }
 }
 /**
  * Common processing logic for operations, handling validation and routing
  * to the appropriate store processors
- *
- * @param {Operation} operation - The operation to process
- * @param {string} actionType - The action to perform ('add', 'update', 'confirm', 'reject')
  */
 function processOperation(operation, actionType) {
     if (!operation || !operation.queryset || !operation.queryset.ModelClass) {
@@ -418,12 +150,14 @@ function processOperation(operation, actionType) {
     if (operation.doNotPropagate) {
         return;
     }
-    // Process model store first
+    // 1. Snapshot metric values BEFORE changes
+    const metricSnapshots = snapshotMetrics(operation);
+    // 2. Model store — updates instance data, triggers renders on ALL queryset stores
     processModelStore(operation, actionType);
-    // Then process queryset stores with improved routing
+    // 3. Queryset stores — track membership state for sync
     processQuerysetStores(operation, actionType);
-    // Finally process metric stores
-    processMetricStores(operation, actionType);
+    // 4. Metrics (before/after diff)
+    processMetricStores(operation, actionType, metricSnapshots);
 }
 // Define handlers as named arrow functions at the top level
 const handleOperationCreated = operation => processOperation(operation, 'add');

package/dist/syncEngine/stores/querysetStore.d.ts

@@ -7,6 +7,7 @@ export class QuerysetStore {
     groundTruthPks: never[];
     isSyncing: boolean;
     lastSync: number | null;
+    _createdAt: number;
     isTemp: any;
     pruneThreshold: any;
     includedPks: Map<any, any>;
@@ -43,13 +44,17 @@ export class QuerysetStore {
      * @private
      */
     private _getValidatedAndFilteredPks;
-    render(optimistic?: boolean, fromCache?: boolean): any[];
-    renderFromData(optimistic?: boolean): any[];
     /**
-     * Render by getting all instances from the model store and filtering locally.
-     * Used when a queryset has no ground truth (temp stores, newly created stores, etc.)
+     * For offset pages that aren't full, expand with creates from the model store
+     * that sort after the first item on the page (direction-adjusted).
      */
-    renderFromModelStore(): any[];
+    _fillOffsetPage(currentPks: any, optimistic: any): any[] | null;
+    render(optimistic?: boolean, fromCache?: boolean): any[];
+    renderFromData(optimistic?: boolean): any;
+    /** Are there any inflight or recent confirmed ops in the model store? */
+    _hasRecentOps(modelStore: any): boolean;
+    /** Build set of PKs from ops newer than lastSync. Only used for no-offset querysets. */
+    _buildFreshPks(modelStore: any, optimistic: any): Set<any>;
     applyOperation(operation: any, currentPks: any): any;
     /**
      * Sync this queryset with the database.