@statezero/core 0.2.38 → 0.2.39

package/dist/syncEngine/stores/modelStore.js

@@ -0,0 +1,565 @@
+import { Operation, Status, Type, operationRegistry } from './operation.js';
+import { isNil, isEmpty, trim, isEqual } from 'lodash-es';
+import { modelEventEmitter } from './reactivity.js';
+import { Cache } from '../cache/cache.js';
+import { replaceTempPks, containsTempPk, resolveToRealPk, onTempPkResolved } from '../../flavours/django/tempPk.js';
+const emitEvents = (store, event) => {
+    if (!event || !event.pks || event.pks.length === 0)
+        return;
+    // Get prior values for these pks in one pass (before computing new)
+    const lastRenderedDataArray = event.pks.map(pk => store._lastRenderedData.get(pk) ?? null);
+    // Batch render all pks - useCache=false to get fresh data for comparison
+    const newRenderedDataArray = store.render(event.pks, true, false);
+    // Single equality check on the whole batch
+    if (!isEqual(newRenderedDataArray, lastRenderedDataArray)) {
+        // Update cache for all pks
+        const pkField = store.pkField;
+        for (const item of newRenderedDataArray) {
+            if (item && item[pkField] != null) {
+                store._lastRenderedData.set(item[pkField], item);
+            }
+        }
+        // Also mark any pks that are now null (deleted)
+        for (const pk of event.pks) {
+            if (!newRenderedDataArray.some(item => item && item[pkField] === pk)) {
+                store._lastRenderedData.set(pk, null);
+            }
+        }
+        modelEventEmitter.emit(`${store.modelClass.configKey}::${store.modelClass.modelName}::render`, event);
+        store.renderCallbacks.forEach((callback) => {
+            try {
+                callback();
+            }
+            catch (error) {
+                console.warn("Error in model store render callback:", error);
+            }
+        });
+    }
+};
+class EventData {
+    constructor(ModelClass, pks) {
+        this.ModelClass = ModelClass;
+        this.pks = Array.isArray(pks) ? pks : [pks];
+    }
+    /**
+     * Single event containing all PKs from an operation
+     */
+    static fromOperation(operation) {
+        const ModelClass = operation.queryset.ModelClass;
+        const pkField = ModelClass.primaryKeyField;
+        const pks = operation.instances
+            .filter(instance => instance != null && typeof instance === 'object' && pkField in instance)
+            .map(instance => instance[pkField]);
+        return pks.length > 0 ? new EventData(ModelClass, pks) : null;
+    }
+    /**
+     * Single event containing all unique PKs across multiple operations
+     */
+    static fromOperations(operations) {
+        if (!operations.length)
+            return null;
+        const ModelClass = operations[0].queryset.ModelClass;
+        const pkField = ModelClass.primaryKeyField;
+        const uniquePks = new Set();
+        for (const op of operations) {
+            for (const inst of op.instances) {
+                if (inst != null && typeof inst === 'object' && pkField in inst) {
+                    uniquePks.add(inst[pkField]);
+                }
+            }
+        }
+        return uniquePks.size > 0 ? new EventData(ModelClass, Array.from(uniquePks)) : null;
+    }
+    /**
+     * Single event containing all unique PKs from an array of instances
+     */
+    static fromInstances(instances, ModelClass) {
+        const pkField = ModelClass.primaryKeyField;
+        const uniquePks = new Set(instances
+            .filter(inst => inst && inst[pkField] != null)
+            .map(inst => inst[pkField]));
+        return uniquePks.size > 0 ? new EventData(ModelClass, Array.from(uniquePks)) : null;
+    }
+}
+export class ModelStore {
+    constructor(modelClass, fetchFn, initialGroundTruth = null, initialOperations = null, options = {}) {
+        this.modelClass = modelClass;
+        this.fetchFn = fetchFn;
+        this.isSyncing = false;
+        this.pruneThreshold = options.pruneThreshold || 10;
+        this.groundTruthArray = initialGroundTruth || [];
+        this.operationsMap = new Map();
+        // Handle initial operations if provided
+        if (initialOperations && initialOperations.length > 0) {
+            this._loadOperations(initialOperations);
+        }
+        this.modelCache = new Cache("model-cache", {}, this.onHydrated.bind(this));
+        this._lastRenderedData = new Map();
+        this.renderCallbacks = new Set();
+        // Migrate cache entries when temp pks are resolved to real pks
+        this._unsubscribeTempPk = onTempPkResolved((tempPk, realPk) => {
+            if (this._lastRenderedData.has(tempPk)) {
+                const data = this._lastRenderedData.get(tempPk);
+                // Update the pk field in the cached data
+                if (data && typeof data === 'object') {
+                    data[this.pkField] = realPk;
+                }
+                this._lastRenderedData.set(realPk, data);
+                this._lastRenderedData.delete(tempPk);
+            }
+        });
+    }
+    registerRenderCallback(callback) {
+        this.renderCallbacks.add(callback);
+        return () => this.renderCallbacks.delete(callback);
+    }
+    /**
+     * Load operations from data and add them to the operations map,
+     * reusing existing operations from the registry if they exist
+     */
+    _loadOperations(operationsData) {
+        operationsData.forEach((opData) => {
+            const existingOp = operationRegistry.get(opData.operationId);
+            if (existingOp) {
+                // If the operation exists in the registry, use it
+                this.operationsMap.set(existingOp.operationId, existingOp);
+            }
+            else {
+                // Otherwise just use the plain object data
+                this.operationsMap.set(opData.operationId, new Operation(opData, true));
+            }
+        });
+    }
+    // Caching
+    get cacheKey() {
+        return `${this.modelClass.configKey}::${this.modelClass.modelName}`;
+    }
+    onHydrated() {
+        if (this.groundTruthArray.length === 0 && this.operationsMap.size === 0) {
+            let cached = this.modelCache.get(this.cacheKey);
+            console.log(`[ModelStore] Hydrated ${this.modelClass.modelName} with ${(cached || []).length} items from cache`);
+            if (!isNil(cached) && !isEmpty(cached)) {
+                this.setGroundTruth(cached);
+            }
+        }
+    }
+    setCache(result) {
+        const pkField = this.pkField;
+        let nonTempPkItems = [];
+        result.forEach((item) => {
+            let pk = item[pkField];
+            if (typeof pk === "string" && containsTempPk(pk)) {
+                pk = replaceTempPks(item[pkField]);
+                if (isNil(pk) || isEmpty(trim(pk))) {
+                    return;
+                }
+            }
+            if (item && typeof item.serialize === "function") {
+                // Pass includeRepr=true to preserve repr field in cache
+                const serializedItem = item.serialize(true);
+                serializedItem[pkField] = pk;
+                nonTempPkItems.push(serializedItem);
+            }
+            else {
+                item[pkField] = pk;
+                nonTempPkItems.push(item);
+            }
+        });
+        this.modelCache.set(this.cacheKey, nonTempPkItems);
+    }
+    clearCache() {
+        this.modelCache.delete(this.cacheKey);
+    }
+    updateCache(items, requestedPks) {
+        const pkField = this.pkField;
+        let nonTempPkItems = [];
+        items.forEach((item) => {
+            let pk = item[pkField];
+            if (typeof pk === "string" && containsTempPk(pk)) {
+                pk = replaceTempPks(item[pkField]);
+                if (isNil(pk) || isEmpty(trim(pk))) {
+                    return;
+                }
+            }
+            item[pkField] = pk;
+            nonTempPkItems.push(item);
+        });
+        // If rendering ALL items (requestedPks is null), simply replace the cache
+        if (requestedPks === null) {
+            this.setCache(nonTempPkItems);
+            return;
+        }
+        // Otherwise, we're rendering specific items - update only those items
+        const currentCache = this.modelCache.get(this.cacheKey) || [];
+        // Filter out items that were requested but not in the result (they were deleted)
+        const filteredCache = currentCache.filter((item) => item &&
+            typeof item === "object" &&
+            pkField in item &&
+            (!requestedPks.has(item[pkField]) ||
+                nonTempPkItems.some((newItem) => newItem[pkField] === item[pkField])));
+        // Create a map for faster lookups
+        const cacheMap = new Map(filteredCache.map((item) => [item[pkField], item]));
+        // Add or update items from the result
+        for (const item of nonTempPkItems) {
+            if (item && typeof item === "object" && pkField in item) {
+                cacheMap.set(item[pkField], item);
+            }
+        }
+        // Update the cache
+        const updatedCache = Array.from(cacheMap.values());
+        this.setCache(updatedCache);
+    }
+    // Main modelStore methods
+    get operations() {
+        return Array.from(this.operationsMap.values());
+    }
+    get pkField() {
+        return this.modelClass.primaryKeyField;
+    }
+    // Commit optimistic updates
+    addOperation(operation) {
+        this.operationsMap.set(operation.operationId, operation);
+        if (this.operationsMap.size > this.pruneThreshold) {
+            this.prune();
+        }
+        emitEvents(this, EventData.fromOperation(operation));
+    }
+    updateOperation(operation) {
+        if (!this.operationsMap.has(operation.operationId))
+            return false;
+        this.operationsMap.set(operation.operationId, operation);
+        emitEvents(this, EventData.fromOperation(operation));
+        return true;
+    }
+    confirm(operation) {
+        if (!this.operationsMap.has(operation.operationId))
+            return;
+        this.operationsMap.set(operation.operationId, operation);
+        emitEvents(this, EventData.fromOperation(operation));
+    }
+    reject(operation) {
+        if (!this.operationsMap.has(operation.operationId))
+            return;
+        this.operationsMap.set(operation.operationId, operation);
+        emitEvents(this, EventData.fromOperation(operation));
+    }
+    setOperations(operations = []) {
+        const prevOps = this.operations;
+        this.operationsMap.clear();
+        operations.forEach((op) => {
+            this.operationsMap.set(op.operationId, op);
+        });
+        const allOps = [...prevOps, ...this.operations];
+        emitEvents(this, EventData.fromOperations(allOps));
+    }
+    // Ground truth data methods
+    setGroundTruth(groundTruth) {
+        let prevGroundTruth = this.groundTruthArray;
+        this.groundTruthArray = Array.isArray(groundTruth) ? groundTruth : [];
+        // reactivity - gather all ops
+        const allOps = [...prevGroundTruth, ...this.groundTruthArray];
+        emitEvents(this, EventData.fromInstances(allOps, this.modelClass));
+    }
+    getGroundTruth() {
+        return this.groundTruthArray;
+    }
+    get groundTruthPks() {
+        const pk = this.pkField;
+        return this.groundTruthArray
+            .filter((instance) => instance && typeof instance === "object" && pk in instance)
+            .map((instance) => instance[pk]);
+    }
+    addToGroundTruth(instances) {
+        if (!Array.isArray(instances) || instances.length === 0)
+            return;
+        const pkField = this.pkField;
+        const pkMap = new Map();
+        instances.forEach((inst) => {
+            if (inst && typeof inst === "object" && pkField in inst) {
+                pkMap.set(inst[pkField], inst);
+            }
+            else {
+                console.warn(`[ModelStore ${this.modelClass.modelName}] Skipping invalid instance in addToGroundTruth:`, inst);
+            }
+        });
+        if (pkMap.size === 0)
+            return;
+        const updatedGroundTruth = [];
+        const processedPks = new Set();
+        const checkpointInstances = []; // Track instances that need CHECKPOINT operations
+        for (const existingItem of this.groundTruthArray) {
+            if (!existingItem ||
+                typeof existingItem !== "object" ||
+                !(pkField in existingItem)) {
+                continue;
+            }
+            const pk = existingItem[pkField];
+            if (pkMap.has(pk)) {
+                const updatedInstance = { ...existingItem, ...pkMap.get(pk) };
+                updatedGroundTruth.push(updatedInstance);
+                processedPks.add(pk);
+                // This instance already existed - add it to checkpoint list
+                checkpointInstances.push(updatedInstance);
+                pkMap.delete(pk);
+            }
+            else {
+                updatedGroundTruth.push(existingItem);
+            }
+        }
+        // Add completely new instances (these don't need checkpoint operations)
+        updatedGroundTruth.push(...Array.from(pkMap.values()));
+        this.groundTruthArray = updatedGroundTruth;
+        // Create CHECKPOINT operation for instances that already existed
+        if (checkpointInstances.length > 0) {
+            const checkpointOperation = new Operation({
+                operationId: `checkpoint_${Date.now()}_${Math.random()
+                    .toString(36)
+                    .substr(2, 9)}`,
+                type: Type.CHECKPOINT,
+                instances: checkpointInstances,
+                status: Status.CONFIRMED,
+                timestamp: Date.now(),
+                queryset: this.modelClass.objects.all(),
+            });
+            this.operationsMap.set(checkpointOperation.operationId, checkpointOperation);
+            console.log(`[ModelStore ${this.modelClass.modelName}] Created CHECKPOINT operation for ${checkpointInstances.length} existing instances`);
+        }
+        // reactivity - use all the newly added instances (both new and updated)
+        emitEvents(this, EventData.fromInstances([...checkpointInstances, ...Array.from(pkMap.values())], this.modelClass));
+    }
+    _filteredOperations(pks, operations) {
+        if (!pks)
+            return operations;
+        const pkField = this.pkField;
+        let filteredOps = [];
+        for (const op of operations) {
+            let relevantInstances = op.instances.filter((instance) => pks.has(instance[pkField] || instance));
+            if (relevantInstances.length > 0) {
+                filteredOps.push({
+                    operationId: op.operationId,
+                    instances: relevantInstances,
+                    timestamp: op.timestamp,
+                    queryset: op.queryset,
+                    type: op.type,
+                    status: op.status,
+                    args: op.args,
+                });
+            }
+        }
+        return filteredOps;
+    }
+    _filteredGroundTruth(pks, groundTruthArray) {
+        const pkField = this.pkField;
+        let groundTruthMap = new Map();
+        for (const instance of groundTruthArray) {
+            if (!instance || typeof instance !== "object" || !(pkField in instance)) {
+                continue;
+            }
+            const pk = instance[pkField];
+            if (!pks || pks.has(pk)) {
+                groundTruthMap.set(pk, instance);
+            }
+        }
+        return groundTruthMap;
+    }
+    applyOperation(operation, currentInstances) {
+        const pkField = this.pkField;
+        for (const instance of operation.instances) {
+            if (!instance || typeof instance !== "object" || !(pkField in instance)) {
+                console.warn(`[ModelStore ${this.modelClass.modelName}] Skipping instance ${instance} in operation ${operation.operationId} during applyOperation due to missing PK field '${String(pkField)}' or invalid format.`);
+                continue;
+            }
+            let pk = instance[pkField];
+            switch (operation.type) {
+                case Type.CREATE:
+                case Type.BULK_CREATE:
+                    if (!currentInstances.has(pk)) {
+                        currentInstances.set(pk, instance);
+                    }
+                    break;
+                case Type.CHECKPOINT:
+                case Type.UPDATE_INSTANCE:
+                case Type.UPDATE: {
+                    const existing = currentInstances.get(pk);
+                    if (existing) {
+                        currentInstances.set(pk, { ...existing, ...instance });
+                    }
+                    else {
+                        const wasDeletedLocally = this.operations.some((op) => op.type === Type.DELETE &&
+                            op.status !== Status.REJECTED &&
+                            op.instances.some((inst) => inst && inst[pkField] === pk));
+                        if (!wasDeletedLocally) {
+                            currentInstances.set(pk, instance);
+                        }
+                    }
+                    break;
+                }
+                case Type.DELETE_INSTANCE:
+                case Type.DELETE:
+                    currentInstances.delete(pk);
+                    break;
+                default:
+                    console.error(`[ModelStore ${this.modelClass.modelName}] Unknown operation type: ${operation.type}`);
+            }
+        }
+        return currentInstances;
+    }
+    getTrimmedOperations() {
+        const twoMinutesAgo = Date.now() - 1000 * 60 * 2;
+        return this.operations.filter((operation) => operation.timestamp > twoMinutesAgo);
+    }
+    getInflightOperations() {
+        return this.operations.filter((operation) => operation.status != Status.CONFIRMED &&
+            operation.status != Status.REJECTED);
+    }
+    // Pruning
+    prune() {
+        let renderedPks = this.render(null, false);
+        this.setGroundTruth(renderedPks);
+        this.setOperations(this.getInflightOperations());
+        this.setCache(renderedPks);
+    }
+    /**
+     * Prune model instances that aren't referenced by any queryset store.
+     * This prevents unbounded cache growth from included nested models.
+     *
+     * @param {QuerysetStoreRegistry} querysetStoreRegistry - The registry to check for queryset references
+     */
+    pruneUnreferencedInstances(querysetStoreRegistry) {
+        const pkField = this.pkField;
+        const modelName = this.modelClass.modelName;
+        // Collect all PKs that are needed by ANY queryset (permanent or temporary)
+        const neededPks = new Set();
+        for (const [semanticKey, store] of querysetStoreRegistry._stores.entries()) {
+            // Check top-level PKs (groundTruthPks)
+            if (store.modelClass.modelName === modelName) {
+                store.groundTruthPks.forEach(pk => neededPks.add(pk));
+            }
+            // Check included PKs (nested models)
+            if (store.includedPks.has(modelName)) {
+                const includedPkSet = store.includedPks.get(modelName);
+                includedPkSet.forEach(pk => neededPks.add(pk));
+            }
+        }
+        // Filter ground truth to only keep needed PKs
+        const filteredGroundTruth = this.groundTruthArray.filter(instance => {
+            if (!instance || typeof instance !== 'object' || !(pkField in instance)) {
+                return false;
+            }
+            return neededPks.has(instance[pkField]);
+        });
+        const removedCount = this.groundTruthArray.length - filteredGroundTruth.length;
+        if (removedCount > 0) {
+            console.log(`[ModelStore ${modelName}] Pruned ${removedCount} unreferenced instances (${filteredGroundTruth.length} remaining)`);
+            this.groundTruthArray = filteredGroundTruth;
+            // Update the cache to reflect the pruned data
+            this.setCache(filteredGroundTruth);
+        }
+    }
+    // Render methods
+    render(pks = null, optimistic = true, useCache = true) {
+        // When rendering specific pks, check cache first
+        if (useCache && pks !== null) {
+            const pksArray = Array.isArray(pks) ? pks : [pks];
+            // Resolve temp pks to real pks for cache lookup
+            const resolvedPks = pksArray.map(pk => resolveToRealPk(pk));
+            const uncachedPks = [];
+            for (const pk of resolvedPks) {
+                if (!this._lastRenderedData.has(pk)) {
+                    uncachedPks.push(pk);
+                }
+            }
+            // All pks were cached - return from cache
+            if (uncachedPks.length === 0) {
+                return this._renderFromCache(resolvedPks);
+            }
+            // Some or all pks need fresh render
+            const pkField = this.pkField;
+            if (uncachedPks.length < resolvedPks.length) {
+                // Partial cache hit - get cached and render uncached
+                const cachedPks = resolvedPks.filter(pk => this._lastRenderedData.has(pk));
+                const cached = this._renderFromCache(cachedPks);
+                const fresh = this._renderFresh(uncachedPks, optimistic);
+                // Update cache for freshly rendered items
+                for (const item of fresh) {
+                    if (item && item[pkField] != null) {
+                        this._lastRenderedData.set(item[pkField], item);
+                    }
+                }
+                return [...cached, ...fresh];
+            }
+            else {
+                // All pks uncached - render fresh and update cache
+                const fresh = this._renderFresh(resolvedPks, optimistic);
+                // Update cache for freshly rendered items
+                for (const item of fresh) {
+                    if (item && item[pkField] != null) {
+                        this._lastRenderedData.set(item[pkField], item);
+                    }
+                }
+                return fresh;
+            }
+        }
+        // Full render (no cache, useCache=false, or pks=null for all)
+        return this._renderFresh(pks, optimistic);
+    }
+    _renderFromCache(pks) {
+        // Returns cached rendered data for the given pks
+        // This method can be spied on in tests to track cache hits
+        return pks
+            .map(pk => this._lastRenderedData.get(pk))
+            .filter(item => item !== null && item !== undefined);
+    }
+    _renderFresh(pks = null, optimistic = true) {
+        const pksSet = pks === null
+            ? null
+            : pks instanceof Set
+                ? pks
+                : new Set(Array.isArray(pks) ? pks : [pks]);
+        const renderedInstancesMap = this._filteredGroundTruth(pksSet, this.groundTruthArray);
+        const relevantOperations = this._filteredOperations(pksSet, this.operations);
+        for (const op of relevantOperations) {
+            if (op.status !== Status.REJECTED &&
+                (optimistic || op.status === Status.CONFIRMED)) {
+                this.applyOperation(op, renderedInstancesMap);
+            }
+        }
+        let result = Array.from(renderedInstancesMap.values());
+        if (pks)
+            this.updateCache(result, pksSet);
+        if (isNil(pks))
+            this.setCache(result);
+        return result;
+    }
+    async sync(pks = null) {
+        const storeIdForLog = this.modelClass.modelName;
+        if (this.isSyncing)
+            return;
+        this.isSyncing = true;
+        try {
+            const currentPks = pks || this.groundTruthPks;
+            if (currentPks.length === 0) {
+                const trimmedOps = this.getTrimmedOperations();
+                this.setOperations(trimmedOps);
+                return;
+            }
+            const newGroundTruth = await this.fetchFn({
+                pks: currentPks,
+                modelClass: this.modelClass,
+            });
+            if (pks) {
+                this.addToGroundTruth(newGroundTruth);
+                return;
+            }
+            this.setGroundTruth(newGroundTruth);
+            const trimmedOps = this.getTrimmedOperations();
+            this.setOperations(trimmedOps);
+        }
+        catch (error) {
+            console.error(`[ModelStore ${storeIdForLog}] Failed to sync ground truth:`, error);
+        }
+        finally {
+            this.isSyncing = false;
+        }
+    }
+}