@statezero/core 0.2.38 → 0.2.39

package/dist/syncEngine/stores/utils.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Guarantees an array of operations, is an array of operations. Handles
+ * the basic failure case of a single operation.
+ */
+export function validateOperationsArray(operationsArray: any): any;
+/**
+ * Extract the minimal signature we're using for “direct‐match” comparison.
+ */
+export function getFingerprint(qs: any): {
+    nodes: any;
+    initialQueryset: any;
+    offset: any;
+    limit: any;
+};

package/dist/syncEngine/stores/utils.js

@@ -0,0 +1,32 @@
+import { isNil, isEqual } from 'lodash-es';
+import { filter } from '../../filtering/localFiltering.js';
+/**
+ * Guarantees an array of operations, is an array of operations. Handles
+ * the basic failure case of a single operation.
+ */
+export function validateOperationsArray(operationsArray) {
+    // verify the operations list
+    if (Array.isArray(operationsArray)) {
+        if (operationsArray.length > 0) {
+            if (!(operationsArray[0] instanceof Operation)) {
+                throw new Error(`operationsArray must be Operations not ${typeof (operationsArray[0])}`);
+            }
+        }
+    }
+    else if (!isNil(operationsArray)) {
+        if (!(operationsArray instanceof Operation)) {
+            throw new Error(`operationsArray must be Operations not ${typeof (operationsArray)}`);
+        }
+        operationsArray = [operationsArray]; // coerce it to be an array
+    }
+    return operationsArray;
+}
+/**
+ * Extract the minimal signature we're using for “direct‐match” comparison.
+ */
+export function getFingerprint(qs) {
+    const { nodes } = qs;
+    const initialQueryset = qs._initialQueryset;
+    const { offset = null, limit = null } = qs._serializerOptions || {};
+    return { nodes, initialQueryset, offset, limit };
+}

package/dist/syncEngine/sync.d.ts

@@ -0,0 +1,46 @@
+export class EventPayload {
+    constructor(data: any);
+    event: any;
+    model: any;
+    operation_id: any;
+    pk_field_name: any;
+    configKey: any;
+    instances: any;
+    _cachedInstances: any;
+    get modelClass(): Function | null;
+    getFullInstances(): Promise<any>;
+}
+export class SyncManager {
+    registries: Map<any, any>;
+    followedModels: Map<any, any>;
+    followAllQuerysets: boolean;
+    followedQuerysets: Map<any, any>;
+    periodicSyncTimer: NodeJS.Timeout | null;
+    eventBatch: Map<any, any>;
+    debounceTimer: NodeJS.Timeout | null;
+    maxWaitTimer: NodeJS.Timeout | null;
+    debounceMs: number;
+    maxWaitMs: number;
+    batchStartTime: number | null;
+    withTimeout(promise: any, ms: any): Promise<any>;
+    /**
+     * Initialize event handlers for all event receivers
+     */
+    initialize(): void;
+    startPeriodicSync(): void;
+    syncStaleQuerysets(): Promise<void>;
+    pruneUnreferencedModels(): void;
+    isStoreFollowed(registry: any, semanticKey: any): boolean;
+    cleanup(): void;
+    followModel(registry: any, modelClass: any): void;
+    unfollowModel(registry: any, modelClass: any): void;
+    manageRegistry(registry: any): void;
+    removeRegistry(registry: any): void;
+    handleEvent: (event: any) => void;
+    processBatch(reason?: string): void;
+    isQuerysetFollowed(queryset: any): boolean;
+    processQuerysetsBatch(representativeEvent: any, allEvents: any): void;
+    processMetrics(event: any): void;
+    processModels(event: any): void;
+}
+export const syncManager: SyncManager;

package/dist/syncEngine/sync.js

@@ -0,0 +1,389 @@
+import { getAllEventReceivers } from "../core/eventReceivers.js";
+import { operationRegistry } from "./stores/operation.js";
+import { initializeAllEventReceivers } from "../config.js";
+import { getEventReceiver } from "../core/eventReceivers.js";
+import { querysetStoreRegistry, QuerysetStoreRegistry, } from "./registries/querysetStoreRegistry.js";
+import { modelStoreRegistry, ModelStoreRegistry, } from "./registries/modelStoreRegistry.js";
+import { metricRegistry, MetricRegistry } from "./registries/metricRegistry.js";
+import { getModelClass, getConfig } from "../config.js";
+import { isNil } from "lodash-es";
+import { QuerysetStore } from "./stores/querysetStore.js";
+import { v7 as uuidv7 } from "uuid";
+export class EventPayload {
+    constructor(data) {
+        this.event = data.event;
+        this.model = data.model;
+        this.operation_id = data.operation_id;
+        this.pk_field_name = data.pk_field_name;
+        this.configKey = data.configKey;
+        // Parse PK fields to numbers in instances
+        this.instances = data.instances?.map(instance => {
+            if (instance && this.pk_field_name && instance[this.pk_field_name] != null) {
+                return {
+                    ...instance,
+                    [this.pk_field_name]: Number(instance[this.pk_field_name])
+                };
+            }
+            return instance;
+        }) || data.instances;
+        this._cachedInstances = null;
+    }
+    get modelClass() {
+        return getModelClass(this.model, this.configKey);
+    }
+    async getFullInstances() {
+        if (this.event === "delete") {
+            throw new Error("Cannot fetch full instances for delete operation bozo...");
+        }
+        if (isNil(this._cachedInstances)) {
+            this._cachedInstances = await this.modelClass.objects
+                .filter({
+                [`${this.modelClass.primaryKeyField}__in`]: this.instances.map((instance) => instance[this.modelClass.primaryKeyField]),
+            })
+                .fetch();
+        }
+        return this._cachedInstances;
+    }
+}
+export class SyncManager {
+    constructor() {
+        this.handleEvent = (event) => {
+            let payload = new EventPayload(event);
+            let isLocalOperation = operationRegistry.has(payload.operation_id);
+            // Always process metrics immediately (they're lightweight)
+            if (this.registries.has(MetricRegistry)) {
+                this.processMetrics(payload);
+            }
+            if (isLocalOperation) {
+                return;
+            }
+            // Add to batch for queryset/model processing
+            const key = `${event.model}::${event.configKey}`;
+            const now = Date.now();
+            // If this is the first event in the batch, start max wait timer
+            if (this.eventBatch.size === 0) {
+                this.batchStartTime = now;
+                this.maxWaitTimer = setTimeout(() => {
+                    this.processBatch("maxWait");
+                }, this.maxWaitMs);
+            }
+            this.eventBatch.set(key, {
+                event: payload,
+                firstSeen: this.eventBatch.has(key)
+                    ? this.eventBatch.get(key).firstSeen
+                    : now,
+            });
+            // Reset debounce timer
+            if (this.debounceTimer) {
+                clearTimeout(this.debounceTimer);
+            }
+            this.debounceTimer = setTimeout(() => {
+                this.processBatch("debounce");
+            }, this.debounceMs);
+        };
+        this.registries = new Map();
+        // Map of registries to model sets
+        this.followedModels = new Map();
+        // Map of querysets to keep synced
+        this.followAllQuerysets = true;
+        this.followedQuerysets = new Map();
+        this.periodicSyncTimer = null;
+        // Batching for event processing
+        this.eventBatch = new Map(); // model::configKey -> { event, firstSeen }
+        this.debounceTimer = null;
+        this.maxWaitTimer = null;
+        this.debounceMs = 100; // Wait for rapid events to settle
+        this.maxWaitMs = 2000; // Maximum time to hold events
+        this.batchStartTime = null;
+    }
+    withTimeout(promise, ms) {
+        // If no timeout specified, use 2x the periodic sync interval, or 30s as fallback
+        if (!ms) {
+            try {
+                const config = getConfig();
+                const intervalSeconds = config.periodicSyncIntervalSeconds;
+                ms = intervalSeconds ? intervalSeconds * 2000 : 30000; // 2x interval in ms, or 30s default
+            }
+            catch {
+                ms = 30000; // 30s fallback if no config
+            }
+        }
+        return Promise.race([
+            promise,
+            new Promise((_, reject) => setTimeout(() => reject(new Error(`Sync timeout after ${ms}ms`)), ms)),
+        ]);
+    }
+    /**
+     * Initialize event handlers for all event receivers
+     */
+    initialize() {
+        // Initialize all event receivers
+        initializeAllEventReceivers();
+        // Get all registered event receivers
+        const eventReceivers = getAllEventReceivers();
+        // Register the event handlers with each receiver
+        eventReceivers.forEach((receiver, configKey) => {
+            if (receiver) {
+                // Model events go to handleEvent
+                receiver.addModelEventHandler(this.handleEvent.bind(this));
+            }
+        });
+        this.startPeriodicSync();
+    }
+    startPeriodicSync() {
+        if (this.periodicSyncTimer)
+            return;
+        try {
+            const config = getConfig();
+            const intervalSeconds = config.periodicSyncIntervalSeconds;
+            // If null or undefined, don't start periodic sync
+            if (!intervalSeconds) {
+                console.log("[SyncManager] Periodic sync disabled (set to null)");
+                return;
+            }
+            const intervalMs = intervalSeconds * 1000;
+            this.periodicSyncTimer = setInterval(() => {
+                this.syncStaleQuerysets();
+            }, intervalMs);
+            console.log(`[SyncManager] Periodic sync started: ${intervalSeconds}s intervals`);
+        }
+        catch (error) {
+            // If no config, don't start periodic sync by default
+            console.log("[SyncManager] No config found, periodic sync disabled by default");
+        }
+    }
+    async syncStaleQuerysets() {
+        const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
+        if (!querysetRegistry)
+            return;
+        // Generate operationId for this sync batch - querysets in same chain will coordinate
+        const operationId = `periodic-sync-${uuidv7()}`;
+        // Get dbSynced keys (followed querysets)
+        const dbSyncedKeys = new Set([...this.followedQuerysets].map(qs => qs.semanticKey));
+        // Collect all stores to sync
+        const storesToSync = [];
+        for (const [semanticKey, store] of querysetRegistry._stores.entries()) {
+            const isFollowed = this.isStoreFollowed(querysetRegistry, semanticKey);
+            if (this.followAllQuerysets || isFollowed) {
+                storesToSync.push(store);
+            }
+        }
+        if (storesToSync.length > 0) {
+            console.log(`[SyncManager] Periodic sync: syncing ${storesToSync.length} stores`);
+            // Run all groupSync calls in parallel - they coordinate via shared promise cache
+            await Promise.all(storesToSync.map(store => querysetRegistry.groupSync(store.queryset, operationId, dbSyncedKeys)));
+        }
+        // Prune unreferenced model instances
+        this.pruneUnreferencedModels();
+    }
+    pruneUnreferencedModels() {
+        const modelRegistry = this.registries.get(ModelStoreRegistry);
+        const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
+        if (!modelRegistry || !querysetRegistry)
+            return;
+        // Prune each model store
+        for (const [modelClass, store] of modelRegistry._stores.entries()) {
+            store.pruneUnreferencedInstances(querysetRegistry);
+        }
+    }
+    isStoreFollowed(registry, semanticKey) {
+        const followingQuerysets = registry.followingQuerysets.get(semanticKey);
+        if (!followingQuerysets)
+            return false;
+        return [...followingQuerysets].some((queryset) => {
+            return this.isQuerysetFollowed(queryset);
+        });
+    }
+    cleanup() {
+        if (this.periodicSyncTimer) {
+            clearInterval(this.periodicSyncTimer);
+            this.periodicSyncTimer = null;
+        }
+        // Clean up batch timers
+        if (this.debounceTimer) {
+            clearTimeout(this.debounceTimer);
+            this.debounceTimer = null;
+        }
+        if (this.maxWaitTimer) {
+            clearTimeout(this.maxWaitTimer);
+            this.maxWaitTimer = null;
+        }
+    }
+    followModel(registry, modelClass) {
+        const models = this.followedModels.get(registry) || new Set();
+        this.followedModels.set(registry, models);
+        if (models.has(modelClass))
+            return;
+        const alreadyFollowed = [...this.followedModels.values()].some((set) => set.has(modelClass));
+        models.add(modelClass);
+        if (!alreadyFollowed) {
+            getEventReceiver(modelClass.configKey)?.subscribe(modelClass.modelName, this.handleEvent);
+        }
+    }
+    unfollowModel(registry, modelClass) {
+        const models = this.followedModels.get(registry);
+        if (!models)
+            return;
+        models.delete(modelClass);
+        const stillFollowed = [...this.followedModels.values()].some((set) => set.has(modelClass));
+        if (!stillFollowed) {
+            getEventReceiver(modelClass.configKey)?.unsubscribe(modelClass.modelName, this.handleEvent);
+        }
+    }
+    manageRegistry(registry) {
+        this.registries.set(registry.constructor, registry);
+        registry.setSyncManager(this);
+    }
+    removeRegistry(registry) {
+        this.registries.delete(registry.constructor);
+    }
+    processBatch(reason = "unknown") {
+        if (this.eventBatch.size === 0)
+            return;
+        // Clear timers
+        if (this.debounceTimer) {
+            clearTimeout(this.debounceTimer);
+            this.debounceTimer = null;
+        }
+        if (this.maxWaitTimer) {
+            clearTimeout(this.maxWaitTimer);
+            this.maxWaitTimer = null;
+        }
+        const events = Array.from(this.eventBatch.values()).map((item) => item.event);
+        const waitTime = Date.now() - this.batchStartTime;
+        this.eventBatch.clear();
+        this.batchStartTime = null;
+        console.log(`[SyncManager] Processing batch of ${events.length} events (reason: ${reason}, waited: ${waitTime}ms)`);
+        // Group events by model for efficient processing
+        const eventsByModel = new Map();
+        events.forEach((event) => {
+            const key = `${event.model}::${event.configKey}`;
+            if (!eventsByModel.has(key)) {
+                eventsByModel.set(key, []);
+            }
+            eventsByModel.get(key).push(event);
+        });
+        // Process each model's events as a batch
+        eventsByModel.forEach((modelEvents, modelKey) => {
+            const event = modelEvents[0]; // Use first event as representative
+            if (this.registries.has(QuerysetStoreRegistry)) {
+                this.processQuerysetsBatch(event, modelEvents);
+            }
+            if (this.registries.has(ModelStoreRegistry)) {
+                this.processModels(event);
+            }
+        });
+    }
+    isQuerysetFollowed(queryset) {
+        const activeSemanticKeys = new Set([...this.followedQuerysets].map((qs) => qs.semanticKey));
+        let current = queryset;
+        while (current) {
+            if (activeSemanticKeys.has(current.semanticKey)) {
+                return true;
+            }
+            current = current.__parent;
+        }
+        return false;
+    }
+    processQuerysetsBatch(representativeEvent, allEvents) {
+        const registry = this.registries.get(QuerysetStoreRegistry);
+        // Collect all stores that need syncing for this model
+        const storesToSync = [];
+        for (const [semanticKey, store] of registry._stores.entries()) {
+            if (store.modelClass.modelName === representativeEvent.model &&
+                store.modelClass.configKey === representativeEvent.configKey) {
+                if (this.followAllQuerysets) {
+                    storesToSync.push(store);
+                    continue;
+                }
+                const followingQuerysets = registry.followingQuerysets.get(semanticKey);
+                if (followingQuerysets) {
+                    const shouldSync = [...followingQuerysets].some((queryset) => {
+                        return this.isQuerysetFollowed(queryset);
+                    });
+                    if (shouldSync) {
+                        storesToSync.push(store);
+                    }
+                }
+            }
+        }
+        if (storesToSync.length === 0)
+            return;
+        // Sync all relevant stores for this model
+        console.log(`[SyncManager] Syncing ${storesToSync.length} queryset stores for ${representativeEvent.model}`);
+        // Generate operationId for this batch - querysets in same chain will coordinate
+        const operationId = `remote-event-${uuidv7()}`;
+        // Get dbSynced keys (followed querysets)
+        const dbSyncedKeys = new Set([...this.followedQuerysets].map(qs => qs.semanticKey));
+        // Run all groupSync calls in parallel - they coordinate via shared promise cache
+        Promise.all(storesToSync.map(store => registry.groupSync(store.queryset, operationId, dbSyncedKeys)));
+    }
+    processMetrics(event) {
+        const registry = this.registries.get(MetricRegistry);
+        for (const [key, entry] of registry._stores.entries()) {
+            // Check if the store has a queryset with the model class AND correct backend
+            if (entry.queryset &&
+                entry.queryset.modelClass.modelName === event.model &&
+                entry.queryset.modelClass.configKey === event.configKey) {
+                if (this.followAllQuerysets) {
+                    entry.store.sync();
+                    continue;
+                }
+                // Check if this queryset (or any parent) is being followed
+                if (this.isQuerysetFollowed(entry.queryset)) {
+                    console.log(`syncing metric store for key: ${key}`);
+                    entry.store.sync();
+                }
+            }
+        }
+    }
+    processModels(event) {
+        const registry = this.registries.get(ModelStoreRegistry);
+        if (!registry)
+            return;
+        const modelStore = registry.getStore(event.modelClass);
+        if (!modelStore)
+            return;
+        // Get PKs from the event
+        const eventPks = new Set((event.instances || [])
+            .filter(inst => inst && inst[event.pk_field_name] != null)
+            .map(inst => inst[event.pk_field_name]));
+        if (eventPks.size === 0)
+            return;
+        // Get PKs that are already in the model store's ground truth
+        const groundTruthPks = new Set(modelStore.groundTruthPks);
+        // Get PKs that are top-level in ANY queryset for this model (not just followed ones)
+        const querysetRegistry = this.registries.get(QuerysetStoreRegistry);
+        const topLevelQuerysetPks = new Set();
+        if (querysetRegistry) {
+            for (const [semanticKey, store] of querysetRegistry._stores.entries()) {
+                // Check ALL querysets for this model (permanent and temporary)
+                if (store.modelClass.modelName === event.model &&
+                    store.modelClass.configKey === event.configKey) {
+                    // Get this queryset's ground truth PKs (top-level instances)
+                    // Don't use render() as it applies operations - we just want ground truth
+                    store.groundTruthPks.forEach(pk => topLevelQuerysetPks.add(pk));
+                }
+            }
+        }
+        // Find PKs that are:
+        // 1. In the event
+        // 2. Already in ground truth (we're tracking them)
+        // 3. NOT top-level in any queryset (they're only nested/included)
+        const pksToSync = [];
+        eventPks.forEach(pk => {
+            if (groundTruthPks.has(pk) && !topLevelQuerysetPks.has(pk)) {
+                pksToSync.push(pk);
+            }
+        });
+        if (pksToSync.length > 0) {
+            console.log(`[SyncManager] Syncing ${pksToSync.length} nested-only PKs for ${event.model}: ${pksToSync}`);
+            modelStore.sync(pksToSync);
+        }
+    }
+}
+const syncManager = new SyncManager();
+syncManager.manageRegistry(querysetStoreRegistry);
+syncManager.manageRegistry(modelStoreRegistry);
+syncManager.manageRegistry(metricRegistry);
+export { syncManager };

package/dist/testing.d.ts

@@ -0,0 +1,63 @@
+export function createTestConfig({ apiUrl, backendKey, generatedTypesDir, generatedActionsDir, getAuthHeaders, testAuthUserId, testAuthHeaders, testAuthHeaderName, eventsType, fileRootURL, fileUploadMode, }: {
+    apiUrl: any;
+    backendKey?: string | undefined;
+    generatedTypesDir?: string | undefined;
+    generatedActionsDir: any;
+    getAuthHeaders: any;
+    testAuthUserId: any;
+    testAuthHeaders: any;
+    testAuthHeaderName?: string | undefined;
+    eventsType?: string | undefined;
+    fileRootURL: any;
+    fileUploadMode: any;
+}): {
+    config: {
+        backendConfigs: {
+            [backendKey]: {
+                API_URL: any;
+                GENERATED_TYPES_DIR: string;
+                getAuthHeaders: () => any;
+                events: {
+                    type: string;
+                };
+            };
+        };
+    };
+    testHeaders: {
+        setSeeding(enabled: any): void;
+        setReset(enabled: any): void;
+        setAuthUserId(userId: any): void;
+        withSeeding(fn: any): any;
+        withReset(fn: any): any;
+        withAuthUserId(userId: any, fn: any): any;
+    };
+};
+export function setupTestStateZero({ apiUrl, backendKey, generatedTypesDir, generatedActionsDir, getAuthHeaders, testAuthUserId, testAuthHeaders, testAuthHeaderName, eventsType, fileRootURL, fileUploadMode, getModelClass, adapters, }: {
+    apiUrl: any;
+    backendKey?: string | undefined;
+    generatedTypesDir?: string | undefined;
+    generatedActionsDir: any;
+    getAuthHeaders: any;
+    testAuthUserId: any;
+    testAuthHeaders: any;
+    testAuthHeaderName: any;
+    eventsType?: string | undefined;
+    fileRootURL: any;
+    fileUploadMode: any;
+    getModelClass: any;
+    adapters: any;
+}): {
+    setSeeding(enabled: any): void;
+    setReset(enabled: any): void;
+    setAuthUserId(userId: any): void;
+    withSeeding(fn: any): any;
+    withReset(fn: any): any;
+    withAuthUserId(userId: any, fn: any): any;
+};
+export function createActionMocker(actionRegistry: any): {
+    mock(actionName: any, handler: any, backendKey?: string): void;
+    restore(actionName: any, backendKey?: string): void;
+    restoreAll(): void;
+};
+export function seedRemote(testHeaders: any, seedFn: any): Promise<any>;
+export function resetRemote(testHeaders: any, resetFn: any): Promise<any>;