@memberjunction/metadata-sync 5.37.0 → 5.39.0

package/dist/lib/sync-metadata-engine.js

@@ -0,0 +1,454 @@
+/**
+ * @fileoverview Upfront preloading + lookup/file caching for `mj sync push`.
+ *
+ * `SyncMetadataEngine` extends `BaseEngine` to piggy-back on its batched
+ * `RunViews` loader and `BaseEntity` event-bus subscription. The push flow
+ * uses it in three ways:
+ *
+ *   1. **Preload** — `Config()` scans every JSON file under each entity
+ *      directory once, collects the set of entity names touched (including
+ *      nested `relatedEntities`), and asks `BaseEngine.Load` to issue one
+ *      unfiltered `RunView` per entity. We load *all* rows of each touched
+ *      entity, not just the PKs mentioned in files — for two reasons:
+ *
+ *        (a) `mj sync` is for METADATA. The entities involved (Actions,
+ *            Prompts, Agents, Templates, etc.) are inherently bounded —
+ *            hundreds to low tens of thousands of rows per tenant, not
+ *            millions. Loading everything is faster than computing and
+ *            shipping a giant `WHERE IN (...)` clause.
+ *        (b) `@lookup:` resolution often targets records that aren't in
+ *            the local files (e.g. a category referenced by name). A
+ *            full preload means those lookups hit the cache too.
+ *
+ *      The loaded `BaseEntity` instances live on dynamic properties named
+ *      via {@link getPropertyNameForEntity}; lookups during the sync phase
+ *      consult those arrays instead of round-tripping the DB.
+ *
+ *   2. **Event-driven mutation** — `BaseEngine` already wires up the global
+ *      `BaseEntity` event bus. Because our configs carry no Filter/OrderBy,
+ *      the base class's default `canUseImmediateMutation` path handles
+ *      save/delete/remote-invalidate events automatically.
+ *
+ *   3. **Lookup + file caches** — Resolved `@lookup` keys are memoized in
+ *      {@link lookupCache} and parsed file contents are memoized in
+ *      {@link fileDataCache}, so validation and sync passes don't re-read
+ *      or re-parse the same file twice.
+ *
+ * @module sync-metadata-engine
+ */
+import { BaseEngine } from '@memberjunction/core';
+import { IsUuidSQLType } from '@memberjunction/sql-dialect';
+import fastGlob from 'fast-glob';
+import fs from 'fs-extra';
+import { JsonPreprocessor } from './json-preprocessor.js';
+import { loadEntityConfig } from '../config.js';
+/** Prefix used to namespace the dynamic per-entity cache properties on the engine instance. */
+const CACHED_ENTITY_PROP_PREFIX = 'cached_';
+/** Concurrency cap for parallel file reads during preload. */
+const FILE_READ_CONCURRENCY = 16;
+/**
+ * Emit a warning when any single preloaded entity comes back with more rows
+ * than this. `mj sync` is for metadata — entities in this benchmark band
+ * (Actions, Prompts, Templates, etc.) are bounded by design. Loading 100k+
+ * rows of one entity into RAM is a signal that something is being synced
+ * that shouldn't be (or the metadata set has grown beyond the tool's intent).
+ */
+const LARGE_ENTITY_WARN_THRESHOLD = 100_000;
+/**
+ * Coordinates upfront batch-preloading of `BaseEntity` instances and in-memory
+ * caches for `mj sync push`. See module docstring for the full picture.
+ */
+export class SyncMetadataEngine extends BaseEngine {
+    constructor() {
+        super(...arguments);
+        this.entityDirs = [];
+        this.preloadedEntityNames = new Set();
+        this.lookupCache = new Map();
+        this.fileDataCache = new Map();
+        /** Reverse index used to scope lookup invalidation on delete: entity name → set of lookup keys touching that entity. */
+        this.lookupKeysByEntity = new Map();
+        /**
+         * O(1) PK lookup index built after preload completes. Keyed by entity
+         * name, then by `serializePrimaryKey(...)` of the entity's own PK.
+         *
+         * Without this, `loadEntity` falls into an `Array.find(... GetAll())`
+         * scan over the entire preloaded slot, which is O(N) per record being
+         * processed — i.e. O(N×K) overall. For entities like `MJ: Integration
+         * Object Fields` where DB-wide row count can be 10×+ the file count,
+         * that turns into billions of comparisons (38 min for the integrations
+         * dir alone in real-world measurement).
+         *
+         * Kept in sync by {@link addEntityToCache} / {@link removeEntityFromCache}.
+         * `findCachedByPrimaryKey` falls back to an array scan on Map miss so
+         * we tolerate drift from `BaseEngine`'s event-driven slot mutations
+         * (which mutate the array directly without going through our helpers).
+         */
+        this.pkIndexes = new Map();
+        /** Non-fatal warnings collected during preload (malformed shapes, oversized entities). Drained by the caller after `Config()`. */
+        this.warnings = [];
+    }
+    initializeEngine(syncEngine) {
+        this.syncEngine = syncEngine;
+    }
+    setEntityDirs(dirs) {
+        this.entityDirs = dirs;
+    }
+    /** Returns and clears the warnings collected during the last preload run. */
+    drainWarnings() {
+        const out = this.warnings;
+        this.warnings = [];
+        return out;
+    }
+    /**
+     * Property name on this engine instance under which cached `BaseEntity[]`
+     * arrays for the given entity are stored. `BaseEngine.applyImmediateMutation`
+     * reads/writes these slots directly, so we use a deterministic naming scheme.
+     */
+    getPropertyNameForEntity(entityName) {
+        return `${CACHED_ENTITY_PROP_PREFIX}${entityName.replace(/[^a-zA-Z0-9]/g, '_')}`;
+    }
+    /**
+     * Build the dynamic configuration list and trigger `BaseEngine.Load` —
+     * which fans the per-entity unfiltered views out over a single `RunViews`
+     * batch under the hood. Then check each loaded slot for size and emit
+     * warnings if anything came back larger than the metadata-scale threshold.
+     */
+    async Config(forceRefresh, contextUser, provider) {
+        const configs = await this.buildDynamicConfigs();
+        const providerToUse = provider || this.ProviderToUse;
+        if (configs.length > 0) {
+            await this.Load(configs, providerToUse, forceRefresh, contextUser);
+            this.buildPKIndexes(configs);
+            this.warnOnOversizedSlots(configs);
+        }
+        return true;
+    }
+    /**
+     * Build the per-entity PK → entity Map from each populated slot.
+     * Runs once after `BaseEngine.Load` completes. The cost (one walk
+     * over the loaded arrays, building Map entries) is trivial compared
+     * to the per-record-lookup savings it enables downstream.
+     */
+    buildPKIndexes(configs) {
+        for (const config of configs) {
+            if (!config.PropertyName || !config.EntityName)
+                continue;
+            const entityInfo = this.syncEngine.getEntityInfo(config.EntityName);
+            if (!entityInfo)
+                continue;
+            const list = this[config.PropertyName];
+            if (!Array.isArray(list))
+                continue;
+            const index = new Map();
+            for (const entity of list) {
+                try {
+                    const pkStr = this.serializePrimaryKey(entityInfo, entity.GetAll());
+                    index.set(pkStr, entity);
+                }
+                catch {
+                    // Defensive: a malformed cached entity shouldn't break index
+                    // construction for the whole entity. The array scan fallback
+                    // will still find it if needed.
+                }
+            }
+            this.pkIndexes.set(config.EntityName, index);
+        }
+    }
+    /**
+     * O(1) PK lookup against the preload cache, with an array-scan
+     * fallback so we tolerate drift from `BaseEngine` event-driven
+     * slot mutations that don't go through our `addEntityToCache` helper.
+     *
+     * Returns `null` when the entity isn't preloaded or the PK isn't in
+     * the cache — callers fall through to a DB load.
+     */
+    findCachedByPrimaryKey(entityName, primaryKey) {
+        if (!this.isEntityPreloaded(entityName))
+            return null;
+        const entityInfo = this.syncEngine.getEntityInfo(entityName);
+        if (!entityInfo)
+            return null;
+        const pkStr = this.serializePrimaryKey(entityInfo, primaryKey);
+        const index = this.pkIndexes.get(entityName);
+        const hit = index?.get(pkStr);
+        if (hit)
+            return hit;
+        // Fallback: someone (BaseEngine event handler) might have spliced
+        // a new entity into the array slot without updating the index.
+        // Linear scan once, and if we find a match, repair the index so
+        // subsequent hits stay O(1).
+        const list = this.getCachedEntities(entityName);
+        for (const entity of list) {
+            try {
+                if (this.serializePrimaryKey(entityInfo, entity.GetAll()) === pkStr) {
+                    index?.set(pkStr, entity);
+                    return entity;
+                }
+            }
+            catch {
+                // skip malformed entries
+            }
+        }
+        return null;
+    }
+    markEntityAsPreloaded(entityName) {
+        this.preloadedEntityNames.add(entityName);
+    }
+    isEntityPreloaded(entityName) {
+        return this.preloadedEntityNames.has(entityName);
+    }
+    getCachedEntities(entityName) {
+        const propName = this.getPropertyNameForEntity(entityName);
+        const list = this[propName];
+        return Array.isArray(list) ? list : [];
+    }
+    getCachedFile(filePath) {
+        return this.fileDataCache.get(filePath);
+    }
+    cacheFile(filePath, rawData, fileData) {
+        this.fileDataCache.set(filePath, { rawData, fileData });
+    }
+    /**
+     * Invalidate the cached file content for a path. Call this immediately
+     * before (or after) any code path that writes back to a metadata JSON
+     * file during the sync, so a subsequent reader gets fresh contents.
+     */
+    invalidateCachedFile(filePath) {
+        this.fileDataCache.delete(filePath);
+    }
+    addEntityToCache(entityName, entity) {
+        const propName = this.getPropertyNameForEntity(entityName);
+        const slot = this;
+        if (!Array.isArray(slot[propName])) {
+            slot[propName] = [];
+        }
+        const list = slot[propName];
+        // `PrimaryKey.Equals` requires the entity to have a populated composite
+        // key; if either side is missing one we fall back to reference equality
+        // only (the entity might not be saved yet, in which case no dedup is
+        // possible anyway).
+        const incomingKey = entity.PrimaryKey;
+        const arrayIdx = list.findIndex(e => {
+            if (e === entity)
+                return true;
+            const existingKey = e.PrimaryKey;
+            if (!incomingKey || !existingKey)
+                return false;
+            return existingKey.Equals(incomingKey);
+        });
+        if (arrayIdx >= 0) {
+            list[arrayIdx] = entity;
+        }
+        else {
+            list.push(entity);
+        }
+        // Keep the PK index in sync. Skip silently when serializePrimaryKey
+        // throws — typically a half-built entity with no PK populated yet.
+        try {
+            const entityInfo = this.syncEngine.getEntityInfo(entityName);
+            if (entityInfo) {
+                const pkStr = this.serializePrimaryKey(entityInfo, entity.GetAll());
+                let index = this.pkIndexes.get(entityName);
+                if (!index) {
+                    index = new Map();
+                    this.pkIndexes.set(entityName, index);
+                }
+                index.set(pkStr, entity);
+            }
+        }
+        catch { /* defensive: malformed entity */ }
+    }
+    removeEntityFromCache(entityName, primaryKey) {
+        const propName = this.getPropertyNameForEntity(entityName);
+        const list = this[propName];
+        const entityInfo = this.syncEngine.getEntityInfo(entityName);
+        let pkStr = null;
+        if (entityInfo) {
+            pkStr = this.serializePrimaryKey(entityInfo, primaryKey);
+            if (list) {
+                const arrayIdx = list.findIndex(e => this.serializePrimaryKey(entityInfo, e.GetAll()) === pkStr);
+                if (arrayIdx >= 0) {
+                    list.splice(arrayIdx, 1);
+                }
+            }
+        }
+        if (pkStr !== null) {
+            this.pkIndexes.get(entityName)?.delete(pkStr);
+        }
+        this.invalidateLookupsForEntity(entityName);
+    }
+    getCachedLookup(key) {
+        return this.lookupCache.get(key);
+    }
+    setCachedLookup(key, id, entityName) {
+        this.lookupCache.set(key, id);
+        if (entityName) {
+            const normalized = entityName.toLowerCase();
+            let bucket = this.lookupKeysByEntity.get(normalized);
+            if (!bucket) {
+                bucket = new Set();
+                this.lookupKeysByEntity.set(normalized, bucket);
+            }
+            bucket.add(key);
+        }
+    }
+    clearLookupCache() {
+        this.lookupCache.clear();
+        this.lookupKeysByEntity.clear();
+    }
+    /**
+     * Invalidate only the lookup entries that touch a given entity. Avoids
+     * the perf cliff of clearing the whole lookup cache on every delete.
+     *
+     * Note: if `setCachedLookup` was called without an `entityName`, those
+     * entries aren't in the reverse index and won't be cleared here. The
+     * `resolveLookup` integration always passes the entity name, so this
+     * is only a concern for callers that bypass that path.
+     */
+    invalidateLookupsForEntity(entityName) {
+        const normalized = entityName.toLowerCase();
+        const bucket = this.lookupKeysByEntity.get(normalized);
+        if (!bucket)
+            return;
+        for (const key of bucket) {
+            this.lookupCache.delete(key);
+        }
+        this.lookupKeysByEntity.delete(normalized);
+    }
+    /**
+     * Canonical string representation of a record's primary key, used as the
+     * key in entity-cache dedup/removal lookups. UUID-typed components are
+     * lower-cased to bridge SQL Server (upper) vs PG (lower).
+     */
+    serializePrimaryKey(entityInfo, primaryKey) {
+        const parts = entityInfo.PrimaryKeys.map((pk) => {
+            const val = primaryKey[pk.Name];
+            let valStr = val !== undefined && val !== null ? String(val) : '';
+            const pkType = pk.Type ?? '';
+            const normalizedType = pkType.trim().toLowerCase();
+            if (IsUuidSQLType(pkType) || normalizedType === 'uniqueidentifier' || normalizedType === 'uuid') {
+                valStr = valStr.toLowerCase().trim();
+            }
+            return `${pk.Name.toLowerCase()}=${valStr}`;
+        });
+        parts.sort();
+        return parts.join('|');
+    }
+    /**
+     * Walk every entity referenced by metadata files (including nested
+     * `relatedEntities`) and build one unfiltered `BaseEngine` config per
+     * entity. Each config triggers a single `SELECT * FROM vw<Entity>`
+     * (capped by `IgnoreMaxRows: true` in BaseEngine), pre-populating the
+     * per-entity slot the sync path will read from.
+     */
+    async buildDynamicConfigs() {
+        const entitiesFound = new Set();
+        for (const entityDir of this.entityDirs) {
+            const entityConfig = await loadEntityConfig(entityDir);
+            if (!entityConfig)
+                continue;
+            const pattern = entityConfig.filePattern || '*.json';
+            const files = await fastGlob(pattern, {
+                cwd: entityDir,
+                absolute: true,
+                onlyFiles: true,
+                dot: true,
+                ignore: ['**/node_modules/**', '**/.mj-*.json']
+            });
+            await this.processFilesInParallel(files, entityConfig.entity, entitiesFound);
+        }
+        const configs = [];
+        for (const entityName of entitiesFound) {
+            const entityInfo = this.syncEngine.getEntityInfo(entityName);
+            if (!entityInfo)
+                continue;
+            this.markEntityAsPreloaded(entityName);
+            configs.push({
+                PropertyName: this.getPropertyNameForEntity(entityName),
+                EntityName: entityName,
+                Type: 'entity',
+                ResultType: 'entity_object',
+                // No Filter — load all rows. Metadata entities are bounded by design;
+                // the few thousand-row tables we touch are faster to bulk-load than
+                // to fence with a giant WHERE clause. See the module docstring for
+                // the full reasoning.
+                AutoRefresh: true
+            });
+        }
+        return configs;
+    }
+    /**
+     * After `BaseEngine.Load` finishes, walk the configs and check whether any
+     * single preloaded entity exceeded the metadata-scale threshold. Emit a
+     * warning so the operator knows their sync is loading something it
+     * probably shouldn't be.
+     */
+    warnOnOversizedSlots(configs) {
+        for (const config of configs) {
+            if (!config.PropertyName || !config.EntityName)
+                continue;
+            const list = this[config.PropertyName];
+            if (!Array.isArray(list))
+                continue;
+            if (list.length > LARGE_ENTITY_WARN_THRESHOLD) {
+                this.warnings.push(`Preloaded ${list.length.toLocaleString()} rows of '${config.EntityName}'. mj sync is intended for metadata-scale entities (hundreds to low tens of thousands); this is large enough that you may want to confirm this entity belongs in a sync workflow.`);
+            }
+        }
+    }
+    /**
+     * Recursively walk records collecting every entity name they (or their
+     * `relatedEntities`) reference. We don't collect primary keys because
+     * preload is unfiltered — knowing the entity is enough.
+     *
+     * Malformed `relatedEntities` shapes (non-array values) produce a warning
+     * pointing at the offending file rather than silently being skipped.
+     */
+    collectEntitiesRecursive(records, entityName, entitiesFound, filePath) {
+        entitiesFound.add(entityName);
+        for (const record of records) {
+            const deleteRecord = record.deleteRecord;
+            if (deleteRecord?.delete === true) {
+                continue;
+            }
+            const relatedEntities = record.relatedEntities;
+            if (!relatedEntities)
+                continue;
+            for (const [relatedEntityName, relatedRecords] of Object.entries(relatedEntities)) {
+                if (Array.isArray(relatedRecords)) {
+                    this.collectEntitiesRecursive(relatedRecords, relatedEntityName, entitiesFound, filePath);
+                }
+                else {
+                    this.warnings.push(`Skipped preload for relatedEntities['${relatedEntityName}'] in ${filePath}: expected an array, got ${typeof relatedRecords}.`);
+                }
+            }
+        }
+    }
+    /**
+     * Process file reads in bounded-parallel batches. `fs.readJson` is I/O-bound;
+     * doing this sequentially was the dominant cost on metadata trees with
+     * thousands of small files.
+     */
+    async processFilesInParallel(files, entityName, entitiesFound) {
+        for (let i = 0; i < files.length; i += FILE_READ_CONCURRENCY) {
+            const batch = files.slice(i, i + FILE_READ_CONCURRENCY);
+            const parsed = await Promise.all(batch.map(filePath => this.readAndPreprocessFile(filePath)));
+            for (const { filePath, rawData, fileData } of parsed) {
+                this.cacheFile(filePath, rawData, fileData);
+                const records = Array.isArray(fileData) ? fileData : [fileData];
+                this.collectEntitiesRecursive(records, entityName, entitiesFound, filePath);
+            }
+        }
+    }
+    async readAndPreprocessFile(filePath) {
+        const rawData = await fs.readJson(filePath);
+        let fileData = rawData;
+        const jsonString = JSON.stringify(rawData);
+        const hasIncludes = jsonString.includes('"@include"') || jsonString.includes('"@include.');
+        if (hasIncludes) {
+            const preprocessor = new JsonPreprocessor();
+            fileData = await preprocessor.processFile(filePath);
+        }
+        return { filePath, rawData, fileData };
+    }
+}
+//# sourceMappingURL=sync-metadata-engine.js.map

package/dist/lib/sync-metadata-engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-metadata-engine.js","sourceRoot":"","sources":["../../src/lib/sync-metadata-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,OAAO,EACL,UAAU,EAOX,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAG7C,+FAA+F;AAC/F,MAAM,yBAAyB,GAAG,SAAS,CAAC;AAE5C,8DAA8D;AAC9D,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,2BAA2B,GAAG,OAAO,CAAC;AAU5C;;;GAGG;AACH,MAAM,OAAO,kBAAmB,SAAQ,UAA8B;IAAtE;;QAEU,eAAU,GAAa,EAAE,CAAC;QAC1B,yBAAoB,GAAgB,IAAI,GAAG,EAAE,CAAC;QAC9C,gBAAW,GAAwB,IAAI,GAAG,EAAE,CAAC;QAC7C,kBAAa,GAA4B,IAAI,GAAG,EAAE,CAAC;QAE3D,wHAAwH;QAChH,uBAAkB,GAA6B,IAAI,GAAG,EAAE,CAAC;QAEjE;;;;;;;;;;;;;;;WAeG;QACK,cAAS,GAAyC,IAAI,GAAG,EAAE,CAAC;QAEpE,kIAAkI;QAC1H,aAAQ,GAAa,EAAE,CAAC;IAuZlC,CAAC;IArZQ,gBAAgB,CAAC,UAAsB;QAC5C,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAEM,aAAa,CAAC,IAAc;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAED,6EAA6E;IACtE,aAAa;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;OAIG;IACI,wBAAwB,CAAC,UAAkB;QAChD,OAAO,GAAG,yBAAyB,GAAG,UAAU,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,EAAE,CAAC;IACnF,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,MAAM,CAAC,YAAsB,EAAE,WAAsB,EAAE,QAA4B;QAC9F,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACjD,MAAM,aAAa,GAAG,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC;QAErD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnE,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC7B,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACK,cAAc,CAAC,OAA4C;QACjE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU;gBAAE,SAAS;YACzD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACpE,IAAI,CAAC,UAAU;gBAAE,SAAS;YAC1B,MAAM,IAAI,GAAI,IAA2C,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC/E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,SAAS;YAEnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;YAC5C,KAAK,MAAM,MAAM,IAAI,IAAoB,EAAE,CAAC;gBAC1C,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;oBACpE,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC3B,CAAC;gBAAC,MAAM,CAAC;oBACP,6DAA6D;oBAC7D,6DAA6D;oBAC7D,gCAAgC;gBAClC,CAAC;YACH,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACI,sBAAsB,CAAC,UAAkB,EAAE,UAAmC;QACnF,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC;YAAE,OAAO,IAAI,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC;QAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QAEpB,kEAAkE;QAClE,+DAA+D;QAC/D,gEAAgE;QAChE,6BAA6B;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAChD,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,IAAI,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;oBACpE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;oBAC1B,OAAO,MAAM,CAAC;gBAChB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEM,qBAAqB,CAAC,UAAkB;QAC7C,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QACzC,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACnD,CAAC;IAEM,iBAAiB,CAAC,UAAkB;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAI,IAA2C,CAAC,QAAQ,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,IAAqB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3D,CAAC;IAEM,aAAa,CAAC,QAAgB;QACnC,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAEM,SAAS,CAAC,QAAgB,EAAE,OAAgB,EAAE,QAAiB;QACpE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;;;OAIG;IACI,oBAAoB,CAAC,QAAgB;QAC1C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAEM,gBAAgB,CAAC,UAAkB,EAAE,MAAkB;QAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAI,IAA2C,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;QACtB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAiB,CAAC;QAC5C,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,oBAAoB;QACpB,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;YAClC,IAAI,CAAC,KAAK,MAAM;gBAAE,OAAO,IAAI,CAAC;YAC9B,MAAM,WAAW,GAAG,CAAC,CAAC,UAAU,CAAC;YACjC,IAAI,CAAC,WAAW,IAAI,CAAC,WAAW;gBAAE,OAAO,KAAK,CAAC;YAC/C,OAAO,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;QACH,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAED,oEAAoE;QACpE,mEAAmE;QACnE,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAC7D,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBACpE,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;oBACtC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;gBACxC,CAAC;gBACD,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,iCAAiC,CAAC,CAAC;IAC/C,CAAC;IAEM,qBAAqB,CAAC,UAAkB,EAAE,UAAmC;QAClF,MAAM,QAAQ,GAAG,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAI,IAA2C,CAAC,QAAQ,CAA6B,CAAC;QAChG,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QAC7D,IAAI,KAAK,GAAkB,IAAI,CAAC;QAChC,IAAI,UAAU,EAAE,CAAC;YACf,KAAK,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YACzD,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;gBACjG,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;oBAClB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAEM,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAEM,eAAe,CAAC,GAAW,EAAE,EAAU,EAAE,UAAmB;QACjE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC5C,IAAI,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACrD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;gBAC3B,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAClD,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAEM,gBAAgB;QACrB,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;IAED;;;;;;;;OAQG;IACI,0BAA0B,CAAC,UAAkB;QAClD,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACI,mBAAmB,CAAC,UAAsB,EAAE,UAAmC;QACpF,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,EAAmB,EAAE,EAAE;YAC/D,MAAM,GAAG,GAAG,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,MAAM,GAAG,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClE,MAAM,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;YAC7B,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACnD,IAAI,aAAa,CAAC,MAAM,CAAC,IAAI,cAAc,KAAK,kBAAkB,IAAI,cAAc,KAAK,MAAM,EAAE,CAAC;gBAChG,MAAM,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;YACvC,CAAC;YACD,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,mBAAmB;QAC/B,MAAM,aAAa,GAAgB,IAAI,GAAG,EAAE,CAAC;QAE7C,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACvD,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,IAAI,QAAQ,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE;gBACpC,GAAG,EAAE,SAAS;gBACd,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,IAAI;gBACf,GAAG,EAAE,IAAI;gBACT,MAAM,EAAE,CAAC,oBAAoB,EAAE,eAAe,CAAC;aAChD,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,OAAO,GAAwC,EAAE,CAAC;QACxD,KAAK,MAAM,UAAU,IAAI,aAAa,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YAC7D,IAAI,CAAC,UAAU;gBAAE,SAAS;YAE1B,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC;gBACX,YAAY,EAAE,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC;gBACvD,UAAU,EAAE,UAAU;gBACtB,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE,eAAe;gBAC3B,sEAAsE;gBACtE,oEAAoE;gBACpE,mEAAmE;gBACnE,sBAAsB;gBACtB,WAAW,EAAE,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,oBAAoB,CAAC,OAA4C;QACvE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,UAAU;gBAAE,SAAS;YACzD,MAAM,IAAI,GAAI,IAA2C,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC/E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,SAAS;YACnC,IAAI,IAAI,CAAC,MAAM,GAAG,2BAA2B,EAAE,CAAC;gBAC9C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAChB,aAAa,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,aAAa,MAAM,CAAC,UAAU,mLAAmL,CAC3P,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,wBAAwB,CAC9B,OAAuC,EACvC,UAAkB,EAClB,aAA0B,EAC1B,QAAgB;QAEhB,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAG,MAAM,CAAC,YAAgD,CAAC;YAC7E,IAAI,YAAY,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,MAAM,eAAe,GAAG,MAAM,CAAC,eAAsD,CAAC;YACtF,IAAI,CAAC,eAAe;gBAAE,SAAS;YAE/B,KAAK,MAAM,CAAC,iBAAiB,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBAClF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;oBAClC,IAAI,CAAC,wBAAwB,CAC3B,cAAgD,EAChD,iBAAiB,EACjB,aAAa,EACb,QAAQ,CACT,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,QAAQ,CAAC,IAAI,CAChB,wCAAwC,iBAAiB,SAAS,QAAQ,4BAA4B,OAAO,cAAc,GAAG,CAC/H,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,sBAAsB,CAClC,KAAe,EACf,UAAkB,EAClB,aAA0B;QAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,qBAAqB,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,qBAAqB,CAAC,CAAC;YACxD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAC9F,KAAK,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,MAAM,EAAE,CAAC;gBACrD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBAChE,IAAI,CAAC,wBAAwB,CAAC,OAAyC,EAAE,UAAU,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAChH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,QAAgB;QAEhB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,QAAQ,GAAY,OAAO,CAAC;QAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE3F,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,YAAY,GAAG,IAAI,gBAAgB,EAAE,CAAC;YAC5C,QAAQ,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IACzC,CAAC;CACF"}

package/dist/services/PushService.d.ts

@@ -48,6 +48,7 @@ export declare class PushService {
     private deferredFileWrites;
     private deferredRecords;
     private stateManager;
+    private syncMetadataEngine;
     constructor(syncEngine: SyncEngine, contextUser: UserInfo, stateManager?: SyncStateManager);
     /** Set or replace the state manager after construction. */
     setStateManager(stateManager: SyncStateManager): void;

package/dist/services/PushService.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"PushService.d.ts","sourceRoot":"","sources":["../../src/services/PushService.ts"],"names":[],"mappings":"AAGA,OAAO,EAAwB,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AAGzF,OAAO,EAAE,UAAU,EAA4E,MAAM,oBAAoB,CAAC;AAa1H,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAY7D,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACvC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AA2CD,qBAAa,WAAW;IACtB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,kBAAkB,CAA6C;IACvE,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,YAAY,CAA+B;gBAEvC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,gBAAgB;IAM1F,2DAA2D;IAC3D,eAAe,CAAC,YAAY,EAAE,gBAAgB,GAAG,IAAI;IAIrD;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAS3B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB;IAKlB,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC;YA2VlE,sBAAsB;YAuRtB,sBAAsB;IAwmBpC,OAAO,CAAC,gBAAgB;YAgBV,mBAAmB;IA8IjC;;;OAGG;YACW,qBAAqB;IAwDnC;;;OAGG;YACW,iBAAiB;IAuL/B;;OAEG;YACW,yBAAyB;IA6DvC;;OAEG;IACH,OAAO,CAAC,eAAe;IAsBvB;;;;;;;;;;;;OAYG;YACW,sBAAsB;IAwFpC;;;OAGG;YACW,kBAAkB;IA0BhC;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAchC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA+B3B;;;OAGG;IACH,OAAO,CAAC,YAAY;IAoBpB,OAAO,CAAC,qBAAqB;CAqB9B"}
+{"version":3,"file":"PushService.d.ts","sourceRoot":"","sources":["../../src/services/PushService.ts"],"names":[],"mappings":"AAGA,OAAO,EAAwB,QAAQ,EAAqB,MAAM,sBAAsB,CAAC;AAGzF,OAAO,EAAE,UAAU,EAA4E,MAAM,oBAAoB,CAAC;AAc1H,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAY7D,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACvC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAClC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AA2CD,qBAAa,WAAW;IACtB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAW;IAC9B,OAAO,CAAC,QAAQ,CAAgB;IAChC,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,kBAAkB,CAA6C;IACvE,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,kBAAkB,CAAqB;gBAEnC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,CAAC,EAAE,gBAAgB;IAW1F,2DAA2D;IAC3D,eAAe,CAAC,YAAY,EAAE,gBAAgB,GAAG,IAAI;IAIrD;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAS3B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,gBAAgB;IAKlB,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC;YAwWlE,sBAAsB;YAkStB,sBAAsB;IAwmBpC,OAAO,CAAC,gBAAgB;YAgBV,mBAAmB;IAiJjC;;;OAGG;YACW,qBAAqB;IAwDnC;;;OAGG;YACW,iBAAiB;IAuL/B;;OAEG;YACW,yBAAyB;IA6DvC;;OAEG;IACH,OAAO,CAAC,eAAe;IAsBvB;;;;;;;;;;;;OAYG;YACW,sBAAsB;IAwFpC;;;OAGG;YACW,kBAAkB;IA4BhC;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAchC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA+B3B;;;OAGG;IACH,OAAO,CAAC,YAAY;IAoBpB,OAAO,CAAC,qBAAqB;CAqB9B"}

package/dist/services/PushService.js

@@ -5,6 +5,7 @@ import { Metadata } from '@memberjunction/core';
 import { UUIDsEqual } from '@memberjunction/global';
 import { IsStringSQLType } from '@memberjunction/sql-dialect';
 import { DeferrableLookupError } from '../lib/sync-engine.js';
+import { SyncMetadataEngine } from '../lib/sync-metadata-engine.js';
 import { BatchContextIndex } from '../lib/batch-context-index.js';
 import { loadEntityConfig, loadSyncConfig } from '../config.js';
 import { FileBackupManager } from '../lib/file-backup-manager.js';
@@ -34,6 +35,10 @@ export class PushService {
         this.syncEngine = syncEngine;
         this.contextUser = contextUser;
         this.stateManager = stateManager;
+        // Initialize SyncMetadataEngine
+        this.syncMetadataEngine = new SyncMetadataEngine();
+        this.syncMetadataEngine.initializeEngine(this.syncEngine);
+        this.syncEngine.setMetadataEngine(this.syncMetadataEngine);
     }
     /** Set or replace the state manager after construction. */
     setStateManager(stateManager) {
@@ -152,6 +157,18 @@ export class PushService {
             if (entityDirs.length === 0) {
                 throw new Error('No entity directories found');
             }
+            // Preload entities and cache files.
+            // We pass the SyncEngine's provider explicitly rather than reaching for
+            // Metadata.Provider — `mj sync` is single-process so they resolve to the
+            // same instance today, but routing through SyncEngine keeps the wiring
+            // self-consistent and makes future provider plumbing trivial.
+            callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
+            this.syncMetadataEngine.setEntityDirs(entityDirs);
+            await this.syncMetadataEngine.Config(true, this.contextUser, this.syncEngine.getProvider());
+            for (const warning of this.syncMetadataEngine.drainWarnings()) {
+                callbacks?.onWarn?.(`   ⚠️  ${warning}`);
+            }
+            callbacks?.onLog?.('✓ Preload completed successfully\n');
             if (options.verbose) {
                 callbacks?.onLog?.(`Found ${entityDirs.length} entity ${entityDirs.length === 1 ? 'directory' : 'directories'} to process`);
             }
@@ -390,20 +407,29 @@ export class PushService {
                 if (!options.dryRun) {
                     await fileBackupManager.backupFile(filePath);
                 }
-                // Read the raw file data first
-                const rawFileData = await fs.readJson(filePath);
+                // Check if file is already cached to avoid double I/O
+                let rawFileData;
+                let fileData;
+                const cachedFile = this.syncMetadataEngine.getCachedFile(filePath);
+                if (cachedFile) {
+                    rawFileData = cachedFile.rawData;
+                    fileData = cachedFile.fileData;
+                }
+                else {
+                    // Fallback (e.g. if not preloaded)
+                    rawFileData = await fs.readJson(filePath);
+                    fileData = rawFileData;
+                    const jsonString = JSON.stringify(rawFileData);
+                    const hasIncludes = jsonString.includes('"@include"') || jsonString.includes('"@include.');
+                    if (hasIncludes) {
+                        const jsonPreprocessor = new JsonPreprocessor();
+                        fileData = await jsonPreprocessor.processFile(filePath);
+                    }
+                    this.syncMetadataEngine.cacheFile(filePath, rawFileData, fileData);
+                }
                 // Keep unprocessed data to write back (preserves @file: references)
                 const unprocessedRecords = Array.isArray(rawFileData) ? rawFileData : [rawFileData];
                 const isArray = Array.isArray(rawFileData);
-                // Preprocess @include directives before checksumming, so changes to
-                // included files are detected by the incremental skip check.
-                let fileData = rawFileData;
-                const jsonString = JSON.stringify(rawFileData);
-                const hasIncludes = jsonString.includes('"@include"') || jsonString.includes('"@include.');
-                if (hasIncludes) {
-                    const jsonPreprocessor = new JsonPreprocessor();
-                    fileData = await jsonPreprocessor.processFile(filePath);
-                }
                 // Compute checksum on the RESOLVED content (after @include preprocessing)
                 // so that changes to included files are properly detected.
                 let cachedChecksum;
@@ -584,6 +610,9 @@ export class PushService {
                         else {
                             await JsonWriteHelper.writeOrderedRecordData(filePath, unprocessedRecords[0]);
                         }
+                        // Drop the cached snapshot — file on disk no longer matches it,
+                        // and any later reader within this push must see fresh contents.
+                        this.syncMetadataEngine.invalidateCachedFile(filePath);
                     }
                 }
                 // Update stored checksum after successful processing (reuse cached value if available)
@@ -1243,6 +1272,8 @@ export class PushService {
                 }
                 throw new Error(`Failed to delete ${entityName} record: ${errorMessage}`);
             }
+            // Remove from metadata cache
+            this.syncMetadataEngine.removeEntityFromCache(entityName, record.primaryKey);
             // Set deletedAt timestamp after successful deletion (only for metadata records)
             if (!isDbOnly) {
                 if (!record.deleteRecord) {
@@ -1654,6 +1685,8 @@ export class PushService {
                 else {
                     await JsonWriteHelper.writeOrderedRecordData(deferredWrite.filePath, deferredWrite.records[0]);
                 }
+                // File contents just changed on disk — drop the stale cache entry.
+                this.syncMetadataEngine.invalidateCachedFile(deferredWrite.filePath);
             }
             catch (error) {
                 callbacks?.onWarn?.(`   ⚠️  Failed to write ${deferredWrite.filePath}: ${error}`);