@itwin/imodel-transformer 1.0.1-customchanges.2 → 1.0.1-customchanges.5

package/lib/cjs/IModelTransformer.js

@@ -220,7 +220,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this._cachedSynchronizationVersion = undefined;
         this._targetClassNameToClassIdCache = new Map();
         // if undefined, it can be initialized by calling [[this.processChangesets]]
-        this._hasElementChangedCache = undefined;
         this._deletedSourceRelationshipData = undefined;
         this._yieldManager = new core_bentley_1.YieldManager();
         /** The directory where schemas will be exported, a random temporary directory */
@@ -913,13 +912,10 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @note A subclass can override this method to provide custom change detection behavior.
      */
     hasElementChanged(sourceElement) {
-        if (this._sourceChangeDataState === "no-changes")
-            return false;
-        if (this._sourceChangeDataState === "unconnected")
-            return true;
-        nodeAssert(this._sourceChangeDataState === "has-changes", "change data should be initialized by now");
-        nodeAssert(this._hasElementChangedCache !== undefined, "has element changed cache should be initialized by now");
-        return this._hasElementChangedCache.has(sourceElement.id);
+        const sourceDbChanges = this.exporter.sourceDbChanges;
+        return (!sourceDbChanges || // are we processing changes? if not then element is considered as changed
+            sourceDbChanges.element.insertIds.has(sourceElement.id) ||
+            sourceDbChanges.element.updateIds.has(sourceElement.id));
     }
     completePartiallyCommittedElements() {
         for (const sourceElementId of this._partiallyCommittedElementIds) {
@@ -1114,8 +1110,10 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 }
             }
         }
-        if (!this.hasElementChanged(sourceElement))
+        if (!this.hasElementChanged(sourceElement)) {
+            core_bentley_1.Logger.logTrace(loggerCategory, `Skipping unchanged element (${sourceElement.id}, ${sourceElement.getDisplayLabel()}).`);
             return;
+        }
         if (!this.doAllReferencesExistInTarget(sourceElement)) {
             this._partiallyCommittedElementIds.add(sourceElement.id);
         }
@@ -1770,39 +1768,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         await this.processChangesets();
         this._initialized = true;
     }
-    async handleCustomChanges(hasElementChangedCache, deleteIdsProcessed) {
-        // The hasElementChangedCache gets populated by changes from this._csFileProps.
-        // Because there is a possibility that someone could manually add ids to exporter.sourceDbChanges, we must separately process exporter.sourceDbChanges and add them to our hasElementChangedCache.
-        // Without this change we risk onExportElement returning early because we use hasElementChangedCache to decide if an element has changed or not.
-        this.exporter.sourceDbChanges?.element.updateIds.forEach((id) => hasElementChangedCache.add(id));
-        this.exporter.sourceDbChanges?.element.insertIds.forEach((id) => hasElementChangedCache.add(id));
-        // This loop is to process all custom deleteIds. Unclear if the special logic is still necessary for relationships or not (TODO!!). For all other entities, we assume that the element is still present in the sourceDb because it is not
-        // a real delete and instead a simulated delete to update filtering criteria between source and target. Since the element is still present, we do not need to call processDeletedOp to find the corresponding targetId.
-        // We can instead rely on `forEachTrackedElement` at the top of processChangesets to find the corresponding targetId.
-        // Note this also assumes we don't need to handle entity recreation for these custom deletes. I.e. a caller of API would not be able to add a custom delete for an entity that was recreated.
-        // a delete followed by an insert.
-        // ASSUME: If a changeset has a deleteId then custom change will never reference it. Is this still true if it was re-inserted? (TODO!!)
-        if (this.exporter.sourceDbChanges?.hasCustomRelationshipChanges) {
-            for (const id of this.exporter.sourceDbChanges?.relationship.deleteIds.keys() ??
-                []) {
-                if (deleteIdsProcessed?.has(id))
-                    continue;
-                const customData = this.exporter.sourceDbChanges?.getCustomRelationshipDataFromId(id);
-                if (customData === undefined) {
-                    core_bentley_1.Logger.logError(loggerCategory, "Custom data not found for relationship.", { id });
-                    continue;
-                }
-                const classFullName = customData.classFullName;
-                const sourceIdOfRelationshipInSource = customData?.sourceIdOfRelationship;
-                const targetIdOfRelationshipInSource = customData?.targetIdOfRelationship;
-                await this.processRelationshipDeleteOp(id, classFullName, sourceIdOfRelationshipInSource, targetIdOfRelationshipInSource);
-            }
-        }
-    }
     /**
-     * Reads all the changeset files in the private member of the transformer: _csFileProps and does two things with these changesets.
-     * Finds the corresponding target entity for any deleted source entities and remaps the sourceId to the targetId.
-     * Populates this._hasElementChangedCache with a set of elementIds that have been updated or inserted into the database.
+     * Reads all the changeset files in the private member of the transformer: _csFileProps
+     * and finds the corresponding target entity for any deleted source entities and remaps the sourceId to the targetId.
      * This function returns early if csFileProps is undefined or is of length 0.
      * @returns void
      */
@@ -1810,7 +1778,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.forEachTrackedElement((sourceElementId, targetElementId) => {
             this.context.remapElement(sourceElementId, targetElementId);
         });
-        await this.addCustomChanges(this.exporter.sourceDbChanges);
+        if (this.exporter.sourceDbChanges)
+            await this.addCustomChanges(this.exporter.sourceDbChanges);
         if (this._csFileProps === undefined || this._csFileProps.length === 0) {
             if (this.exporter.sourceDbChanges === undefined ||
                 !this.exporter.sourceDbChanges.hasChanges)
@@ -1819,7 +1788,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             if (this._sourceChangeDataState === "no-changes")
                 this._sourceChangeDataState = "has-changes";
         }
-        const hasElementChangedCache = new Set();
         const relationshipECClassIdsToSkip = new Set();
         for await (const row of this.sourceDb.createQueryReader("SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementDrivesElement)")) {
             relationshipECClassIdsToSkip.add(row.ECInstanceId);
@@ -1828,10 +1796,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         for await (const row of this.sourceDb.createQueryReader("SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementRefersToElements)")) {
             relationshipECClassIds.add(row.ECInstanceId);
         }
-        const elementECClassIds = new Set();
-        for await (const row of this.sourceDb.createQueryReader("SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.Element)")) {
-            elementECClassIds.add(row.ECInstanceId);
-        }
         // For later use when processing deletes.
         const alreadyImportedElementInserts = new Set();
         const alreadyImportedModelInserts = new Set();
@@ -1846,9 +1810,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 alreadyImportedModelInserts.add(targetModelId);
         });
         this._deletedSourceRelationshipData = new Map();
-        /** a map of element ids to this transformation scope's ESA data for that element, in case the ESA is deleted in the target */
-        const elemIdToScopeEsa = new Map();
-        const deleteIdsProcessed = new Set();
         for (const csFile of this._csFileProps ?? []) {
             const csReader = core_backend_1.SqliteChangesetReader.openFile({
                 fileName: csFile.pathname,
@@ -1861,6 +1822,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 ecChangeUnifier.appendFrom(csAdaptor);
             }
             const changes = [...ecChangeUnifier.instances];
+            /** a map of element ids to this transformation scope's ESA data for that element, in case the ESA is deleted in the target */
+            const elemIdToScopeEsa = new Map();
             for (const change of changes) {
                 if (change.ECClassId !== undefined &&
                     relationshipECClassIdsToSkip.has(change.ECClassId))
@@ -1872,10 +1835,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                     change.Kind === core_backend_1.ExternalSourceAspect.Kind.Element) {
                     elemIdToScopeEsa.set(change.Element.Id, change);
                 }
-                else if ((changeType === "Inserted" || changeType === "Updated") &&
-                    change.ECClassId !== undefined &&
-                    elementECClassIds.has(change.ECClassId))
-                    hasElementChangedCache.add(change.ECInstanceId);
             }
             // Loop to process deletes.
             for (const change of changes) {
@@ -1888,80 +1847,18 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 if (changeType !== "Deleted" ||
                     relationshipECClassIdsToSkip.has(ecClassId))
                     continue;
-                if (relationshipECClassIds.has(ecClassId)) {
-                    if (change.$meta?.classFullName === undefined) {
-                        core_bentley_1.Logger.logError(loggerCategory, "ClassFullName was not found for relationship when reading changes. Relationship delete will not propagate.", { relationshipId: change.ECInstanceId, ecClassId });
-                        continue;
-                    }
-                    if (change.SourceECInstanceId === undefined ||
-                        change.TargetECInstanceId === undefined) {
-                        core_bentley_1.Logger.logError(loggerCategory, "SourceECInstanceId or TargetECInstanceId was not found for relationship when reading changes. Relationship delete will not propagate.", {
-                            relationshipId: change.ECInstanceId,
-                            ecClassId,
-                            classFullName: change.$meta.classFullName,
-                        });
-                        continue;
-                    }
-                    await this.processRelationshipDeleteOp(change.ECInstanceId, change.$meta.classFullName, change.SourceECInstanceId, change.TargetECInstanceId);
-                }
-                else {
-                    await this.processElementDeleteOp(change.ECInstanceId, alreadyImportedElementInserts, alreadyImportedModelInserts, elemIdToScopeEsa, change.FederationGuid);
-                }
-                deleteIdsProcessed.add(change.ECInstanceId);
+                await this.processDeletedOp(change, elemIdToScopeEsa, relationshipECClassIds.has(ecClassId ?? ""), alreadyImportedElementInserts, alreadyImportedModelInserts);
             }
             csReader.close();
         }
-        await this.handleCustomChanges(hasElementChangedCache, deleteIdsProcessed);
-        this._hasElementChangedCache = hasElementChangedCache;
         return;
     }
     /**
-     * Helper function for processChangesets.
-     * Populates the '_deletedSourceRelationshipData' map, whose key is the id of the relationship in the source and the value is an object used to find that relationship in the target.
-     * @param changedInstanceId The id of the relationship that was deleted
-     * @param classFullName classFullName of relationship
-     * @param sourceIdOfRelationshipInSource the element Id acting as the source of the relationship in the sourceDb
-     * @param targetIdOfRelationshipInSource the element Id acting as the target of the relationship in the sourceDb
-     * @returns
-     */
-    async processRelationshipDeleteOp(changedInstanceId, classFullName, sourceIdOfRelationshipInSource, targetIdOfRelationshipInSource) {
-        // we need a connected iModel with changes to remap elements with deletions
-        const notConnectedModel = this.sourceDb.iTwinId === undefined;
-        const noChanges = this.synchronizationVersion.index === this.sourceDb.changeset.index &&
-            (this.exporter.sourceDbChanges === undefined ||
-                !this.exporter.sourceDbChanges.hasChanges);
-        if (notConnectedModel || noChanges)
-            return;
-        const sourceIdOfRelationshipInTarget = await this.getTargetIdFromSourceId(sourceIdOfRelationshipInSource, true);
-        const targetIdOfRelationshipInTarget = await this.getTargetIdFromSourceId(targetIdOfRelationshipInSource, true);
-        if (sourceIdOfRelationshipInTarget && targetIdOfRelationshipInTarget) {
-            this._deletedSourceRelationshipData.set(changedInstanceId, {
-                classFullName,
-                sourceIdInTarget: sourceIdOfRelationshipInTarget,
-                targetIdInTarget: targetIdOfRelationshipInTarget,
-            });
-        }
-        else if (this.sourceDb === this.provenanceSourceDb) {
-            const relProvenance = this._queryProvenanceForRelationship(changedInstanceId, {
-                classFullName,
-                sourceId: sourceIdOfRelationshipInSource,
-                targetId: targetIdOfRelationshipInSource,
-            });
-            if (relProvenance && relProvenance.relationshipId)
-                this._deletedSourceRelationshipData.set(changedInstanceId, {
-                    classFullName,
-                    relId: relProvenance.relationshipId,
-                    provenanceAspectId: relProvenance.aspectId,
-                });
-        }
-    }
-    /**
-     * This function is called by the transformer as it is about to process the changesets passed to it in [[IModelTransformOptions.argsForProcessChanges]].
-     * This would be after the exporter has already processed the same set of changesets passed to the transformer in [[IModelTransformOptions.argsForProcessChanges]].
-     * This function should be used to modify the exporter's sourceDbChanges, if necessary, using [[ChangedInstanceIds.addCustomChange]]. See [[ChangedInstanceIds.addCustomChange]] for more information.
-     * @param sourceDbChanges will only be defined if the transformer was called with [[IModelTransformOptions.argsForProcessChanges]].
-     * @note If defined, sourceDbChanges will already be populated with the changesets passed to the transformer, if any when this function is called by the transformer.
-     * @note The transformer will have built up the remap table between the source and target iModels before calling this function. This means that functions like [[IModelTransformer.context.findTargetElementId]] will return meaningful results.
+     * This will be called when transformer is called with [[IModelTransformOptions.argsForProcessChanges]] to process changes.
+     * It will be executed after changes in changesets are populated into `sourceDbChanges` and before data processing begins.
+     * Remap table between the source and target iModels will be built at that time, meaning that functions like [[IModelTransformer.context.findTargetElementId]] will return meaningful results.
+     * This function should be used to modify the `sourceDbChanges`, if necessary, using `add custom change` methods in [[ChangedInstanceIds]], such as [[ChangedInstanceIds.addCustomElementChange]], [[ChangedInstanceIds.addCustomModelChange]] and other.
+     * @param sourceDbChanges the ChangedInstanceIds already populated by the exporter with the changes in source changesets, if any, passed to the transformer.
      * @note Its expected that this function be overridden by a subclass of transformer if it needs to modify sourceDbChanges.
      */
     async addCustomChanges(_sourceDbChanges) { }
@@ -1970,11 +1867,12 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @param change the change to process, must be of changeType "Deleted"
      * @param mapOfDeletedElemIdToScopeEsas a map of elementIds to changedECInstances (which are ESAs). the elementId is not the id of the esa itself, but the elementid that the esa was stored on before the esa's deletion.
      * All ESAs in this map are part of the transformer's scope / ESA data and are tracked in case the ESA is deleted in the target.
+     * @param isRelationship is relationship or not
      * @param alreadyImportedElementInserts used to handle entity recreation and not delete already handled element inserts.
      * @param alreadyImportedModelInserts used to handle entity recreation and not delete already handled model inserts.
      * @returns void
      */
-    async processElementDeleteOp(changedInstanceId, alreadyImportedElementInserts, alreadyImportedModelInserts, mapOfDeletedElemIdToScopeEsas, federationGuid) {
+    async processDeletedOp(change, mapOfDeletedElemIdToScopeEsas, isRelationship, alreadyImportedElementInserts, alreadyImportedModelInserts) {
         // we need a connected iModel with changes to remap elements with deletions
         const notConnectedModel = this.sourceDb.iTwinId === undefined;
         const noChanges = this.synchronizationVersion.index === this.sourceDb.changeset.index &&
@@ -1982,69 +1880,92 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 !this.exporter.sourceDbChanges.hasChanges);
         if (notConnectedModel || noChanges)
             return;
-        let targetId = await this.getTargetIdFromSourceId(changedInstanceId, false, mapOfDeletedElemIdToScopeEsas, federationGuid);
-        if (targetId === undefined && this.sourceDb === this.provenanceSourceDb) {
-            targetId = this._queryProvenanceForElement(changedInstanceId);
-        }
-        // since we are processing one changeset at a time, we can see local source deletes
-        // of entities that were never synced and can be safely ignored
-        const deletionNotInTarget = !targetId;
-        if (deletionNotInTarget)
-            return;
-        this.context.remapElement(changedInstanceId, targetId);
-        // If an entity insert and an entity delete both point to the same entity in target iModel, that means that entity was recreated.
-        // In such case an entity update will be triggered and we no longer need to delete the entity.
-        if (alreadyImportedElementInserts.has(targetId)) {
-            this.exporter.sourceDbChanges?.element.deleteIds.delete(changedInstanceId);
-        }
-        if (alreadyImportedModelInserts.has(targetId)) {
-            this.exporter.sourceDbChanges?.model.deleteIds.delete(changedInstanceId);
-        }
-    }
-    /**
-     * Find the corresponding id in the targetDb given a id from the sourceDb
-     * @param id the id in the source that we want to find the target id for
-     * @param isRelationship Changes the way we look for the federationGuid , if true we look for the federationGuid on the element itself, if false we expect it to be passed in because it was part of the ChangedECInstance.
-     * Typically the source and targetIds of the relationship and not the relationshipId itself is passed to this function
-     * @param mapOfDeletedElemIdToScopeEsas a map of elementIds to changedECInstances (which are ESAs). the elementId is not the id of the esa itself, but the elementid that the esa was stored on before the esa's deletion.
-     * All ESAs in this map are part of the transformer's scope / ESA data and are tracked in case the ESA is deleted in the target.
-     * @param federationGuid
-     * @returns id of the corresponding entity in the targetDb or undefined if not found
-     */
-    async getTargetIdFromSourceId(id, isRelationship, mapOfDeletedElemIdToScopeEsas, federationGuid) {
         /**
          * if our ChangedECInstance is in the provenanceDb, then we can use the ids we find in the ChangedECInstance to query for ESAs.
          * This is because the ESAs are stored on an element Id thats present in the provenanceDb.
          */
         const changeDataInProvenanceDb = this.sourceDb === this.provenanceDb;
-        let identifierValue;
-        let element;
-        if (isRelationship) {
-            element = this.sourceDb.elements.tryGetElement(id);
-        }
-        const fedGuid = isRelationship ? element?.federationGuid : federationGuid;
-        // Check for targetId using sourceId's fedguid
-        if (fedGuid) {
-            const targetId = this._queryElemIdByFedGuid(this.targetDb, fedGuid);
-            if (targetId !== undefined)
+        const getTargetIdFromSourceId = async (id) => {
+            let identifierValue;
+            let element;
+            if (isRelationship) {
+                element = this.sourceDb.elements.tryGetElement(id);
+            }
+            const fedGuid = isRelationship
+                ? element?.federationGuid
+                : change.FederationGuid;
+            if (changeDataInProvenanceDb) {
+                // TODO: clarify what happens if there are multiple (e.g. elements were merged)
+                for await (const row of this.sourceDb.createQueryReader("SELECT esa.Identifier FROM bis.ExternalSourceAspect esa WHERE Scope.Id=:scopeId AND Kind=:kind AND Element.Id=:relatedElementId LIMIT 1", core_common_1.QueryBinder.from([
+                    this.targetScopeElementId,
+                    core_backend_1.ExternalSourceAspect.Kind.Element,
+                    id,
+                ]))) {
+                    identifierValue = row.Identifier;
+                }
+                identifierValue =
+                    identifierValue ?? mapOfDeletedElemIdToScopeEsas.get(id)?.Identifier;
+            }
+            // Check for targetId by an esa first
+            if (changeDataInProvenanceDb && identifierValue) {
+                const targetId = identifierValue;
+                return targetId;
+            }
+            // Check for targetId using sourceId's fedguid if we didn't find an esa.
+            if (fedGuid) {
+                const targetId = this._queryElemIdByFedGuid(this.targetDb, fedGuid);
                 return targetId;
+            }
+            return undefined;
+        };
+        const changedInstanceId = change.ECInstanceId;
+        if (isRelationship) {
+            const sourceIdOfRelationshipInSource = change.SourceECInstanceId;
+            const targetIdOfRelationshipInSource = change.TargetECInstanceId;
+            const classFullName = change.$meta?.classFullName;
+            const sourceIdOfRelationshipInTarget = await getTargetIdFromSourceId(sourceIdOfRelationshipInSource);
+            const targetIdOfRelationshipInTarget = await getTargetIdFromSourceId(targetIdOfRelationshipInSource);
+            if (sourceIdOfRelationshipInTarget && targetIdOfRelationshipInTarget) {
+                this._deletedSourceRelationshipData.set(changedInstanceId, {
+                    classFullName: classFullName ?? "",
+                    sourceIdInTarget: sourceIdOfRelationshipInTarget,
+                    targetIdInTarget: targetIdOfRelationshipInTarget,
+                });
+            }
+            else if (this.sourceDb === this.provenanceSourceDb) {
+                const relProvenance = this._queryProvenanceForRelationship(changedInstanceId, {
+                    classFullName: classFullName ?? "",
+                    sourceId: sourceIdOfRelationshipInSource,
+                    targetId: targetIdOfRelationshipInSource,
+                });
+                if (relProvenance && relProvenance.relationshipId)
+                    this._deletedSourceRelationshipData.set(changedInstanceId, {
+                        classFullName: classFullName ?? "",
+                        relId: relProvenance.relationshipId,
+                        provenanceAspectId: relProvenance.aspectId,
+                    });
+            }
         }
-        // Check for targetId by esa
-        if (changeDataInProvenanceDb) {
-            // TODO: clarify what happens if there are multiple (e.g. elements were merged)
-            for await (const row of this.sourceDb.createQueryReader("SELECT esa.Identifier FROM bis.ExternalSourceAspect esa WHERE Scope.Id=:scopeId AND Kind=:kind AND Element.Id=:relatedElementId LIMIT 1", core_common_1.QueryBinder.from([
-                this.targetScopeElementId,
-                core_backend_1.ExternalSourceAspect.Kind.Element,
-                id,
-            ]))) {
-                identifierValue = row.Identifier;
+        else {
+            let targetId = await getTargetIdFromSourceId(changedInstanceId);
+            if (targetId === undefined && this.sourceDb === this.provenanceSourceDb) {
+                targetId = this._queryProvenanceForElement(changedInstanceId);
+            }
+            // since we are processing one changeset at a time, we can see local source deletes
+            // of entities that were never synced and can be safely ignored
+            const deletionNotInTarget = !targetId;
+            if (deletionNotInTarget)
+                return;
+            this.context.remapElement(changedInstanceId, targetId);
+            // If an entity insert and an entity delete both point to the same entity in target iModel, that means that entity was recreated.
+            // In such case an entity update will be triggered and we no longer need to delete the entity.
+            if (alreadyImportedElementInserts.has(targetId)) {
+                this.exporter.sourceDbChanges?.element.deleteIds.delete(changedInstanceId);
+            }
+            if (alreadyImportedModelInserts.has(targetId)) {
+                this.exporter.sourceDbChanges?.model.deleteIds.delete(changedInstanceId);
             }
-            identifierValue =
-                identifierValue ?? mapOfDeletedElemIdToScopeEsas?.get(id)?.Identifier;
-            if (identifierValue)
-                return identifierValue;
         }
-        return undefined;
     }
     async _tryInitChangesetData(args) {
         if (!args ||