@itwin/imodel-transformer 1.0.1-dev.0 → 1.1.1-dev.0

package/lib/cjs/IModelExporter.js

@@ -55,7 +55,9 @@ class IModelExportHandler {
     shouldExportElement(_element) {
         return true;
     }
-    /** Called when element is skipped instead of exported. */
+    /** Called when element is skipped instead of exported.
+     * @note When an element is skipped, exporter will not export any of its child elements. Because of this, [[onSkipElement]] will not be invoked for any children of a "skipped" element.
+     */
     onSkipElement(_elementId) { }
     /** Called when an element should be exported.
      * @param element The element to export
@@ -130,7 +132,7 @@ exports.IModelExportHandler = IModelExportHandler;
 class IModelExporter {
     /**
      * Retrieve the cached entity change information.
-     * @note This will only be initialized after [IModelExporter.exportChanges] is invoked.
+     * @note This will only be initialized after [IModelExporter.exportChanges] is invoked or [IModelExporter.initialize] is called.
      */
     get sourceDbChanges() {
         return this._sourceDbChanges;
@@ -252,19 +254,24 @@ class IModelExporter {
         await this.exportModel(core_common_1.IModel.repositoryModelId);
         await this.exportRelationships(core_backend_1.ElementRefersToElements.classFullName);
     }
-    async exportChanges(accessTokenOrOpts, startChangesetId) {
+    /** Export changes from the source iModel.
+     * Inserts, updates, and deletes are determined by inspecting the changeset(s).
+     * @note To form a range of versions to process, set `startChangesetId` for the start (inclusive) of the desired
+     *       range and open the source iModel as of the end (inclusive) of the desired range.
+     * @note the changedInstanceIds are just for this call to exportChanges, so you must continue to pass it in
+     *       for consecutive calls
+     */
+    async exportChanges(args) {
         if (!this.sourceDb.isBriefcaseDb())
             throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Must be a briefcase to export changes");
         if ("" === this.sourceDb.changeset.id) {
             await this.exportAll(); // no changesets, so revert to exportAll
             return;
         }
-        const initOpts = typeof accessTokenOrOpts === "object"
-            ? accessTokenOrOpts
-            : {
-                accessToken: accessTokenOrOpts,
-                startChangeset: { id: startChangesetId },
-            };
+        const startChangeset = args && "startChangeset" in args ? args.startChangeset : undefined;
+        const initOpts = {
+            startChangeset: { id: startChangeset?.id },
+        };
         await this.initialize(initOpts);
         // _sourceDbChanges are initialized in this.initialize
         nodeAssert(this._sourceDbChanges !== undefined, "sourceDbChanges must be initialized.");
@@ -592,6 +599,12 @@ class IModelExporter {
             core_bentley_1.Logger.logTrace(loggerCategory, `visitElements=false, skipping exportElement(${elementId})`);
             return;
         }
+        // Return early if the elementId is already in the excludedElementIds, that way we don't need to load the element from the db.
+        if (this._excludedElementIds.has(elementId)) {
+            core_bentley_1.Logger.logInfo(loggerCategory, `Excluded element ${elementId} by Id`);
+            this.handler.onSkipElement(elementId);
+            return;
+        }
         // are we processing changes?
         const isUpdate = this._sourceDbChanges?.element.insertIds.has(elementId)
             ? false
@@ -702,70 +715,10 @@ class IModelExporter {
             return this.handler.onProgress();
         }
     }
-    /**
-     * You may override this to store arbitrary json state in a exporter state dump, useful for some resumptions
-     * @see [[IModelTransformer.saveStateToFile]]
-     */
-    getAdditionalStateJson() {
-        return {};
-    }
-    /**
-     * You may override this to load arbitrary json state in a transformer state dump, useful for some resumptions
-     * @see [[IModelTransformer.loadStateFromFile]]
-     */
-    loadAdditionalStateJson(_additionalState) { }
-    /**
-     * Reload our state from a JSON object
-     * Intended for [[IModelTransformer.resumeTransformation]]
-     * @internal
-     * You can load custom json from the exporter save state for custom exporters by overriding [[IModelExporter.loadAdditionalStateJson]]
-     */
-    loadStateFromJson(state) {
-        if (state.exporterClass !== this.constructor.name)
-            throw Error("resuming from a differently named exporter class, it is not necessarily valid to resume with a different exporter class");
-        this.wantGeometry = state.wantGeometry;
-        this.wantTemplateModels = state.wantTemplateModels;
-        this.wantSystemSchemas = state.wantSystemSchemas;
-        this.visitElements = state.visitElements;
-        this.visitRelationships = state.visitRelationships;
-        this._excludedCodeSpecNames = new Set(state.excludedCodeSpecNames);
-        (this._excludedElementIds = core_bentley_1.CompressedId64Set.decompressSet(state.excludedElementIds)),
-            (this._excludedElementCategoryIds = core_bentley_1.CompressedId64Set.decompressSet(state.excludedElementCategoryIds)),
-            (this._excludedElementClasses = new Set(state.excludedElementClassNames.map((c) => this.sourceDb.getJsClass(c))));
-        this._exportElementAspectsStrategy.loadExcludedElementAspectClasses(state.excludedElementAspectClassFullNames);
-        this._excludedRelationshipClasses = new Set(state.excludedRelationshipClassNames.map((c) => this.sourceDb.getJsClass(c)));
-        this.loadAdditionalStateJson(state.additionalState);
-    }
-    /**
-     * Serialize state to a JSON object
-     * Intended for [[IModelTransformer.resumeTransformation]]
-     * @internal
-     * You can add custom json to the exporter save state for custom exporters by overriding [[IModelExporter.getAdditionalStateJson]]
-     */
-    saveStateToJson() {
-        return {
-            exporterClass: this.constructor.name,
-            wantGeometry: this.wantGeometry,
-            wantTemplateModels: this.wantTemplateModels,
-            wantSystemSchemas: this.wantSystemSchemas,
-            visitElements: this.visitElements,
-            visitRelationships: this.visitRelationships,
-            excludedCodeSpecNames: [...this._excludedCodeSpecNames],
-            excludedElementIds: core_bentley_1.CompressedId64Set.compressSet(this._excludedElementIds),
-            excludedElementCategoryIds: core_bentley_1.CompressedId64Set.compressSet(this._excludedElementCategoryIds),
-            excludedElementClassNames: Array.from(this._excludedElementClasses, (cls) => cls.classFullName),
-            excludedElementAspectClassFullNames: [
-                ...this._exportElementAspectsStrategy
-                    .excludedElementAspectClassFullNames,
-            ],
-            excludedRelationshipClassNames: Array.from(this._excludedRelationshipClasses, (cls) => cls.classFullName),
-            additionalState: this.getAdditionalStateJson(),
-        };
-    }
 }
 exports.IModelExporter = IModelExporter;
 /** Class for holding change information.
- * @beta
+ * @public
  */
 class ChangedInstanceOps {
     constructor() {
@@ -784,11 +737,20 @@ class ChangedInstanceOps {
                 val.delete.forEach((id) => this.deleteIds.add(id));
         }
     }
+    /**
+     * Checks if empty.
+     * @returns true if there no ids in the ChangedInstanceOps object.
+     */
+    get isEmpty() {
+        return (0 === this.insertIds.size &&
+            0 === this.updateIds.size &&
+            0 === this.deleteIds.size);
+    }
 }
 exports.ChangedInstanceOps = ChangedInstanceOps;
 /**
  * Class for discovering modified elements between 2 versions of an iModel.
- * @beta
+ * @public
  */
 class ChangedInstanceIds {
     constructor(db) {
@@ -806,6 +768,7 @@ class ChangedInstanceIds {
         this._elementSubclassIds = new Set();
         this._aspectSubclassIds = new Set();
         this._relationshipSubclassIds = new Set();
+        this._relationshipSubclassIdsToSkip = new Set();
         const addECClassIdsToSet = async (setToModify, baseClass) => {
             for await (const row of this._db.createQueryReader(`SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (${baseClass})`)) {
                 setToModify.add(row.ECInstanceId);
@@ -818,6 +781,7 @@ class ChangedInstanceIds {
             addECClassIdsToSet(this._aspectSubclassIds, "BisCore.ElementUniqueAspect"),
             addECClassIdsToSet(this._aspectSubclassIds, "BisCore.ElementMultiAspect"),
             addECClassIdsToSet(this._relationshipSubclassIds, "BisCore.ElementRefersToElements"),
+            addECClassIdsToSet(this._relationshipSubclassIdsToSkip, "BisCore.ElementDrivesElement"),
         ];
         await Promise.all(promises);
     }
@@ -826,7 +790,8 @@ class ChangedInstanceIds {
             this._modelSubclassIds &&
             this._elementSubclassIds &&
             this._aspectSubclassIds &&
-            this._relationshipSubclassIds);
+            this._relationshipSubclassIds &&
+            this._relationshipSubclassIdsToSkip);
     }
     isRelationship(ecClassId) {
         return this._relationshipSubclassIds?.has(ecClassId);
@@ -843,6 +808,17 @@ class ChangedInstanceIds {
     isElement(ecClassId) {
         return this._elementSubclassIds?.has(ecClassId);
     }
+    /** Checks if there are any changes.
+     * @returns true if there are any changes in the ChangedInstanceIds object.
+     */
+    get hasChanges() {
+        return (!this.codeSpec.isEmpty ||
+            !this.model.isEmpty ||
+            !this.element.isEmpty ||
+            !this.aspect.isEmpty ||
+            !this.relationship.isEmpty ||
+            !this.font.isEmpty);
+    }
     /**
      * Adds the provided [[ChangedECInstance]] to the appropriate set of changes by class type (codeSpec, model, element, aspect, or relationship) maintained by this instance of ChangedInstanceIds.
      * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x'
@@ -858,6 +834,8 @@ class ChangedInstanceIds {
         const changeType = change.$meta?.op;
         if (changeType === undefined)
             throw new Error(`ChangeType was undefined for id: ${change.ECInstanceId}.`);
+        if (this._relationshipSubclassIdsToSkip?.has(ecClassId))
+            return;
         if (this.isRelationship(ecClassId))
             this.handleChange(this.relationship, changeType, change.ECInstanceId);
         else if (this.isCodeSpec(ecClassId))
@@ -869,6 +847,128 @@ class ChangedInstanceIds {
         else if (this.isElement(ecClassId))
             this.handleChange(this.element, changeType, change.ECInstanceId);
     }
+    /**
+     * This method should only be called inside [[IModelTransformer.addCustomChanges]].
+     * It adds the provided change to the element changes maintained by this instance of ChangedInstanceIds.
+     * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x'
+     * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type.
+     * @note Custom element 'Insert' and 'Update' will mark element's parent model hierarchy and their modeled elements as 'Updated' in [[ChangedInstanceIds.model]] and [[ChangedInstanceIds.element]]. Parent models have to be marked as 'Updated' to make sure that added change is not skipped by transformer. Transformer starts processing elements from RepositoryModel and then visits all child models. Modeled elements hierarchy is marked as updated to trigger their inserts in case a new model (or its parent) needs to be inserted.
+     * @note Custom element 'Insert' will also mark element aspects and all element relationships as inserted.
+     * @note It is the responsibility of the caller to ensure that the provided id is, in fact an element.
+     * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship.
+     * @note In data processing with filter criteria scenarios it is important to consistently filter out models and their modeled elements that were previously removed from target via [[addCustomModelChange]] or [[shouldExportElement]] apis.
+     * @beta
+     */
+    async addCustomElementChange(changeType, ids) {
+        if (core_bentley_1.Id64.sizeOf(ids) === 0) {
+            return;
+        }
+        for (const id of core_bentley_1.Id64.iterable(ids)) {
+            this.handleChange(this.element, changeType, id);
+        }
+        if (changeType === "Deleted") {
+            return;
+        }
+        const idsSet = core_bentley_1.Id64.toIdSet(ids);
+        // Parent models have to be marked as 'Updated' to make sure that added change is not skipped by transformer. Transformer starts processing elements from RepositoryModel and then visits all child models.
+        // Transformer handles update as insert if element is not found in target, for this reason modeled elements will be also marked as updated to trigger their inserts in case a new model (or its parent) needs to be inserted. Otherwise error would be thrown about missing modeled element while inserting new model.
+        const parentModelIds = await this.markParentModelsAsUpdated(idsSet);
+        // Aspects and relationships of inserted data needs to be marked as inserted otherwise those would not be exported
+        if (changeType === "Inserted") {
+            // Adding parents as well as we are not sure if those were inserted or updated
+            parentModelIds.forEach((parentId) => {
+                idsSet.add(parentId);
+            });
+            await this.markElementAspectsAsInserted(idsSet);
+            // Marking only ElementRefersToElements.classFullName as only those are exported in exportRelationships()
+            await this.markElementRelationshipsAsInserted(core_backend_1.ElementRefersToElements.classFullName, idsSet);
+        }
+    }
+    /**
+     * This method should only be called inside [IModelTransformer.addCustomChanges].
+     * Adds the provided change to the model changes maintained by this instance of ChangedInstanceIds.
+     * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x'
+     * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type.
+     * Will add same change to the model's modeledElement by calling [[ChangedInstanceIds.addCustomElementChange]] which will register more needed changes. This is to ensure the changes from the model and its modeledElement get exported together.
+     * @note It is the responsibility of the caller to ensure that the provided id is, in fact a model.
+     * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship.
+     * @note In data processing with filter criteria scenarios it is important to consistently filter out models and their modeled elements that were previously removed from target via [[addCustomModelChange]] or [[shouldExportElement]] apis.
+     * @beta
+     */
+    async addCustomModelChange(changeType, ids) {
+        // Also add the model's modeledElement to the element changes. The modeledElement and model go hand in hand and have the same id.
+        await this.addCustomElementChange(changeType, ids);
+        for (const id of core_bentley_1.Id64.iterable(ids)) {
+            this.handleChange(this.model, changeType, id);
+        }
+    }
+    /**
+     * This method should only be called inside [IModelTransformer.addCustomChanges].
+     * Adds the provided change to the aspect changes maintained by this instance of ChangedInstanceIds
+     * If the same ECInstanceId is seen multiple times, the changedInstanceIds will be modified accordingly, i.e. if an id 'x' was updated but now we see 'x' was deleted, we will remove 'x'
+     * from the set of updatedIds and add it to the set of deletedIds for the appropriate class type.
+     * @note It is the responsibility of the caller to ensure that the provided id is, in fact an aspect.
+     * @note In most cases, this method does not need to be called. Its only for consumers to mimic changes as if they were found in a changeset, which should only be useful in certain cases such as the changing of filter criteria for a preexisting master branch relationship.
+     * @beta
+     */
+    addCustomAspectChange(changeType, ids) {
+        for (const id of core_bentley_1.Id64.iterable(ids)) {
+            this.handleChange(this.aspect, changeType, id);
+        }
+    }
+    /**
+     * There is an optimization in [IModelExporter.exportModelContents] which doesn't try to export elements within a model unless the model itself is marked as `Updated` or 'Inserted' in sourceDbChanges. This method is used in [[addCustomElementChange]] and [[addCustomModelChange]] to add the parent model hierarchy to the 'updatedIds' so that the custom element changes are exported.
+     * Transformer will insert 'Updated' model to target if it does not exist there already. To handle such case, modeled elements of parent models are also marked as updated. This is done, because model can not be inserted without it's modeled element.
+     */
+    async markParentModelsAsUpdated(elementIds) {
+        const params = new core_common_1.QueryBinder().bindIdSet("elementIds", elementIds);
+        const ecQuery = `
+    WITH RECURSIVE hierarchy (parentId) AS (
+        SELECT Model.Id FROM bis.Element WHERE InVirtualSet(:elementIds, ECInstanceId)
+        UNION
+        SELECT ParentModel.id
+        FROM bis.Model e
+            INNER JOIN hierarchy h ON h.parentId = e.ECInstanceId
+        )
+        SELECT parentId FROM hierarchy where parentId is not null
+    `;
+        const parentModelIds = new Set();
+        for await (const row of this._db.createQueryReader(ecQuery, params)) {
+            // Transformer handles update as insert when element does not exist in target.
+            // Which means that in scenario where child and parent model are filtered out from target,
+            // and child element is inserted trough custom change, its parent model will be marked as updated.
+            // Transformer then will:
+            //  1. Handle parent update as insert (since it does not exist in target).
+            //  2. Will insert child element (otherwise this insert would be ignored due to missing parent).
+            this.handleChange(this.model, "Updated", row.parentId);
+            this.handleChange(this.element, "Updated", row.parentId);
+            parentModelIds.add(row.parentId);
+        }
+        return parentModelIds;
+    }
+    async markElementRelationshipsAsInserted(relationshipClassName, elementIds) {
+        const ecQuery = `SELECT ECInstanceId FROM ${relationshipClassName}
+        WHERE InVirtualSet(:elementIds, TargetECInstanceId)
+        OR InVirtualSet(:elementIds, SourceECInstanceId)`;
+        const queryBinder = new core_common_1.QueryBinder().bindIdSet("elementIds", elementIds);
+        const queryReader = this._db.createQueryReader(ecQuery, queryBinder);
+        for await (const row of queryReader) {
+            this.handleChange(this.relationship, "Inserted", row.ECInstanceId);
+        }
+    }
+    async markElementAspectsAsInserted(elementIds) {
+        for (const aspectClassName of [
+            core_backend_1.ElementUniqueAspect.classFullName,
+            core_backend_1.ElementMultiAspect.classFullName,
+        ]) {
+            const ecQuery = `Select ECInstanceId from ${aspectClassName} where InVirtualSet(:elementIds, Element.Id)`;
+            const queryBinder = new core_common_1.QueryBinder().bindIdSet("elementIds", elementIds);
+            const queryReader = this._db.createQueryReader(ecQuery, queryBinder);
+            for await (const row of queryReader) {
+                this.addCustomAspectChange("Inserted", row.toArray()[0]);
+            }
+        }
+    }
     handleChange(changedInstanceOps, changeType, id) {
         // if changeType is a delete and we already have the id in the inserts then we can remove the id from the inserts.
         // if changeType is a delete and we already have the id in the updates then we can remove the id from the updates AND add it to the deletes.
@@ -893,12 +993,12 @@ class ChangedInstanceIds {
     }
     /**
      * Initializes a new ChangedInstanceIds object with information taken from a range of changesets.
+     * @public
      */
     static async initialize(opts) {
         if ("changedInstanceIds" in opts)
             return opts.changedInstanceIds;
         const iModelId = opts.iModel.iModelId;
-        const accessToken = opts.accessToken;
         const startChangeset = "startChangeset" in opts ? opts.startChangeset : undefined;
         const changesetRanges = startChangeset !== undefined
             ? [
@@ -909,13 +1009,11 @@ class ChangedInstanceIds {
                             changeset: {
                                 id: startChangeset.id ?? opts.iModel.changeset.id,
                             },
-                            accessToken,
                         })).index,
                     opts.iModel.changeset.index ??
                         (await core_backend_1.IModelHost.hubAccess.queryChangeset({
                             iModelId,
                             changeset: { id: opts.iModel.changeset.id },
-                            accessToken,
                         })).index,
                 ],
             ]
@@ -924,7 +1022,6 @@ class ChangedInstanceIds {
                 : undefined;
         const csFileProps = changesetRanges !== undefined
             ? (await Promise.all(changesetRanges.map(async ([first, end]) => core_backend_1.IModelHost.hubAccess.downloadChangesets({
-                accessToken,
                 iModelId,
                 range: { first, end },
                 targetDir: core_backend_1.BriefcaseManager.getChangeSetsPath(iModelId),
@@ -935,10 +1032,6 @@ class ChangedInstanceIds {
         if (csFileProps === undefined)
             return undefined;
         const changedInstanceIds = new ChangedInstanceIds(opts.iModel);
-        const relationshipECClassIdsToSkip = new Set();
-        for await (const row of opts.iModel.createQueryReader("SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementDrivesElement)")) {
-            relationshipECClassIdsToSkip.add(row.ECInstanceId);
-        }
         for (const csFile of csFileProps) {
             const csReader = core_backend_1.SqliteChangesetReader.openFile({
                 fileName: csFile.pathname,
@@ -952,9 +1045,6 @@ class ChangedInstanceIds {
             }
             const changes = [...ecChangeUnifier.instances];
             for (const change of changes) {
-                if (change.ECClassId !== undefined &&
-                    relationshipECClassIdsToSkip.has(change.ECClassId))
-                    continue;
                 await changedInstanceIds.addChange(change);
             }
             csReader.close();