@itwin/imodel-transformer 2.0.0-dev.50 → 2.0.0-dev.52

package/lib/cjs/IModelExporter.js

@@ -18,6 +18,7 @@ const ElementAspectExportProcessor_1 = require("./ElementAspectExportProcessor")
 const ElementAspectExportCoordinator_1 = require("./ElementAspectExportCoordinator");
 const IModelTransformerError_1 = require("./IModelTransformerError");
 const ChangesetScanner_1 = require("./ChangesetScanner");
+const ChangedElementForest_1 = require("./ChangedElementForest");
 const loggerCategory = TransformerLoggerCategory_1.TransformerLoggerCategory.IModelExporter;
 /** Handles the events generated by IModelExporter.
  * @note Change information is available when `IModelExportHandler` methods are invoked via [IModelExporter.exportChanges]($transformer), but not available when invoked via [IModelExporter.exportAll]($transformer).
@@ -75,8 +76,8 @@ class IModelExportHandler {
      * @internal
      */
     async preExportElement(_element) { }
-    /** Called when an element should be deleted. */
-    async onDeleteElement(_elementId) { }
+    /** Called once with the source IDs of all elements deleted by the exported changes. */
+    async onDeleteElements(_elementIds) { }
     /** If `true` is returned, then the ElementAspect will be exported.
      * @note This method can optionally be overridden to exclude an individual ElementAspect from the export. The base implementation always returns `true`.
      */
@@ -172,6 +173,10 @@ class IModelExporter {
     _progressCounter = 0;
     /** Optionally cached entity change information */
     _sourceDbChanges;
+    /** State shared by nested calls in one change-processing element traversal. */
+    _changedElementTraversal;
+    /** Bounds the additional hierarchy retained by the sparse changed-element path. */
+    _changedElementForestElementLimit = 100_000;
     /**
      * Retrieve the cached entity change information.
      * @note This will only be initialized after [IModelExporter.exportChanges] is invoked or [IModelExporter.initialize] is called.
@@ -262,7 +267,9 @@ class IModelExporter {
     excludeCodeSpec(codeSpecName) {
         this._excludedCodeSpecNames.add(codeSpecName);
     }
-    /** Add a rule to exclude a specific Element. */
+    /** Add a rule to exclude a specific Element.
+     * @note Configure exclusions before starting an export operation. Changing exclusions while an export is in progress is unsupported.
+     */
     excludeElement(elementId) {
         this._excludedElementIds.add(elementId);
     }
@@ -337,27 +344,33 @@ class IModelExporter {
         (0, node_assert_1.strict)(this._sourceDbChanges !== undefined, "sourceDbChanges must be initialized.");
         await this.exportCodeSpecs();
         await this.exportFonts();
-        await this._elementAspectExportCoordinator.run(async () => {
-            if (initOpts.skipPropagateChangesToRootElements) {
-                // The root Subject is in the RepositoryModel. Traverse its children
-                // separately, then export other top-level repository elements while
-                // excluding the root so no element is visited twice.
-                await this.exportChildElements(core_common_1.IModel.rootSubjectId);
-                await this.exportModelContents(core_common_1.IModel.repositoryModelId, core_backend_1.Element.classFullName, true);
-                await this.exportSubModels(core_common_1.IModel.repositoryModelId);
+        (0, node_assert_1.strict)(this.beginChangedElementTraversalScope(), "exportChanges must own its changed-element traversal scope");
+        try {
+            await this._elementAspectExportCoordinator.run(async () => {
+                if (initOpts.skipPropagateChangesToRootElements) {
+                    // The root Subject is in the RepositoryModel. Traverse its children
+                    // separately, then export other top-level repository elements while
+                    // excluding the root so no element is visited twice.
+                    await this.exportChildElements(core_common_1.IModel.rootSubjectId);
+                    await this.exportModelContents(core_common_1.IModel.repositoryModelId, core_backend_1.Element.classFullName, true);
+                    await this.exportSubModels(core_common_1.IModel.repositoryModelId);
+                }
+                else {
+                    await this.exportModel(core_common_1.IModel.repositoryModelId);
+                }
+            });
+            const aspectOnlyOwnerElementIds = new Set(this._sourceDbChanges.aspectOwnerElementIds);
+            for (const elementId of this._sourceDbChanges.element.insertIds) {
+                aspectOnlyOwnerElementIds.delete(elementId);
             }
-            else {
-                await this.exportModel(core_common_1.IModel.repositoryModelId);
+            for (const elementId of this._sourceDbChanges.element.updateIds) {
+                aspectOnlyOwnerElementIds.delete(elementId);
             }
-        });
-        const aspectOnlyOwnerElementIds = new Set(this._sourceDbChanges.aspectOwnerElementIds);
-        for (const elementId of this._sourceDbChanges.element.insertIds) {
-            aspectOnlyOwnerElementIds.delete(elementId);
+            await this.exportAspectsForOwners(await this.filterOwnerElementIdsForAspectExport(aspectOnlyOwnerElementIds));
         }
-        for (const elementId of this._sourceDbChanges.element.updateIds) {
-            aspectOnlyOwnerElementIds.delete(elementId);
+        finally {
+            this.endChangedElementTraversalScope();
         }
-        await this.exportAspectsForOwners(await this.filterOwnerElementIdsForAspectExport(aspectOnlyOwnerElementIds));
         await this.exportRelationships(core_backend_1.ElementRefersToElements.classFullName);
         // handle deletes
         if (this.visitElements) {
@@ -365,22 +378,8 @@ class IModelExporter {
             for (const modelId of this._sourceDbChanges.model.deleteIds) {
                 await this.handler.onDeleteModel(modelId);
             }
-            for (const elementId of this._sourceDbChanges.element.deleteIds) {
-                // We don't know how the handler wants to handle deletions, and we don't have enough information
-                // to know if deleted entities were related, so when processing changes, ignore errors from deletion.
-                // Technically, to keep the ignored error scope small, we ignore only the error of looking up a missing element,
-                // that approach works at least for the IModelTransformer.
-                // In the future, the handler may be responsible for doing the work of finding out which elements were cascade deleted,
-                // and returning them for the exporter to use to avoid double-deleting with error ignoring
-                try {
-                    await this.handler.onDeleteElement(elementId);
-                }
-                catch (err) {
-                    const isMissingErr = err instanceof core_common_1.IModelError &&
-                        err.errorNumber === core_bentley_1.IModelStatus.NotFound;
-                    if (!isMissingErr)
-                        throw err;
-                }
+            if (this._sourceDbChanges.element.deleteIds.size > 0) {
+                await this.handler.onDeleteElements(new Set(this._sourceDbChanges.element.deleteIds));
             }
         }
         if (this.visitRelationships) {
@@ -623,6 +622,20 @@ class IModelExporter {
                 !this._sourceDbChanges.model.updateIds.has(modelId)) {
                 return; // this optimization assumes that the Model changes (LastMod) any time an Element in the Model changes
             }
+            if (this.canUseChangedElementForest() &&
+                elementClassFullName === core_backend_1.Element.classFullName) {
+                const forest = await this.getChangedElementForest();
+                if (forest !== undefined) {
+                    core_bentley_1.Logger.logTrace(loggerCategory, `exportModelContents(${modelId}) via changed-element index`);
+                    for (const rootId of forest.getModelRoots(modelId)) {
+                        if (skipRootSubject && rootId === core_common_1.IModel.rootSubjectId)
+                            continue;
+                        await this.exportElement(rootId);
+                        await this._yieldManager.allowYield();
+                    }
+                    return;
+                }
+            }
         }
         core_bentley_1.Logger.logTrace(loggerCategory, `exportModelContents(${modelId})`);
         let sql;
@@ -718,20 +731,19 @@ class IModelExporter {
             core_bentley_1.Logger.logTrace(loggerCategory, `visitElements=false, skipping exportElement(${elementId})`);
             return;
         }
-        // Return early if the elementId is already in the excludedElementIds, that way we don't need to load the element from the db.
+        // Return early if the elementId is already excluded so it does not need to be loaded.
         if (this._excludedElementIds.has(elementId)) {
             core_bentley_1.Logger.logInfo(loggerCategory, `Excluded element ${elementId} by Id`);
             await this.handler.onSkipElement(elementId);
             return;
         }
-        // are we processing changes?
         const isUpdate = this._sourceDbChanges?.element.insertIds.has(elementId)
             ? false
             : this._sourceDbChanges?.element.updateIds.has(elementId)
                 ? true
                 : undefined;
-        // Short-circuit: element is not in the changeset, skip its own export but still visit children
-        if (undefined !== this._sourceDbChanges && undefined === isUpdate) {
+        // An unchanged element may still connect a changed descendant to its model root.
+        if (this._sourceDbChanges !== undefined && isUpdate === undefined) {
             return this.exportChildElements(elementId);
         }
         const element = this.sourceDb.elements.getElement({
@@ -748,9 +760,7 @@ class IModelExporter {
             await this._elementAspectExportCoordinator.addAcceptedOwner(elementId);
             return this.exportChildElements(elementId);
         }
-        else {
-            await this.handler.onSkipElement(element.id);
-        }
+        await this.handler.onSkipElement(element.id);
     }
     /** Export the child elements of the specified element from the source iModel.
      * @note This method is called from [[exportChanges]] and [[exportAll]], so it only needs to be called directly when exporting a subset of an iModel.
@@ -763,7 +773,15 @@ class IModelExporter {
             core_bentley_1.Logger.logTrace(loggerCategory, `visitElements=false, skipping exportChildElements(${elementId})`);
             return;
         }
-        const childElementIds = this.sourceDb.elements.queryChildren(elementId);
+        let childElementIds;
+        if (this.canUseChangedElementForest()) {
+            const forest = await this.getChangedElementForest();
+            if (forest !== undefined) {
+                core_bentley_1.Logger.logTrace(loggerCategory, `exportChildElements(${elementId}) via changed-element index`);
+                childElementIds = forest.getChildren(elementId);
+            }
+        }
+        childElementIds ??= this.sourceDb.elements.queryChildren(elementId);
         if (childElementIds.length > 0) {
             core_bentley_1.Logger.logTrace(loggerCategory, `exportChildElements(${elementId})`);
             for (const childElementId of childElementIds) {
@@ -771,6 +789,54 @@ class IModelExporter {
             }
         }
     }
+    /** Whether change processing may use the sparse hierarchy as the child-id source.
+     * Custom traversal overrides retain the previous full element dispatch.
+     */
+    canUseChangedElementForest() {
+        return (this._sourceDbChanges !== undefined &&
+            this._changedElementTraversal?.useForest === true &&
+            this.exportElement === IModelExporter.prototype.exportElement &&
+            this.exportChildElements === IModelExporter.prototype.exportChildElements);
+    }
+    getOverriddenElementTraversalMethods() {
+        const methods = [];
+        if (this.exportElement !== IModelExporter.prototype.exportElement)
+            methods.push("exportElement");
+        if (this.exportChildElements !== IModelExporter.prototype.exportChildElements)
+            methods.push("exportChildElements");
+        return methods;
+    }
+    disableChangedElementForest(reason) {
+        if (!this._changedElementTraversal?.useForest)
+            return;
+        this._changedElementTraversal.useForest = false;
+        core_bentley_1.Logger.logInfo(loggerCategory, `Using full element traversal for change processing because ${reason}.`);
+    }
+    async createChangedElementForest() {
+        (0, node_assert_1.strict)(this._sourceDbChanges !== undefined, "changed-element traversal requires sourceDbChanges");
+        const excludedElementIds = this._excludedElementIds;
+        const candidateCountUpperBound = this._sourceDbChanges.element.insertIds.size +
+            this._sourceDbChanges.element.updateIds.size +
+            excludedElementIds.size;
+        if (candidateCountUpperBound > this._changedElementForestElementLimit) {
+            this.disableChangedElementForest(`${candidateCountUpperBound} candidate element references exceed the ${this._changedElementForestElementLimit}-element in-memory index limit`);
+            return undefined;
+        }
+        const forest = await ChangedElementForest_1.ChangedElementForest.create(this.sourceDb, [
+            ...this._sourceDbChanges.element.insertIds,
+            ...this._sourceDbChanges.element.updateIds,
+            ...excludedElementIds,
+        ], this._changedElementForestElementLimit);
+        if (forest === undefined) {
+            this.disableChangedElementForest(`the required parent hierarchy exceeds the ${this._changedElementForestElementLimit}-element in-memory index limit`);
+        }
+        return forest;
+    }
+    async getChangedElementForest() {
+        const traversal = this._changedElementTraversal;
+        (0, node_assert_1.strict)(traversal !== undefined, "changed-element index requires an active traversal scope");
+        return (traversal.forest ??= this.createChangedElementForest());
+    }
     /** Exports all aspects owned by the supplied elements. */
     async exportAspectsForOwners(ownerElementIds) {
         return this._elementAspectExportCoordinator.exportOwners(ownerElementIds);
@@ -831,8 +897,30 @@ class IModelExporter {
         }
         return this.shouldExportModelForAspect(modelFact.parentModelId, elementFacts, modelFacts);
     }
+    beginChangedElementTraversalScope() {
+        if (this._sourceDbChanges === undefined ||
+            this._changedElementTraversal !== undefined) {
+            return false;
+        }
+        this._changedElementTraversal = { useForest: true };
+        const overriddenMethods = this.getOverriddenElementTraversalMethods();
+        if (this.visitElements && overriddenMethods.length > 0) {
+            this.disableChangedElementForest(`${overriddenMethods.join(" and ")} ${overriddenMethods.length === 1 ? "is" : "are"} overridden by the exporter subclass`);
+        }
+        return true;
+    }
+    endChangedElementTraversalScope() {
+        this._changedElementTraversal = undefined;
+    }
     async runScopedElementExport(exportElements) {
-        await this._elementAspectExportCoordinator.run(exportElements);
+        const ownsChangedElementTraversal = this.beginChangedElementTraversalScope();
+        try {
+            await this._elementAspectExportCoordinator.run(exportElements);
+        }
+        finally {
+            if (ownsChangedElementTraversal)
+                this.endChangedElementTraversalScope();
+        }
     }
     /** Apply the element export filter when deciding whether to process an aspect owner. @internal */
     async shouldExportElementForAspect(elementId, elementFact) {
@@ -910,8 +998,7 @@ class IModelExporter {
                 ec_className(e.ECClassId, 's') schemaName,
                 ec_className(e.ECClassId, 'c') className
          FROM bis.Element e
-         INNER JOIN IdSet(:elementIds) ids ON ids.id = e.ECInstanceId
-         OPTIONS ENABLE_EXPERIMENTAL_FEATURES`, queryParams, { usePrimaryConn: true })) {
+         INNER JOIN IdSet(:elementIds) ids ON ids.id = e.ECInstanceId`, queryParams, { usePrimaryConn: true })) {
                 elementFacts.set(row.id, {
                     id: row.id,
                     parentId: row.parentId ?? undefined,
@@ -923,8 +1010,7 @@ class IModelExporter {
                 const categoryQueryParams = new core_common_1.QueryBinder().bindIdSet("elementIds", elementIdBatch);
                 for await (const row of this.sourceDb.createQueryReader(`SELECT g.ECInstanceId id, g.Category.Id categoryId
            FROM bis.GeometricElement g
-           INNER JOIN IdSet(:elementIds) ids ON ids.id = g.ECInstanceId
-           OPTIONS ENABLE_EXPERIMENTAL_FEATURES`, categoryQueryParams, { usePrimaryConn: true })) {
+           INNER JOIN IdSet(:elementIds) ids ON ids.id = g.ECInstanceId`, categoryQueryParams, { usePrimaryConn: true })) {
                     const elementFact = elementFacts.get(row.id);
                     if (elementFact !== undefined) {
                         elementFact.categoryId = row.categoryId;
@@ -944,8 +1030,7 @@ class IModelExporter {
                 const queryParams = new core_common_1.QueryBinder().bindIdSet("modelIds", modelIdBatch);
                 for await (const row of this.sourceDb.createQueryReader(`SELECT model.ECInstanceId id, model.ParentModel.Id parentModelId, model.IsTemplate isTemplate
            FROM bis.Model model
-           INNER JOIN IdSet(:modelIds) ids ON ids.id = model.ECInstanceId
-           OPTIONS ENABLE_EXPERIMENTAL_FEATURES`, queryParams, { usePrimaryConn: true })) {
+           INNER JOIN IdSet(:modelIds) ids ON ids.id = model.ECInstanceId`, queryParams, { usePrimaryConn: true })) {
                     modelFacts.set(row.id, {
                         id: row.id,
                         parentModelId: row.parentModelId ?? undefined,
@@ -1448,7 +1533,6 @@ class ChangedInstanceIds {
             INNER JOIN hierarchy h ON h.parentId = e.ECInstanceId
         )
         SELECT parentId FROM hierarchy where parentId is not null
-        OPTIONS ENABLE_EXPERIMENTAL_FEATURES
     `;
         const parentModelIds = new Set();
         for await (const row of this._db.createQueryReader(ecQuery, params, {
@@ -1475,8 +1559,7 @@ class ChangedInstanceIds {
         SELECT relationship.ECInstanceId
         FROM ${relationshipClassName} relationship
         INNER JOIN IdSet(:elementIds) ids
-          ON ids.id = relationship.SourceECInstanceId
-        OPTIONS ENABLE_EXPERIMENTAL_FEATURES`;
+          ON ids.id = relationship.SourceECInstanceId`;
         const queryBinder = new core_common_1.QueryBinder().bindIdSet("elementIds", elementIds);
         const queryReader = this._db.createQueryReader(ecQuery, queryBinder, {
             usePrimaryConn: true,
@@ -1492,8 +1575,7 @@ class ChangedInstanceIds {
         ]) {
             const ecQuery = `SELECT aspect.ECInstanceId, aspect.Element.Id
         FROM ${aspectClassName} aspect
-        INNER JOIN IdSet(:elementIds) ids ON ids.id = aspect.Element.Id
-        OPTIONS ENABLE_EXPERIMENTAL_FEATURES`;
+        INNER JOIN IdSet(:elementIds) ids ON ids.id = aspect.Element.Id`;
             const queryBinder = new core_common_1.QueryBinder().bindIdSet("elementIds", elementIds);
             const queryReader = this._db.createQueryReader(ecQuery, queryBinder, {
                 usePrimaryConn: true,