@itwin/imodel-transformer 1.1.4-dev.0 → 1.2.1-dev.0

package/lib/cjs/IModelTransformer.js

@@ -13,8 +13,8 @@ const Semver = require("semver");
 const nodeAssert = require("assert");
 const core_bentley_1 = require("@itwin/core-bentley");
 const core_geometry_1 = require("@itwin/core-geometry");
+const coreBackendPkgJson = require("@itwin/core-backend/package.json");
 const core_backend_1 = require("@itwin/core-backend");
-const Symbols_1 = require("@itwin/core-backend/lib/cjs/internal/Symbols");
 const core_common_1 = require("@itwin/core-common");
 const IModelExporter_1 = require("./IModelExporter");
 const IModelImporter_1 = require("./IModelImporter");
@@ -66,37 +66,10 @@ function mapId64(idContainer, func) {
  * @beta
  */
 class IModelTransformer extends IModelExporter_1.IModelExportHandler {
-    /** The IModelExporter that will export from the source iModel. */
-    exporter;
-    /** The IModelImporter that will import into the target iModel. */
-    importer;
-    /** The normally read-only source iModel.
-     * @note The source iModel will need to be read/write when provenance is being stored during a reverse synchronization.
-     */
-    sourceDb;
-    /** The read/write target iModel. */
-    targetDb;
-    /** The IModelTransformContext for this IModelTransformer. */
-    context;
-    _syncType;
     /** The Id of the Element in the **target** iModel that represents the **source** repository as a whole and scopes its [ExternalSourceAspect]($backend) instances. */
     get targetScopeElementId() {
         return this._options.targetScopeElementId;
     }
-    /** a set of elements for which source provenance will be explicitly tracked by ExternalSourceAspects */
-    _elementsWithExplicitlyTrackedProvenance = new Set();
-    _partiallyCommittedElementIds = new Set();
-    _partiallyCommittedAspectIds = new Set();
-    /** the options that were used to initialize this transformer */
-    _options;
-    /**
-     * A private variable meant to be set by tests which have an outdated way of setting up transforms. In all synchronizations today we expect to find an ESA in the branch db which describes the master -> branch relationship.
-     * The exception to this is the first transform aka the provenance initializing transform which requires that the master imodel and the branch imodel are identical at the time of provenance initialization.
-     * A couple ofoutdated tests run their first transform providing a source and targetdb that are slightly different which is no longer supported. In order to not remove these tests which are still providing value
-     * this private property on the IModelTransformer exists.
-     */
-    _allowNoScopingESA = false;
-    static noEsaSyncDirectionErrorMessage = "Couldn't find an external source aspect to determine sync direction. This often means that the master->branch relationship has not been established. Consider running the transformer with wasSourceIModelCopiedToTarget set to true.";
     /**
      * Queries for an esa which matches the props in the provided aspectProps.
      * @param dbToQuery db to run the query on for scope external source
@@ -199,12 +172,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             this._syncType = this.determineSyncType();
         return this._syncType === "forward";
     }
-    _changesetRanges = undefined;
-    /**
-     * Set if the transformer is being used to perform the provenance initialization step of a fork initialization.
-     * In general don't use the transformer for that, prefer [[BranchProvenanceInitializer.initializeBranchProvenance]]
-     */
-    _isProvenanceInitTransform;
     /** The element classes that are considered to define provenance in the iModel */
     static get provenanceElementClasses() {
         return [
@@ -225,6 +192,47 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      */
     constructor(source, target, options) {
         super();
+        /** a set of elements for which source provenance will be explicitly tracked by ExternalSourceAspects */
+        this._elementsWithExplicitlyTrackedProvenance = new Set();
+        this._partiallyCommittedElementIds = new Set();
+        this._partiallyCommittedAspectIds = new Set();
+        /**
+         * A private variable meant to be set by tests which have an outdated way of setting up transforms. In all synchronizations today we expect to find an ESA in the branch db which describes the master -> branch relationship.
+         * The exception to this is the first transform aka the provenance initializing transform which requires that the master imodel and the branch imodel are identical at the time of provenance initialization.
+         * A couple ofoutdated tests run their first transform providing a source and targetdb that are slightly different which is no longer supported. In order to not remove these tests which are still providing value
+         * this private property on the IModelTransformer exists.
+         */
+        this._allowNoScopingESA = false;
+        this._changesetRanges = undefined;
+        /**
+         * Previously the transformer would insert provenance always pointing to the "target" relationship.
+         * It should (and now by default does) instead insert provenance pointing to the provenanceSource
+         * SEE: https://github.com/iTwin/imodel-transformer/issues/54
+         * This exists only to facilitate testing that the transformer can handle the older, flawed method
+         */
+        this._forceOldRelationshipProvenanceMethod = false;
+        /** NOTE: the json properties must be converted to string before insertion */
+        this._targetScopeProvenanceProps = undefined;
+        /**
+         * Index of the changeset that the transformer was at when the transformation begins (was constructed).
+         * Used to determine at the end which changesets were part of a synchronization.
+         */
+        this._startingChangesetIndices = undefined;
+        this._cachedSynchronizationVersion = undefined;
+        this._targetClassNameToClassIdCache = new Map();
+        // if undefined, it can be initialized by calling [[this.processChangesets]]
+        this._deletedSourceRelationshipData = undefined;
+        this._yieldManager = new core_bentley_1.YieldManager();
+        /** The directory where schemas will be exported, a random temporary directory */
+        this._schemaExportDir = path.join(core_backend_1.KnownLocations.tmpdir, core_bentley_1.Guid.createValue());
+        this._longNamedSchemasMap = new Map();
+        /** state to prevent reinitialization, @see [[initialize]] */
+        this._initialized = false;
+        this._sourceChangeDataState = "uninited";
+        /** length === 0 when _changeDataState = "no-change", length > 0 means "has-changes", otherwise undefined  */
+        this._csFileProps = undefined;
+        /** previous provenance, either a federation guid, a `${sourceFedGuid}/${targetFedGuid}` pair, or required aspect props */
+        this._lastProvenanceEntityInfo = nullLastProvenanceEntityInfo;
         // initialize IModelTransformOptions
         this._options = {
             ...options,
@@ -235,6 +243,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             danglingReferencesBehavior: options?.danglingReferencesBehavior ?? "reject",
             branchRelationshipDataBehavior: options?.branchRelationshipDataBehavior ?? "reject",
             skipPropagateChangesToRootElements: options?.skipPropagateChangesToRootElements ?? true,
+            tryAlignGeolocation: options?.tryAlignGeolocation ?? false,
         };
         // check if authorization client is defined
         if (core_backend_1.IModelHost.authorizationClient === undefined) {
@@ -290,6 +299,20 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             this.targetDb.codeValueBehavior = "exact";
         }
         /* eslint-enable @itwin/no-internal */
+        if (this._options.tryAlignGeolocation) {
+            if (this.sourceDb.geographicCoordinateSystem ||
+                this.targetDb.geographicCoordinateSystem) {
+                core_bentley_1.Logger.logTrace(loggerCategory, "Aligning Additional transforms between imodels due to imodels containing GeographicCoordinateSystem data");
+                this._linearSpatialTransform =
+                    this.calculateTransformFromHelmertTransforms();
+            }
+            else if (this.sourceDb.ecefLocation && this.targetDb.ecefLocation) {
+                core_bentley_1.Logger.logTrace(loggerCategory, "Aligning ECEF Location's between imodels due to imodels not containing GeographicCoordinateSystem data");
+                this._linearSpatialTransform = this.calculateEcefTransform();
+            }
+            else
+                core_bentley_1.Logger.logTrace(loggerCategory, "No Geolcation data to align, both GCS and ECEF are undefined");
+        }
     }
     /** validates that the importer set on the transformer has the same values for its shared options as the transformer.
      *  @note This expects that the importer is already set on the transformer.
@@ -394,13 +417,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         };
         return aspectProps;
     }
-    /**
-     * Previously the transformer would insert provenance always pointing to the "target" relationship.
-     * It should (and now by default does) instead insert provenance pointing to the provenanceSource
-     * SEE: https://github.com/iTwin/imodel-transformer/issues/54
-     * This exists only to facilitate testing that the transformer can handle the older, flawed method
-     */
-    _forceOldRelationshipProvenanceMethod = false;
     /** Create an ExternalSourceAspectProps in a standard way for an Element in an iModel --> iModel transformation. */
     initElementProvenance(sourceElementId, targetElementId) {
         return IModelTransformer.initElementProvenanceOptions(sourceElementId, targetElementId, {
@@ -424,14 +440,16 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             forceOldRelationshipProvenanceMethod: this._forceOldRelationshipProvenanceMethod,
         });
     }
-    /** NOTE: the json properties must be converted to string before insertion */
-    _targetScopeProvenanceProps = undefined;
     /**
-     * Index of the changeset that the transformer was at when the transformation begins (was constructed).
-     * Used to determine at the end which changesets were part of a synchronization.
+     * As of itwinjs 4.6.0, definitionContainers are now deleted as if they were DefinitionPartitions as opposed to Definitions.
+     * This variable being true will be used to special case the deletion of DefinitionContainers the same way DefinitionPartitions are deleted.
      */
-    _startingChangesetIndices = undefined;
-    _cachedSynchronizationVersion = undefined;
+    get hasDefinitionContainerDeletionFeature() {
+        if (this._hasDefinitionContainerDeletionFeature === undefined) {
+            this._hasDefinitionContainerDeletionFeature = Semver.satisfies(coreBackendPkgJson.version, "^4.6.0");
+        }
+        return this._hasDefinitionContainerDeletionFeature;
+    }
     /**
      * We cache the synchronization version to avoid querying the target scoping ESA multiple times.
      * If the target scoping ESA is ever updated we need to clear any potentially cached sync version otherwise we will get stale values.
@@ -805,7 +823,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             return stmt.getValue(0).getId();
         });
     }
-    _targetClassNameToClassIdCache = new Map();
     _targetClassNameToClassId(classFullName) {
         let classId = this._targetClassNameToClassIdCache.get(classFullName);
         if (classId === undefined) {
@@ -919,10 +936,87 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 targetElementProps.jsonProperties.Subject.Job = undefined;
             }
         }
+        if (this._linearSpatialTransform !== undefined &&
+            sourceElement instanceof core_backend_1.GeometricElement3d) {
+            // can check the sourceElement since this IModelTransformer does not remap classes
+            const placement = core_common_1.Placement3d.fromJSON(targetElementProps.placement);
+            if (placement.isValid) {
+                placement.multiplyTransform(this._linearSpatialTransform);
+                targetElementProps.placement = placement;
+            }
+        }
         return targetElementProps;
     }
-    // if undefined, it can be initialized by calling [[this.processChangesets]]
-    _deletedSourceRelationshipData = undefined;
+    /**
+     * Calculate the transform between two ECEF locations
+     * @param srcDb
+     * @param targetDb
+     * @returns Transform that converts relative coordinates in the source iModel to relative coordinates in the target iModel.
+     * @note This can only be used if both source and target iModels are linearly located
+     */
+    calculateEcefTransform() {
+        const srcEcefLoc = this.sourceDb.ecefLocation;
+        const targetEcefLoc = this.targetDb.ecefLocation;
+        if (srcEcefLoc === undefined || targetEcefLoc === undefined) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.NoGeoLocation, "Both source and target ECEF locations must be defined to calculate the transform.");
+        }
+        if (srcEcefLoc.getTransform().isAlmostEqual(targetEcefLoc.getTransform())) {
+            core_bentley_1.Logger.logTrace(loggerCategory, "ECEF data is already aligned. No spatial transforms needed.");
+            return undefined;
+        }
+        const srcSpatialToECEF = srcEcefLoc.getTransform(); // converts relative to ECEF in relation to source
+        const targetECEFToSpatial = targetEcefLoc.getTransform().inverse(); // converts ECEF to relative in relation to target
+        if (!targetECEFToSpatial) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.NoGeoLocation, "Failed to invert target ECEF transform.");
+        }
+        const ecefTransform = targetECEFToSpatial.multiplyTransformTransform(srcSpatialToECEF); // chain both transforms
+        return ecefTransform;
+    }
+    static convertHelmertToTransform(helmert) {
+        if (!helmert) {
+            return core_geometry_1.Transform.createIdentity();
+        }
+        const rotationXY = core_geometry_1.Matrix3d.createRotationAroundAxisIndex(core_geometry_1.AxisIndex.Z, core_geometry_1.Angle.createDegrees(helmert?.rotDeg));
+        rotationXY.scaleColumnsInPlace(helmert.scale, helmert.scale, 1.0);
+        const translation = core_geometry_1.Vector3d.create(helmert.translationX, helmert.translationY, helmert.translationZ);
+        const helmertTransform = core_geometry_1.Transform.createRefs(translation, rotationXY);
+        return helmertTransform;
+    }
+    calculateTransformFromHelmertTransforms() {
+        if (this.sourceDb.geographicCoordinateSystem?.horizontalCRS === undefined ||
+            this.sourceDb.geographicCoordinateSystem?.verticalCRS === undefined) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Source iModel does not have a geographic coordinate system defined.");
+        }
+        if (this.targetDb.geographicCoordinateSystem?.horizontalCRS === undefined ||
+            this.targetDb.geographicCoordinateSystem.verticalCRS === undefined) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Target iModel does not have a geographic coordinate system defined.");
+        }
+        if (!this.sourceDb.geographicCoordinateSystem.horizontalCRS.equals(this.targetDb.geographicCoordinateSystem.horizontalCRS) ||
+            !this.sourceDb.geographicCoordinateSystem.verticalCRS.equals(this.targetDb.geographicCoordinateSystem.verticalCRS)) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.MismatchGcs, "Source and target geographic coordinate systems must match to calculate the spatial transform.");
+        }
+        if (this.sourceDb.geographicCoordinateSystem.additionalTransform ===
+            this.targetDb.geographicCoordinateSystem.additionalTransform) {
+            core_bentley_1.Logger.logTrace(loggerCategory, "Geolocation data is already aligned. No spatial transforms needed.");
+            return undefined;
+        }
+        const srcScale = this.sourceDb.geographicCoordinateSystem.additionalTransform
+            ?.helmert2DWithZOffset?.scale ?? 1;
+        const targetScale = this.targetDb.geographicCoordinateSystem.additionalTransform
+            ?.helmert2DWithZOffset?.scale ?? 1;
+        if (srcScale !== targetScale) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.MismatchGcs, "Spatial transform is non rigid. Source and target Helmert transforms must have the same scale to calculate a rigid spatial transform.");
+        }
+        const srcTransform = IModelTransformer.convertHelmertToTransform(this.sourceDb.geographicCoordinateSystem.additionalTransform
+            ?.helmert2DWithZOffset); // moves elements to where src helmert transform would move them at render time
+        const targetTransformInv = IModelTransformer.convertHelmertToTransform(this.targetDb.geographicCoordinateSystem.additionalTransform
+            ?.helmert2DWithZOffset).inverse(); // negates target helmert transform that is applied at render time
+        if (!targetTransformInv) {
+            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.NoGeoLocation, "Failed to invert target Helmert transform.");
+        }
+        const combinedTransform = targetTransformInv.multiplyTransformTransform(srcTransform);
+        return combinedTransform;
+    }
     /** Returns true if a change within sourceElement is detected.
      * @param sourceElement The Element from the source iModel
      * @note A subclass can override this method to provide custom change detection behavior.
@@ -1224,7 +1318,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const targetModelId = this.context.findTargetElementId(sourceModelId);
         if (!core_bentley_1.Id64.isValidId64(targetModelId))
             return;
-        const sql = `
+        let sql;
+        if (this.hasDefinitionContainerDeletionFeature) {
+            sql = `
       SELECT 1
       FROM bis.DefinitionPartition
       WHERE ECInstanceId=:targetModelId
@@ -1233,6 +1329,14 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
       FROM bis.DefinitionContainer
       WHERE ECInstanceId=:targetModelId
     `;
+        }
+        else {
+            sql = `
+      SELECT 1
+      FROM bis.DefinitionPartition
+      WHERE ECInstanceId=:targetModelId
+    `;
+        }
         if (this.exporter.sourceDbChanges?.element.deleteIds.has(sourceModelId)) {
             // eslint-disable-next-line @itwin/no-internal, deprecation/deprecation
             const isDefinitionPartition = this.targetDb.withPreparedStatement(sql, (stmt) => {
@@ -1534,7 +1638,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             }
         }
     }
-    _yieldManager = new core_bentley_1.YieldManager();
     /** Detect Relationship deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against relationships in the source iModel.
      * @deprecated in 1.x. Don't use this anymore
      * @see [[process]] with [[IModelTransformOptions.argsForProcessChanges]] provided.
@@ -1647,8 +1750,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const targetElementAspectProps = this.context.cloneElementAspect(sourceElementAspect);
         return targetElementAspectProps;
     }
-    /** The directory where schemas will be exported, a random temporary directory */
-    _schemaExportDir = path.join(core_backend_1.KnownLocations.tmpdir, core_bentley_1.Guid.createValue());
     /** Override of [IModelExportHandler.shouldExportSchema]($transformer) that is called to determine if a schema should be exported
      * @note the default behavior doesn't import schemas older than those already in the target
      */
@@ -1658,7 +1759,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             return true;
         return Semver.gt(`${schemaKey.version.read}.${schemaKey.version.write}.${schemaKey.version.minor}`, core_backend_1.Schema.toSemverString(versionInTarget));
     }
-    _longNamedSchemasMap = new Map();
     /** Override of [IModelExportHandler.onExportSchema]($transformer) that serializes a schema to disk for [[processSchemas]] to import into
      * the target iModel when it is exported from the source iModel.
      * @returns {Promise<ExportSchemaResult>} Although the type is possibly void for backwards compatibility of subclasses,
@@ -1687,11 +1787,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             this._longNamedSchemasMap.set(schema.name, schemaFileName);
         }
         /* eslint-disable-next-line deprecation/deprecation */
-        this.sourceDb.exportSchema({
-            schemaName: schema.name,
-            outputDirectory: this._schemaExportDir,
-            outputFileName: schemaFileName,
-        });
+        this.sourceDb.nativeDb.exportSchema(schema.name, this._schemaExportDir, schemaFileName);
         return { schemaPath: path.join(this._schemaExportDir, schemaFileName) };
     }
     _makeLongNameResolvingSchemaCtx() {
@@ -1777,11 +1873,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.completePartiallyCommittedElements();
         this.completePartiallyCommittedAspects();
     }
-    /** state to prevent reinitialization, @see [[initialize]] */
-    _initialized = false;
-    _sourceChangeDataState = "uninited";
-    /** length === 0 when _changeDataState = "no-change", length > 0 means "has-changes", otherwise undefined  */
-    _csFileProps = undefined;
     /**
      * Initialize prerequisites of processing, you must initialize with an [[InitOptions]] if you
      * are intending to process changes. Callers may wish to explicitly call initialize if they need to execute code after initialize but before [[process]] is called.
@@ -2021,7 +2112,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const endChangesetId = this.sourceDb.changeset.id;
         const [startChangesetIndex, endChangesetIndex] = await Promise.all([startChangesetIndexOrId, endChangesetId].map(async (indexOrId) => typeof indexOrId === "number"
             ? indexOrId
-            : core_backend_1.IModelHost[Symbols_1._hubAccess]
+            : core_backend_1.IModelHost.hubAccess
                 .queryChangeset({
                 iModelId: this.sourceDb.iModelId,
                 // eslint-disable-next-line deprecation/deprecation
@@ -2053,7 +2144,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const csFileProps = [];
         for (const [first, end] of this._changesetRanges) {
             // TODO: should the first changeset in a reverse sync really be included even though its 'initialized branch provenance'? The answer is no, its a bug that needs to be fixed.
-            const fileProps = await core_backend_1.IModelHost[Symbols_1._hubAccess].downloadChangesets({
+            const fileProps = await core_backend_1.IModelHost.hubAccess.downloadChangesets({
                 iModelId: this.sourceDb.iModelId,
                 targetDir: core_backend_1.BriefcaseManager.getChangeSetsPath(this.sourceDb.iModelId),
                 range: { first, end },
@@ -2126,8 +2217,6 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.importer.computeProjectExtents();
         this.finalizeTransformation();
     }
-    /** previous provenance, either a federation guid, a `${sourceFedGuid}/${targetFedGuid}` pair, or required aspect props */
-    _lastProvenanceEntityInfo = nullLastProvenanceEntityInfo;
     markLastProvenance(sourceAspect, { isRelationship = false }) {
         this._lastProvenanceEntityInfo =
             typeof sourceAspect === "string"
@@ -2199,14 +2288,11 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     }
 }
 exports.IModelTransformer = IModelTransformer;
+IModelTransformer.noEsaSyncDirectionErrorMessage = "Couldn't find an external source aspect to determine sync direction. This often means that the master->branch relationship has not been established. Consider running the transformer with wasSourceIModelCopiedToTarget set to true.";
 /** IModelTransformer that clones the contents of a template model.
  * @beta
  */
 class TemplateModelCloner extends IModelTransformer {
-    /** The Placement to apply to the template. */
-    _transform3d;
-    /** Accumulates the mapping of sourceElementIds to targetElementIds from the elements in the template model that were cloned. */
-    _sourceIdToTargetIdMap;
     /** Construct a new TemplateModelCloner
      * @param sourceDb The source IModelDb that contains the templates to clone
      * @param targetDb Optionally specify the target IModelDb where the cloned template will be inserted.