@itwin/imodel-transformer 0.4.4-dev.0 → 0.4.18-dev.0

package/lib/cjs/IModelTransformer.js

@@ -2,9 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TemplateModelCloner = exports.IModelTransformer = void 0;
 /*---------------------------------------------------------------------------------------------
-* Copyright (c) Bentley Systems, Incorporated. All rights reserved.
-* See LICENSE.md in the project root for license terms and full copyright notice.
-*--------------------------------------------------------------------------------------------*/
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
 /** @packageDocumentation
  * @module iModels
  */
@@ -22,6 +22,7 @@ const PendingReferenceMap_1 = require("./PendingReferenceMap");
 const EntityMap_1 = require("./EntityMap");
 const IModelCloneContext_1 = require("./IModelCloneContext");
 const EntityUnifier_1 = require("./EntityUnifier");
+const Algo_1 = require("./Algo");
 const loggerCategory = TransformerLoggerCategory_1.TransformerLoggerCategory.IModelTransformer;
 const nullLastProvenanceEntityInfo = {
     entityId: core_bentley_1.Id64.invalid,
@@ -94,9 +95,115 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     get targetScopeElementId() {
         return this._options.targetScopeElementId;
     }
+    /**
+     * Queries for an esa which matches the props in the provided aspectProps.
+     * @param dbToQuery db to run the query on for scope external source
+     * @param aspectProps aspectProps to search for @see ExternalSourceAspectProps
+     */
+    static queryScopeExternalSourceAspect(dbToQuery, aspectProps) {
+        const sql = `
+      SELECT ECInstanceId, Version, JsonProperties
+      FROM ${core_backend_1.ExternalSourceAspect.classFullName}
+      WHERE Element.Id=:elementId
+        AND Scope.Id=:scopeId
+        AND Kind=:kind
+        AND Identifier=:identifier
+      LIMIT 1
+    `;
+        return dbToQuery.withPreparedStatement(sql, (statement) => {
+            statement.bindId("elementId", aspectProps.element.id);
+            if (aspectProps.scope === undefined)
+                return undefined; // return instead of binding an invalid id
+            statement.bindId("scopeId", aspectProps.scope.id);
+            statement.bindString("kind", aspectProps.kind);
+            statement.bindString("identifier", aspectProps.identifier);
+            if (core_bentley_1.DbResult.BE_SQLITE_ROW !== statement.step())
+                return undefined;
+            const aspectId = statement.getValue(0).getId();
+            const versionValue = statement.getValue(1);
+            const version = versionValue.isNull
+                ? undefined
+                : versionValue.getString();
+            const jsonPropsValue = statement.getValue(2);
+            const jsonProperties = jsonPropsValue.isNull
+                ? undefined
+                : jsonPropsValue.getString();
+            return { aspectId, version, jsonProperties };
+        });
+    }
+    /**
+     * Determines the sync direction "forward" or "reverse" of a given sourceDb and targetDb by looking for the scoping ESA.
+     * If the sourceDb's iModelId is found as the identifier of the expected scoping ESA in the targetDb, then it is a forward synchronization.
+     * If the targetDb's iModelId is found as the identifier of the expected scoping ESA in the sourceDb, then it is a reverse synchronization.
+     * @throws if no scoping ESA can be found in either the sourceDb or targetDb which describes a master branch relationship between the two databases.
+     * @returns "forward" or "reverse"
+     */
+    static determineSyncType(sourceDb, targetDb, 
+    /** @see [[IModelTransformOptions.targetScopeElementId]] */
+    targetScopeElementId) {
+        const aspectProps = {
+            id: undefined,
+            version: undefined,
+            classFullName: core_backend_1.ExternalSourceAspect.classFullName,
+            element: {
+                id: targetScopeElementId,
+                relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName,
+            },
+            scope: { id: core_common_1.IModel.rootSubjectId },
+            identifier: sourceDb.iModelId,
+            kind: core_backend_1.ExternalSourceAspect.Kind.Scope,
+            jsonProperties: undefined,
+        };
+        /** First check if the targetDb is the branch (branch is the @see provenanceDb) */
+        const esaPropsFromTargetDb = this.queryScopeExternalSourceAspect(targetDb, aspectProps);
+        if (esaPropsFromTargetDb !== undefined) {
+            return "forward"; // we found an esa assuming targetDb is the provenanceDb/branch so this is a forward sync.
+        }
+        // Now check if the sourceDb is the branch
+        aspectProps.identifier = targetDb.iModelId;
+        const esaPropsFromSourceDb = this.queryScopeExternalSourceAspect(sourceDb, aspectProps);
+        if (esaPropsFromSourceDb !== undefined) {
+            return "reverse"; // we found an esa assuming sourceDb is the provenanceDb/branch so this is a reverse sync.
+        }
+        throw new Error(this.noEsaSyncDirectionErrorMessage);
+    }
+    determineSyncType() {
+        if (this._isProvenanceInitTransform) {
+            return "forward";
+        }
+        if (!this._isSynchronization) {
+            return "not-sync";
+        }
+        try {
+            return IModelTransformer.determineSyncType(this.sourceDb, this.targetDb, this.targetScopeElementId);
+        }
+        catch (err) {
+            if (err instanceof Error &&
+                err.message === IModelTransformer.noEsaSyncDirectionErrorMessage &&
+                this._allowNoScopingESA) {
+                return "forward";
+            }
+            throw err;
+        }
+    }
+    get isReverseSynchronization() {
+        if (this._syncType === undefined)
+            this._syncType = this.determineSyncType();
+        return this._syncType === "reverse";
+    }
+    get isForwardSynchronization() {
+        if (this._syncType === undefined)
+            this._syncType = this.determineSyncType();
+        return this._syncType === "forward";
+    }
     /** The element classes that are considered to define provenance in the iModel */
     static get provenanceElementClasses() {
-        return [core_backend_1.FolderLink, core_backend_1.SynchronizationConfigLink, core_backend_1.ExternalSource, core_backend_1.ExternalSourceAttachment];
+        return [
+            core_backend_1.FolderLink,
+            core_backend_1.SynchronizationConfigLink,
+            core_backend_1.ExternalSource,
+            core_backend_1.ExternalSourceAttachment,
+        ];
     }
     /** The element aspect classes that are considered to define provenance in the iModel */
     static get provenanceElementAspectClasses() {
@@ -112,18 +219,52 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         /** map of (unprocessed element, referencing processed element) pairs to the partially committed element that needs the reference resolved
          * and have some helper methods below for now */
         this._pendingReferences = new PendingReferenceMap_1.PendingReferenceMap();
+        /** a set of elements for which source provenance will be explicitly tracked by ExternalSourceAspects */
+        this._elementsWithExplicitlyTrackedProvenance = new Set();
         /** map of partially committed entities to their partial commit progress */
         this._partiallyCommittedEntities = new EntityMap_1.EntityMap();
+        this._isSynchronization = false;
+        /**
+         * A private variable meant to be set by tests which have an outdated way of setting up transforms. In all synchronizations today we expect to find an ESA in the branch db which describes the master -> branch relationship.
+         * The exception to this is the first transform aka the provenance initializing transform which requires that the master imodel and the branch imodel are identical at the time of provenance initialization.
+         * A couple ofoutdated tests run their first transform providing a source and targetdb that are slightly different which is no longer supported. In order to not remove these tests which are still providing value
+         * this private property on the IModelTransformer exists.
+         */
+        this._allowNoScopingESA = false;
+        this._changesetRanges = undefined;
         /** Set of entity keys which were not exported and don't need to be tracked for pending reference resolution.
          * @note Currently only tracks elements which were not exported.
          */
         this._skippedEntities = new Set();
+        /**
+         * Previously the transformer would insert provenance always pointing to the "target" relationship.
+         * It should (and now by default does) instead insert provenance pointing to the provenanceSource
+         * SEE: https://github.com/iTwin/imodel-transformer/issues/54
+         * This exists only to facilitate testing that the transformer can handle the older, flawed method
+         */
+        this._forceOldRelationshipProvenanceMethod = false;
+        /** NOTE: the json properties must be converted to string before insertion */
+        this._targetScopeProvenanceProps = undefined;
+        /**
+         * Index of the changeset that the transformer was at when the transformation begins (was constructed).
+         * Used to determine at the end which changesets were part of a synchronization.
+         */
+        this._startingChangesetIndices = undefined;
+        this._cachedSynchronizationVersion = undefined;
+        this._targetClassNameToClassIdCache = new Map();
+        // if undefined, it can be initialized by calling [[this.processChangesets]]
+        this._hasElementChangedCache = undefined;
+        this._deletedSourceRelationshipData = undefined;
         this._yieldManager = new core_bentley_1.YieldManager();
         /** The directory where schemas will be exported, a random temporary directory */
         this._schemaExportDir = path.join(core_backend_1.KnownLocations.tmpdir, core_bentley_1.Guid.createValue());
         this._longNamedSchemasMap = new Map();
         /** state to prevent reinitialization, @see [[initialize]] */
         this._initialized = false;
+        this._sourceChangeDataState = "uninited";
+        /** length === 0 when _changeDataState = "no-change", length > 0 means "has-changes", otherwise undefined  */
+        this._csFileProps = undefined;
+        /** previous provenance, either a federation guid, a `${sourceFedGuid}/${targetFedGuid}` pair, or required aspect props */
         this._lastProvenanceEntityInfo = nullLastProvenanceEntityInfo;
         // initialize IModelTransformOptions
         this._options = {
@@ -132,9 +273,15 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             cloneUsingBinaryGeometry: options?.cloneUsingBinaryGeometry ?? true,
             targetScopeElementId: options?.targetScopeElementId ?? core_common_1.IModel.rootSubjectId,
             // eslint-disable-next-line deprecation/deprecation
-            danglingReferencesBehavior: options?.danglingReferencesBehavior ?? options?.danglingPredecessorsBehavior ?? "reject",
+            danglingReferencesBehavior: options?.danglingReferencesBehavior ??
+                options?.danglingPredecessorsBehavior ??
+                "reject",
+            branchRelationshipDataBehavior: options?.branchRelationshipDataBehavior ?? "reject",
         };
-        this._isFirstSynchronization = this._options.wasSourceIModelCopiedToTarget ? true : undefined;
+        this._isProvenanceInitTransform = this._options
+            .wasSourceIModelCopiedToTarget
+            ? true
+            : undefined;
         // initialize exporter and sourceDb
         if (source instanceof core_backend_1.IModelDb) {
             this.exporter = new IModelExporter_1.IModelExporter(source);
@@ -145,7 +292,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.sourceDb = this.exporter.sourceDb;
         this.exporter.registerHandler(this);
         this.exporter.wantGeometry = options?.loadSourceGeometry ?? false; // optimization to not load source GeometryStreams by default
-        if (!this._options.includeSourceProvenance) { // clone provenance from the source iModel into the target iModel?
+        if (!this._options.includeSourceProvenance) {
+            // clone provenance from the source iModel into the target iModel?
             IModelTransformer.provenanceElementClasses.forEach((cls) => this.exporter.excludeElementClass(cls.classFullName));
             IModelTransformer.provenanceElementAspectClasses.forEach((cls) => this.exporter.excludeElementAspectClass(cls.classFullName));
         }
@@ -153,32 +301,49 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.exporter.excludeElementAspectClass("BisCore:TextAnnotationData"); // This ElementAspect is auto-created by the BisCore:TextAnnotation2d/3d element handlers
         // initialize importer and targetDb
         if (target instanceof core_backend_1.IModelDb) {
-            this.importer = new IModelImporter_1.IModelImporter(target, { preserveElementIdsForFiltering: this._options.preserveElementIdsForFiltering });
+            this.importer = new IModelImporter_1.IModelImporter(target, {
+                preserveElementIdsForFiltering: this._options.preserveElementIdsForFiltering,
+                skipPropagateChangesToRootElements: this._options.skipPropagateChangesToRootElements,
+            });
         }
         else {
             this.importer = target;
-            /* eslint-disable deprecation/deprecation */
-            if (Boolean(this._options.preserveElementIdsForFiltering) !== this.importer.preserveElementIdsForFiltering) {
-                core_bentley_1.Logger.logWarning(loggerCategory, [
-                    "A custom importer was passed as a target but its 'preserveElementIdsForFiltering' option is out of sync with the transformer's option.",
-                    "The custom importer target's option will be force updated to use the transformer's value.",
-                    "This behavior is deprecated and will be removed in a future version, throwing an error if they are out of sync.",
-                ].join("\n"));
-                this.importer.preserveElementIdsForFiltering = Boolean(this._options.preserveElementIdsForFiltering);
-            }
-            /* eslint-enable deprecation/deprecation */
+            this.validateSharedOptionsMatch();
         }
         this.targetDb = this.importer.targetDb;
         // create the IModelCloneContext, it must be initialized later
         this.context = new IModelCloneContext_1.IModelCloneContext(this.sourceDb, this.targetDb);
+        if (this.sourceDb.isBriefcase && this.targetDb.isBriefcase) {
+            nodeAssert(this.sourceDb.changeset.index !== undefined &&
+                this.targetDb.changeset.index !== undefined, "database has no changeset index");
+            this._startingChangesetIndices = {
+                target: this.targetDb.changeset.index,
+                source: this.sourceDb.changeset.index,
+            };
+        }
         // this internal is guaranteed stable for just transformer usage
         /* eslint-disable @itwin/no-internal */
-        if ("codeValueBehavior" in this.sourceDb) {
+        if (("codeValueBehavior" in this.sourceDb)) {
             this.sourceDb.codeValueBehavior = "exact";
             this.targetDb.codeValueBehavior = "exact";
         }
         /* eslint-enable @itwin/no-internal */
     }
+    /** validates that the importer set on the transformer has the same values for its shared options as the transformer.
+     *  @note This expects that the importer is already set on the transformer.
+     */
+    validateSharedOptionsMatch() {
+        if (Boolean(this._options.preserveElementIdsForFiltering) !==
+            this.importer.options.preserveElementIdsForFiltering) {
+            const errMessage = "A custom importer was passed as a target but its 'preserveElementIdsForFiltering' option is out of sync with the transformer's option.";
+            throw new Error(errMessage);
+        }
+        if (Boolean(this._options.skipPropagateChangesToRootElements) !==
+            this.importer.options.skipPropagateChangesToRootElements) {
+            const errMessage = "A custom importer was passed as a target but its 'skipPropagateChangesToRootElements' option is out of sync with the transformer's option.";
+            throw new Error(errMessage);
+        }
+    }
     /** Dispose any native resources associated with this IModelTransformer. */
     dispose() {
         core_bentley_1.Logger.logTrace(loggerCategory, "dispose()");
@@ -204,24 +369,31 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @note This will be [[targetDb]] except when it is a reverse synchronization. In that case it be [[sourceDb]].
      */
     get provenanceDb() {
-        return this._options.isReverseSynchronization ? this.sourceDb : this.targetDb;
+        return this.isReverseSynchronization ? this.sourceDb : this.targetDb;
     }
     /** Return the IModelDb where IModelTransformer looks for entities referred to by stored provenance.
      * @note This will be [[sourceDb]] except when it is a reverse synchronization. In that case it be [[targetDb]].
      */
     get provenanceSourceDb() {
-        return this._options.isReverseSynchronization ? this.targetDb : this.sourceDb;
+        return this.isReverseSynchronization ? this.targetDb : this.sourceDb;
     }
     /** Create an ExternalSourceAspectProps in a standard way for an Element in an iModel --> iModel transformation. */
     static initElementProvenanceOptions(sourceElementId, targetElementId, args) {
-        const elementId = args.isReverseSynchronization ? sourceElementId : targetElementId;
+        const elementId = args.isReverseSynchronization
+            ? sourceElementId
+            : targetElementId;
         const version = args.isReverseSynchronization
             ? args.targetDb.elements.queryLastModifiedTime(targetElementId)
             : args.sourceDb.elements.queryLastModifiedTime(sourceElementId);
-        const aspectIdentifier = args.isReverseSynchronization ? targetElementId : sourceElementId;
+        const aspectIdentifier = args.isReverseSynchronization
+            ? targetElementId
+            : sourceElementId;
         const aspectProps = {
             classFullName: core_backend_1.ExternalSourceAspect.classFullName,
-            element: { id: elementId, relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName },
+            element: {
+                id: elementId,
+                relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName,
+            },
             scope: { id: args.targetScopeElementId },
             identifier: aspectIdentifier,
             kind: core_backend_1.ExternalSourceAspect.Kind.Element,
@@ -229,10 +401,41 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         };
         return aspectProps;
     }
+    static initRelationshipProvenanceOptions(sourceRelInstanceId, targetRelInstanceId, args) {
+        const provenanceDb = args.isReverseSynchronization
+            ? args.sourceDb
+            : args.targetDb;
+        const aspectIdentifier = args.isReverseSynchronization
+            ? targetRelInstanceId
+            : sourceRelInstanceId;
+        const provenanceRelInstanceId = args.isReverseSynchronization
+            ? sourceRelInstanceId
+            : targetRelInstanceId;
+        const elementId = provenanceDb.withPreparedStatement("SELECT SourceECInstanceId FROM bis.ElementRefersToElements WHERE ECInstanceId=?", (stmt) => {
+            stmt.bindId(1, provenanceRelInstanceId);
+            nodeAssert(stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW);
+            return stmt.getValue(0).getId();
+        });
+        const jsonProperties = args.forceOldRelationshipProvenanceMethod
+            ? { targetRelInstanceId }
+            : { provenanceRelInstanceId };
+        const aspectProps = {
+            classFullName: core_backend_1.ExternalSourceAspect.classFullName,
+            element: {
+                id: elementId,
+                relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName,
+            },
+            scope: { id: args.targetScopeElementId },
+            identifier: aspectIdentifier,
+            kind: core_backend_1.ExternalSourceAspect.Kind.Relationship,
+            jsonProperties: JSON.stringify(jsonProperties),
+        };
+        return aspectProps;
+    }
     /** Create an ExternalSourceAspectProps in a standard way for an Element in an iModel --> iModel transformation. */
     initElementProvenance(sourceElementId, targetElementId) {
         return IModelTransformer.initElementProvenanceOptions(sourceElementId, targetElementId, {
-            isReverseSynchronization: !!this._options.isReverseSynchronization,
+            isReverseSynchronization: this.isReverseSynchronization,
             targetScopeElementId: this.targetScopeElementId,
             sourceDb: this.sourceDb,
             targetDb: this.targetDb,
@@ -240,36 +443,115 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     }
     /** Create an ExternalSourceAspectProps in a standard way for a Relationship in an iModel --> iModel transformations.
      * The ExternalSourceAspect is meant to be owned by the Element in the target iModel that is the `sourceId` of transformed relationship.
-     * The `identifier` property of the ExternalSourceAspect will be the ECInstanceId of the relationship in the source iModel.
-     * The ECInstanceId of the relationship in the target iModel will be stored in the JsonProperties of the ExternalSourceAspect.
+     * The `identifier` property of the ExternalSourceAspect will be the ECInstanceId of the relationship in the master iModel.
+     * The ECInstanceId of the relationship in the branch iModel will be stored in the JsonProperties of the ExternalSourceAspect.
      */
     initRelationshipProvenance(sourceRelationship, targetRelInstanceId) {
-        const targetRelationship = this.targetDb.relationships.getInstance(core_backend_1.ElementRefersToElements.classFullName, targetRelInstanceId);
-        const elementId = this._options.isReverseSynchronization ? sourceRelationship.sourceId : targetRelationship.sourceId;
-        const aspectIdentifier = this._options.isReverseSynchronization ? targetRelInstanceId : sourceRelationship.id;
-        const aspectProps = {
+        return IModelTransformer.initRelationshipProvenanceOptions(sourceRelationship.id, targetRelInstanceId, {
+            sourceDb: this.sourceDb,
+            targetDb: this.targetDb,
+            isReverseSynchronization: this.isReverseSynchronization,
+            targetScopeElementId: this.targetScopeElementId,
+            forceOldRelationshipProvenanceMethod: this._forceOldRelationshipProvenanceMethod,
+        });
+    }
+    /** the changeset in the scoping element's source version found for this transformation
+     * @note: the version depends on whether this is a reverse synchronization or not, as
+     * it is stored separately for both synchronization directions.
+     * @note: must call [[initScopeProvenance]] before using this property.
+     * @note: empty string and -1 for changeset and index if it has never been transformed or was transformed before federation guid update (pre 1.x).
+     */
+    get _synchronizationVersion() {
+        if (!this._cachedSynchronizationVersion) {
+            nodeAssert(this._targetScopeProvenanceProps, "_targetScopeProvenanceProps was not set yet");
+            const version = this.isReverseSynchronization
+                ? this._targetScopeProvenanceProps.jsonProperties?.reverseSyncVersion
+                : this._targetScopeProvenanceProps.version;
+            nodeAssert(version !== undefined, "no version contained in target scope");
+            const [id, index] = version === "" ? ["", -1] : version.split(";");
+            this._cachedSynchronizationVersion = { index: Number(index), id };
+            nodeAssert(!Number.isNaN(this._cachedSynchronizationVersion.index), "bad parse: invalid index in version");
+        }
+        return this._cachedSynchronizationVersion;
+    }
+    /** the changeset in the scoping element's source version found for this transformation
+     * @note: the version depends on whether this is a reverse synchronization or not, as
+     * it is stored separately for both synchronization directions.
+     * @note: empty string and -1 for changeset and index if it has never been transformed, or was transformed before federation guid update (pre 1.x).
+     */
+    get synchronizationVersion() {
+        if (this._cachedSynchronizationVersion === undefined) {
+            const provenanceScopeAspect = this.tryGetProvenanceScopeAspect();
+            if (!provenanceScopeAspect) {
+                return { index: -1, id: "" }; // first synchronization.
+            }
+            const version = this.isReverseSynchronization
+                ? JSON.parse(provenanceScopeAspect.jsonProperties ?? "{}").reverseSyncVersion
+                : provenanceScopeAspect.version;
+            if (!version) {
+                return { index: -1, id: "" }; // previous synchronization was done before fed guid update.
+            }
+            const [id, index] = version.split(";");
+            if (Number.isNaN(Number(index)))
+                throw new Error("Could not parse version data from scope aspect");
+            this._cachedSynchronizationVersion = { index: Number(index), id }; // synchronization version found and cached.
+        }
+        return this._cachedSynchronizationVersion;
+    }
+    /**
+     * @returns provenance scope aspect if it exists in the provenanceDb.
+     * Provenance scope aspect is created and inserted into provenanceDb when [[initScopeProvenance]] is invoked.
+     */
+    tryGetProvenanceScopeAspect() {
+        const scopeProvenanceAspectProps = IModelTransformer.queryScopeExternalSourceAspect(this.provenanceDb, {
+            id: undefined,
             classFullName: core_backend_1.ExternalSourceAspect.classFullName,
-            element: { id: elementId, relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName },
-            scope: { id: this.targetScopeElementId },
-            identifier: aspectIdentifier,
-            kind: core_backend_1.ExternalSourceAspect.Kind.Relationship,
-            jsonProperties: JSON.stringify({ targetRelInstanceId }),
-        };
-        aspectProps.id = this.queryExternalSourceAspectId(aspectProps);
-        return aspectProps;
+            scope: { id: core_common_1.IModel.rootSubjectId },
+            kind: core_backend_1.ExternalSourceAspect.Kind.Scope,
+            element: { id: this.targetScopeElementId ?? core_common_1.IModel.rootSubjectId },
+            identifier: this.provenanceSourceDb.iModelId,
+        });
+        return scopeProvenanceAspectProps !== undefined
+            ? this.provenanceDb.elements.getAspect(scopeProvenanceAspectProps.aspectId)
+            : undefined;
     }
-    validateScopeProvenance() {
+    /**
+     * Make sure there are no conflicting other scope-type external source aspects on the *target scope element*,
+     * If there are none at all, insert one, then this must be a first synchronization.
+     * @returns the last synced version (changesetId) on the target scope's external source aspect,
+     *          if this was a [BriefcaseDb]($backend)
+     */
+    initScopeProvenance() {
         const aspectProps = {
+            id: undefined,
+            version: undefined,
             classFullName: core_backend_1.ExternalSourceAspect.classFullName,
-            element: { id: this.targetScopeElementId, relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName },
+            element: {
+                id: this.targetScopeElementId,
+                relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName,
+            },
             scope: { id: core_common_1.IModel.rootSubjectId },
-            identifier: this._options.isReverseSynchronization ? this.targetDb.iModelId : this.sourceDb.iModelId,
+            identifier: this.provenanceSourceDb.iModelId,
             kind: core_backend_1.ExternalSourceAspect.Kind.Scope,
+            jsonProperties: undefined,
         };
-        aspectProps.id = this.queryExternalSourceAspectId(aspectProps); // this query includes "identifier"
-        if (undefined === aspectProps.id) {
+        const foundEsaProps = IModelTransformer.queryScopeExternalSourceAspect(this.provenanceDb, aspectProps); // this query includes "identifier"
+        if (foundEsaProps === undefined) {
+            aspectProps.version = ""; // empty since never before transformed. Will be updated in [[finalizeTransformation]]
+            aspectProps.jsonProperties = {
+                pendingReverseSyncChangesetIndices: [],
+                pendingSyncChangesetIndices: [],
+                reverseSyncVersion: "", // empty since never before transformed. Will be updated in first reverse sync
+            };
             // this query does not include "identifier" to find possible conflicts
-            const sql = `SELECT ECInstanceId FROM ${core_backend_1.ExternalSourceAspect.classFullName} WHERE Element.Id=:elementId AND Scope.Id=:scopeId AND Kind=:kind LIMIT 1`;
+            const sql = `
+        SELECT ECInstanceId
+        FROM ${core_backend_1.ExternalSourceAspect.classFullName}
+        WHERE Element.Id=:elementId
+          AND Scope.Id=:scopeId
+          AND Kind=:kind
+        LIMIT 1
+      `;
             const hasConflictingScope = this.provenanceDb.withPreparedStatement(sql, (statement) => {
                 statement.bindId("elementId", aspectProps.element.id);
                 statement.bindId("scopeId", aspectProps.scope.id); // this scope.id can never be invalid, we create it above
@@ -280,162 +562,276 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 throw new core_common_1.IModelError(core_bentley_1.IModelStatus.InvalidId, "Provenance scope conflict");
             }
             if (!this._options.noProvenance) {
-                this.provenanceDb.elements.insertAspect(aspectProps);
+                const id = this.provenanceDb.elements.insertAspect({
+                    ...aspectProps,
+                    jsonProperties: JSON.stringify(aspectProps.jsonProperties),
+                });
+                aspectProps.id = id;
             }
         }
+        else {
+            // foundEsaProps is defined.
+            aspectProps.id = foundEsaProps.aspectId;
+            aspectProps.version =
+                foundEsaProps.version ??
+                    (this._options.branchRelationshipDataBehavior === "unsafe-migrate"
+                        ? ""
+                        : undefined);
+            aspectProps.jsonProperties = foundEsaProps.jsonProperties
+                ? JSON.parse(foundEsaProps.jsonProperties)
+                : this._options.branchRelationshipDataBehavior === "unsafe-migrate"
+                    ? {
+                        pendingReverseSyncChangesetIndices: [],
+                        pendingSyncChangesetIndices: [],
+                        reverseSyncVersion: "",
+                    }
+                    : undefined;
+        }
+        this._targetScopeProvenanceProps =
+            aspectProps;
     }
-    queryExternalSourceAspectId(aspectProps) {
-        const sql = `SELECT ECInstanceId FROM ${core_backend_1.ExternalSourceAspect.classFullName} WHERE Element.Id=:elementId AND Scope.Id=:scopeId AND Kind=:kind AND Identifier=:identifier LIMIT 1`;
-        return this.provenanceDb.withPreparedStatement(sql, (statement) => {
-            statement.bindId("elementId", aspectProps.element.id);
-            if (aspectProps.scope === undefined)
-                return undefined; // return undefined instead of binding an invalid id
-            statement.bindId("scopeId", aspectProps.scope.id);
-            statement.bindString("kind", aspectProps.kind);
-            statement.bindString("identifier", aspectProps.identifier);
-            return (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) ? statement.getValue(0).getId() : undefined;
-        });
-    }
-    /** Iterate all matching ExternalSourceAspects in the provenance iModel (target unless reverse sync) and call a function for each one. */
+    /**
+     * Iterate all matching federation guids and ExternalSourceAspects in the provenance iModel (target unless reverse sync)
+     * and call a function for each one.
+     * @note provenance is done by federation guids where possible
+     * @note this may execute on each element more than once! Only use in cases where that is handled
+     */
     static forEachTrackedElement(args) {
+        if (args.provenanceDb === args.provenanceSourceDb)
+            return;
         if (!args.provenanceDb.containsClass(core_backend_1.ExternalSourceAspect.classFullName)) {
             throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadSchema, "The BisCore schema version of the target database is too old");
         }
-        const sql = `SELECT Identifier,Element.Id FROM ${core_backend_1.ExternalSourceAspect.classFullName} WHERE Scope.Id=:scopeId AND Kind=:kind`;
-        args.provenanceDb.withPreparedStatement(sql, (statement) => {
-            statement.bindId("scopeId", args.targetScopeElementId);
-            statement.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Element);
-            while (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) {
-                const aspectIdentifier = statement.getValue(0).getString(); // ExternalSourceAspect.Identifier is of type string
-                const elementId = statement.getValue(1).getId();
-                if (args.isReverseSynchronization) {
-                    args.fn(elementId, aspectIdentifier); // provenance coming from the sourceDb
+        const sourceDb = args.isReverseSynchronization
+            ? args.provenanceDb
+            : args.provenanceSourceDb;
+        const targetDb = args.isReverseSynchronization
+            ? args.provenanceSourceDb
+            : args.provenanceDb;
+        // query for provenanceDb
+        const elementIdByFedGuidQuery = `
+      SELECT e.ECInstanceId, FederationGuid
+      FROM bis.Element e
+      ${args.skipPropagateChangesToRootElements
+            ? "WHERE e.ECInstanceId NOT IN (0x1, 0xe, 0x10) -- special static elements"
+            : ""}
+      ORDER BY FederationGuid
+    `;
+        // iterate through sorted list of fed guids from both dbs to get the intersection
+        // NOTE: if we exposed the native attach database support,
+        // we could get the intersection of fed guids in one query, not sure if it would be faster
+        // OR we could do a raw sqlite query...
+        sourceDb.withStatement(elementIdByFedGuidQuery, (sourceStmt) => targetDb.withStatement(elementIdByFedGuidQuery, (targetStmt) => {
+            if (sourceStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return;
+            let sourceRow = sourceStmt.getRow();
+            if (targetStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return;
+            let targetRow = targetStmt.getRow();
+            // NOTE: these comparisons rely upon the lowercase of the guid,
+            // and the fact that '0' < '9' < a' < 'f' in ascii/utf8
+            while (true) {
+                const currSourceRow = sourceRow, currTargetRow = targetRow;
+                if (currSourceRow.federationGuid !== undefined &&
+                    currTargetRow.federationGuid !== undefined &&
+                    currSourceRow.federationGuid === currTargetRow.federationGuid) {
+                    // data flow direction is always sourceDb -> targetDb and it does not depend on where the explicit element provenance is stored
+                    args.fn(sourceRow.id, targetRow.id);
                 }
-                else {
-                    args.fn(aspectIdentifier, elementId); // provenance coming from the targetDb
+                if (currTargetRow.federationGuid === undefined ||
+                    (currSourceRow.federationGuid !== undefined &&
+                        currSourceRow.federationGuid >= currTargetRow.federationGuid)) {
+                    if (targetStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                        return;
+                    targetRow = targetStmt.getRow();
+                }
+                if (currSourceRow.federationGuid === undefined ||
+                    (currTargetRow.federationGuid !== undefined &&
+                        currSourceRow.federationGuid <= currTargetRow.federationGuid)) {
+                    if (sourceStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                        return;
+                    sourceRow = sourceStmt.getRow();
                 }
             }
+        }));
+        // query for provenanceDb
+        const provenanceAspectsQuery = `
+      SELECT esa.Identifier, Element.Id
+      FROM bis.ExternalSourceAspect esa
+      WHERE Scope.Id=:scopeId
+        AND Kind=:kind
+    `;
+        // Technically this will a second time call the function (as documented) on
+        // victims of the old provenance method that have both fedguids and an inserted aspect.
+        // But this is a private function with one known caller where that doesn't matter
+        args.provenanceDb.withPreparedStatement(provenanceAspectsQuery, (stmt) => {
+            const runFnInDataFlowDirection = (sourceId, targetId) => args.isReverseSynchronization
+                ? args.fn(sourceId, targetId)
+                : args.fn(targetId, sourceId);
+            stmt.bindId("scopeId", args.targetScopeElementId);
+            stmt.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Element);
+            while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
+                // ExternalSourceAspect.Identifier is of type string
+                const aspectIdentifier = stmt.getValue(0).getString();
+                const elementId = stmt.getValue(1).getId();
+                runFnInDataFlowDirection(elementId, aspectIdentifier);
+            }
         });
     }
     forEachTrackedElement(fn) {
         return IModelTransformer.forEachTrackedElement({
-            provenanceSourceDb: this._options.isReverseSynchronization ? this.sourceDb : this.targetDb,
+            provenanceSourceDb: this.provenanceSourceDb,
             provenanceDb: this.provenanceDb,
             targetScopeElementId: this.targetScopeElementId,
-            isReverseSynchronization: !!this._options.isReverseSynchronization,
+            isReverseSynchronization: this.isReverseSynchronization,
             fn,
+            skipPropagateChangesToRootElements: this._options.skipPropagateChangesToRootElements ?? false,
         });
     }
-    /** Initialize the source to target Element mapping from ExternalSourceAspects in the target iModel.
-     * @note This method is called from all `process*` functions and should never need to be called directly.
-     * @deprecated in 3.x. call [[initialize]] instead, it does the same thing among other initialization
-     * @note Passing an [[InitFromExternalSourceAspectsArgs]] is required when processing changes, to remap any elements that may have been deleted.
-     *       You must await the returned promise as well in this case. The synchronous behavior has not changed but is deprecated and won't process everything.
-     */
-    initFromExternalSourceAspects(args) {
-        this.forEachTrackedElement((sourceElementId, targetElementId) => {
-            this.context.remapElement(sourceElementId, targetElementId);
+    _queryProvenanceForElement(entityInProvenanceSourceId) {
+        return this.provenanceDb.withPreparedStatement(`
+        SELECT esa.Element.Id
+        FROM Bis.ExternalSourceAspect esa
+        WHERE esa.Kind=?
+          AND esa.Scope.Id=?
+          AND esa.Identifier=?
+      `, (stmt) => {
+            stmt.bindString(1, core_backend_1.ExternalSourceAspect.Kind.Element);
+            stmt.bindId(2, this.targetScopeElementId);
+            stmt.bindString(3, entityInProvenanceSourceId);
+            if (stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return stmt.getValue(0).getId();
+            else
+                return undefined;
         });
-        if (args)
-            return this.remapDeletedSourceElements(args);
     }
-    /** When processing deleted elements in a reverse synchronization, the [[provenanceDb]] (usually a branch iModel) has already
-     * deleted the [ExternalSourceAspect]($backend)s that tell us which elements in the reverse synchronization target (usually
-     * a master iModel) should be deleted. We must use the changesets to get the values of those before they were deleted.
-     */
-    async remapDeletedSourceElements(args) {
-        // we need a connected iModel with changes to remap elements with deletions
-        if (this.sourceDb.iTwinId === undefined)
-            return;
-        try {
-            const startChangesetIndexOrId = args.startChangeset?.index
-                ?? args.startChangeset?.id
-                ?? this.sourceDb.changeset.index
-                ?? this.sourceDb.changeset.id;
-            const endChangesetId = this.sourceDb.changeset.id;
-            const [firstChangesetIndex, endChangesetIndex] = await Promise.all(([startChangesetIndexOrId, endChangesetId])
-                .map(async (indexOrId) => typeof indexOrId === "number"
-                ? indexOrId
-                : core_backend_1.IModelHost.hubAccess
-                    .queryChangeset({
-                    iModelId: this.sourceDb.iModelId,
-                    // eslint-disable-next-line deprecation/deprecation
-                    changeset: { id: indexOrId },
-                    accessToken: args.accessToken,
-                })
-                    .then((changeset) => changeset.index)));
-            const changesetIds = await core_backend_1.ChangeSummaryManager.createChangeSummaries({
-                accessToken: args.accessToken,
-                iModelId: this.sourceDb.iModelId,
-                iTwinId: this.sourceDb.iTwinId,
-                range: { first: firstChangesetIndex, end: endChangesetIndex },
-            });
-            core_backend_1.ChangeSummaryManager.attachChangeCache(this.sourceDb);
-            for (const changesetId of changesetIds) {
-                this.sourceDb.withPreparedStatement(`
-          SELECT esac.Element.Id, esac.Identifier
-          FROM ecchange.change.InstanceChange ic
-          JOIN BisCore.ExternalSourceAspect.Changes(:changesetId, 'BeforeDelete') esac
-            ON ic.ChangedInstance.Id=esac.ECInstanceId
-          WHERE ic.OpCode=:opcode
-            AND ic.Summary.Id=:changesetId
-            AND esac.Scope.Id=:targetScopeElementId
-            -- not yet documented ecsql feature to check class id
-            AND ic.ChangedInstance.ClassId IS (ONLY BisCore.ExternalSourceAspect)
-          `, (stmt) => {
-                    stmt.bindInteger("opcode", core_common_1.ChangeOpCode.Delete);
-                    stmt.bindInteger("changesetId", changesetId);
-                    stmt.bindInteger("targetScopeElementId", this.targetScopeElementId);
-                    while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
-                        const targetId = stmt.getValue(0).getId();
-                        const sourceId = stmt.getValue(1).getString(); // BisCore.ExternalSourceAspect.Identifier stores a hex Id64String
-                        // TODO: maybe delete and don't just remap
-                        this.context.remapElement(targetId, sourceId);
-                    }
-                });
-            }
-        }
-        finally {
-            if (core_backend_1.ChangeSummaryManager.isChangeCacheAttached(this.sourceDb))
-                core_backend_1.ChangeSummaryManager.detachChangeCache(this.sourceDb);
+    _queryProvenanceForRelationship(entityInProvenanceSourceId, sourceRelInfo) {
+        return this.provenanceDb.withPreparedStatement(`
+        SELECT
+          ECInstanceId,
+          JSON_EXTRACT(JsonProperties, '$.targetRelInstanceId'),
+          JSON_EXTRACT(JsonProperties, '$.provenanceRelInstanceId')
+        FROM Bis.ExternalSourceAspect
+        WHERE Kind=?
+          AND Scope.Id=?
+          AND Identifier=?
+      `, (stmt) => {
+            stmt.bindString(1, core_backend_1.ExternalSourceAspect.Kind.Relationship);
+            stmt.bindId(2, this.targetScopeElementId);
+            stmt.bindString(3, entityInProvenanceSourceId);
+            if (stmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return undefined;
+            const aspectId = stmt.getValue(0).getId();
+            const provenanceRelInstIdVal = stmt.getValue(2);
+            const provenanceRelInstanceId = !provenanceRelInstIdVal.isNull
+                ? provenanceRelInstIdVal.getString()
+                : this._queryTargetRelId(sourceRelInfo);
+            return {
+                aspectId,
+                relationshipId: provenanceRelInstanceId,
+            };
+        });
+    }
+    _queryTargetRelId(sourceRelInfo) {
+        const targetRelInfo = {
+            sourceId: this.context.findTargetElementId(sourceRelInfo.sourceId),
+            targetId: this.context.findTargetElementId(sourceRelInfo.targetId),
+        };
+        if (targetRelInfo.sourceId === undefined ||
+            targetRelInfo.targetId === undefined)
+            return undefined; // couldn't find an element, rel is invalid or deleted
+        return this.targetDb.withPreparedStatement(`
+      SELECT ECInstanceId
+      FROM bis.ElementRefersToElements
+      WHERE SourceECInstanceId=?
+        AND TargetECInstanceId=?
+        AND ECClassId=?
+    `, (stmt) => {
+            stmt.bindId(1, targetRelInfo.sourceId);
+            stmt.bindId(2, targetRelInfo.targetId);
+            stmt.bindId(3, this._targetClassNameToClassId(sourceRelInfo.classFullName));
+            if (stmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return undefined;
+            return stmt.getValue(0).getId();
+        });
+    }
+    _targetClassNameToClassId(classFullName) {
+        let classId = this._targetClassNameToClassIdCache.get(classFullName);
+        if (classId === undefined) {
+            classId = this._getRelClassId(this.targetDb, classFullName);
+            this._targetClassNameToClassIdCache.set(classFullName, classId);
         }
+        return classId;
+    }
+    // NOTE: this doesn't handle remapped element classes,
+    // but is only used for relationships rn
+    _getRelClassId(db, classFullName) {
+        return db.withPreparedStatement(`
+      SELECT c.ECInstanceId
+      FROM ECDbMeta.ECClassDef c
+      JOIN ECDbMeta.ECSchemaDef s ON c.Schema.Id=s.ECInstanceId
+      WHERE s.Name=? AND c.Name=?
+    `, (stmt) => {
+            const [schemaName, className] = classFullName.indexOf(".") !== -1
+                ? classFullName.split(".")
+                : classFullName.split(":");
+            stmt.bindString(1, schemaName);
+            stmt.bindString(2, className);
+            if (stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return stmt.getValue(0).getId();
+            (0, core_bentley_1.assert)(false, "relationship was not found");
+        });
+    }
+    _queryElemIdByFedGuid(db, fedGuid) {
+        return db.withPreparedStatement("SELECT ECInstanceId FROM Bis.Element WHERE FederationGuid=?", (stmt) => {
+            stmt.bindGuid(1, fedGuid);
+            if (stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return stmt.getValue(0).getId();
+            else
+                return undefined;
+        });
     }
     /** Returns `true` if *brute force* delete detections should be run.
+     * @note This is only called if [[IModelTransformOptions.forceExternalSourceAspectProvenance]] option is true
      * @note Not relevant for processChanges when change history is known.
      */
     shouldDetectDeletes() {
-        if (this._isFirstSynchronization)
-            return false; // not necessary the first time since there are no deletes to detect
-        if (this._options.isReverseSynchronization)
-            return false; // not possible for a reverse synchronization since provenance will be deleted when element is deleted
-        return true;
+        nodeAssert(this._syncType !== undefined);
+        return this._syncType === "not-sync";
     }
-    /** Detect Element deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against Elements in the source iModel.
-     * @see processChanges
-     * @note This method is called from [[processAll]] and is not needed by [[processChanges]], so it only needs to be called directly when processing a subset of an iModel.
+    /**
+     * Detect Element deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against Elements
+     * in the source iModel.
+     * @deprecated in 1.x. Do not use this. // FIXME<MIKE>: how to better explain this?
+     * This method is only called during [[processAll]] when the option
+     * [[IModelTransformOptions.forceExternalSourceAspectProvenance]] is enabled. It is not
+     * necessary when using [[processChanges]] since changeset information is sufficient.
+     * @note you do not need to call this directly unless processing a subset of an iModel.
      * @throws [[IModelError]] If the required provenance information is not available to detect deletes.
      */
     async detectElementDeletes() {
-        if (this._options.isReverseSynchronization) {
-            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Cannot detect deletes when isReverseSynchronization=true");
-        }
-        const targetElementsToDelete = [];
-        this.forEachTrackedElement((sourceElementId, targetElementId) => {
-            if (undefined === this.sourceDb.elements.tryGetElementProps(sourceElementId)) {
-                // if the sourceElement is not found, then it must have been deleted, so propagate the delete to the target iModel
-                targetElementsToDelete.push(targetElementId);
-            }
-        });
-        targetElementsToDelete.forEach((targetElementId) => {
-            try {
-                // TODO: make it possible to delete more elements at once to prevent redundant expensive
-                // element reference scanning
-                this.importer.deleteElement(targetElementId);
-            }
-            catch (err) {
-                // ignore not found elements, iterative element tree deletion might have already deleted them
-                if (err.name !== "Not Found")
-                    throw err;
+        const sql = `
+      SELECT Identifier, Element.Id
+      FROM BisCore.ExternalSourceAspect
+      WHERE Scope.Id=:scopeId
+        AND Kind=:kind
+    `;
+        nodeAssert(!this.isReverseSynchronization, "synchronizations with processChanges already detect element deletes, don't call detectElementDeletes");
+        this.provenanceDb.withPreparedStatement(sql, (stmt) => {
+            stmt.bindId("scopeId", this.targetScopeElementId);
+            stmt.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Element);
+            while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
+                // ExternalSourceAspect.Identifier is of type string
+                const aspectIdentifier = stmt.getValue(0).getString();
+                if (!core_bentley_1.Id64.isValidId64(aspectIdentifier)) {
+                    continue;
+                }
+                const targetElemId = stmt.getValue(1).getId();
+                const wasDeletedInSource = !EntityUnifier_1.EntityUnifier.exists(this.sourceDb, {
+                    entityReference: `e${aspectIdentifier}`,
+                });
+                if (wasDeletedInSource)
+                    this.importer.deleteElement(targetElemId);
             }
         });
     }
@@ -443,7 +839,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @deprecated in 3.x, this no longer has any effect except emitting a warning
      */
     skipElement(_sourceElement) {
-        core_bentley_1.Logger.logWarning(loggerCategory, `Tried to defer/skip an element, which is no longer necessary`);
+        core_bentley_1.Logger.logWarning(loggerCategory, "Tried to defer/skip an element, which is no longer necessary");
     }
     /** Transform the specified sourceElement into ElementProps for the target iModel.
      * @param sourceElement The Element from the source iModel to transform.
@@ -467,19 +863,14 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @param targetElementId The Element from the target iModel to compare against.
      * @note A subclass can override this method to provide custom change detection behavior.
      */
-    hasElementChanged(sourceElement, targetElementId) {
-        const sourceAspects = this.targetDb.elements.getAspects(targetElementId, core_backend_1.ExternalSourceAspect.classFullName);
-        for (const sourceAspect of sourceAspects) {
-            if (sourceAspect.scope === undefined) // if the scope was lost, we can't correlate so assume it changed
-                return true;
-            if (sourceAspect.identifier === sourceElement.id &&
-                sourceAspect.scope.id === this.targetScopeElementId &&
-                sourceAspect.kind === core_backend_1.ExternalSourceAspect.Kind.Element) {
-                const lastModifiedTime = sourceElement.iModel.elements.queryLastModifiedTime(sourceElement.id);
-                return lastModifiedTime !== sourceAspect.version;
-            }
-        }
-        return true;
+    hasElementChanged(sourceElement, _targetElementId) {
+        if (this._sourceChangeDataState === "no-changes")
+            return false;
+        if (this._sourceChangeDataState === "unconnected")
+            return true;
+        nodeAssert(this._sourceChangeDataState === "has-changes", "change data should be initialized by now");
+        nodeAssert(this._hasElementChangedCache !== undefined, "has element changed cache should be initialized by now");
+        return this._hasElementChangedCache.has(sourceElement.id);
     }
     static transformCallbackFor(transformer, entity) {
         if (entity instanceof core_backend_1.Element)
@@ -505,8 +896,10 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             const updateEntity = EntityUnifier_1.EntityUnifier.updaterFor(this.targetDb, sourceEntity);
             const targetProps = onEntityTransform.call(this, sourceEntity);
             if (sourceEntity instanceof core_backend_1.Relationship) {
-                targetProps.sourceId = this.context.findTargetElementId(sourceEntity.sourceId);
-                targetProps.targetId = this.context.findTargetElementId(sourceEntity.targetId);
+                targetProps.sourceId =
+                    this.context.findTargetElementId(sourceEntity.sourceId);
+                targetProps.targetId =
+                    this.context.findTargetElementId(sourceEntity.targetId);
             }
             updateEntity({ ...targetProps, id: core_backend_1.EntityReferences.toId64(targetId) });
             this._partiallyCommittedEntities.delete(sourceEntity);
@@ -522,11 +915,14 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         for (const referenceId of entity.getReferenceConcreteIds()) {
             // TODO: probably need to rename from 'id' to 'ref' so these names aren't so ambiguous
             const referenceIdInTarget = this.context.findTargetEntityId(referenceId);
-            const alreadyProcessed = core_backend_1.EntityReferences.isValid(referenceIdInTarget) || this._skippedEntities.has(referenceId);
+            const alreadyProcessed = core_backend_1.EntityReferences.isValid(referenceIdInTarget) ||
+                this._skippedEntities.has(referenceId);
             if (alreadyProcessed)
                 continue;
             core_bentley_1.Logger.logTrace(loggerCategory, `Deferring resolution of reference '${referenceId}' of element '${entity.id}'`);
-            const referencedExistsInSource = EntityUnifier_1.EntityUnifier.exists(this.sourceDb, { entityReference: referenceId });
+            const referencedExistsInSource = EntityUnifier_1.EntityUnifier.exists(this.sourceDb, {
+                entityReference: referenceId,
+            });
             if (!referencedExistsInSource) {
                 core_bentley_1.Logger.logWarning(loggerCategory, `Source ${EntityUnifier_1.EntityUnifier.getReadableType(entity)} (${entity.id}) has a dangling reference to (${referenceId})`);
                 switch (this._options.danglingReferencesBehavior) {
@@ -536,7 +932,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                         throw new core_common_1.IModelError(core_bentley_1.IModelStatus.NotFound, [
                             `Found a reference to an element "${referenceId}" that doesn't exist while looking for references of "${entity.id}".`,
                             "This must have been caused by an upstream application that changed the iModel.",
-                            "You can set the IModelTransformerOptions.danglingReferencesBehavior option to 'ignore' to ignore this, but this will leave the iModel",
+                            "You can set the IModelTransformOptions.danglingReferencesBehavior option to 'ignore' to ignore this, but this will leave the iModel",
                             "in a state where downstream consuming applications will need to handle the invalidity themselves. In some cases, writing a custom",
                             "transformer to remove the reference and fix affected elements may be suitable.",
                         ].join("\n"));
@@ -574,7 +970,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     /** Override of [IModelExportHandler.shouldExportElement]($transformer) that is called to determine if an element should be exported from the source iModel.
      * @note Reaching this point means that the element has passed the standard exclusion checks in IModelExporter.
      */
-    shouldExportElement(_sourceElement) { return true; }
+    shouldExportElement(_sourceElement) {
+        return true;
+    }
     onSkipElement(sourceElementId) {
         if (this.context.findTargetElementId(sourceElementId) !== core_bentley_1.Id64.invalid) {
             // element already has provenance
@@ -605,7 +1003,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             const referenceType = elemClass.requiredReferenceKeyTypeMap[referenceKey];
             // For now we just consider all required references to be elements (as they are in biscore), and do not support
             // entities that refuse to be inserted without a different kind of entity (e.g. aspect or relationship) first being inserted
-            (0, core_bentley_1.assert)(referenceType === core_common_1.ConcreteEntityTypes.Element || referenceType === core_common_1.ConcreteEntityTypes.Model);
+            (0, core_bentley_1.assert)(referenceType === core_common_1.ConcreteEntityTypes.Element ||
+                referenceType === core_common_1.ConcreteEntityTypes.Model);
             return mapId64(idContainer, (id) => {
                 if (id === core_bentley_1.Id64.invalid || id === core_common_1.IModel.rootSubjectId)
                     return undefined; // not allowed to directly export the root subject
@@ -614,13 +1013,13 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                     // This is relied upon by the TemplateModelCloner
                     // TODO: extract this out to only be in the TemplateModelCloner
                     const asDefinitionElem = this.sourceDb.elements.tryGetElement(id, core_backend_1.DefinitionElement);
-                    if (asDefinitionElem && !(asDefinitionElem instanceof core_backend_1.RecipeDefinitionElement)) {
+                    if (asDefinitionElem &&
+                        !(asDefinitionElem instanceof core_backend_1.RecipeDefinitionElement)) {
                         this.context.remapElement(id, id);
                     }
                 }
                 return id;
-            })
-                .filter((sourceReferenceId) => {
+            }).filter((sourceReferenceId) => {
                 if (sourceReferenceId === undefined)
                     return false;
                 const referenceInTargetId = this.context.findTargetElementId(sourceReferenceId);
@@ -648,7 +1047,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const isSubModeled = dbHasModel(this.sourceDb, elementId);
         const idOfElemInTarget = this.context.findTargetElementId(elementId);
         const isElemInTarget = core_bentley_1.Id64.invalid !== idOfElemInTarget;
-        const needsModelImport = isSubModeled && (!isElemInTarget || !dbHasModel(this.targetDb, idOfElemInTarget));
+        const needsModelImport = isSubModeled &&
+            (!isElemInTarget || !dbHasModel(this.targetDb, idOfElemInTarget));
         return { needsElemImport: !isElemInTarget, needsModelImport };
     }
     /** Override of [IModelExportHandler.onExportElement]($transformer) that imports an element into the target iModel when it is exported from the source iModel.
@@ -663,57 +1063,80 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         }
         else if (this._options.wasSourceIModelCopiedToTarget) {
             targetElementId = sourceElement.id;
-            targetElementProps = this.targetDb.elements.getElementProps(targetElementId);
+            targetElementProps =
+                this.targetDb.elements.getElementProps(targetElementId);
         }
         else {
             targetElementId = this.context.findTargetElementId(sourceElement.id);
             targetElementProps = this.onTransformElement(sourceElement);
         }
+        // if an existing remapping was not yet found, check by FederationGuid
+        if (this.context.isBetweenIModels &&
+            !core_bentley_1.Id64.isValid(targetElementId) &&
+            sourceElement.federationGuid !== undefined) {
+            targetElementId =
+                this._queryElemIdByFedGuid(this.targetDb, sourceElement.federationGuid) ?? core_bentley_1.Id64.invalid;
+            if (core_bentley_1.Id64.isValid(targetElementId))
+                this.context.remapElement(sourceElement.id, targetElementId); // record that the targetElement was found
+        }
         // if an existing remapping was not yet found, check by Code as long as the CodeScope is valid (invalid means a missing reference so not worth checking)
-        if (!core_bentley_1.Id64.isValidId64(targetElementId) && core_bentley_1.Id64.isValidId64(targetElementProps.code.scope)) {
+        if (!core_bentley_1.Id64.isValidId64(targetElementId) &&
+            core_bentley_1.Id64.isValidId64(targetElementProps.code.scope)) {
             // respond the same way to undefined code value as the @see Code class, but don't use that class because it trims
             // whitespace from the value, and there are iModels out there with untrimmed whitespace that we ought not to trim
             targetElementProps.code.value = targetElementProps.code.value ?? "";
-            targetElementId = this.targetDb.elements.queryElementIdByCode(targetElementProps.code);
-            if (undefined !== targetElementId) {
-                const targetElement = this.targetDb.elements.getElement(targetElementId);
-                if (targetElement.classFullName === targetElementProps.classFullName) { // ensure code remapping doesn't change the target class
+            const maybeTargetElementId = this.targetDb.elements.queryElementIdByCode(targetElementProps.code);
+            if (undefined !== maybeTargetElementId) {
+                const maybeTargetElem = this.targetDb.elements.getElement(maybeTargetElementId);
+                if (maybeTargetElem.classFullName === targetElementProps.classFullName) {
+                    // ensure code remapping doesn't change the target class
+                    targetElementId = maybeTargetElementId;
                     this.context.remapElement(sourceElement.id, targetElementId); // record that the targetElement was found by Code
                 }
                 else {
-                    targetElementId = undefined;
                     targetElementProps.code = core_common_1.Code.createEmpty(); // clear out invalid code
                 }
             }
         }
-        if (undefined !== targetElementId && core_bentley_1.Id64.isValidId64(targetElementId)) {
-            // compare LastMod of sourceElement to ExternalSourceAspect of targetElement to see there are changes to import
-            if (!this.hasElementChanged(sourceElement, targetElementId)) {
-                return;
-            }
-        }
+        if (core_bentley_1.Id64.isValid(targetElementId) &&
+            !this.hasElementChanged(sourceElement, targetElementId))
+            return;
         this.collectUnmappedReferences(sourceElement);
-        // TODO: untangle targetElementId state...
-        if (targetElementId === core_bentley_1.Id64.invalid)
-            targetElementId = undefined;
-        targetElementProps.id = targetElementId; // targetElementId will be valid (indicating update) or undefined (indicating insert)
+        // targetElementId will be valid (indicating update) or undefined (indicating insert)
+        targetElementProps.id = core_bentley_1.Id64.isValid(targetElementId)
+            ? targetElementId
+            : undefined;
         if (!this._options.wasSourceIModelCopiedToTarget) {
             this.importer.importElement(targetElementProps); // don't need to import if iModel was copied
         }
         this.context.remapElement(sourceElement.id, targetElementProps.id); // targetElementProps.id assigned by importElement
         // now that we've mapped this elem we can fix unmapped references to it
         this.resolvePendingReferences(sourceElement);
+        // the transformer does not currently 'split' or 'join' any elements, therefore, it does not
+        // insert external source aspects because federation guids are sufficient for this.
+        // Other transformer subclasses must insert the appropriate aspect (as provided by a TBD API)
+        // when splitting/joining elements
+        // physical consolidation is an example of a 'joining' transform
+        // FIXME: verify at finalization time that we don't lose provenance on new elements
+        // FIXME: make public and improve `initElementProvenance` API for usage by consolidators
         if (!this._options.noProvenance) {
-            const aspectProps = this.initElementProvenance(sourceElement.id, targetElementProps.id);
-            let aspectId = this.queryExternalSourceAspectId(aspectProps);
-            if (aspectId === undefined) {
-                aspectId = this.provenanceDb.elements.insertAspect(aspectProps);
-            }
-            else {
-                this.provenanceDb.elements.updateAspect(aspectProps);
+            let provenance = this._options.forceExternalSourceAspectProvenance ||
+                this._elementsWithExplicitlyTrackedProvenance.has(sourceElement.id)
+                ? undefined
+                : sourceElement.federationGuid;
+            if (!provenance) {
+                const aspectProps = this.initElementProvenance(sourceElement.id, targetElementProps.id);
+                const foundEsaProps = IModelTransformer.queryScopeExternalSourceAspect(this.provenanceDb, aspectProps);
+                if (foundEsaProps === undefined)
+                    aspectProps.id = this.provenanceDb.elements.insertAspect(aspectProps);
+                else {
+                    // Since initElementProvenance sets a property 'version' on the aspectProps that we wish to persist in the provenanceDb, only grab the id from the foundEsaProps.
+                    aspectProps.id = foundEsaProps.aspectId;
+                    this.provenanceDb.elements.updateAspect(aspectProps);
+                }
+                provenance = aspectProps;
             }
-            aspectProps.id = aspectId;
-            this.markLastProvenance(aspectProps, { isRelationship: false });
+            this.markLastProvenance(provenance, { isRelationship: false });
         }
     }
     resolvePendingReferences(entity) {
@@ -739,10 +1162,15 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * This override calls [[onTransformModel]] and then [IModelImporter.importModel]($transformer) to update the target iModel.
      */
     onExportModel(sourceModel) {
-        if (core_common_1.IModel.repositoryModelId === sourceModel.id) {
+        if (this._options.skipPropagateChangesToRootElements &&
+            core_common_1.IModel.repositoryModelId === sourceModel.id)
             return; // The RepositoryModel should not be directly imported
-        }
         const targetModeledElementId = this.context.findTargetElementId(sourceModel.id);
+        // there can only be one repositoryModel per database, so ignore the repo model on remapped subjects
+        const isRemappedRootSubject = sourceModel.id === core_common_1.IModel.repositoryModelId &&
+            targetModeledElementId != sourceModel.id;
+        if (isRemappedRootSubject)
+            return;
         const targetModelProps = this.onTransformModel(sourceModel, targetModeledElementId);
         this.importer.importModel(targetModelProps);
         this.resolvePendingReferences(sourceModel);
@@ -751,7 +1179,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     onDeleteModel(sourceModelId) {
         // It is possible and apparently occasionally sensical to delete a model without deleting its underlying element.
         // - If only the model is deleted, [[initFromExternalSourceAspects]] will have already remapped the underlying element since it still exists.
-        // - If both were deleted, [[remapDeletedSourceElements]] will find and remap the deleted element making this operation valid
+        // - If both were deleted, [[remapDeletedSourceEntities]] will find and remap the deleted element making this operation valid
         const targetModelId = this.context.findTargetElementId(sourceModelId);
         if (!core_bentley_1.Id64.isValidId64(targetModelId))
             return;
@@ -764,9 +1192,12 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 stmt.bindId(1, targetModelId);
                 const val = stmt.step();
                 switch (val) {
-                    case core_bentley_1.DbResult.BE_SQLITE_ROW: return true;
-                    case core_bentley_1.DbResult.BE_SQLITE_DONE: return false;
-                    default: (0, core_bentley_1.assert)(false, `unexpected db result: '${stmt}'`);
+                    case core_bentley_1.DbResult.BE_SQLITE_ROW:
+                        return true;
+                    case core_bentley_1.DbResult.BE_SQLITE_DONE:
+                        return false;
+                    default:
+                        (0, core_bentley_1.assert)(false, `unexpected db result: '${stmt}'`);
                 }
             });
             if (isDefinitionPartition) {
@@ -780,7 +1211,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             this.importer.deleteModel(targetModelId);
         }
         catch (error) {
-            const isDeletionProhibitedErr = error instanceof core_common_1.IModelError && (error.errorNumber === core_bentley_1.IModelStatus.DeletionProhibited || error.errorNumber === core_bentley_1.IModelStatus.ForeignKeyConstraint);
+            const isDeletionProhibitedErr = error instanceof core_common_1.IModelError &&
+                (error.errorNumber === core_bentley_1.IModelStatus.DeletionProhibited ||
+                    error.errorNumber === core_bentley_1.IModelStatus.ForeignKeyConstraint);
             if (!isDeletionProhibitedErr)
                 throw error;
             // Transformer tries to delete models before it deletes elements. Definition models cannot be deleted unless all of their modeled elements are deleted first.
@@ -791,7 +1224,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     }
     /** Schedule modeled partition deletion */
     scheduleModeledPartitionDeletion(sourceModelId) {
-        const deletedElements = this.exporter.sourceDbChanges?.element.deleteIds;
+        const deletedElements = this.exporter.sourceDbChanges?.element
+            .deleteIds;
         if (!deletedElements.has(sourceModelId)) {
             deletedElements.add(sourceModelId);
         }
@@ -857,7 +1291,10 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     onTransformModel(sourceModel, targetModeledElementId) {
         const targetModelProps = sourceModel.toJSON();
         // don't directly edit deep object since toJSON performs a shallow clone
-        targetModelProps.modeledElement = { ...targetModelProps.modeledElement, id: targetModeledElementId };
+        targetModelProps.modeledElement = {
+            ...targetModelProps.modeledElement,
+            id: targetModeledElementId,
+        };
         targetModelProps.id = targetModeledElementId;
         targetModelProps.parentModel = this.context.findTargetElementId(targetModelProps.parentModel);
         return targetModelProps;
@@ -866,25 +1303,131 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @deprecated in 3.x. This method is no longer necessary since the transformer no longer needs to defer elements
      */
     async processDeferredElements(_numRetries = 3) { }
-    finalizeTransformation() {
+    /** called at the end ([[finalizeTransformation]]) of a transformation,
+     * updates the target scope element to say that transformation up through the
+     * source's changeset has been performed. Also stores all changesets that occurred
+     * during the transformation as "pending synchronization changeset indices" @see TargetScopeProvenanceJsonProps
+     *
+     * You generally should not call this function yourself and use [[processChanges]] instead.
+     * It is public for unsupported use cases of custom synchronization transforms.
+     * @note if you are not running processChanges in this transformation, this will fail
+     * without setting the `force` option to `true`
+     */
+    updateSynchronizationVersion({ force = false } = {}) {
+        const notForcedAndHasNoChangesAndIsntProvenanceInit = !force &&
+            this._sourceChangeDataState !== "has-changes" &&
+            !this._isProvenanceInitTransform;
+        if (notForcedAndHasNoChangesAndIsntProvenanceInit)
+            return;
+        nodeAssert(this._targetScopeProvenanceProps);
+        const sourceVersion = `${this.sourceDb.changeset.id};${this.sourceDb.changeset.index}`;
+        const targetVersion = `${this.targetDb.changeset.id};${this.targetDb.changeset.index}`;
+        if (this._isProvenanceInitTransform) {
+            this._targetScopeProvenanceProps.version = sourceVersion;
+            this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion =
+                targetVersion;
+        }
+        else if (this.isReverseSynchronization) {
+            const oldVersion = this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion;
+            core_bentley_1.Logger.logInfo(loggerCategory, `updating reverse version from ${oldVersion} to ${sourceVersion}`);
+            this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion =
+                sourceVersion;
+        }
+        else if (!this.isReverseSynchronization) {
+            core_bentley_1.Logger.logInfo(loggerCategory, `updating sync version from ${this._targetScopeProvenanceProps.version} to ${sourceVersion}`);
+            this._targetScopeProvenanceProps.version = sourceVersion;
+        }
+        if (this._isSynchronization ||
+            (this._startingChangesetIndices && this._isProvenanceInitTransform)) {
+            nodeAssert(this.targetDb.changeset.index !== undefined &&
+                this._startingChangesetIndices !== undefined, "updateSynchronizationVersion was called without change history");
+            const jsonProps = this._targetScopeProvenanceProps.jsonProperties;
+            core_bentley_1.Logger.logTrace(loggerCategory, `previous pendingReverseSyncChanges: ${jsonProps.pendingReverseSyncChangesetIndices}`);
+            core_bentley_1.Logger.logTrace(loggerCategory, `previous pendingSyncChanges: ${jsonProps.pendingSyncChangesetIndices}`);
+            const [syncChangesetsToClear, syncChangesetsToUpdate] = this
+                .isReverseSynchronization
+                ? [
+                    jsonProps.pendingReverseSyncChangesetIndices,
+                    jsonProps.pendingSyncChangesetIndices,
+                ]
+                : [
+                    jsonProps.pendingSyncChangesetIndices,
+                    jsonProps.pendingReverseSyncChangesetIndices,
+                ];
+            for (let i = this._startingChangesetIndices.target + 1; i <= this.targetDb.changeset.index + 1; i++)
+                syncChangesetsToUpdate.push(i);
+            // FIXME: add test to synchronize an iModel that is not at the tip, since then clearning syncChangesets is
+            // probably wrong, and we should filter it instead
+            syncChangesetsToClear.length = 0;
+            // if reverse sync then we may have received provenance changes which should be marked as sync changes
+            if (this.isReverseSynchronization) {
+                nodeAssert(this.sourceDb.changeset.index !== undefined, "changeset didn't exist");
+                for (let i = this._startingChangesetIndices.source + 1; i <= this.sourceDb.changeset.index + 1; i++)
+                    jsonProps.pendingReverseSyncChangesetIndices.push(i);
+            }
+            core_bentley_1.Logger.logTrace(loggerCategory, `new pendingReverseSyncChanges: ${jsonProps.pendingReverseSyncChangesetIndices}`);
+            core_bentley_1.Logger.logTrace(loggerCategory, `new pendingSyncChanges: ${jsonProps.pendingSyncChangesetIndices}`);
+        }
+        this.provenanceDb.elements.updateAspect({
+            ...this._targetScopeProvenanceProps,
+            jsonProperties: JSON.stringify(this._targetScopeProvenanceProps.jsonProperties),
+        });
+    }
+    // FIXME<MIKE>: is this necessary when manually using low level transform APIs? (document if so)
+    async finalizeTransformation(options) {
+        this.importer.finalize();
+        this.updateSynchronizationVersion();
         if (this._partiallyCommittedEntities.size > 0) {
-            core_bentley_1.Logger.logWarning(loggerCategory, [
+            const message = [
                 "The following elements were never fully resolved:",
                 [...this._partiallyCommittedEntities.keys()].join(","),
                 "This indicates that either some references were excluded from the transformation",
                 "or the source has dangling references.",
-            ].join("\n"));
+            ].join("\n");
+            if (this._options.danglingReferencesBehavior === "reject")
+                throw new Error(message);
+            core_bentley_1.Logger.logWarning(loggerCategory, message);
             for (const partiallyCommittedElem of this._partiallyCommittedEntities.values()) {
                 partiallyCommittedElem.forceComplete();
             }
         }
+        // TODO: ignore if we remove change cache usage
+        if (!this._options.noDetachChangeCache) {
+            if (core_backend_1.ChangeSummaryManager.isChangeCacheAttached(this.sourceDb))
+                core_backend_1.ChangeSummaryManager.detachChangeCache(this.sourceDb);
+        }
         // this internal is guaranteed stable for just transformer usage
         /* eslint-disable @itwin/no-internal */
-        if ("codeValueBehavior" in this.sourceDb) {
+        if (("codeValueBehavior" in this.sourceDb)) {
             this.sourceDb.codeValueBehavior = "trim-unicode-whitespace";
             this.targetDb.codeValueBehavior = "trim-unicode-whitespace";
         }
         /* eslint-enable @itwin/no-internal */
+        const defaultSaveTargetChanges = () => this.targetDb.saveChanges();
+        await (options?.saveTargetChanges ?? defaultSaveTargetChanges)(this);
+        if (this.isReverseSynchronization)
+            this.sourceDb.saveChanges();
+        const description = `${this._isProvenanceInitTransform
+            ? options?.provenanceInitTransformChangesetDescription ??
+                `initialized branch provenance with master iModel: ${this.sourceDb.iModelId}`
+            : this.isForwardSynchronization
+                ? options?.forwardSyncBranchChangesetDescription ??
+                    `Forward sync of iModel: ${this.sourceDb.iModelId}`
+                : options?.reverseSyncMasterChangesetDescription ??
+                    `Reverse sync of iModel: ${this.sourceDb.iModelId}`}`;
+        if (this.targetDb.isBriefcaseDb()) {
+            // This relies on authorizationClient on iModelHost being defined, otherwise this will fail
+            await this.targetDb.pushChanges({
+                description,
+            });
+        }
+        if (this.isReverseSynchronization && this.sourceDb.isBriefcaseDb()) {
+            // This relies on authorizationClient on iModelHost being defined, otherwise this will fail
+            await this.sourceDb.pushChanges({
+                description: options?.reverseSyncBranchChangesetDescription ??
+                    `Update provenance in response to a reverse sync to iModel: ${this.targetDb.iModelId}`,
+            });
+        }
     }
     /** Imports all relationships that subclass from the specified base class.
      * @param baseRelClassFullName The specified base relationship class.
@@ -897,64 +1440,98 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     /** Override of [IModelExportHandler.shouldExportRelationship]($transformer) that is called to determine if a [Relationship]($backend) should be exported.
      * @note Reaching this point means that the relationship has passed the standard exclusion checks in [IModelExporter]($transformer).
      */
-    shouldExportRelationship(_sourceRelationship) { return true; }
+    shouldExportRelationship(_sourceRelationship) {
+        return true;
+    }
     /** Override of [IModelExportHandler.onExportRelationship]($transformer) that imports a relationship into the target iModel when it is exported from the source iModel.
      * This override calls [[onTransformRelationship]] and then [IModelImporter.importRelationship]($transformer) to update the target iModel.
      */
     onExportRelationship(sourceRelationship) {
+        const sourceFedGuid = queryElemFedGuid(this.sourceDb, sourceRelationship.sourceId);
+        const targetFedGuid = queryElemFedGuid(this.sourceDb, sourceRelationship.targetId);
         const targetRelationshipProps = this.onTransformRelationship(sourceRelationship);
         const targetRelationshipInstanceId = this.importer.importRelationship(targetRelationshipProps);
-        if (!this._options.noProvenance && core_bentley_1.Id64.isValidId64(targetRelationshipInstanceId)) {
-            const aspectProps = this.initRelationshipProvenance(sourceRelationship, targetRelationshipInstanceId);
-            if (undefined === aspectProps.id) {
-                aspectProps.id = this.provenanceDb.elements.insertAspect(aspectProps);
+        if (!this._options.noProvenance &&
+            core_bentley_1.Id64.isValid(targetRelationshipInstanceId)) {
+            let provenance = !this._options.forceExternalSourceAspectProvenance
+                ? sourceFedGuid && targetFedGuid && `${sourceFedGuid}/${targetFedGuid}`
+                : undefined;
+            if (!provenance) {
+                const aspectProps = this.initRelationshipProvenance(sourceRelationship, targetRelationshipInstanceId);
+                const foundEsaProps = IModelTransformer.queryScopeExternalSourceAspect(this.provenanceDb, aspectProps);
+                // onExportRelationship doesn't need to call updateAspect if esaProps were found, because relationship provenance doesn't have the same concept of a version as element provenance (which uses last mod time on the elements).
+                if (undefined === foundEsaProps) {
+                    aspectProps.id = this.provenanceDb.elements.insertAspect(aspectProps);
+                }
+                provenance = aspectProps;
             }
-            (0, core_bentley_1.assert)(aspectProps.id !== undefined);
-            this.markLastProvenance(aspectProps, { isRelationship: true });
+            this.markLastProvenance(provenance, { isRelationship: true });
         }
     }
     /** Override of [IModelExportHandler.onDeleteRelationship]($transformer) that is called when [IModelExporter]($transformer) detects that a [Relationship]($backend) has been deleted from the source iModel.
      * This override propagates the delete to the target iModel via [IModelImporter.deleteRelationship]($transformer).
      */
     onDeleteRelationship(sourceRelInstanceId) {
-        const sql = `SELECT ECInstanceId,JsonProperties FROM ${core_backend_1.ExternalSourceAspect.classFullName} aspect` +
-            ` WHERE aspect.Scope.Id=:scopeId AND aspect.Kind=:kind AND aspect.Identifier=:identifier LIMIT 1`;
-        this.targetDb.withPreparedStatement(sql, (statement) => {
-            statement.bindId("scopeId", this.targetScopeElementId);
-            statement.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Relationship);
-            statement.bindString("identifier", sourceRelInstanceId);
-            if (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) {
-                const json = JSON.parse(statement.getValue(1).getString());
-                if (undefined !== json.targetRelInstanceId) {
-                    const targetRelationship = this.targetDb.relationships.tryGetInstance(core_backend_1.ElementRefersToElements.classFullName, json.targetRelInstanceId);
-                    if (targetRelationship) {
-                        this.importer.deleteRelationship(targetRelationship.toJSON());
-                    }
-                    this.targetDb.elements.deleteAspect(statement.getValue(0).getId());
-                }
+        nodeAssert(this._deletedSourceRelationshipData, "should be defined at initialization by now");
+        const deletedRelData = this._deletedSourceRelationshipData.get(sourceRelInstanceId);
+        if (!deletedRelData) {
+            // this can occur if both the source and target deleted it
+            core_bentley_1.Logger.logWarning(loggerCategory, "tried to delete a relationship that wasn't in change data");
+            return;
+        }
+        const relArg = deletedRelData.relId ??
+            {
+                sourceId: deletedRelData.sourceIdInTarget,
+                targetId: deletedRelData.targetIdInTarget,
+            };
+        // FIXME: make importer.deleteRelationship not need full props
+        const targetRelationship = this.targetDb.relationships.tryGetInstance(deletedRelData.classFullName, relArg);
+        if (targetRelationship) {
+            this.importer.deleteRelationship(targetRelationship.toJSON());
+        }
+        if (deletedRelData.provenanceAspectId) {
+            try {
+                this.provenanceDb.elements.deleteAspect(deletedRelData.provenanceAspectId);
             }
-        });
+            catch (error) {
+                // This aspect may no longer exist if it was deleted at some other point during the transformation. This is fine.
+                if (error.errorNumber === core_bentley_1.IModelStatus.NotFound)
+                    return;
+                throw error;
+            }
+        }
     }
     /** Detect Relationship deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against relationships in the source iModel.
+     * @deprecated in 1.x. Don't use this anymore
      * @see processChanges
      * @note This method is called from [[processAll]] and is not needed by [[processChanges]], so it only needs to be called directly when processing a subset of an iModel.
      * @throws [[IModelError]] If the required provenance information is not available to detect deletes.
      */
     async detectRelationshipDeletes() {
-        if (this._options.isReverseSynchronization) {
+        if (this.isReverseSynchronization) {
             throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Cannot detect deletes when isReverseSynchronization=true");
         }
         const aspectDeleteIds = [];
-        const sql = `SELECT ECInstanceId,Identifier,JsonProperties FROM ${core_backend_1.ExternalSourceAspect.classFullName} aspect WHERE aspect.Scope.Id=:scopeId AND aspect.Kind=:kind`;
+        const sql = `
+      SELECT ECInstanceId, Identifier, JsonProperties
+      FROM ${core_backend_1.ExternalSourceAspect.classFullName} aspect
+      WHERE aspect.Scope.Id=:scopeId
+        AND aspect.Kind=:kind
+    `;
         await this.targetDb.withPreparedStatement(sql, async (statement) => {
             statement.bindId("scopeId", this.targetScopeElementId);
             statement.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Relationship);
             while (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) {
                 const sourceRelInstanceId = core_bentley_1.Id64.fromJSON(statement.getValue(1).getString());
-                if (undefined === this.sourceDb.relationships.tryGetInstanceProps(core_backend_1.ElementRefersToElements.classFullName, sourceRelInstanceId)) {
+                if (undefined ===
+                    this.sourceDb.relationships.tryGetInstanceProps(core_backend_1.ElementRefersToElements.classFullName, sourceRelInstanceId)) {
+                    // this function exists only to support some in-imodel transformations, which must
+                    // use the old (external source aspect) provenance method anyway so we don't need to support
+                    // new provenance
                     const json = JSON.parse(statement.getValue(2).getString());
-                    if (undefined !== json.targetRelInstanceId) {
-                        const targetRelationship = this.targetDb.relationships.getInstance(core_backend_1.ElementRefersToElements.classFullName, json.targetRelInstanceId);
+                    const targetRelInstanceId = json.targetRelInstanceId ?? json.provenanceRelInstanceId;
+                    if (targetRelInstanceId) {
+                        const targetRelationship = this.targetDb.relationships.getInstance(core_backend_1.ElementRefersToElements.classFullName, targetRelInstanceId);
                         this.importer.deleteRelationship(targetRelationship.toJSON());
                     }
                     aspectDeleteIds.push(statement.getValue(0).getId());
@@ -973,9 +1550,12 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const targetRelationshipProps = sourceRelationship.toJSON();
         targetRelationshipProps.sourceId = this.context.findTargetElementId(sourceRelationship.sourceId);
         targetRelationshipProps.targetId = this.context.findTargetElementId(sourceRelationship.targetId);
+        // TODO: move to cloneRelationship in IModelCloneContext
         sourceRelationship.forEachProperty((propertyName, propertyMetaData) => {
-            if ((core_common_1.PrimitiveTypeCode.Long === propertyMetaData.primitiveType) && ("Id" === propertyMetaData.extendedType)) {
-                targetRelationshipProps[propertyName] = this.context.findTargetElementId(sourceRelationship.asAny[propertyName]);
+            if (core_common_1.PrimitiveTypeCode.Long === propertyMetaData.primitiveType &&
+                "Id" === propertyMetaData.extendedType) {
+                targetRelationshipProps[propertyName] =
+                    this.context.findTargetElementId(sourceRelationship.asAny[propertyName]);
             }
         });
         return targetRelationshipProps;
@@ -1007,8 +1587,10 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         sourceAspects.forEach((a) => this.collectUnmappedReferences(a));
         // const targetAspectsToImport = targetAspectPropsArray.filter((targetAspect, i) => hasEntityChanged(sourceAspects[i], targetAspect));
         const targetIds = this.importer.importElementMultiAspects(targetAspectPropsArray, (a) => {
-            const isExternalSourceAspectFromTransformer = a instanceof core_backend_1.ExternalSourceAspect && a.scope?.id === this.targetScopeElementId;
-            return !this._options.includeSourceProvenance || !isExternalSourceAspectFromTransformer;
+            const isExternalSourceAspectFromTransformer = a instanceof core_backend_1.ExternalSourceAspect &&
+                a.scope?.id === this.targetScopeElementId;
+            return (!this._options.includeSourceProvenance ||
+                !isExternalSourceAspectFromTransformer);
         });
         for (let i = 0; i < targetIds.length; ++i) {
             this.context.remapElementAspect(sourceAspects[i].id, targetIds[i]);
@@ -1047,9 +1629,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         let schemaFileName = schema.name + ext;
         // many file systems have a max file-name/path-segment size of 255, so we workaround that on all systems
         const systemMaxPathSegmentSize = 255;
-        // windows usually has a limit for the total path length of 260
-        const windowsMaxPathLimit = 260;
-        if (schemaFileName.length > systemMaxPathSegmentSize || path.join(this._schemaExportDir, schemaFileName).length >= windowsMaxPathLimit) {
+        if (schemaFileName.length > systemMaxPathSegmentSize) {
             // this name should be well under 255 bytes
             // ( 100 + (Number.MAX_SAFE_INTEGER.toString().length = 16) + (ext.length = 13) ) = 129 which is less than 255
             // You'd have to be past 2**53-1 (Number.MAX_SAFE_INTEGER) long named schemas in order to hit decimal formatting,
@@ -1089,7 +1669,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             const maybeLongNameResolvingSchemaCtx = this._longNamedSchemasMap.size > 0
                 ? this._makeLongNameResolvingSchemaCtx()
                 : undefined;
-            return await this.targetDb.importSchemas(schemaFullPaths, { ecSchemaXmlContext: maybeLongNameResolvingSchemaCtx });
+            return await this.targetDb.importSchemas(schemaFullPaths, {
+                ecSchemaXmlContext: maybeLongNameResolvingSchemaCtx,
+            });
         }
         finally {
             core_backend_1.IModelJsFs.removeSync(this._schemaExportDir);
@@ -1097,8 +1679,8 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         }
     }
     /** Cause all fonts to be exported from the source iModel and imported into the target iModel.
-   * @note This method is called from [[processChanges]] and [[processAll]], so it only needs to be called directly when processing a subset of an iModel.
-   */
+     * @note This method is called from [[processChanges]] and [[processAll]], so it only needs to be called directly when processing a subset of an iModel.
+     */
     async processFonts() {
         // we do not need to initialize for this since no entities are exported
         await this.initialize();
@@ -1125,7 +1707,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     /** Override of [IModelExportHandler.shouldExportCodeSpec]($transformer) that is called to determine if a CodeSpec should be exported from the source iModel.
      * @note Reaching this point means that the CodeSpec has passed the standard exclusion checks in [IModelExporter]($transformer).
      */
-    shouldExportCodeSpec(_sourceCodeSpec) { return true; }
+    shouldExportCodeSpec(_sourceCodeSpec) {
+        return true;
+    }
     /** Override of [IModelExportHandler.onExportCodeSpec]($transformer) that imports a CodeSpec into the target iModel when it is exported from the source iModel. */
     onExportCodeSpec(sourceCodeSpec) {
         this.context.importCodeSpec(sourceCodeSpec.id);
@@ -1141,52 +1725,331 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         return this.processDeferredElements(); // eslint-disable-line deprecation/deprecation
     }
     /**
-     * Initialize prerequisites of processing, you must initialize with an [[InitFromExternalSourceAspectsArgs]] if you
-     * are intending process changes, but prefer using [[processChanges]]
-     * Called by all `process*` functions implicitly.
+     * Initialize prerequisites of processing, you must initialize with an [[InitOptions]] if you
+     * are intending to process changes, but prefer using [[processChanges]] explicitly since it calls this.
+     * @note Called by all `process*` functions implicitly.
      * Overriders must call `super.initialize()` first
      */
     async initialize(args) {
         if (this._initialized)
             return;
+        await this._tryInitChangesetData(args);
         await this.context.initialize();
-        // eslint-disable-next-line deprecation/deprecation
-        await this.initFromExternalSourceAspects(args);
+        // need exporter initialized to do remapdeletedsourceentities.
+        await this.exporter.initialize(this.getExportInitOpts(args ?? {}));
+        // Exporter must be initialized prior to processing changesets in order to properly handle entity recreations (an entity delete followed by an insert of that same entity).
+        await this.processChangesets();
         this._initialized = true;
     }
+    /**
+     * Reads all the changeset files in the private member of the transformer: _csFileProps and does two things with these changesets.
+     * Finds the corresponding target entity for any deleted source entities and remaps the sourceId to the targetId.
+     * Populates this._hasElementChangedCache with a set of elementIds that have been updated or inserted into the database.
+     * This function returns early if csFileProps is undefined or is of length 0.
+     * @returns void
+     */
+    async processChangesets() {
+        this.forEachTrackedElement((sourceElementId, targetElementId) => {
+            this.context.remapElement(sourceElementId, targetElementId);
+        });
+        if (this._csFileProps === undefined || this._csFileProps.length === 0)
+            return;
+        const hasElementChangedCache = new Set();
+        const relationshipECClassIdsToSkip = new Set();
+        for await (const row of this.sourceDb.createQueryReader("SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementDrivesElement)")) {
+            relationshipECClassIdsToSkip.add(row.ECInstanceId);
+        }
+        const relationshipECClassIds = new Set();
+        for await (const row of this.sourceDb.createQueryReader("SELECT ECInstanceId FROM ECDbMeta.ECClassDef where ECInstanceId IS (BisCore.ElementRefersToElements)")) {
+            relationshipECClassIds.add(row.ECInstanceId);
+        }
+        // For later use when processing deletes.
+        const alreadyImportedElementInserts = new Set();
+        const alreadyImportedModelInserts = new Set();
+        this.exporter.sourceDbChanges?.element.insertIds.forEach((insertedSourceElementId) => {
+            const targetElementId = this.context.findTargetElementId(insertedSourceElementId);
+            if (core_bentley_1.Id64.isValid(targetElementId))
+                alreadyImportedElementInserts.add(targetElementId);
+        });
+        this.exporter.sourceDbChanges?.model.insertIds.forEach((insertedSourceModelId) => {
+            const targetModelId = this.context.findTargetElementId(insertedSourceModelId);
+            if (core_bentley_1.Id64.isValid(targetModelId))
+                alreadyImportedModelInserts.add(targetModelId);
+        });
+        this._deletedSourceRelationshipData = new Map();
+        for (const csFile of this._csFileProps) {
+            const csReader = core_backend_1.SqliteChangesetReader.openFile({
+                fileName: csFile.pathname,
+                db: this.sourceDb,
+                disableSchemaCheck: true,
+            });
+            const csAdaptor = new core_backend_1.ChangesetECAdaptor(csReader);
+            const ecChangeUnifier = new core_backend_1.PartialECChangeUnifier();
+            while (csAdaptor.step()) {
+                ecChangeUnifier.appendFrom(csAdaptor);
+            }
+            const changes = [...ecChangeUnifier.instances];
+            /** a map of element ids to this transformation scope's ESA data for that element, in case the ESA is deleted in the target */
+            const elemIdToScopeEsa = new Map();
+            for (const change of changes) {
+                if (change.ECClassId !== undefined &&
+                    relationshipECClassIdsToSkip.has(change.ECClassId))
+                    continue;
+                const changeType = change.$meta?.op;
+                if (changeType === "Deleted" &&
+                    change?.$meta?.classFullName === core_backend_1.ExternalSourceAspect.classFullName &&
+                    change.Scope.Id === this.targetScopeElementId) {
+                    elemIdToScopeEsa.set(change.Element.Id, change);
+                }
+                else if (changeType === "Inserted" || changeType === "Updated")
+                    hasElementChangedCache.add(change.ECInstanceId);
+            }
+            // Loop to process deletes.
+            for (const change of changes) {
+                const changeType = change.$meta?.op;
+                const ecClassId = change.ECClassId ?? change.$meta?.fallbackClassId;
+                if (ecClassId === undefined)
+                    throw new Error(`ECClassId was not found for id: ${change.ECInstanceId}! Table is : ${change?.$meta?.tables}`);
+                if (changeType === undefined)
+                    throw new Error(`ChangeType was undefined for id: ${change.ECInstanceId}.`);
+                if (changeType !== "Deleted" ||
+                    relationshipECClassIdsToSkip.has(ecClassId))
+                    continue;
+                this.processDeletedOp(change, elemIdToScopeEsa, relationshipECClassIds.has(ecClassId ?? ""), alreadyImportedElementInserts, alreadyImportedModelInserts);
+            }
+            csReader.close();
+        }
+        this._hasElementChangedCache = hasElementChangedCache;
+        return;
+    }
+    /**
+     * Helper function for processChangesets. Remaps the id of element deleted found in the 'change' to an element in the targetDb.
+     * @param change the change to process, must be of changeType "Deleted"
+     * @param mapOfDeletedElemIdToScopeEsas a map of elementIds to changedECInstances (which are ESAs). the elementId is not the id of the esa itself, but the elementid that the esa was stored on before the esa's deletion.
+     * All ESAs in this map are part of the transformer's scope / ESA data and are tracked in case the ESA is deleted in the target.
+     * @param isRelationship is relationship or not
+     * @param alreadyImportedElementInserts used to handle entity recreation and not delete already handled element inserts.
+     * @param alreadyImportedModelInserts used to handle entity recreation and not delete already handled model inserts.
+     * @returns void
+     */
+    processDeletedOp(change, mapOfDeletedElemIdToScopeEsas, isRelationship, alreadyImportedElementInserts, alreadyImportedModelInserts) {
+        // we need a connected iModel with changes to remap elements with deletions
+        const notConnectedModel = this.sourceDb.iTwinId === undefined;
+        const noChanges = this._synchronizationVersion.index === this.sourceDb.changeset.index;
+        if (notConnectedModel || noChanges)
+            return;
+        // optimization: if we have provenance, use it to avoid more querying later
+        // eventually when itwin.js supports attaching a second iModelDb in JS,
+        // this won't have to be a conditional part of the query, and we can always have it by attaching
+        const queryCanAccessProvenance = this.sourceDb === this.provenanceDb;
+        const instId = change.ECInstanceId;
+        if (!isRelationship) {
+            const sourceElemFedGuid = change.FederationGuid;
+            let identifierValue;
+            if (queryCanAccessProvenance) {
+                const aspects = this.sourceDb.elements.getAspects(instId, core_backend_1.ExternalSourceAspect.classFullName);
+                for (const aspect of aspects) {
+                    // look for aspect where the ecInstanceId = the aspect.element.id
+                    if (aspect.element.id === instId &&
+                        aspect.scope.id === this.targetScopeElementId)
+                        identifierValue = aspect.identifier;
+                }
+                // Think I need to query the esas given the instId.. not sure what db to do it on though.. soruce or target.. or provenance?
+                // I need to know the id of the element dpeneding on which db its stored in.
+            }
+            if (queryCanAccessProvenance && !identifierValue) {
+                if (mapOfDeletedElemIdToScopeEsas.get(instId) !== undefined)
+                    identifierValue =
+                        mapOfDeletedElemIdToScopeEsas.get(instId).Identifier;
+            }
+            const targetId = (queryCanAccessProvenance && identifierValue) ||
+                // maybe batching these queries would perform better but we should
+                // try to attach the second db and query both together anyway
+                (sourceElemFedGuid &&
+                    this._queryElemIdByFedGuid(this.targetDb, sourceElemFedGuid)) ||
+                // FIXME<MIKE>: describe why it's safe to assume nothing has been deleted in provenanceDb
+                this._queryProvenanceForElement(instId);
+            // since we are processing one changeset at a time, we can see local source deletes
+            // of entities that were never synced and can be safely ignored
+            const deletionNotInTarget = !targetId;
+            if (deletionNotInTarget)
+                return;
+            this.context.remapElement(instId, targetId);
+            // If an entity insert and an entity delete both point to the same entity in target iModel, that means that entity was recreated.
+            // In such case an entity update will be triggered and we no longer need to delete the entity.
+            if (alreadyImportedElementInserts.has(targetId)) {
+                this.exporter.sourceDbChanges?.element.deleteIds.delete(instId);
+            }
+            if (alreadyImportedModelInserts.has(targetId)) {
+                this.exporter.sourceDbChanges?.model.deleteIds.delete(instId);
+            }
+        }
+        else {
+            // is deleted relationship
+            const classFullName = change.$meta?.classFullName;
+            const sourceIdOfRelationshipInSource = change.SourceECInstanceId;
+            const targetIdOfRelationshipInSource = change.TargetECInstanceId;
+            const [sourceIdInTarget, targetIdInTarget] = [
+                sourceIdOfRelationshipInSource,
+                targetIdOfRelationshipInSource,
+            ].map((id) => {
+                let element;
+                try {
+                    element = this.sourceDb.elements.getElement(id);
+                }
+                catch (err) {
+                    return undefined;
+                }
+                const fedGuid = element.federationGuid;
+                let identifierValue;
+                if (queryCanAccessProvenance) {
+                    const aspects = this.sourceDb.elements.getAspects(id, core_backend_1.ExternalSourceAspect.classFullName);
+                    for (const aspect of aspects) {
+                        if (aspect.element.id === id &&
+                            aspect.scope.id === this.targetScopeElementId)
+                            identifierValue = aspect.identifier;
+                    }
+                    if (identifierValue === undefined) {
+                        if (mapOfDeletedElemIdToScopeEsas.get(id) !== undefined)
+                            identifierValue =
+                                mapOfDeletedElemIdToScopeEsas.get(id).Identifier;
+                    }
+                }
+                return ((queryCanAccessProvenance && identifierValue) ||
+                    // maybe batching these queries would perform better but we should
+                    // try to attach the second db and query both together anyway
+                    (fedGuid && this._queryElemIdByFedGuid(this.targetDb, fedGuid)));
+            });
+            if (sourceIdInTarget && targetIdInTarget) {
+                this._deletedSourceRelationshipData.set(instId, {
+                    classFullName: classFullName ?? "",
+                    sourceIdInTarget,
+                    targetIdInTarget,
+                });
+            }
+            else {
+                // FIXME<MIKE>: describe why it's safe to assume nothing has been deleted in provenanceDb
+                const relProvenance = this._queryProvenanceForRelationship(instId, {
+                    classFullName: classFullName ?? "",
+                    sourceId: sourceIdOfRelationshipInSource,
+                    targetId: targetIdOfRelationshipInSource,
+                });
+                if (relProvenance && relProvenance.relationshipId)
+                    this._deletedSourceRelationshipData.set(instId, {
+                        classFullName: classFullName ?? "",
+                        relId: relProvenance.relationshipId,
+                        provenanceAspectId: relProvenance.aspectId,
+                    });
+            }
+        }
+    }
+    async _tryInitChangesetData(args) {
+        if (!args ||
+            this.sourceDb.iTwinId === undefined ||
+            this.sourceDb.changeset.index === undefined) {
+            this._sourceChangeDataState = "unconnected";
+            return;
+        }
+        const noChanges = this._synchronizationVersion.index === this.sourceDb.changeset.index;
+        if (noChanges) {
+            this._sourceChangeDataState = "no-changes";
+            this._csFileProps = [];
+            return;
+        }
+        // NOTE: that we do NOT download the changesummary for the last transformed version, we want
+        // to ignore those already processed changes
+        const startChangesetIndexOrId = args.startChangeset?.index ??
+            args.startChangeset?.id ??
+            this._synchronizationVersion.index + 1;
+        const endChangesetId = this.sourceDb.changeset.id;
+        const [startChangesetIndex, endChangesetIndex] = await Promise.all([startChangesetIndexOrId, endChangesetId].map(async (indexOrId) => typeof indexOrId === "number"
+            ? indexOrId
+            : core_backend_1.IModelHost.hubAccess
+                .queryChangeset({
+                iModelId: this.sourceDb.iModelId,
+                // eslint-disable-next-line deprecation/deprecation
+                changeset: { id: indexOrId },
+                accessToken: args.accessToken,
+            })
+                .then((changeset) => changeset.index)));
+        const missingChangesets = startChangesetIndex > this._synchronizationVersion.index + 1;
+        if (!this._options.ignoreMissingChangesetsInSynchronizations &&
+            startChangesetIndex !== this._synchronizationVersion.index + 1 &&
+            this._synchronizationVersion.index !== -1) {
+            throw Error(`synchronization is ${missingChangesets ? "missing changesets" : ""},` +
+                " startChangesetId should be" +
+                " exactly the first changeset *after* the previous synchronization to not miss data." +
+                ` You specified '${startChangesetIndexOrId}' which is changeset #${startChangesetIndex}` +
+                ` but the previous synchronization for this targetScopeElement was '${this._synchronizationVersion.id}'` +
+                ` which is changeset #${this._synchronizationVersion.index}. The transformer expected` +
+                ` #${this._synchronizationVersion.index + 1}.`);
+        }
+        nodeAssert(this._targetScopeProvenanceProps, "_targetScopeProvenanceProps should be set by now");
+        const changesetsToSkip = this.isReverseSynchronization
+            ? this._targetScopeProvenanceProps.jsonProperties
+                .pendingReverseSyncChangesetIndices
+            : this._targetScopeProvenanceProps.jsonProperties
+                .pendingSyncChangesetIndices;
+        core_bentley_1.Logger.logTrace(loggerCategory, `changesets to skip: ${changesetsToSkip}`);
+        this._changesetRanges = (0, Algo_1.rangesFromRangeAndSkipped)(startChangesetIndex, endChangesetIndex, changesetsToSkip);
+        core_bentley_1.Logger.logTrace(loggerCategory, `ranges: ${this._changesetRanges}`);
+        const csFileProps = [];
+        for (const [first, end] of this._changesetRanges) {
+            // TODO: should the first changeset in a reverse sync really be included even though its 'initialized branch provenance'? The answer is no, its a bug that needs to be fixed.
+            const fileProps = await core_backend_1.IModelHost.hubAccess.downloadChangesets({
+                iModelId: this.sourceDb.iModelId,
+                targetDir: core_backend_1.BriefcaseManager.getChangeSetsPath(this.sourceDb.iModelId),
+                range: { first, end },
+            });
+            csFileProps.push(...fileProps);
+        }
+        this._csFileProps = csFileProps;
+        this._sourceChangeDataState = "has-changes";
+    }
     /** Export everything from the source iModel and import the transformed entities into the target iModel.
-   * @note [[processSchemas]] is not called automatically since the target iModel may want a different collection of schemas.
-   */
-    async processAll() {
-        core_bentley_1.Logger.logTrace(loggerCategory, "processAll()");
+     * @note [[processSchemas]] is not called automatically since the target iModel may want a different collection of schemas.
+     */
+    async processAll(options) {
         this.logSettings();
-        this.validateScopeProvenance();
+        this.initScopeProvenance();
         await this.initialize();
         await this.exporter.exportCodeSpecs();
         await this.exporter.exportFonts();
-        // The RepositoryModel and root Subject of the target iModel should not be transformed.
-        await this.exporter.exportChildElements(core_common_1.IModel.rootSubjectId); // start below the root Subject
-        await this.exporter.exportModelContents(core_common_1.IModel.repositoryModelId, core_backend_1.Element.classFullName, true); // after the Subject hierarchy, process the other elements of the RepositoryModel
-        await this.exporter.exportSubModels(core_common_1.IModel.repositoryModelId); // start below the RepositoryModel
+        if (this._options.skipPropagateChangesToRootElements) {
+            // FIXME<NICK>: This option in exportAll was a maybe.
+            // The RepositoryModel and root Subject of the target iModel should not be transformed.
+            await this.exporter.exportChildElements(core_common_1.IModel.rootSubjectId); // start below the root Subject
+            await this.exporter.exportModelContents(core_common_1.IModel.repositoryModelId, core_backend_1.Element.classFullName, true); // after the Subject hierarchy, process the other elements of the RepositoryModel
+            await this.exporter.exportSubModels(core_common_1.IModel.repositoryModelId); // start below the RepositoryModel
+        }
+        else {
+            await this.exporter.exportModel(core_common_1.IModel.repositoryModelId);
+        }
         await this.exporter["exportAllAspects"](); // eslint-disable-line @typescript-eslint/dot-notation
         await this.exporter.exportRelationships(core_backend_1.ElementRefersToElements.classFullName);
         await this.processDeferredElements(); // eslint-disable-line deprecation/deprecation
-        if (this.shouldDetectDeletes()) {
+        if (this._options.forceExternalSourceAspectProvenance &&
+            this.shouldDetectDeletes()) {
             await this.detectElementDeletes();
             await this.detectRelationshipDeletes();
         }
         if (this._options.optimizeGeometry)
             this.importer.optimizeGeometry(this._options.optimizeGeometry);
         this.importer.computeProjectExtents();
-        this.finalizeTransformation();
+        await this.finalizeTransformation(options);
     }
     markLastProvenance(sourceAspect, { isRelationship = false }) {
-        this._lastProvenanceEntityInfo = {
-            entityId: sourceAspect.element.id,
-            aspectId: sourceAspect.id,
-            aspectVersion: sourceAspect.version ?? "",
-            aspectKind: isRelationship ? core_backend_1.ExternalSourceAspect.Kind.Relationship : core_backend_1.ExternalSourceAspect.Kind.Element,
-        };
+        this._lastProvenanceEntityInfo =
+            typeof sourceAspect === "string"
+                ? sourceAspect
+                : {
+                    entityId: sourceAspect.element.id,
+                    aspectId: sourceAspect.id,
+                    aspectVersion: sourceAspect.version ?? "",
+                    aspectKind: isRelationship
+                        ? core_backend_1.ExternalSourceAspect.Kind.Relationship
+                        : core_backend_1.ExternalSourceAspect.Kind.Element,
+                };
     }
     /**
      * Load the state of the active transformation from an open SQLiteDb
@@ -1198,17 +2061,35 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const lastProvenanceEntityInfo = db.withSqliteStatement(`SELECT entityId, aspectId, aspectVersion, aspectKind FROM ${IModelTransformer.lastProvenanceEntityInfoTable}`, (stmt) => {
             if (core_bentley_1.DbResult.BE_SQLITE_ROW !== stmt.step())
                 throw Error("expected row when getting lastProvenanceEntityId from target state table");
-            return {
-                entityId: stmt.getValueString(0),
-                aspectId: stmt.getValueString(1),
-                aspectVersion: stmt.getValueString(2),
-                aspectKind: stmt.getValueString(3),
-            };
+            const entityId = stmt.getValueString(0);
+            const isGuidOrGuidPair = entityId.includes("-");
+            return isGuidOrGuidPair
+                ? entityId
+                : {
+                    entityId,
+                    aspectId: stmt.getValueString(1),
+                    aspectVersion: stmt.getValueString(2),
+                    aspectKind: stmt.getValueString(3),
+                };
         });
-        const targetHasCorrectLastProvenance = 
-        // ignore provenance check if it's null since we can't bind those ids
-        !core_bentley_1.Id64.isValidId64(lastProvenanceEntityInfo.aspectId) ||
+        /*
+        // TODO: maybe save transformer state resumption state based on target changset and require calls
+        // to saveChanges
+        if () {
+          const [sourceFedGuid, targetFedGuid, relClassFullName] = lastProvenanceEntityInfo.split("/");
+          const isRelProvenance = targetFedGuid !== undefined;
+          const instanceId = isRelProvenance
+            ? this.targetDb.elements.getElement({federationGuid: sourceFedGuid})
+            : "";
+          //const classId =
+          if (isRelProvenance) {
+          }
+        }
+        */
+        const targetHasCorrectLastProvenance = typeof lastProvenanceEntityInfo === "string" ||
+            // ignore provenance check if it's null since we can't bind those ids
             !core_bentley_1.Id64.isValidId64(lastProvenanceEntityInfo.entityId) ||
+            !core_bentley_1.Id64.isValidId64(lastProvenanceEntityInfo.aspectId) ||
             this.provenanceDb.withPreparedStatement(`
         SELECT Version FROM ${core_backend_1.ExternalSourceAspect.classFullName}
         WHERE Scope.Id=:scopeId
@@ -1249,10 +2130,12 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.context.loadStateFromDb(db);
         this.importer.loadStateFromJson(state.importerState);
         this.exporter.loadStateFromJson(state.exporterState);
+        this._elementsWithExplicitlyTrackedProvenance =
+            core_bentley_1.CompressedId64Set.decompressSet(state.explicitlyTrackedElements);
         this.loadAdditionalStateJson(state.additionalState);
     }
     /**
-     * @deprecated in 0.1.x, this is buggy, and with 1.x it will be equivalently efficient to simply restart the transformation
+     * @deprecated in 0.1.x, this is buggy, and it is now equivalently efficient to simply restart the transformation
      * from the original changeset
      *
      * Return a new transformer instance with the same remappings state as saved from a previous [[IModelTransformer.saveStateToFile]] call.
@@ -1299,17 +2182,21 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const jsonState = {
             transformerClass: this.constructor.name,
             options: this._options,
+            explicitlyTrackedElements: core_bentley_1.CompressedId64Set.compressSet(this._elementsWithExplicitlyTrackedProvenance),
             importerState: this.importer.saveStateToJson(),
             exporterState: this.exporter.saveStateToJson(),
             additionalState: this.getAdditionalStateJson(),
         };
         this.context.saveStateToDb(db);
-        if (core_bentley_1.DbResult.BE_SQLITE_DONE !== db.executeSQL(`CREATE TABLE ${IModelTransformer.jsStateTable} (data TEXT)`))
+        if (core_bentley_1.DbResult.BE_SQLITE_DONE !==
+            db.executeSQL(`CREATE TABLE ${IModelTransformer.jsStateTable} (data TEXT)`))
             throw Error("Failed to create the js state table in the state database");
-        if (core_bentley_1.DbResult.BE_SQLITE_DONE !== db.executeSQL(`
+        if (core_bentley_1.DbResult.BE_SQLITE_DONE !==
+            db.executeSQL(`
       CREATE TABLE ${IModelTransformer.lastProvenanceEntityInfoTable} (
-        -- because we cannot bind the invalid id which we use for our null state, we actually store the id as a hex string
+        -- either the invalid id for null provenance state, federation guid (or pair for rels) of the entity, or a hex element id
         entityId TEXT,
+        -- the following are only valid if the above entityId is a hex id representation
         aspectId TEXT,
         aspectVersion TEXT,
         aspectKind TEXT
@@ -1323,17 +2210,20 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 throw Error("Failed to insert options into the state database");
         });
         db.withSqliteStatement(`INSERT INTO ${IModelTransformer.lastProvenanceEntityInfoTable} (entityId, aspectId, aspectVersion, aspectKind) VALUES (?,?,?,?)`, (stmt) => {
-            stmt.bindString(1, this._lastProvenanceEntityInfo.entityId);
-            stmt.bindString(2, this._lastProvenanceEntityInfo.aspectId);
-            stmt.bindString(3, this._lastProvenanceEntityInfo.aspectVersion);
-            stmt.bindString(4, this._lastProvenanceEntityInfo.aspectKind);
+            const lastProvenanceEntityInfo = this
+                ._lastProvenanceEntityInfo;
+            stmt.bindString(1, lastProvenanceEntityInfo?.entityId ??
+                this._lastProvenanceEntityInfo);
+            stmt.bindString(2, lastProvenanceEntityInfo?.aspectId ?? "");
+            stmt.bindString(3, lastProvenanceEntityInfo?.aspectVersion ?? "");
+            stmt.bindString(4, lastProvenanceEntityInfo?.aspectKind ?? "");
             if (core_bentley_1.DbResult.BE_SQLITE_DONE !== stmt.step())
                 throw Error("Failed to insert options into the state database");
         });
         db.saveChanges();
     }
     /**
-     * @deprecated in 0.1.x, this is buggy, and with 1.x it will be equivalently efficient to simply restart the transformation
+     * @deprecated in 0.1.x, this is buggy, and it is now equivalently efficient to simply restart the transformation
      * from the original changeset
      *
      * Save the state of the active transformation to a file path, if a file at the path already exists, it will be overwritten
@@ -1357,28 +2247,62 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             db.closeDb();
         }
     }
-    async processChanges(accessTokenOrArgs, startChangesetId) {
-        core_bentley_1.Logger.logTrace(loggerCategory, "processChanges()");
+    /** Export changes from the source iModel and import the transformed entities into the target iModel.
+     * Inserts, updates, and deletes are determined by inspecting the changeset(s).
+     * @note the transformer saves and pushes changes when its work is complete.
+     * @note if no startChangesetId or startChangeset option is provided as part of the ProcessChangesOptions, the next unsynchronized changeset
+     * will automatically be determined and used
+     * @note To form a range of versions to process, set `startChangesetId` for the start (inclusive) of the desired range and open the source iModel as of the end (inclusive) of the desired range.
+     */
+    async processChanges(options) {
+        this._isSynchronization = true;
+        this.initScopeProvenance();
         this.logSettings();
-        this.validateScopeProvenance();
-        const options = typeof accessTokenOrArgs === "string"
-            ? {
-                accessToken: accessTokenOrArgs,
-                startChangeset: startChangesetId ? { id: startChangesetId } : this.sourceDb.changeset,
-                changedInstanceIds: undefined,
-            }
-            : accessTokenOrArgs;
         await this.initialize(options);
-        await this.exporter.exportChanges(options);
+        // must wait for initialization of synchronization provenance data
+        await this.exporter.exportChanges(this.getExportInitOpts(options));
         await this.processDeferredElements(); // eslint-disable-line deprecation/deprecation
-        await this.exporter["exportAllAspects"](); // eslint-disable-line @typescript-eslint/dot-notation
         if (this._options.optimizeGeometry)
             this.importer.optimizeGeometry(this._options.optimizeGeometry);
         this.importer.computeProjectExtents();
-        this.finalizeTransformation();
+        await this.finalizeTransformation(options);
+    }
+    /** Changeset data must be initialized in order to build correct changeOptions.
+     * Call [[IModelTransformer.initialize]] for initialization of synchronization provenance data
+     */
+    getExportInitOpts(opts) {
+        if (!this._isSynchronization)
+            return {};
+        return {
+            skipPropagateChangesToRootElements: this._options.skipPropagateChangesToRootElements ?? false,
+            accessToken: opts.accessToken,
+            ...(this._csFileProps
+                ? { csFileProps: this._csFileProps }
+                : this._changesetRanges
+                    ? { changesetRanges: this._changesetRanges }
+                    : opts.startChangeset
+                        ? { startChangeset: opts.startChangeset }
+                        : {
+                            startChangeset: {
+                                index: this._synchronizationVersion.index + 1,
+                            },
+                        }),
+        };
+    }
+    /** Combine an array of source elements into a single target element.
+     * All source and target elements must be created before calling this method.
+     * The "combine" operation is a remap and no properties from the source elements will be exported into the target
+     * and provenance will be explicitly tracked by ExternalSourceAspects
+     */
+    combineElements(sourceElementIds, targetElementId) {
+        for (const elementId of sourceElementIds) {
+            this.context.remapElement(elementId, targetElementId);
+            this._elementsWithExplicitlyTrackedProvenance.add(elementId);
+        }
     }
 }
 exports.IModelTransformer = IModelTransformer;
+IModelTransformer.noEsaSyncDirectionErrorMessage = "Couldn't find an external source aspect to determine sync direction. This often means that the master->branch relationship has not been established. Consider running the transformer with wasSourceIModelCopiedToTarget set to true.";
 /** @internal the name of the table where javascript state of the transformer is serialized in transformer state dumps */
 IModelTransformer.jsStateTable = "TransformerJsState";
 /** @internal the name of the table where the target state heuristics is serialized in transformer state dumps */
@@ -1407,6 +2331,7 @@ class TemplateModelCloner extends IModelTransformer {
      * @returns The mapping of sourceElementIds from the template model to the instantiated targetElementIds in the targetDb in case further processing is required.
      */
     async placeTemplate3d(sourceTemplateModelId, targetModelId, placement) {
+        await this.initialize();
         this.context.remapElement(sourceTemplateModelId, targetModelId);
         this._transform3d = core_geometry_1.Transform.createOriginAndMatrix(placement.origin, placement.angles.toMatrix3d());
         this._sourceIdToTargetIdMap = new Map();
@@ -1427,6 +2352,7 @@ class TemplateModelCloner extends IModelTransformer {
      * @returns The mapping of sourceElementIds from the template model to the instantiated targetElementIds in the targetDb in case further processing is required.
      */
     async placeTemplate2d(sourceTemplateModelId, targetModelId, placement) {
+        await this.initialize();
         this.context.remapElement(sourceTemplateModelId, targetModelId);
         this._transform3d = core_geometry_1.Transform.createOriginAndMatrix(core_geometry_1.Point3d.createFrom(placement.origin), placement.rotation);
         this._sourceIdToTargetIdMap = new Map();
@@ -1444,14 +2370,15 @@ class TemplateModelCloner extends IModelTransformer {
         // eslint-disable-next-line deprecation/deprecation
         const referenceIds = sourceElement.getReferenceConcreteIds();
         referenceIds.forEach((referenceId) => {
-            // FIXME: consider going through all definition elements at once and remapping them to themselves
+            // TODO: consider going through all definition elements at once and remapping them to themselves
             if (!core_backend_1.EntityReferences.isValid(this.context.findTargetEntityId(referenceId))) {
                 if (this.context.isBetweenIModels) {
                     throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, `Remapping for source dependency ${referenceId} not found for target iModel`);
                 }
                 else {
                     const definitionElement = this.sourceDb.elements.tryGetElement(referenceId, core_backend_1.DefinitionElement);
-                    if (definitionElement && !(definitionElement instanceof core_backend_1.RecipeDefinitionElement)) {
+                    if (definitionElement &&
+                        !(definitionElement instanceof core_backend_1.RecipeDefinitionElement)) {
                         this.context.remapElement(referenceId, referenceId); // when in the same iModel, can use existing DefinitionElements without remapping
                     }
                     else {
@@ -1463,16 +2390,12 @@ class TemplateModelCloner extends IModelTransformer {
         const targetElementProps = super.onTransformElement(sourceElement);
         targetElementProps.federationGuid = core_bentley_1.Guid.createValue(); // clone from template should create a new federationGuid
         targetElementProps.code = core_common_1.Code.createEmpty(); // clone from template should not maintain codes
-        if (sourceElement instanceof core_backend_1.GeometricElement3d) {
-            const placement = core_common_1.Placement3d.fromJSON(targetElementProps.placement);
-            if (placement.isValid) {
-                placement.multiplyTransform(this._transform3d);
-                targetElementProps.placement = placement;
-            }
-        }
-        else if (sourceElement instanceof core_backend_1.GeometricElement2d) {
-            const placement = core_common_1.Placement2d.fromJSON(targetElementProps.placement);
+        if (sourceElement instanceof core_backend_1.GeometricElement) {
+            const is3d = sourceElement instanceof core_backend_1.GeometricElement3d;
+            const placementClass = is3d ? core_common_1.Placement3d : core_common_1.Placement2d;
+            const placement = placementClass.fromJSON(targetElementProps.placement);
             if (placement.isValid) {
+                nodeAssert(this._transform3d);
                 placement.multiplyTransform(this._transform3d);
                 targetElementProps.placement = placement;
             }
@@ -1482,4 +2405,17 @@ class TemplateModelCloner extends IModelTransformer {
     }
 }
 exports.TemplateModelCloner = TemplateModelCloner;
+function queryElemFedGuid(db, elemId) {
+    return db.withPreparedStatement(`
+    SELECT FederationGuid
+    FROM bis.Element
+    WHERE ECInstanceId=?
+  `, (stmt) => {
+        stmt.bindId(1, elemId);
+        (0, core_bentley_1.assert)(stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW);
+        const result = stmt.getValue(0).getGuid();
+        (0, core_bentley_1.assert)(stmt.step() === core_bentley_1.DbResult.BE_SQLITE_DONE);
+        return result;
+    });
+}
 //# sourceMappingURL=IModelTransformer.js.map