@itwin/imodel-transformer 0.3.2-dev.0 → 0.3.18-fedguidopt.6

package/lib/cjs/IModelTransformer.js

@@ -22,6 +22,7 @@ const PendingReferenceMap_1 = require("./PendingReferenceMap");
 const EntityMap_1 = require("./EntityMap");
 const IModelCloneContext_1 = require("./IModelCloneContext");
 const EntityUnifier_1 = require("./EntityUnifier");
+const Algo_1 = require("./Algo");
 const loggerCategory = TransformerLoggerCategory_1.TransformerLoggerCategory.IModelTransformer;
 const nullLastProvenanceEntityInfo = {
     entityId: core_bentley_1.Id64.invalid,
@@ -94,6 +95,12 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     get targetScopeElementId() {
         return this._options.targetScopeElementId;
     }
+    get _isReverseSynchronization() {
+        return this._isSynchronization && this._options.isReverseSynchronization;
+    }
+    get _isForwardSynchronization() {
+        return this._isSynchronization && !this._options.isReverseSynchronization;
+    }
     /** The element classes that are considered to define provenance in the iModel */
     static get provenanceElementClasses() {
         return [core_backend_1.FolderLink, core_backend_1.SynchronizationConfigLink, core_backend_1.ExternalSource, core_backend_1.ExternalSourceAttachment];
@@ -112,18 +119,46 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         /** map of (unprocessed element, referencing processed element) pairs to the partially committed element that needs the reference resolved
          * and have some helper methods below for now */
         this._pendingReferences = new PendingReferenceMap_1.PendingReferenceMap();
+        /** a set of elements for which source provenance will be explicitly tracked by ExternalSourceAspects */
+        this._elementsWithExplicitlyTrackedProvenance = new Set();
         /** map of partially committed entities to their partial commit progress */
         this._partiallyCommittedEntities = new EntityMap_1.EntityMap();
+        this._isSynchronization = false;
+        this._changesetRanges = undefined;
         /** Set of entity keys which were not exported and don't need to be tracked for pending reference resolution.
          * @note Currently only tracks elements which were not exported.
          */
         this._skippedEntities = new Set();
+        // FIXME: add test using this
+        /**
+         * Previously the transformer would insert provenance always pointing to the "target" relationship.
+         * It should (and now by default does) instead insert provenance pointing to the provenanceSource
+         * SEE: https://github.com/iTwin/imodel-transformer/issues/54
+         * This exists only to facilitate testing that the transformer can handle the older, flawed method
+         */
+        this._forceOldRelationshipProvenanceMethod = false;
+        /** NOTE: the json properties must be converted to string before insertion */
+        this._targetScopeProvenanceProps = undefined;
+        /**
+         * Index of the changeset that the transformer was at when the transformation begins (was constructed).
+         * Used to determine at the end which changesets were part of a synchronization.
+         */
+        this._startingChangesetIndices = undefined;
+        this._cachedSynchronizationVersion = undefined;
+        this._targetClassNameToClassIdCache = new Map();
+        // if undefined, it can be initialized by calling [[this._cacheSourceChanges]]
+        this._hasElementChangedCache = undefined;
+        this._deletedSourceRelationshipData = undefined;
         this._yieldManager = new core_bentley_1.YieldManager();
         /** The directory where schemas will be exported, a random temporary directory */
         this._schemaExportDir = path.join(core_backend_1.KnownLocations.tmpdir, core_bentley_1.Guid.createValue());
         this._longNamedSchemasMap = new Map();
         /** state to prevent reinitialization, @see [[initialize]] */
         this._initialized = false;
+        /** length === 0 when _changeDataState = "no-change", length > 0 means "has-changes", otherwise undefined  */
+        this._changeSummaryIds = undefined;
+        this._sourceChangeDataState = "uninited";
+        /** previous provenance, either a federation guid, a `${sourceFedGuid}/${targetFedGuid}` pair, or required aspect props */
         this._lastProvenanceEntityInfo = nullLastProvenanceEntityInfo;
         // initialize IModelTransformOptions
         this._options = {
@@ -171,6 +206,13 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.targetDb = this.importer.targetDb;
         // create the IModelCloneContext, it must be initialized later
         this.context = new IModelCloneContext_1.IModelCloneContext(this.sourceDb, this.targetDb);
+        if (this.sourceDb.isBriefcase && this.targetDb.isBriefcase) {
+            nodeAssert(this.sourceDb.changeset.index !== undefined && this.targetDb.changeset.index !== undefined, "database has no changeset index");
+            this._startingChangesetIndices = {
+                target: this.targetDb.changeset.index,
+                source: this.sourceDb.changeset.index,
+            };
+        }
     }
     /** Dispose any native resources associated with this IModelTransformer. */
     dispose() {
@@ -219,9 +261,32 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         };
         return aspectProps;
     }
+    static initRelationshipProvenanceOptions(sourceRelInstanceId, targetRelInstanceId, args) {
+        const provenanceDb = args.isReverseSynchronization ? args.sourceDb : args.targetDb;
+        const aspectIdentifier = args.isReverseSynchronization ? targetRelInstanceId : sourceRelInstanceId;
+        const provenanceRelInstanceId = args.isReverseSynchronization ? sourceRelInstanceId : targetRelInstanceId;
+        const elementId = provenanceDb.withPreparedStatement("SELECT SourceECInstanceId FROM bis.ElementRefersToElements WHERE ECInstanceId=?", (stmt) => {
+            stmt.bindId(1, provenanceRelInstanceId);
+            nodeAssert(stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW);
+            return stmt.getValue(0).getId();
+        });
+        const jsonProperties = args.forceOldRelationshipProvenanceMethod
+            ? { targetRelInstanceId }
+            : { provenanceRelInstanceId };
+        const aspectProps = {
+            classFullName: core_backend_1.ExternalSourceAspect.classFullName,
+            element: { id: elementId, relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName },
+            scope: { id: args.targetScopeElementId },
+            identifier: aspectIdentifier,
+            kind: core_backend_1.ExternalSourceAspect.Kind.Relationship,
+            jsonProperties: JSON.stringify(jsonProperties),
+        };
+        return aspectProps;
+    }
     /** Create an ExternalSourceAspectProps in a standard way for an Element in an iModel --> iModel transformation. */
     initElementProvenance(sourceElementId, targetElementId) {
         return IModelTransformer.initElementProvenanceOptions(sourceElementId, targetElementId, {
+            // FIXME: deprecate isReverseSync option and instead detect from targetScopeElement provenance
             isReverseSynchronization: !!this._options.isReverseSynchronization,
             targetScopeElementId: this.targetScopeElementId,
             sourceDb: this.sourceDb,
@@ -233,32 +298,73 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * The ECInstanceId of the relationship in the target iModel will be stored in the JsonProperties of the ExternalSourceAspect.
      */
     initRelationshipProvenance(sourceRelationship, targetRelInstanceId) {
-        const targetRelationship = this.targetDb.relationships.getInstance(core_backend_1.ElementRefersToElements.classFullName, targetRelInstanceId);
-        const elementId = this._options.isReverseSynchronization ? sourceRelationship.sourceId : targetRelationship.sourceId;
-        const aspectIdentifier = this._options.isReverseSynchronization ? targetRelInstanceId : sourceRelationship.id;
-        const aspectProps = {
-            classFullName: core_backend_1.ExternalSourceAspect.classFullName,
-            element: { id: elementId, relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName },
-            scope: { id: this.targetScopeElementId },
-            identifier: aspectIdentifier,
-            kind: core_backend_1.ExternalSourceAspect.Kind.Relationship,
-            jsonProperties: JSON.stringify({ targetRelInstanceId }),
-        };
-        aspectProps.id = this.queryExternalSourceAspectId(aspectProps);
-        return aspectProps;
+        return IModelTransformer.initRelationshipProvenanceOptions(sourceRelationship.id, targetRelInstanceId, {
+            sourceDb: this.sourceDb,
+            targetDb: this.targetDb,
+            isReverseSynchronization: !!this._options.isReverseSynchronization,
+            targetScopeElementId: this.targetScopeElementId,
+            forceOldRelationshipProvenanceMethod: this._forceOldRelationshipProvenanceMethod,
+        });
     }
-    validateScopeProvenance() {
+    /** the changeset in the scoping element's source version found for this transformation
+     * @note: the version depends on whether this is a reverse synchronization or not, as
+     * it is stored separately for both synchronization directions
+     * @note: empty string and -1 for changeset and index if it has never been transformed
+     */
+    get _synchronizationVersion() {
+        if (!this._cachedSynchronizationVersion) {
+            nodeAssert(this._targetScopeProvenanceProps, "_targetScopeProvenanceProps was not set yet");
+            const version = this._options.isReverseSynchronization
+                ? this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion
+                : this._targetScopeProvenanceProps.version;
+            nodeAssert(version !== undefined, "no version contained in target scope");
+            const [id, index] = version === ""
+                ? ["", -1]
+                : version.split(";");
+            this._cachedSynchronizationVersion = { index: Number(index), id };
+            nodeAssert(!Number.isNaN(this._cachedSynchronizationVersion.index), "bad parse: invalid index in version");
+        }
+        return this._cachedSynchronizationVersion;
+    }
+    /**
+     * Make sure there are no conflicting other scope-type external source aspects on the *target scope element*,
+     * If there are none at all, insert one, then this must be a first synchronization.
+     * @returns the last synced version (changesetId) on the target scope's external source aspect,
+     *          if this was a [BriefcaseDb]($backend)
+     */
+    initScopeProvenance() {
         const aspectProps = {
+            id: undefined,
+            version: undefined,
             classFullName: core_backend_1.ExternalSourceAspect.classFullName,
             element: { id: this.targetScopeElementId, relClassName: core_backend_1.ElementOwnsExternalSourceAspects.classFullName },
             scope: { id: core_common_1.IModel.rootSubjectId },
-            identifier: this._options.isReverseSynchronization ? this.targetDb.iModelId : this.sourceDb.iModelId,
+            identifier: this.provenanceSourceDb.iModelId,
             kind: core_backend_1.ExternalSourceAspect.Kind.Scope,
+            jsonProperties: undefined,
         };
-        aspectProps.id = this.queryExternalSourceAspectId(aspectProps); // this query includes "identifier"
+        // FIXME: handle older transformed iModels which do NOT have the version
+        // or reverseSyncVersion set correctly
+        const externalSource = this.queryScopeExternalSource(aspectProps, { getJsonProperties: true }); // this query includes "identifier"
+        aspectProps.id = externalSource.aspectId;
+        aspectProps.version = externalSource.version;
+        aspectProps.jsonProperties = externalSource.jsonProperties ? JSON.parse(externalSource.jsonProperties) : {};
         if (undefined === aspectProps.id) {
+            aspectProps.version = ""; // empty since never before transformed. Will be updated in [[finalizeTransformation]]
+            aspectProps.jsonProperties = {
+                pendingReverseSyncChangesetIndices: [],
+                pendingSyncChangesetIndices: [],
+                reverseSyncVersion: "", // empty since never before transformed. Will be updated in first reverse sync
+            };
             // this query does not include "identifier" to find possible conflicts
-            const sql = `SELECT ECInstanceId FROM ${core_backend_1.ExternalSourceAspect.classFullName} WHERE Element.Id=:elementId AND Scope.Id=:scopeId AND Kind=:kind LIMIT 1`;
+            const sql = `
+        SELECT ECInstanceId
+        FROM ${core_backend_1.ExternalSourceAspect.classFullName}
+        WHERE Element.Id=:elementId
+          AND Scope.Id=:scopeId
+          AND Kind=:kind
+        LIMIT 1
+      `;
             const hasConflictingScope = this.provenanceDb.withPreparedStatement(sql, (statement) => {
                 statement.bindId("elementId", aspectProps.element.id);
                 statement.bindId("scopeId", aspectProps.scope.id); // this scope.id can never be invalid, we create it above
@@ -269,46 +375,128 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 throw new core_common_1.IModelError(core_bentley_1.IModelStatus.InvalidId, "Provenance scope conflict");
             }
             if (!this._options.noProvenance) {
-                this.provenanceDb.elements.insertAspect(aspectProps);
+                this.provenanceDb.elements.insertAspect({
+                    ...aspectProps,
+                    jsonProperties: JSON.stringify(aspectProps.jsonProperties),
+                });
             }
         }
+        this._targetScopeProvenanceProps = aspectProps;
     }
-    queryExternalSourceAspectId(aspectProps) {
-        const sql = `SELECT ECInstanceId FROM ${core_backend_1.ExternalSourceAspect.classFullName} WHERE Element.Id=:elementId AND Scope.Id=:scopeId AND Kind=:kind AND Identifier=:identifier LIMIT 1`;
+    /**
+     * @returns the id and version of an aspect with the given element, scope, kind, and identifier
+     * May also return a reverseSyncVersion from json properties if requested
+     */
+    queryScopeExternalSource(aspectProps, { getJsonProperties = false } = {}) {
+        const sql = `
+      SELECT ECInstanceId, Version
+      ${getJsonProperties ? ", JsonProperties" : ""}
+      FROM ${core_backend_1.ExternalSourceAspect.classFullName}
+      WHERE Element.Id=:elementId
+        AND Scope.Id=:scopeId
+        AND Kind=:kind
+        AND Identifier=:identifier
+      LIMIT 1
+    `;
+        const emptyResult = { aspectId: undefined, version: undefined, jsonProperties: undefined };
         return this.provenanceDb.withPreparedStatement(sql, (statement) => {
             statement.bindId("elementId", aspectProps.element.id);
             if (aspectProps.scope === undefined)
-                return undefined; // return undefined instead of binding an invalid id
+                return emptyResult; // return undefined instead of binding an invalid id
             statement.bindId("scopeId", aspectProps.scope.id);
             statement.bindString("kind", aspectProps.kind);
             statement.bindString("identifier", aspectProps.identifier);
-            return (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) ? statement.getValue(0).getId() : undefined;
+            if (core_bentley_1.DbResult.BE_SQLITE_ROW !== statement.step())
+                return emptyResult;
+            const aspectId = statement.getValue(0).getId();
+            const version = statement.getValue(1).getString();
+            const jsonProperties = getJsonProperties ? statement.getValue(2).getString() : undefined;
+            return { aspectId, version, jsonProperties };
         });
     }
-    /** Iterate all matching ExternalSourceAspects in the provenance iModel (target unless reverse sync) and call a function for each one. */
+    /**
+     * Iterate all matching federation guids and ExternalSourceAspects in the provenance iModel (target unless reverse sync)
+     * and call a function for each one.
+     * @note provenance is done by federation guids where possible
+     * @note this may execute on each element more than once! Only use in cases where that is handled
+     */
     static forEachTrackedElement(args) {
+        if (args.provenanceDb === args.provenanceSourceDb)
+            return;
         if (!args.provenanceDb.containsClass(core_backend_1.ExternalSourceAspect.classFullName)) {
             throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadSchema, "The BisCore schema version of the target database is too old");
         }
-        const sql = `SELECT Identifier,Element.Id FROM ${core_backend_1.ExternalSourceAspect.classFullName} WHERE Scope.Id=:scopeId AND Kind=:kind`;
-        args.provenanceDb.withPreparedStatement(sql, (statement) => {
-            statement.bindId("scopeId", args.targetScopeElementId);
-            statement.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Element);
-            while (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) {
-                const aspectIdentifier = statement.getValue(0).getString(); // ExternalSourceAspect.Identifier is of type string
-                const elementId = statement.getValue(1).getId();
-                if (args.isReverseSynchronization) {
-                    args.fn(elementId, aspectIdentifier); // provenance coming from the sourceDb
+        const sourceDb = args.isReverseSynchronization ? args.provenanceDb : args.provenanceSourceDb;
+        const targetDb = args.isReverseSynchronization ? args.provenanceSourceDb : args.provenanceDb;
+        // query for provenanceDb
+        const elementIdByFedGuidQuery = `
+      SELECT e.ECInstanceId, FederationGuid
+      FROM bis.Element e
+      WHERE e.ECInstanceId NOT IN (0x1, 0xe, 0x10) -- special static elements
+      ORDER BY FederationGuid
+    `;
+        // iterate through sorted list of fed guids from both dbs to get the intersection
+        // NOTE: if we exposed the native attach database support,
+        // we could get the intersection of fed guids in one query, not sure if it would be faster
+        // OR we could do a raw sqlite query...
+        sourceDb.withStatement(elementIdByFedGuidQuery, (sourceStmt) => targetDb.withStatement(elementIdByFedGuidQuery, (targetStmt) => {
+            if (sourceStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return;
+            let sourceRow = sourceStmt.getRow();
+            if (targetStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return;
+            let targetRow = targetStmt.getRow();
+            // NOTE: these comparisons rely upon the lowercase of the guid,
+            // and the fact that '0' < '9' < a' < 'f' in ascii/utf8
+            while (true) {
+                const currSourceRow = sourceRow, currTargetRow = targetRow;
+                if (currSourceRow.federationGuid !== undefined
+                    && currTargetRow.federationGuid !== undefined
+                    && currSourceRow.federationGuid === currTargetRow.federationGuid) {
+                    // data flow direction is always sourceDb -> targetDb and it does not depend on where the explicit element provenance is stored
+                    args.fn(sourceRow.id, targetRow.id);
                 }
-                else {
-                    args.fn(aspectIdentifier, elementId); // provenance coming from the targetDb
+                if (currTargetRow.federationGuid === undefined
+                    || (currSourceRow.federationGuid !== undefined
+                        && currSourceRow.federationGuid >= currTargetRow.federationGuid)) {
+                    if (targetStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                        return;
+                    targetRow = targetStmt.getRow();
+                }
+                if (currSourceRow.federationGuid === undefined
+                    || (currTargetRow.federationGuid !== undefined
+                        && currSourceRow.federationGuid <= currTargetRow.federationGuid)) {
+                    if (sourceStmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                        return;
+                    sourceRow = sourceStmt.getRow();
                 }
             }
+        }));
+        // query for provenanceDb
+        const provenanceAspectsQuery = `
+      SELECT esa.Identifier, Element.Id
+      FROM bis.ExternalSourceAspect esa
+      WHERE Scope.Id=:scopeId
+        AND Kind=:kind
+    `;
+        // Technically this will a second time call the function (as documented) on
+        // victims of the old provenance method that have both fedguids and an inserted aspect.
+        // But this is a private function with one known caller where that doesn't matter
+        args.provenanceDb.withPreparedStatement(provenanceAspectsQuery, (stmt) => {
+            const runFnInDataFlowDirection = (sourceId, targetId) => args.isReverseSynchronization ? args.fn(sourceId, targetId) : args.fn(targetId, sourceId);
+            stmt.bindId("scopeId", args.targetScopeElementId);
+            stmt.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Element);
+            while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
+                // ExternalSourceAspect.Identifier is of type string
+                const aspectIdentifier = stmt.getValue(0).getString();
+                const elementId = stmt.getValue(1).getId();
+                runFnInDataFlowDirection(elementId, aspectIdentifier);
+            }
         });
     }
     forEachTrackedElement(fn) {
         return IModelTransformer.forEachTrackedElement({
-            provenanceSourceDb: this._options.isReverseSynchronization ? this.sourceDb : this.targetDb,
+            provenanceSourceDb: this.provenanceSourceDb,
             provenanceDb: this.provenanceDb,
             targetScopeElementId: this.targetScopeElementId,
             isReverseSynchronization: !!this._options.isReverseSynchronization,
@@ -326,105 +514,349 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             this.context.remapElement(sourceElementId, targetElementId);
         });
         if (args)
-            return this.remapDeletedSourceElements(args);
+            return this.remapDeletedSourceEntities();
     }
-    /** When processing deleted elements in a reverse synchronization, the [[provenanceDb]] (usually a branch iModel) has already
-     * deleted the [ExternalSourceAspect]($backend)s that tell us which elements in the reverse synchronization target (usually
-     * a master iModel) should be deleted. We must use the changesets to get the values of those before they were deleted.
+    /**
+     * Scan changesets for deleted entities, if in a reverse synchronization, provenance has
+     * already been deleted, so we must scan for that as well.
      */
-    async remapDeletedSourceElements(args) {
+    async remapDeletedSourceEntities() {
         // we need a connected iModel with changes to remap elements with deletions
-        if (this.sourceDb.iTwinId === undefined)
+        const notConnectedModel = this.sourceDb.iTwinId === undefined;
+        const noChanges = this._synchronizationVersion.index === this.sourceDb.changeset.index;
+        if (notConnectedModel || noChanges)
             return;
-        try {
-            const startChangesetIndexOrId = args.startChangeset?.index
-                ?? args.startChangeset?.id
-                ?? this.sourceDb.changeset.index
-                ?? this.sourceDb.changeset.id;
-            const endChangesetId = this.sourceDb.changeset.id;
-            const [firstChangesetIndex, endChangesetIndex] = await Promise.all(([startChangesetIndexOrId, endChangesetId])
-                .map(async (indexOrId) => typeof indexOrId === "number"
-                ? indexOrId
-                : core_backend_1.IModelHost.hubAccess
-                    .queryChangeset({
-                    iModelId: this.sourceDb.iModelId,
-                    // eslint-disable-next-line deprecation/deprecation
-                    changeset: { id: indexOrId },
-                    accessToken: args.accessToken,
-                })
-                    .then((changeset) => changeset.index)));
-            const changesetIds = await core_backend_1.ChangeSummaryManager.createChangeSummaries({
-                accessToken: args.accessToken,
-                iModelId: this.sourceDb.iModelId,
-                iTwinId: this.sourceDb.iTwinId,
-                range: { first: firstChangesetIndex, end: endChangesetIndex },
-            });
-            core_backend_1.ChangeSummaryManager.attachChangeCache(this.sourceDb);
-            for (const changesetId of changesetIds) {
-                this.sourceDb.withPreparedStatement(`
-          SELECT esac.Element.Id, esac.Identifier
-          FROM ecchange.change.InstanceChange ic
-          JOIN BisCore.ExternalSourceAspect.Changes(:changesetId, 'BeforeDelete') esac
-            ON ic.ChangedInstance.Id=esac.ECInstanceId
-          WHERE ic.OpCode=:opcode
-            AND ic.Summary.Id=:changesetId
-            AND esac.Scope.Id=:targetScopeElementId
-            -- not yet documented ecsql feature to check class id
-            AND ic.ChangedInstance.ClassId IS (ONLY BisCore.ExternalSourceAspect)
-          `, (stmt) => {
-                    stmt.bindInteger("opcode", core_common_1.ChangeOpCode.Delete);
-                    stmt.bindInteger("changesetId", changesetId);
-                    stmt.bindInteger("targetScopeElementId", this.targetScopeElementId);
-                    while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
-                        const targetId = stmt.getValue(0).getId();
-                        const sourceId = stmt.getValue(1).getString(); // BisCore.ExternalSourceAspect.Identifier stores a hex Id64String
-                        // TODO: maybe delete and don't just remap
-                        this.context.remapElement(targetId, sourceId);
+        this._deletedSourceRelationshipData = new Map();
+        nodeAssert(this._changeSummaryIds, "change summaries should be initialized before we get here");
+        nodeAssert(this._changeSummaryIds.length > 0, "change summaries should have at least one");
+        const alreadyImportedElementInserts = new Set();
+        const alreadyImportedModelInserts = new Set();
+        this.exporter.sourceDbChanges?.element.insertIds.forEach((insertedSourceElementId) => {
+            const targetElementId = this.context.findTargetElementId(insertedSourceElementId);
+            if (core_bentley_1.Id64.isValid(targetElementId))
+                alreadyImportedElementInserts.add(targetElementId);
+        });
+        this.exporter.sourceDbChanges?.model.insertIds.forEach((insertedSourceModelId) => {
+            const targetModelId = this.context.findTargetElementId(insertedSourceModelId);
+            if (core_bentley_1.Id64.isValid(targetModelId))
+                alreadyImportedModelInserts.add(targetModelId);
+        });
+        // optimization: if we have provenance, use it to avoid more querying later
+        // eventually when itwin.js supports attaching a second iModelDb in JS,
+        // this won't have to be a conditional part of the query, and we can always have it by attaching
+        const queryCanAccessProvenance = this.sourceDb === this.provenanceDb;
+        const deletedEntitySql = `
+      SELECT
+        1 AS IsElemNotRel,
+        ic.ChangedInstance.Id AS InstanceId,
+        NULL AS InstId2, -- need these columns for relationship ends in the unioned query
+        NULL AS InstId3,
+        ec.FederationGuid AS FedGuid,
+        NULL AS FedGuid2,
+        ic.ChangedInstance.ClassId AS ClassId
+        ${queryCanAccessProvenance ? `
+        , coalesce(esa.Identifier, esac.Identifier) AS Identifier1
+        , NULL AS Identifier2
+        ` : ""}
+      FROM ecchange.change.InstanceChange ic
+        LEFT JOIN bis.Element.Changes(:changeSummaryId, 'BeforeDelete') ec
+          ON ic.ChangedInstance.Id=ec.ECInstanceId
+        ${queryCanAccessProvenance ? `
+          LEFT JOIN bis.ExternalSourceAspect esa
+            ON ec.ECInstanceId=esa.Element.Id
+          LEFT JOIN bis.ExternalSourceAspect.Changes(:changeSummaryId, 'BeforeDelete') esac
+            ON ec.ECInstanceId=esac.Element.Id
+        ` : ""}
+      WHERE ic.OpCode=:opDelete
+        AND ic.Summary.Id=:changeSummaryId
+        AND ic.ChangedInstance.ClassId IS (BisCore.Element)
+        ${queryCanAccessProvenance ? `
+          AND (esa.Scope.Id=:targetScopeElement OR esa.Scope.Id IS NULL)
+          AND (esa.Kind='Element' OR esa.Kind IS NULL)
+          AND (esac.Scope.Id=:targetScopeElement OR esac.Scope.Id IS NULL)
+          AND (esac.Kind='Element' OR esac.Kind IS NULL)
+          ` : ""}
+
+      UNION ALL
+
+      SELECT
+        0 AS IsElemNotRel,
+        ic.ChangedInstance.Id AS InstanceId,
+        coalesce(se.ECInstanceId, sec.ECInstanceId) AS InstId2,
+        coalesce(te.ECInstanceId, tec.ECInstanceId) AS InstId3,
+        coalesce(se.FederationGuid, sec.FederationGuid) AS FedGuid1,
+        coalesce(te.FederationGuid, tec.FederationGuid) AS FedGuid2,
+        ic.ChangedInstance.ClassId AS ClassId
+        ${queryCanAccessProvenance ? `
+        , coalesce(sesa.Identifier, sesac.Identifier) AS Identifier1
+        , coalesce(tesa.Identifier, tesac.Identifier) AS Identifier2
+        ` : ""}
+      FROM ecchange.change.InstanceChange ic
+        LEFT JOIN bis.ElementRefersToElements.Changes(:changeSummaryId, 'BeforeDelete') ertec
+          ON ic.ChangedInstance.Id=ertec.ECInstanceId
+        -- FIXME: test a deletion of both an element and a relationship at the same time
+        LEFT JOIN bis.Element se
+          ON se.ECInstanceId=ertec.SourceECInstanceId
+        LEFT JOIN bis.Element te
+          ON te.ECInstanceId=ertec.TargetECInstanceId
+        LEFT JOIN bis.Element.Changes(:changeSummaryId, 'BeforeDelete') sec
+          ON sec.ECInstanceId=ertec.SourceECInstanceId
+        LEFT JOIN bis.Element.Changes(:changeSummaryId, 'BeforeDelete') tec
+          ON tec.ECInstanceId=ertec.TargetECInstanceId
+        ${queryCanAccessProvenance ? `
+          -- NOTE: need to join on both se/te and sec/tec incase the element was deleted
+          LEFT JOIN bis.ExternalSourceAspect sesa
+            ON se.ECInstanceId=sesa.Element.Id -- don't use *esac*.Identifier because it's a string
+          LEFT JOIN bis.ExternalSourceAspect.Changes(:changeSummaryId, 'BeforeDelete') sesac
+            ON sec.ECInstanceId=sesac.Element.Id
+          LEFT JOIN bis.ExternalSourceAspect tesa
+            ON te.ECInstanceId=tesa.Element.Id
+          LEFT JOIN bis.ExternalSourceAspect.Changes(:changeSummaryId, 'BeforeDelete') tesac
+            ON tec.ECInstanceId=tesac.Element.Id
+          ` : ""}
+      WHERE ic.OpCode=:opDelete
+        AND ic.Summary.Id=:changeSummaryId
+        AND ic.ChangedInstance.ClassId IS (BisCore.ElementRefersToElements)
+        ${queryCanAccessProvenance ? `
+          AND (sesa.Scope.Id=:targetScopeElement OR sesa.Scope.Id IS NULL)
+          AND (sesa.Kind='Relationship' OR sesa.Kind IS NULL)
+          AND (sesac.Scope.Id=:targetScopeElement OR sesac.Scope.Id IS NULL)
+          AND (sesac.Kind='Relationship' OR sesac.Kind IS NULL)
+          AND (tesa.Scope.Id=:targetScopeElement OR tesa.Scope.Id IS NULL)
+          AND (tesa.Kind='Relationship' OR tesa.Kind IS NULL)
+          AND (tesac.Scope.Id=:targetScopeElement OR tesac.Scope.Id IS NULL)
+          AND (tesac.Kind='Relationship' OR tesac.Kind IS NULL)
+          ` : ""}
+    `;
+        for (const changeSummaryId of this._changeSummaryIds) {
+            // FIXME: test deletion in both forward and reverse sync
+            this.sourceDb.withPreparedStatement(deletedEntitySql, (stmt) => {
+                stmt.bindInteger("opDelete", core_common_1.ChangeOpCode.Delete);
+                if (queryCanAccessProvenance)
+                    stmt.bindId("targetScopeElement", this.targetScopeElementId);
+                stmt.bindId("changeSummaryId", changeSummaryId);
+                while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
+                    const isElemNotRel = stmt.getValue(0).getBoolean();
+                    const instId = stmt.getValue(1).getId();
+                    if (isElemNotRel) {
+                        const sourceElemFedGuid = stmt.getValue(4).getGuid();
+                        // "Identifier" is a string, so null value returns '' which doesn't work with ??, and I don't like ||
+                        let identifierValue;
+                        // TODO: if I could attach the second db, will probably be much faster to get target id
+                        // as part of the whole query rather than with _queryElemIdByFedGuid
+                        const targetId = (queryCanAccessProvenance
+                            && (identifierValue = stmt.getValue(7))
+                            && !identifierValue.isNull
+                            && identifierValue.getString())
+                            // maybe batching these queries would perform better but we should
+                            // try to attach the second db and query both together anyway
+                            || (sourceElemFedGuid && this._queryElemIdByFedGuid(this.targetDb, sourceElemFedGuid))
+                            // FIXME: describe why it's safe to assume nothing has been deleted in provenanceDb
+                            || this._queryProvenanceForElement(instId);
+                        // since we are processing one changeset at a time, we can see local source deletes
+                        // of entities that were never synced and can be safely ignored
+                        const deletionNotInTarget = !targetId;
+                        if (deletionNotInTarget)
+                            continue;
+                        this.context.remapElement(instId, targetId);
+                        // If an entity insert and an entity delete both point to the same entity in target iModel, that means that entity was recreated.
+                        // In such case an entity update will be triggered and we no longer need to delete the entity.
+                        if (alreadyImportedElementInserts.has(targetId)) {
+                            this.exporter.sourceDbChanges?.element.deleteIds.delete(instId);
+                        }
+                        if (alreadyImportedModelInserts.has(targetId)) {
+                            this.exporter.sourceDbChanges?.model.deleteIds.delete(instId);
+                        }
                     }
-                });
-            }
+                    else { // is deleted relationship
+                        const classFullName = stmt.getValue(6).getClassNameForClassId();
+                        const [sourceIdInTarget, targetIdInTarget] = [
+                            { guidColumn: 4, identifierColumn: 7, isTarget: false },
+                            { guidColumn: 5, identifierColumn: 8, isTarget: true },
+                        ].map(({ guidColumn, identifierColumn }) => {
+                            const fedGuid = stmt.getValue(guidColumn).getGuid();
+                            let identifierValue;
+                            return ((queryCanAccessProvenance
+                                // FIXME: this is really far from idiomatic, try to undo that
+                                && (identifierValue = stmt.getValue(identifierColumn))
+                                && !identifierValue.isNull
+                                && identifierValue.getString())
+                                // maybe batching these queries would perform better but we should
+                                // try to attach the second db and query both together anyway
+                                || (fedGuid && this._queryElemIdByFedGuid(this.targetDb, fedGuid)));
+                        });
+                        // since we are processing one changeset at a time, we can see local source deletes
+                        // of entities that were never synced and can be safely ignored
+                        if (sourceIdInTarget && targetIdInTarget) {
+                            this._deletedSourceRelationshipData.set(instId, {
+                                classFullName,
+                                sourceIdInTarget,
+                                targetIdInTarget,
+                            });
+                        }
+                        else {
+                            // FIXME: describe why it's safe to assume nothing has been deleted in provenanceDb
+                            const relProvenance = this._queryProvenanceForRelationship(instId, {
+                                classFullName,
+                                sourceId: stmt.getValue(2).getId(),
+                                targetId: stmt.getValue(3).getId(),
+                            });
+                            if (relProvenance && relProvenance.relationshipId)
+                                this._deletedSourceRelationshipData.set(instId, {
+                                    classFullName,
+                                    relId: relProvenance.relationshipId,
+                                    provenanceAspectId: relProvenance.aspectId,
+                                });
+                        }
+                    }
+                }
+                // NEXT: remap sourceId and targetId to target, get provenance there
+                // NOTE: it is possible during a forward sync for the target to already have deleted
+                // something that the source deleted, in which case we can safely ignore the gone provenance
+            });
         }
-        finally {
-            if (core_backend_1.ChangeSummaryManager.isChangeCacheAttached(this.sourceDb))
-                core_backend_1.ChangeSummaryManager.detachChangeCache(this.sourceDb);
+    }
+    _queryProvenanceForElement(entityInProvenanceSourceId) {
+        return this.provenanceDb.withPreparedStatement(`
+        SELECT esa.Element.Id
+        FROM Bis.ExternalSourceAspect esa
+        WHERE esa.Kind=?
+          AND esa.Scope.Id=?
+          AND esa.Identifier=?
+      `, (stmt) => {
+            stmt.bindString(1, core_backend_1.ExternalSourceAspect.Kind.Element);
+            stmt.bindId(2, this.targetScopeElementId);
+            stmt.bindString(3, entityInProvenanceSourceId);
+            if (stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return stmt.getValue(0).getId();
+            else
+                return undefined;
+        });
+    }
+    _queryProvenanceForRelationship(entityInProvenanceSourceId, sourceRelInfo) {
+        return this.provenanceDb.withPreparedStatement(`
+        SELECT
+          ECInstanceId,
+          JSON_EXTRACT(JsonProperties, '$.targetRelInstanceId'),
+          JSON_EXTRACT(JsonProperties, '$.provenanceRelInstanceId')
+        FROM Bis.ExternalSourceAspect
+        WHERE Kind=?
+          AND Scope.Id=?
+          AND Identifier=?
+      `, (stmt) => {
+            stmt.bindString(1, core_backend_1.ExternalSourceAspect.Kind.Relationship);
+            stmt.bindId(2, this.targetScopeElementId);
+            stmt.bindString(3, entityInProvenanceSourceId);
+            if (stmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return undefined;
+            const aspectId = stmt.getValue(0).getId();
+            const provenanceRelInstIdVal = stmt.getValue(2);
+            const provenanceRelInstanceId = !provenanceRelInstIdVal.isNull
+                ? provenanceRelInstIdVal.getString()
+                : this._queryTargetRelId(sourceRelInfo);
+            return {
+                aspectId,
+                relationshipId: provenanceRelInstanceId,
+            };
+        });
+    }
+    _queryTargetRelId(sourceRelInfo) {
+        const targetRelInfo = {
+            sourceId: this.context.findTargetElementId(sourceRelInfo.sourceId),
+            targetId: this.context.findTargetElementId(sourceRelInfo.targetId),
+        };
+        if (targetRelInfo.sourceId === undefined || targetRelInfo.targetId === undefined)
+            return undefined; // couldn't find an element, rel is invalid or deleted
+        return this.targetDb.withPreparedStatement(`
+      SELECT ECInstanceId
+      FROM bis.ElementRefersToElements
+      WHERE SourceECInstanceId=?
+        AND TargetECInstanceId=?
+        AND ECClassId=?
+    `, (stmt) => {
+            stmt.bindId(1, targetRelInfo.sourceId);
+            stmt.bindId(2, targetRelInfo.targetId);
+            stmt.bindId(3, this._targetClassNameToClassId(sourceRelInfo.classFullName));
+            if (stmt.step() !== core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return undefined;
+            return stmt.getValue(0).getId();
+        });
+    }
+    _targetClassNameToClassId(classFullName) {
+        let classId = this._targetClassNameToClassIdCache.get(classFullName);
+        if (classId === undefined) {
+            classId = this._getRelClassId(this.targetDb, classFullName);
+            this._targetClassNameToClassIdCache.set(classFullName, classId);
         }
+        return classId;
+    }
+    // NOTE: this doesn't handle remapped element classes,
+    // but is only used for relationships rn
+    _getRelClassId(db, classFullName) {
+        return db.withPreparedStatement(`
+      SELECT c.ECInstanceId
+      FROM ECDbMeta.ECClassDef c
+      JOIN ECDbMeta.ECSchemaDef s ON c.Schema.Id=s.ECInstanceId
+      WHERE s.Name=? AND c.Name=?
+    `, (stmt) => {
+            const [schemaName, className] = classFullName.split(".");
+            stmt.bindString(1, schemaName);
+            stmt.bindString(2, className);
+            if (stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return stmt.getValue(0).getId();
+            (0, core_bentley_1.assert)(false, "relationship was not found");
+        });
+    }
+    _queryElemIdByFedGuid(db, fedGuid) {
+        return db.withPreparedStatement("SELECT ECInstanceId FROM Bis.Element WHERE FederationGuid=?", (stmt) => {
+            stmt.bindGuid(1, fedGuid);
+            if (stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW)
+                return stmt.getValue(0).getId();
+            else
+                return undefined;
+        });
     }
     /** Returns `true` if *brute force* delete detections should be run.
      * @note Not relevant for processChanges when change history is known.
      */
     shouldDetectDeletes() {
+        // FIXME: all synchronizations should mark this as false
         if (this._isFirstSynchronization)
             return false; // not necessary the first time since there are no deletes to detect
         if (this._options.isReverseSynchronization)
             return false; // not possible for a reverse synchronization since provenance will be deleted when element is deleted
         return true;
     }
-    /** Detect Element deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against Elements in the source iModel.
-     * @see processChanges
-     * @note This method is called from [[processAll]] and is not needed by [[processChanges]], so it only needs to be called directly when processing a subset of an iModel.
+    /**
+     * Detect Element deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against Elements
+     * in the source iModel.
+     * @deprecated in 0.1.x. This method is only called during [[processAll]] when the option
+     * [[IModelTransformerOptions.forceExternalSourceAspectProvenance]] is enabled. It is not
+     * necessary when using [[processChanges]] since changeset information is sufficient.
+     * @note you do not need to call this directly unless processing a subset of an iModel.
      * @throws [[IModelError]] If the required provenance information is not available to detect deletes.
      */
     async detectElementDeletes() {
-        if (this._options.isReverseSynchronization) {
-            throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Cannot detect deletes when isReverseSynchronization=true");
-        }
-        const targetElementsToDelete = [];
-        this.forEachTrackedElement((sourceElementId, targetElementId) => {
-            if (undefined === this.sourceDb.elements.tryGetElementProps(sourceElementId)) {
-                // if the sourceElement is not found, then it must have been deleted, so propagate the delete to the target iModel
-                targetElementsToDelete.push(targetElementId);
-            }
-        });
-        targetElementsToDelete.forEach((targetElementId) => {
-            try {
-                // TODO: make it possible to delete more elements at once to prevent redundant expensive
-                // element reference scanning
-                this.importer.deleteElement(targetElementId);
-            }
-            catch (err) {
-                // ignore not found elements, iterative element tree deletion might have already deleted them
-                if (err.name !== "Not Found")
-                    throw err;
+        const sql = `
+      SELECT Identifier, Element.Id
+      FROM BisCore.ExternalSourceAspect
+      WHERE Scope.Id=:scopeId
+        AND Kind=:kind
+    `;
+        nodeAssert(!this._options.isReverseSynchronization, "synchronizations with processChagnes already detect element deletes, don't call detectElementDeletes");
+        this.provenanceDb.withPreparedStatement(sql, (stmt) => {
+            stmt.bindId("scopeId", this.targetScopeElementId);
+            stmt.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Element);
+            while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
+                // ExternalSourceAspect.Identifier is of type string
+                const aspectIdentifier = stmt.getValue(0).getString();
+                if (!core_bentley_1.Id64.isId64(aspectIdentifier)) {
+                    continue;
+                }
+                const targetElemId = stmt.getValue(1).getId();
+                const wasDeletedInSource = !EntityUnifier_1.EntityUnifier.exists(this.sourceDb, { entityReference: `e${aspectIdentifier}` });
+                if (wasDeletedInSource)
+                    this.importer.deleteElement(targetElemId);
             }
         });
     }
@@ -451,24 +883,53 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         }
         return targetElementProps;
     }
+    // FIXME: this is a PoC, see if we minimize memory usage
+    _cacheSourceChanges() {
+        nodeAssert(this._changeSummaryIds && this._changeSummaryIds.length > 0, "should have changeset data by now");
+        this._hasElementChangedCache = new Set();
+        const query = `
+      SELECT
+        ic.ChangedInstance.Id AS InstId
+      FROM ecchange.change.InstanceChange ic
+      JOIN iModelChange.Changeset imc ON ic.Summary.Id=imc.Summary.Id
+      -- FIXME: do relationship entities also need this cache optimization?
+      WHERE ic.ChangedInstance.ClassId IS (BisCore.Element)
+        AND InVirtualSet(:changeSummaryIds, ic.Summary.Id)
+        -- ignore deleted, we take care of those in remapDeletedSourceEntities
+        -- include inserted since inserted code-colliding elements should be considered
+        -- a change so that the colliding element is exported to the target
+        AND ic.OpCode<>:opDelete
+    `;
+        // there is a single mega-query multi-join+coalescing hack that I used originally to get around
+        // only being able to run table.Changes() on one changeset at once, but sqlite only supports up to 64
+        // tables in a join. Need to talk to core about .Changes being able to take a set of changesets
+        // You can find this version in the `federation-guid-optimization-megaquery` branch
+        // I wouldn't use it unless we prove via profiling that it speeds things up significantly
+        // And even then let's first try scanning the raw changesets instead of applying them as these queries
+        // require
+        this.sourceDb.withPreparedStatement(query, (stmt) => {
+            stmt.bindInteger("opDelete", core_common_1.ChangeOpCode.Delete);
+            stmt.bindIdSet("changeSummaryIds", this._changeSummaryIds);
+            while (core_bentley_1.DbResult.BE_SQLITE_ROW === stmt.step()) {
+                const instId = stmt.getValue(0).getId();
+                this._hasElementChangedCache.add(instId);
+            }
+        });
+    }
     /** Returns true if a change within sourceElement is detected.
      * @param sourceElement The Element from the source iModel
      * @param targetElementId The Element from the target iModel to compare against.
      * @note A subclass can override this method to provide custom change detection behavior.
      */
-    hasElementChanged(sourceElement, targetElementId) {
-        const sourceAspects = this.targetDb.elements.getAspects(targetElementId, core_backend_1.ExternalSourceAspect.classFullName);
-        for (const sourceAspect of sourceAspects) {
-            if (sourceAspect.scope === undefined) // if the scope was lost, we can't correlate so assume it changed
-                return true;
-            if (sourceAspect.identifier === sourceElement.id &&
-                sourceAspect.scope.id === this.targetScopeElementId &&
-                sourceAspect.kind === core_backend_1.ExternalSourceAspect.Kind.Element) {
-                const lastModifiedTime = sourceElement.iModel.elements.queryLastModifiedTime(sourceElement.id);
-                return lastModifiedTime !== sourceAspect.version;
-            }
-        }
-        return true;
+    hasElementChanged(sourceElement, _targetElementId) {
+        if (this._sourceChangeDataState === "no-changes")
+            return false;
+        if (this._sourceChangeDataState === "unconnected")
+            return true;
+        nodeAssert(this._sourceChangeDataState === "has-changes", "change data should be initialized by now");
+        if (this._hasElementChangedCache === undefined)
+            this._cacheSourceChanges();
+        return this._hasElementChangedCache.has(sourceElement.id);
     }
     static transformCallbackFor(transformer, entity) {
         if (entity instanceof core_backend_1.Element)
@@ -658,51 +1119,68 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             targetElementId = this.context.findTargetElementId(sourceElement.id);
             targetElementProps = this.onTransformElement(sourceElement);
         }
+        // if an existing remapping was not yet found, check by FederationGuid
+        if (this.context.isBetweenIModels && !core_bentley_1.Id64.isValid(targetElementId) && sourceElement.federationGuid !== undefined) {
+            targetElementId = this._queryElemIdByFedGuid(this.targetDb, sourceElement.federationGuid) ?? core_bentley_1.Id64.invalid;
+            if (core_bentley_1.Id64.isValid(targetElementId))
+                this.context.remapElement(sourceElement.id, targetElementId); // record that the targetElement was found
+        }
         // if an existing remapping was not yet found, check by Code as long as the CodeScope is valid (invalid means a missing reference so not worth checking)
-        if (!core_bentley_1.Id64.isValidId64(targetElementId) && core_bentley_1.Id64.isValidId64(targetElementProps.code.scope)) {
+        if (!core_bentley_1.Id64.isValid(targetElementId) && core_bentley_1.Id64.isValidId64(targetElementProps.code.scope)) {
             // respond the same way to undefined code value as the @see Code class, but don't use that class because is trims
             // whitespace from the value, and there are iModels out there with untrimmed whitespace that we ought not to trim
             targetElementProps.code.value = targetElementProps.code.value ?? "";
-            targetElementId = this.targetDb.elements.queryElementIdByCode(targetElementProps.code);
-            if (undefined !== targetElementId) {
-                const targetElement = this.targetDb.elements.getElement(targetElementId);
-                if (targetElement.classFullName === targetElementProps.classFullName) { // ensure code remapping doesn't change the target class
+            const maybeTargetElementId = this.targetDb.elements.queryElementIdByCode(targetElementProps.code);
+            if (undefined !== maybeTargetElementId) {
+                const maybeTargetElem = this.targetDb.elements.getElement(maybeTargetElementId);
+                if (maybeTargetElem.classFullName === targetElementProps.classFullName) { // ensure code remapping doesn't change the target class
+                    targetElementId = maybeTargetElementId;
                     this.context.remapElement(sourceElement.id, targetElementId); // record that the targetElement was found by Code
                 }
                 else {
-                    targetElementId = undefined;
                     targetElementProps.code = core_common_1.Code.createEmpty(); // clear out invalid code
                 }
             }
         }
-        if (undefined !== targetElementId && core_bentley_1.Id64.isValidId64(targetElementId)) {
-            // compare LastMod of sourceElement to ExternalSourceAspect of targetElement to see there are changes to import
-            if (!this.hasElementChanged(sourceElement, targetElementId)) {
-                return;
-            }
-        }
+        if (core_bentley_1.Id64.isValid(targetElementId) && !this.hasElementChanged(sourceElement, targetElementId))
+            return;
         this.collectUnmappedReferences(sourceElement);
-        // TODO: untangle targetElementId state...
-        if (targetElementId === core_bentley_1.Id64.invalid)
-            targetElementId = undefined;
-        targetElementProps.id = targetElementId; // targetElementId will be valid (indicating update) or undefined (indicating insert)
+        // targetElementId will be valid (indicating update) or undefined (indicating insert)
+        targetElementProps.id
+            = core_bentley_1.Id64.isValid(targetElementId)
+                ? targetElementId
+                : undefined;
         if (!this._options.wasSourceIModelCopiedToTarget) {
             this.importer.importElement(targetElementProps); // don't need to import if iModel was copied
         }
         this.context.remapElement(sourceElement.id, targetElementProps.id); // targetElementProps.id assigned by importElement
         // now that we've mapped this elem we can fix unmapped references to it
         this.resolvePendingReferences(sourceElement);
+        // the transformer does not currently 'split' or 'join' any elements, therefore, it does not
+        // insert external source aspects because federation guids are sufficient for this.
+        // Other transformer subclasses must insert the appropriate aspect (as provided by a TBD API)
+        // when splitting/joining elements
+        // physical consolidation is an example of a 'joining' transform
+        // FIXME: document this externally!
+        // verify at finalization time that we don't lose provenance on new elements
+        // make public and improve `initElementProvenance` API for usage by consolidators
         if (!this._options.noProvenance) {
-            const aspectProps = this.initElementProvenance(sourceElement.id, targetElementProps.id);
-            let aspectId = this.queryExternalSourceAspectId(aspectProps);
-            if (aspectId === undefined) {
-                aspectId = this.provenanceDb.elements.insertAspect(aspectProps);
-            }
-            else {
-                this.provenanceDb.elements.updateAspect(aspectProps);
+            let provenance = this._options.forceExternalSourceAspectProvenance || this._elementsWithExplicitlyTrackedProvenance.has(sourceElement.id)
+                ? undefined
+                : sourceElement.federationGuid;
+            if (!provenance) {
+                const aspectProps = this.initElementProvenance(sourceElement.id, targetElementProps.id);
+                const aspectId = this.queryScopeExternalSource(aspectProps).aspectId;
+                if (aspectId === undefined) {
+                    aspectProps.id = this.provenanceDb.elements.insertAspect(aspectProps);
+                }
+                else {
+                    aspectProps.id = aspectId;
+                    this.provenanceDb.elements.updateAspect(aspectProps);
+                }
+                provenance = aspectProps;
             }
-            aspectProps.id = aspectId;
-            this.markLastProvenance(aspectProps, { isRelationship: false });
+            this.markLastProvenance(provenance, { isRelationship: false });
         }
     }
     resolvePendingReferences(entity) {
@@ -740,50 +1218,11 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
     onDeleteModel(sourceModelId) {
         // It is possible and apparently occasionally sensical to delete a model without deleting its underlying element.
         // - If only the model is deleted, [[initFromExternalSourceAspects]] will have already remapped the underlying element since it still exists.
-        // - If both were deleted, [[remapDeletedSourceElements]] will find and remap the deleted element making this operation valid
+        // - If both were deleted, [[remapDeletedSourceEntities]] will find and remap the deleted element making this operation valid
         const targetModelId = this.context.findTargetElementId(sourceModelId);
-        if (!core_bentley_1.Id64.isValidId64(targetModelId))
-            return;
-        if (this.exporter.sourceDbChanges?.element.deleteIds.has(sourceModelId)) {
-            const isDefinitionPartition = this.targetDb.withPreparedStatement(`
-        SELECT 1
-        FROM bis.DefinitionPartition
-        WHERE ECInstanceId=?
-      `, (stmt) => {
-                stmt.bindId(1, targetModelId);
-                const val = stmt.step();
-                switch (val) {
-                    case core_bentley_1.DbResult.BE_SQLITE_ROW: return true;
-                    case core_bentley_1.DbResult.BE_SQLITE_DONE: return false;
-                    default: (0, core_bentley_1.assert)(false, `unexpected db result: '${stmt}'`);
-                }
-            });
-            if (isDefinitionPartition) {
-                // Skipping model deletion because model's partition will also be deleted.
-                // It expects that model will be present and will fail if it's missing.
-                // Model will be deleted when its partition will be deleted.
-                return;
-            }
-        }
-        try {
+        if (core_bentley_1.Id64.isValidId64(targetModelId)) {
             this.importer.deleteModel(targetModelId);
         }
-        catch (error) {
-            const isDeletionProhibitedErr = error instanceof core_common_1.IModelError && (error.errorNumber === core_bentley_1.IModelStatus.DeletionProhibited || error.errorNumber === core_bentley_1.IModelStatus.ForeignKeyConstraint);
-            if (!isDeletionProhibitedErr)
-                throw error;
-            // Transformer tries to delete models before it deletes elements. Definition models cannot be deleted unless all of their modeled elements are deleted first.
-            // In case a definition model needs to be deleted we need to skip it for now and register its modeled partition for deletion.
-            // The `OnDeleteElement` calls `DeleteElementTree` Which deletes the model together with its partition after deleting all of the modeled elements.
-            this.scheduleModeledPartitionDeletion(sourceModelId);
-        }
-    }
-    /** Schedule modeled partition deletion */
-    scheduleModeledPartitionDeletion(sourceModelId) {
-        const deletedElements = this.exporter.sourceDbChanges?.element.deleteIds;
-        if (!deletedElements.has(sourceModelId)) {
-            deletedElements.add(sourceModelId);
-        }
     }
     /** Cause the model container, contents, and sub-models to be exported from the source iModel and imported into the target iModel.
      * @param sourceModeledElementId Import this [Model]($backend) from the source IModelDb.
@@ -855,7 +1294,68 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * @deprecated in 3.x. This method is no longer necessary since the transformer no longer needs to defer elements
      */
     async processDeferredElements(_numRetries = 3) { }
+    /** called at the end ([[finalizeTransformation]]) of a transformation,
+     * updates the target scope element to say that transformation up through the
+     * source's changeset has been performed. Also stores all changesets that occurred
+     * during the transformation as "pending synchronization changeset indices"
+     *
+     * You generally should not call this function yourself and use [[processChanges]] instead.
+     * It is public for unsupported use cases of custom synchronization transforms.
+     * @note if you are not running processChanges in this transformation, this will fail
+     * without setting the `force` option to `true`
+     */
+    updateSynchronizationVersion({ force = false } = {}) {
+        if (!force && (this._sourceChangeDataState !== "has-changes" && !this._isFirstSynchronization))
+            return;
+        nodeAssert(this._targetScopeProvenanceProps);
+        const sourceVersion = `${this.sourceDb.changeset.id};${this.sourceDb.changeset.index}`;
+        const targetVersion = `${this.targetDb.changeset.id};${this.targetDb.changeset.index}`;
+        if (this._isFirstSynchronization) {
+            this._targetScopeProvenanceProps.version = sourceVersion;
+            this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion = targetVersion;
+        }
+        else if (this._options.isReverseSynchronization) {
+            const oldVersion = this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion;
+            core_bentley_1.Logger.logInfo(loggerCategory, `updating reverse version from ${oldVersion} to ${sourceVersion}`);
+            this._targetScopeProvenanceProps.jsonProperties.reverseSyncVersion = sourceVersion;
+        }
+        else if (!this._options.isReverseSynchronization) {
+            core_bentley_1.Logger.logInfo(loggerCategory, `updating sync version from ${this._targetScopeProvenanceProps.version} to ${sourceVersion}`);
+            this._targetScopeProvenanceProps.version = sourceVersion;
+        }
+        if (this._isSynchronization) {
+            (0, core_bentley_1.assert)(this.targetDb.changeset.index !== undefined && this._startingChangesetIndices !== undefined, "updateSynchronizationVersion was called without change history");
+            const jsonProps = this._targetScopeProvenanceProps.jsonProperties;
+            core_bentley_1.Logger.logTrace(loggerCategory, `previous pendingReverseSyncChanges: ${jsonProps.pendingReverseSyncChangesetIndices}`);
+            core_bentley_1.Logger.logTrace(loggerCategory, `previous pendingSyncChanges: ${jsonProps.pendingSyncChangesetIndices}`);
+            const [syncChangesetsToClear, syncChangesetsToUpdate] = this._isReverseSynchronization
+                ? [jsonProps.pendingReverseSyncChangesetIndices, jsonProps.pendingSyncChangesetIndices]
+                : [jsonProps.pendingSyncChangesetIndices, jsonProps.pendingReverseSyncChangesetIndices];
+            // NOTE that as documented in [[processChanges]], this assumes that right after
+            // transformation finalization, the work will be saved immediately, otherwise we've
+            // just marked this changeset as a synchronization to ignore, and the user can add other
+            // stuff to it which would break future synchronizations
+            // FIXME: force save for the user to prevent that
+            for (let i = this._startingChangesetIndices.target + 1; i <= this.targetDb.changeset.index + 1; i++)
+                syncChangesetsToUpdate.push(i);
+            syncChangesetsToClear.length = 0;
+            // if reverse sync then we may have received provenance changes which should be marked as sync changes
+            if (this._isReverseSynchronization) {
+                nodeAssert(this.sourceDb.changeset.index, "changeset didn't exist");
+                for (let i = this._startingChangesetIndices.source + 1; i <= this.sourceDb.changeset.index + 1; i++)
+                    jsonProps.pendingReverseSyncChangesetIndices.push(i);
+            }
+            core_bentley_1.Logger.logTrace(loggerCategory, `new pendingReverseSyncChanges: ${jsonProps.pendingReverseSyncChangesetIndices}`);
+            core_bentley_1.Logger.logTrace(loggerCategory, `new pendingSyncChanges: ${jsonProps.pendingSyncChangesetIndices}`);
+        }
+        this.provenanceDb.elements.updateAspect({
+            ...this._targetScopeProvenanceProps,
+            jsonProperties: JSON.stringify(this._targetScopeProvenanceProps.jsonProperties),
+        });
+    }
+    // FIXME: is this necessary when manually using lowlevel transform APIs?
     finalizeTransformation() {
+        this.updateSynchronizationVersion();
         if (this._partiallyCommittedEntities.size > 0) {
             core_bentley_1.Logger.logWarning(loggerCategory, [
                 "The following elements were never fully resolved:",
@@ -867,6 +1367,11 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 partiallyCommittedElem.forceComplete();
             }
         }
+        // FIXME: make processAll have a try {} finally {} that cleans this up
+        if (!this._options.noDetachChangeCache) {
+            if (core_backend_1.ChangeSummaryManager.isChangeCacheAttached(this.sourceDb))
+                core_backend_1.ChangeSummaryManager.detachChangeCache(this.sourceDb);
+        }
     }
     /** Imports all relationships that subclass from the specified base class.
      * @param baseRelClassFullName The specified base relationship class.
@@ -884,40 +1389,52 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
      * This override calls [[onTransformRelationship]] and then [IModelImporter.importRelationship]($transformer) to update the target iModel.
      */
     onExportRelationship(sourceRelationship) {
+        const sourceFedGuid = queryElemFedGuid(this.sourceDb, sourceRelationship.sourceId);
+        const targetFedGuid = queryElemFedGuid(this.sourceDb, sourceRelationship.targetId);
         const targetRelationshipProps = this.onTransformRelationship(sourceRelationship);
         const targetRelationshipInstanceId = this.importer.importRelationship(targetRelationshipProps);
-        if (!this._options.noProvenance && core_bentley_1.Id64.isValidId64(targetRelationshipInstanceId)) {
-            const aspectProps = this.initRelationshipProvenance(sourceRelationship, targetRelationshipInstanceId);
-            if (undefined === aspectProps.id) {
-                aspectProps.id = this.provenanceDb.elements.insertAspect(aspectProps);
+        if (!this._options.noProvenance && core_bentley_1.Id64.isValid(targetRelationshipInstanceId)) {
+            let provenance = !this._options.forceExternalSourceAspectProvenance
+                ? sourceFedGuid && targetFedGuid && `${sourceFedGuid}/${targetFedGuid}`
+                : undefined;
+            if (!provenance) {
+                const aspectProps = this.initRelationshipProvenance(sourceRelationship, targetRelationshipInstanceId);
+                aspectProps.id = this.queryScopeExternalSource(aspectProps).aspectId;
+                if (undefined === aspectProps.id) {
+                    aspectProps.id = this.provenanceDb.elements.insertAspect(aspectProps);
+                }
+                provenance = aspectProps;
             }
-            (0, core_bentley_1.assert)(aspectProps.id !== undefined);
-            this.markLastProvenance(aspectProps, { isRelationship: true });
+            this.markLastProvenance(provenance, { isRelationship: true });
         }
     }
     /** Override of [IModelExportHandler.onDeleteRelationship]($transformer) that is called when [IModelExporter]($transformer) detects that a [Relationship]($backend) has been deleted from the source iModel.
      * This override propagates the delete to the target iModel via [IModelImporter.deleteRelationship]($transformer).
      */
     onDeleteRelationship(sourceRelInstanceId) {
-        const sql = `SELECT ECInstanceId,JsonProperties FROM ${core_backend_1.ExternalSourceAspect.classFullName} aspect` +
-            ` WHERE aspect.Scope.Id=:scopeId AND aspect.Kind=:kind AND aspect.Identifier=:identifier LIMIT 1`;
-        this.targetDb.withPreparedStatement(sql, (statement) => {
-            statement.bindId("scopeId", this.targetScopeElementId);
-            statement.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Relationship);
-            statement.bindString("identifier", sourceRelInstanceId);
-            if (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) {
-                const json = JSON.parse(statement.getValue(1).getString());
-                if (undefined !== json.targetRelInstanceId) {
-                    const targetRelationship = this.targetDb.relationships.tryGetInstance(core_backend_1.ElementRefersToElements.classFullName, json.targetRelInstanceId);
-                    if (targetRelationship) {
-                        this.importer.deleteRelationship(targetRelationship.toJSON());
-                    }
-                    this.targetDb.elements.deleteAspect(statement.getValue(0).getId());
-                }
-            }
-        });
+        nodeAssert(this._deletedSourceRelationshipData, "should be defined at initialization by now");
+        const deletedRelData = this._deletedSourceRelationshipData.get(sourceRelInstanceId);
+        if (!deletedRelData) {
+            // this can occur if both the source and target deleted it
+            core_bentley_1.Logger.logWarning(loggerCategory, "tried to delete a relationship that wasn't in change data");
+            return;
+        }
+        const relArg = deletedRelData.relId ?? {
+            sourceId: deletedRelData.sourceIdInTarget,
+            targetId: deletedRelData.targetIdInTarget,
+        };
+        //
+        // FIXME: make importer.deleteRelationship not need full props
+        const targetRelationship = this.targetDb.relationships.tryGetInstance(deletedRelData.classFullName, relArg);
+        if (targetRelationship) {
+            this.importer.deleteRelationship(targetRelationship.toJSON());
+        }
+        if (deletedRelData.provenanceAspectId) {
+            this.provenanceDb.elements.deleteAspect(deletedRelData.provenanceAspectId);
+        }
     }
     /** Detect Relationship deletes using ExternalSourceAspects in the target iModel and a *brute force* comparison against relationships in the source iModel.
+     * @deprecated
      * @see processChanges
      * @note This method is called from [[processAll]] and is not needed by [[processChanges]], so it only needs to be called directly when processing a subset of an iModel.
      * @throws [[IModelError]] If the required provenance information is not available to detect deletes.
@@ -927,13 +1444,20 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             throw new core_common_1.IModelError(core_bentley_1.IModelStatus.BadRequest, "Cannot detect deletes when isReverseSynchronization=true");
         }
         const aspectDeleteIds = [];
-        const sql = `SELECT ECInstanceId,Identifier,JsonProperties FROM ${core_backend_1.ExternalSourceAspect.classFullName} aspect WHERE aspect.Scope.Id=:scopeId AND aspect.Kind=:kind`;
+        const sql = `
+      SELECT ECInstanceId, Identifier, JsonProperties
+      FROM ${core_backend_1.ExternalSourceAspect.classFullName} aspect
+      WHERE aspect.Scope.Id=:scopeId
+        AND aspect.Kind=:kind
+    `;
         await this.targetDb.withPreparedStatement(sql, async (statement) => {
             statement.bindId("scopeId", this.targetScopeElementId);
             statement.bindString("kind", core_backend_1.ExternalSourceAspect.Kind.Relationship);
             while (core_bentley_1.DbResult.BE_SQLITE_ROW === statement.step()) {
                 const sourceRelInstanceId = core_bentley_1.Id64.fromJSON(statement.getValue(1).getString());
                 if (undefined === this.sourceDb.relationships.tryGetInstanceProps(core_backend_1.ElementRefersToElements.classFullName, sourceRelInstanceId)) {
+                    // FIXME: make sure matches new provenance-based method
+                    // FIXME: use sql JSON_EXTRACT
                     const json = JSON.parse(statement.getValue(2).getString());
                     if (undefined !== json.targetRelInstanceId) {
                         const targetRelationship = this.targetDb.relationships.getInstance(core_backend_1.ElementRefersToElements.classFullName, json.targetRelInstanceId);
@@ -955,6 +1479,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const targetRelationshipProps = sourceRelationship.toJSON();
         targetRelationshipProps.sourceId = this.context.findTargetElementId(sourceRelationship.sourceId);
         targetRelationshipProps.targetId = this.context.findTargetElementId(sourceRelationship.targetId);
+        // TODO: move to cloneRelationship in IModelCloneContext
         sourceRelationship.forEachProperty((propertyName, propertyMetaData) => {
             if ((core_common_1.PrimitiveTypeCode.Long === propertyMetaData.primitiveType) && ("Id" === propertyMetaData.extendedType)) {
                 targetRelationshipProps[propertyName] = this.context.findTargetElementId(sourceRelationship.asAny[propertyName]);
@@ -1116,26 +1641,86 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         return this.processDeferredElements(); // eslint-disable-line deprecation/deprecation
     }
     /**
-     * Initialize prerequisites of processing, you must initialize with an [[InitFromExternalSourceAspectsArgs]] if you
-     * are intending process changes, but prefer using [[processChanges]]
-     * Called by all `process*` functions implicitly.
+     * Initialize prerequisites of processing, you must initialize with an [[InitOptions]] if you
+     * are intending to process changes, but prefer using [[processChanges]] explicitly since it calls this.
+     * @note Called by all `process*` functions implicitly.
      * Overriders must call `super.initialize()` first
      */
     async initialize(args) {
         if (this._initialized)
             return;
         await this.context.initialize();
+        await this._tryInitChangesetData(args);
+        await this.exporter.initialize(this.getExportInitOpts(args ?? {}));
+        // Exporter must be initialized prior to `initFromExternalSourceAspects` in order to handle entity recreations.
         // eslint-disable-next-line deprecation/deprecation
         await this.initFromExternalSourceAspects(args);
         this._initialized = true;
     }
+    async _tryInitChangesetData(args) {
+        if (!args || this.sourceDb.iTwinId === undefined || this.sourceDb.changeset.index === undefined) {
+            this._sourceChangeDataState = "unconnected";
+            return;
+        }
+        const noChanges = this._synchronizationVersion.index === this.sourceDb.changeset.index;
+        if (noChanges) {
+            this._sourceChangeDataState = "no-changes";
+            this._changeSummaryIds = [];
+            return;
+        }
+        // NOTE: that we do NOT download the changesummary for the last transformed version, we want
+        // to ignore those already processed changes
+        const startChangesetIndexOrId = args.startChangeset?.index
+            ?? args.startChangeset?.id
+            ?? this._synchronizationVersion.index + 1;
+        const endChangesetId = this.sourceDb.changeset.id;
+        const [startChangesetIndex, endChangesetIndex] = await Promise.all(([startChangesetIndexOrId, endChangesetId])
+            .map(async (indexOrId) => typeof indexOrId === "number"
+            ? indexOrId
+            : core_backend_1.IModelHost.hubAccess
+                .queryChangeset({
+                iModelId: this.sourceDb.iModelId,
+                // eslint-disable-next-line deprecation/deprecation
+                changeset: { id: indexOrId },
+                accessToken: args.accessToken,
+            })
+                .then((changeset) => changeset.index)));
+        const missingChangesets = startChangesetIndex > this._synchronizationVersion.index + 1;
+        if (!this._options.ignoreMissingChangesetsInSynchronizations
+            && startChangesetIndex !== this._synchronizationVersion.index + 1
+            && this._synchronizationVersion.index !== -1) {
+            throw Error(`synchronization is ${missingChangesets ? "missing changesets" : ""},`
+                + " startChangesetId should be"
+                + " exactly the first changeset *after* the previous synchronization to not miss data."
+                + ` You specified '${startChangesetIndexOrId}' which is changeset #${startChangesetIndex}`
+                + ` but the previous synchronization for this targetScopeElement was '${this._synchronizationVersion.id}'`
+                + ` which is changeset #${this._synchronizationVersion.index}. The transformer expected`
+                + ` #${this._synchronizationVersion.index + 1}.`);
+        }
+        nodeAssert(this._targetScopeProvenanceProps, "_targetScopeProvenanceProps should be set by now");
+        const changesetsToSkip = this._isReverseSynchronization
+            ? this._targetScopeProvenanceProps.jsonProperties.pendingReverseSyncChangesetIndices
+            : this._targetScopeProvenanceProps.jsonProperties.pendingSyncChangesetIndices;
+        core_bentley_1.Logger.logTrace(loggerCategory, `changesets to skip: ${changesetsToSkip}`);
+        this._changesetRanges = (0, Algo_1.rangesFromRangeAndSkipped)(startChangesetIndex, endChangesetIndex, changesetsToSkip);
+        core_bentley_1.Logger.logTrace(loggerCategory, `ranges: ${this._changesetRanges}`);
+        for (const [first, end] of this._changesetRanges) {
+            this._changeSummaryIds = await core_backend_1.ChangeSummaryManager.createChangeSummaries({
+                accessToken: args.accessToken,
+                iModelId: this.sourceDb.iModelId,
+                iTwinId: this.sourceDb.iTwinId,
+                range: { first, end },
+            });
+        }
+        core_backend_1.ChangeSummaryManager.attachChangeCache(this.sourceDb);
+        this._sourceChangeDataState = "has-changes";
+    }
     /** Export everything from the source iModel and import the transformed entities into the target iModel.
    * @note [[processSchemas]] is not called automatically since the target iModel may want a different collection of schemas.
    */
     async processAll() {
-        core_bentley_1.Logger.logTrace(loggerCategory, "processAll()");
         this.logSettings();
-        this.validateScopeProvenance();
+        this.initScopeProvenance();
         await this.initialize();
         await this.exporter.exportCodeSpecs();
         await this.exporter.exportFonts();
@@ -1155,12 +1740,15 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.finalizeTransformation();
     }
     markLastProvenance(sourceAspect, { isRelationship = false }) {
-        this._lastProvenanceEntityInfo = {
-            entityId: sourceAspect.element.id,
-            aspectId: sourceAspect.id,
-            aspectVersion: sourceAspect.version ?? "",
-            aspectKind: isRelationship ? core_backend_1.ExternalSourceAspect.Kind.Relationship : core_backend_1.ExternalSourceAspect.Kind.Element,
-        };
+        this._lastProvenanceEntityInfo
+            = typeof sourceAspect === "string"
+                ? sourceAspect
+                : {
+                    entityId: sourceAspect.element.id,
+                    aspectId: sourceAspect.id,
+                    aspectVersion: sourceAspect.version ?? "",
+                    aspectKind: isRelationship ? core_backend_1.ExternalSourceAspect.Kind.Relationship : core_backend_1.ExternalSourceAspect.Kind.Element,
+                };
     }
     /**
      * Load the state of the active transformation from an open SQLiteDb
@@ -1172,17 +1760,35 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const lastProvenanceEntityInfo = db.withSqliteStatement(`SELECT entityId, aspectId, aspectVersion, aspectKind FROM ${IModelTransformer.lastProvenanceEntityInfoTable}`, (stmt) => {
             if (core_bentley_1.DbResult.BE_SQLITE_ROW !== stmt.step())
                 throw Error("expected row when getting lastProvenanceEntityId from target state table");
-            return {
-                entityId: stmt.getValueString(0),
-                aspectId: stmt.getValueString(1),
-                aspectVersion: stmt.getValueString(2),
-                aspectKind: stmt.getValueString(3),
-            };
+            const entityId = stmt.getValueString(0);
+            const isGuidOrGuidPair = entityId.includes("-");
+            return isGuidOrGuidPair
+                ? entityId
+                : {
+                    entityId,
+                    aspectId: stmt.getValueString(1),
+                    aspectVersion: stmt.getValueString(2),
+                    aspectKind: stmt.getValueString(3),
+                };
         });
-        const targetHasCorrectLastProvenance = 
-        // ignore provenance check if it's null since we can't bind those ids
-        !core_bentley_1.Id64.isValidId64(lastProvenanceEntityInfo.aspectId) ||
+        /*
+        // TODO: maybe save transformer state resumption state based on target changset and require calls
+        // to saveChanges
+        if () {
+          const [sourceFedGuid, targetFedGuid, relClassFullName] = lastProvenanceEntityInfo.split("/");
+          const isRelProvenance = targetFedGuid !== undefined;
+          const instanceId = isRelProvenance
+            ? this.targetDb.elements.getElement({federationGuid: sourceFedGuid})
+            : "";
+          //const classId =
+          if (isRelProvenance) {
+          }
+        }
+        */
+        const targetHasCorrectLastProvenance = typeof lastProvenanceEntityInfo === "string" ||
+            // ignore provenance check if it's null since we can't bind those ids
             !core_bentley_1.Id64.isValidId64(lastProvenanceEntityInfo.entityId) ||
+            !core_bentley_1.Id64.isValidId64(lastProvenanceEntityInfo.aspectId) ||
             this.provenanceDb.withPreparedStatement(`
         SELECT Version FROM ${core_backend_1.ExternalSourceAspect.classFullName}
         WHERE Scope.Id=:scopeId
@@ -1223,9 +1829,13 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         this.context.loadStateFromDb(db);
         this.importer.loadStateFromJson(state.importerState);
         this.exporter.loadStateFromJson(state.exporterState);
+        this._elementsWithExplicitlyTrackedProvenance = core_bentley_1.CompressedId64Set.decompressSet(state.explicitlyTrackedElements);
         this.loadAdditionalStateJson(state.additionalState);
     }
     /**
+     * @deprecated in 0.1.x, this is buggy, and it is now equivalently efficient to simply restart the transformation
+     * from the original changeset
+     *
      * Return a new transformer instance with the same remappings state as saved from a previous [[IModelTransformer.saveStateToFile]] call.
      * This allows you to "resume" an iModel transformation, you will have to call [[IModelTransformer.processChanges]]/[[IModelTransformer.processAll]]
      * again but the remapping state will cause already mapped elements to be skipped.
@@ -1270,6 +1880,7 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
         const jsonState = {
             transformerClass: this.constructor.name,
             options: this._options,
+            explicitlyTrackedElements: core_bentley_1.CompressedId64Set.compressSet(this._elementsWithExplicitlyTrackedProvenance),
             importerState: this.importer.saveStateToJson(),
             exporterState: this.exporter.saveStateToJson(),
             additionalState: this.getAdditionalStateJson(),
@@ -1279,8 +1890,9 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             throw Error("Failed to create the js state table in the state database");
         if (core_bentley_1.DbResult.BE_SQLITE_DONE !== db.executeSQL(`
       CREATE TABLE ${IModelTransformer.lastProvenanceEntityInfoTable} (
-        -- because we cannot bind the invalid id which we use for our null state, we actually store the id as a hex string
+        -- either the invalid id for null provenance state, federation guid (or pair for rels) of the entity, or a hex element id
         entityId TEXT,
+        -- the following are only valid if the above entityId is a hex id representation
         aspectId TEXT,
         aspectVersion TEXT,
         aspectKind TEXT
@@ -1294,16 +1906,20 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
                 throw Error("Failed to insert options into the state database");
         });
         db.withSqliteStatement(`INSERT INTO ${IModelTransformer.lastProvenanceEntityInfoTable} (entityId, aspectId, aspectVersion, aspectKind) VALUES (?,?,?,?)`, (stmt) => {
-            stmt.bindString(1, this._lastProvenanceEntityInfo.entityId);
-            stmt.bindString(2, this._lastProvenanceEntityInfo.aspectId);
-            stmt.bindString(3, this._lastProvenanceEntityInfo.aspectVersion);
-            stmt.bindString(4, this._lastProvenanceEntityInfo.aspectKind);
+            const lastProvenanceEntityInfo = this._lastProvenanceEntityInfo;
+            stmt.bindString(1, lastProvenanceEntityInfo?.entityId ?? this._lastProvenanceEntityInfo);
+            stmt.bindString(2, lastProvenanceEntityInfo?.aspectId ?? "");
+            stmt.bindString(3, lastProvenanceEntityInfo?.aspectVersion ?? "");
+            stmt.bindString(4, lastProvenanceEntityInfo?.aspectKind ?? "");
             if (core_bentley_1.DbResult.BE_SQLITE_DONE !== stmt.step())
                 throw Error("Failed to insert options into the state database");
         });
         db.saveChanges();
     }
     /**
+     * @deprecated in 0.1.x, this is buggy, and it is now equivalently efficient to simply restart the transformation
+     * from the original changeset
+     *
      * Save the state of the active transformation to a file path, if a file at the path already exists, it will be overwritten
      * This state can be used by [[IModelTransformer.resumeTransformation]] to resume a transformation from this point.
      * The serialization format is a custom sqlite database.
@@ -1325,25 +1941,54 @@ class IModelTransformer extends IModelExporter_1.IModelExportHandler {
             db.closeDb();
         }
     }
-    async processChanges(accessTokenOrArgs, startChangesetId) {
-        core_bentley_1.Logger.logTrace(loggerCategory, "processChanges()");
-        this.logSettings();
-        this.validateScopeProvenance();
-        const options = typeof accessTokenOrArgs === "string"
+    async processChanges(optionsOrAccessToken, startChangesetId) {
+        this._isSynchronization = true;
+        // FIXME: we used to validateScopeProvenance... does initing it cover that?
+        this.initScopeProvenance();
+        const args = typeof optionsOrAccessToken === "string"
             ? {
-                accessToken: accessTokenOrArgs,
-                startChangeset: startChangesetId ? { id: startChangesetId } : this.sourceDb.changeset,
-                changedInstanceIds: undefined,
+                accessToken: optionsOrAccessToken,
+                startChangeset: startChangesetId
+                    ? { id: startChangesetId }
+                    : { index: this._synchronizationVersion.index + 1 },
             }
-            : accessTokenOrArgs;
-        await this.initialize(options);
-        await this.exporter.exportChanges(options);
+            : optionsOrAccessToken;
+        this.logSettings();
+        await this.initialize(args);
+        // must wait for initialization of synchronization provenance data
+        await this.exporter.exportChanges(this.getExportInitOpts(args));
         await this.processDeferredElements(); // eslint-disable-line deprecation/deprecation
         if (this._options.optimizeGeometry)
             this.importer.optimizeGeometry(this._options.optimizeGeometry);
         this.importer.computeProjectExtents();
         this.finalizeTransformation();
     }
+    /** Changeset data must be initialized in order to build correct changeOptions.
+     * Call [[IModelTransformer.initialize]] for initialization of synchronization provenance data
+     */
+    getExportInitOpts(opts) {
+        if (!this._isSynchronization)
+            return {};
+        return {
+            accessToken: opts.accessToken,
+            ...this._changesetRanges
+                ? { changesetRanges: this._changesetRanges }
+                : opts.startChangeset
+                    ? { startChangeset: opts.startChangeset }
+                    : { startChangeset: { index: this._synchronizationVersion.index + 1 } },
+        };
+    }
+    /** Combine an array of source elements into a single target element.
+     * All source and target elements must be created before calling this method.
+     * The "combine" operation is a remap and no properties from the source elements will be exported into the target
+     * and provenance will be explicitly tracked by ExternalSourceAspects
+     */
+    combineElements(sourceElementIds, targetElementId) {
+        for (const elementId of sourceElementIds) {
+            this.context.remapElement(elementId, targetElementId);
+            this._elementsWithExplicitlyTrackedProvenance.add(elementId);
+        }
+    }
 }
 exports.IModelTransformer = IModelTransformer;
 /** @internal the name of the table where javascript state of the transformer is serialized in transformer state dumps */
@@ -1374,6 +2019,7 @@ class TemplateModelCloner extends IModelTransformer {
      * @returns The mapping of sourceElementIds from the template model to the instantiated targetElementIds in the targetDb in case further processing is required.
      */
     async placeTemplate3d(sourceTemplateModelId, targetModelId, placement) {
+        await this.initialize();
         this.context.remapElement(sourceTemplateModelId, targetModelId);
         this._transform3d = core_geometry_1.Transform.createOriginAndMatrix(placement.origin, placement.angles.toMatrix3d());
         this._sourceIdToTargetIdMap = new Map();
@@ -1394,6 +2040,7 @@ class TemplateModelCloner extends IModelTransformer {
      * @returns The mapping of sourceElementIds from the template model to the instantiated targetElementIds in the targetDb in case further processing is required.
      */
     async placeTemplate2d(sourceTemplateModelId, targetModelId, placement) {
+        await this.initialize();
         this.context.remapElement(sourceTemplateModelId, targetModelId);
         this._transform3d = core_geometry_1.Transform.createOriginAndMatrix(core_geometry_1.Point3d.createFrom(placement.origin), placement.rotation);
         this._sourceIdToTargetIdMap = new Map();
@@ -1430,16 +2077,12 @@ class TemplateModelCloner extends IModelTransformer {
         const targetElementProps = super.onTransformElement(sourceElement);
         targetElementProps.federationGuid = core_bentley_1.Guid.createValue(); // clone from template should create a new federationGuid
         targetElementProps.code = core_common_1.Code.createEmpty(); // clone from template should not maintain codes
-        if (sourceElement instanceof core_backend_1.GeometricElement3d) {
-            const placement = core_common_1.Placement3d.fromJSON(targetElementProps.placement);
-            if (placement.isValid) {
-                placement.multiplyTransform(this._transform3d);
-                targetElementProps.placement = placement;
-            }
-        }
-        else if (sourceElement instanceof core_backend_1.GeometricElement2d) {
-            const placement = core_common_1.Placement2d.fromJSON(targetElementProps.placement);
+        if (sourceElement instanceof core_backend_1.GeometricElement) {
+            const is3d = sourceElement instanceof core_backend_1.GeometricElement3d;
+            const placementClass = is3d ? core_common_1.Placement3d : core_common_1.Placement2d;
+            const placement = (placementClass).fromJSON(targetElementProps.placement);
             if (placement.isValid) {
+                nodeAssert(this._transform3d);
                 placement.multiplyTransform(this._transform3d);
                 targetElementProps.placement = placement;
             }
@@ -1449,4 +2092,17 @@ class TemplateModelCloner extends IModelTransformer {
     }
 }
 exports.TemplateModelCloner = TemplateModelCloner;
+function queryElemFedGuid(db, elemId) {
+    return db.withPreparedStatement(`
+    SELECT FederationGuid
+    FROM bis.Element
+    WHERE ECInstanceId=?
+  `, (stmt) => {
+        stmt.bindId(1, elemId);
+        (0, core_bentley_1.assert)(stmt.step() === core_bentley_1.DbResult.BE_SQLITE_ROW);
+        const result = stmt.getValue(0).getGuid();
+        (0, core_bentley_1.assert)(stmt.step() === core_bentley_1.DbResult.BE_SQLITE_DONE);
+        return result;
+    });
+}
 //# sourceMappingURL=IModelTransformer.js.map