@peers-app/peers-device 0.7.25 → 0.7.27

package/dist/tracked-data-source.js

@@ -1,20 +1,32 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TrackedDataSource = void 0;
-const lodash_1 = require("lodash");
 const peers_sdk_1 = require("@peers-app/peers-sdk");
+const lodash_1 = require("lodash");
 const json_diff_1 = require("./json-diff");
+/**
+ * TrackedDataSource wraps a data source and tracks all changes using granular change records.
+ *
+ * Key improvements over V1:
+ * - Each JSON patch operation gets its own change record
+ * - Superseding logic prevents temporal inconsistencies
+ * - More efficient cleanup via supersededAt timestamps
+ */
 class TrackedDataSource {
     dataSource;
     changeTrackingTable;
     tableName;
     primaryKeyName;
+    preserveHistory = false;
     constructor(dataSource, changeTrackingTable) {
         this.dataSource = dataSource;
         this.changeTrackingTable = changeTrackingTable;
         this.tableName = dataSource.tableName;
         this.primaryKeyName = dataSource.primaryKeyName;
     }
+    //================================================================================================
+    // Change Tracking Methods
+    //================================================================================================
     async listChanges(filter = {}, opts) {
         filter = { $and: [{ tableName: this.tableName }, filter] };
         return this.changeTrackingTable.list(filter, opts);
@@ -23,145 +35,32 @@ class TrackedDataSource {
         filter = { $and: [{ tableName: this.tableName }, filter] };
         return this.changeTrackingTable.cursor(filter, opts);
     }
-    applyChangesPromise = Promise.resolve();
-    async applyChanges(changes) {
-        this.applyChangesPromise = this.applyChangesPromise.finally(() => this._applyChanges(changes));
-        return this.applyChangesPromise;
-    }
-    async _applyChanges(changes) {
-        changes = (0, lodash_1.uniqBy)(changes, 'changeId');
-        const oldestTimestamp = Math.min(...changes.map(c => c.timestamp));
-        const recordIds = (0, lodash_1.uniq)(changes.map(c => c.recordId)).sort();
-        const existingChanges = await this.listChanges({ timestamp: { $gte: oldestTimestamp }, recordId: { $in: recordIds } });
-        if (existingChanges.length > 10000) {
-            console.warn(`Merging a very large number of changes (${existingChanges.length}) for table ${this.tableName}. This can cause severe performance degradation or system down.  This indicates there may be a small number of records with a very large number of changes.  Use full record writes (snapshot or delete) to clear out excessive partial changes.`, { tableName: this.tableName, recordIds, });
-        }
-        const newChanges = changes.filter(c => !existingChanges.find(ec => ec.changeId === c.changeId));
-        if (newChanges.length === 0)
-            return;
-        const allChanges = (0, lodash_1.sortBy)([...newChanges, ...existingChanges], 'timestamp');
-        const recordIdsNotBeingDeletedOrRestored = (0, lodash_1.uniq)(allChanges.filter(c => c.changeType !== 'delete' && c.changeType !== 'restore').map(c => c.recordId));
-        const deletedRecordIds = await this.filterToDeletedIds(recordIdsNotBeingDeletedOrRestored);
-        const existingRecordIds = recordIds.filter(r => !deletedRecordIds.includes(r));
-        const existingRecords = await this.dataSource.list({ [this.primaryKeyName]: { $in: existingRecordIds } });
-        const groupedChanges = (0, lodash_1.groupBy)(allChanges, c => c.recordId);
-        let changesApplied = false;
-        for (const recordId of Object.keys(groupedChanges)) {
-            const changes = groupedChanges[recordId];
-            let record = existingRecords.find(r => r[this.primaryKeyName] === recordId);
-            let deletedRecord = deletedRecordIds.includes(recordId) ? { [this.primaryKeyName]: recordId } : undefined;
-            const lastFullWriteTimestamp = (0, lodash_1.findLast)(changes, c => c.changeType !== 'update')?.timestamp || 0;
-            for (const change of changes) {
-                // if the change is older than the last full write, skip it because even if we don't have it, it will immediately be deleted
-                if (change.timestamp < lastFullWriteTimestamp) {
-                    continue;
-                }
-                switch (change.changeType) {
-                    case 'insert':
-                    case 'snapshot':
-                        if (!deletedRecord) {
-                            changesApplied = true;
-                            record = change.newRecord;
-                        }
-                        break;
-                    case 'update':
-                        if (!deletedRecord) {
-                            changesApplied = true;
-                            if (record) {
-                                record = (0, json_diff_1.applyJsonDiff)(record, change.jsonDiff);
-                            }
-                            else {
-                                /* istanbul ignore next */
-                                record = change.newRecord;
-                            }
-                        }
-                        break;
-                    case 'delete':
-                        changesApplied = true;
-                        deletedRecord = (change.oldRecord || record || { [this.primaryKeyName]: recordId });
-                        record = undefined;
-                        break;
-                    case 'restore':
-                        changesApplied = true;
-                        deletedRecord = undefined;
-                        record = change.newRecord;
-                        break;
-                    default:
-                        throw new Error(`Unsupported change type: ${JSON.stringify(change, null, 2)}`);
-                }
-                try {
-                    if (newChanges.find(ec => ec.changeId === change.changeId)) {
-                        change.timestampApplied = (0, peers_sdk_1.getTimestamp)();
-                        await this.changeTrackingTable.insert(change);
-                        this.changeTrackingTable.dataChangedEmitter.emit(change);
-                    }
-                }
-                catch (err) {
-                    if (String(err).includes('UNIQUE constraint failed')) {
-                        console.warn(`Change already exists in table (are you using the same changeTracking table for syncing tables with different names?): ${change.changeId}: ` + JSON.stringify(change));
-                    }
-                    else if (String(err).includes('Validation on insert failed ')) {
-                        console.warn(`Validation on insert failed. Continuing with the change applied but not saved - ${change.changeId}: ` + JSON.stringify(change));
-                    }
-                    else {
-                        throw err;
-                    }
-                }
-            }
-            if (lastFullWriteTimestamp > 0) {
-                await this.changeTrackingTable.db.exec(`
-          DELETE FROM "${this.changeTrackingTable.tableName}" 
-          WHERE
-            tableName = ? AND   
-            recordId = ? AND 
-            timestamp < ?
-            `, [this.tableName, recordId, lastFullWriteTimestamp]);
-            }
-            if (changesApplied) {
-                try {
-                    // TODO if this fails the data will be out of sync and won't be back in sync until the orphaned changes are fully overwritten
-                    if (record) {
-                        await this.dataSource.save(record);
-                    }
-                    else if (deletedRecord) {
-                        await this.dataSource.delete(deletedRecord || recordId);
-                    }
-                    else {
-                        throw new Error(`record and deletedRecord are both undefined. This should never happen.`);
-                    }
-                }
-                catch (err) {
-                    console.error(`Error committing changes for recordId: ${recordId}`, err);
-                }
-            }
-        }
-    }
-    async filterToDeletedIds(recordIds) {
-        return this.changeTrackingTable.filterToDeletedIds(recordIds);
-    }
+    //================================================================================================
+    // Write Operations
+    //================================================================================================
     async save(data, opts) {
         const primaryKeyName = this.primaryKeyName;
         const recordId = data[primaryKeyName];
         if (!recordId) {
-            return this.insert(data);
+            return this.insert(data, opts);
         }
         const oldData = await this.dataSource.get(data[primaryKeyName]);
         if (oldData) {
             return this._update(data, oldData, opts?.saveAsSnapshot);
         }
-        const deletedIds = await this.filterToDeletedIds([recordId]);
+        const deletedIds = await this.changeTrackingTable.filterToDeletedIds([recordId]);
         if (deletedIds.length) {
             if (opts?.restoreIfDeleted) {
                 return this.restore(data);
             }
-            throw new Error(`Cannot save record with id ${recordId} because it has been deleted.  Use restore instead.`);
+            throw new Error(`Cannot save record with id ${recordId} because it has been deleted. Use restore instead.`);
         }
-        return this._insert(data, true);
+        return this._insert(data, true, opts);
     }
-    async insert(data) {
-        return this._insert(data);
+    async insert(data, opts) {
+        return this._insert(data, false, opts);
     }
-    async _insert(data, skipDeletedCheck = false) {
+    async _insert(data, skipDeletedCheck = false, opts) {
         let recordId = data[this.primaryKeyName];
         if (!recordId) {
             recordId = (0, peers_sdk_1.newid)();
@@ -169,24 +68,29 @@ class TrackedDataSource {
             data[this.primaryKeyName] = recordId;
         }
         else if (!skipDeletedCheck) {
-            const deletedIds = await this.filterToDeletedIds([recordId]);
+            const deletedIds = await this.changeTrackingTable.filterToDeletedIds([recordId]);
             if (deletedIds.length) {
-                throw new Error(`Cannot insert record with id ${recordId} because it has been deleted.  Use restore instead.`);
+                throw new Error(`Cannot insert record with id ${recordId} because it has been deleted. Use restore instead.`);
             }
         }
-        const timestamp = (0, peers_sdk_1.getTimestamp)();
-        const change = await this.changeTrackingTable.insert({
+        // Use weakInsert timestamp (1 + Math.random()) if weakInsert is true, otherwise use normal timestamp
+        const timestamp = opts?.weakInsert ? 1 + Math.random() : (0, peers_sdk_1.getTimestamp)();
+        const insertedData = await this.dataSource.insert(data);
+        // Store a single 'add' operation at path "/" with the full object
+        // This is more efficient than storing individual field operations
+        const change = {
             changeId: (0, peers_sdk_1.newid)(),
-            changeType: 'insert',
-            timestamp,
-            timestampApplied: timestamp,
             tableName: this.tableName,
             recordId,
-            newRecord: data,
-        });
+            op: 'set',
+            path: '/', // Root path for full object
+            value: data,
+            createdAt: timestamp,
+            appliedAt: timestamp,
+        };
+        await this.changeTrackingTable.insert(change);
         this.changeTrackingTable.dataChangedEmitter.emit(change);
-        return this.dataSource.insert(data);
-        // TODO handle errors if insert fails
+        return insertedData;
     }
     async update(newData) {
         return this._update(newData);
@@ -200,62 +104,103 @@ class TrackedDataSource {
         if (!oldData) {
             throw new Error(`No record found to update: ${recordId}: ` + JSON.stringify(newData));
         }
-        const jsonDiff = (0, json_diff_1.createJsonDiff)(oldData, newData);
-        if (!jsonDiff) {
-            return newData;
-        }
         const timestamp = (0, peers_sdk_1.getTimestamp)();
-        const change = await this.changeTrackingTable.insert({
-            changeId: (0, peers_sdk_1.newid)(),
-            changeType: saveAsSnapshot ? 'snapshot' : 'update',
-            timestamp,
-            timestampApplied: timestamp,
-            tableName: this.tableName,
-            recordId,
-            newRecord: newData,
-            jsonDiff,
-        });
+        const updatedObject = await this.dataSource.update(newData);
         if (saveAsSnapshot) {
-            // cleans up any old changes since they are all overshadowed by the snapshot
-            await this.changeTrackingTable.db.exec(`
-        DELETE FROM "${this.changeTrackingTable.tableName}" 
-        WHERE
-          tableName = ? AND   
-          recordId = ? AND 
-          timestamp < ?
-          `, [this.tableName, recordId, timestamp]);
+            // If saving as snapshot, we store a single 'set' operation at path "/" with the full object
+            const change = await this.changeTrackingTable.insert({
+                changeId: (0, peers_sdk_1.newid)(),
+                tableName: this.tableName,
+                recordId,
+                op: 'set',
+                path: '/', // Root path for full object
+                value: updatedObject,
+                createdAt: timestamp,
+                appliedAt: timestamp,
+            });
+            this.changeTrackingTable.dataChangedEmitter.emit(change);
+            await this.changeTrackingTable.markAllPriorChangesSuperseded(this.tableName, recordId, timestamp);
         }
-        this.changeTrackingTable.dataChangedEmitter.emit(change);
-        return this.dataSource.update(newData);
-        // TODO handle errors if update fails
+        else {
+            const jsonDiff = (0, json_diff_1.createJsonDiff)(oldData, updatedObject);
+            if (jsonDiff) {
+                // Store each operation as a separate change record
+                const changes = [];
+                for (let i = 0; i < jsonDiff.length; i++) {
+                    const op = jsonDiff[i];
+                    const changeOp = op.op === 'patch-text' ? 'patch-text'
+                        : op.op === 'remove' ? 'delete'
+                            : 'set';
+                    const changePath = op.path || '/';
+                    const value = op.op === 'add' || op.op === 'replace' || op.op === 'patch-text' ? op.value : undefined;
+                    const change = {
+                        changeId: (0, peers_sdk_1.newid)(),
+                        tableName: this.tableName,
+                        recordId,
+                        op: changeOp,
+                        path: changePath,
+                        value,
+                        createdAt: timestamp, // timestamp should be the same for all changes
+                        appliedAt: timestamp,
+                    };
+                    changes.push(change);
+                }
+                // get all active changes for these records
+                const activeExistingChanges = await this.changeTrackingTable.list({
+                    tableName: this.tableName,
+                    recordId,
+                    supersededAt: { $exists: false },
+                }, { sortBy: ['createdAt'] });
+                const supersededChangeIds = [];
+                for (const existingChange of activeExistingChanges) {
+                    const superseded = changes.some(newChange => (0, json_diff_1.isChildPathOrSame)(newChange.path, existingChange.path));
+                    if (superseded) {
+                        supersededChangeIds.push(existingChange.changeId);
+                    }
+                }
+                // TODO make this a bulk insert
+                for (const change of changes) {
+                    await this.changeTrackingTable.insert(change);
+                    this.changeTrackingTable.dataChangedEmitter.emit(change);
+                }
+                if (supersededChangeIds.length > 0) {
+                    await this.changeTrackingTable.batchMarkSuperseded(supersededChangeIds.map(changeId => ({
+                        changeId,
+                        supersededAt: timestamp,
+                    })));
+                }
+            }
+        }
+        return updatedObject;
     }
     async delete(data) {
         const primaryKeyName = this.primaryKeyName;
-        let oldData = data;
         let recordId = '';
         if (typeof data === 'string') {
             recordId = data;
-            oldData = (await this.dataSource.get(data));
-            if (!oldData) {
-                console.warn(`Record not found: ${data}.  Possibly already deleted or was never created.`);
-            }
         }
         else {
             recordId = data[primaryKeyName];
         }
         const timestamp = (0, peers_sdk_1.getTimestamp)();
-        const change = await this.changeTrackingTable.insert({
+        // we write this to the db first in case there are errors
+        await this.dataSource.delete(data);
+        // A delete operation supersedes all prior changes for this record
+        // Store a 'remove' operation
+        const change = {
             changeId: (0, peers_sdk_1.newid)(),
-            changeType: 'delete',
-            timestamp,
-            timestampApplied: timestamp,
             tableName: this.tableName,
             recordId,
-            oldRecord: oldData
-        });
+            op: 'delete',
+            path: '/', // Root path for delete
+            value: undefined,
+            createdAt: timestamp,
+            appliedAt: timestamp,
+        };
+        await this.changeTrackingTable.insert(change);
         this.changeTrackingTable.dataChangedEmitter.emit(change);
-        await this.dataSource.delete(data);
-        // TODO handle errors if the delete fails
+        // Mark all old changes as superseded
+        await this.changeTrackingTable.markAllPriorChangesSuperseded(this.tableName, recordId, timestamp);
     }
     async restore(data) {
         let recordId = data[this.primaryKeyName];
@@ -263,76 +208,251 @@ class TrackedDataSource {
             throw new Error(`No recordId provided to restore: ` + JSON.stringify(data));
         }
         const timestamp = (0, peers_sdk_1.getTimestamp)();
-        const change = await this.changeTrackingTable.insert({
+        // we write this to the db first in case there are errors
+        const insertedData = await this.dataSource.insert(data);
+        // Store a single 'add' operation at path "/" with the full object
+        // Using 'add' since we're restoring (adding back) the record
+        const change = {
             changeId: (0, peers_sdk_1.newid)(),
-            changeType: 'restore',
-            timestamp,
-            timestampApplied: timestamp,
             tableName: this.tableName,
             recordId,
-            newRecord: data,
-        });
+            op: 'set',
+            path: '/', // Root path for full object
+            value: insertedData,
+            createdAt: timestamp,
+            appliedAt: timestamp,
+        };
+        await this.changeTrackingTable.insert(change);
         this.changeTrackingTable.dataChangedEmitter.emit(change);
-        return this.dataSource.insert(data);
-        // TODO handle errors if restore fails
+        // Mark all old changes as superseded
+        await this.changeTrackingTable.markAllPriorChangesSuperseded(this.tableName, recordId, timestamp);
+        return insertedData;
     }
-    /**
-     * WARNING: This isn't well tested.
-     */
-    async compact(beforeTimestamp) {
-        if (!beforeTimestamp) {
-            const twoWeeksAgo = new Date();
-            twoWeeksAgo.setDate(twoWeeksAgo.getDate() - 14);
-            beforeTimestamp = twoWeeksAgo.getTime();
+    //================================================================================================
+    // Apply Changes (Sync)
+    //================================================================================================
+    applyChangesPromise = Promise.resolve();
+    async applyChanges(changes) {
+        this.applyChangesPromise = this.applyChangesPromise.finally(() => this._applyChanges(changes));
+        return this.applyChangesPromise;
+    }
+    async _applyChanges(changes) {
+        if (changes.length === 0) {
+            return;
         }
-        for (let i = 0; i < 100; i++) {
-            const recordIdsWithOldUpdates = await this.changeTrackingTable.list({
-                timestamp: { $lt: beforeTimestamp },
-                tableName: this.tableName,
-                changeType: 'update',
-            }, { pageSize: 100 });
-            if (!recordIdsWithOldUpdates.length) {
-                return;
+        changes = (0, lodash_1.uniqBy)(changes, c => c.changeId);
+        const missingChangeIds = await this.changeTrackingTable.filterToMissingChangeIds(changes.map(c => c.changeId));
+        if (missingChangeIds.length === 0) {
+            // All changes are already applied
+            return;
+        }
+        const newChanges = changes.filter(c => missingChangeIds.some(id => id === c.changeId));
+        // Get affected recordIds (sorted by order in which they were changed)
+        const recordIds = (0, lodash_1.chain)(newChanges)
+            // .sortBy(c => c.changeId) // this is relatively expensive and not needed
+            .map(c => c.recordId)
+            .uniq()
+            .value();
+        // get all active changes for these records
+        const activeExistingChanges = await this.changeTrackingTable.list({
+            tableName: this.tableName,
+            recordId: { $in: recordIds },
+            supersededAt: { $exists: false },
+        }, { sortBy: ['createdAt'] });
+        // Process changes by record
+        const allChanges = (0, lodash_1.chain)([...activeExistingChanges, ...newChanges])
+            .uniqBy("changeId")
+            .value();
+        // write superseded changes to db (if preserving history)
+        const newSupersededChanges = newChanges.filter(c => c.supersededAt);
+        if (this.preserveHistory && newSupersededChanges.length) {
+            for (const newSupersededChange of newSupersededChanges) {
+                try {
+                    await this.changeTrackingTable.insert(newSupersededChange);
+                }
+                catch (e) {
+                    console.warn(`Skipping invalid superseded change for recordId: ${newSupersededChange.recordId}, changeId: ${newSupersededChange.changeId}`, e);
+                }
             }
-            const recordIds = (0, lodash_1.uniq)(recordIdsWithOldUpdates.map(c => c.recordId));
-            for (const recordId of recordIds) {
-                const newestOldChanges = await this.changeTrackingTable.list({
-                    timestamp: { $lt: beforeTimestamp },
-                    changeType: { $in: ['update', 'insert'] },
-                    tableName: this.tableName,
-                    recordId,
-                }, { pageSize: 1, sortBy: ['-timestamp'] });
-                // }, { pageSize: 3, sortBy: ['-timestamp'] });
-                // if (newestOldChanges.length < 3) {
-                //   // if there are less than 3 changes, don't bother, it's not worth the network noise
-                //   continue;
-                // }
-                const newestOldChange = newestOldChanges[0];
-                if (!newestOldChange) {
-                    console.warn(`No changes found for recordId: ${recordId}`);
+        }
+        const newActiveChanges = newChanges.filter(c => !c.supersededAt);
+        if (newActiveChanges.length === 0) {
+            // No new active changes to apply
+            return;
+        }
+        const allActiveChanges = allChanges.filter(c => !c.supersededAt);
+        const batchTimestamp = (0, peers_sdk_1.getTimestamp)();
+        const groupedActiveChanges = (0, lodash_1.groupBy)(allActiveChanges, "recordId");
+        const supersededChanges = [];
+        for (const recordId of Object.keys(groupedActiveChanges)) {
+            // changes for this record sorted in reverse order (newest first) for superseding logic
+            const recordChangesActiveReverse = (0, lodash_1.sortBy)(groupedActiveChanges[recordId], ['createdAt', 'changeId']).reverse();
+            // get the last full write for this record (which is the first found since we're in reverse order)
+            const lastFullWrite = recordChangesActiveReverse.find(c => c.path === '/');
+            if (!lastFullWrite) {
+                console.error(`No full write found for recordId: ${recordId}, skipping changes`);
+                continue;
+            }
+            let finalRecord = undefined;
+            let isDeleted = lastFullWrite.op === 'delete';
+            if (isDeleted) {
+                // mark new and existing changes as superseded except the last full write
+                for (const c of recordChangesActiveReverse) {
+                    if (c !== lastFullWrite) {
+                        supersededChanges.push(c);
+                    }
+                }
+                if (newActiveChanges.includes(lastFullWrite)) {
+                    // if this delete is a new change, we need to apply it to the data source
+                    await this.dataSource.delete(recordId);
+                }
+                continue;
+            }
+            else {
+                finalRecord = lastFullWrite.value;
+            }
+            if (!finalRecord) {
+                console.error(`Last full write has no value for recordId: ${recordId}, skipping changes`);
+                continue;
+            }
+            // Apply changes in reverse order to detect superseding
+            const writtenPaths = [];
+            const writtenChanges = [];
+            for (const change of recordChangesActiveReverse) {
+                // validate the change before applying
+                if (newActiveChanges.includes(change)) {
+                    try {
+                        this.changeTrackingTable.schema.parse(change);
+                    }
+                    catch (e) {
+                        console.warn(`Skipping invalid change for recordId: ${recordId}, changeId: ${change.changeId}`, e);
+                        newActiveChanges.splice(newActiveChanges.indexOf(change), 1);
+                        continue;
+                    }
+                }
+                const superseded = writtenPaths.some(parentPath => (0, json_diff_1.isChildPathOrSame)(parentPath, change.path));
+                if (superseded) {
+                    supersededChanges.push(change);
                     continue;
                 }
-                const snapshot = {
-                    changeId: (0, peers_sdk_1.newid)(),
-                    changeType: 'snapshot',
-                    timestamp: newestOldChange.timestamp + 0.001,
-                    timestampApplied: (0, peers_sdk_1.getTimestamp)(),
-                    tableName: this.tableName,
-                    recordId,
-                    newRecord: newestOldChange.newRecord,
+                writtenPaths.push(change.path);
+                if (change !== lastFullWrite) {
+                    writtenChanges.push(change);
+                }
+            }
+            // convert changes to diff ops
+            const changesToApply = (0, lodash_1.chain)(writtenChanges)
+                .sortBy(['createdAt', 'changeId'])
+                .map(c => {
+                const diffOp = c.op === 'delete' ? 'remove'
+                    : c.op === 'set' ? 'add'
+                        : c.op;
+                const diff = {
+                    op: diffOp,
+                    path: c.path,
+                    value: c.value,
                 };
-                await this.applyChanges([snapshot]);
+                return diff;
+            })
+                .value();
+            if (changesToApply.length) {
+                finalRecord = (0, json_diff_1.applyJsonDiff)(finalRecord, changesToApply);
+            }
+            if (!finalRecord) {
+                console.error(`Failed to apply changes for recordId: ${recordId}, skipping`);
+                continue;
+            }
+            // Save the final record state first to ensure the underlying dataSource accepts it
+            // TODO if an error is thrown we shouldn't save the changes or update superseded status
+            try {
+                await this.dataSource.save(finalRecord);
+            }
+            catch (e) {
+                // TODO we could improve this by trying to find which change caused the failure and just skip that one
+                console.warn(`Failed to save final record for recordId: ${recordId}, skipping changes`, e);
+                for (const change of writtenChanges) {
+                    if (newActiveChanges.includes(change)) {
+                        newActiveChanges.splice(newActiveChanges.indexOf(change), 1);
+                    }
+                }
+            }
+        }
+        // Now save all new changes and mark superseded changes
+        // TODO: make this a bulk insert
+        for (const change of newActiveChanges) {
+            change.appliedAt = batchTimestamp;
+            await this.changeTrackingTable.insert(change).catch(e => {
+                if (e.code === 'SQLITE_CONSTRAINT_PRIMARYKEY') {
+                    console.warn(`Attempted to insert a change that already exists for recordId: ${change.recordId}, changeId: ${change.changeId}`);
+                    return;
+                }
+                throw new Error(`Failed to save change for recordId: ${change.recordId}, changeId: ${change.changeId}, state corrupted: ${e.message}`, { cause: e });
+            });
+            if (!supersededChanges.includes(change)) {
+                this.changeTrackingTable.dataChangedEmitter.emit(change);
+            }
+        }
+        // NOTE: if something interrupts this, it will will just do it the next time applyChanges is called
+        //       because the superseding flag is just an optimization    
+        if (supersededChanges.length > 0) {
+            if (this.preserveHistory) {
+                // update superseded changes in place
+                await this.changeTrackingTable.batchMarkSuperseded(supersededChanges.map(change => ({
+                    changeId: change.changeId,
+                    supersededAt: change.supersededAt || batchTimestamp,
+                })));
             }
+            else {
+                // delete superseded changes
+                await this.changeTrackingTable.deleteChanges(supersededChanges.map(change => change.changeId));
+                await this.changeTrackingTable.deleteSupersededChangesOlderThan(this.tableName, batchTimestamp);
+            }
+        }
+    }
+    //================================================================================================
+    // Read Operations (pass-through to underlying data source)
+    //================================================================================================
+    async get(id) {
+        return this.dataSource.get(id);
+    }
+    async list(filter = {}, opts = {}) {
+        return this.dataSource.list(filter, opts);
+    }
+    async count(filter = {}) {
+        if (this.dataSource.count) {
+            return this.dataSource.count(filter);
+        }
+        console.warn(`count not implemented for data source: ${this.dataSource.constructor.name}, falling back to cursor`);
+        let cnt = 0;
+        const cursor = this.dataSource.cursor?.(filter) ?? (0, peers_sdk_1.dataSourceCursor)(this.dataSource, filter);
+        for await (const _ of cursor) {
+            cnt++;
         }
-        console.warn(`Compaction failed to complete in 100 iterations.  Try running it again to see if it eventually finishes.`);
+        return cnt;
     }
+    cursor(filter = {}, opts = {}) {
+        if (this.dataSource.cursor) {
+            return this.dataSource.cursor(filter, opts);
+        }
+        return (0, peers_sdk_1.dataSourceCursor)(this.dataSource, filter, opts);
+    }
+    //================================================================================================
+    // Utility Methods
+    //================================================================================================
     /**
-     * WARNING: This iterates through all records in the data source to check for missing records in the change tracking table.
-     * This can be slow for large data sources, so use with caution.
-     * This could be improved by using a SQL query somewhere with direct access to the database.
-     * Also, under normal operation this should not be necessary, as the change tracking table should always be in sync with the data source.
+     * Find all records in the data source that don't have change tracking records,
+     * and create initial change records for them.
+     *
+     * This is useful for:
+     * - Initializing change tracking for existing data
+     * - Recovery after change tracking table corruption
+     * - Migration scenarios
+     *
+     * @param batchSize Number of records to process at once (default: 100)
      */
     async findAndTrackRecords(batchSize = 100) {
+        // TEMPORARY: Take this out
+        await this.changeTrackingTable.db.exec('DROP TABLE IF EXISTS ChangeTracking');
         const cursor = this.cursor();
         const nextBatch = async () => {
             const batch = [];
@@ -350,49 +470,53 @@ class TrackedDataSource {
             const recordIds = batch.map(b => b[this.primaryKeyName]);
             const missingRecordIds = await this.changeTrackingTable.filterToMissingIds(recordIds);
             for (const recordId of missingRecordIds) {
+                const record = batch.find(b => b[this.primaryKeyName] === recordId);
+                if (!record)
+                    continue;
+                // Create a change record with very low timestamp so it gets superseded by any real changes
+                // Use random decimal as tie-breaker between multiple snapshots
+                const lowTimestamp = 1 + Math.random();
+                const now = (0, peers_sdk_1.getTimestamp)();
                 await this.changeTrackingTable.insert({
                     changeId: (0, peers_sdk_1.newid)(),
-                    changeType: 'snapshot',
-                    timestamp: 1, // keep this low so it gets overwritten by any real changes
-                    timestampApplied: (0, peers_sdk_1.getTimestamp)(),
                     tableName: this.tableName,
                     recordId,
-                    newRecord: batch.find(b => b[this.primaryKeyName] === recordId),
+                    op: 'set',
+                    path: '/',
+                    value: record,
+                    createdAt: lowTimestamp,
+                    appliedAt: now,
                 });
                 newlyTrackedRecords++;
             }
             batch = await nextBatch();
         }
-        if (newlyTrackedRecords) {
-            console.log(`Found ${newlyTrackedRecords} new records to track in table: ${this.tableName}`);
-        }
-    }
-    //================================================================================================
-    // reads don't get tracked
-    //================================================================================================
-    async get(id) {
-        return this.dataSource.get(id);
-    }
-    async list(filter = {}, opts = {}) {
-        return this.dataSource.list(filter, opts);
-    }
-    async count(filter = {}) {
-        if (this.dataSource.count) {
-            return this.dataSource.count(filter);
-        }
-        console.warn(`count not implemented for data source: ${this.dataSource.constructor.name}, falling back to cursor`);
-        let cnt = 0;
-        const cursor = this.dataSource.cursor?.(filter) ?? (0, peers_sdk_1.dataSourceCursor)(this.dataSource, filter);
-        for await (const _ of cursor) {
-            cnt++;
+        if (newlyTrackedRecords > 0) {
+            console.log(`Tracked ${newlyTrackedRecords} previously untracked records in table ${this.tableName}`);
         }
-        return cnt;
     }
-    cursor(filter = {}, opts = {}) {
-        if (this.dataSource.cursor) {
-            return this.dataSource.cursor(filter, opts);
+    /**
+     * Remove superseded change records older than the given timestamp.
+     *
+     * This helps keep the change tracking table from growing indefinitely by cleaning up
+     * changes that have been superseded and are no longer needed for synchronization.
+     *
+     * Only removes changes where supersededAt is set AND supersededAt < beforeTimestamp.
+     *
+     * @param beforeTimestamp Remove changes superseded before this timestamp (default: two weeks ago)
+     */
+    async compact(beforeTimestamp) {
+        if (!beforeTimestamp) {
+            if (this.preserveHistory) {
+                // Default to two weeks ago
+                beforeTimestamp = Date.now() - (14 * 24 * 60 * 60 * 1000);
+            }
+            else {
+                beforeTimestamp = Date.now();
+            }
         }
-        return (0, peers_sdk_1.dataSourceCursor)(this.dataSource, filter, opts);
+        // Delete all superseded changes for this table in a single SQL query
+        await this.changeTrackingTable.deleteSupersededChangesOlderThan(this.tableName, beforeTimestamp);
     }
 }
 exports.TrackedDataSource = TrackedDataSource;