@powersync/service-module-mongodb 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219110735

package/CHANGELOG.md

@@ -1,12 +1,77 @@
 # @powersync/service-module-mongodb
 
-## 0.0.0-dev-20241128134723
+## 0.0.0-dev-20241219110735
+
+### Minor Changes
+
+- 697d44b: Moved MongoDB sync bucket storage implementation to the MongoDB module.
+
+### Patch Changes
+
+- Updated dependencies [697d44b]
+- Updated dependencies [697d44b]
+- Updated dependencies [a66be3b]
+- Updated dependencies [697d44b]
+  - @powersync/service-core@0.0.0-dev-20241219110735
+  - @powersync/lib-services-framework@0.0.0-dev-20241219110735
+  - @powersync/service-sync-rules@0.0.0-dev-20241219110735
+  - @powersync/service-types@0.0.0-dev-20241219110735
+
+## 0.1.7
+
+### Patch Changes
+
+- Updated dependencies [320e646]
+- Updated dependencies [e3a9343]
+  - @powersync/service-core@0.12.2
+
+## 0.1.6
+
+### Patch Changes
+
+- 2043447: Use short timeout for testing mongodb connections.
+
+## 0.1.5
+
+### Patch Changes
+
+- 889ac46: Fix "BSONObj size is invalid" error during replication.
+- Updated dependencies [889ac46]
+  - @powersync/service-core@0.12.1
+
+## 0.1.4
+
+### Patch Changes
+
+- Updated dependencies [ebc62ff]
+  - @powersync/service-core@0.12.0
+  - @powersync/service-types@0.5.0
+
+## 0.1.3
+
+### Patch Changes
+
+- Updated dependencies [62e97f3]
+- Updated dependencies [a235c9f]
+- Updated dependencies [8c6ce90]
+  - @powersync/service-core@0.11.0
+  - @powersync/service-sync-rules@0.22.0
+
+## 0.1.2
+
+### Patch Changes
+
+- Updated dependencies [2a4f020]
+  - @powersync/service-core@0.10.1
+
+## 0.1.1
 
 ### Patch Changes
 
-- Updated dependencies [74548e4]
-  - @powersync/service-core@0.0.0-dev-20241128134723
-  - @powersync/service-types@0.0.0-dev-20241128134723
+- Updated dependencies [2c18ad2]
+- Updated dependencies [35c267f]
+  - @powersync/service-core@0.10.0
+  - @powersync/service-types@0.4.0
 
 ## 0.1.0
 

package/dist/db/db-index.d.ts

@@ -0,0 +1 @@
+export * as mongo from './mongo.js';

package/dist/db/db-index.js

@@ -0,0 +1,2 @@
+export * as mongo from './mongo.js';
+//# sourceMappingURL=db-index.js.map

package/dist/db/db-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db-index.js","sourceRoot":"","sources":["../../src/db/db-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,YAAY,CAAC"}

package/dist/db/mongo.d.ts

@@ -0,0 +1,35 @@
+import * as mongo from 'mongodb';
+import { configFile } from '@powersync/service-types';
+/**
+ * Time for new connection to timeout.
+ */
+export declare const MONGO_CONNECT_TIMEOUT_MS = 10000;
+/**
+ * Time for individual requests to timeout the socket.
+ */
+export declare const MONGO_SOCKET_TIMEOUT_MS = 60000;
+/**
+ * Time for individual requests to timeout the operation.
+ *
+ * This is time spent on the cursor, not total time.
+ *
+ * Must be less than MONGO_SOCKET_TIMEOUT_MS to ensure proper error handling.
+ */
+export declare const MONGO_OPERATION_TIMEOUT_MS = 30000;
+/**
+ * Same as above, but specifically for clear operations.
+ *
+ * These are retried when reaching the timeout.
+ */
+export declare const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5000;
+export declare function createMongoClient(config: configFile.MongoStorageConfig): mongo.MongoClient;
+/**
+ * Wait up to a minute for authentication errors to resolve.
+ *
+ * There can be a delay between an Atlas user being created, and that user being
+ * available on the database cluster. This works around it.
+ *
+ * This is specifically relevant for migrations and teardown - other parts of the stack
+ * can generate handle these failures and just retry or restart.
+ */
+export declare function waitForAuth(db: mongo.Db): Promise<void>;

package/dist/db/mongo.js

@@ -0,0 +1,73 @@
+import * as mongo from 'mongodb';
+import * as timers from 'timers/promises';
+import { normalizeMongoConfig } from '../storage/storage-index.js';
+/**
+ * Time for new connection to timeout.
+ */
+export const MONGO_CONNECT_TIMEOUT_MS = 10000;
+/**
+ * Time for individual requests to timeout the socket.
+ */
+export const MONGO_SOCKET_TIMEOUT_MS = 60000;
+/**
+ * Time for individual requests to timeout the operation.
+ *
+ * This is time spent on the cursor, not total time.
+ *
+ * Must be less than MONGO_SOCKET_TIMEOUT_MS to ensure proper error handling.
+ */
+export const MONGO_OPERATION_TIMEOUT_MS = 30000;
+/**
+ * Same as above, but specifically for clear operations.
+ *
+ * These are retried when reaching the timeout.
+ */
+export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5000;
+export function createMongoClient(config) {
+    const normalized = normalizeMongoConfig(config);
+    return new mongo.MongoClient(normalized.uri, {
+        auth: {
+            username: normalized.username,
+            password: normalized.password
+        },
+        // Time for connection to timeout
+        connectTimeoutMS: MONGO_CONNECT_TIMEOUT_MS,
+        // Time for individual requests to timeout
+        socketTimeoutMS: MONGO_SOCKET_TIMEOUT_MS,
+        // How long to wait for new primary selection
+        serverSelectionTimeoutMS: 30000,
+        // Avoid too many connections:
+        // 1. It can overwhelm the source database.
+        // 2. Processing too many queries in parallel can cause the process to run out of memory.
+        maxPoolSize: 8,
+        maxConnecting: 3,
+        maxIdleTimeMS: 60000
+    });
+}
+/**
+ * Wait up to a minute for authentication errors to resolve.
+ *
+ * There can be a delay between an Atlas user being created, and that user being
+ * available on the database cluster. This works around it.
+ *
+ * This is specifically relevant for migrations and teardown - other parts of the stack
+ * can generate handle these failures and just retry or restart.
+ */
+export async function waitForAuth(db) {
+    const start = Date.now();
+    while (Date.now() - start < 60000) {
+        try {
+            await db.command({ ping: 1 });
+            // Success
+            break;
+        }
+        catch (e) {
+            if (e.codeName == 'AuthenticationFailed') {
+                await timers.setTimeout(1000);
+                continue;
+            }
+            throw e;
+        }
+    }
+}
+//# sourceMappingURL=mongo.js.map

package/dist/db/mongo.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mongo.js","sourceRoot":"","sources":["../../src/db/mongo.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,SAAS,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAG1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,KAAM,CAAC;AAE/C;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAM,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,KAAM,CAAC;AAEjD;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,IAAK,CAAC;AAEtD,MAAM,UAAU,iBAAiB,CAAC,MAAqC;IACrE,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAChD,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE;QAC3C,IAAI,EAAE;YACJ,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;SAC9B;QACD,iCAAiC;QACjC,gBAAgB,EAAE,wBAAwB;QAC1C,0CAA0C;QAC1C,eAAe,EAAE,uBAAuB;QACxC,6CAA6C;QAC7C,wBAAwB,EAAE,KAAM;QAEhC,8BAA8B;QAC9B,2CAA2C;QAC3C,yFAAyF;QACzF,WAAW,EAAE,CAAC;QAEd,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,KAAM;KACtB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY;IAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,KAAM,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9B,UAAU;YACV,MAAM;QACR,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,CAAC,QAAQ,IAAI,sBAAsB,EAAE,CAAC;gBACzC,MAAM,MAAM,CAAC,UAAU,CAAC,IAAK,CAAC,CAAC;gBAC/B,SAAS;YACX,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -1 +1,3 @@
 export * from './module/MongoModule.js';
+export * as db from './db/db-index.js';
+export * as storage from './storage/storage-index.js';

package/dist/index.js

@@ -1,2 +1,4 @@
 export * from './module/MongoModule.js';
+export * as db from './db/db-index.js';
+export * as storage from './storage/storage-index.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AAExC,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,4BAA4B,CAAC"}

package/dist/locks/MonogLocks.d.ts

@@ -0,0 +1,36 @@
+import * as framework from '@powersync/lib-services-framework';
+import * as bson from 'bson';
+import * as mongo from 'mongodb';
+/**
+ * Lock Document Schema
+ */
+export type Lock = {
+    name: string;
+    active_lock?: {
+        lock_id: bson.ObjectId;
+        ts: Date;
+    };
+};
+export type Collection = mongo.Collection<Lock>;
+export type AcquireLockParams = {
+    /**
+     * Name of the process/user trying to acquire the lock.
+     */
+    name: string;
+    /**
+     * The TTL of the lock (ms). Default: 60000 ms (1 min)
+     */
+    timeout?: number;
+};
+export declare const releaseLock: (collection: Collection, lock_id: string) => Promise<void>;
+export type CreateLockManagerParams = {
+    /**
+     * Name of the process/user trying to acquire the lock.
+     */
+    name: string;
+    /**
+     * The TTL for each lock (ms). Default: 60000 ms (1 min)
+     */
+    timeout?: number;
+};
+export declare const createMongoLockManager: (collection: Collection, params: CreateLockManagerParams) => framework.locks.LockManager;

package/dist/locks/MonogLocks.js

@@ -0,0 +1,83 @@
+import * as framework from '@powersync/lib-services-framework';
+import * as bson from 'bson';
+const DEFAULT_LOCK_TIMEOUT = 60 * 1000; // 1 minute
+const acquireLock = async (collection, params) => {
+    const now = new Date();
+    const lock_timeout = params.timeout ?? DEFAULT_LOCK_TIMEOUT;
+    const lock_id = new bson.ObjectId();
+    await collection.updateOne({
+        name: params.name
+    }, {
+        $setOnInsert: {
+            name: params.name
+        }
+    }, {
+        upsert: true
+    });
+    const expired_ts = now.getTime() - lock_timeout;
+    const res = await collection.updateOne({
+        $and: [
+            { name: params.name },
+            {
+                $or: [{ active_lock: { $exists: false } }, { 'active_lock.ts': { $lte: new Date(expired_ts) } }]
+            }
+        ]
+    }, {
+        $set: {
+            active_lock: {
+                lock_id: lock_id,
+                ts: now
+            }
+        }
+    });
+    if (res.modifiedCount === 0) {
+        return null;
+    }
+    return lock_id.toString();
+};
+const refreshLock = async (collection, lock_id) => {
+    const lockId = new bson.ObjectId(lock_id);
+    const res = await collection.updateOne({
+        'active_lock.lock_id': lockId
+    }, {
+        $set: {
+            'active_lock.ts': new Date()
+        }
+    });
+    if (res.modifiedCount === 0) {
+        throw new Error('Lock not found, could not refresh');
+    }
+};
+export const releaseLock = async (collection, lock_id) => {
+    const lockId = new bson.ObjectId(lock_id);
+    const res = await collection.updateOne({
+        'active_lock.lock_id': lockId
+    }, {
+        $unset: {
+            active_lock: true
+        }
+    });
+    if (res.modifiedCount === 0) {
+        throw new Error('Lock not found, could not release');
+    }
+};
+export const createMongoLockManager = (collection, params) => {
+    return {
+        acquire: () => acquireLock(collection, params),
+        refresh: (lock_id) => refreshLock(collection, lock_id),
+        release: (lock_id) => releaseLock(collection, lock_id),
+        lock: async (handler) => {
+            const lock_id = await acquireLock(collection, params);
+            if (!lock_id) {
+                throw new framework.locks.LockActiveError();
+            }
+            try {
+                await handler(() => refreshLock(collection, lock_id));
+            }
+            finally {
+                await releaseLock(collection, lock_id);
+            }
+        }
+    };
+};
+//# sourceMappingURL=MonogLocks.js.map

package/dist/locks/MonogLocks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MonogLocks.js","sourceRoot":"","sources":["../../src/locks/MonogLocks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AA2B7B,MAAM,oBAAoB,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;AAEnD,MAAM,WAAW,GAAG,KAAK,EAAE,UAAsB,EAAE,MAAyB,EAAE,EAAE;IAC9E,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,IAAI,oBAAoB,CAAC;IAC5D,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;IAEpC,MAAM,UAAU,CAAC,SAAS,CACxB;QACE,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,EACD;QACE,YAAY,EAAE;YACZ,IAAI,EAAE,MAAM,CAAC,IAAI;SAClB;KACF,EACD;QACE,MAAM,EAAE,IAAI;KACb,CACF,CAAC;IAEF,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,YAAY,CAAC;IAEhD,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,SAAS,CACpC;QACE,IAAI,EAAE;YACJ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;YACrB;gBACE,GAAG,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,gBAAgB,EAAE,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;aACjG;SACF;KACF,EACD;QACE,IAAI,EAAE;YACJ,WAAW,EAAE;gBACX,OAAO,EAAE,OAAO;gBAChB,EAAE,EAAE,GAAG;aACR;SACF;KACF,CACF,CAAC;IAEF,IAAI,GAAG,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,OAAO,CAAC,QAAQ,EAAE,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EAAE,UAAsB,EAAE,OAAe,EAAE,EAAE;IACpE,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,SAAS,CACpC;QACE,qBAAqB,EAAE,MAAM;KAC9B,EACD;QACE,IAAI,EAAE;YACJ,gBAAgB,EAAE,IAAI,IAAI,EAAE;SAC7B;KACF,CACF,CAAC;IAEF,IAAI,GAAG,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,UAAsB,EAAE,OAAe,EAAE,EAAE;IAC3E,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,SAAS,CACpC;QACE,qBAAqB,EAAE,MAAM;KAC9B,EACD;QACE,MAAM,EAAE;YACN,WAAW,EAAE,IAAI;SAClB;KACF,CACF,CAAC;IAEF,IAAI,GAAG,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;AACH,CAAC,CAAC;AAaF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,UAAsB,EACtB,MAA+B,EACF,EAAE;IAC/B,OAAO;QACL,OAAO,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC;QAC9C,OAAO,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC;QAC9D,OAAO,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC;QAE9D,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACtB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YACtD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;YAC9C,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;YACxD,CAAC;oBAAS,CAAC;gBACT,MAAM,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC,CAAC"}

package/dist/migrations/MonogMigrationAgent.d.ts

@@ -0,0 +1,12 @@
+import * as framework from '@powersync/lib-services-framework';
+import { migrations } from '@powersync/service-core';
+import { configFile } from '@powersync/service-types';
+import { PowerSyncMongo } from '../storage/storage-index.js';
+export declare class MongoMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent {
+    store: framework.MigrationStore;
+    locks: framework.LockManager;
+    protected client: PowerSyncMongo;
+    constructor(mongoConfig: configFile.MongoStorageConfig);
+    getInternalScriptsDir(): string;
+    [Symbol.asyncDispose](): Promise<void>;
+}

package/dist/migrations/MonogMigrationAgent.js

@@ -0,0 +1,25 @@
+import { migrations } from '@powersync/service-core';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { createMongoLockManager } from '../locks/MonogLocks.js';
+import { createPowerSyncMongo } from '../storage/storage-index.js';
+import { createMongoMigrationStore } from './mongo-migration-store.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const MONGO_LOCK_PROCESS = 'migrations';
+const MIGRATIONS_DIR = path.join(__dirname, '/db/migrations');
+export class MongoMigrationAgent extends migrations.AbstractPowerSyncMigrationAgent {
+    constructor(mongoConfig) {
+        super();
+        this.client = createPowerSyncMongo(mongoConfig);
+        this.store = createMongoMigrationStore(this.client.db);
+        this.locks = createMongoLockManager(this.client.locks, { name: MONGO_LOCK_PROCESS });
+    }
+    getInternalScriptsDir() {
+        return MIGRATIONS_DIR;
+    }
+    async [Symbol.asyncDispose]() {
+        await this.client.client.close();
+    }
+}
+//# sourceMappingURL=MonogMigrationAgent.js.map

package/dist/migrations/MonogMigrationAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MonogMigrationAgent.js","sourceRoot":"","sources":["../../src/migrations/MonogMigrationAgent.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAkB,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,kBAAkB,GAAG,YAAY,CAAC;AACxC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;AAE9D,MAAM,OAAO,mBAAoB,SAAQ,UAAU,CAAC,+BAA+B;IAMjF,YAAY,WAA0C;QACpD,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;QAEhD,IAAI,CAAC,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACvF,CAAC;IAED,qBAAqB;QACnB,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACzB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACnC,CAAC;CACF"}

package/dist/migrations/db/migrations/1684951997326-init.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1684951997326-init.js

@@ -0,0 +1,30 @@
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const database = storage.createPowerSyncMongo(configuration.storage);
+    await mongo.waitForAuth(database.db);
+    try {
+        await database.bucket_parameters.createIndex({
+            'key.g': 1,
+            lookup: 1,
+            _id: 1
+        }, { name: 'lookup1' });
+    }
+    finally {
+        await database.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const database = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        if (await database.bucket_parameters.indexExists('lookup')) {
+            await database.bucket_parameters.dropIndex('lookup1');
+        }
+    }
+    finally {
+        await database.client.close();
+    }
+};
+//# sourceMappingURL=1684951997326-init.js.map

package/dist/migrations/db/migrations/1684951997326-init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1684951997326-init.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1684951997326-init.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAC9C,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAE7D,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,QAAQ,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IACtG,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAC1C;YACE,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,CAAC;YACT,GAAG,EAAE,CAAC;SACP,EACD,EAAE,IAAI,EAAE,SAAS,EAAE,CACpB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IACtG,IAAI,CAAC;QACH,IAAI,MAAM,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3D,MAAM,QAAQ,CAAC,iBAAiB,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;AACH,CAAC,CAAC"}

package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.d.ts

@@ -0,0 +1,2 @@
+export declare const up: () => Promise<void>;
+export declare const down: () => Promise<void>;

package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js

@@ -0,0 +1,5 @@
+export const up = async () => {
+    // No-op - we don't auto-create sync rules anymore
+};
+export const down = async () => { };
+//# sourceMappingURL=1688556755264-initial-sync-rules.js.map

package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1688556755264-initial-sync-rules.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1688556755264-initial-sync-rules.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,IAAI,EAAE;IAC3B,kDAAkD;AACpD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,GAAE,CAAC,CAAC"}

package/dist/migrations/db/migrations/1702295701188-sync-rule-state.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js

@@ -0,0 +1,54 @@
+import { storage as core_storage } from '@powersync/service-core';
+import * as mongo from '../../../db/mongo.js';
+import * as storage from '../../../storage/storage-index.js';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    await mongo.waitForAuth(db.db);
+    try {
+        // We keep the old flags for existing deployments still shutting down.
+        // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
+        await db.sync_rules.updateMany({
+            active: { $ne: true },
+            replicating: true,
+            auto_activate: true
+        }, { $set: { state: core_storage.SyncRuleState.PROCESSING } });
+        // 2. Snapshot done: `active = true, snapshot_done = true, replicating = true, auto_activate = false`
+        await db.sync_rules.updateMany({
+            active: true
+        }, { $set: { state: core_storage.SyncRuleState.ACTIVE } });
+        // 3. Stopped: `active = false, snapshot_done = true, replicating = false, auto_activate = false`.
+        await db.sync_rules.updateMany({
+            active: { $ne: true },
+            replicating: { $ne: true },
+            auto_activate: { $ne: true }
+        }, { $set: { state: core_storage.SyncRuleState.STOP } });
+        const remaining = await db.sync_rules.find({ state: null }).toArray();
+        if (remaining.length > 0) {
+            const slots = remaining.map((doc) => doc.slot_name).join(', ');
+            throw new Error(`Invalid state for sync rules: ${slots}`);
+        }
+    }
+    finally {
+        await db.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        await db.sync_rules.updateMany({
+            state: core_storage.SyncRuleState.ACTIVE
+        }, { $set: { active: true, replicating: true } });
+        await db.sync_rules.updateMany({
+            state: core_storage.SyncRuleState.PROCESSING
+        }, { $set: { active: false, replicating: true, auto_activate: true } });
+        await db.sync_rules.updateMany({
+            $or: [{ state: core_storage.SyncRuleState.STOP }, { state: core_storage.SyncRuleState.TERMINATED }]
+        }, { $set: { active: false, replicating: false, auto_activate: false } });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+//# sourceMappingURL=1702295701188-sync-rule-state.js.map

package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1702295701188-sync-rule-state.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1702295701188-sync-rule-state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,YAAY,EAAc,MAAM,yBAAyB,CAAC;AAE9E,OAAO,KAAK,KAAK,MAAM,sBAAsB,CAAC;AAC9C,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAuB7D,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IAEhG,MAAM,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,sEAAsE;QAEtE,uGAAuG;QACvG,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B;YACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YACrB,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI;SACpB,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,UAAU,EAAE,EAAE,CAC3D,CAAC;QAEF,qGAAqG;QACrG,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B;YACE,MAAM,EAAE,IAAI;SACb,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CACvD,CAAC;QAEF,kGAAkG;QAClG,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B;YACE,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YACrB,WAAW,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YAC1B,aAAa,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;SAC7B,EACD,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CACrD,CAAC;QAEF,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAC7E,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IAChG,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B;YACE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,MAAM;SACzC,EACD,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,EAAE,CAC9C,CAAC;QAEF,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B;YACE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,UAAU;SAC7C,EACD,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,EAAE,CACpE,CAAC;QAEF,MAAM,EAAE,CAAC,UAAU,CAAC,UAAU,CAC5B;YACE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;SACpG,EACD,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,CACtE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}

package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js

@@ -0,0 +1,26 @@
+import * as storage from '../../../storage/storage-index.js';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        await db.write_checkpoints.createIndex({
+            user_id: 1
+        }, { name: 'user_id' });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        if (await db.write_checkpoints.indexExists('user_id')) {
+            await db.write_checkpoints.dropIndex('user_id');
+        }
+    }
+    finally {
+        await db.client.close();
+    }
+};
+//# sourceMappingURL=1711543888062-write-checkpoint-index.js.map

package/dist/migrations/db/migrations/1711543888062-write-checkpoint-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1711543888062-write-checkpoint-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1711543888062-write-checkpoint-index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAE7D,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IAEhG,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CACpC;YACE,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,SAAS,EAAE,CACpB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IAEhG,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;YACtD,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}

package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.d.ts

@@ -0,0 +1,3 @@
+import { migrations } from '@powersync/service-core';
+export declare const up: migrations.PowerSyncMigrationFunction;
+export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js

@@ -0,0 +1,28 @@
+import * as storage from '../../../storage/storage-index.js';
+const INDEX_NAME = 'user_sync_rule_unique';
+export const up = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        await db.custom_write_checkpoints.createIndex({
+            user_id: 1,
+            sync_rules_id: 1
+        }, { name: INDEX_NAME, unique: true });
+    }
+    finally {
+        await db.client.close();
+    }
+};
+export const down = async (context) => {
+    const { service_context: { configuration } } = context;
+    const db = storage.createPowerSyncMongo(configuration.storage);
+    try {
+        if (await db.custom_write_checkpoints.indexExists(INDEX_NAME)) {
+            await db.custom_write_checkpoints.dropIndex(INDEX_NAME);
+        }
+    }
+    finally {
+        await db.client.close();
+    }
+};
+//# sourceMappingURL=1727099539247-custom-write-checkpoint-index.js.map

package/dist/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"1727099539247-custom-write-checkpoint-index.js","sourceRoot":"","sources":["../../../../src/migrations/db/migrations/1727099539247-custom-write-checkpoint-index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,mCAAmC,CAAC;AAE7D,MAAM,UAAU,GAAG,uBAAuB,CAAC;AAE3C,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IACZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IAEhG,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAC3C;YACE,OAAO,EAAE,CAAC;YACV,aAAa,EAAE,CAAC;SACjB,EACD,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CACnC,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;IAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;IAEZ,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,aAAa,CAAC,OAAwC,CAAC,CAAC;IAEhG,IAAI,CAAC;QACH,IAAI,MAAM,EAAE,CAAC,wBAAwB,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC,CAAC"}

package/dist/migrations/mongo-migration-store.d.ts

@@ -0,0 +1,7 @@
+import { migrations } from '@powersync/lib-services-framework';
+import { Db } from 'mongodb';
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export declare const createMongoMigrationStore: (db: Db) => migrations.MigrationStore;

package/dist/migrations/mongo-migration-store.js

@@ -0,0 +1,49 @@
+import * as path from 'path';
+/**
+ * A custom store for node-migrate which is used to save and load migrations that have
+ * been operated on to mongo.
+ */
+export const createMongoMigrationStore = (db) => {
+    const collection = db.collection('migrations');
+    return {
+        load: async () => {
+            const state_entry = await collection.findOne();
+            if (!state_entry) {
+                return;
+            }
+            const { _id, ...state } = state_entry;
+            /**
+             * This is for backwards compatibility. A previous version of the migration tool used to save
+             * state as `lastRun`.
+             */
+            let last_run = state.last_run;
+            if ('lastRun' in state) {
+                last_run = state.lastRun;
+            }
+            /**
+             * This is for backwards compatibility. A previous version of the migration tool used to include the
+             * file extension in migration names. This strips that extension off if it exists
+             */
+            const extension = path.extname(last_run);
+            if (extension) {
+                last_run = last_run.replace(extension, '');
+            }
+            return {
+                last_run,
+                log: state.log || []
+            };
+        },
+        clear: async () => {
+            await collection.deleteMany({});
+        },
+        save: async (state) => {
+            await collection.replaceOne({}, {
+                last_run: state.last_run,
+                log: state.log
+            }, {
+                upsert: true
+            });
+        }
+    };
+};
+//# sourceMappingURL=mongo-migration-store.js.map