@powersync/service-module-postgres-storage 0.1.0

package/dist/storage/PostgresCompactor.js

@@ -0,0 +1,298 @@
+import { logger } from '@powersync/lib-services-framework';
+import { utils } from '@powersync/service-core';
+import { BIGINT_MAX } from '../types/codecs.js';
+import { models } from '../types/types.js';
+import { sql } from '../utils/db.js';
+import { pick } from '../utils/ts-codec.js';
+import { encodedCacheKey } from './batch/OperationBatch.js';
+const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
+const DEFAULT_MOVE_BATCH_LIMIT = 2000;
+const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10_000;
+/** This default is primarily for tests. */
+const DEFAULT_MEMORY_LIMIT_MB = 64;
+export class PostgresCompactor {
+    db;
+    group_id;
+    updates = [];
+    idLimitBytes;
+    moveBatchLimit;
+    moveBatchQueryLimit;
+    clearBatchLimit;
+    maxOpId;
+    buckets;
+    constructor(db, group_id, options) {
+        this.db = db;
+        this.group_id = group_id;
+        this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
+        this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
+        this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
+        this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+        this.maxOpId = options?.maxOpId;
+        this.buckets = options?.compactBuckets;
+    }
+    /**
+     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
+     *
+     * See /docs/compacting-operations.md for details.
+     */
+    async compact() {
+        if (this.buckets) {
+            for (let bucket of this.buckets) {
+                // We can make this more efficient later on by iterating
+                // through the buckets in a single query.
+                // That makes batching more tricky, so we leave for later.
+                await this.compactInternal(bucket);
+            }
+        }
+        else {
+            await this.compactInternal(undefined);
+        }
+    }
+    async compactInternal(bucket) {
+        const idLimitBytes = this.idLimitBytes;
+        let currentState = null;
+        let bucketLower = null;
+        let bucketUpper = null;
+        if (bucket?.includes('[')) {
+            // Exact bucket name
+            bucketLower = bucket;
+            bucketUpper = bucket;
+        }
+        else if (bucket) {
+            // Bucket definition name
+            bucketLower = `${bucket}[`;
+            bucketUpper = `${bucket}[\uFFFF`;
+        }
+        let upperOpIdLimit = BIGINT_MAX;
+        while (true) {
+            const batch = await this.db.sql `
+        SELECT
+          op,
+          op_id,
+          source_table,
+          table_name,
+          row_id,
+          source_key,
+          bucket_name
+        FROM
+          bucket_data
+        WHERE
+          group_id = ${{ type: 'int4', value: this.group_id }}
+          AND bucket_name LIKE COALESCE(${{ type: 'varchar', value: bucketLower }}, '%')
+          AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
+        ORDER BY
+          bucket_name,
+          op_id DESC
+        LIMIT
+          ${{ type: 'int4', value: this.moveBatchQueryLimit }}
+      `
+                .decoded(pick(models.BucketData, ['op', 'source_table', 'table_name', 'source_key', 'row_id', 'op_id', 'bucket_name']))
+                .rows();
+            if (batch.length == 0) {
+                // We've reached the end
+                break;
+            }
+            // Set upperBound for the next batch
+            upperOpIdLimit = batch[batch.length - 1].op_id;
+            for (const doc of batch) {
+                if (currentState == null || doc.bucket_name != currentState.bucket) {
+                    if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
+                        // Important to flush before clearBucket()
+                        await this.flush();
+                        logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
+                        const bucket = currentState.bucket;
+                        const clearOp = currentState.lastNotPut;
+                        // Free memory before clearing bucket
+                        currentState = null;
+                        await this.clearBucket(bucket, clearOp);
+                    }
+                    currentState = {
+                        bucket: doc.bucket_name,
+                        seen: new Map(),
+                        trackingSize: 0,
+                        lastNotPut: null,
+                        opsSincePut: 0
+                    };
+                }
+                if (this.maxOpId != null && doc.op_id > this.maxOpId) {
+                    continue;
+                }
+                let isPersistentPut = doc.op == 'PUT';
+                if (doc.op == 'REMOVE' || doc.op == 'PUT') {
+                    const key = `${doc.table_name}/${doc.row_id}/${encodedCacheKey(doc.source_table, doc.source_key)}`;
+                    const targetOp = currentState.seen.get(utils.flatstr(key));
+                    if (targetOp) {
+                        // Will convert to MOVE, so don't count as PUT
+                        isPersistentPut = false;
+                        this.updates.push(sql `
+              UPDATE bucket_data
+              SET
+                op = 'MOVE',
+                target_op = ${{ type: 'int8', value: targetOp }},
+                table_name = NULL,
+                row_id = NULL,
+                data = NULL,
+                source_table = NULL,
+                source_key = NULL
+              WHERE
+                group_id = ${{ type: 'int4', value: this.group_id }}
+                AND bucket_name = ${{ type: 'varchar', value: doc.bucket_name }}
+                AND op_id = ${{ type: 'int8', value: doc.op_id }}
+            `);
+                    }
+                    else {
+                        if (currentState.trackingSize >= idLimitBytes) {
+                            // Reached memory limit.
+                            // Keep the highest seen values in this case.
+                        }
+                        else {
+                            // flatstr reduces the memory usage by flattening the string
+                            currentState.seen.set(utils.flatstr(key), doc.op_id);
+                            // length + 16 for the string
+                            // 24 for the bigint
+                            // 50 for map overhead
+                            // 50 for additional overhead
+                            currentState.trackingSize += key.length + 140;
+                        }
+                    }
+                }
+                if (isPersistentPut) {
+                    currentState.lastNotPut = null;
+                    currentState.opsSincePut = 0;
+                }
+                else if (doc.op != 'CLEAR') {
+                    if (currentState.lastNotPut == null) {
+                        currentState.lastNotPut = doc.op_id;
+                    }
+                    currentState.opsSincePut += 1;
+                }
+                if (this.updates.length >= this.moveBatchLimit) {
+                    await this.flush();
+                }
+            }
+        }
+        await this.flush();
+        currentState?.seen.clear();
+        if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
+            logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
+            const bucket = currentState.bucket;
+            const clearOp = currentState.lastNotPut;
+            // Free memory before clearing bucket
+            currentState = null;
+            await this.clearBucket(bucket, clearOp);
+        }
+    }
+    async flush() {
+        if (this.updates.length > 0) {
+            logger.info(`Compacting ${this.updates.length} ops`);
+            await this.db.query(...this.updates);
+            this.updates = [];
+        }
+    }
+    /**
+     * Perform a CLEAR compact for a bucket.
+     *
+     * @param bucket bucket name
+     * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
+     */
+    async clearBucket(bucket, op) {
+        /**
+         * This entire method could be implemented as a Postgres function, but this might make debugging
+         * a bit more challenging.
+         */
+        let done = false;
+        while (!done) {
+            await this.db.lockConnection(async (db) => {
+                /**
+                 * Start a transaction where each read returns the state at the start of the transaction,.
+                 * Similar to the MongoDB readConcern: { level: 'snapshot' } mode.
+                 */
+                await db.sql `BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ`.execute();
+                try {
+                    let checksum = 0;
+                    let lastOpId = null;
+                    let targetOp = null;
+                    let gotAnOp = false;
+                    const codec = pick(models.BucketData, ['op', 'source_table', 'source_key', 'op_id', 'checksum', 'target_op']);
+                    for await (const operations of db.streamRows(sql `
+            SELECT
+              source_table,
+              source_key,
+              op,
+              op_id,
+              checksum,
+              target_op
+            FROM
+              bucket_data
+            WHERE
+              group_id = ${{ type: 'int4', value: this.group_id }}
+              AND bucket_name = ${{ type: 'varchar', value: bucket }}
+              AND op_id <= ${{ type: 'int8', value: op }}
+            ORDER BY
+              op_id
+            LIMIT
+              ${{ type: 'int4', value: this.clearBatchLimit }}
+          `)) {
+                        const decodedOps = operations.map((o) => codec.decode(o));
+                        for (const op of decodedOps) {
+                            if ([models.OpType.MOVE, models.OpType.REMOVE, models.OpType.CLEAR].includes(op.op)) {
+                                checksum = utils.addChecksums(checksum, Number(op.checksum));
+                                lastOpId = op.op_id;
+                                if (op.op != models.OpType.CLEAR) {
+                                    gotAnOp = true;
+                                }
+                                if (op.target_op != null) {
+                                    if (targetOp == null || op.target_op > targetOp) {
+                                        targetOp = op.target_op;
+                                    }
+                                }
+                            }
+                            else {
+                                throw new Error(`Unexpected ${op.op} operation at ${this.group_id}:${bucket}:${op.op_id}`);
+                            }
+                        }
+                    }
+                    if (!gotAnOp) {
+                        await db.sql `COMMIT`.execute();
+                        done = true;
+                        return;
+                    }
+                    logger.info(`Flushing CLEAR at ${lastOpId}`);
+                    await db.sql `
+            DELETE FROM bucket_data
+            WHERE
+              group_id = ${{ type: 'int4', value: this.group_id }}
+              AND bucket_name = ${{ type: 'varchar', value: bucket }}
+              AND op_id <= ${{ type: 'int8', value: lastOpId }}
+          `.execute();
+                    await db.sql `
+            INSERT INTO
+              bucket_data (
+                group_id,
+                bucket_name,
+                op_id,
+                op,
+                checksum,
+                target_op
+              )
+            VALUES
+              (
+                ${{ type: 'int4', value: this.group_id }},
+                ${{ type: 'varchar', value: bucket }},
+                ${{ type: 'int8', value: lastOpId }},
+                ${{ type: 'varchar', value: models.OpType.CLEAR }},
+                ${{ type: 'int8', value: checksum }},
+                ${{ type: 'int8', value: targetOp }}
+              )
+          `.execute();
+                    await db.sql `COMMIT`.execute();
+                }
+                catch (ex) {
+                    await db.sql `ROLLBACK`.execute();
+                    throw ex;
+                }
+            });
+        }
+    }
+}
+//# sourceMappingURL=PostgresCompactor.js.map

package/dist/storage/PostgresCompactor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PostgresCompactor.js","sourceRoot":"","sources":["../../src/storage/PostgresCompactor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAW,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGzD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAqC5D,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,iBAAiB;IAWlB;IACA;IAXF,OAAO,GAAuB,EAAE,CAAC;IAEjC,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAAqB;IAC5B,OAAO,CAAuB;IAEtC,YACU,EAA+B,EAC/B,QAAgB,EACxB,OAAgC;QAFxB,OAAE,GAAF,EAAE,CAA6B;QAC/B,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,wDAAwD;gBACxD,yCAAyC;gBACzC,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAA0B;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,IAAI,WAAW,GAAkB,IAAI,CAAC;QACtC,IAAI,WAAW,GAAkB,IAAI,CAAC;QAEtC,IAAI,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,oBAAoB;YACpB,WAAW,GAAG,MAAM,CAAC;YACrB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YAClB,yBAAyB;YACzB,WAAW,GAAG,GAAG,MAAM,GAAG,CAAC;YAC3B,WAAW,GAAG,GAAG,MAAM,SAAS,CAAC;QACnC,CAAC;QAED,IAAI,cAAc,GAAG,UAAU,CAAC;QAEhC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;uBAYd,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;0CACnB,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,WAAW,EAAE;wBACzD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE;;;;;YAKnD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE;OACtD;iBACE,OAAO,CACN,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAC9G;iBACA,IAAI,EAAE,CAAC;YAEV,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,cAAc,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;YAE/C,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,WAAW,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;oBACnE,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC;wBAC7F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;oBAC1C,CAAC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,WAAW;wBACvB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;gBACJ,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACrG,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC3D,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAA;;;;8BAIH,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;;;;;;;6BAOlC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;oCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,EAAE;8BACjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE;aACnD,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAU;QAClD;;;WAGG;QACH,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,OAAO,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBACxC;;;mBAGG;gBACH,MAAM,EAAE,CAAC,GAAG,CAAA,mDAAmD,CAAC,OAAO,EAAE,CAAC;gBAE1E,IAAI,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;oBAC9G,IAAI,KAAK,EAAE,MAAM,UAAU,IAAI,EAAE,CAAC,UAAU,CAA0B,GAAG,CAAA;;;;;;;;;;;2BAWxD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACvC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;;;;gBAIxC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE;WAClD,CAAC,EAAE,CAAC;wBACH,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC1D,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;4BAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gCACpF,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;gCAC7D,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC;gCACpB,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oCACjC,OAAO,GAAG,IAAI,CAAC;gCACjB,CAAC;gCACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;oCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;wCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;oCAC1B,CAAC;gCACH,CAAC;4BACH,CAAC;iCAAM,CAAC;gCACN,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC;4BAC7F,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,MAAM,EAAE,CAAC,GAAG,CAAA,QAAQ,CAAC,OAAO,EAAE,CAAC;wBAC/B,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;oBAE7C,MAAM,EAAE,CAAC,GAAG,CAAA;;;2BAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACvC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;WACnD,CAAC,OAAO,EAAE,CAAC;oBAEZ,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;kBAYJ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kBACtC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;kBAClC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;kBACjC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;kBAC/C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;kBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;;WAExC,CAAC,OAAO,EAAE,CAAC;oBAEZ,MAAM,EAAE,CAAC,GAAG,CAAA,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACjC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,MAAM,EAAE,CAAC,GAAG,CAAA,UAAU,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/storage/PostgresStorageProvider.js

@@ -0,0 +1,35 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { logger } from '@powersync/lib-services-framework';
+import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
+import { dropTables } from '../utils/db.js';
+import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
+export class PostgresStorageProvider {
+    get type() {
+        return lib_postgres.POSTGRES_CONNECTION_TYPE;
+    }
+    async getStorage(options) {
+        const { resolvedConfig } = options;
+        const { storage } = resolvedConfig;
+        if (!isPostgresStorageConfig(storage)) {
+            // This should not be reached since the generation should be managed externally.
+            throw new Error(`Cannot create Postgres bucket storage with provided config ${storage.type} !== ${lib_postgres.POSTGRES_CONNECTION_TYPE}`);
+        }
+        const decodedConfig = PostgresStorageConfig.decode(storage);
+        const normalizedConfig = normalizePostgresStorageConfig(decodedConfig);
+        const storageFactory = new PostgresBucketStorageFactory({
+            config: normalizedConfig,
+            slot_name_prefix: options.resolvedConfig.slot_name_prefix
+        });
+        return {
+            storage: storageFactory,
+            shutDown: async () => storageFactory.db[Symbol.asyncDispose](),
+            tearDown: async () => {
+                logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`);
+                await dropTables(storageFactory.db);
+                await storageFactory.db[Symbol.asyncDispose]();
+                return true;
+            }
+        };
+    }
+}
+//# sourceMappingURL=PostgresStorageProvider.js.map

package/dist/storage/PostgresStorageProvider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PostgresStorageProvider.js","sourceRoot":"","sources":["../../src/storage/PostgresStorageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAG3D,OAAO,EAAE,uBAAuB,EAAE,8BAA8B,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AACnH,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,4BAA4B,EAAE,MAAM,mCAAmC,CAAC;AAEjF,MAAM,OAAO,uBAAuB;IAClC,IAAI,IAAI;QACN,OAAO,YAAY,CAAC,wBAAwB,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAkC;QACjD,MAAM,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC;QAEnC,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC;QACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,gFAAgF;YAChF,MAAM,IAAI,KAAK,CACb,8DAA8D,OAAO,CAAC,IAAI,QAAQ,YAAY,CAAC,wBAAwB,EAAE,CAC1H,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC5D,MAAM,gBAAgB,GAAG,8BAA8B,CAAC,aAAa,CAAC,CAAC;QACvE,MAAM,cAAc,GAAG,IAAI,4BAA4B,CAAC;YACtD,MAAM,EAAE,gBAAgB;YACxB,gBAAgB,EAAE,OAAO,CAAC,cAAc,CAAC,gBAAgB;SAC1D,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,cAAc;YACvB,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE;YAC9D,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACnB,MAAM,CAAC,IAAI,CAAC,kCAAkC,gBAAgB,CAAC,QAAQ,KAAK,CAAC,CAAC;gBAC9E,MAAM,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;gBACpC,MAAM,cAAc,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;gBAC/C,OAAO,IAAI,CAAC;YACd,CAAC;SAC8B,CAAC;IACpC,CAAC;CACF"}