@streamr/node 100.0.0-rc.0

package/dist/src/plugins/storage/DeleteExpiredCmd.js

@@ -0,0 +1,155 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeleteExpiredCmd = void 0;
+const cassandra_driver_1 = __importDefault(require("cassandra-driver"));
+const p_limit_1 = __importDefault(require("p-limit"));
+const utils_1 = require("@streamr/utils");
+const logger = new utils_1.Logger(module);
+const totalSizeOfBuckets = (buckets) => buckets.reduce((mem, { size }) => mem + size, 0) / (1024 * 1024);
+const totalRecordCount = (buckets) => buckets.reduce((mem, { records }) => mem + records, 0);
+class DeleteExpiredCmd {
+    streamrBaseUrl;
+    dryRun;
+    bucketLimit;
+    cassandraClient;
+    limit;
+    constructor({ streamrBaseUrl, cassandraUsername, cassandraPassword, cassandraHosts, cassandraDatacenter, cassandraKeyspace, bucketLimit, dryRun = true }) {
+        this.streamrBaseUrl = streamrBaseUrl;
+        this.dryRun = dryRun;
+        this.bucketLimit = bucketLimit || 10000000;
+        const authProvider = new cassandra_driver_1.default.auth.PlainTextAuthProvider(cassandraUsername, cassandraPassword);
+        this.cassandraClient = new cassandra_driver_1.default.Client({
+            contactPoints: [...cassandraHosts],
+            localDataCenter: cassandraDatacenter,
+            keyspace: cassandraKeyspace,
+            authProvider,
+        });
+        // used for limited concurrency
+        this.limit = (0, p_limit_1.default)(5);
+    }
+    async run(client) {
+        const streams = await this.getStreams();
+        logger.info(`Found ${streams.length} unique streams`);
+        const streamsInfo = await this.fetchStreamsInfo(streams, client);
+        const potentialBuckets = await this.getPotentiallyExpiredBuckets(streamsInfo);
+        logger.info(`Found ${potentialBuckets.length} potentially expired buckets`);
+        const cutPotentialBuckets = potentialBuckets.slice(0, this.bucketLimit);
+        logger.info(`Left with ${cutPotentialBuckets.length} potentially expired buckets after cutting`);
+        const expiredBuckets = await this.filterExpiredBuckets(cutPotentialBuckets);
+        logger.info(`Found ${expiredBuckets.length} expired buckets`, {
+            totalRecords: totalRecordCount(expiredBuckets),
+            totalSizeOfBucketsInMb: totalSizeOfBuckets(expiredBuckets)
+        });
+        if (!this.dryRun) {
+            await this.deleteExpired(expiredBuckets);
+        }
+        await this.cassandraClient.shutdown();
+    }
+    async getStreams() {
+        const query = 'SELECT DISTINCT stream_id, partition FROM bucket';
+        const resultSet = await this.cassandraClient.execute(query, [], {
+            fetchSize: 100000
+        });
+        return resultSet.rows.map((row) => ({
+            streamId: row.stream_id,
+            partition: row.partition
+        }));
+    }
+    async fetchStreamsInfo(streams, client) {
+        const tasks = streams.filter(Boolean).map((stream) => {
+            return this.limit(async () => {
+                try {
+                    const streamFromChain = await client.getStream(stream.streamId);
+                    return {
+                        streamId: stream.streamId,
+                        partition: stream.partition,
+                        storageDays: streamFromChain.getMetadata().storageDays ?? 365
+                    };
+                }
+                catch (err) {
+                    logger.error('Failed to fetch stream info', { err });
+                }
+            });
+        });
+        return Promise.all(tasks);
+    }
+    async getPotentiallyExpiredBuckets(streamsInfo) {
+        const result = [];
+        const query = 'SELECT * FROM bucket WHERE stream_id = ? AND partition = ? AND date_create <= ?';
+        // @ts-expect-error void filtering
+        const tasks = streamsInfo.filter(Boolean).map((stream) => {
+            const { streamId, partition, storageDays } = stream;
+            const timestampBefore = Date.now() - 1000 * 60 * 60 * 24 * storageDays;
+            const params = [streamId, partition, timestampBefore];
+            return this.limit(async () => {
+                const resultSet = await this.cassandraClient.execute(query, params, {
+                    prepare: true,
+                }).catch((err) => logger.error('Failed to execute query', { err, query }));
+                if (resultSet) {
+                    resultSet.rows.forEach((row) => {
+                        result.push({
+                            bucketId: row.id,
+                            dateCreate: row.date_create,
+                            streamId: row.stream_id,
+                            partition: row.partition,
+                            records: row.records,
+                            size: row.size,
+                            storageDays
+                        });
+                    });
+                }
+            });
+        });
+        await Promise.all(tasks);
+        return result;
+    }
+    async filterExpiredBuckets(potentialBuckets) {
+        const result = [];
+        const query = 'SELECT MAX(ts) AS m FROM stream_data WHERE stream_id = ? AND partition = ? AND bucket_id = ?';
+        const tasks = potentialBuckets.filter(Boolean).map((bucket) => {
+            const { streamId, partition, bucketId, storageDays } = bucket;
+            const timestampBefore = Date.now() - 1000 * 60 * 60 * 24 * storageDays;
+            const params = [streamId, partition, bucketId];
+            return this.limit(async () => {
+                const resultSet = await this.cassandraClient.execute(query, params, {
+                    prepare: true,
+                }).catch((err) => logger.error('Failed to execute query', { err, query }));
+                if (resultSet && (resultSet.rows.length === 0
+                    || resultSet.rows[0].m === null
+                    || resultSet.rows[0].m.getTime() < timestampBefore)) {
+                    result.push(bucket);
+                }
+            });
+        });
+        await Promise.all(tasks);
+        return result;
+    }
+    async deleteExpired(expiredBuckets) {
+        const tasks = expiredBuckets.filter(Boolean).map((stream) => {
+            const { bucketId, dateCreate, streamId, partition } = stream;
+            const queries = [
+                {
+                    query: 'DELETE FROM bucket WHERE stream_id = ? AND partition = ? AND date_create = ?',
+                    params: [streamId, partition, dateCreate]
+                },
+                {
+                    query: 'DELETE FROM stream_data WHERE stream_id = ? AND partition = ? AND bucket_id = ?',
+                    params: [streamId, partition, bucketId]
+                }
+            ];
+            logger.info('Delete expired bucket', { streamId, partition, bucketId });
+            return this.limit(async () => {
+                await this.cassandraClient.batch(queries, {
+                    prepare: true
+                }).catch((err) => logger.error('Failed to delete expired buckets', { err, queries }));
+                return undefined;
+            });
+        });
+        return Promise.all(tasks);
+    }
+}
+exports.DeleteExpiredCmd = DeleteExpiredCmd;
+//# sourceMappingURL=DeleteExpiredCmd.js.map

package/dist/src/plugins/storage/DeleteExpiredCmd.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeleteExpiredCmd.js","sourceRoot":"","sources":["../../../../src/plugins/storage/DeleteExpiredCmd.ts"],"names":[],"mappings":";;;;;;AAAA,wEAAoD;AACpD,sDAAuC;AAEvC,0CAAuC;AAEvC,MAAM,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,CAAC,CAAA;AAEjC,MAAM,kBAAkB,GAAG,CAAC,OAAqB,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;AAEtH,MAAM,gBAAgB,GAAG,CAAC,OAAqB,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,EAAE,CAAC,CAAC,CAAA;AAgC1G,MAAa,gBAAgB;IAEzB,cAAc,CAAQ;IACtB,MAAM,CAAS;IACf,WAAW,CAAQ;IACnB,eAAe,CAAQ;IACvB,KAAK,CAAO;IAEZ,YAAY,EACR,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,MAAM,GAAG,IAAI,EACP;QACN,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,WAAW,GAAG,WAAW,IAAI,QAAQ,CAAA;QAE1C,MAAM,YAAY,GAAG,IAAI,0BAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;QACnG,IAAI,CAAC,eAAe,GAAG,IAAI,0BAAS,CAAC,MAAM,CAAC;YACxC,aAAa,EAAE,CAAC,GAAG,cAAc,CAAC;YAClC,eAAe,EAAE,mBAAmB;YACpC,QAAQ,EAAE,iBAAiB;YAC3B,YAAY;SACf,CAAC,CAAA;QAEF,+BAA+B;QAC/B,IAAI,CAAC,KAAK,GAAG,IAAA,iBAAM,EAAC,CAAC,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,MAAqB;QAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,iBAAiB,CAAC,CAAA;QAErD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAChE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,4BAA4B,CAAC,WAAW,CAAC,CAAA;QAC7E,MAAM,CAAC,IAAI,CAAC,SAAS,gBAAgB,CAAC,MAAM,8BAA8B,CAAC,CAAA;QAE3E,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;QACvE,MAAM,CAAC,IAAI,CAAC,aAAa,mBAAmB,CAAC,MAAM,4CAA4C,CAAC,CAAA;QAEhG,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,mBAAmB,CAAC,CAAA;QAC3E,MAAM,CAAC,IAAI,CAAC,SAAS,cAAc,CAAC,MAAM,kBAAkB,EAAE;YAC1D,YAAY,EAAE,gBAAgB,CAAC,cAAc,CAAC;YAC9C,sBAAsB,EAAE,kBAAkB,CAAC,cAAc,CAAC;SAC7D,CAAC,CAAA;QAEF,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC5C,CAAC;QAED,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAA;IACzC,CAAC;IAEO,KAAK,CAAC,UAAU;QACpB,MAAM,KAAK,GAAG,kDAAkD,CAAA;QAChE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE;YAC5D,SAAS,EAAE,MAAM;SACpB,CAAC,CAAA;QACF,OAAO,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAChC,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,SAAS,EAAE,GAAG,CAAC,SAAS;SAC3B,CAAC,CAAC,CAAA;IACP,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAqB,EAAE,MAAqB;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAkB,EAAE,EAAE;YAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBACzB,IAAI,CAAC;oBACD,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;oBAC/D,OAAO;wBACH,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,SAAS,EAAE,MAAM,CAAC,SAAS;wBAC3B,WAAW,EAAE,eAAe,CAAC,WAAW,EAAE,CAAC,WAAW,IAAI,GAAG;qBAChE,CAAA;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;gBAAC,CAAC;YAC1E,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;IAEO,KAAK,CAAC,4BAA4B,CAAC,WAA2C;QAClF,MAAM,MAAM,GAAiB,EAAE,CAAA;QAE/B,MAAM,KAAK,GAAG,iFAAiF,CAAA;QAE/F,kCAAkC;QAClC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAsB,EAAE,EAAE;YACrE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,CAAA;YACnD,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,WAAW,CAAA;YACtE,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,eAAe,CAAC,CAAA;YAErD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE;oBAChE,OAAO,EAAE,IAAI;iBAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;gBAE1E,IAAI,SAAS,EAAE,CAAC;oBACZ,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAwB,EAAE,EAAE;wBAChD,MAAM,CAAC,IAAI,CAAC;4BACR,QAAQ,EAAE,GAAG,CAAC,EAAE;4BAChB,UAAU,EAAE,GAAG,CAAC,WAAW;4BAC3B,QAAQ,EAAE,GAAG,CAAC,SAAS;4BACvB,SAAS,EAAE,GAAG,CAAC,SAAS;4BACxB,OAAO,EAAE,GAAG,CAAC,OAAO;4BACpB,IAAI,EAAE,GAAG,CAAC,IAAI;4BACd,WAAW;yBACd,CAAC,CAAA;oBACN,CAAC,CAAC,CAAA;gBACN,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACxB,OAAO,MAAM,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,gBAA8B;QAC7D,MAAM,MAAM,GAAiB,EAAE,CAAA;QAE/B,MAAM,KAAK,GAAG,8FAA8F,CAAA;QAE5G,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAkB,EAAE,EAAE;YACtE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAA;YAC7D,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,WAAW,CAAA;YACtE,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAA;YAE9C,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE;oBAChE,OAAO,EAAE,IAAI;iBAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;gBAE1E,IAAI,SAAS,IAAI,CACb,SAAS,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;uBACxB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI;uBAC5B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,eAAe,CAAC,EAAE,CAAC;oBACtD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACvB,CAAC;YACL,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACxB,OAAO,MAAM,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,cAA4B;QACpD,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACxD,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,MAAM,CAAA;YAC5D,MAAM,OAAO,GAAG;gBACZ;oBACI,KAAK,EAAE,8EAA8E;oBACrF,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAC;iBAC5C;gBACD;oBACI,KAAK,EAAE,iFAAiF;oBACxF,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC;iBAC1C;aACJ,CAAA;YAED,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;YAEvE,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;gBACzB,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,OAAO,EAAE;oBACtC,OAAO,EAAE,IAAI;iBAChB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAA;gBACrF,OAAO,SAAS,CAAA;YACpB,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;CACJ;AAhLD,4CAgLC"}

package/dist/src/plugins/storage/SetMembershipSynchronizer.d.ts

@@ -0,0 +1,33 @@
+export interface Diff<E extends string> {
+    added: E[];
+    removed: E[];
+}
+export declare const EMPTY_DIFF: Diff<never>;
+/**
+ * Represents a set that evolves over (discrete) time. The set is updated in
+ * two distinct ways:
+ *      1. Snapshots that define the set at a given time step.
+ *      2. Patches that add or remove a subset at a given time step.
+ */
+export declare class SetMembershipSynchronizer<E extends string> {
+    private lastSnapshotSequenceNo;
+    private state;
+    private lastSequenceNoByElement;
+    /**
+     * Get up-to-date view of state
+     */
+    getState(): ReadonlySet<E>;
+    /**
+     * Ingest a snapshot at a given sequence number
+     * @param elements the set of _all_ elements in this snapshot
+     * @param sequenceNo the sequence number for this snapshot
+     */
+    ingestSnapshot(elements: Set<E>, sequenceNo: number): Diff<E>;
+    /**
+     * Ingest a patch
+     * @param elements the set of elements relevant to this patch
+     * @param operation indicates whether this is an addition or a removal
+     * @param sequenceNo the sequence number for this patch
+     */
+    ingestPatch(elements: Set<E>, operation: 'added' | 'removed', sequenceNo: number): Diff<E>;
+}

package/dist/src/plugins/storage/SetMembershipSynchronizer.js

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SetMembershipSynchronizer = exports.EMPTY_DIFF = void 0;
+const utils_1 = require("@streamr/utils");
+const logger = new utils_1.Logger(module);
+exports.EMPTY_DIFF = Object.freeze({
+    added: [],
+    removed: []
+});
+function setDifference(setA, setB) {
+    return [...setA].filter((el) => !setB.has(el));
+}
+/**
+ * Represents a set that evolves over (discrete) time. The set is updated in
+ * two distinct ways:
+ *      1. Snapshots that define the set at a given time step.
+ *      2. Patches that add or remove a subset at a given time step.
+ */
+class SetMembershipSynchronizer {
+    lastSnapshotSequenceNo = 0;
+    state = new Set(); // snapshot + patches
+    lastSequenceNoByElement = new Map();
+    /**
+     * Get up-to-date view of state
+     */
+    getState() {
+        return this.state;
+    }
+    /**
+     * Ingest a snapshot at a given sequence number
+     * @param elements the set of _all_ elements in this snapshot
+     * @param sequenceNo the sequence number for this snapshot
+     */
+    ingestSnapshot(elements, sequenceNo) {
+        if (sequenceNo <= this.lastSnapshotSequenceNo) {
+            logger.warn('Ignore snapshot (stale sequenceNo)', {
+                sequenceNo,
+                lastSnapshotSequenceNo: this.lastSnapshotSequenceNo
+            });
+            return exports.EMPTY_DIFF;
+        }
+        this.lastSnapshotSequenceNo = sequenceNo;
+        // delete stale patches
+        this.lastSequenceNoByElement.forEach((patchSequenceNo, element) => {
+            if (patchSequenceNo <= sequenceNo) {
+                this.lastSequenceNoByElement.delete(element);
+            }
+        });
+        // apply patches
+        const previousState = this.state;
+        this.state = new Set(elements);
+        this.lastSequenceNoByElement.forEach((_, element) => {
+            if (previousState.has(element)) {
+                this.state.add(element);
+            }
+            else {
+                this.state.delete(element);
+            }
+        });
+        return {
+            added: setDifference(this.state, previousState),
+            removed: setDifference(previousState, this.state)
+        };
+    }
+    /**
+     * Ingest a patch
+     * @param elements the set of elements relevant to this patch
+     * @param operation indicates whether this is an addition or a removal
+     * @param sequenceNo the sequence number for this patch
+     */
+    ingestPatch(elements, operation, sequenceNo) {
+        if (sequenceNo <= this.lastSnapshotSequenceNo) {
+            return exports.EMPTY_DIFF;
+        }
+        const nonStaleElements = [...elements].filter((element) => (sequenceNo > (this.lastSequenceNoByElement.get(element) || 0)));
+        nonStaleElements.forEach((element) => {
+            this.lastSequenceNoByElement.set(element, sequenceNo);
+        });
+        if (operation === 'added') {
+            return {
+                removed: [],
+                added: nonStaleElements.filter((element) => {
+                    const didStateChange = !this.state.has(element);
+                    this.state.add(element);
+                    return didStateChange;
+                }),
+            };
+        }
+        else {
+            return {
+                added: [],
+                removed: nonStaleElements.filter((element) => {
+                    const didStateChange = this.state.has(element);
+                    this.state.delete(element);
+                    return didStateChange;
+                })
+            };
+        }
+    }
+}
+exports.SetMembershipSynchronizer = SetMembershipSynchronizer;
+//# sourceMappingURL=SetMembershipSynchronizer.js.map

package/dist/src/plugins/storage/SetMembershipSynchronizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SetMembershipSynchronizer.js","sourceRoot":"","sources":["../../../../src/plugins/storage/SetMembershipSynchronizer.ts"],"names":[],"mappings":";;;AAAA,0CAAuC;AAEvC,MAAM,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,CAAC,CAAA;AAOpB,QAAA,UAAU,GAAgB,MAAM,CAAC,MAAM,CAAC;IACjD,KAAK,EAAE,EAAE;IACT,OAAO,EAAE,EAAE;CACd,CAAC,CAAA;AAEF,SAAS,aAAa,CAAI,IAAY,EAAE,IAAY;IAChD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AAClD,CAAC;AAED;;;;;GAKG;AACH,MAAa,yBAAyB;IAC1B,sBAAsB,GAAG,CAAC,CAAA;IAC1B,KAAK,GAAG,IAAI,GAAG,EAAK,CAAA,CAAC,qBAAqB;IAC1C,uBAAuB,GAAG,IAAI,GAAG,EAAa,CAAA;IAEtD;;OAEG;IACH,QAAQ;QACJ,OAAO,IAAI,CAAC,KAAK,CAAA;IACrB,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,QAAgB,EAAE,UAAkB;QAC/C,IAAI,UAAU,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC5C,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBAC9C,UAAU;gBACV,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;aACtD,CAAC,CAAA;YACF,OAAO,kBAAU,CAAA;QACrB,CAAC;QAED,IAAI,CAAC,sBAAsB,GAAG,UAAU,CAAA;QAExC,uBAAuB;QACvB,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,eAAe,EAAE,OAAO,EAAE,EAAE;YAC9D,IAAI,eAAe,IAAI,UAAU,EAAE,CAAC;gBAChC,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAChD,CAAC;QACL,CAAC,CAAC,CAAA;QAEF,gBAAgB;QAChB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAA;QAChC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,CAAI,QAAQ,CAAC,CAAA;QACjC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YAChD,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAC3B,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAC9B,CAAC;QACL,CAAC,CAAC,CAAA;QAEF,OAAO;YACH,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,CAAC;YAC/C,OAAO,EAAE,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC;SACpD,CAAA;IACL,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,QAAgB,EAAE,SAA8B,EAAE,UAAkB;QAC5E,IAAI,UAAU,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC5C,OAAO,kBAAU,CAAA;QACrB,CAAC;QAED,MAAM,gBAAgB,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CACvD,UAAU,GAAG,CAAC,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAChE,CAAC,CAAA;QAEF,gBAAgB,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACjC,IAAI,CAAC,uBAAuB,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;QACzD,CAAC,CAAC,CAAA;QAEF,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;YACxB,OAAO;gBACH,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;oBACvC,MAAM,cAAc,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;oBAC/C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;oBACvB,OAAO,cAAc,CAAA;gBACzB,CAAC,CAAC;aACL,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,OAAO;gBACH,KAAK,EAAE,EAAE;gBACT,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;oBACzC,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;oBAC9C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;oBAC1B,OAAO,cAAc,CAAA;gBACzB,CAAC,CAAC;aACL,CAAA;QACL,CAAC;IACL,CAAC;CACJ;AA3FD,8DA2FC"}

package/dist/src/plugins/storage/Storage.d.ts

@@ -0,0 +1,47 @@
+/// <reference types="node" />
+/// <reference types="node" />
+/// <reference types="node" />
+import { Client } from 'cassandra-driver';
+import { MetricsContext } from '@streamr/utils';
+import { BatchManager } from './BatchManager';
+import { Readable } from 'stream';
+import { EventEmitter } from 'events';
+import { StreamMessage } from '@streamr/protocol';
+import { BucketManager, BucketManagerOptions } from './BucketManager';
+export interface StartCassandraOptions {
+    contactPoints: string[];
+    localDataCenter: string;
+    keyspace: string;
+    username?: string;
+    password?: string;
+    opts?: Partial<BucketManagerOptions & {
+        useTtl: boolean;
+    }>;
+}
+export type StorageOptions = Partial<BucketManagerOptions> & {
+    useTtl?: boolean;
+    retriesIntervalMilliseconds?: number;
+};
+export declare class Storage extends EventEmitter {
+    opts: StorageOptions;
+    cassandraClient: Client;
+    bucketManager: BucketManager;
+    batchManager: BatchManager;
+    pendingStores: Map<string, NodeJS.Timeout>;
+    constructor(cassandraClient: Client, opts: StorageOptions);
+    store(streamMessage: StreamMessage): Promise<boolean>;
+    requestLast(streamId: string, partition: number, limit: number): Readable;
+    requestFrom(streamId: string, partition: number, fromTimestamp: number, fromSequenceNo: number, publisherId?: string): Readable;
+    requestRange(streamId: string, partition: number, fromTimestamp: number, fromSequenceNo: number, toTimestamp: number, toSequenceNo: number, publisherId: string | undefined, msgChainId: string | undefined): Readable;
+    enableMetrics(metricsContext: MetricsContext): void;
+    close(): Promise<void>;
+    private fetchRange;
+    private queryWithStreamingResults;
+    private parseRow;
+    private createResultStream;
+    getFirstMessageTimestampInStream(streamId: string, partition: number): Promise<number>;
+    getLastMessageTimestampInStream(streamId: string, partition: number): Promise<number>;
+    getNumberOfMessagesInStream(streamId: string, partition: number): Promise<number>;
+    getTotalBytesInStream(streamId: string, partition: number): Promise<number>;
+}
+export declare const startCassandraStorage: ({ contactPoints, localDataCenter, keyspace, username, password, opts }: StartCassandraOptions) => Promise<Storage>;