@streamr/node 100.0.0-rc.0

package/dist/src/plugins/storage/Storage.js

@@ -0,0 +1,459 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startCassandraStorage = exports.Storage = void 0;
+const cassandra_driver_1 = require("cassandra-driver");
+const utils_1 = require("@streamr/utils");
+const BatchManager_1 = require("./BatchManager");
+const stream_1 = require("stream");
+const events_1 = require("events");
+const stream_2 = require("stream");
+const uuid_1 = require("uuid");
+const merge2_1 = __importDefault(require("merge2"));
+const BucketManager_1 = require("./BucketManager");
+const utils_2 = require("@streamr/utils");
+const dataQueryEndpoint_1 = require("./dataQueryEndpoint");
+const trackerless_network_1 = require("@streamr/trackerless-network");
+const logger = new utils_2.Logger(module);
+const MAX_TIMESTAMP_VALUE = 8640000000000000; // https://262.ecma-international.org/5.1/#sec-15.9.1.1
+const MAX_RESEND_LAST = 10000;
+const bucketsToIds = (buckets) => buckets.map((bucket) => bucket.getId());
+class Storage extends events_1.EventEmitter {
+    opts;
+    cassandraClient;
+    bucketManager;
+    batchManager;
+    pendingStores;
+    constructor(cassandraClient, opts) {
+        super();
+        const defaultOptions = {
+            useTtl: false,
+            retriesIntervalMilliseconds: 500
+        };
+        this.opts = {
+            ...defaultOptions,
+            ...opts
+        };
+        this.cassandraClient = cassandraClient;
+        this.bucketManager = new BucketManager_1.BucketManager(cassandraClient, opts);
+        this.batchManager = new BatchManager_1.BatchManager(cassandraClient, {
+            useTtl: this.opts.useTtl
+        });
+        this.pendingStores = new Map();
+    }
+    async store(streamMessage) {
+        logger.debug('Store message', { msgId: streamMessage.messageId });
+        const bucketId = this.bucketManager.getBucketId(streamMessage.getStreamId(), streamMessage.getStreamPartition(), streamMessage.getTimestamp());
+        return new Promise((resolve, reject) => {
+            if (bucketId) {
+                logger.trace('Found bucket', { bucketId });
+                const record = {
+                    streamId: streamMessage.getStreamId(),
+                    partition: streamMessage.getStreamPartition(),
+                    timestamp: streamMessage.getTimestamp(),
+                    sequenceNo: streamMessage.getSequenceNumber(),
+                    publisherId: streamMessage.getPublisherId(),
+                    msgChainId: streamMessage.getMsgChainId(),
+                    payload: Buffer.from((0, trackerless_network_1.convertStreamMessageToBytes)(streamMessage))
+                };
+                this.bucketManager.incrementBucket(bucketId, record.payload.length);
+                setImmediate(() => this.batchManager.store(bucketId, record, (err) => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        this.emit('write', record.payload);
+                        resolve(true);
+                    }
+                }));
+            }
+            else {
+                logger.trace('Move message to pending messages (bucket not found)', {
+                    messageId: JSON.stringify(streamMessage.messageId)
+                });
+                const uuid = (0, uuid_1.v1)();
+                const timeout = setTimeout(() => {
+                    this.pendingStores.delete(uuid);
+                    this.store(streamMessage).then(resolve, reject);
+                }, this.opts.retriesIntervalMilliseconds);
+                this.pendingStores.set(uuid, timeout);
+            }
+        });
+    }
+    requestLast(streamId, partition, limit) {
+        if (limit > MAX_RESEND_LAST) {
+            // eslint-disable-next-line no-param-reassign
+            limit = MAX_RESEND_LAST;
+        }
+        const GET_LAST_N_MESSAGES = 'SELECT payload FROM stream_data WHERE '
+            + 'stream_id = ? AND partition = ? AND bucket_id IN ? '
+            + 'ORDER BY ts DESC, sequence_no DESC '
+            + 'LIMIT ?';
+        const COUNT_MESSAGES = 'SELECT COUNT(*) AS total FROM stream_data WHERE stream_id = ? AND partition = ? AND bucket_id = ?';
+        const GET_BUCKETS = 'SELECT id FROM bucket WHERE stream_id = ? AND partition = ?';
+        let total = 0;
+        const options = {
+            prepare: true, fetchSize: 1
+        };
+        const resultStream = this.createResultStream({ streamId, partition, limit });
+        const makeLastQuery = async (bucketIds) => {
+            try {
+                const params = [streamId, partition, bucketIds, limit];
+                const resultSet = await this.cassandraClient.execute(GET_LAST_N_MESSAGES, params, {
+                    prepare: true,
+                    fetchSize: 0 // disable paging
+                });
+                resultSet.rows.reverse().forEach((r) => {
+                    resultStream.write(r);
+                });
+                resultStream.end();
+            }
+            catch (err) {
+                resultStream.destroy(err);
+            }
+        };
+        let bucketId;
+        const bucketIds = [];
+        /**
+         * Process:
+         * - get latest bucketId => count number of messages in this bucket
+         * - if enough => get all messages and return
+         * - if not => move to the next bucket and repeat cycle
+         */
+        this.cassandraClient.eachRow(GET_BUCKETS, [streamId, partition], options, (_n, row) => {
+            bucketId = row.id;
+            bucketIds.push(bucketId);
+        }, async (err, result) => {
+            // do nothing if resultStream ended
+            if (resultStream.writableEnded || resultStream.readableEnded) {
+                return;
+            }
+            if (err) {
+                resultStream.destroy(err);
+            }
+            else {
+                // no buckets found at all
+                if (!bucketId) {
+                    resultStream.end();
+                    return;
+                }
+                try {
+                    // get total stored message in bucket
+                    const resultSet = await this.cassandraClient.execute(COUNT_MESSAGES, [streamId, partition, bucketId], {
+                        prepare: true,
+                        fetchSize: 0 // disable paging
+                    });
+                    const row = resultSet.first();
+                    total += row.total.low;
+                    // if not enough messages and we next page exists, repeat eachRow
+                    if (result.nextPage && total < limit && total < MAX_RESEND_LAST) {
+                        result.nextPage();
+                    }
+                    else {
+                        makeLastQuery(bucketIds);
+                    }
+                }
+                catch (err) {
+                    resultStream.destroy(err);
+                }
+            }
+        });
+        return resultStream;
+    }
+    requestFrom(streamId, partition, fromTimestamp, fromSequenceNo, publisherId) {
+        return this.fetchRange(streamId, partition, fromTimestamp, fromSequenceNo, MAX_TIMESTAMP_VALUE, dataQueryEndpoint_1.MAX_SEQUENCE_NUMBER_VALUE, publisherId);
+    }
+    requestRange(streamId, partition, fromTimestamp, fromSequenceNo, toTimestamp, toSequenceNo, publisherId, msgChainId) {
+        // TODO is there any reason why we shouldn't allow range queries which contain publisherId, but not msgChainId?
+        // (or maybe even queries with msgChain but without publisherId)
+        const isValidRequest = (publisherId !== undefined && msgChainId !== undefined) || (publisherId === undefined && msgChainId === undefined);
+        if (!isValidRequest) {
+            throw new Error('Invalid combination of requestFrom arguments');
+        }
+        return this.fetchRange(streamId, partition, fromTimestamp, fromSequenceNo, toTimestamp, toSequenceNo, publisherId, msgChainId);
+    }
+    enableMetrics(metricsContext) {
+        const metrics = {
+            readMessagesPerSecond: new utils_1.RateMetric(),
+            readBytesPerSecond: new utils_1.RateMetric(),
+            writeMessagesPerSecond: new utils_1.RateMetric(),
+            writeBytesPerSecond: new utils_1.RateMetric()
+        };
+        metricsContext.addMetrics('broker.plugin.storage', metrics);
+        this.on('read', (streamMessage) => {
+            metrics.readMessagesPerSecond.record(1);
+            metrics.readBytesPerSecond.record(streamMessage.length);
+        });
+        this.on('write', (streamMessage) => {
+            metrics.writeMessagesPerSecond.record(1);
+            metrics.writeBytesPerSecond.record(streamMessage.length);
+        });
+    }
+    close() {
+        const keys = [...this.pendingStores.keys()];
+        keys.forEach((key) => {
+            const timeout = this.pendingStores.get(key);
+            clearTimeout(timeout);
+            this.pendingStores.delete(key);
+        });
+        this.bucketManager.stop();
+        this.batchManager.stop();
+        return this.cassandraClient.shutdown();
+    }
+    fetchRange(streamId, partition, fromTimestamp, fromSequenceNo, toTimestamp, toSequenceNo, publisherId, msgChainId) {
+        const resultStream = this.createResultStream({
+            streamId,
+            partition,
+            fromTimestamp,
+            fromSequenceNo,
+            toTimestamp,
+            toSequenceNo,
+            publisherId,
+            msgChainId,
+        });
+        this.bucketManager.getBucketsByTimestamp(streamId, partition, fromTimestamp, toTimestamp).then((buckets) => {
+            if (buckets.length === 0) {
+                resultStream.end();
+                return;
+            }
+            const bucketIds = bucketsToIds(buckets);
+            let queries;
+            // optimize the typical case where the sequenceNumber doesn't filter out anything
+            if ((fromSequenceNo === dataQueryEndpoint_1.MIN_SEQUENCE_NUMBER_VALUE) && (toSequenceNo === dataQueryEndpoint_1.MAX_SEQUENCE_NUMBER_VALUE)) {
+                queries = [
+                    {
+                        where: 'WHERE stream_id = ? AND partition = ? AND bucket_id IN ? AND ts >= ? AND ts <= ?',
+                        params: [streamId, partition, bucketIds, fromTimestamp, toTimestamp]
+                    }
+                ];
+            }
+            else {
+                queries = [
+                    {
+                        where: 'WHERE stream_id = ? AND partition = ? AND bucket_id IN ? AND ts = ? AND sequence_no >= ?',
+                        params: [streamId, partition, bucketIds, fromTimestamp, fromSequenceNo]
+                    },
+                    {
+                        where: 'WHERE stream_id = ? AND partition = ? AND bucket_id IN ? AND ts > ? AND ts < ?',
+                        params: [streamId, partition, bucketIds, fromTimestamp, toTimestamp]
+                    },
+                    {
+                        where: 'WHERE stream_id = ? AND partition = ? AND bucket_id IN ? AND ts = ? AND sequence_no <= ?',
+                        params: [streamId, partition, bucketIds, toTimestamp, toSequenceNo]
+                    }
+                ];
+            }
+            queries.forEach((q) => {
+                if (publisherId !== undefined) {
+                    q.where += ' AND publisher_id = ?';
+                    q.params.push(publisherId);
+                }
+                if (msgChainId !== undefined) {
+                    q.where += ' AND msg_chain_id = ?';
+                    q.params.push(msgChainId);
+                }
+            });
+            const streams = queries.map((q) => {
+                const select = `SELECT payload FROM stream_data ${q.where} ALLOW FILTERING`;
+                return this.queryWithStreamingResults(select, q.params);
+            });
+            return (0, stream_2.pipeline)(
+            // @ts-expect-error options not in type
+            (0, merge2_1.default)(...streams, {
+                pipeError: true,
+            }), resultStream, (err) => {
+                if (err) {
+                    resultStream.destroy(err);
+                    streams.forEach((s) => s.destroy(undefined));
+                }
+            });
+        })
+            .catch((e) => {
+            resultStream.destroy(e);
+        });
+        return resultStream;
+    }
+    queryWithStreamingResults(query, queryParams) {
+        return this.cassandraClient.stream(query, queryParams, {
+            prepare: true,
+            // force small page sizes, otherwise gives RangeError [ERR_OUT_OF_RANGE]: The value of "offset" is out of range.
+            fetchSize: 128,
+            readTimeout: 0,
+        });
+    }
+    parseRow(row, debugInfo) {
+        if (row.payload === null) {
+            logger.error('Found unexpected message with NULL payload in Cassandra', { debugInfo });
+            return null;
+        }
+        this.emit('read', row.payload);
+        return row.payload;
+    }
+    createResultStream(debugInfo) {
+        const self = this; // eslint-disable-line @typescript-eslint/no-this-alias
+        let last = Date.now();
+        return new stream_1.Transform({
+            highWaterMark: 1024, // buffer up to 1024 messages
+            objectMode: true,
+            transform(row, _, done) {
+                const now = Date.now();
+                const message = self.parseRow(row, debugInfo);
+                if (message !== null) {
+                    this.push(message);
+                }
+                // To avoid blocking main thread for too long, after every 100ms
+                // pause & resume the cassandraStream to give other events in the event
+                // queue a chance to be handled.
+                if ((now - last) > 100) {
+                    setImmediate(() => {
+                        last = Date.now();
+                        done();
+                    });
+                }
+                else {
+                    done();
+                }
+            }
+        });
+    }
+    async getFirstMessageTimestampInStream(streamId, partition) {
+        const bucketQuery = 'SELECT id FROM bucket WHERE stream_id=? AND partition =? ORDER BY date_create ASC LIMIT 1';
+        const queryParams = [streamId, partition];
+        const buckets = await this.cassandraClient.execute(bucketQuery, queryParams, {
+            prepare: true,
+        });
+        if (buckets.rows.length !== 1) {
+            return 0;
+        }
+        const bucketId = buckets.rows[0].id;
+        const query = 'SELECT ts FROM stream_data WHERE stream_id=? AND partition=? AND bucket_id=? ORDER BY ts ASC LIMIT 1';
+        const streams = await this.cassandraClient.execute(query, [
+            streamId,
+            partition,
+            bucketId
+        ], {
+            prepare: true
+        });
+        if (streams.rows.length !== 1) {
+            return 0;
+        }
+        const { ts } = streams.rows[0];
+        return new Date(ts).getTime();
+    }
+    async getLastMessageTimestampInStream(streamId, partition) {
+        const bucketQuery = 'SELECT id FROM bucket WHERE stream_id=? AND partition =? ORDER BY date_create DESC LIMIT 1';
+        const queryParams = [streamId, partition];
+        const buckets = await this.cassandraClient.execute(bucketQuery, queryParams, {
+            prepare: true,
+        });
+        if (buckets.rows.length !== 1) {
+            return 0;
+        }
+        const bucketId = buckets.rows[0].id;
+        const query = 'SELECT ts FROM stream_data WHERE stream_id=? AND partition=? AND bucket_id=? ORDER BY ts DESC LIMIT 1';
+        const streams = await this.cassandraClient.execute(query, [
+            streamId,
+            partition,
+            bucketId
+        ], {
+            prepare: true
+        });
+        if (streams.rows.length !== 1) {
+            return 0;
+        }
+        const { ts } = streams.rows[0];
+        return new Date(ts).getTime();
+    }
+    async getNumberOfMessagesInStream(streamId, partition) {
+        const query = 'SELECT SUM(records) as count FROM bucket WHERE stream_id=? AND partition=?';
+        const queryParams = [
+            streamId,
+            partition
+        ];
+        const res = await this.cassandraClient.execute(query, queryParams, {
+            prepare: true
+        });
+        if (res.rows.length !== 1) {
+            return 0;
+        }
+        const { count } = res.rows[0];
+        return count;
+    }
+    async getTotalBytesInStream(streamId, partition) {
+        const query = 'SELECT SUM(size) as count FROM bucket WHERE stream_id=? AND partition=?';
+        const queryParams = [
+            streamId,
+            partition
+        ];
+        const res = await this.cassandraClient.execute(query, queryParams, {
+            prepare: true
+        });
+        if (res.rows.length !== 1) {
+            return 0;
+        }
+        let { count } = res.rows[0];
+        // Cassandra's integer has overflown, calculate fetching row by row
+        if (count < 0) {
+            count = 0;
+            const query = 'SELECT size FROM bucket WHERE stream_id=? AND partition=?';
+            const queryParams = [
+                streamId,
+                partition
+            ];
+            const res = await this.cassandraClient.execute(query, queryParams, {
+                prepare: true
+            });
+            for (const row of res.rows) {
+                count += row.size;
+            }
+        }
+        return count;
+    }
+}
+exports.Storage = Storage;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(() => resolve(undefined), ms));
+}
+const startCassandraStorage = async ({ contactPoints, localDataCenter, keyspace, username, password, opts }) => {
+    const authProvider = new cassandra_driver_1.auth.PlainTextAuthProvider(username || '', password || '');
+    const requestLogger = new cassandra_driver_1.tracker.RequestLogger({
+        slowThreshold: 10 * 1000, // 10 secs
+    });
+    // @ts-expect-error 'emitter' field is missing in type definition file
+    requestLogger.emitter.on('slow', (message) => {
+        logger.warn('Encountered "slow" event from cassandraClient', { message });
+    });
+    const cassandraClient = new cassandra_driver_1.Client({
+        contactPoints,
+        localDataCenter,
+        keyspace,
+        authProvider,
+        requestTracker: requestLogger,
+        pooling: {
+            maxRequestsPerConnection: 32768
+        }
+    });
+    const nbTrials = 20;
+    let retryCount = nbTrials;
+    let lastError = '';
+    while (retryCount > 0) {
+        /* eslint-disable no-await-in-loop */
+        try {
+            await cassandraClient.connect().catch((err) => { throw err; });
+            return new Storage(cassandraClient, opts || {});
+        }
+        catch (err) {
+            // eslint-disable-next-line no-console
+            console.log('Cassandra not responding yet...');
+            retryCount -= 1;
+            await sleep(5000);
+            lastError = err;
+        }
+        /* eslint-enable no-await-in-loop */
+    }
+    throw new Error(`Failed to connect to Cassandra after ${nbTrials} trials: ${lastError.toString()}`);
+};
+exports.startCassandraStorage = startCassandraStorage;
+//# sourceMappingURL=Storage.js.map

package/dist/src/plugins/storage/Storage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Storage.js","sourceRoot":"","sources":["../../../../src/plugins/storage/Storage.ts"],"names":[],"mappings":";;;;;;AAAA,uDAA+D;AAC/D,0CAA2D;AAC3D,iDAA6C;AAC7C,mCAA4C;AAC5C,mCAAqC;AACrC,mCAAiC;AACjC,+BAAmC;AACnC,oDAA2B;AAE3B,mDAAqE;AACrE,0CAAuC;AAEvC,2DAA0F;AAC1F,sEAA0E;AAE1E,MAAM,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,CAAC,CAAA;AAEjC,MAAM,mBAAmB,GAAG,gBAAgB,CAAA,CAAC,uDAAuD;AACpG,MAAM,eAAe,GAAG,KAAK,CAAA;AAW7B,MAAM,YAAY,GAAG,CAAC,OAAiB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;AAoB3F,MAAa,OAAQ,SAAQ,qBAAY;IAErC,IAAI,CAAgB;IACpB,eAAe,CAAQ;IACvB,aAAa,CAAe;IAC5B,YAAY,CAAc;IAC1B,aAAa,CAA6B;IAE1C,YAAY,eAAuB,EAAE,IAAoB;QACrD,KAAK,EAAE,CAAA;QAEP,MAAM,cAAc,GAAG;YACnB,MAAM,EAAE,KAAK;YACb,2BAA2B,EAAE,GAAG;SACnC,CAAA;QAED,IAAI,CAAC,IAAI,GAAG;YACR,GAAG,cAAc;YACjB,GAAG,IAAI;SACV,CAAA;QAED,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QACtC,IAAI,CAAC,aAAa,GAAG,IAAI,6BAAa,CAAC,eAAe,EAAE,IAAI,CAAC,CAAA;QAC7D,IAAI,CAAC,YAAY,GAAG,IAAI,2BAAY,CAAC,eAAe,EAAE;YAClD,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM;SAC3B,CAAC,CAAA;QACF,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAE,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,aAA4B;QACpC,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC,SAAS,EAAE,CAAC,CAAA;QAEjE,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,EAAE,EAAE,aAAa,CAAC,kBAAkB,EAAE,EAAE,aAAa,CAAC,YAAY,EAAE,CAAC,CAAA;QAE9I,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAA;gBAE1C,MAAM,MAAM,GAAG;oBACX,QAAQ,EAAE,aAAa,CAAC,WAAW,EAAE;oBACrC,SAAS,EAAE,aAAa,CAAC,kBAAkB,EAAE;oBAC7C,SAAS,EAAE,aAAa,CAAC,YAAY,EAAE;oBACvC,UAAU,EAAE,aAAa,CAAC,iBAAiB,EAAE;oBAC7C,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;oBAC3C,UAAU,EAAE,aAAa,CAAC,aAAa,EAAE;oBACzC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAA,iDAA2B,EAAC,aAAa,CAAC,CAAC;iBACnE,CAAA;gBAED,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;gBACnE,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,GAAW,EAAE,EAAE;oBACzE,IAAI,GAAG,EAAE,CAAC;wBACN,MAAM,CAAC,GAAG,CAAC,CAAA;oBACf,CAAC;yBAAM,CAAC;wBACJ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;wBAClC,OAAO,CAAC,IAAI,CAAC,CAAA;oBACjB,CAAC;gBACL,CAAC,CAAC,CAAC,CAAA;YACP,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,qDAAqD,EAAE;oBAChE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,CAAC;iBACrD,CAAC,CAAA;gBAEF,MAAM,IAAI,GAAG,IAAA,SAAM,GAAE,CAAA;gBACrB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;oBAC/B,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;gBACnD,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA;gBACzC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACzC,CAAC;QACL,CAAC,CAAC,CAAA;IACN,CAAC;IAED,WAAW,CAAC,QAAgB,EAAE,SAAiB,EAAE,KAAa;QAC1D,IAAI,KAAK,GAAG,eAAe,EAAE,CAAC;YAC1B,6CAA6C;YAC7C,KAAK,GAAG,eAAe,CAAA;QAC3B,CAAC;QAED,MAAM,mBAAmB,GAAG,wCAAwC;cAC9D,qDAAqD;cACrD,qCAAqC;cACrC,SAAS,CAAA;QACf,MAAM,cAAc,GAAG,mGAAmG,CAAA;QAC1H,MAAM,WAAW,GAAG,6DAA6D,CAAA;QAEjF,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,OAAO,GAAG;YACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;SAC9B,CAAA;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAA;QAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,SAAqB,EAAE,EAAE;YAClD,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;gBACtD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,EAAE;oBAC9E,OAAO,EAAE,IAAI;oBACb,SAAS,EAAE,CAAC,CAAC,iBAAiB;iBACjC,CAAC,CAAA;gBACF,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAY,EAAE,EAAE;oBAC9C,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;gBACzB,CAAC,CAAC,CAAA;gBACF,YAAY,CAAC,GAAG,EAAE,CAAA;YACtB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YAC7B,CAAC;QACL,CAAC,CAAA;QAED,IAAI,QAAkB,CAAA;QACtB,MAAM,SAAS,GAAe,EAAE,CAAA;QAChC;;;;;WAKG;QACH,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,GAAc,EAAE,EAAE;YAC7F,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAA;YACjB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC5B,CAAC,EAAE,KAAK,EAAE,GAAsB,EAAE,MAAuB,EAAE,EAAE;YACzD,mCAAmC;YACnC,IAAI,YAAY,CAAC,aAAa,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;gBAAC,OAAM;YAAC,CAAC;YACxE,IAAI,GAAG,EAAE,CAAC;gBACN,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YAC7B,CAAC;iBAAM,CAAC;gBACJ,0BAA0B;gBAC1B,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACZ,YAAY,CAAC,GAAG,EAAE,CAAA;oBAClB,OAAM;gBACV,CAAC;gBACD,IAAI,CAAC;oBACD,qCAAqC;oBACrC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE;wBAClG,OAAO,EAAE,IAAI;wBACb,SAAS,EAAE,CAAC,CAAC,iBAAiB;qBACjC,CAAC,CAAA;oBACF,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,CAAA;oBAC7B,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAA;oBAEtB,iEAAiE;oBACjE,IAAI,MAAM,CAAC,QAAQ,IAAI,KAAK,GAAG,KAAK,IAAI,KAAK,GAAG,eAAe,EAAE,CAAC;wBAC9D,MAAM,CAAC,QAAQ,EAAE,CAAA;oBACrB,CAAC;yBAAM,CAAC;wBACJ,aAAa,CAAC,SAAS,CAAC,CAAA;oBAC5B,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;gBAC7B,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAA;QAEF,OAAO,YAAY,CAAA;IACvB,CAAC;IAED,WAAW,CAAC,QAAgB,EAAE,SAAiB,EAAE,aAAqB,EAAE,cAAsB,EAAE,WAAoB;QAChH,OAAO,IAAI,CAAC,UAAU,CAClB,QAAQ,EACR,SAAS,EACT,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,6CAAyB,EACzB,WAAW,CACd,CAAA;IACL,CAAC;IAED,YAAY,CACR,QAAgB,EAChB,SAAiB,EACjB,aAAqB,EACrB,cAAsB,EACtB,WAAmB,EACnB,YAAoB,EACpB,WAA+B,EAC/B,UAA8B;QAE9B,+GAA+G;QAC/G,gEAAgE;QAChE,MAAM,cAAc,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,CAAC,CAAA;QACzI,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;QACnE,CAAC;QACD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,EACrD,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,CAAC,CAAA;IAC3E,CAAC;IAED,aAAa,CAAC,cAA8B;QACxC,MAAM,OAAO,GAAG;YACZ,qBAAqB,EAAE,IAAI,kBAAU,EAAE;YACvC,kBAAkB,EAAE,IAAI,kBAAU,EAAE;YACpC,sBAAsB,EAAE,IAAI,kBAAU,EAAE;YACxC,mBAAmB,EAAE,IAAI,kBAAU,EAAE;SACxC,CAAA;QACD,cAAc,CAAC,UAAU,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAA;QAC3D,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,aAAyB,EAAE,EAAE;YAC1C,OAAO,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YACvC,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,aAAyB,EAAE,EAAE;YAC3C,OAAO,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YACxC,OAAO,CAAC,mBAAmB,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;QAC5D,CAAC,CAAC,CAAA;IACN,CAAC;IAED,KAAK;QACD,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAA;QAC3C,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC3C,YAAY,CAAC,OAAO,CAAC,CAAA;YACrB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAClC,CAAC,CAAC,CAAA;QAEF,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAA;QACzB,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;QACxB,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAA;IAC1C,CAAC;IAEO,UAAU,CACd,QAAgB,EAChB,SAAiB,EACjB,aAAqB,EACrB,cAAsB,EACtB,WAAmB,EACnB,YAAoB,EACpB,WAAoB,EACpB,UAAmB;QAEnB,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC;YACzC,QAAQ;YACR,SAAS;YACT,aAAa;YACb,cAAc;YACd,WAAW;YACX,YAAY;YACZ,WAAW;YACX,UAAU;SACb,CAAC,CAAA;QAEF,IAAI,CAAC,aAAa,CAAC,qBAAqB,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,OAAiB,EAAE,EAAE;YACjH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,YAAY,CAAC,GAAG,EAAE,CAAA;gBAClB,OAAM;YACV,CAAC;YAED,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,CAAA;YAEvC,IAAI,OAAO,CAAA;YACX,iFAAiF;YACjF,IAAI,CAAC,cAAc,KAAK,6CAAyB,CAAC,IAAI,CAAC,YAAY,KAAK,6CAAyB,CAAC,EAAE,CAAC;gBACjG,OAAO,GAAG;oBACN;wBACI,KAAK,EAAE,kFAAkF;wBACzF,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC;qBACvE;iBACJ,CAAA;YACL,CAAC;iBAAM,CAAC;gBACJ,OAAO,GAAG;oBACN;wBACI,KAAK,EAAE,0FAA0F;wBACjG,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,cAAc,CAAC;qBAC1E;oBACD;wBACI,KAAK,EAAE,gFAAgF;wBACvF,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,WAAW,CAAC;qBACvE;oBACD;wBACI,KAAK,EAAE,0FAA0F;wBACjG,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC;qBACtE;iBACJ,CAAA;YACL,CAAC;YAED,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBAClB,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;oBAC5B,CAAC,CAAC,KAAK,IAAI,uBAAuB,CAAA;oBAClC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;gBAC9B,CAAC;gBACD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;oBAC3B,CAAC,CAAC,KAAK,IAAI,uBAAuB,CAAA;oBAClC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBAC7B,CAAC;YACL,CAAC,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC9B,MAAM,MAAM,GAAG,mCAAmC,CAAC,CAAC,KAAK,kBAAkB,CAAA;gBAC3E,OAAO,IAAI,CAAC,yBAAyB,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAA;YAC3D,CAAC,CAAC,CAAA;YAEF,OAAO,IAAA,iBAAQ;YACX,uCAAuC;YACvC,IAAA,gBAAM,EAAC,GAAG,OAAO,EAAE;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,EACF,YAAY,EACZ,CAAC,GAAiB,EAAE,EAAE;gBAClB,IAAI,GAAG,EAAE,CAAC;oBACN,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;oBACzB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAA;gBAChD,CAAC;YACL,CAAC,CACJ,CAAA;QACL,CAAC,CAAC;aACG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEN,OAAO,YAAY,CAAA;IACvB,CAAC;IAEO,yBAAyB,CAAC,KAAa,EAAE,WAAkB;QAC/D,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,EAAE;YACnD,OAAO,EAAE,IAAI;YACb,gHAAgH;YAChH,SAAS,EAAE,GAAG;YACd,WAAW,EAAE,CAAC;SACjB,CAAa,CAAA;IAClB,CAAC;IAEO,QAAQ,CAAC,GAAc,EAAE,SAA0B;QACvD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,KAAK,CAAC,yDAAyD,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;YACtF,OAAO,IAAI,CAAA;QACf,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAA;QAC9B,OAAO,GAAG,CAAC,OAAO,CAAA;IACtB,CAAC;IAEO,kBAAkB,CAAC,SAA0B;QACjD,MAAM,IAAI,GAAG,IAAI,CAAA,CAAC,uDAAuD;QACzE,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACrB,OAAO,IAAI,kBAAS,CAAC;YACjB,aAAa,EAAE,IAAI,EAAE,6BAA6B;YAClD,UAAU,EAAE,IAAI;YAChB,SAAS,CAAC,GAAc,EAAE,CAAC,EAAE,IAAI;gBAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;gBACtB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;gBAC7C,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;oBACnB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBACtB,CAAC;gBACD,gEAAgE;gBAChE,uEAAuE;gBACvE,gCAAgC;gBAChC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;oBACrB,YAAY,CAAC,GAAG,EAAE;wBACd,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;wBACjB,IAAI,EAAE,CAAA;oBACV,CAAC,CAAC,CAAA;gBACN,CAAC;qBAAM,CAAC;oBACJ,IAAI,EAAE,CAAA;gBACV,CAAC;YACL,CAAC;SACJ,CAAC,CAAA;IACN,CAAC;IAED,KAAK,CAAC,gCAAgC,CAAC,QAAgB,EAAE,SAAiB;QACtE,MAAM,WAAW,GAAG,2FAA2F,CAAA;QAE/G,MAAM,WAAW,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;QAEzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,WAAW,EAAE,WAAW,EAAE;YACzE,OAAO,EAAE,IAAI;SAChB,CAAC,CAAA;QAEF,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAA;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEnC,MAAM,KAAK,GAAG,sGAAsG,CAAA;QAEpH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE;YACtD,QAAQ;YACR,SAAS;YACT,QAAQ;SACX,EAAE;YACC,OAAO,EAAE,IAAI;SAChB,CAAC,CAAA;QAEF,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAA;QACZ,CAAC;QAED,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9B,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,+BAA+B,CAAC,QAAgB,EAAE,SAAiB;QACrE,MAAM,WAAW,GAAG,4FAA4F,CAAA;QAEhH,MAAM,WAAW,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;QAEzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,WAAW,EAAE,WAAW,EAAE;YACzE,OAAO,EAAE,IAAI;SAChB,CAAC,CAAA;QAEF,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAA;QACZ,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEnC,MAAM,KAAK,GAAG,uGAAuG,CAAA;QAErH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE;YACtD,QAAQ;YACR,SAAS;YACT,QAAQ;SACX,EAAE;YACC,OAAO,EAAE,IAAI;SAChB,CAAC,CAAA;QAEF,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAA;QACZ,CAAC;QAED,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9B,OAAO,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,2BAA2B,CAAC,QAAgB,EAAE,SAAiB;QACjE,MAAM,KAAK,GAAG,4EAA4E,CAAA;QAC1F,MAAM,WAAW,GAAG;YAChB,QAAQ;YACR,SAAS;SACZ,CAAA;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE;YAC/D,OAAO,EAAE,IAAI;SAChB,CAAC,CAAA;QAEF,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,CAAA;QACZ,CAAC;QAED,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7B,OAAO,KAAK,CAAA;IAChB,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,QAAgB,EAAE,SAAiB;QAC3D,MAAM,KAAK,GAAG,yEAAyE,CAAA;QACvF,MAAM,WAAW,GAAG;YAChB,QAAQ;YACR,SAAS;SACZ,CAAA;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE;YAC/D,OAAO,EAAE,IAAI;SAChB,CAAC,CAAA;QAEF,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,CAAA;QACZ,CAAC;QAED,IAAI,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE3B,mEAAmE;QACnE,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACZ,KAAK,GAAG,CAAC,CAAA;YAET,MAAM,KAAK,GAAG,2DAA2D,CAAA;YACzE,MAAM,WAAW,GAAG;gBAChB,QAAQ;gBACR,SAAS;aACZ,CAAA;YAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE;gBAC/D,OAAO,EAAE,IAAI;aAChB,CAAC,CAAA;YAEF,KAAK,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACzB,KAAK,IAAI,GAAG,CAAC,IAAI,CAAA;YACrB,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAA;IAChB,CAAC;CAEJ;AAjeD,0BAieC;AAED,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;AAC7E,CAAC;AAEM,MAAM,qBAAqB,GAAG,KAAK,EAAE,EACxC,aAAa,EACb,eAAe,EACf,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,IAAI,EACgB,EAAoB,EAAE;IAC1C,MAAM,YAAY,GAAG,IAAI,uBAAI,CAAC,qBAAqB,CAAC,QAAQ,IAAI,EAAE,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAA;IACnF,MAAM,aAAa,GAAG,IAAI,0BAAO,CAAC,aAAa,CAAC;QAC5C,aAAa,EAAE,EAAE,GAAG,IAAI,EAAE,UAAU;KACvC,CAAC,CAAA;IACF,sEAAsE;IACtE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAa,EAAE,EAAE;QAC/C,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;IAC7E,CAAC,CAAC,CAAA;IACF,MAAM,eAAe,GAAG,IAAI,yBAAM,CAAC;QAC/B,aAAa;QACb,eAAe;QACf,QAAQ;QACR,YAAY;QACZ,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAE;YACL,wBAAwB,EAAE,KAAK;SAClC;KACJ,CAAC,CAAA;IACF,MAAM,QAAQ,GAAG,EAAE,CAAA;IACnB,IAAI,UAAU,GAAG,QAAQ,CAAA;IACzB,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,OAAO,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,qCAAqC;QACrC,IAAI,CAAC;YACD,MAAM,eAAe,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,GAAG,MAAM,GAAG,CAAA,CAAC,CAAC,CAAC,CAAA;YAC7D,OAAO,IAAI,OAAO,CAAC,eAAe,EAAE,IAAI,IAAI,EAAE,CAAC,CAAA;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,sCAAsC;YACtC,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAA;YAC9C,UAAU,IAAI,CAAC,CAAA;YACf,MAAM,KAAK,CAAC,IAAI,CAAC,CAAA;YACjB,SAAS,GAAG,GAAG,CAAA;QACnB,CAAC;QACD,oCAAoC;IACxC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,wCAAwC,QAAQ,YAAY,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;AACvG,CAAC,CAAA;AA5CY,QAAA,qBAAqB,yBA4CjC"}

package/dist/src/plugins/storage/StorageConfig.d.ts

@@ -0,0 +1,43 @@
+import { EthereumAddress } from '@streamr/utils';
+import { StreamPartID } from '@streamr/protocol';
+import { StreamrClient } from '@streamr/sdk';
+export interface StorageConfigListener {
+    onStreamPartAdded: (streamPart: StreamPartID) => void;
+    onStreamPartRemoved: (streamPart: StreamPartID) => void;
+}
+/**
+ * Manages the two data sources for storage node assignments (poll-based and
+ * event-based), feeding the received full state and partial state updates to
+ * `StorageAssignmentSynchronizer`. The state diffs produced by the
+ * synchronizer are then further delivered to the user of this class via
+ * listeners.
+ *
+ * The two data sources, heterogeneous in nature, are:
+ *
+ *  (1) Poll-based storage node assignments occurring on a scheduled interval
+ *      (reliable, large payload, infrequent, may be stale)
+ *
+ *  (2) Event-based storage node assignments picked up in real-time
+ *      (intermittent, small payload, frequent, up-to-date)
+ *
+ *  Event-based assignments are great for picking up on changes quickly.
+ *  However, there is a risk of not receiving updates due to, e.g. connectivity
+ *  issues. Therefore, if the real-time system fails, polling acts as a sort-of
+ *  backup system.
+ */
+export declare class StorageConfig {
+    private readonly listener;
+    private readonly synchronizer;
+    private readonly clusterSize;
+    private readonly myIndexInCluster;
+    private readonly storagePoller;
+    private readonly storageEventListener;
+    private readonly abortController;
+    constructor(clusterId: EthereumAddress, clusterSize: number, myIndexInCluster: number, pollInterval: number, streamrClient: StreamrClient, listener: StorageConfigListener);
+    start(): Promise<void>;
+    destroy(): void;
+    hasStreamPart(streamPart: StreamPartID): boolean;
+    getStreamParts(): ReadonlySet<StreamPartID>;
+    private createMyStreamParts;
+    private handleDiff;
+}

package/dist/src/plugins/storage/StorageConfig.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StorageConfig = void 0;
+const utils_1 = require("@streamr/utils");
+const SetMembershipSynchronizer_1 = require("./SetMembershipSynchronizer");
+const StoragePoller_1 = require("./StoragePoller");
+const StorageEventListener_1 = require("./StorageEventListener");
+const logger = new utils_1.Logger(module);
+/**
+ * Manages the two data sources for storage node assignments (poll-based and
+ * event-based), feeding the received full state and partial state updates to
+ * `StorageAssignmentSynchronizer`. The state diffs produced by the
+ * synchronizer are then further delivered to the user of this class via
+ * listeners.
+ *
+ * The two data sources, heterogeneous in nature, are:
+ *
+ *  (1) Poll-based storage node assignments occurring on a scheduled interval
+ *      (reliable, large payload, infrequent, may be stale)
+ *
+ *  (2) Event-based storage node assignments picked up in real-time
+ *      (intermittent, small payload, frequent, up-to-date)
+ *
+ *  Event-based assignments are great for picking up on changes quickly.
+ *  However, there is a risk of not receiving updates due to, e.g. connectivity
+ *  issues. Therefore, if the real-time system fails, polling acts as a sort-of
+ *  backup system.
+ */
+class StorageConfig {
+    listener;
+    synchronizer = new SetMembershipSynchronizer_1.SetMembershipSynchronizer();
+    clusterSize;
+    myIndexInCluster;
+    storagePoller;
+    storageEventListener;
+    abortController;
+    constructor(clusterId, clusterSize, myIndexInCluster, pollInterval, streamrClient, listener) {
+        this.clusterSize = clusterSize;
+        this.myIndexInCluster = myIndexInCluster;
+        this.listener = listener;
+        this.storagePoller = new StoragePoller_1.StoragePoller(clusterId, pollInterval, streamrClient, (streams, block) => {
+            const streamParts = streams.flatMap((stream) => ([
+                ...this.createMyStreamParts(stream)
+            ]));
+            this.handleDiff(this.synchronizer.ingestSnapshot(new Set(streamParts), block));
+        });
+        this.storageEventListener = new StorageEventListener_1.StorageEventListener(clusterId, streamrClient, (stream, type, block) => {
+            const streamParts = this.createMyStreamParts(stream);
+            this.handleDiff(this.synchronizer.ingestPatch(streamParts, type, block));
+        });
+        this.abortController = new AbortController();
+    }
+    async start() {
+        this.storageEventListener.start();
+        await this.storagePoller.start(this.abortController.signal);
+    }
+    destroy() {
+        this.abortController.abort();
+        this.storageEventListener.destroy();
+    }
+    hasStreamPart(streamPart) {
+        return this.getStreamParts().has(streamPart);
+    }
+    getStreamParts() {
+        return this.synchronizer.getState();
+    }
+    createMyStreamParts(stream) {
+        return new Set(stream.getStreamParts().filter((streamPart) => {
+            const hashedIndex = (0, utils_1.keyToArrayIndex)(this.clusterSize, streamPart);
+            return hashedIndex === this.myIndexInCluster;
+        }));
+    }
+    handleDiff({ added, removed }) {
+        added.forEach((streamPart) => this.listener.onStreamPartAdded(streamPart));
+        removed.forEach((streamPart) => this.listener.onStreamPartRemoved(streamPart));
+        logger.info('Updated state', {
+            added,
+            removed
+        });
+    }
+}
+exports.StorageConfig = StorageConfig;
+//# sourceMappingURL=StorageConfig.js.map

package/dist/src/plugins/storage/StorageConfig.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StorageConfig.js","sourceRoot":"","sources":["../../../../src/plugins/storage/StorageConfig.ts"],"names":[],"mappings":";;;AAAA,0CAAyE;AAGzE,2EAA6E;AAC7E,mDAA+C;AAC/C,iEAA6D;AAE7D,MAAM,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,CAAC,CAAA;AAOjC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAa,aAAa;IACL,QAAQ,CAAuB;IAC/B,YAAY,GAAG,IAAI,qDAAyB,EAAgB,CAAA;IAC5D,WAAW,CAAQ;IACnB,gBAAgB,CAAQ;IACxB,aAAa,CAAe;IAC5B,oBAAoB,CAAsB;IAC1C,eAAe,CAAiB;IAEjD,YACI,SAA0B,EAC1B,WAAmB,EACnB,gBAAwB,EACxB,YAAoB,EACpB,aAA4B,EAC5B,QAA+B;QAE/B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAA;QAC9B,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;QACxB,IAAI,CAAC,aAAa,GAAG,IAAI,6BAAa,CAAC,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YAC9F,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAc,EAAE,EAAE,CAAC,CAAC;gBACrD,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;aACtC,CAAC,CAAC,CAAA;YACH,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,IAAI,GAAG,CAAe,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC,CAAA;QAChG,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,oBAAoB,GAAG,IAAI,2CAAoB,CAAC,SAAS,EAAE,aAAa,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACnG,MAAM,WAAW,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAA;YACpD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;QAC5E,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;IAChD,CAAC;IAED,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,CAAA;QACjC,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;IAC/D,CAAC;IAED,OAAO;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;QAC5B,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,CAAA;IACvC,CAAC;IAED,aAAa,CAAC,UAAwB;QAClC,OAAO,IAAI,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;IAChD,CAAC;IAED,cAAc;QACV,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAA;IACvC,CAAC;IAEO,mBAAmB,CAAC,MAAc;QACtC,OAAO,IAAI,GAAG,CAAe,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE;YACvE,MAAM,WAAW,GAAG,IAAA,uBAAe,EAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAA;YACjE,OAAO,WAAW,KAAK,IAAI,CAAC,gBAAgB,CAAA;QAChD,CAAC,CAAC,CAAC,CAAA;IACP,CAAC;IAEO,UAAU,CAAC,EAAE,KAAK,EAAE,OAAO,EAAsB;QACrD,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAA;QAC1E,OAAO,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC,CAAA;QAC9E,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE;YACzB,KAAK;YACL,OAAO;SACV,CAAC,CAAA;IACN,CAAC;CACJ;AAlED,sCAkEC"}

package/dist/src/plugins/storage/StorageEventListener.d.ts

@@ -0,0 +1,17 @@
+import { Stream, StreamrClient } from '@streamr/sdk';
+import { EthereumAddress } from '@streamr/utils';
+/**
+ * Hooks up to StreamrClient event listener to learn about
+ * stream assignment and removal events in real-time.
+ */
+export declare class StorageEventListener {
+    private readonly clusterId;
+    private readonly streamrClient;
+    private readonly onEvent;
+    private readonly onAddToStorageNode;
+    private readonly onRemoveFromStorageNode;
+    constructor(clusterId: EthereumAddress, streamrClient: StreamrClient, onEvent: (stream: Stream, type: 'added' | 'removed', block: number) => void);
+    private handleEvent;
+    start(): void;
+    destroy(): void;
+}