@eclesia/indexer-engine 2.7.6 → 2.8.0

package/dist/indexer/index.js

@@ -1,719 +1,553 @@
-/* eslint-disable max-lines */
-import { createHash, } from "node:crypto";
-import fs from "node:fs";
-import { toRfc3339WithNanoseconds, } from "@cosmjs/tendermint-rpc";
-import { connectComet, } from "@cosmjs/tendermint-rpc";
-import { MsgExec, } from "cosmjs-types/cosmos/authz/v1beta1/tx";
-import { QueryValidatorsRequest, QueryValidatorsResponse, } from "cosmjs-types/cosmos/staking/v1beta1/query";
-import { Tx, } from "cosmjs-types/cosmos/tx/v1beta1/tx";
+import { EclesiaEmitter } from "../emitter/index.js";
+import { CircularBuffer } from "../promise-queue/index.js";
+import { decodeAttr } from "../utils/text.js";
+import "../utils/index.js";
+import { createHash } from "node:crypto";
+import * as fs from "node:fs";
+import { connectComet, toRfc3339WithNanoseconds } from "@cosmjs/tendermint-rpc";
+import { MsgExec } from "cosmjs-types/cosmos/authz/v1beta1/tx.js";
+import { QueryValidatorsRequest, QueryValidatorsResponse } from "cosmjs-types/cosmos/staking/v1beta1/query.js";
+import { Tx } from "cosmjs-types/cosmos/tx/v1beta1/tx.js";
 import Fastify from "fastify";
-import { chain, } from "stream-chain";
-import { parser, } from "stream-json";
-import { pick, } from "stream-json/filters/Pick";
-import { streamArray, } from "stream-json/streamers/StreamArray";
-import { streamValues, } from "stream-json/streamers/StreamValues";
-import { batch, } from "stream-json/utils/Batch";
-import { v4 as uuidv4, } from "uuid";
-import winston from "winston";
-import { EclesiaEmitter, } from "../emitter";
-import { CircularBuffer, } from "../promise-queue";
-import { decodeAttr, } from "../utils";
-export const defaultIndexerConfig = {
-    startHeight: 1,
-    batchSize: 500,
-    modules: [],
-    getNextHeight: () => 1,
-    logLevel: "info",
-    usePolling: false,
-    pollingInterval: 5000,
-    processGenesis: false,
-    minimal: true,
-    init: () => Promise.resolve(),
-    beginTransaction: () => Promise.resolve(),
-    endTransaction: (_status) => Promise.resolve(),
+import { chain } from "stream-chain";
+import * as Parser from "stream-json";
+import * as Pick from "stream-json/filters/Pick.js";
+import * as StreamArray from "stream-json/streamers/StreamArray.js";
+import * as StreamValues from "stream-json/streamers/StreamValues.js";
+import * as Batch from "stream-json/utils/Batch.js";
+import { v4 } from "uuid";
+import * as winston from "winston";
+
+//#region src/indexer/index.ts
+const defaultIndexerConfig = {
+	startHeight: 1,
+	batchSize: 500,
+	modules: [],
+	getNextHeight: () => 1,
+	logLevel: "info",
+	usePolling: false,
+	pollingInterval: 5e3,
+	processGenesis: false,
+	minimal: true,
+	init: () => Promise.resolve(),
+	beginTransaction: () => Promise.resolve(),
+	endTransaction: (_status) => Promise.resolve()
 };
-export class EcleciaIndexer extends EclesiaEmitter {
-    constructor(config) {
-        super();
-        this.initialized = false;
-        this.retryCount = 0;
-        this.tryToRecover = false;
-        this.healthCheck = {
-            status: "CONNECTING",
-        };
-        this.subscription = null;
-        this.blockListener = {
-            next: (data) => {
-                this.newBlockReceived(data.header.height);
-            },
-        };
-        this.asyncEmit = async (type, event) => {
-            event.uuid = uuidv4();
-            /*
-             * More than 1 listener can be registered for an event type
-             * Fortunately these are all set up during module init() so we have a consistent count
-             * so we can count responses to resolve when complete
-             * values are irrelevant as promise resolution is only used for flow control
-             */
-            let listenerCount = this.handled.get(type);
-            if (!listenerCount) {
-                // Setting listenerCount to 1 (the unhandled listener)
-                listenerCount = 1;
-            }
-            let listenersResponded = 0;
-            const prom = new Promise((resolve, reject) => {
-                const returnFunc = (ev) => {
-                    if (ev.uuid == event.uuid) {
-                        if (ev.status) {
-                            listenersResponded++;
-                            if (listenersResponded == listenerCount) {
-                                // All listeners have done their thing so we can remove listener, resolve and continue execution
-                                this.off("uuid", returnFunc);
-                                resolve();
-                            }
-                        }
-                        else {
-                            // At least 1 listener is reporting an error. Reject and handle exception at the original asyncEmit location
-                            reject(ev.error);
-                        }
-                    }
-                };
-                this.on("uuid", returnFunc);
-            });
-            this.emit(type, event);
-            return prom;
-        };
-        this.config = {
-            ...defaultIndexerConfig,
-            ...config,
-        };
-        if (this.config.minimal) {
-            this.blockQueue = new CircularBuffer(this.config.batchSize);
-        }
-        else {
-            this.blockQueue = new CircularBuffer(this.config.batchSize);
-        }
-        const { printf, } = winston.format;
-        const eclesiaFormat = printf(({ level, message, timestamp, }) => {
-            return `${timestamp} [${level.toUpperCase()}]:\t${message}`;
-        });
-        this.log = winston.createLogger({
-            level: this.config.logLevel,
-            defaultMeta: {
-                service: "Eclesia Indexer",
-            },
-            transports: [
-                new winston.transports.File({
-                    filename: "error.log",
-                    level: "error",
-                }),
-                new winston.transports.File({
-                    filename: "combined.log",
-                }),
-                new winston.transports.Console({
-                    format: winston.format.combine(winston.format.splat(), winston.format.timestamp(), eclesiaFormat, winston.format.colorize({
-                        all: true,
-                    })),
-                }),
-            ],
-        });
-        this.fastify = Fastify({
-            logger: false,
-        });
-        this.on("_unhandled", (msg) => {
-            if (msg.uuid) {
-                this.log.verbose("Unhandled event: " + msg.type);
-                this.emit("uuid", {
-                    status: true,
-                    uuid: msg.uuid,
-                });
-            }
-        });
-        this.fastify.get("/health", async (_request, reply) => {
-            const code = this.healthCheck.status == "OK"
-                ? 200
-                : 503;
-            reply.code(code).send(this.healthCheck);
-        });
-        this.fastify.listen({
-            port: 80,
-            host: "0.0.0.0",
-        }, (err) => {
-            if (err) {
-                this.log.error(err);
-                process.exit(1);
-            }
-        });
-    }
-    setStatus(status) {
-        this.healthCheck.status = status;
-    }
-    isMinimal(_blockqueue) {
-        if (this.config.minimal) {
-            return true;
-        }
-        else {
-            return false;
-        }
-    }
-    async connect() {
-        try {
-            if (this.client && this.tryToRecover) {
-                this.log.verbose("Recover from error. Attempting to disconnect from RPC");
-                this.client.disconnect();
-                this.blockClient.disconnect();
-                this.log.verbose("Disconnected from RPC");
-            }
-            this.client = await connectComet(this.config.rpcUrl);
-            this.log.info("Connected to RPC for ad hoc queries");
-            this.blockClient = await connectComet(this.config.rpcUrl);
-            this.log.info("Connected to RPC for block & validator info");
-            return true;
-        }
-        catch (error) {
-            this.log.error(error);
-            this.tryToRecover = true;
-            return false;
-        }
-    }
-    async start() {
-        if (this.blockQueue) {
-            this.blockQueue.clear();
-            this.log.verbose("Starting, clearing block queue");
-        }
-        if (!this.initialized) {
-            try {
-                if (this.config.init) {
-                    await this.config.init();
-                }
-            }
-            catch (e) {
-                this.log.error("Failed to initialize indexer: " + e);
-                this.setStatus("FAILED");
-                throw e;
-            }
-            if (this.config.processGenesis) {
-                try {
-                    if (this.config.genesisPath) {
-                        await this.parseGenesis();
-                    }
-                }
-                catch (e) {
-                    this.log.error("Failed to parse genesis: " + e);
-                    this.setStatus("FAILED");
-                    throw e;
-                }
-            }
-            this.initialized = true;
-        }
-        try {
-            await this.connect();
-            if (!this.config.usePolling && this.subscription) {
-                this.subscription.removeListener(this.blockListener);
-                this.subscription = null;
-                this.log.verbose("Removed existing block listener and subscription");
-            }
-            this.subscription = this.client.subscribeNewBlock
-                ? this.client.subscribeNewBlock()
-                : null;
-            const status = await this.client.status();
-            this.latestHeight = status.syncInfo.latestBlockHeight;
-            this.log.info("Current chain height: " + this.latestHeight);
-            this.heightToProcess = await this.config.getNextHeight();
-            if (this.config.usePolling) {
-                this.pollForBlock();
-            }
-            else {
-                if (this.subscription) {
-                    this.subscription.addListener(this.blockListener);
-                }
-                else {
-                    throw new Error("Could not subscribe to new blocks");
-                }
-            }
-        }
-        catch (e) {
-            this.log.error("Failed to set up block listening: " + e);
-            this.setStatus("FAILED");
-            throw e;
-        }
-        this.tryToRecover = false;
-        this.fetcher().catch((e) => {
-            this.setStatus("FAILED");
-            throw new Error("Error in fetching service: " + e);
-        });
-        const hrTime = process.hrtime();
-        let ms = hrTime[0] * 1000000 + hrTime[1] / 1000;
-        while (this.blockQueue.size() > 0 && !this.tryToRecover) {
-            // await the dequeued promise is essentially awaiting fetched data for that block
-            try {
-                if (this.tryToRecover) {
-                    throw new Error("Exiting processing loop. Attempting to recover indexer");
-                }
-                // Index block inside a db transaction to ensure data consistency
-                await this.config.beginTransaction();
-                this.log.silly("Started db tx");
-                let height, timestamp;
-                if (this.isMinimal(this.blockQueue)) {
-                    const toProcess = await this.blockQueue.dequeue();
-                    this.log.silly("Retrieved block data");
-                    if (!toProcess || !toProcess[0] || !toProcess[1]) {
-                        throw new Error("Could not fetch block");
-                    }
-                    height = toProcess[0].block.header.height;
-                    timestamp = toRfc3339WithNanoseconds(toProcess[0].block.header.time);
-                    await this.processBlock(toProcess[0], toProcess[1]);
-                }
-                else {
-                    const toProcess = await this.blockQueue.dequeue();
-                    this.log.silly("Retrieved block data");
-                    if (!toProcess || !toProcess[0] || !toProcess[1] || !toProcess[2]) {
-                        throw new Error("Could not fetch block");
-                    }
-                    this.log.silly("Decoded block");
-                    height = toProcess[0].block.header.height;
-                    timestamp = toRfc3339WithNanoseconds(toProcess[0].block.header.time);
-                    await this.processBlock(toProcess[0], toProcess[1], QueryValidatorsResponse.decode(toProcess[2]).validators);
-                }
-                // Emit events to trigger periodic operations every 50, 100 and 1000 blocks
-                if (height % 1000 == 0) {
-                    const hrTime = process.hrtime();
-                    const newms = hrTime[0] * 1000000 + hrTime[1] / 1000;
-                    const duration = newms - ms;
-                    ms = newms;
-                    const rate = 1000000000 / duration;
-                    this.log.info("Processing:" + rate.toFixed(2) + "blocks/sec");
-                    await this.asyncEmit("periodic/1000", {
-                        value: null,
-                        height,
-                        timestamp,
-                    });
-                }
-                if (height % 100 == 0) {
-                    await this.asyncEmit("periodic/100", {
-                        value: null,
-                        height,
-                        timestamp,
-                    });
-                }
-                if (height % 50 == 0) {
-                    await this.asyncEmit("periodic/50", {
-                        value: null,
-                        height,
-                        timestamp,
-                    });
-                }
-                this.log.silly("Handled periodic events");
-                await this.config.endTransaction(true);
-                this.log.silly("Committed db tx");
-            }
-            catch (e) {
-                this.log.error("" + e);
-                this.setStatus("FAILED");
-                try {
-                    await this.config.endTransaction(false);
-                }
-                catch (dbe) {
-                    this.log.error("Error ending transaction. Must be a DB error: " + dbe);
-                }
-                this.tryToRecover = true;
-                this.retryCount++;
-                break;
-            }
-            this.retryCount = 0;
-            this.setStatus("OK");
-        }
-        if (this.retryCount < 3) {
-            this.log.debug("Indexer retryCount: " + this.retryCount);
-            this.log.info("Indexer is restarting");
-            setTimeout(() => this.start(), this.retryCount * 5000);
-        }
-        else {
-            this.log.info("Indexer failed too many times. Exiting.");
-            process.exit(1);
-        }
-    }
-    async processBlock(block, block_results, validators) {
-        let processStart = [0, 0];
-        if (this.config.logLevel == "silly") {
-            processStart = process.hrtime();
-        }
-        const height = block.block.header.height;
-        this.log.debug("Processing block: %d", height);
-        this.log.silly("Started db tx");
-        // Initialize height & timestamp to be used for this block-processing run
-        const timestamp = toRfc3339WithNanoseconds(block.block.header.time);
-        // Use & await asyncEmit to ensure db insertions in order
-        /*
-         * Emit block information to any interested modules.
-         * Primarily the required block module listens to this
-         */
-        await this.asyncEmit("block", {
-            value: {
-                block,
-                block_results,
-            },
-            height,
-            timestamp,
-        });
-        this.log.silly("Modules handled block event");
-        // Deal with begin_block events first
-        await this.asyncEmit("begin_block", {
-            value: {
-                events: block_results.beginBlockEvents,
-                validators,
-            },
-            height,
-            timestamp,
-        });
-        this.log.silly("Modules handled begin_block events");
-        // Then individual tx_events
-        await this.asyncEmit("tx_events", {
-            value: block_results.results,
-            height,
-            timestamp,
-        });
-        this.log.silly("Modules handled tx events");
-        // Emit details and result for each tx msg separately
-        for (let t = 0; t < block.block.txs.length; t++) {
-            const tx = Tx.decode(block.block.txs[t]);
-            const result = block_results.results[t].code;
-            const txlog = block_results.results[t].log;
-            if (result != 0) {
-                //  Tx failed. Ignore
-                continue;
-            }
-            if (tx.body && tx.body.memo != "") {
-                const txHash = createHash("sha256").update(block.block.txs[t])
-                    .digest("hex");
-                await this.asyncEmit("tx_memo", {
-                    value: {
-                        txHash,
-                        txBody: tx.body,
-                    },
-                    height,
-                    timestamp,
-                });
-            }
-            // parsing log rather than using events directly in order to have msg_index available to filter appropriate events for each msg
-            const events = txlog
-                ? JSON.parse(txlog)
-                : [];
-            const msgs = tx.body?.messages;
-            if (msgs) {
-                for (let i = 0; i < msgs.length; i++) {
-                    this.log.silly("Indexer broadcasting msg for handling: " + msgs[i].typeUrl);
-                    const msgevents = msgs.length > 1
-                        ? events.find(x => x.msg_index == i)?.events
-                        : events[0].events;
-                    await this.asyncEmit(msgs[i].typeUrl, {
-                        value: {
-                            tx: msgs[i].value,
-                            events: msgevents,
-                        },
-                        height,
-                        timestamp,
-                    });
-                    if (msgs[i].typeUrl == "/cosmos.authz.v1beta1.MsgExec") {
-                        const authzMsgs = MsgExec.decode(msgs[i].value).msgs;
-                        if (authzMsgs) {
-                            for (let r = 0; r < authzMsgs.length; r++) {
-                                this.log.silly("Indexer broadcasting msg for handling: " + authzMsgs[r].typeUrl);
-                                const authzMsgEvents = msgevents?.reduce((events, evt) => {
-                                    if (evt.attributes.filter(x => decodeAttr(x.key) == "authz_msg_index" && decodeAttr(x.value) == "" + r).length > 0) {
-                                        events.push(evt);
-                                    }
-                                    return events;
-                                }, []);
-                                await this.asyncEmit(authzMsgs[r].typeUrl, {
-                                    value: {
-                                        tx: authzMsgs[r].value,
-                                        events: authzMsgEvents,
-                                    },
-                                    height,
-                                    timestamp,
-                                });
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        this.log.silly("Modules handled msg events");
-        // Then deal with end_block events
-        await this.asyncEmit("end_block", {
-            value: block_results.endBlockEvents,
-            height,
-            timestamp,
-        });
-        this.log.silly("Modules handled end_block events");
-        if (this.config.logLevel == "silly") {
-            const processEnd = process.hrtime(processStart);
-            const processTime = processEnd[0] * 1000 + processEnd[1] / 1000000;
-            this.log.silly("Processed block %d in %d ms", height, processTime.toFixed(2));
-        }
-    }
-    async fetcher() {
-        for (let i = this.heightToProcess; i <= this.latestHeight; i++) {
-            this.log.debug("Fetching: " + i);
-            if (this.tryToRecover) {
-                this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
-                break;
-            }
-            try {
-                if (this.isMinimal(this.blockQueue)) {
-                    const timeoutPromise = new Promise((resolve, reject) => {
-                        setTimeout(reject, 20000, false);
-                    });
-                    const toIndex = Promise.race([Promise.all([this.blockClient.block(i), this.blockClient.blockResults(i)]), timeoutPromise]).catch((e) => {
-                        this.log.error("Error fetching block: " + i + " : " + e);
-                        this.tryToRecover = true;
-                        return Promise.resolve([]);
-                    });
-                    this.blockQueue.enqueue(toIndex);
-                }
-                else {
-                    const timeoutPromise = new Promise((resolve, reject) => {
-                        setTimeout(reject, 20000, false);
-                    });
-                    const q = QueryValidatorsRequest.fromPartial({
-                        pagination: {
-                            limit: 1000n,
-                        },
-                    });
-                    const vals = QueryValidatorsRequest.encode(q).finish();
-                    const toIndex = Promise.race([
-                        Promise.all([
-                            this.blockClient.block(i),
-                            this.blockClient.blockResults(i),
-                            this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, i, false),
-                        ]),
-                        timeoutPromise,
-                    ]).catch((e) => {
-                        this.log.error("Error fetching block: " + i + " : " + e);
-                        this.tryToRecover = true;
-                        return Promise.resolve([]);
-                    });
-                    this.blockQueue.enqueue(toIndex);
-                }
-                await this.blockQueue.continue();
-                if (this.tryToRecover) {
-                    this.retryCount++;
-                    throw new Error("RPC not responding");
-                }
-            }
-            catch (e) {
-                this.log.error(e);
-                break;
-            }
-        }
-        if (!this.tryToRecover) {
-            this.blockQueue.setSynced();
-            this.log.info("Synced to latest height");
-        }
-    }
-    async callABCI(path, data, height, adHoc = true) {
-        try {
-            const abciq = await (adHoc
-                ? this.client
-                : this.blockClient).abciQuery({
-                path,
-                data,
-                height: height,
-            });
-            if (abciq) {
-                return abciq.value;
-            }
-            else {
-                this.tryToRecover = true;
-                this.setStatus("FAILED");
-                throw new Error("RPC not responding. Query at: " + path);
-            }
-        }
-        catch (_e) {
-            this.tryToRecover = true;
-            this.setStatus("FAILED");
-            this.retryCount++;
-            throw new Error("RPC not responding. Query at: " + path);
-        }
-    }
-    newBlockReceived(height) {
-        this.log.info("Received new block: %d", height);
-        // If we are synced, add to end of queue
-        if (this.blockQueue.synced && !this.tryToRecover) {
-            if (this.blockQueue.size() + 1 == this.config.batchSize) {
-                this.log.error("Block queue is full. Cannot add new block");
-                this.tryToRecover = true;
-                this.retryCount++;
-                return;
-            }
-            try {
-                if (this.isMinimal(this.blockQueue)) {
-                    this.blockQueue.enqueue(Promise.all([this.client.block(height), this.client.blockResults(height)]).catch((e) => {
-                        this.log.error("Error fetching block: " + height + " : " + e);
-                        return Promise.resolve([]);
-                    }));
-                }
-                else {
-                    const q = QueryValidatorsRequest.fromPartial({
-                        pagination: {
-                            limit: 1000n,
-                        },
-                    });
-                    const vals = QueryValidatorsRequest.encode(q).finish();
-                    this.blockQueue.enqueue(Promise.all([
-                        this.client.block(height),
-                        this.client.blockResults(height),
-                        this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, height, false),
-                    ]).catch((e) => {
-                        this.log.error("Error fetching block: " + height + " : " + e);
-                        return Promise.resolve([]);
-                    }));
-                }
-            }
-            catch (e) {
-                this.log.error("" + e);
-            }
-        }
-        else {
-            this.latestHeight = height;
-        }
-    }
-    async pollForBlock() {
-        const status = await this.client.status();
-        if (status.syncInfo.latestBlockHeight > this.latestHeight) {
-            while (this.latestHeight < status.syncInfo.latestBlockHeight) {
-                this.newBlockReceived(this.latestHeight + 1);
-            }
-        }
-        setTimeout(() => {
-            this.pollForBlock();
-        }, this.config.pollingInterval);
-    }
-    readGenesis() {
-        if (this.config.genesisPath) {
-            return fs.createReadStream(this.config.genesisPath).pipe(parser());
-        }
-        else {
-            throw new Error("Genesis path not set");
-        }
-    }
-    async setArrayReader(path, processor) {
-        const readPromise = new Promise((resolve, reject) => {
-            try {
-                const filters = path.split(".");
-                const pickers = filters.map(filter => pick({
-                    filter,
-                }));
-                let counter = 0;
-                chain([
-                    this.readGenesis(),
-                    ...pickers,
-                    streamArray(),
-                    batch({
-                        batchSize: 1000,
-                    }),
-                    processor,
-                ])
-                    .on("data", (data) => {
-                    if (data && Array.isArray(data)) {
-                        counter = counter + data.length;
-                    }
-                })
-                    .on("end", () => {
-                    this.log.info(`Processed ${counter} entries`);
-                    resolve(true);
-                });
-            }
-            catch (_e) {
-                this.log.verbose("Error in setArrayReader: " + _e);
-                reject();
-            }
-        });
-        return readPromise;
-    }
-    async setValueReader(path, processor) {
-        const readPromise = new Promise((resolve, reject) => {
-            try {
-                const filters = path.split(".");
-                const pickers = filters.map(filter => pick({
-                    filter,
-                }));
-                let counter = 0;
-                chain([this.readGenesis(), ...pickers, streamValues(), processor])
-                    .on("data", (_data) => {
-                    counter++;
-                })
-                    .on("end", () => {
-                    this.log.info(`Processed ${counter} entries`);
-                    resolve(true);
-                });
-            }
-            catch (_e) {
-                reject();
-            }
-        });
-        return readPromise;
-    }
-    async parseGenesis() {
-        this.log.info("Parsing genesis");
-        await this.config.beginTransaction();
-        try {
-            this.log.info("Starting genesis import");
-            this.log.debug("Importing genesis file...");
-            for (const [key, _value] of this.handled) {
-                if (key.startsWith("genesis/")) {
-                    const genesisEntry = key.split("/");
-                    this.log.verbose("Importing " + key + "...");
-                    if (genesisEntry[1] == "array") {
-                        await this.setArrayReader(genesisEntry[2], 
-                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                        async (data) => {
-                            await this.asyncEmit(key, {
-                                value: data.map((x) => x.value),
-                            });
-                            return data;
-                        });
-                    }
-                    else {
-                        await this.setValueReader(genesisEntry[2], 
-                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                        async (data) => {
-                            await this.asyncEmit(key, {
-                                value: data.value,
-                            });
-                            return data;
-                        });
-                    }
-                }
-            }
-            this.log.info("Importing gen TXs...");
-            await this.setArrayReader("app_state.genutil.gen_txs", 
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            async (data) => {
-                for (let j = 0; j < data.length; j++) {
-                    const gentx = data[j].value;
-                    for (let i = 0; i < gentx.body.messages.length; i++) {
-                        const msg = gentx.body.messages[i];
-                        await this.asyncEmit(("gentx" + msg["@type"]), {
-                            value: msg,
-                        });
-                    }
-                }
-                return data;
-            });
-            await this.config.endTransaction(true);
-            this.log.info("Finished importing");
-        }
-        catch (e) {
-            try {
-                await this.config.endTransaction(false);
-            }
-            catch (dbe) {
-                this.log.error("Error ending transaction. Must be a DB error: " + dbe);
-            }
-            this.log.error("Failed to import genesis");
-            throw e;
-        }
-    }
-}
+var EcleciaIndexer = class extends EclesiaEmitter {
+	constructor(config) {
+		super();
+		this.initialized = false;
+		this.retryCount = 0;
+		this.tryToRecover = false;
+		this.healthCheck = { status: "CONNECTING" };
+		this.subscription = null;
+		this.blockListener = { next: (data) => {
+			this.newBlockReceived(data.header.height);
+		} };
+		this.asyncEmit = async (type, event) => {
+			event.uuid = v4();
+			let listenerCount = this.handled.get(type);
+			if (!listenerCount) listenerCount = 1;
+			let listenersResponded = 0;
+			const prom = new Promise((resolve, reject) => {
+				const returnFunc = (ev) => {
+					if (ev.uuid == event.uuid) if (ev.status) {
+						listenersResponded++;
+						if (listenersResponded == listenerCount) {
+							this.off("uuid", returnFunc);
+							resolve();
+						}
+					} else reject(ev.error);
+				};
+				this.on("uuid", returnFunc);
+			});
+			this.emit(type, event);
+			return prom;
+		};
+		this.config = {
+			...defaultIndexerConfig,
+			...config
+		};
+		if (this.config.minimal) this.blockQueue = new CircularBuffer(this.config.batchSize);
+		else this.blockQueue = new CircularBuffer(this.config.batchSize);
+		const { printf } = winston.format;
+		const eclesiaFormat = printf(({ level, message, timestamp }) => {
+			return `${timestamp} [${level.toUpperCase()}]:\t${message}`;
+		});
+		this.log = winston.createLogger({
+			level: this.config.logLevel,
+			defaultMeta: { service: "Eclesia Indexer" },
+			transports: [
+				new winston.transports.File({
+					filename: "error.log",
+					level: "error"
+				}),
+				new winston.transports.File({ filename: "combined.log" }),
+				new winston.transports.Console({ format: winston.format.combine(winston.format.splat(), winston.format.timestamp(), eclesiaFormat, winston.format.colorize({ all: true })) })
+			]
+		});
+		this.fastify = Fastify({ logger: false });
+		this.on("_unhandled", (msg) => {
+			if (msg.uuid) {
+				this.log.verbose("Unhandled event: " + msg.type);
+				this.emit("uuid", {
+					status: true,
+					uuid: msg.uuid
+				});
+			}
+		});
+		this.fastify.get("/health", async (_request, reply) => {
+			const code = this.healthCheck.status == "OK" ? 200 : 503;
+			reply.code(code).send(this.healthCheck);
+		});
+		this.fastify.listen({
+			port: 80,
+			host: "0.0.0.0"
+		}, (err) => {
+			if (err) {
+				this.log.error(err);
+				process.exit(1);
+			}
+		});
+	}
+	setStatus(status) {
+		this.healthCheck.status = status;
+	}
+	isMinimal(_blockqueue) {
+		if (this.config.minimal) return true;
+		else return false;
+	}
+	async connect() {
+		try {
+			if (this.client && this.tryToRecover) {
+				this.log.verbose("Recover from error. Attempting to disconnect from RPC");
+				this.client.disconnect();
+				this.blockClient.disconnect();
+				this.log.verbose("Disconnected from RPC");
+			}
+			this.client = await connectComet(this.config.rpcUrl);
+			this.log.info("Connected to RPC for ad hoc queries");
+			this.blockClient = await connectComet(this.config.rpcUrl);
+			this.log.info("Connected to RPC for block & validator info");
+			return true;
+		} catch (error) {
+			this.log.error(error);
+			this.tryToRecover = true;
+			return false;
+		}
+	}
+	async start() {
+		if (this.blockQueue) {
+			this.blockQueue.clear();
+			this.log.verbose("Starting, clearing block queue");
+		}
+		if (!this.initialized) {
+			try {
+				if (this.config.init) await this.config.init();
+			} catch (e) {
+				this.log.error("Failed to initialize indexer: " + e);
+				this.setStatus("FAILED");
+				throw e;
+			}
+			if (this.config.processGenesis) try {
+				if (this.config.genesisPath) await this.parseGenesis();
+			} catch (e) {
+				this.log.error("Failed to parse genesis: " + e);
+				this.setStatus("FAILED");
+				throw e;
+			}
+			this.initialized = true;
+		}
+		try {
+			await this.connect();
+			if (!this.config.usePolling && this.subscription) {
+				this.subscription.removeListener(this.blockListener);
+				this.subscription = null;
+				this.log.verbose("Removed existing block listener and subscription");
+			}
+			this.subscription = this.client.subscribeNewBlock ? this.client.subscribeNewBlock() : null;
+			this.latestHeight = (await this.client.status()).syncInfo.latestBlockHeight;
+			this.log.info("Current chain height: " + this.latestHeight);
+			this.heightToProcess = await this.config.getNextHeight();
+			if (this.config.usePolling) this.pollForBlock();
+			else if (this.subscription) this.subscription.addListener(this.blockListener);
+			else throw new Error("Could not subscribe to new blocks");
+		} catch (e) {
+			this.log.error("Failed to set up block listening: " + e);
+			this.setStatus("FAILED");
+			throw e;
+		}
+		this.tryToRecover = false;
+		this.fetcher().catch((e) => {
+			this.setStatus("FAILED");
+			throw new Error("Error in fetching service: " + e);
+		});
+		const hrTime = process.hrtime();
+		let ms = hrTime[0] * 1e6 + hrTime[1] / 1e3;
+		while (this.blockQueue.size() > 0 && !this.tryToRecover) {
+			try {
+				if (this.tryToRecover) throw new Error("Exiting processing loop. Attempting to recover indexer");
+				await this.config.beginTransaction();
+				this.log.silly("Started db tx");
+				let height, timestamp;
+				if (this.isMinimal(this.blockQueue)) {
+					const toProcess = await this.blockQueue.dequeue();
+					this.log.silly("Retrieved block data");
+					if (!toProcess || !toProcess[0] || !toProcess[1]) throw new Error("Could not fetch block");
+					height = toProcess[0].block.header.height;
+					timestamp = toRfc3339WithNanoseconds(toProcess[0].block.header.time);
+					await this.processBlock(toProcess[0], toProcess[1]);
+				} else {
+					const toProcess = await this.blockQueue.dequeue();
+					this.log.silly("Retrieved block data");
+					if (!toProcess || !toProcess[0] || !toProcess[1] || !toProcess[2]) throw new Error("Could not fetch block");
+					this.log.silly("Decoded block");
+					height = toProcess[0].block.header.height;
+					timestamp = toRfc3339WithNanoseconds(toProcess[0].block.header.time);
+					await this.processBlock(toProcess[0], toProcess[1], QueryValidatorsResponse.decode(toProcess[2]).validators);
+				}
+				if (height % 1e3 == 0) {
+					const hrTime$1 = process.hrtime();
+					const newms = hrTime$1[0] * 1e6 + hrTime$1[1] / 1e3;
+					const duration = newms - ms;
+					ms = newms;
+					const rate = 1e9 / duration;
+					this.log.info("Processing:" + rate.toFixed(2) + "blocks/sec");
+					await this.asyncEmit("periodic/1000", {
+						value: null,
+						height,
+						timestamp
+					});
+				}
+				if (height % 100 == 0) await this.asyncEmit("periodic/100", {
+					value: null,
+					height,
+					timestamp
+				});
+				if (height % 50 == 0) await this.asyncEmit("periodic/50", {
+					value: null,
+					height,
+					timestamp
+				});
+				this.log.silly("Handled periodic events");
+				await this.config.endTransaction(true);
+				this.log.silly("Committed db tx");
+			} catch (e) {
+				this.log.error("" + e);
+				this.setStatus("FAILED");
+				try {
+					await this.config.endTransaction(false);
+				} catch (dbe) {
+					this.log.error("Error ending transaction. Must be a DB error: " + dbe);
+				}
+				this.tryToRecover = true;
+				this.retryCount++;
+				break;
+			}
+			this.retryCount = 0;
+			this.setStatus("OK");
+		}
+		if (this.retryCount < 3) {
+			this.log.debug("Indexer retryCount: " + this.retryCount);
+			this.log.info("Indexer is restarting");
+			setTimeout(() => this.start(), this.retryCount * 5e3);
+		} else {
+			this.log.info("Indexer failed too many times. Exiting.");
+			process.exit(1);
+		}
+	}
+	async processBlock(block, block_results, validators) {
+		let processStart = [0, 0];
+		if (this.config.logLevel == "silly") processStart = process.hrtime();
+		const height = block.block.header.height;
+		this.log.debug("Processing block: %d", height);
+		this.log.silly("Started db tx");
+		const timestamp = toRfc3339WithNanoseconds(block.block.header.time);
+		await this.asyncEmit("block", {
+			value: {
+				block,
+				block_results
+			},
+			height,
+			timestamp
+		});
+		this.log.silly("Modules handled block event");
+		await this.asyncEmit("begin_block", {
+			value: {
+				events: block_results.beginBlockEvents,
+				validators
+			},
+			height,
+			timestamp
+		});
+		this.log.silly("Modules handled begin_block events");
+		await this.asyncEmit("tx_events", {
+			value: block_results.results,
+			height,
+			timestamp
+		});
+		this.log.silly("Modules handled tx events");
+		for (let t = 0; t < block.block.txs.length; t++) {
+			const tx = Tx.decode(block.block.txs[t]);
+			const result = block_results.results[t].code;
+			const txlog = block_results.results[t].log;
+			if (result != 0) continue;
+			if (tx.body && tx.body.memo != "") {
+				const txHash = createHash("sha256").update(block.block.txs[t]).digest("hex");
+				await this.asyncEmit("tx_memo", {
+					value: {
+						txHash,
+						txBody: tx.body
+					},
+					height,
+					timestamp
+				});
+			}
+			const events = txlog ? JSON.parse(txlog) : [];
+			const msgs = tx.body?.messages;
+			if (msgs) for (let i = 0; i < msgs.length; i++) {
+				this.log.silly("Indexer broadcasting msg for handling: " + msgs[i].typeUrl);
+				const msgevents = msgs.length > 1 ? events.find((x) => x.msg_index == i)?.events : events[0].events;
+				await this.asyncEmit(msgs[i].typeUrl, {
+					value: {
+						tx: msgs[i].value,
+						events: msgevents
+					},
+					height,
+					timestamp
+				});
+				if (msgs[i].typeUrl == "/cosmos.authz.v1beta1.MsgExec") {
+					const authzMsgs = MsgExec.decode(msgs[i].value).msgs;
+					if (authzMsgs) for (let r = 0; r < authzMsgs.length; r++) {
+						this.log.silly("Indexer broadcasting msg for handling: " + authzMsgs[r].typeUrl);
+						const authzMsgEvents = msgevents?.reduce((events$1, evt) => {
+							if (evt.attributes.filter((x) => decodeAttr(x.key) == "authz_msg_index" && decodeAttr(x.value) == "" + r).length > 0) events$1.push(evt);
+							return events$1;
+						}, []);
+						await this.asyncEmit(authzMsgs[r].typeUrl, {
+							value: {
+								tx: authzMsgs[r].value,
+								events: authzMsgEvents
+							},
+							height,
+							timestamp
+						});
+					}
+				}
+			}
+		}
+		this.log.silly("Modules handled msg events");
+		await this.asyncEmit("end_block", {
+			value: block_results.endBlockEvents,
+			height,
+			timestamp
+		});
+		this.log.silly("Modules handled end_block events");
+		if (this.config.logLevel == "silly") {
+			const processEnd = process.hrtime(processStart);
+			const processTime = processEnd[0] * 1e3 + processEnd[1] / 1e6;
+			this.log.silly("Processed block %d in %d ms", height, processTime.toFixed(2));
+		}
+	}
+	async fetcher() {
+		for (let i = this.heightToProcess; i <= this.latestHeight; i++) {
+			this.log.debug("Fetching: " + i);
+			if (this.tryToRecover) {
+				this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
+				break;
+			}
+			try {
+				if (this.isMinimal(this.blockQueue)) {
+					const timeoutPromise = new Promise((resolve, reject) => {
+						setTimeout(reject, 2e4, false);
+					});
+					const toIndex = Promise.race([Promise.all([this.blockClient.block(i), this.blockClient.blockResults(i)]), timeoutPromise]).catch((e) => {
+						this.log.error("Error fetching block: " + i + " : " + e);
+						this.tryToRecover = true;
+						return Promise.resolve([]);
+					});
+					this.blockQueue.enqueue(toIndex);
+				} else {
+					const timeoutPromise = new Promise((resolve, reject) => {
+						setTimeout(reject, 2e4, false);
+					});
+					const q = QueryValidatorsRequest.fromPartial({ pagination: { limit: 1000n } });
+					const vals = QueryValidatorsRequest.encode(q).finish();
+					const toIndex = Promise.race([Promise.all([
+						this.blockClient.block(i),
+						this.blockClient.blockResults(i),
+						this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, i, false)
+					]), timeoutPromise]).catch((e) => {
+						this.log.error("Error fetching block: " + i + " : " + e);
+						this.tryToRecover = true;
+						return Promise.resolve([]);
+					});
+					this.blockQueue.enqueue(toIndex);
+				}
+				await this.blockQueue.continue();
+				if (this.tryToRecover) {
+					this.retryCount++;
+					throw new Error("RPC not responding");
+				}
+			} catch (e) {
+				this.log.error(e);
+				break;
+			}
+		}
+		if (!this.tryToRecover) {
+			this.blockQueue.setSynced();
+			this.log.info("Synced to latest height");
+		}
+	}
+	async callABCI(path, data, height, adHoc = true) {
+		try {
+			const abciq = await (adHoc ? this.client : this.blockClient).abciQuery({
+				path,
+				data,
+				height
+			});
+			if (abciq) return abciq.value;
+			else {
+				this.tryToRecover = true;
+				this.setStatus("FAILED");
+				throw new Error("RPC not responding. Query at: " + path);
+			}
+		} catch (_e) {
+			this.tryToRecover = true;
+			this.setStatus("FAILED");
+			this.retryCount++;
+			throw new Error("RPC not responding. Query at: " + path);
+		}
+	}
+	newBlockReceived(height) {
+		this.log.info("Received new block: %d", height);
+		if (this.blockQueue.synced && !this.tryToRecover) {
+			if (this.blockQueue.size() + 1 == this.config.batchSize) {
+				this.log.error("Block queue is full. Cannot add new block");
+				this.tryToRecover = true;
+				this.retryCount++;
+				return;
+			}
+			try {
+				if (this.isMinimal(this.blockQueue)) this.blockQueue.enqueue(Promise.all([this.client.block(height), this.client.blockResults(height)]).catch((e) => {
+					this.log.error("Error fetching block: " + height + " : " + e);
+					return Promise.resolve([]);
+				}));
+				else {
+					const q = QueryValidatorsRequest.fromPartial({ pagination: { limit: 1000n } });
+					const vals = QueryValidatorsRequest.encode(q).finish();
+					this.blockQueue.enqueue(Promise.all([
+						this.client.block(height),
+						this.client.blockResults(height),
+						this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, height, false)
+					]).catch((e) => {
+						this.log.error("Error fetching block: " + height + " : " + e);
+						return Promise.resolve([]);
+					}));
+				}
+			} catch (e) {
+				this.log.error("" + e);
+			}
+		} else this.latestHeight = height;
+	}
+	async pollForBlock() {
+		const status = await this.client.status();
+		if (status.syncInfo.latestBlockHeight > this.latestHeight) while (this.latestHeight < status.syncInfo.latestBlockHeight) this.newBlockReceived(this.latestHeight + 1);
+		setTimeout(() => {
+			this.pollForBlock();
+		}, this.config.pollingInterval);
+	}
+	readGenesis() {
+		if (this.config.genesisPath) return fs.createReadStream(this.config.genesisPath).pipe(Parser.parser());
+		else throw new Error("Genesis path not set");
+	}
+	async setArrayReader(path, processor) {
+		return new Promise((resolve, reject) => {
+			try {
+				const pickers = path.split(".").map((filter) => Pick.pick({ filter }));
+				let counter = 0;
+				chain([
+					this.readGenesis(),
+					...pickers,
+					StreamArray.streamArray(),
+					Batch.batch({ batchSize: 1e3 }),
+					processor
+				]).on("data", (data) => {
+					if (data && Array.isArray(data)) counter = counter + data.length;
+				}).on("end", () => {
+					this.log.info(`Processed ${counter} entries`);
+					resolve(true);
+				});
+			} catch (_e) {
+				this.log.verbose("Error in setArrayReader: " + _e);
+				reject();
+			}
+		});
+	}
+	async setValueReader(path, processor) {
+		return new Promise((resolve, reject) => {
+			try {
+				const pickers = path.split(".").map((filter) => Pick.pick({ filter }));
+				let counter = 0;
+				chain([
+					this.readGenesis(),
+					...pickers,
+					StreamValues.streamValues(),
+					processor
+				]).on("data", (_data) => {
+					counter++;
+				}).on("end", () => {
+					this.log.info(`Processed ${counter} entries`);
+					resolve(true);
+				});
+			} catch (_e) {
+				reject();
+			}
+		});
+	}
+	async parseGenesis() {
+		this.log.info("Parsing genesis");
+		await this.config.beginTransaction();
+		try {
+			this.log.info("Starting genesis import");
+			this.log.debug("Importing genesis file...");
+			for (const [key, _value] of this.handled) if (key.startsWith("genesis/")) {
+				const genesisEntry = key.split("/");
+				this.log.verbose("Importing " + key + "...");
+				if (genesisEntry[1] == "array") await this.setArrayReader(genesisEntry[2], async (data) => {
+					await this.asyncEmit(key, { value: data.map((x) => x.value) });
+					return data;
+				});
+				else await this.setValueReader(genesisEntry[2], async (data) => {
+					await this.asyncEmit(key, { value: data.value });
+					return data;
+				});
+			}
+			this.log.info("Importing gen TXs...");
+			await this.setArrayReader("app_state.genutil.gen_txs", async (data) => {
+				for (let j = 0; j < data.length; j++) {
+					const gentx = data[j].value;
+					for (let i = 0; i < gentx.body.messages.length; i++) {
+						const msg = gentx.body.messages[i];
+						await this.asyncEmit("gentx" + msg["@type"], { value: msg });
+					}
+				}
+				return data;
+			});
+			await this.config.endTransaction(true);
+			this.log.info("Finished importing");
+		} catch (e) {
+			try {
+				await this.config.endTransaction(false);
+			} catch (dbe) {
+				this.log.error("Error ending transaction. Must be a DB error: " + dbe);
+			}
+			this.log.error("Failed to import genesis");
+			throw e;
+		}
+	}
+};
+
+//#endregion
+export { EcleciaIndexer, defaultIndexerConfig };
+//# sourceMappingURL=index.js.map