@eclesia/indexer-engine 2.14.4 → 2.16.0

package/dist/indexer/index.mjs

@@ -1,9 +1,10 @@
-import { CONNECT_TIMEOUT_MS, DEFAULT_BATCH_SIZE, DEFAULT_HEALTH_CHECK_PORT, DEFAULT_POLLING_INTERVAL_MS, DEFAULT_PROMETHEUS_PORT, DEFAULT_START_HEIGHT, GENESIS_BATCH_SIZE, PAGINATION_LIMITS, PERIODIC_INTERVALS, QUEUE_DEQUEUE_TIMEOUT_MS, RPC_TIMEOUT_MS } from "../constants.mjs";
+import { CONNECT_TIMEOUT_MS, DEFAULT_BATCH_SIZE, DEFAULT_BIND_HOST, DEFAULT_HEALTH_CHECK_PORT, DEFAULT_POLLING_INTERVAL_MS, DEFAULT_PROMETHEUS_PORT, DEFAULT_START_HEIGHT, GENESIS_BATCH_SIZE, IDLE_CHECK_INTERVAL_MS, MAX_FAILURES_PER_BLOCK, PAGINATION_LIMITS, PERIODIC_INTERVALS, RETRY_BASE_DELAY_MS, RETRY_MAX_DELAY_MS, RPC_TIMEOUT_MS } from "../constants.mjs";
 import { EclesiaEmitter } from "../emitter/index.mjs";
-import { RPCError } from "../errors/index.mjs";
+import { ConfigurationError, RPCError } from "../errors/index.mjs";
 import { IndexerMetrics } from "../metrics/index.mjs";
 import { CircularBuffer } from "../promise-queue/index.mjs";
-import { decodeAttr } from "../utils/text.mjs";
+import { decodeAttr, hasBlockEventMode, redactUrl } from "../utils/text.mjs";
+import { retryDelay, withTimeout } from "../utils/timeout.mjs";
 import "../utils/index.mjs";
 import { validateFilePath, validatePort, validatePositiveInteger, validateUrl } from "../validation/index.mjs";
 import { createHash } from "node:crypto";
@@ -14,12 +15,11 @@ import { QueryValidatorsRequest, QueryValidatorsResponse } from "cosmjs-types/co
 import { Tx } from "cosmjs-types/cosmos/tx/v1beta1/tx.js";
 import Fastify from "fastify";
 import { chain } from "stream-chain";
-import Parser from "stream-json";
-import Pick from "stream-json/filters/Pick.js";
-import StreamArray from "stream-json/streamers/StreamArray.js";
-import StreamValues from "stream-json/streamers/StreamValues.js";
-import Batch from "stream-json/utils/Batch.js";
-import { v4 } from "uuid";
+import pick from "stream-json/filters/pick.js";
+import parser from "stream-json/parser.js";
+import streamArray from "stream-json/streamers/stream-array.js";
+import streamValues from "stream-json/streamers/stream-values.js";
+import batch from "stream-json/utils/batch.js";
 import * as winston from "winston";
 
 //#region src/indexer/index.ts
@@ -46,7 +46,7 @@ const defaultIndexerConfig = {
 * Core blockchain indexer that connects to Tendermint RPC and processes blocks
 * Extends EclesiaEmitter to provide event-driven architecture for modules
 */
-var EcleciaIndexer = class extends EclesiaEmitter {
+var EclesiaIndexer = class extends EclesiaEmitter {
 	/** Indexer configuration settings */
 	config;
 	/** Fastify HTTP server for health checks */
@@ -81,6 +81,31 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 	subscription = null;
 	/** Timeout handler for block reception */
 	blockTimeout = null;
+	/** Timer for the next poll in polling mode */
+	pollTimer = null;
+	/** Bumped on every (re)start and stop so a polling chain from a previous run exits */
+	pollGeneration = 0;
+	/** Bumped on every start() so callbacks left over from a previous run cannot trigger recovery in this one */
+	runGeneration = 0;
+	/**
+	* Rejecters of waits parked in waitForBlockData(). Each entry is removed as soon as its block
+	* arrives, so a healthy run keeps this empty instead of accumulating one entry per block.
+	*/
+	blockWaiters = /* @__PURE__ */ new Set();
+	/** Pending restart timer */
+	retryTimer = null;
+	/** Resolves when the indexer has stopped for good: stop() was called, endHeight was reached, or it gave up */
+	stopped = Promise.resolve();
+	resolveStopped = () => {};
+	/** Next height the fetcher will request; advances as fetches are enqueued */
+	nextFetchHeight = 0;
+	/** Whether a fetcher loop is active, and for which run */
+	fetcherRunning = false;
+	fetcherGeneration = 0;
+	fetcherToken = 0;
+	/** Height of the block whose processing failed most recently, and how many times in a row */
+	lastFailedHeight;
+	sameHeightFailures = 0;
 	/**
 	* Creates a new Eclesia indexer instance
 	* @param config - Indexer configuration options
@@ -94,40 +119,44 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 		if (config.prometheusPort !== void 0) validatePort(config.prometheusPort, "prometheusPort");
 		if (config.startHeight !== void 0) validatePositiveInteger(config.startHeight, "startHeight");
 		if (config.pollingInterval !== void 0) validatePositiveInteger(config.pollingInterval, "pollingInterval");
+		const provided = Object.fromEntries(Object.entries(config).filter(([, value]) => value !== void 0));
 		this.config = {
 			...defaultIndexerConfig,
-			...config
+			...provided
 		};
-		const { printf } = winston.format;
-		const eclesiaFormat = printf(({ level, message, timestamp }) => {
-			return `${timestamp} [${level.toUpperCase()}]:\t${message}`;
+		const errorFormat = winston.format((info) => {
+			const error = info.error;
+			if (error instanceof Error) info.error = {
+				name: error.name,
+				message: error.message,
+				stack: error.stack
+			};
+			else if (error !== void 0 && (typeof error !== "object" || error === null)) info.error = { message: String(error) };
+			return info;
+		});
+		const textFormat = winston.format.printf(({ level, message, timestamp, error }) => {
+			const detail = error;
+			const suffix = detail ? "\n" + (detail.stack ?? detail.message ?? "") : "";
+			return `${timestamp} [${level.toUpperCase()}]:\t${message}${suffix}`;
 		});
 		this.log = winston.createLogger({
 			level: this.config.logLevel,
 			defaultMeta: { service: "Eclesia Indexer" },
-			transports: [
-				new winston.transports.File({
-					filename: "error.log",
-					level: "error"
-				}),
-				new winston.transports.File({ filename: "combined.log" }),
-				new winston.transports.Console({ format: winston.format.combine(winston.format.splat(), winston.format.timestamp(), eclesiaFormat, winston.format.colorize({ all: true })) })
-			]
+			transports: [new winston.transports.Console({ format: this.config.logFormat === "json" ? winston.format.combine(winston.format.splat(), winston.format.timestamp(), errorFormat(), winston.format.json()) : winston.format.combine(winston.format.splat(), winston.format.timestamp(), errorFormat(), textFormat, winston.format.colorize({ all: true })) })]
 		});
+		const protocol = new URL(this.config.rpcUrl).protocol;
+		if (!this.config.usePolling && (protocol === "http:" || protocol === "https:")) {
+			this.log.warn("rpcUrl " + redactUrl(this.config.rpcUrl) + " is HTTP, which cannot deliver block subscriptions; polling every " + this.config.pollingInterval + " ms instead (use a ws:// or wss:// URL for WebSocket mode)");
+			this.config.usePolling = true;
+		}
 		const queueErrorHandler = (e) => {
 			this.prometheus?.recordError("rpc");
-			this.log.error("Error enqueueing block data: " + e);
+			this.log.error("Error enqueueing block data", { error: e });
 		};
 		if (this.config.minimal) this.blockQueue = new CircularBuffer(this.config.batchSize, queueErrorHandler);
 		else this.blockQueue = new CircularBuffer(this.config.batchSize, queueErrorHandler);
 		this.on("_unhandled", (msg) => {
-			if (msg.uuid) {
-				this.log.verbose("Unhandled event: " + msg.type);
-				this.emit("uuid", {
-					status: true,
-					uuid: msg.uuid
-				});
-			}
+			if (msg.type !== "uuid" && this.log.isVerboseEnabled()) this.log.verbose("Unhandled event: " + msg.type);
 		});
 		if (this.config.enablePrometheus) {
 			this.prometheus = new IndexerMetrics();
@@ -139,10 +168,10 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 			const prometheusPort = this.config.prometheusPort ?? (process.env.PROMETHEUS_PORT ? parseInt(process.env.PROMETHEUS_PORT, 10) : DEFAULT_PROMETHEUS_PORT);
 			this.prometheusServer.listen({
 				port: prometheusPort,
-				host: "0.0.0.0"
+				host: this.config.prometheusHost ?? DEFAULT_BIND_HOST
 			}, (err) => {
 				if (err) {
-					this.log.error("Prometheus error: " + err);
+					this.log.error("Prometheus server error", { error: err });
 					this.prometheus?.recordError("metrics_server");
 					this.emit("fatal-error", {
 						error: err,
@@ -154,16 +183,16 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 		if (this.config.enableHealthcheck) {
 			this.fastify = Fastify({ logger: false });
 			this.fastify.get("/health", async (_request, reply) => {
-				const code = this.healthCheck.status == "OK" ? 200 : 503;
+				const code = this.healthCheck.status == "OK" || this.healthCheck.status == "WAITING" ? 200 : 503;
 				reply.code(code).send(this.healthCheck);
 			});
 			const healthPort = this.config.healthCheckPort ?? (process.env.HEALTH_CHECK_PORT ? parseInt(process.env.HEALTH_CHECK_PORT, 10) : DEFAULT_HEALTH_CHECK_PORT);
 			this.fastify.listen({
 				port: healthPort,
-				host: "0.0.0.0"
+				host: this.config.healthCheckHost ?? DEFAULT_BIND_HOST
 			}, (err) => {
 				if (err) {
-					this.log.error("Health check server error: " + err);
+					this.log.error("Health check server error", { error: err });
 					this.prometheus?.recordError("health_check_server");
 					this.emit("fatal-error", {
 						error: err,
@@ -175,10 +204,110 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 	}
 	setStatus(status) {
 		this.healthCheck.status = status;
+		this.prometheus?.setWaiting(status === "WAITING");
+	}
+	/**
+	* Marks the current run for recovery and wakes the main loop if it is parked waiting for a
+	* block. Callbacks left over from a previous run pass their generation and are ignored.
+	*/
+	requestRecovery(reason, generation = this.runGeneration) {
+		if (generation !== this.runGeneration) {
+			this.log.debug("Ignoring recovery request from a previous run: " + reason);
+			return;
+		}
+		if (!this.tryToRecover) this.log.warn("Recovery requested: " + reason);
+		this.tryToRecover = true;
+		this.wakeBlockWaiters("Recovery requested while waiting for block data");
+	}
+	/** Rejects every wait parked in waitForBlockData() */
+	wakeBlockWaiters(reason) {
+		const waiters = [...this.blockWaiters];
+		this.blockWaiters.clear();
+		for (const reject of waiters) reject(new RPCError(reason));
+	}
+	/**
+	* Refuses to index a chain other than the configured one. An RPC pool that mixes networks, or
+	* a wrong URL, would otherwise write a different chain's blocks into the database.
+	*/
+	assertChainId(network) {
+		if (this.config.chainId !== void 0 && network !== this.config.chainId) throw new ConfigurationError("RPC serves chain " + network + " but chainId is configured as " + this.config.chainId, {
+			expected: this.config.chainId,
+			actual: network
+		});
+	}
+	/** Counts consecutive processing failures per block height */
+	noteBlockFailure(height) {
+		if (height === this.lastFailedHeight) this.sameHeightFailures++;
+		else {
+			this.lastFailedHeight = height;
+			this.sameHeightFailures = 1;
+		}
+	}
+	/** True once one block has failed maxFailuresPerBlock times in a row */
+	isStuck() {
+		return this.lastFailedHeight !== void 0 && this.sameHeightFailures >= (this.config.maxFailuresPerBlock ?? MAX_FAILURES_PER_BLOCK);
+	}
+	/**
+	* Waits for the next dequeued block but wakes early when recovery or stop is requested, so a
+	* loop parked on an empty queue never waits for a block that will not come.
+	*/
+	waitForBlockData(dequeued) {
+		if (this.tryToRecover || !this.started) return Promise.reject(new RPCError("Recovery requested while waiting for block data"));
+		return new Promise((resolve, reject) => {
+			this.blockWaiters.add(reject);
+			dequeued.then((value) => {
+				this.blockWaiters.delete(reject);
+				resolve(value);
+			}, (error) => {
+				this.blockWaiters.delete(reject);
+				reject(error);
+			});
+		});
+	}
+	/** (Re)arms the idle check that runs when no block has been announced for a while */
+	armIdleCheck() {
+		if (this.blockTimeout) clearTimeout(this.blockTimeout);
+		this.blockTimeout = setTimeout(() => {
+			this.checkLiveness();
+		}, IDLE_CHECK_INTERVAL_MS);
+	}
+	/**
+	* Runs when no block has been announced for IDLE_CHECK_INTERVAL_MS. A chain that has stopped
+	* producing blocks (halt, upgrade, slow chain) is not an error: the indexer reports WAITING and
+	* checks again later. Recovery is requested only when the chain has moved on without us, which
+	* means the subscription is dead, or when the RPC cannot be reached at all.
+	*/
+	async checkLiveness() {
+		if (!this.started) return;
+		const generation = this.runGeneration;
+		try {
+			const status = await withTimeout(this.client.status(), RPC_TIMEOUT_MS, new RPCError("RPC status call timed out"));
+			if (!this.started || generation !== this.runGeneration) return;
+			const chainHeight = status.syncInfo.latestBlockHeight;
+			if (chainHeight > this.latestHeight) {
+				this.requestRecovery("chain is at " + chainHeight + " but nothing was announced since " + this.latestHeight, generation);
+				return;
+			}
+			this.log.info("No new block for " + IDLE_CHECK_INTERVAL_MS / 1e3 + " s, chain height is still " + chainHeight);
+			this.setStatus("WAITING");
+			this.armIdleCheck();
+		} catch (e) {
+			this.log.error("Liveness check failed", { error: e });
+			this.requestRecovery("RPC unreachable during liveness check", generation);
+		}
 	}
-	blockListener = { next: (data) => {
-		this.newBlockReceived(data.header.height);
-	} };
+	blockListener = {
+		next: (data) => {
+			this.newBlockReceived(data.header.height);
+		},
+		error: (error) => {
+			this.log.error("Block subscription error", { error });
+			this.requestRecovery("block subscription errored");
+		},
+		complete: () => {
+			if (this.started) this.requestRecovery("block subscription closed by the node");
+		}
+	};
 	isMinimal(_blockqueue) {
 		if (this.config.minimal) return true;
 		else return false;
@@ -195,38 +324,55 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 				} catch (_e) {}
 				this.log.verbose("Disconnected from RPC");
 			}
-			const connectTimeoutPromise = new Promise((resolve, reject) => {
-				setTimeout(reject, CONNECT_TIMEOUT_MS, new RPCError("RPC connection timed out"));
-			});
-			this.log.info("Attempting to connected to RPC: " + this.config.rpcUrl);
-			this.client = await Promise.race([connectComet(this.config.rpcUrl), connectTimeoutPromise]);
-			await Promise.race([this.client.status(), connectTimeoutPromise]);
+			this.log.info("Attempting to connect to RPC: " + redactUrl(this.config.rpcUrl));
+			this.client = await this.connectWithTimeout();
+			await withTimeout(this.client.status(), CONNECT_TIMEOUT_MS, new RPCError("RPC status call timed out"));
 			this.log.info("Connected to RPC for ad hoc queries");
-			this.blockClient = await Promise.race([connectComet(this.config.rpcUrl), connectTimeoutPromise]);
-			await Promise.race([this.blockClient.status(), connectTimeoutPromise]);
+			this.blockClient = await this.connectWithTimeout();
+			await withTimeout(this.blockClient.status(), CONNECT_TIMEOUT_MS, new RPCError("RPC status call timed out"));
 			this.log.info("Connected to RPC for block & validator info");
 			return true;
 		} catch (error) {
-			this.log.error("RPC connection error: " + error);
+			this.log.error("RPC connection error", { error });
 			this.prometheus?.recordError("rpc");
-			this.tryToRecover = true;
+			this.requestRecovery("RPC connection failed");
 			return false;
 		}
 	}
+	/**
+	* Opens one CometBFT client with its own timeout. If the timeout wins, the client
+	* that may still arrive is disconnected so a slow RPC never leaks a socket.
+	*/
+	async connectWithTimeout() {
+		let timedOut = false;
+		const pending = connectComet(this.config.rpcUrl);
+		pending.then((client) => {
+			if (timedOut) try {
+				client.disconnect();
+			} catch (_e) {}
+		}).catch(() => {});
+		try {
+			return await withTimeout(pending, CONNECT_TIMEOUT_MS, new RPCError("RPC connection timed out"));
+		} catch (e) {
+			timedOut = true;
+			throw e;
+		}
+	}
 	async initialize() {
 		if (!this.initialized) {
 			try {
 				if (this.config.init) await this.config.init();
 			} catch (e) {
-				this.log.error("Failed to initialize indexer: " + e);
+				this.log.error("Failed to initialize indexer", { error: e });
 				this.prometheus?.recordError("init_error");
 				this.setStatus("FAILED");
 				throw e;
 			}
 			if (await this.config.shouldProcessGenesis()) try {
 				if (this.config.genesisPath) await this.parseGenesis();
+				else this.log.warn("shouldProcessGenesis() returned true but no genesisPath is configured, skipping genesis import");
 			} catch (e) {
-				this.log.error("Failed to parse genesis: " + e);
+				this.log.error("Failed to parse genesis", { error: e });
 				this.prometheus?.recordError("genesis_error");
 				this.setStatus("FAILED");
 				throw e;
@@ -234,8 +380,40 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 			this.initialized = true;
 		}
 	}
-	stop() {
+	/**
+	* Stops the indexer and releases everything that would keep the process alive:
+	* the block subscription, polling and inactivity timers, both RPC clients and the
+	* health and metrics servers. Safe to call more than once.
+	*/
+	async stop() {
 		this.started = false;
+		this.resolveStopped();
+		if (this.retryTimer) {
+			clearTimeout(this.retryTimer);
+			this.retryTimer = null;
+		}
+		this.wakeBlockWaiters("Indexer stopped while waiting for block data");
+		this.stopPolling();
+		if (this.blockTimeout) {
+			clearTimeout(this.blockTimeout);
+			this.blockTimeout = null;
+		}
+		if (this.subscription) {
+			try {
+				this.subscription.removeListener(this.blockListener);
+			} catch (_e) {}
+			this.subscription = null;
+		}
+		for (const client of [this.client, this.blockClient]) try {
+			client?.disconnect();
+		} catch (_e) {}
+		const servers = [this.fastify, this.prometheusServer];
+		this.fastify = null;
+		this.prometheusServer = null;
+		await Promise.all(servers.map((server) => server?.close().catch((e) => {
+			this.log.warn("Error closing HTTP server", { error: e });
+		})));
+		this.log.info("Indexer stopped");
 	}
 	clearBlockQueue() {
 		if (this.blockQueue) {
@@ -255,65 +433,88 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 				this.log.verbose("Removed existing block listener and subscription");
 			}
 			if (!this.config.usePolling) this.subscription = this.client.subscribeNewBlock ? this.client.subscribeNewBlock() : null;
-			const statusPromise = new Promise((resolve, reject) => {
-				setTimeout(reject, RPC_TIMEOUT_MS, new RPCError("RPC status call timed out"));
-			});
-			this.latestHeight = (await Promise.race([this.client.status(), statusPromise])).syncInfo.latestBlockHeight;
-			this.log.info("Current chain height: " + this.latestHeight);
+			const status = await withTimeout(this.client.status(), RPC_TIMEOUT_MS, new RPCError("RPC status call timed out"));
+			this.assertChainId(status.nodeInfo.network);
+			this.latestHeight = status.syncInfo.latestBlockHeight;
+			this.log.info("Connected to " + status.nodeInfo.network + ", current chain height: " + this.latestHeight);
 			this.heightToProcess = await this.config.getNextHeight();
-			if (this.config.usePolling) this.pollForBlock();
+			this.nextFetchHeight = this.heightToProcess;
+			if (this.config.usePolling) this.startPolling();
 			else if (this.subscription) this.subscription.addListener(this.blockListener);
 			else {
 				this.prometheus?.recordError("rpc");
 				throw new Error("Could not subscribe to new blocks");
 			}
+			this.armIdleCheck();
 		} catch (e) {
-			this.log.error("Failed to set up block listening: " + e);
+			this.log.error("Failed to set up block listening", { error: e });
 			this.prometheus?.recordError("rpc");
 			this.setStatus("FAILED");
 			throw e;
 		}
 	}
+	/**
+	* Resolves once the indexer has stopped for good: stop() was called, endHeight was reached, or
+	* a fatal-error was emitted. Restarts with backoff do not resolve it. Use it to keep a caller
+	* waiting for the whole run rather than for the first loop exit.
+	*/
+	whenStopped() {
+		return this.stopped;
+	}
 	async start() {
+		if (!this.started) this.stopped = new Promise((resolve) => {
+			this.resolveStopped = resolve;
+		});
 		this.started = true;
+		this.runGeneration++;
+		const generation = this.runGeneration;
+		this.tryToRecover = false;
+		this.blockWaiters.clear();
 		this.clearBlockQueue();
 		await this.initialize();
 		try {
 			await this.setupBlockListening();
 			this.log.debug("Starting main processing loop");
-			this.tryToRecover = false;
 			this.fetcher().catch((e) => {
 				this.setStatus("FAILED");
 				this.prometheus?.recordError("rpc");
-				throw new Error("Error in fetching service: " + e);
+				this.log.error("Error in fetching service", { error: e });
+				this.requestRecovery("fetcher failed", generation);
 			});
-		} catch (_e) {}
-		while (this.started && !this.tryToRecover && (this.config.endHeight === void 0 || this.heightToProcess <= this.config.endHeight) && this.blockQueue.size() > 0) {
+		} catch (e) {
+			this.requestRecovery("block listening setup failed: " + e, generation);
+		}
+		let lastProcessed;
+		while (this.started && !this.tryToRecover) {
+			let txOpen = false;
+			let failingHeight;
 			try {
 				this.prometheus?.updateRetryCount(this.retryCount);
-				await this.config.beginTransaction();
-				this.log.silly("Started db tx");
-				let height, timestamp;
+				if (this.blockQueue.synced && this.blockQueue.size() <= 1) this.setStatus("WAITING");
+				let height;
+				let timestamp;
 				if (this.isMinimal(this.blockQueue)) {
-					const timeoutPromise = new Promise((resolve, reject) => {
-						setTimeout(reject, QUEUE_DEQUEUE_TIMEOUT_MS, []);
-					});
-					const toProcess = await Promise.race([this.blockQueue.dequeue(), timeoutPromise]);
+					const toProcess = await this.waitForBlockData(this.blockQueue.dequeue());
 					this.log.silly("Retrieved block data");
 					if (!toProcess || !toProcess[0] || !toProcess[1]) throw new RPCError("Could not fetch block(minimal)");
 					height = toProcess[0].block.header.height;
+					failingHeight = height;
 					timestamp = toRfc3339WithNanoseconds(toProcess[0].block.header.time);
+					await this.config.beginTransaction();
+					txOpen = true;
+					this.log.silly("Started db tx");
 					await this.processBlock(toProcess[0], toProcess[1]);
 				} else {
-					const timeoutPromise = new Promise((resolve, reject) => {
-						setTimeout(reject, QUEUE_DEQUEUE_TIMEOUT_MS, []);
-					});
-					const toProcess = await Promise.race([this.blockQueue.dequeue(), timeoutPromise]);
+					const toProcess = await this.waitForBlockData(this.blockQueue.dequeue());
 					this.log.silly("Retrieved block data");
 					if (!toProcess || !toProcess[0] || !toProcess[1] || !toProcess[2]) throw new RPCError("Could not fetch block(full)");
 					this.log.silly("Decoded block");
 					height = toProcess[0].block.header.height;
+					failingHeight = height;
 					timestamp = toRfc3339WithNanoseconds(toProcess[0].block.header.time);
+					await this.config.beginTransaction();
+					txOpen = true;
+					this.log.silly("Started db tx");
 					await this.processBlock(toProcess[0], toProcess[1], QueryValidatorsResponse.decode(toProcess[2]).validators);
 				}
 				if (height % PERIODIC_INTERVALS.LARGE == 0) await this.asyncEmit("periodic/large", {
@@ -333,65 +534,86 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 				});
 				this.log.silly("Handled periodic events");
 				await this.config.endTransaction(true);
+				txOpen = false;
+				lastProcessed = height;
+				this.lastFailedHeight = void 0;
+				this.sameHeightFailures = 0;
 				this.log.silly("Committed db tx");
 			} catch (e) {
-				this.prometheus?.recordError("block");
-				this.log.error("Block processing error: " + e);
-				this.setStatus("FAILED");
-				try {
+				if (txOpen) try {
 					await this.config.endTransaction(false);
 				} catch (dbe) {
 					this.prometheus?.recordError("database");
-					this.log.error("Error ending transaction. Must be a DB error: " + dbe);
+					this.log.error("Error ending transaction", { error: dbe });
 				}
-				this.tryToRecover = true;
+				if (!this.started) break;
+				if (failingHeight !== void 0) this.noteBlockFailure(failingHeight);
+				this.prometheus?.recordError("block");
+				this.log.error("Block processing error", { error: e });
+				this.setStatus("FAILED");
+				this.requestRecovery("block processing failed", generation);
 				break;
 			}
 			this.retryCount = 0;
 			this.setStatus("OK");
+			if (this.config.endHeight !== void 0 && lastProcessed !== void 0 && lastProcessed >= this.config.endHeight) {
+				this.log.info("Reached configured end height " + this.config.endHeight + ". Stopping indexer.");
+				await this.stop();
+				return;
+			}
 		}
 		if (!this.started) {
 			this.log.info("Indexer manually stopped.");
 			return;
 		}
-		if (this.config.endHeight !== void 0 && this.heightToProcess >= this.config.endHeight) {
-			this.log.info("Reached configured end height. Stopping indexer.");
-			this.stop();
+		if (this.isStuck()) {
+			const height = this.lastFailedHeight;
+			this.log.error("Block " + height + " failed " + this.sameHeightFailures + " times in a row. This is a deterministic failure in a handler or the data, not an outage. Giving up.");
+			this.started = false;
+			this.emit("fatal-error", {
+				error: /* @__PURE__ */ new Error("Block " + height + " failed " + this.sameHeightFailures + " times in a row"),
+				message: "Block processing is stuck",
+				retryCount: this.retryCount,
+				height
+			});
+			this.resolveStopped();
 			return;
 		}
-		if (this.retryCount < 3) {
-			this.log.debug("Indexer retryCount: " + this.retryCount);
-			this.retryCount++;
-			this.log.info("Indexer is restarting");
-			setTimeout(() => this.start(), this.retryCount * 5e3);
-		} else {
-			this.log.info("Indexer failed too many times. Exiting.");
+		this.retryCount++;
+		if (this.config.maxRetries !== void 0 && this.retryCount > this.config.maxRetries) {
+			this.log.error("Indexer failed " + this.retryCount + " times in a row, giving up (maxRetries=" + this.config.maxRetries + ")");
+			this.started = false;
 			this.emit("fatal-error", {
 				error: /* @__PURE__ */ new Error("Max retry attempts exceeded"),
 				message: "Indexer failed too many times",
 				retryCount: this.retryCount
 			});
+			this.resolveStopped();
+			return;
 		}
+		const delay = retryDelay(this.retryCount, RETRY_BASE_DELAY_MS, RETRY_MAX_DELAY_MS);
+		this.log.warn("Indexer is restarting in " + delay / 1e3 + " s (attempt " + this.retryCount + ")");
+		this.retryTimer = setTimeout(() => {
+			this.retryTimer = null;
+			this.start().catch((e) => {
+				this.log.error("Restart failed", { error: e });
+			});
+		}, delay);
 	}
+	/**
+	* Emits an event and waits for its handlers. Handlers run one after another in registration
+	* order: they share one database connection and one transaction, so interleaving them at
+	* await points would let two handlers read and write the same rows in an unpredictable order,
+	* and a failure in one would leave the others mid-flight while the block is rolled back. The
+	* first rejection propagates and stops the remaining handlers.
+	*/
 	asyncEmit = async (type, event) => {
-		event.uuid = v4();
-		let listenerCount = this.handled.get(type);
-		if (!listenerCount) listenerCount = 1;
-		let listenersResponded = 0;
-		const prom = new Promise((resolve, reject) => {
-			const returnFunc = (ev) => {
-				if (ev.uuid == event.uuid) if (ev.status) {
-					listenersResponded++;
-					if (listenersResponded == listenerCount) {
-						this.off("uuid", returnFunc);
-						resolve();
-					}
-				} else reject(ev.error);
-			};
-			this.on("uuid", returnFunc);
-		});
-		this.emit(type, event);
-		return prom;
+		const handlers = this.handlersFor(type);
+		if (handlers.length === 0) {
+			this.emit(type, event);
+			return;
+		}
+		for (const handler of handlers) await handler(event);
 	};
 	async processBlock(block, block_results, validators) {
 		const endTimer = this.prometheus?.timeBlockProcessing();
@@ -411,8 +633,8 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 		let beginBlockEvents;
 		let endBlockEvents;
 		if (block_results.finalizeBlockEvents) {
-			beginBlockEvents = block_results.finalizeBlockEvents.filter((x) => x.attributes.find((a) => decodeAttr(a.key) == "mode" && decodeAttr(a.value) == "begin_block"));
-			endBlockEvents = block_results.finalizeBlockEvents.filter((x) => x.attributes.find((a) => decodeAttr(a.key) == "mode" && decodeAttr(a.value) == "end_block"));
+			beginBlockEvents = block_results.finalizeBlockEvents.filter((x) => hasBlockEventMode(x, "BeginBlock"));
+			endBlockEvents = block_results.finalizeBlockEvents.filter((x) => hasBlockEventMode(x, "EndBlock"));
 		} else {
 			beginBlockEvents = block_results.beginBlockEvents;
 			endBlockEvents = block_results.endBlockEvents;
@@ -448,7 +670,13 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 					timestamp
 				});
 			}
-			let events = txlog ? JSON.parse(txlog) : [];
+			let events = [];
+			if (txlog) try {
+				const parsed = JSON.parse(txlog);
+				if (Array.isArray(parsed)) events = parsed;
+			} catch (_e) {
+				this.log.silly("Tx log is not JSON, using msg_index attributes instead");
+			}
 			if (events.length == 0) {
 				const eventsToAdd = [];
 				this.log.silly("No events found in tx log. Parsing events for msg_index");
@@ -470,8 +698,8 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 			}
 			const msgs = tx.body?.messages;
 			if (msgs) for (let i = 0; i < msgs.length; i++) {
-				this.log.silly("Indexer broadcasting msg for handling: " + msgs[i].typeUrl);
-				const msgevents = msgs.length > 1 ? events.find((x) => x.msg_index == i)?.events : events[0].events;
+				if (this.log.isSillyEnabled()) this.log.silly("Indexer broadcasting msg for handling: " + msgs[i].typeUrl);
+				const msgevents = msgs.length > 1 ? events.find((x) => x.msg_index == i)?.events : events[0]?.events ?? [];
 				await this.asyncEmit(msgs[i].typeUrl, {
 					value: {
 						tx: msgs[i].value,
@@ -483,7 +711,7 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 				if (msgs[i].typeUrl == "/cosmos.authz.v1beta1.MsgExec") {
 					const authzMsgs = MsgExec.decode(msgs[i].value).msgs;
 					if (authzMsgs) for (let r = 0; r < authzMsgs.length; r++) {
-						this.log.silly("Indexer broadcasting msg for handling: " + authzMsgs[r].typeUrl);
+						if (this.log.isSillyEnabled()) this.log.silly("Indexer broadcasting msg for handling: " + authzMsgs[r].typeUrl);
 						const authzMsgEvents = msgevents?.reduce((events$1, evt) => {
 							if (evt.attributes.filter((x) => decodeAttr(x.key) == "authz_msg_index" && decodeAttr(x.value) == "" + r).length > 0) events$1.push(evt);
 							return events$1;
@@ -511,136 +739,166 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 		endTimer?.();
 		this.prometheus?.updateBlockMetrics(height, this.latestHeight, this.blockQueue.size());
 	}
+	/**
+	* Fetches every height from nextFetchHeight up to latestHeight, waiting for queue space
+	* before each fetch. Serves the initial catch-up and live blocks alike: a new announcement
+	* only moves latestHeight and starts this loop if it is not already running, so bursts and
+	* skipped announcements are handled by the same code and the queue can never overflow.
+	*/
 	async fetcher() {
-		for (let i = this.heightToProcess; i <= this.latestHeight; i++) {
-			if (this.tryToRecover) {
-				this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
-				break;
-			}
-			this.log.debug("Fetching: " + i);
-			try {
-				if (this.isMinimal(this.blockQueue)) {
-					const timeoutPromise = new Promise((resolve, reject) => {
-						setTimeout(reject, RPC_TIMEOUT_MS, false);
-					});
-					const toIndex = Promise.race([Promise.all([this.blockClient.block(i), this.blockClient.blockResults(i)]), timeoutPromise]).catch((e) => {
-						this.log.error("Error fetching block: " + i + " : " + e);
-						this.prometheus?.recordError("rpc");
-						this.tryToRecover = true;
-						return Promise.resolve([]);
-					});
-					this.blockQueue.enqueue(toIndex);
-				} else {
-					const timeoutPromise = new Promise((resolve, reject) => {
-						setTimeout(reject, RPC_TIMEOUT_MS, false);
-					});
-					const q = QueryValidatorsRequest.fromPartial({ pagination: { limit: PAGINATION_LIMITS.VALIDATORS } });
-					const vals = QueryValidatorsRequest.encode(q).finish();
-					const toIndex = Promise.race([Promise.all([
-						this.blockClient.block(i),
-						this.blockClient.blockResults(i),
-						this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, i, false)
-					]), timeoutPromise]).catch((e) => {
-						this.log.error("Error fetching block: " + i + " : " + e);
-						this.tryToRecover = true;
-						return Promise.resolve([]);
-					});
-					this.blockQueue.enqueue(toIndex);
+		if (this.fetcherRunning && this.fetcherGeneration === this.runGeneration) return;
+		const generation = this.runGeneration;
+		const token = ++this.fetcherToken;
+		this.fetcherRunning = true;
+		this.fetcherGeneration = generation;
+		try {
+			while (this.nextFetchHeight <= this.latestHeight && (this.config.endHeight === void 0 || this.nextFetchHeight <= this.config.endHeight)) {
+				if (this.tryToRecover || !this.started || generation !== this.runGeneration) {
+					this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
+					break;
 				}
-			} catch (e) {
-				this.log.error("Fetching error: " + e);
-				break;
+				const i = this.nextFetchHeight;
+				this.log.debug("Fetching: " + i);
+				try {
+					if (this.isMinimal(this.blockQueue)) {
+						const toIndex = withTimeout(Promise.all([this.blockClient.block(i), this.blockClient.blockResults(i)]), RPC_TIMEOUT_MS, new RPCError("Timed out fetching block " + i)).catch((e) => {
+							this.log.error("Error fetching block " + i, { error: e });
+							this.prometheus?.recordError("rpc");
+							this.requestRecovery("fetch failed for block " + i, generation);
+							return Promise.resolve([]);
+						});
+						this.blockQueue.enqueue(toIndex);
+					} else {
+						const toIndex = withTimeout(Promise.all([
+							this.blockClient.block(i),
+							this.blockClient.blockResults(i),
+							this.fetchValidatorSet(i)
+						]), RPC_TIMEOUT_MS, new RPCError("Timed out fetching block " + i)).catch((e) => {
+							this.log.error("Error fetching block " + i, { error: e });
+							this.prometheus?.recordError("rpc");
+							this.requestRecovery("fetch failed for block " + i, generation);
+							return Promise.resolve([]);
+						});
+						this.blockQueue.enqueue(toIndex);
+					}
+				} catch (e) {
+					this.log.error("Fetching error", { error: e });
+					break;
+				}
+				this.nextFetchHeight = i + 1;
+				await this.blockQueue.continue();
 			}
-			await this.blockQueue.continue();
-			if (this.tryToRecover) {
-				this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
-				break;
+			if (!this.tryToRecover && this.started && generation === this.runGeneration && !this.blockQueue.synced) {
+				this.blockQueue.setSynced();
+				this.log.info("Synced to latest height");
 			}
-		}
-		if (!this.tryToRecover) {
-			this.blockQueue.setSynced();
-			this.log.info("Synced to latest height");
+		} finally {
+			if (this.fetcherToken === token) this.fetcherRunning = false;
 		}
 	}
+	/**
+	* Runs an ABCI query. A transport failure (RPC down, timeout, empty reply) requests a recovery.
+	* A reply with a non-zero code is the chain answering "no" (pruned height, unknown path, bad
+	* key): it is thrown as an RPCError with the code and log, and no recovery is requested for
+	* ad-hoc queries, so modules can catch it. Block-pipeline queries reject into the fetcher,
+	* which requests recovery itself.
+	*/
 	async callABCI(path, data, height, adHoc = true) {
+		let abciq;
+		const endTimer = this.prometheus?.timeRpcCall(path) ?? void 0;
 		try {
-			const endTimer = this.prometheus?.timeRpcCall(path) ?? void 0;
-			const abciq = await (adHoc ? this.client : this.blockClient).abciQuery({
+			abciq = await (adHoc ? this.client : this.blockClient).abciQuery({
 				path,
 				data,
 				height
 			});
+		} catch (e) {
+			this.setStatus("FAILED");
+			this.prometheus?.recordError("rpc");
+			this.requestRecovery("ABCI query failed for " + path);
+			throw new RPCError("RPC not responding. Query at: " + path + " (" + e + ")");
+		} finally {
 			endTimer?.();
-			if (abciq) return abciq.value;
-			else {
-				this.tryToRecover = true;
-				this.setStatus("FAILED");
-				this.prometheus?.recordError("rpc");
-				throw new Error("RPC not responding. Query at: " + path);
-			}
-		} catch (_e) {
-			this.tryToRecover = true;
+		}
+		if (!abciq) {
 			this.setStatus("FAILED");
-			throw new Error("RPC not responding. Query at: " + path);
+			this.prometheus?.recordError("rpc");
+			this.requestRecovery("empty ABCI response for " + path);
+			throw new RPCError("RPC not responding. Query at: " + path);
 		}
+		if (abciq.code) {
+			this.prometheus?.recordError("rpc");
+			throw new RPCError("ABCI query " + path + " failed with code " + abciq.code + (abciq.log ? ": " + abciq.log : ""));
+		}
+		return abciq.value;
+	}
+	/**
+	* Fetches the complete validator set at a height, following pagination, and returns it
+	* re-encoded as a single QueryValidatorsResponse so the block queue payload keeps its shape.
+	* Chains with more validators than one page (1000) were silently truncated before.
+	*/
+	async fetchValidatorSet(height) {
+		const validators = [];
+		let key;
+		do {
+			const request = QueryValidatorsRequest.fromPartial({ pagination: key ? {
+				limit: PAGINATION_LIMITS.VALIDATORS,
+				key
+			} : { limit: PAGINATION_LIMITS.VALIDATORS } });
+			const page = QueryValidatorsResponse.decode(await this.callABCI("/cosmos.staking.v1beta1.Query/Validators", QueryValidatorsRequest.encode(request).finish(), height, false));
+			validators.push(...page.validators);
+			key = page.pagination?.nextKey && page.pagination.nextKey.length > 0 ? page.pagination.nextKey : void 0;
+		} while (key);
+		return QueryValidatorsResponse.encode(QueryValidatorsResponse.fromPartial({ validators })).finish();
 	}
 	newBlockReceived(height) {
-		if (this.blockTimeout) clearTimeout(this.blockTimeout);
-		this.blockTimeout = setTimeout(() => {
-			this.tryToRecover = true;
-		}, 3e4);
+		this.armIdleCheck();
 		this.log.info("Received new block: %d", height);
-		if (height == this.latestHeight) return;
-		if (this.blockQueue.synced && !this.tryToRecover) {
-			this.latestHeight = height;
-			if (this.blockQueue.size() + 1 == this.config.batchSize) {
-				this.prometheus?.recordError("block");
-				this.log.error("Block queue is full. Cannot add new block");
-				this.tryToRecover = true;
-				return;
-			}
-			try {
-				if (this.isMinimal(this.blockQueue)) this.blockQueue.enqueue(Promise.all([this.client.block(height), this.client.blockResults(height)]).catch((e) => {
-					this.log.error("Error fetching block: " + height + " : " + e);
-					return Promise.resolve([]);
-				}));
-				else {
-					const q = QueryValidatorsRequest.fromPartial({ pagination: { limit: 1000n } });
-					const vals = QueryValidatorsRequest.encode(q).finish();
-					this.blockQueue.enqueue(Promise.all([
-						this.client.block(height),
-						this.client.blockResults(height),
-						this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, height, false)
-					]).catch((e) => {
-						this.log.error("Error fetching block: " + height + " : " + e);
-						return Promise.resolve([]);
-					}));
-				}
-			} catch (e) {
-				this.log.error("New Block Received error: " + e);
-			}
-		} else this.latestHeight = height;
+		if (height <= this.latestHeight) return;
+		this.latestHeight = height;
+		if (this.tryToRecover || !this.started) return;
+		this.fetcher().catch((e) => {
+			this.log.error("Error in fetching service", { error: e });
+			this.requestRecovery("fetcher failed");
+		});
+	}
+	/** Starts a single polling chain, retiring any chain left over from a previous run */
+	startPolling() {
+		this.stopPolling();
+		const generation = ++this.pollGeneration;
+		this.pollForBlock(generation);
 	}
-	async pollForBlock() {
+	stopPolling() {
+		this.pollGeneration++;
+		if (this.pollTimer) {
+			clearTimeout(this.pollTimer);
+			this.pollTimer = null;
+		}
+	}
+	async pollForBlock(generation) {
+		if (!this.started || generation !== this.pollGeneration) return;
 		try {
 			const status = await this.client.status();
-			if (status.syncInfo.latestBlockHeight > this.latestHeight) while (this.latestHeight < status.syncInfo.latestBlockHeight) this.newBlockReceived(this.latestHeight + 1);
-			setTimeout(() => {
-				this.pollForBlock();
-			}, this.config.pollingInterval);
+			if (!this.started || generation !== this.pollGeneration) return;
+			if (status.syncInfo.latestBlockHeight > this.latestHeight) this.newBlockReceived(status.syncInfo.latestBlockHeight);
 		} catch (e) {
-			this.log.error("Error polling for new block: " + e);
-			this.tryToRecover = true;
+			this.log.error("Error polling for new block", { error: e });
+			this.prometheus?.recordError("rpc");
+			this.requestRecovery("polling failed");
+			return;
 		}
+		this.pollTimer = setTimeout(() => {
+			this.pollForBlock(generation);
+		}, this.config.pollingInterval);
 	}
 	readGenesis() {
-		if (this.config.genesisPath) return fs.createReadStream(this.config.genesisPath).pipe(Parser.parser());
+		if (this.config.genesisPath) return fs.createReadStream(this.config.genesisPath);
 		else throw new Error("Genesis path not set");
 	}
 	async setArrayReader(path, processor) {
 		return new Promise((resolve, reject) => {
 			try {
-				const pickers = path.split(".").map((filter) => Pick.pick({ filter }));
+				const pickers = path.split(".").map((filter) => pick({ filter }));
 				let counter = 0;
 				let chunkCounter = 0;
 				const chunkProcessor = async (data) => {
@@ -652,48 +910,59 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 						await this.config.endTransaction(true);
 						await this.config.beginTransaction();
 					}
+					return data;
 				};
 				chain([
 					this.readGenesis(),
+					parser(),
 					...pickers,
-					StreamArray.streamArray(),
-					Batch.batch({ batchSize: GENESIS_BATCH_SIZE }),
+					streamArray(),
+					batch({ batchSize: GENESIS_BATCH_SIZE }),
 					chunkProcessor
 				]).on("data", (data) => {
 					if (data && Array.isArray(data)) counter = counter + data.length;
 				}).on("end", () => {
 					this.log.info(`Processed ${counter} entries in ${chunkCounter} chunks`);
 					resolve(true);
+				}).on("error", (e) => {
+					this.log.error("Error reading genesis array " + path, { error: e });
+					reject(e);
 				});
-			} catch (_e) {
-				this.log.verbose("Error in setArrayReader: " + _e);
-				reject();
+			} catch (e) {
+				this.log.verbose("Error in setArrayReader: " + e);
+				reject(e);
 			}
 		});
 	}
 	async setValueReader(path, processor) {
 		return new Promise((resolve, reject) => {
 			try {
-				const pickers = path.split(".").map((filter) => Pick.pick({ filter }));
+				const pickers = path.split(".").map((filter) => pick({ filter }));
 				let counter = 0;
 				chain([
 					this.readGenesis(),
+					parser(),
 					...pickers,
-					StreamValues.streamValues(),
+					streamValues(),
 					processor
 				]).on("data", (_data) => {
 					counter++;
 				}).on("end", () => {
 					this.log.info(`Processed ${counter} entries`);
 					resolve(true);
+				}).on("error", (e) => {
+					this.log.error("Error reading genesis value " + path, { error: e });
+					reject(e);
 				});
-			} catch (_e) {
-				reject();
+			} catch (e) {
+				this.log.verbose("Error in setValueReader: " + e);
+				reject(e);
 			}
 		});
 	}
 	async parseGenesis() {
 		this.log.info("Parsing genesis");
+		await this.config.onGenesisStart?.();
 		await this.config.beginTransaction();
 		try {
 			this.log.info("Starting genesis import");
@@ -721,6 +990,7 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 				}
 				return data;
 			});
+			await this.config.onGenesisComplete?.();
 			await this.config.endTransaction(true);
 			this.log.info("Finished importing");
 		} catch (e) {
@@ -728,14 +998,16 @@ var EcleciaIndexer = class extends EclesiaEmitter {
 				await this.config.endTransaction(false);
 			} catch (dbe) {
 				this.prometheus?.recordError("database");
-				this.log.error("Error ending transaction. Must be a DB error: " + dbe);
+				this.log.error("Error ending transaction", { error: dbe });
 			}
 			this.log.error("Failed to import genesis");
 			throw e;
 		}
 	}
 };
+/** @deprecated Misspelling kept for compatibility, use EclesiaIndexer. Removed in 3.0. */
+const EcleciaIndexer = EclesiaIndexer;
 
 //#endregion
-export { EcleciaIndexer, defaultIndexerConfig };
+export { EcleciaIndexer, EclesiaIndexer, defaultIndexerConfig };
 //# sourceMappingURL=index.mjs.map