@eclesia/indexer-engine 2.14.4 → 2.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/_virtual/rolldown_runtime.cjs +20 -15
  2. package/dist/_virtual/rolldown_runtime.mjs +10 -30
  3. package/dist/constants.cjs +31 -2
  4. package/dist/constants.cjs.map +1 -1
  5. package/dist/constants.d.cts +27 -3
  6. package/dist/constants.d.cts.map +1 -1
  7. package/dist/constants.d.mts +27 -3
  8. package/dist/constants.d.mts.map +1 -1
  9. package/dist/constants.mjs +27 -3
  10. package/dist/constants.mjs.map +1 -1
  11. package/dist/emitter/index.cjs +25 -7
  12. package/dist/emitter/index.cjs.map +1 -1
  13. package/dist/emitter/index.d.cts +8 -1
  14. package/dist/emitter/index.d.cts.map +1 -1
  15. package/dist/emitter/index.d.mts +8 -1
  16. package/dist/emitter/index.d.mts.map +1 -1
  17. package/dist/emitter/index.mjs +25 -7
  18. package/dist/emitter/index.mjs.map +1 -1
  19. package/dist/index.cjs +6 -0
  20. package/dist/index.d.cts +3 -3
  21. package/dist/index.d.mts +3 -3
  22. package/dist/index.mjs +3 -3
  23. package/dist/indexer/index.cjs +498 -225
  24. package/dist/indexer/index.cjs.map +1 -1
  25. package/dist/indexer/index.d.cts +111 -7
  26. package/dist/indexer/index.d.cts.map +1 -1
  27. package/dist/indexer/index.d.mts +111 -7
  28. package/dist/indexer/index.d.mts.map +1 -1
  29. package/dist/indexer/index.mjs +496 -224
  30. package/dist/indexer/index.mjs.map +1 -1
  31. package/dist/metrics/index.cjs +14 -0
  32. package/dist/metrics/index.cjs.map +1 -1
  33. package/dist/metrics/index.d.cts +7 -0
  34. package/dist/metrics/index.d.cts.map +1 -1
  35. package/dist/metrics/index.d.mts +7 -0
  36. package/dist/metrics/index.d.mts.map +1 -1
  37. package/dist/metrics/index.mjs +14 -0
  38. package/dist/metrics/index.mjs.map +1 -1
  39. package/dist/mocks/index.cjs +3 -1
  40. package/dist/mocks/index.cjs.map +1 -1
  41. package/dist/mocks/index.d.cts +2 -2
  42. package/dist/mocks/index.d.mts +2 -2
  43. package/dist/mocks/index.mjs +4 -2
  44. package/dist/mocks/index.mjs.map +1 -1
  45. package/dist/mocks/rpc-client.cjs +185 -20
  46. package/dist/mocks/rpc-client.cjs.map +1 -1
  47. package/dist/mocks/rpc-client.d.cts +26 -10
  48. package/dist/mocks/rpc-client.d.cts.map +1 -1
  49. package/dist/mocks/rpc-client.d.mts +26 -10
  50. package/dist/mocks/rpc-client.d.mts.map +1 -1
  51. package/dist/mocks/rpc-client.mjs +184 -22
  52. package/dist/mocks/rpc-client.mjs.map +1 -1
  53. package/dist/promise-queue/index.cjs +10 -4
  54. package/dist/promise-queue/index.cjs.map +1 -1
  55. package/dist/promise-queue/index.d.cts +1 -4
  56. package/dist/promise-queue/index.d.cts.map +1 -1
  57. package/dist/promise-queue/index.d.mts +1 -4
  58. package/dist/promise-queue/index.d.mts.map +1 -1
  59. package/dist/promise-queue/index.mjs +10 -4
  60. package/dist/promise-queue/index.mjs.map +1 -1
  61. package/dist/types/index.cjs.map +1 -1
  62. package/dist/types/index.d.cts +28 -5
  63. package/dist/types/index.d.cts.map +1 -1
  64. package/dist/types/index.d.mts +28 -5
  65. package/dist/types/index.d.mts.map +1 -1
  66. package/dist/types/index.mjs.map +1 -1
  67. package/dist/utils/index.cjs +6 -1
  68. package/dist/utils/index.cjs.map +1 -1
  69. package/dist/utils/index.d.cts +3 -2
  70. package/dist/utils/index.d.mts +3 -2
  71. package/dist/utils/index.mjs +7 -2
  72. package/dist/utils/index.mjs.map +1 -1
  73. package/dist/utils/text.cjs +31 -0
  74. package/dist/utils/text.cjs.map +1 -1
  75. package/dist/utils/text.d.cts +24 -1
  76. package/dist/utils/text.d.cts.map +1 -1
  77. package/dist/utils/text.d.mts +24 -1
  78. package/dist/utils/text.d.mts.map +1 -1
  79. package/dist/utils/text.mjs +30 -1
  80. package/dist/utils/text.mjs.map +1 -1
  81. package/dist/utils/timeout.cjs +32 -0
  82. package/dist/utils/timeout.cjs.map +1 -0
  83. package/dist/utils/timeout.d.cts +18 -0
  84. package/dist/utils/timeout.d.cts.map +1 -0
  85. package/dist/utils/timeout.d.mts +18 -0
  86. package/dist/utils/timeout.d.mts.map +1 -0
  87. package/dist/utils/timeout.mjs +30 -0
  88. package/dist/utils/timeout.mjs.map +1 -0
  89. package/package.json +35 -34
  90. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/ascii.cjs +0 -32
  91. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/ascii.cjs.map +0 -1
  92. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/ascii.mjs +0 -29
  93. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/ascii.mjs.map +0 -1
  94. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/base64.cjs +0 -67
  95. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/base64.cjs.map +0 -1
  96. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/base64.mjs +0 -64
  97. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/base64.mjs.map +0 -1
  98. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/bech32.cjs +0 -80
  99. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/bech32.cjs.map +0 -1
  100. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/bech32.mjs +0 -77
  101. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/bech32.mjs.map +0 -1
  102. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/hex.cjs +0 -33
  103. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/hex.cjs.map +0 -1
  104. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/hex.mjs +0 -30
  105. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/hex.mjs.map +0 -1
  106. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/index.cjs +0 -106
  107. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/index.cjs.map +0 -1
  108. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/index.mjs +0 -103
  109. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/index.mjs.map +0 -1
  110. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/rfc3339.cjs +0 -52
  111. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/rfc3339.cjs.map +0 -1
  112. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/rfc3339.mjs +0 -49
  113. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/rfc3339.mjs.map +0 -1
  114. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/utf8.cjs +0 -31
  115. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/utf8.cjs.map +0 -1
  116. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/utf8.mjs +0 -28
  117. package/dist/node_modules/.pnpm/@cosmjs_encoding@0.36.0/node_modules/@cosmjs/encoding/build/utf8.mjs.map +0 -1
  118. package/dist/node_modules/.pnpm/base64-js@1.5.1/node_modules/base64-js/index.cjs +0 -98
  119. package/dist/node_modules/.pnpm/base64-js@1.5.1/node_modules/base64-js/index.cjs.map +0 -1
  120. package/dist/node_modules/.pnpm/base64-js@1.5.1/node_modules/base64-js/index.mjs +0 -95
  121. package/dist/node_modules/.pnpm/base64-js@1.5.1/node_modules/base64-js/index.mjs.map +0 -1
@@ -5,6 +5,7 @@ const require_index$1 = require('../errors/index.cjs');
5
5
  const require_index$2 = require('../metrics/index.cjs');
6
6
  const require_index$3 = require('../promise-queue/index.cjs');
7
7
  const require_text = require('../utils/text.cjs');
8
+ const require_timeout = require('../utils/timeout.cjs');
8
9
  require('../utils/index.cjs');
9
10
  const require_index$5 = require('../validation/index.cjs');
10
11
  let crypto = require("crypto");
@@ -17,17 +18,16 @@ let cosmjs_types_cosmos_tx_v1beta1_tx_js = require("cosmjs-types/cosmos/tx/v1bet
17
18
  let fastify = require("fastify");
18
19
  fastify = require_rolldown_runtime.__toESM(fastify);
19
20
  let stream_chain = require("stream-chain");
20
- let stream_json = require("stream-json");
21
- stream_json = require_rolldown_runtime.__toESM(stream_json);
22
- let stream_json_filters_Pick_js = require("stream-json/filters/Pick.js");
23
- stream_json_filters_Pick_js = require_rolldown_runtime.__toESM(stream_json_filters_Pick_js);
24
- let stream_json_streamers_StreamArray_js = require("stream-json/streamers/StreamArray.js");
25
- stream_json_streamers_StreamArray_js = require_rolldown_runtime.__toESM(stream_json_streamers_StreamArray_js);
26
- let stream_json_streamers_StreamValues_js = require("stream-json/streamers/StreamValues.js");
27
- stream_json_streamers_StreamValues_js = require_rolldown_runtime.__toESM(stream_json_streamers_StreamValues_js);
28
- let stream_json_utils_Batch_js = require("stream-json/utils/Batch.js");
29
- stream_json_utils_Batch_js = require_rolldown_runtime.__toESM(stream_json_utils_Batch_js);
30
- let uuid = require("uuid");
21
+ let stream_json_filters_pick_js = require("stream-json/filters/pick.js");
22
+ stream_json_filters_pick_js = require_rolldown_runtime.__toESM(stream_json_filters_pick_js);
23
+ let stream_json_parser_js = require("stream-json/parser.js");
24
+ stream_json_parser_js = require_rolldown_runtime.__toESM(stream_json_parser_js);
25
+ let stream_json_streamers_stream_array_js = require("stream-json/streamers/stream-array.js");
26
+ stream_json_streamers_stream_array_js = require_rolldown_runtime.__toESM(stream_json_streamers_stream_array_js);
27
+ let stream_json_streamers_stream_values_js = require("stream-json/streamers/stream-values.js");
28
+ stream_json_streamers_stream_values_js = require_rolldown_runtime.__toESM(stream_json_streamers_stream_values_js);
29
+ let stream_json_utils_batch_js = require("stream-json/utils/batch.js");
30
+ stream_json_utils_batch_js = require_rolldown_runtime.__toESM(stream_json_utils_batch_js);
31
31
  let winston = require("winston");
32
32
  winston = require_rolldown_runtime.__toESM(winston);
33
33
 
@@ -55,7 +55,7 @@ const defaultIndexerConfig = {
55
55
  * Core blockchain indexer that connects to Tendermint RPC and processes blocks
56
56
  * Extends EclesiaEmitter to provide event-driven architecture for modules
57
57
  */
58
- var EcleciaIndexer = class extends require_index.EclesiaEmitter {
58
+ var EclesiaIndexer = class extends require_index.EclesiaEmitter {
59
59
  /** Indexer configuration settings */
60
60
  config;
61
61
  /** Fastify HTTP server for health checks */
@@ -90,6 +90,31 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
90
90
  subscription = null;
91
91
  /** Timeout handler for block reception */
92
92
  blockTimeout = null;
93
+ /** Timer for the next poll in polling mode */
94
+ pollTimer = null;
95
+ /** Bumped on every (re)start and stop so a polling chain from a previous run exits */
96
+ pollGeneration = 0;
97
+ /** Bumped on every start() so callbacks left over from a previous run cannot trigger recovery in this one */
98
+ runGeneration = 0;
99
+ /**
100
+ * Rejecters of waits parked in waitForBlockData(). Each entry is removed as soon as its block
101
+ * arrives, so a healthy run keeps this empty instead of accumulating one entry per block.
102
+ */
103
+ blockWaiters = /* @__PURE__ */ new Set();
104
+ /** Pending restart timer */
105
+ retryTimer = null;
106
+ /** Resolves when the indexer has stopped for good: stop() was called, endHeight was reached, or it gave up */
107
+ stopped = Promise.resolve();
108
+ resolveStopped = () => {};
109
+ /** Next height the fetcher will request; advances as fetches are enqueued */
110
+ nextFetchHeight = 0;
111
+ /** Whether a fetcher loop is active, and for which run */
112
+ fetcherRunning = false;
113
+ fetcherGeneration = 0;
114
+ fetcherToken = 0;
115
+ /** Height of the block whose processing failed most recently, and how many times in a row */
116
+ lastFailedHeight;
117
+ sameHeightFailures = 0;
93
118
  /**
94
119
  * Creates a new Eclesia indexer instance
95
120
  * @param config - Indexer configuration options
@@ -103,40 +128,44 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
103
128
  if (config.prometheusPort !== void 0) require_index$5.validatePort(config.prometheusPort, "prometheusPort");
104
129
  if (config.startHeight !== void 0) require_index$5.validatePositiveInteger(config.startHeight, "startHeight");
105
130
  if (config.pollingInterval !== void 0) require_index$5.validatePositiveInteger(config.pollingInterval, "pollingInterval");
131
+ const provided = Object.fromEntries(Object.entries(config).filter(([, value]) => value !== void 0));
106
132
  this.config = {
107
133
  ...defaultIndexerConfig,
108
- ...config
134
+ ...provided
109
135
  };
110
- const { printf } = winston.format;
111
- const eclesiaFormat = printf(({ level, message, timestamp }) => {
112
- return `${timestamp} [${level.toUpperCase()}]:\t${message}`;
136
+ const errorFormat = winston.format((info) => {
137
+ const error = info.error;
138
+ if (error instanceof Error) info.error = {
139
+ name: error.name,
140
+ message: error.message,
141
+ stack: error.stack
142
+ };
143
+ else if (error !== void 0 && (typeof error !== "object" || error === null)) info.error = { message: String(error) };
144
+ return info;
145
+ });
146
+ const textFormat = winston.format.printf(({ level, message, timestamp, error }) => {
147
+ const detail = error;
148
+ const suffix = detail ? "\n" + (detail.stack ?? detail.message ?? "") : "";
149
+ return `${timestamp} [${level.toUpperCase()}]:\t${message}${suffix}`;
113
150
  });
114
151
  this.log = winston.createLogger({
115
152
  level: this.config.logLevel,
116
153
  defaultMeta: { service: "Eclesia Indexer" },
117
- transports: [
118
- new winston.transports.File({
119
- filename: "error.log",
120
- level: "error"
121
- }),
122
- new winston.transports.File({ filename: "combined.log" }),
123
- new winston.transports.Console({ format: winston.format.combine(winston.format.splat(), winston.format.timestamp(), eclesiaFormat, winston.format.colorize({ all: true })) })
124
- ]
154
+ transports: [new winston.transports.Console({ format: this.config.logFormat === "json" ? winston.format.combine(winston.format.splat(), winston.format.timestamp(), errorFormat(), winston.format.json()) : winston.format.combine(winston.format.splat(), winston.format.timestamp(), errorFormat(), textFormat, winston.format.colorize({ all: true })) })]
125
155
  });
156
+ const protocol = new URL(this.config.rpcUrl).protocol;
157
+ if (!this.config.usePolling && (protocol === "http:" || protocol === "https:")) {
158
+ this.log.warn("rpcUrl " + require_text.redactUrl(this.config.rpcUrl) + " is HTTP, which cannot deliver block subscriptions; polling every " + this.config.pollingInterval + " ms instead (use a ws:// or wss:// URL for WebSocket mode)");
159
+ this.config.usePolling = true;
160
+ }
126
161
  const queueErrorHandler = (e) => {
127
162
  this.prometheus?.recordError("rpc");
128
- this.log.error("Error enqueueing block data: " + e);
163
+ this.log.error("Error enqueueing block data", { error: e });
129
164
  };
130
165
  if (this.config.minimal) this.blockQueue = new require_index$3.CircularBuffer(this.config.batchSize, queueErrorHandler);
131
166
  else this.blockQueue = new require_index$3.CircularBuffer(this.config.batchSize, queueErrorHandler);
132
167
  this.on("_unhandled", (msg) => {
133
- if (msg.uuid) {
134
- this.log.verbose("Unhandled event: " + msg.type);
135
- this.emit("uuid", {
136
- status: true,
137
- uuid: msg.uuid
138
- });
139
- }
168
+ if (msg.type !== "uuid" && this.log.isVerboseEnabled()) this.log.verbose("Unhandled event: " + msg.type);
140
169
  });
141
170
  if (this.config.enablePrometheus) {
142
171
  this.prometheus = new require_index$2.IndexerMetrics();
@@ -148,10 +177,10 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
148
177
  const prometheusPort = this.config.prometheusPort ?? (process.env.PROMETHEUS_PORT ? parseInt(process.env.PROMETHEUS_PORT, 10) : require_constants.DEFAULT_PROMETHEUS_PORT);
149
178
  this.prometheusServer.listen({
150
179
  port: prometheusPort,
151
- host: "0.0.0.0"
180
+ host: this.config.prometheusHost ?? require_constants.DEFAULT_BIND_HOST
152
181
  }, (err) => {
153
182
  if (err) {
154
- this.log.error("Prometheus error: " + err);
183
+ this.log.error("Prometheus server error", { error: err });
155
184
  this.prometheus?.recordError("metrics_server");
156
185
  this.emit("fatal-error", {
157
186
  error: err,
@@ -163,16 +192,16 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
163
192
  if (this.config.enableHealthcheck) {
164
193
  this.fastify = (0, fastify.default)({ logger: false });
165
194
  this.fastify.get("/health", async (_request, reply) => {
166
- const code = this.healthCheck.status == "OK" ? 200 : 503;
195
+ const code = this.healthCheck.status == "OK" || this.healthCheck.status == "WAITING" ? 200 : 503;
167
196
  reply.code(code).send(this.healthCheck);
168
197
  });
169
198
  const healthPort = this.config.healthCheckPort ?? (process.env.HEALTH_CHECK_PORT ? parseInt(process.env.HEALTH_CHECK_PORT, 10) : require_constants.DEFAULT_HEALTH_CHECK_PORT);
170
199
  this.fastify.listen({
171
200
  port: healthPort,
172
- host: "0.0.0.0"
201
+ host: this.config.healthCheckHost ?? require_constants.DEFAULT_BIND_HOST
173
202
  }, (err) => {
174
203
  if (err) {
175
- this.log.error("Health check server error: " + err);
204
+ this.log.error("Health check server error", { error: err });
176
205
  this.prometheus?.recordError("health_check_server");
177
206
  this.emit("fatal-error", {
178
207
  error: err,
@@ -184,10 +213,110 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
184
213
  }
185
214
  setStatus(status) {
186
215
  this.healthCheck.status = status;
216
+ this.prometheus?.setWaiting(status === "WAITING");
217
+ }
218
+ /**
219
+ * Marks the current run for recovery and wakes the main loop if it is parked waiting for a
220
+ * block. Callbacks left over from a previous run pass their generation and are ignored.
221
+ */
222
+ requestRecovery(reason, generation = this.runGeneration) {
223
+ if (generation !== this.runGeneration) {
224
+ this.log.debug("Ignoring recovery request from a previous run: " + reason);
225
+ return;
226
+ }
227
+ if (!this.tryToRecover) this.log.warn("Recovery requested: " + reason);
228
+ this.tryToRecover = true;
229
+ this.wakeBlockWaiters("Recovery requested while waiting for block data");
230
+ }
231
+ /** Rejects every wait parked in waitForBlockData() */
232
+ wakeBlockWaiters(reason) {
233
+ const waiters = [...this.blockWaiters];
234
+ this.blockWaiters.clear();
235
+ for (const reject of waiters) reject(new require_index$1.RPCError(reason));
236
+ }
237
+ /**
238
+ * Refuses to index a chain other than the configured one. An RPC pool that mixes networks, or
239
+ * a wrong URL, would otherwise write a different chain's blocks into the database.
240
+ */
241
+ assertChainId(network) {
242
+ if (this.config.chainId !== void 0 && network !== this.config.chainId) throw new require_index$1.ConfigurationError("RPC serves chain " + network + " but chainId is configured as " + this.config.chainId, {
243
+ expected: this.config.chainId,
244
+ actual: network
245
+ });
246
+ }
247
+ /** Counts consecutive processing failures per block height */
248
+ noteBlockFailure(height) {
249
+ if (height === this.lastFailedHeight) this.sameHeightFailures++;
250
+ else {
251
+ this.lastFailedHeight = height;
252
+ this.sameHeightFailures = 1;
253
+ }
254
+ }
255
+ /** True once one block has failed maxFailuresPerBlock times in a row */
256
+ isStuck() {
257
+ return this.lastFailedHeight !== void 0 && this.sameHeightFailures >= (this.config.maxFailuresPerBlock ?? require_constants.MAX_FAILURES_PER_BLOCK);
258
+ }
259
+ /**
260
+ * Waits for the next dequeued block but wakes early when recovery or stop is requested, so a
261
+ * loop parked on an empty queue never waits for a block that will not come.
262
+ */
263
+ waitForBlockData(dequeued) {
264
+ if (this.tryToRecover || !this.started) return Promise.reject(new require_index$1.RPCError("Recovery requested while waiting for block data"));
265
+ return new Promise((resolve, reject) => {
266
+ this.blockWaiters.add(reject);
267
+ dequeued.then((value) => {
268
+ this.blockWaiters.delete(reject);
269
+ resolve(value);
270
+ }, (error) => {
271
+ this.blockWaiters.delete(reject);
272
+ reject(error);
273
+ });
274
+ });
275
+ }
276
+ /** (Re)arms the idle check that runs when no block has been announced for a while */
277
+ armIdleCheck() {
278
+ if (this.blockTimeout) clearTimeout(this.blockTimeout);
279
+ this.blockTimeout = setTimeout(() => {
280
+ this.checkLiveness();
281
+ }, require_constants.IDLE_CHECK_INTERVAL_MS);
282
+ }
283
+ /**
284
+ * Runs when no block has been announced for IDLE_CHECK_INTERVAL_MS. A chain that has stopped
285
+ * producing blocks (halt, upgrade, slow chain) is not an error: the indexer reports WAITING and
286
+ * checks again later. Recovery is requested only when the chain has moved on without us, which
287
+ * means the subscription is dead, or when the RPC cannot be reached at all.
288
+ */
289
+ async checkLiveness() {
290
+ if (!this.started) return;
291
+ const generation = this.runGeneration;
292
+ try {
293
+ const status = await require_timeout.withTimeout(this.client.status(), require_constants.RPC_TIMEOUT_MS, new require_index$1.RPCError("RPC status call timed out"));
294
+ if (!this.started || generation !== this.runGeneration) return;
295
+ const chainHeight = status.syncInfo.latestBlockHeight;
296
+ if (chainHeight > this.latestHeight) {
297
+ this.requestRecovery("chain is at " + chainHeight + " but nothing was announced since " + this.latestHeight, generation);
298
+ return;
299
+ }
300
+ this.log.info("No new block for " + require_constants.IDLE_CHECK_INTERVAL_MS / 1e3 + " s, chain height is still " + chainHeight);
301
+ this.setStatus("WAITING");
302
+ this.armIdleCheck();
303
+ } catch (e) {
304
+ this.log.error("Liveness check failed", { error: e });
305
+ this.requestRecovery("RPC unreachable during liveness check", generation);
306
+ }
187
307
  }
188
- blockListener = { next: (data) => {
189
- this.newBlockReceived(data.header.height);
190
- } };
308
+ blockListener = {
309
+ next: (data) => {
310
+ this.newBlockReceived(data.header.height);
311
+ },
312
+ error: (error) => {
313
+ this.log.error("Block subscription error", { error });
314
+ this.requestRecovery("block subscription errored");
315
+ },
316
+ complete: () => {
317
+ if (this.started) this.requestRecovery("block subscription closed by the node");
318
+ }
319
+ };
191
320
  isMinimal(_blockqueue) {
192
321
  if (this.config.minimal) return true;
193
322
  else return false;
@@ -204,38 +333,55 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
204
333
  } catch (_e) {}
205
334
  this.log.verbose("Disconnected from RPC");
206
335
  }
207
- const connectTimeoutPromise = new Promise((resolve, reject) => {
208
- setTimeout(reject, require_constants.CONNECT_TIMEOUT_MS, new require_index$1.RPCError("RPC connection timed out"));
209
- });
210
- this.log.info("Attempting to connected to RPC: " + this.config.rpcUrl);
211
- this.client = await Promise.race([(0, __cosmjs_tendermint_rpc.connectComet)(this.config.rpcUrl), connectTimeoutPromise]);
212
- await Promise.race([this.client.status(), connectTimeoutPromise]);
336
+ this.log.info("Attempting to connect to RPC: " + require_text.redactUrl(this.config.rpcUrl));
337
+ this.client = await this.connectWithTimeout();
338
+ await require_timeout.withTimeout(this.client.status(), require_constants.CONNECT_TIMEOUT_MS, new require_index$1.RPCError("RPC status call timed out"));
213
339
  this.log.info("Connected to RPC for ad hoc queries");
214
- this.blockClient = await Promise.race([(0, __cosmjs_tendermint_rpc.connectComet)(this.config.rpcUrl), connectTimeoutPromise]);
215
- await Promise.race([this.blockClient.status(), connectTimeoutPromise]);
340
+ this.blockClient = await this.connectWithTimeout();
341
+ await require_timeout.withTimeout(this.blockClient.status(), require_constants.CONNECT_TIMEOUT_MS, new require_index$1.RPCError("RPC status call timed out"));
216
342
  this.log.info("Connected to RPC for block & validator info");
217
343
  return true;
218
344
  } catch (error) {
219
- this.log.error("RPC connection error: " + error);
345
+ this.log.error("RPC connection error", { error });
220
346
  this.prometheus?.recordError("rpc");
221
- this.tryToRecover = true;
347
+ this.requestRecovery("RPC connection failed");
222
348
  return false;
223
349
  }
224
350
  }
351
+ /**
352
+ * Opens one CometBFT client with its own timeout. If the timeout wins, the client
353
+ * that may still arrive is disconnected so a slow RPC never leaks a socket.
354
+ */
355
+ async connectWithTimeout() {
356
+ let timedOut = false;
357
+ const pending = (0, __cosmjs_tendermint_rpc.connectComet)(this.config.rpcUrl);
358
+ pending.then((client) => {
359
+ if (timedOut) try {
360
+ client.disconnect();
361
+ } catch (_e) {}
362
+ }).catch(() => {});
363
+ try {
364
+ return await require_timeout.withTimeout(pending, require_constants.CONNECT_TIMEOUT_MS, new require_index$1.RPCError("RPC connection timed out"));
365
+ } catch (e) {
366
+ timedOut = true;
367
+ throw e;
368
+ }
369
+ }
225
370
  async initialize() {
226
371
  if (!this.initialized) {
227
372
  try {
228
373
  if (this.config.init) await this.config.init();
229
374
  } catch (e) {
230
- this.log.error("Failed to initialize indexer: " + e);
375
+ this.log.error("Failed to initialize indexer", { error: e });
231
376
  this.prometheus?.recordError("init_error");
232
377
  this.setStatus("FAILED");
233
378
  throw e;
234
379
  }
235
380
  if (await this.config.shouldProcessGenesis()) try {
236
381
  if (this.config.genesisPath) await this.parseGenesis();
382
+ else this.log.warn("shouldProcessGenesis() returned true but no genesisPath is configured, skipping genesis import");
237
383
  } catch (e) {
238
- this.log.error("Failed to parse genesis: " + e);
384
+ this.log.error("Failed to parse genesis", { error: e });
239
385
  this.prometheus?.recordError("genesis_error");
240
386
  this.setStatus("FAILED");
241
387
  throw e;
@@ -243,8 +389,40 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
243
389
  this.initialized = true;
244
390
  }
245
391
  }
246
- stop() {
392
+ /**
393
+ * Stops the indexer and releases everything that would keep the process alive:
394
+ * the block subscription, polling and inactivity timers, both RPC clients and the
395
+ * health and metrics servers. Safe to call more than once.
396
+ */
397
+ async stop() {
247
398
  this.started = false;
399
+ this.resolveStopped();
400
+ if (this.retryTimer) {
401
+ clearTimeout(this.retryTimer);
402
+ this.retryTimer = null;
403
+ }
404
+ this.wakeBlockWaiters("Indexer stopped while waiting for block data");
405
+ this.stopPolling();
406
+ if (this.blockTimeout) {
407
+ clearTimeout(this.blockTimeout);
408
+ this.blockTimeout = null;
409
+ }
410
+ if (this.subscription) {
411
+ try {
412
+ this.subscription.removeListener(this.blockListener);
413
+ } catch (_e) {}
414
+ this.subscription = null;
415
+ }
416
+ for (const client of [this.client, this.blockClient]) try {
417
+ client?.disconnect();
418
+ } catch (_e) {}
419
+ const servers = [this.fastify, this.prometheusServer];
420
+ this.fastify = null;
421
+ this.prometheusServer = null;
422
+ await Promise.all(servers.map((server) => server?.close().catch((e) => {
423
+ this.log.warn("Error closing HTTP server", { error: e });
424
+ })));
425
+ this.log.info("Indexer stopped");
248
426
  }
249
427
  clearBlockQueue() {
250
428
  if (this.blockQueue) {
@@ -264,65 +442,88 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
264
442
  this.log.verbose("Removed existing block listener and subscription");
265
443
  }
266
444
  if (!this.config.usePolling) this.subscription = this.client.subscribeNewBlock ? this.client.subscribeNewBlock() : null;
267
- const statusPromise = new Promise((resolve, reject) => {
268
- setTimeout(reject, require_constants.RPC_TIMEOUT_MS, new require_index$1.RPCError("RPC status call timed out"));
269
- });
270
- this.latestHeight = (await Promise.race([this.client.status(), statusPromise])).syncInfo.latestBlockHeight;
271
- this.log.info("Current chain height: " + this.latestHeight);
445
+ const status = await require_timeout.withTimeout(this.client.status(), require_constants.RPC_TIMEOUT_MS, new require_index$1.RPCError("RPC status call timed out"));
446
+ this.assertChainId(status.nodeInfo.network);
447
+ this.latestHeight = status.syncInfo.latestBlockHeight;
448
+ this.log.info("Connected to " + status.nodeInfo.network + ", current chain height: " + this.latestHeight);
272
449
  this.heightToProcess = await this.config.getNextHeight();
273
- if (this.config.usePolling) this.pollForBlock();
450
+ this.nextFetchHeight = this.heightToProcess;
451
+ if (this.config.usePolling) this.startPolling();
274
452
  else if (this.subscription) this.subscription.addListener(this.blockListener);
275
453
  else {
276
454
  this.prometheus?.recordError("rpc");
277
455
  throw new Error("Could not subscribe to new blocks");
278
456
  }
457
+ this.armIdleCheck();
279
458
  } catch (e) {
280
- this.log.error("Failed to set up block listening: " + e);
459
+ this.log.error("Failed to set up block listening", { error: e });
281
460
  this.prometheus?.recordError("rpc");
282
461
  this.setStatus("FAILED");
283
462
  throw e;
284
463
  }
285
464
  }
465
+ /**
466
+ * Resolves once the indexer has stopped for good: stop() was called, endHeight was reached, or
467
+ * a fatal-error was emitted. Restarts with backoff do not resolve it. Use it to keep a caller
468
+ * waiting for the whole run rather than for the first loop exit.
469
+ */
470
+ whenStopped() {
471
+ return this.stopped;
472
+ }
286
473
  async start() {
474
+ if (!this.started) this.stopped = new Promise((resolve) => {
475
+ this.resolveStopped = resolve;
476
+ });
287
477
  this.started = true;
478
+ this.runGeneration++;
479
+ const generation = this.runGeneration;
480
+ this.tryToRecover = false;
481
+ this.blockWaiters.clear();
288
482
  this.clearBlockQueue();
289
483
  await this.initialize();
290
484
  try {
291
485
  await this.setupBlockListening();
292
486
  this.log.debug("Starting main processing loop");
293
- this.tryToRecover = false;
294
487
  this.fetcher().catch((e) => {
295
488
  this.setStatus("FAILED");
296
489
  this.prometheus?.recordError("rpc");
297
- throw new Error("Error in fetching service: " + e);
490
+ this.log.error("Error in fetching service", { error: e });
491
+ this.requestRecovery("fetcher failed", generation);
298
492
  });
299
- } catch (_e) {}
300
- while (this.started && !this.tryToRecover && (this.config.endHeight === void 0 || this.heightToProcess <= this.config.endHeight) && this.blockQueue.size() > 0) {
493
+ } catch (e) {
494
+ this.requestRecovery("block listening setup failed: " + e, generation);
495
+ }
496
+ let lastProcessed;
497
+ while (this.started && !this.tryToRecover) {
498
+ let txOpen = false;
499
+ let failingHeight;
301
500
  try {
302
501
  this.prometheus?.updateRetryCount(this.retryCount);
303
- await this.config.beginTransaction();
304
- this.log.silly("Started db tx");
305
- let height, timestamp;
502
+ if (this.blockQueue.synced && this.blockQueue.size() <= 1) this.setStatus("WAITING");
503
+ let height;
504
+ let timestamp;
306
505
  if (this.isMinimal(this.blockQueue)) {
307
- const timeoutPromise = new Promise((resolve, reject) => {
308
- setTimeout(reject, require_constants.QUEUE_DEQUEUE_TIMEOUT_MS, []);
309
- });
310
- const toProcess = await Promise.race([this.blockQueue.dequeue(), timeoutPromise]);
506
+ const toProcess = await this.waitForBlockData(this.blockQueue.dequeue());
311
507
  this.log.silly("Retrieved block data");
312
508
  if (!toProcess || !toProcess[0] || !toProcess[1]) throw new require_index$1.RPCError("Could not fetch block(minimal)");
313
509
  height = toProcess[0].block.header.height;
510
+ failingHeight = height;
314
511
  timestamp = (0, __cosmjs_tendermint_rpc.toRfc3339WithNanoseconds)(toProcess[0].block.header.time);
512
+ await this.config.beginTransaction();
513
+ txOpen = true;
514
+ this.log.silly("Started db tx");
315
515
  await this.processBlock(toProcess[0], toProcess[1]);
316
516
  } else {
317
- const timeoutPromise = new Promise((resolve, reject) => {
318
- setTimeout(reject, require_constants.QUEUE_DEQUEUE_TIMEOUT_MS, []);
319
- });
320
- const toProcess = await Promise.race([this.blockQueue.dequeue(), timeoutPromise]);
517
+ const toProcess = await this.waitForBlockData(this.blockQueue.dequeue());
321
518
  this.log.silly("Retrieved block data");
322
519
  if (!toProcess || !toProcess[0] || !toProcess[1] || !toProcess[2]) throw new require_index$1.RPCError("Could not fetch block(full)");
323
520
  this.log.silly("Decoded block");
324
521
  height = toProcess[0].block.header.height;
522
+ failingHeight = height;
325
523
  timestamp = (0, __cosmjs_tendermint_rpc.toRfc3339WithNanoseconds)(toProcess[0].block.header.time);
524
+ await this.config.beginTransaction();
525
+ txOpen = true;
526
+ this.log.silly("Started db tx");
326
527
  await this.processBlock(toProcess[0], toProcess[1], cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsResponse.decode(toProcess[2]).validators);
327
528
  }
328
529
  if (height % require_constants.PERIODIC_INTERVALS.LARGE == 0) await this.asyncEmit("periodic/large", {
@@ -342,65 +543,86 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
342
543
  });
343
544
  this.log.silly("Handled periodic events");
344
545
  await this.config.endTransaction(true);
546
+ txOpen = false;
547
+ lastProcessed = height;
548
+ this.lastFailedHeight = void 0;
549
+ this.sameHeightFailures = 0;
345
550
  this.log.silly("Committed db tx");
346
551
  } catch (e) {
347
- this.prometheus?.recordError("block");
348
- this.log.error("Block processing error: " + e);
349
- this.setStatus("FAILED");
350
- try {
552
+ if (txOpen) try {
351
553
  await this.config.endTransaction(false);
352
554
  } catch (dbe) {
353
555
  this.prometheus?.recordError("database");
354
- this.log.error("Error ending transaction. Must be a DB error: " + dbe);
556
+ this.log.error("Error ending transaction", { error: dbe });
355
557
  }
356
- this.tryToRecover = true;
558
+ if (!this.started) break;
559
+ if (failingHeight !== void 0) this.noteBlockFailure(failingHeight);
560
+ this.prometheus?.recordError("block");
561
+ this.log.error("Block processing error", { error: e });
562
+ this.setStatus("FAILED");
563
+ this.requestRecovery("block processing failed", generation);
357
564
  break;
358
565
  }
359
566
  this.retryCount = 0;
360
567
  this.setStatus("OK");
568
+ if (this.config.endHeight !== void 0 && lastProcessed !== void 0 && lastProcessed >= this.config.endHeight) {
569
+ this.log.info("Reached configured end height " + this.config.endHeight + ". Stopping indexer.");
570
+ await this.stop();
571
+ return;
572
+ }
361
573
  }
362
574
  if (!this.started) {
363
575
  this.log.info("Indexer manually stopped.");
364
576
  return;
365
577
  }
366
- if (this.config.endHeight !== void 0 && this.heightToProcess >= this.config.endHeight) {
367
- this.log.info("Reached configured end height. Stopping indexer.");
368
- this.stop();
578
+ if (this.isStuck()) {
579
+ const height = this.lastFailedHeight;
580
+ this.log.error("Block " + height + " failed " + this.sameHeightFailures + " times in a row. This is a deterministic failure in a handler or the data, not an outage. Giving up.");
581
+ this.started = false;
582
+ this.emit("fatal-error", {
583
+ error: /* @__PURE__ */ new Error("Block " + height + " failed " + this.sameHeightFailures + " times in a row"),
584
+ message: "Block processing is stuck",
585
+ retryCount: this.retryCount,
586
+ height
587
+ });
588
+ this.resolveStopped();
369
589
  return;
370
590
  }
371
- if (this.retryCount < 3) {
372
- this.log.debug("Indexer retryCount: " + this.retryCount);
373
- this.retryCount++;
374
- this.log.info("Indexer is restarting");
375
- setTimeout(() => this.start(), this.retryCount * 5e3);
376
- } else {
377
- this.log.info("Indexer failed too many times. Exiting.");
591
+ this.retryCount++;
592
+ if (this.config.maxRetries !== void 0 && this.retryCount > this.config.maxRetries) {
593
+ this.log.error("Indexer failed " + this.retryCount + " times in a row, giving up (maxRetries=" + this.config.maxRetries + ")");
594
+ this.started = false;
378
595
  this.emit("fatal-error", {
379
596
  error: /* @__PURE__ */ new Error("Max retry attempts exceeded"),
380
597
  message: "Indexer failed too many times",
381
598
  retryCount: this.retryCount
382
599
  });
600
+ this.resolveStopped();
601
+ return;
383
602
  }
603
+ const delay = require_timeout.retryDelay(this.retryCount, require_constants.RETRY_BASE_DELAY_MS, require_constants.RETRY_MAX_DELAY_MS);
604
+ this.log.warn("Indexer is restarting in " + delay / 1e3 + " s (attempt " + this.retryCount + ")");
605
+ this.retryTimer = setTimeout(() => {
606
+ this.retryTimer = null;
607
+ this.start().catch((e) => {
608
+ this.log.error("Restart failed", { error: e });
609
+ });
610
+ }, delay);
384
611
  }
612
+ /**
613
+ * Emits an event and waits for its handlers. Handlers run one after another in registration
614
+ * order: they share one database connection and one transaction, so interleaving them at
615
+ * await points would let two handlers read and write the same rows in an unpredictable order,
616
+ * and a failure in one would leave the others mid-flight while the block is rolled back. The
617
+ * first rejection propagates and stops the remaining handlers.
618
+ */
385
619
  asyncEmit = async (type, event) => {
386
- event.uuid = (0, uuid.v4)();
387
- let listenerCount = this.handled.get(type);
388
- if (!listenerCount) listenerCount = 1;
389
- let listenersResponded = 0;
390
- const prom = new Promise((resolve, reject) => {
391
- const returnFunc = (ev) => {
392
- if (ev.uuid == event.uuid) if (ev.status) {
393
- listenersResponded++;
394
- if (listenersResponded == listenerCount) {
395
- this.off("uuid", returnFunc);
396
- resolve();
397
- }
398
- } else reject(ev.error);
399
- };
400
- this.on("uuid", returnFunc);
401
- });
402
- this.emit(type, event);
403
- return prom;
620
+ const handlers = this.handlersFor(type);
621
+ if (handlers.length === 0) {
622
+ this.emit(type, event);
623
+ return;
624
+ }
625
+ for (const handler of handlers) await handler(event);
404
626
  };
405
627
  async processBlock(block, block_results, validators) {
406
628
  const endTimer = this.prometheus?.timeBlockProcessing();
@@ -420,8 +642,8 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
420
642
  let beginBlockEvents;
421
643
  let endBlockEvents;
422
644
  if (block_results.finalizeBlockEvents) {
423
- beginBlockEvents = block_results.finalizeBlockEvents.filter((x) => x.attributes.find((a) => require_text.decodeAttr(a.key) == "mode" && require_text.decodeAttr(a.value) == "begin_block"));
424
- endBlockEvents = block_results.finalizeBlockEvents.filter((x) => x.attributes.find((a) => require_text.decodeAttr(a.key) == "mode" && require_text.decodeAttr(a.value) == "end_block"));
645
+ beginBlockEvents = block_results.finalizeBlockEvents.filter((x) => require_text.hasBlockEventMode(x, "BeginBlock"));
646
+ endBlockEvents = block_results.finalizeBlockEvents.filter((x) => require_text.hasBlockEventMode(x, "EndBlock"));
425
647
  } else {
426
648
  beginBlockEvents = block_results.beginBlockEvents;
427
649
  endBlockEvents = block_results.endBlockEvents;
@@ -457,7 +679,13 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
457
679
  timestamp
458
680
  });
459
681
  }
460
- let events = txlog ? JSON.parse(txlog) : [];
682
+ let events = [];
683
+ if (txlog) try {
684
+ const parsed = JSON.parse(txlog);
685
+ if (Array.isArray(parsed)) events = parsed;
686
+ } catch (_e) {
687
+ this.log.silly("Tx log is not JSON, using msg_index attributes instead");
688
+ }
461
689
  if (events.length == 0) {
462
690
  const eventsToAdd = [];
463
691
  this.log.silly("No events found in tx log. Parsing events for msg_index");
@@ -479,8 +707,8 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
479
707
  }
480
708
  const msgs = tx.body?.messages;
481
709
  if (msgs) for (let i = 0; i < msgs.length; i++) {
482
- this.log.silly("Indexer broadcasting msg for handling: " + msgs[i].typeUrl);
483
- const msgevents = msgs.length > 1 ? events.find((x) => x.msg_index == i)?.events : events[0].events;
710
+ if (this.log.isSillyEnabled()) this.log.silly("Indexer broadcasting msg for handling: " + msgs[i].typeUrl);
711
+ const msgevents = msgs.length > 1 ? events.find((x) => x.msg_index == i)?.events : events[0]?.events ?? [];
484
712
  await this.asyncEmit(msgs[i].typeUrl, {
485
713
  value: {
486
714
  tx: msgs[i].value,
@@ -492,7 +720,7 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
492
720
  if (msgs[i].typeUrl == "/cosmos.authz.v1beta1.MsgExec") {
493
721
  const authzMsgs = cosmjs_types_cosmos_authz_v1beta1_tx_js.MsgExec.decode(msgs[i].value).msgs;
494
722
  if (authzMsgs) for (let r = 0; r < authzMsgs.length; r++) {
495
- this.log.silly("Indexer broadcasting msg for handling: " + authzMsgs[r].typeUrl);
723
+ if (this.log.isSillyEnabled()) this.log.silly("Indexer broadcasting msg for handling: " + authzMsgs[r].typeUrl);
496
724
  const authzMsgEvents = msgevents?.reduce((events$1, evt) => {
497
725
  if (evt.attributes.filter((x) => require_text.decodeAttr(x.key) == "authz_msg_index" && require_text.decodeAttr(x.value) == "" + r).length > 0) events$1.push(evt);
498
726
  return events$1;
@@ -520,136 +748,166 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
520
748
  endTimer?.();
521
749
  this.prometheus?.updateBlockMetrics(height, this.latestHeight, this.blockQueue.size());
522
750
  }
751
+ /**
752
+ * Fetches every height from nextFetchHeight up to latestHeight, waiting for queue space
753
+ * before each fetch. Serves the initial catch-up and live blocks alike: a new announcement
754
+ * only moves latestHeight and starts this loop if it is not already running, so bursts and
755
+ * skipped announcements are handled by the same code and the queue can never overflow.
756
+ */
523
757
  async fetcher() {
524
- for (let i = this.heightToProcess; i <= this.latestHeight; i++) {
525
- if (this.tryToRecover) {
526
- this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
527
- break;
528
- }
529
- this.log.debug("Fetching: " + i);
530
- try {
531
- if (this.isMinimal(this.blockQueue)) {
532
- const timeoutPromise = new Promise((resolve, reject) => {
533
- setTimeout(reject, require_constants.RPC_TIMEOUT_MS, false);
534
- });
535
- const toIndex = Promise.race([Promise.all([this.blockClient.block(i), this.blockClient.blockResults(i)]), timeoutPromise]).catch((e) => {
536
- this.log.error("Error fetching block: " + i + " : " + e);
537
- this.prometheus?.recordError("rpc");
538
- this.tryToRecover = true;
539
- return Promise.resolve([]);
540
- });
541
- this.blockQueue.enqueue(toIndex);
542
- } else {
543
- const timeoutPromise = new Promise((resolve, reject) => {
544
- setTimeout(reject, require_constants.RPC_TIMEOUT_MS, false);
545
- });
546
- const q = cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsRequest.fromPartial({ pagination: { limit: require_constants.PAGINATION_LIMITS.VALIDATORS } });
547
- const vals = cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsRequest.encode(q).finish();
548
- const toIndex = Promise.race([Promise.all([
549
- this.blockClient.block(i),
550
- this.blockClient.blockResults(i),
551
- this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, i, false)
552
- ]), timeoutPromise]).catch((e) => {
553
- this.log.error("Error fetching block: " + i + " : " + e);
554
- this.tryToRecover = true;
555
- return Promise.resolve([]);
556
- });
557
- this.blockQueue.enqueue(toIndex);
758
+ if (this.fetcherRunning && this.fetcherGeneration === this.runGeneration) return;
759
+ const generation = this.runGeneration;
760
+ const token = ++this.fetcherToken;
761
+ this.fetcherRunning = true;
762
+ this.fetcherGeneration = generation;
763
+ try {
764
+ while (this.nextFetchHeight <= this.latestHeight && (this.config.endHeight === void 0 || this.nextFetchHeight <= this.config.endHeight)) {
765
+ if (this.tryToRecover || !this.started || generation !== this.runGeneration) {
766
+ this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
767
+ break;
558
768
  }
559
- } catch (e) {
560
- this.log.error("Fetching error: " + e);
561
- break;
769
+ const i = this.nextFetchHeight;
770
+ this.log.debug("Fetching: " + i);
771
+ try {
772
+ if (this.isMinimal(this.blockQueue)) {
773
+ const toIndex = require_timeout.withTimeout(Promise.all([this.blockClient.block(i), this.blockClient.blockResults(i)]), require_constants.RPC_TIMEOUT_MS, new require_index$1.RPCError("Timed out fetching block " + i)).catch((e) => {
774
+ this.log.error("Error fetching block " + i, { error: e });
775
+ this.prometheus?.recordError("rpc");
776
+ this.requestRecovery("fetch failed for block " + i, generation);
777
+ return Promise.resolve([]);
778
+ });
779
+ this.blockQueue.enqueue(toIndex);
780
+ } else {
781
+ const toIndex = require_timeout.withTimeout(Promise.all([
782
+ this.blockClient.block(i),
783
+ this.blockClient.blockResults(i),
784
+ this.fetchValidatorSet(i)
785
+ ]), require_constants.RPC_TIMEOUT_MS, new require_index$1.RPCError("Timed out fetching block " + i)).catch((e) => {
786
+ this.log.error("Error fetching block " + i, { error: e });
787
+ this.prometheus?.recordError("rpc");
788
+ this.requestRecovery("fetch failed for block " + i, generation);
789
+ return Promise.resolve([]);
790
+ });
791
+ this.blockQueue.enqueue(toIndex);
792
+ }
793
+ } catch (e) {
794
+ this.log.error("Fetching error", { error: e });
795
+ break;
796
+ }
797
+ this.nextFetchHeight = i + 1;
798
+ await this.blockQueue.continue();
562
799
  }
563
- await this.blockQueue.continue();
564
- if (this.tryToRecover) {
565
- this.log.verbose("Exiting fetcher loop. Attempting to recover indexer");
566
- break;
800
+ if (!this.tryToRecover && this.started && generation === this.runGeneration && !this.blockQueue.synced) {
801
+ this.blockQueue.setSynced();
802
+ this.log.info("Synced to latest height");
567
803
  }
568
- }
569
- if (!this.tryToRecover) {
570
- this.blockQueue.setSynced();
571
- this.log.info("Synced to latest height");
804
+ } finally {
805
+ if (this.fetcherToken === token) this.fetcherRunning = false;
572
806
  }
573
807
  }
808
+ /**
809
+ * Runs an ABCI query. A transport failure (RPC down, timeout, empty reply) requests a recovery.
810
+ * A reply with a non-zero code is the chain answering "no" (pruned height, unknown path, bad
811
+ * key): it is thrown as an RPCError with the code and log, and no recovery is requested for
812
+ * ad-hoc queries, so modules can catch it. Block-pipeline queries reject into the fetcher,
813
+ * which requests recovery itself.
814
+ */
574
815
  async callABCI(path, data, height, adHoc = true) {
816
+ let abciq;
817
+ const endTimer = this.prometheus?.timeRpcCall(path) ?? void 0;
575
818
  try {
576
- const endTimer = this.prometheus?.timeRpcCall(path) ?? void 0;
577
- const abciq = await (adHoc ? this.client : this.blockClient).abciQuery({
819
+ abciq = await (adHoc ? this.client : this.blockClient).abciQuery({
578
820
  path,
579
821
  data,
580
822
  height
581
823
  });
824
+ } catch (e) {
825
+ this.setStatus("FAILED");
826
+ this.prometheus?.recordError("rpc");
827
+ this.requestRecovery("ABCI query failed for " + path);
828
+ throw new require_index$1.RPCError("RPC not responding. Query at: " + path + " (" + e + ")");
829
+ } finally {
582
830
  endTimer?.();
583
- if (abciq) return abciq.value;
584
- else {
585
- this.tryToRecover = true;
586
- this.setStatus("FAILED");
587
- this.prometheus?.recordError("rpc");
588
- throw new Error("RPC not responding. Query at: " + path);
589
- }
590
- } catch (_e) {
591
- this.tryToRecover = true;
831
+ }
832
+ if (!abciq) {
592
833
  this.setStatus("FAILED");
593
- throw new Error("RPC not responding. Query at: " + path);
834
+ this.prometheus?.recordError("rpc");
835
+ this.requestRecovery("empty ABCI response for " + path);
836
+ throw new require_index$1.RPCError("RPC not responding. Query at: " + path);
594
837
  }
838
+ if (abciq.code) {
839
+ this.prometheus?.recordError("rpc");
840
+ throw new require_index$1.RPCError("ABCI query " + path + " failed with code " + abciq.code + (abciq.log ? ": " + abciq.log : ""));
841
+ }
842
+ return abciq.value;
843
+ }
844
+ /**
845
+ * Fetches the complete validator set at a height, following pagination, and returns it
846
+ * re-encoded as a single QueryValidatorsResponse so the block queue payload keeps its shape.
847
+ * Chains with more validators than one page (1000) were silently truncated before.
848
+ */
849
+ async fetchValidatorSet(height) {
850
+ const validators = [];
851
+ let key;
852
+ do {
853
+ const request = cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsRequest.fromPartial({ pagination: key ? {
854
+ limit: require_constants.PAGINATION_LIMITS.VALIDATORS,
855
+ key
856
+ } : { limit: require_constants.PAGINATION_LIMITS.VALIDATORS } });
857
+ const page = cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsResponse.decode(await this.callABCI("/cosmos.staking.v1beta1.Query/Validators", cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsRequest.encode(request).finish(), height, false));
858
+ validators.push(...page.validators);
859
+ key = page.pagination?.nextKey && page.pagination.nextKey.length > 0 ? page.pagination.nextKey : void 0;
860
+ } while (key);
861
+ return cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsResponse.encode(cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsResponse.fromPartial({ validators })).finish();
595
862
  }
596
863
  newBlockReceived(height) {
597
- if (this.blockTimeout) clearTimeout(this.blockTimeout);
598
- this.blockTimeout = setTimeout(() => {
599
- this.tryToRecover = true;
600
- }, 3e4);
864
+ this.armIdleCheck();
601
865
  this.log.info("Received new block: %d", height);
602
- if (height == this.latestHeight) return;
603
- if (this.blockQueue.synced && !this.tryToRecover) {
604
- this.latestHeight = height;
605
- if (this.blockQueue.size() + 1 == this.config.batchSize) {
606
- this.prometheus?.recordError("block");
607
- this.log.error("Block queue is full. Cannot add new block");
608
- this.tryToRecover = true;
609
- return;
610
- }
611
- try {
612
- if (this.isMinimal(this.blockQueue)) this.blockQueue.enqueue(Promise.all([this.client.block(height), this.client.blockResults(height)]).catch((e) => {
613
- this.log.error("Error fetching block: " + height + " : " + e);
614
- return Promise.resolve([]);
615
- }));
616
- else {
617
- const q = cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsRequest.fromPartial({ pagination: { limit: 1000n } });
618
- const vals = cosmjs_types_cosmos_staking_v1beta1_query_js.QueryValidatorsRequest.encode(q).finish();
619
- this.blockQueue.enqueue(Promise.all([
620
- this.client.block(height),
621
- this.client.blockResults(height),
622
- this.callABCI("/cosmos.staking.v1beta1.Query/Validators", vals, height, false)
623
- ]).catch((e) => {
624
- this.log.error("Error fetching block: " + height + " : " + e);
625
- return Promise.resolve([]);
626
- }));
627
- }
628
- } catch (e) {
629
- this.log.error("New Block Received error: " + e);
630
- }
631
- } else this.latestHeight = height;
866
+ if (height <= this.latestHeight) return;
867
+ this.latestHeight = height;
868
+ if (this.tryToRecover || !this.started) return;
869
+ this.fetcher().catch((e) => {
870
+ this.log.error("Error in fetching service", { error: e });
871
+ this.requestRecovery("fetcher failed");
872
+ });
873
+ }
874
+ /** Starts a single polling chain, retiring any chain left over from a previous run */
875
+ startPolling() {
876
+ this.stopPolling();
877
+ const generation = ++this.pollGeneration;
878
+ this.pollForBlock(generation);
632
879
  }
633
- async pollForBlock() {
880
+ stopPolling() {
881
+ this.pollGeneration++;
882
+ if (this.pollTimer) {
883
+ clearTimeout(this.pollTimer);
884
+ this.pollTimer = null;
885
+ }
886
+ }
887
+ async pollForBlock(generation) {
888
+ if (!this.started || generation !== this.pollGeneration) return;
634
889
  try {
635
890
  const status = await this.client.status();
636
- if (status.syncInfo.latestBlockHeight > this.latestHeight) while (this.latestHeight < status.syncInfo.latestBlockHeight) this.newBlockReceived(this.latestHeight + 1);
637
- setTimeout(() => {
638
- this.pollForBlock();
639
- }, this.config.pollingInterval);
891
+ if (!this.started || generation !== this.pollGeneration) return;
892
+ if (status.syncInfo.latestBlockHeight > this.latestHeight) this.newBlockReceived(status.syncInfo.latestBlockHeight);
640
893
  } catch (e) {
641
- this.log.error("Error polling for new block: " + e);
642
- this.tryToRecover = true;
894
+ this.log.error("Error polling for new block", { error: e });
895
+ this.prometheus?.recordError("rpc");
896
+ this.requestRecovery("polling failed");
897
+ return;
643
898
  }
899
+ this.pollTimer = setTimeout(() => {
900
+ this.pollForBlock(generation);
901
+ }, this.config.pollingInterval);
644
902
  }
645
903
  readGenesis() {
646
- if (this.config.genesisPath) return fs.createReadStream(this.config.genesisPath).pipe(stream_json.default.parser());
904
+ if (this.config.genesisPath) return fs.createReadStream(this.config.genesisPath);
647
905
  else throw new Error("Genesis path not set");
648
906
  }
649
907
  async setArrayReader(path, processor) {
650
908
  return new Promise((resolve, reject) => {
651
909
  try {
652
- const pickers = path.split(".").map((filter) => stream_json_filters_Pick_js.default.pick({ filter }));
910
+ const pickers = path.split(".").map((filter) => (0, stream_json_filters_pick_js.default)({ filter }));
653
911
  let counter = 0;
654
912
  let chunkCounter = 0;
655
913
  const chunkProcessor = async (data) => {
@@ -661,48 +919,59 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
661
919
  await this.config.endTransaction(true);
662
920
  await this.config.beginTransaction();
663
921
  }
922
+ return data;
664
923
  };
665
924
  (0, stream_chain.chain)([
666
925
  this.readGenesis(),
926
+ (0, stream_json_parser_js.default)(),
667
927
  ...pickers,
668
- stream_json_streamers_StreamArray_js.default.streamArray(),
669
- stream_json_utils_Batch_js.default.batch({ batchSize: require_constants.GENESIS_BATCH_SIZE }),
928
+ (0, stream_json_streamers_stream_array_js.default)(),
929
+ (0, stream_json_utils_batch_js.default)({ batchSize: require_constants.GENESIS_BATCH_SIZE }),
670
930
  chunkProcessor
671
931
  ]).on("data", (data) => {
672
932
  if (data && Array.isArray(data)) counter = counter + data.length;
673
933
  }).on("end", () => {
674
934
  this.log.info(`Processed ${counter} entries in ${chunkCounter} chunks`);
675
935
  resolve(true);
936
+ }).on("error", (e) => {
937
+ this.log.error("Error reading genesis array " + path, { error: e });
938
+ reject(e);
676
939
  });
677
- } catch (_e) {
678
- this.log.verbose("Error in setArrayReader: " + _e);
679
- reject();
940
+ } catch (e) {
941
+ this.log.verbose("Error in setArrayReader: " + e);
942
+ reject(e);
680
943
  }
681
944
  });
682
945
  }
683
946
  async setValueReader(path, processor) {
684
947
  return new Promise((resolve, reject) => {
685
948
  try {
686
- const pickers = path.split(".").map((filter) => stream_json_filters_Pick_js.default.pick({ filter }));
949
+ const pickers = path.split(".").map((filter) => (0, stream_json_filters_pick_js.default)({ filter }));
687
950
  let counter = 0;
688
951
  (0, stream_chain.chain)([
689
952
  this.readGenesis(),
953
+ (0, stream_json_parser_js.default)(),
690
954
  ...pickers,
691
- stream_json_streamers_StreamValues_js.default.streamValues(),
955
+ (0, stream_json_streamers_stream_values_js.default)(),
692
956
  processor
693
957
  ]).on("data", (_data) => {
694
958
  counter++;
695
959
  }).on("end", () => {
696
960
  this.log.info(`Processed ${counter} entries`);
697
961
  resolve(true);
962
+ }).on("error", (e) => {
963
+ this.log.error("Error reading genesis value " + path, { error: e });
964
+ reject(e);
698
965
  });
699
- } catch (_e) {
700
- reject();
966
+ } catch (e) {
967
+ this.log.verbose("Error in setValueReader: " + e);
968
+ reject(e);
701
969
  }
702
970
  });
703
971
  }
704
972
  async parseGenesis() {
705
973
  this.log.info("Parsing genesis");
974
+ await this.config.onGenesisStart?.();
706
975
  await this.config.beginTransaction();
707
976
  try {
708
977
  this.log.info("Starting genesis import");
@@ -730,6 +999,7 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
730
999
  }
731
1000
  return data;
732
1001
  });
1002
+ await this.config.onGenesisComplete?.();
733
1003
  await this.config.endTransaction(true);
734
1004
  this.log.info("Finished importing");
735
1005
  } catch (e) {
@@ -737,15 +1007,18 @@ var EcleciaIndexer = class extends require_index.EclesiaEmitter {
737
1007
  await this.config.endTransaction(false);
738
1008
  } catch (dbe) {
739
1009
  this.prometheus?.recordError("database");
740
- this.log.error("Error ending transaction. Must be a DB error: " + dbe);
1010
+ this.log.error("Error ending transaction", { error: dbe });
741
1011
  }
742
1012
  this.log.error("Failed to import genesis");
743
1013
  throw e;
744
1014
  }
745
1015
  }
746
1016
  };
1017
+ /** @deprecated Misspelling kept for compatibility, use EclesiaIndexer. Removed in 3.0. */
1018
+ const EcleciaIndexer = EclesiaIndexer;
747
1019
 
748
1020
  //#endregion
749
1021
  exports.EcleciaIndexer = EcleciaIndexer;
1022
+ exports.EclesiaIndexer = EclesiaIndexer;
750
1023
  exports.defaultIndexerConfig = defaultIndexerConfig;
751
1024
  //# sourceMappingURL=index.cjs.map