effective-indexer 0.2.8 → 0.2.10

package/dist/index.cjs

@@ -117,13 +117,16 @@ var pickLoggerLayer = (format) => {
       return effect.Logger.pretty;
   }
 };
-var LoggerLive = (config) => {
-  const level = config.enableTelemetry ? parseLogLevel(config.logLevel) : effect.LogLevel.Error;
-  return effect.Layer.merge(
-    pickLoggerLayer(config.logFormat),
-    effect.Logger.minimumLogLevel(level)
-  );
-};
+var LoggerLive = effect.Layer.unwrapEffect(
+  effect.Effect.gen(function* () {
+    const config = yield* Config;
+    const level = config.enableTelemetry ? parseLogLevel(config.logLevel) : effect.LogLevel.Error;
+    return effect.Layer.merge(
+      pickLoggerLayer(config.logFormat),
+      effect.Logger.minimumLogLevel(level)
+    );
+  })
+);
 var toHexQuantity = (value) => `0x${value.toString(16)}`;
 var RpcProvider = class extends effect.Context.Tag("effective-indexer/RpcProvider")() {
 };
@@ -179,12 +182,22 @@ var RpcProviderLive = effect.Layer.effect(
         cause: e
       })
     }).pipe(
-      effect.Effect.map((block) => ({
-        number: block.number,
-        hash: block.hash,
-        parentHash: block.parentHash,
-        timestamp: block.timestamp
-      }))
+      effect.Effect.flatMap((block) => {
+        if (!block) {
+          return effect.Effect.fail(
+            new RpcError({
+              reason: `Block ${blockNumber} not found`,
+              method: "eth_getBlockByNumber"
+            })
+          );
+        }
+        return effect.Effect.succeed({
+          number: block.number,
+          hash: block.hash,
+          parentHash: block.parentHash,
+          timestamp: block.timestamp
+        });
+      })
     );
     return { getBlockNumber, getLogs, getBlock };
   })
@@ -203,7 +216,7 @@ var BlockCursorLive = effect.Layer.effect(
     const pollOnce = effect.Effect.gen(function* () {
       const current = yield* rpc.getBlockNumber;
       const confirmations = BigInt(config.network.polling.confirmations);
-      const confirmed = current - confirmations;
+      const confirmed = current > confirmations ? current - confirmations : 0n;
       const prev = yield* effect.Ref.get(lastSeen);
       const isInitialized = yield* effect.Ref.get(initialized);
       if (!isInitialized) {
@@ -220,10 +233,11 @@ var BlockCursorLive = effect.Layer.effect(
         yield* effect.Effect.logTrace("No new confirmed blocks");
         return [];
       }
-      const blocks = [];
-      for (let block = prev + 1n; block <= confirmed; block += 1n) {
-        blocks.push(block);
-      }
+      const count = Number(confirmed - prev);
+      const blocks = Array.from(
+        { length: count },
+        (_, i) => prev + 1n + BigInt(i)
+      );
       yield* effect.Ref.set(lastSeen, confirmed);
       yield* effect.Effect.logTrace("Blocks emitted").pipe(
         effect.Effect.annotateLogs({
@@ -246,15 +260,31 @@ var wrapSqlError = (operation) => (e) => new StorageError({
   operation,
   cause: e
 });
+var buildWhereClause = (query) => {
+  const pairs = [
+    [query.contractName, "contract_name = ?"],
+    [query.eventName, "event_name = ?"],
+    [
+      query.fromBlock !== void 0 ? Number(query.fromBlock) : void 0,
+      "block_number >= ?"
+    ],
+    [
+      query.toBlock !== void 0 ? Number(query.toBlock) : void 0,
+      "block_number <= ?"
+    ],
+    [query.txHash, "tx_hash = ?"]
+  ];
+  const active = pairs.filter(([v]) => v !== void 0);
+  const where = active.length > 0 ? `WHERE ${active.map(([, c]) => c).join(" AND ")}` : "";
+  const params = active.map(([v]) => v);
+  return { where, params };
+};
 var toJsonValue = (_key, value) => typeof value === "bigint" ? value.toString() : value;
 var INSERT_BATCH_SIZE = 250;
-var chunkEvents = (events, size) => {
-  const chunks = [];
-  for (let i = 0; i < events.length; i += size) {
-    chunks.push(events.slice(i, i + size));
-  }
-  return chunks;
-};
+var chunkEvents = (events, size) => Array.from(
+  { length: Math.ceil(events.length / size) },
+  (_, i) => events.slice(i * size, (i + 1) * size)
+);
 var Storage = class extends effect.Context.Tag("effective-indexer/Storage")() {
 };
 var StorageLive = effect.Layer.effect(
@@ -262,6 +292,7 @@ var StorageLive = effect.Layer.effect(
   effect.Effect.gen(function* () {
     const sql$1 = yield* sql.SqlClient.SqlClient;
     const initialize = effect.Effect.gen(function* () {
+      yield* sql$1`PRAGMA journal_mode=WAL`;
       yield* sql$1`
         CREATE TABLE IF NOT EXISTS events (
           id            INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -297,7 +328,7 @@ var StorageLive = effect.Layer.effect(
       if (events.length === 0) {
         return;
       }
-      for (const batch of chunkEvents(events, INSERT_BATCH_SIZE)) {
+      yield* effect.Effect.forEach(chunkEvents(events, INSERT_BATCH_SIZE), (batch) => {
         const placeholders = batch.map(() => "(?, ?, ?, ?, ?, ?, ?)").join(", ");
         const params = batch.flatMap((event) => {
           const argsJson = JSON.stringify(event.args, toJsonValue);
@@ -313,44 +344,22 @@ var StorageLive = effect.Layer.effect(
             argsJson
           ];
         });
-        yield* sql$1.unsafe(
+        return sql$1.unsafe(
           `INSERT OR IGNORE INTO events (contract_name, event_name, block_number, tx_hash, log_index, timestamp, args)
             VALUES ${placeholders}`,
           params
         );
-      }
+      });
     }).pipe(effect.Effect.mapError(wrapSqlError("insertEvents")));
     const deleteEventsFrom = (blockNumber) => sql$1`DELETE FROM events WHERE block_number >= ${Number(blockNumber)}`.pipe(
       effect.Effect.asVoid,
       effect.Effect.mapError(wrapSqlError("deleteEventsFrom"))
     );
     const queryEvents = (query) => effect.Effect.gen(function* () {
-      const conditions = [];
-      const params = [];
-      if (query.contractName !== void 0) {
-        conditions.push("contract_name = ?");
-        params.push(query.contractName);
-      }
-      if (query.eventName !== void 0) {
-        conditions.push("event_name = ?");
-        params.push(query.eventName);
-      }
-      if (query.fromBlock !== void 0) {
-        conditions.push("block_number >= ?");
-        params.push(Number(query.fromBlock));
-      }
-      if (query.toBlock !== void 0) {
-        conditions.push("block_number <= ?");
-        params.push(Number(query.toBlock));
-      }
-      if (query.txHash !== void 0) {
-        conditions.push("tx_hash = ?");
-        params.push(query.txHash);
-      }
-      const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+      const { where, params } = buildWhereClause(query);
       const order = query.order === "desc" ? "DESC" : "ASC";
-      const limit = query.limit ?? 1e3;
-      const offset = query.offset ?? 0;
+      const limit = Math.max(1, Math.min(query.limit ?? 1e3, 1e4));
+      const offset = Math.max(0, query.offset ?? 0);
       const rows = yield* sql$1.unsafe(
         `SELECT * FROM events ${where} ORDER BY block_number ${order}, log_index ASC LIMIT ? OFFSET ?`,
         [...params, limit, offset]
@@ -358,36 +367,10 @@ var StorageLive = effect.Layer.effect(
       return rows;
     }).pipe(effect.Effect.mapError(wrapSqlError("queryEvents")));
     const countEvents = (query) => effect.Effect.gen(function* () {
-      if (!query || !query.contractName && !query.eventName && !query.fromBlock && !query.toBlock && !query.txHash) {
-        const rows2 = yield* sql$1`SELECT COUNT(*) as count FROM events`;
-        return rows2[0]?.count ?? 0;
-      }
-      const conditions = [];
-      const params = [];
-      if (query.contractName !== void 0) {
-        conditions.push("contract_name = ?");
-        params.push(query.contractName);
-      }
-      if (query.eventName !== void 0) {
-        conditions.push("event_name = ?");
-        params.push(query.eventName);
-      }
-      if (query.fromBlock !== void 0) {
-        conditions.push("block_number >= ?");
-        params.push(Number(query.fromBlock));
-      }
-      if (query.toBlock !== void 0) {
-        conditions.push("block_number <= ?");
-        params.push(Number(query.toBlock));
-      }
-      if (query.txHash !== void 0) {
-        conditions.push("tx_hash = ?");
-        params.push(query.txHash);
-      }
-      const where = `WHERE ${conditions.join(" AND ")}`;
-      const rows = yield* sql$1.unsafe(
+      const { where, params } = buildWhereClause(query ?? {});
+      const rows = where === "" ? yield* sql$1`SELECT COUNT(*) as count FROM events` : yield* sql$1.unsafe(
         `SELECT COUNT(*) as count FROM events ${where}`,
-        params
+        [...params]
       );
       return rows[0]?.count ?? 0;
     }).pipe(effect.Effect.mapError(wrapSqlError("countEvents")));
@@ -672,9 +655,7 @@ var ProgressRendererLive = effect.Layer.effect(
         });
         logUpdate(lines.join("\n"));
       } else {
-        for (const s of snapshots) {
-          yield* effect.Effect.log(buildLine(s, config));
-        }
+        yield* effect.Effect.forEach(snapshots, (s) => effect.Effect.log(buildLine(s, config)));
       }
     });
     return {
@@ -745,24 +726,32 @@ var fetchLogs = (params) => effect.Stream.unwrap(
     const config = yield* Config;
     const rpc = yield* RpcProvider;
     const chunkSize = BigInt(config.network.logs.chunkSize);
-    const chunks = [];
-    let cursor = params.fromBlock;
-    while (cursor <= params.toBlock) {
-      const end = cursor + chunkSize - 1n > params.toBlock ? params.toBlock : cursor + chunkSize - 1n;
-      chunks.push({ from: cursor, to: end });
-      cursor = end + 1n;
-    }
+    const { baseDelayMs, maxDelayMs } = config.network.logs.retry;
+    const maxRetries = config.network.logs.maxRetries;
+    const concurrency = config.network.logs.parallelRequests;
+    const retrySchedule = effect.Schedule.exponential(
+      effect.Duration.millis(baseDelayMs)
+    ).pipe(
+      effect.Schedule.delayed(
+        (d) => effect.Duration.millis(Math.min(effect.Duration.toMillis(d), maxDelayMs))
+      ),
+      effect.Schedule.compose(effect.Schedule.recurs(maxRetries))
+    );
+    const span = params.toBlock - params.fromBlock + 1n;
+    const numChunks = span > 0n ? Number((span + chunkSize - 1n) / chunkSize) : 0;
+    const chunks = Array.from({ length: numChunks }, (_, i) => {
+      const from = params.fromBlock + BigInt(i) * chunkSize;
+      const to = from + chunkSize - 1n > params.toBlock ? params.toBlock : from + chunkSize - 1n;
+      return { from, to };
+    });
     if (chunks.length === 0) {
       return effect.Stream.empty;
     }
-    const concurrency = config.network.logs.parallelRequests;
     return effect.Stream.fromIterable(chunks).pipe(
       effect.Stream.mapEffect(
         (chunk) => effect.Effect.gen(function* () {
-          const { baseDelayMs, maxDelayMs } = config.network.logs.retry;
-          const maxRetries = config.network.logs.maxRetries;
           const attempt = yield* effect.Ref.make(0);
-          return yield* rpc.getLogs({
+          const logs = yield* rpc.getLogs({
             address: params.address,
             topics: [params.topics],
             fromBlock: chunk.from,
@@ -789,26 +778,18 @@ var fetchLogs = (params) => effect.Stream.unwrap(
                 );
               })
             ),
-            effect.Effect.retry(
-              effect.Schedule.exponential(effect.Duration.millis(baseDelayMs)).pipe(
-                effect.Schedule.delayed(
-                  (d) => effect.Duration.millis(
-                    Math.min(effect.Duration.toMillis(d), maxDelayMs)
-                  )
-                ),
-                effect.Schedule.compose(effect.Schedule.recurs(maxRetries))
-              )
-            ),
+            effect.Effect.retry(retrySchedule),
             effect.Effect.tap(
-              (logs) => effect.Effect.logTrace("Logs fetched").pipe(
+              (result) => effect.Effect.logTrace("Logs fetched").pipe(
                 effect.Effect.annotateLogs({
                   from: chunk.from.toString(),
                   to: chunk.to.toString(),
-                  count: logs.length.toString()
+                  count: result.length.toString()
                 })
               )
             )
           );
+          return { logs, chunkEnd: chunk.to };
         }),
         { concurrency }
       )
@@ -859,13 +840,11 @@ var ReorgDetectorLive = effect.Layer.effect(
         }
       }
       yield* effect.Ref.update(blockHashBuffer, (buf) => {
-        const newBuf = new Map(buf);
-        newBuf.set(block.number, block.hash);
         const minBlock = block.number - BigInt(reorgDepth);
-        for (const key of newBuf.keys()) {
-          if (key < minBlock) newBuf.delete(key);
-        }
-        return newBuf;
+        const entries = Array.from(buf.entries()).filter(
+          ([k]) => k >= minBlock
+        );
+        return new Map([...entries, [block.number, block.hash]]);
       });
       yield* storage.insertBlockHash(block.number, block.hash);
     });
@@ -949,48 +928,48 @@ var indexContract = (contract) => effect.Stream.unwrap(
       toBlock: currentHead
     }).pipe(
       effect.Stream.mapEffect(
-        (rawLogs) => effect.Effect.gen(function* () {
-          if (rawLogs.length === 0) return [];
-          const decoded = yield* decoder.decodeBatch(
-            contract.name,
-            contract.abi,
-            rawLogs
-          );
-          const blockNumbers = [
-            ...new Set(decoded.map((d) => d.blockNumber))
-          ];
-          const withTimestamp = [];
-          for (const bn of blockNumbers) {
-            const blockInfo = yield* getBlockWithRetry(bn);
-            yield* reorgDetector.verifyBlock(blockInfo);
-            for (const event of decoded) {
-              if (event.blockNumber === bn) {
-                withTimestamp.push({
-                  ...event,
-                  timestamp: blockInfo.timestamp
-                });
-              }
-            }
-          }
-          yield* storage.insertEvents(
-            withTimestamp.map((e) => ({
-              contractName: e.contractName,
-              eventName: e.eventName,
-              blockNumber: e.blockNumber,
-              txHash: e.txHash,
-              logIndex: e.logIndex,
-              timestamp: e.timestamp,
-              args: e.args
-            }))
-          );
-          const lastBlock = withTimestamp[withTimestamp.length - 1];
-          if (lastBlock) {
-            yield* checkpoint.save(
+        (chunk) => effect.Effect.gen(function* () {
+          const { logs: rawLogs, chunkEnd } = chunk;
+          const blockCache = /* @__PURE__ */ new Map();
+          const withTimestamp = rawLogs.length > 0 ? yield* effect.Effect.gen(function* () {
+            const decoded = yield* decoder.decodeBatch(
               contract.name,
-              lastBlock.blockNumber,
-              lastBlock.blockHash
+              contract.abi,
+              rawLogs
             );
-          }
+            const blockNumbers = [
+              ...new Set(decoded.map((d) => d.blockNumber))
+            ];
+            const enriched = yield* effect.Effect.forEach(
+              blockNumbers,
+              (bn) => effect.Effect.gen(function* () {
+                const blockInfo = yield* getBlockWithRetry(bn);
+                blockCache.set(bn, blockInfo);
+                yield* reorgDetector.verifyBlock(blockInfo);
+                return decoded.filter((e) => e.blockNumber === bn).map((e) => ({
+                  ...e,
+                  timestamp: blockInfo.timestamp
+                }));
+              }),
+              { concurrency: 1 }
+            );
+            const events = enriched.flat();
+            yield* storage.insertEvents(
+              events.map((e) => ({
+                contractName: e.contractName,
+                eventName: e.eventName,
+                blockNumber: e.blockNumber,
+                txHash: e.txHash,
+                logIndex: e.logIndex,
+                timestamp: e.timestamp,
+                args: e.args
+              }))
+            );
+            return events;
+          }) : [];
+          const cached = blockCache.get(chunkEnd);
+          const chunkEndHash = cached ? cached.hash : (yield* getBlockWithRetry(chunkEnd)).hash;
+          yield* checkpoint.save(contract.name, chunkEnd, chunkEndHash);
           const chunkSize = BigInt(config.network.logs.chunkSize);
           const lastProcessed = yield* effect.Ref.modify(
             processedBlocksRef,
@@ -1009,7 +988,7 @@ var indexContract = (contract) => effect.Stream.unwrap(
           yield* effect.Effect.logDebug("Chunk indexed").pipe(
             effect.Effect.annotateLogs({
               events: withTimestamp.length.toString(),
-              checkpoint: lastBlock ? lastBlock.blockNumber.toString() : "none"
+              checkpoint: chunkEnd.toString()
             })
           );
           return withTimestamp;
@@ -1190,15 +1169,11 @@ var QueryApiLive = effect.Layer.effect(
     return { getEvents, getEventCount, getLatestBlock };
   })
 );
-var toConfigMap = (env) => {
-  const map = /* @__PURE__ */ new Map();
-  for (const [key, value] of Object.entries(env)) {
-    if (typeof value === "string") {
-      map.set(key, value);
-    }
-  }
-  return map;
-};
+var toConfigMap = (env) => new Map(
+  Object.entries(env).filter(
+    (entry) => typeof entry[1] === "string"
+  )
+);
 var getProvider = (env) => effect.ConfigProvider.fromMap(toConfigMap(env ?? process.env));
 var readOptionalString = (name, provider) => effect.Effect.runSync(
   effect.Effect.withConfigProvider(provider)(
@@ -1266,12 +1241,10 @@ var createWebhookNotifier = (webhookUrl, init) => async (notification) => {
   }
 };
 var buildLayers = (config) => {
-  const resolved = resolveConfig(config);
   const ConfigLayer = ConfigLive(config);
   const SqliteLayer = sqlSqliteNode.SqliteClient.layer({
     filename: config.dbPath ?? "./indexer.db"
   });
-  const LoggerLayer = LoggerLive(resolved);
   const FoundationLayer = effect.Layer.merge(ConfigLayer, SqliteLayer);
   const StorageLayer = StorageLive.pipe(effect.Layer.provide(FoundationLayer));
   const RpcLayer = RpcProviderLive.pipe(effect.Layer.provide(ConfigLayer));
@@ -1290,6 +1263,7 @@ var buildLayers = (config) => {
   const ProgressRendererLayer = ProgressRendererLive.pipe(
     effect.Layer.provide(effect.Layer.merge(ConfigLayer, ProgressReporterLayer))
   );
+  const LoggerLayer = LoggerLive.pipe(effect.Layer.provide(ConfigLayer));
   return effect.Layer.mergeAll(
     ConfigLayer,
     StorageLayer,
@@ -1410,27 +1384,22 @@ var runIndexerWorker = async (config, options) => {
     maxRetryDelayMs: options?.recovery?.maxRetryDelayMs ?? config.worker?.recovery?.maxRetryDelayMs ?? 3e4,
     backoffFactor: options?.recovery?.backoffFactor ?? config.worker?.recovery?.backoffFactor ?? 2
   };
-  const signalHandlers = /* @__PURE__ */ new Map();
   let stopRequested = false;
   let activeIndexer = null;
-  const stopSignalPromise = new Promise((resolve) => {
-    for (const signal of signals) {
-      const handler = () => {
-        stopRequested = true;
-        resolve();
-      };
-      signalHandlers.set(signal, handler);
-      runtime.process.on(signal, handler);
-    }
+  let resolveStopSignal;
+  const stopSignalPromise = new Promise((r) => {
+    resolveStopSignal = r;
   });
-  const cleanup = () => {
-    for (const signal of signals) {
-      const handler = signalHandlers.get(signal);
-      if (handler !== void 0) {
-        runtime.process.off(signal, handler);
-      }
-    }
+  const requestStop = () => {
+    stopRequested = true;
+    resolveStopSignal?.();
   };
+  signals.forEach((signal) => {
+    runtime.process.on(signal, requestStop);
+  });
+  const cleanup = () => signals.forEach((signal) => {
+    runtime.process.off(signal, requestStop);
+  });
   const stopActiveIndexer = async () => {
     if (activeIndexer !== null) {
       try {