effective-indexer 0.2.4 → 0.2.6

package/LICENSE

@@ -1,7 +1,12 @@
+Effective Indexer License
 Copyright (c) 2026 Aleksandr Shenshin
-All rights reserved.
 
-This software and associated documentation files are proprietary and confidential.
-No part of this software may be copied, modified, distributed, sublicensed,
-published, sold, or used in any form without prior written permission from
-the copyright holder.
+This project is available under the PolyForm Noncommercial License 1.0.0.
+License text: https://polyformproject.org/licenses/noncommercial/1.0.0/
+
+You may use, modify, and distribute this software only for noncommercial purposes
+as defined by that license.
+
+Commercial use requires a separate paid commercial license agreement.
+For commercial licensing inquiries, contact:
+Aleksandr Shenshin <shenshin@me.com>

package/README.md

@@ -20,6 +20,12 @@ npm install effective-indexer effect
 
 `effect` is a peer dependency.
 
+## License
+
+Free for noncommercial use under PolyForm Noncommercial 1.0.0.
+Commercial use requires a paid commercial license (see `LICENSE`).
+Contact: Aleksandr Shenshin <shenshin@me.com>.
+
 ## 5-minute setup
 
 ### 1) Create `indexer.config.ts`
@@ -112,13 +118,13 @@ await indexer.stop()
 
 ## Public API
 
-- `defineIndexerConfig(config)`  
+- `defineIndexerConfig(config)`
   Identity helper for typed config files (Hardhat-style).
-- `resolveIndexerConfigFromEnv(config, options?)`  
+- `resolveIndexerConfigFromEnv(config, options?)`
   Resolves `{{ENV_VAR}}` placeholders and optional RPC URL override.
-- `runIndexerWorker(config, options?)`  
+- `runIndexerWorker(config, options?)`
   Runs long-lived worker with built-in DB directory creation and graceful shutdown.
-- `Indexer.create(config)`  
+- `Indexer.create(config)`
   Returns handle: `start()`, `stop()`, `query()`, `count()`.
 
 ## Config essentials
@@ -483,11 +489,3 @@ npm run typecheck
 npm run test
 npm run check
 ```
-
-### Live Integration Tests
-
-Integration tests read RPC URLs from `.env` (mainnet) and `.env.test` (testnet) using `EVM_RPC_URL`.
-
-```bash
-npm run test:integration
-```

package/dist/index.cjs

@@ -53,22 +53,26 @@ var resolveTelemetry = (config) => {
     }
   };
 };
-var resolveConfig = (config) => {
+var resolveConfigEffect = (config) => effect.Effect.gen(function* () {
   const network = resolveNetwork(config);
   const telemetry = resolveTelemetry(config);
   const pr = network.logs.parallelRequests;
   if (!Number.isInteger(pr) || pr < 1) {
-    throw new ConfigError({
-      reason: "parallelRequests must be an integer >= 1",
-      field: "network.logs.parallelRequests"
-    });
+    return yield* effect.Effect.fail(
+      new ConfigError({
+        reason: "parallelRequests must be an integer >= 1",
+        field: "network.logs.parallelRequests"
+      })
+    );
   }
   const pi = telemetry.progress.intervalMs;
   if (!Number.isInteger(pi) || !Number.isFinite(pi) || pi < 500) {
-    throw new ConfigError({
-      reason: "telemetry.progress.intervalMs must be an integer >= 500",
-      field: "telemetry.progress.intervalMs"
-    });
+    return yield* effect.Effect.fail(
+      new ConfigError({
+        reason: "telemetry.progress.intervalMs must be an integer >= 500",
+        field: "telemetry.progress.intervalMs"
+      })
+    );
   }
   return {
     rpcUrl: config.rpcUrl,
@@ -80,10 +84,11 @@ var resolveConfig = (config) => {
     logFormat: config.logFormat ?? "pretty",
     enableTelemetry: config.enableTelemetry ?? true
   };
-};
+});
+var resolveConfig = (config) => effect.Effect.runSync(resolveConfigEffect(config));
 var Config = class extends effect.Context.Tag("effective-indexer/Config")() {
 };
-var ConfigLive = (raw) => effect.Layer.succeed(Config, resolveConfig(raw));
+var ConfigLive = (raw) => effect.Layer.effect(Config, resolveConfigEffect(raw));
 var parseLogLevel = (level) => {
   switch (level) {
     case "trace":
@@ -242,6 +247,14 @@ var wrapSqlError = (operation) => (e) => new StorageError({
   cause: e
 });
 var toJsonValue = (_key, value) => typeof value === "bigint" ? value.toString() : value;
+var INSERT_BATCH_SIZE = 250;
+var chunkEvents = (events, size) => {
+  const chunks = [];
+  for (let i = 0; i < events.length; i += size) {
+    chunks.push(events.slice(i, i + size));
+  }
+  return chunks;
+};
 var Storage = class extends effect.Context.Tag("effective-indexer/Storage")() {
 };
 var StorageLive = effect.Layer.effect(
@@ -281,14 +294,30 @@ var StorageLive = effect.Layer.effect(
       yield* effect.Effect.logDebug("Storage schema initialized");
     }).pipe(effect.Effect.mapError(wrapSqlError("initialize")));
     const insertEvents = (events) => effect.Effect.gen(function* () {
-      for (const event of events) {
-        const argsJson = JSON.stringify(event.args, toJsonValue);
-        const blockNum = Number(event.blockNumber);
-        const ts = event.timestamp !== null ? Number(event.timestamp) : null;
-        yield* sql$1`
-            INSERT OR IGNORE INTO events (contract_name, event_name, block_number, tx_hash, log_index, timestamp, args)
-            VALUES (${event.contractName}, ${event.eventName}, ${blockNum}, ${event.txHash}, ${event.logIndex}, ${ts}, ${argsJson})
-          `;
+      if (events.length === 0) {
+        return;
+      }
+      for (const batch of chunkEvents(events, INSERT_BATCH_SIZE)) {
+        const placeholders = batch.map(() => "(?, ?, ?, ?, ?, ?, ?)").join(", ");
+        const params = batch.flatMap((event) => {
+          const argsJson = JSON.stringify(event.args, toJsonValue);
+          const blockNum = Number(event.blockNumber);
+          const ts = event.timestamp !== null ? Number(event.timestamp) : null;
+          return [
+            event.contractName,
+            event.eventName,
+            blockNum,
+            event.txHash,
+            event.logIndex,
+            ts,
+            argsJson
+          ];
+        });
+        yield* sql$1.unsafe(
+          `INSERT OR IGNORE INTO events (contract_name, event_name, block_number, tx_hash, log_index, timestamp, args)
+            VALUES ${placeholders}`,
+          params
+        );
       }
     }).pipe(effect.Effect.mapError(wrapSqlError("insertEvents")));
     const deleteEventsFrom = (blockNumber) => sql$1`DELETE FROM events WHERE block_number >= ${Number(blockNumber)}`.pipe(
@@ -479,8 +508,20 @@ var EventDecoderLive = effect.Layer.succeed(EventDecoder, {
     try: () => decodeLog(contractName, abi, log),
     catch: (e) => new DecodeError({ reason: String(e), log, cause: e })
   }),
-  decodeBatch: (contractName, abi, logs) => effect.Effect.succeed(
-    logs.map((log) => decodeLog(contractName, abi, log)).filter((e) => e !== null)
+  decodeBatch: (contractName, abi, logs) => effect.Effect.forEach(
+    logs,
+    (log) => effect.Effect.gen(function* () {
+      const decoded = decodeLog(contractName, abi, log);
+      if (decoded === null) {
+        return yield* effect.Effect.fail(
+          new DecodeError({
+            reason: "Failed to decode log with provided ABI",
+            log
+          })
+        );
+      }
+      return decoded;
+    })
   )
 });
 var computeSnapshot = (p) => {
@@ -672,22 +713,33 @@ var ProgressRendererLive = effect.Layer.effect(
     };
   })
 );
-var buildTopicFilter = (abi, eventNames) => {
-  const topics = [];
-  for (const name of eventNames) {
+var buildTopicFilterEffect = (abi, eventNames) => effect.Effect.forEach(
+  eventNames,
+  (name) => effect.Effect.gen(function* () {
     const abiEvent = abi.find(
       (item) => item.type === "event" && item.name === name
     );
     if (!abiEvent || abiEvent.type !== "event") {
-      throw new Error(`Event "${name}" not found in ABI`);
+      return yield* effect.Effect.fail(
+        new ConfigError({
+          reason: `Event "${name}" not found in ABI`,
+          field: "contracts.events"
+        })
+      );
     }
     const encoded = viem.encodeEventTopics({ abi: [abiEvent], eventName: name });
-    if (encoded[0]) {
-      topics.push(encoded[0]);
+    const topic = encoded[0];
+    if (topic === void 0) {
+      return yield* effect.Effect.fail(
+        new ConfigError({
+          reason: `Failed to encode topic for event "${name}"`,
+          field: "contracts.events"
+        })
+      );
     }
-  }
-  return topics;
-};
+    return topic;
+  })
+);
 var fetchLogs = (params) => effect.Stream.unwrap(
   effect.Effect.gen(function* () {
     const config = yield* Config;
@@ -847,6 +899,28 @@ var indexContract = (contract) => effect.Stream.unwrap(
     const blockCursor = yield* BlockCursor;
     const progress = yield* ProgressReporter;
     const renderer = yield* ProgressRenderer;
+    const { baseDelayMs, maxDelayMs } = config.network.logs.retry;
+    const maxRetries = config.network.logs.maxRetries;
+    const blockRetrySchedule = effect.Schedule.exponential(
+      effect.Duration.millis(baseDelayMs)
+    ).pipe(
+      effect.Schedule.delayed(
+        (duration) => effect.Duration.millis(Math.min(effect.Duration.toMillis(duration), maxDelayMs))
+      ),
+      effect.Schedule.compose(effect.Schedule.recurs(maxRetries))
+    );
+    const getBlockWithRetry = (blockNumber) => rpc.getBlock(blockNumber).pipe(
+      effect.Effect.tapError(
+        (err) => effect.Effect.logDebug("RPC getBlock failed, retrying").pipe(
+          effect.Effect.annotateLogs({
+            block: blockNumber.toString(),
+            reason: err.reason,
+            method: "eth_getBlockByNumber"
+          })
+        )
+      ),
+      effect.Effect.retry(blockRetrySchedule)
+    );
     const startBlock = yield* checkpoint.getStartBlock(
       contract.name,
       contract.startBlock ?? 0n
@@ -858,7 +932,10 @@ var indexContract = (contract) => effect.Stream.unwrap(
         toBlock: currentHead.toString()
       })
     );
-    const topics = buildTopicFilter(contract.abi, contract.events);
+    const topics = yield* buildTopicFilterEffect(
+      contract.abi,
+      contract.events
+    );
     const needsBackfill = startBlock <= currentHead;
     const totalBackfillBlocks = currentHead - startBlock + 1n;
     if (needsBackfill) {
@@ -884,7 +961,7 @@ var indexContract = (contract) => effect.Stream.unwrap(
           ];
           const withTimestamp = [];
           for (const bn of blockNumbers) {
-            const blockInfo = yield* rpc.getBlock(bn);
+            const blockInfo = yield* getBlockWithRetry(bn);
             yield* reorgDetector.verifyBlock(blockInfo);
             for (const event of decoded) {
               if (event.blockNumber === bn) {
@@ -943,7 +1020,7 @@ var indexContract = (contract) => effect.Stream.unwrap(
     const liveStream = blockCursor.liveBlocks.pipe(
       effect.Stream.mapEffect(
         (blockNumber) => effect.Effect.gen(function* () {
-          const blockInfo = yield* rpc.getBlock(blockNumber);
+          const blockInfo = yield* getBlockWithRetry(blockNumber);
           const reorgResult = yield* effect.Effect.either(
             reorgDetector.verifyBlock(blockInfo)
           );
@@ -1062,6 +1139,14 @@ var parseStoredEvent = (row) => ({
   timestamp: row.timestamp,
   args: JSON.parse(row.args)
 });
+var parseStoredEventEffect = (row) => effect.Effect.try({
+  try: () => parseStoredEvent(row),
+  catch: (e) => new StorageError({
+    reason: String(e),
+    operation: "parseStoredEvent",
+    cause: e
+  })
+});
 var QueryApi = class extends effect.Context.Tag("effective-indexer/QueryApi")() {
 };
 var QueryApiLive = effect.Layer.effect(
@@ -1073,7 +1158,7 @@ var QueryApiLive = effect.Layer.effect(
       (_, v) => typeof v === "bigint" ? v.toString() : v
     );
     const getEvents = (query) => storage.queryEvents(query ?? {}).pipe(
-      effect.Effect.map((rows) => rows.map(parseStoredEvent)),
+      effect.Effect.flatMap((rows) => effect.Effect.forEach(rows, parseStoredEventEffect)),
       effect.Effect.timed,
       effect.Effect.tap(
         ([duration, results]) => effect.Effect.logDebug("Query executed").pipe(
@@ -1165,6 +1250,21 @@ var resolveIndexerConfigFromEnv = (config, options) => {
 };
 
 // src/index.ts
+var createWebhookNotifier = (webhookUrl, init) => async (notification) => {
+  const response = await fetch(webhookUrl, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      ...init?.headers ?? {}
+    },
+    body: JSON.stringify(notification)
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Notification webhook failed with status ${response.status}`
+    );
+  }
+};
 var buildLayers = (config) => {
   const resolved = resolveConfig(config);
   const ConfigLayer = ConfigLive(config);
@@ -1209,6 +1309,8 @@ var createIndexer = (config) => {
   let abortController = null;
   const runtime = effect.ManagedRuntime.make(ServicesLive);
   let runningPromise = null;
+  let stopPromise = null;
+  let disposed = false;
   return {
     start: async () => {
       if (runningPromise !== null) {
@@ -1226,21 +1328,41 @@ var createIndexer = (config) => {
       });
     },
     stop: async () => {
-      abortController?.abort();
-      const wasAborted = abortController?.signal.aborted ?? false;
-      if (runningPromise !== null) {
-        try {
-          await runningPromise;
-        } catch (error) {
-          if (!wasAborted) {
-            throw error;
+      if (disposed) {
+        return;
+      }
+      if (stopPromise !== null) {
+        return stopPromise;
+      }
+      stopPromise = (async () => {
+        abortController?.abort();
+        const wasAborted = abortController?.signal.aborted ?? false;
+        if (runningPromise !== null) {
+          try {
+            await runningPromise;
+          } catch (error) {
+            if (!wasAborted) {
+              throw error;
+            }
+          } finally {
+            runningPromise = null;
           }
-        } finally {
-          runningPromise = null;
         }
+        await runtime.dispose();
+        abortController = null;
+        disposed = true;
+      })();
+      try {
+        await stopPromise;
+      } finally {
+        stopPromise = null;
       }
-      await runtime.dispose();
-      abortController = null;
+    },
+    waitForExit: async () => {
+      if (runningPromise === null) {
+        return;
+      }
+      await runningPromise;
     },
     query: async (q) => {
       return runtime.runPromise(
@@ -1261,9 +1383,7 @@ var createIndexer = (config) => {
   };
 };
 var defaultWorkerRuntime = {
-  process,
-  setInterval,
-  clearInterval
+  process
 };
 var resolveDbPath = (config) => config.dbPath ?? "./indexer.db";
 var ensureDbDirectory = async (config) => {
@@ -1280,46 +1400,92 @@ var runIndexerWorker = async (config, options) => {
   const runtime = options?.runtime ?? defaultWorkerRuntime;
   const create = options?.createIndexer ?? createIndexer;
   const signals = options?.shutdownSignals ?? ["SIGINT", "SIGTERM"];
-  const keepAliveIntervalMs = options?.keepAliveIntervalMs ?? 6e4;
-  const indexer = create(config);
-  await indexer.start();
-  await new Promise((resolve, reject) => {
-    const keepAliveTimer = runtime.setInterval(
-      () => void 0,
-      keepAliveIntervalMs
-    );
-    let stopping = false;
-    const signalHandlers = /* @__PURE__ */ new Map();
-    const cleanup = () => {
-      runtime.clearInterval(keepAliveTimer);
-      for (const signal of signals) {
-        const handler = signalHandlers.get(signal);
-        if (handler !== void 0) {
-          runtime.process.off(signal, handler);
-        }
-      }
-    };
-    const handleStop = async () => {
-      if (stopping) {
-        return;
-      }
-      stopping = true;
-      cleanup();
-      try {
-        await indexer.stop();
-        resolve();
-      } catch (error) {
-        reject(error);
-      }
-    };
+  const webhookUrl = config.worker?.alert?.webhookUrl;
+  const configNotifier = webhookUrl && webhookUrl.length > 0 ? createWebhookNotifier(webhookUrl) : null;
+  const onRecoveryFailure = options?.onRecoveryFailure ?? configNotifier ?? void 0;
+  const recovery = {
+    enabled: options?.recovery?.enabled ?? config.worker?.recovery?.enabled ?? true,
+    maxRecoveryDurationMs: options?.recovery?.maxRecoveryDurationMs ?? config.worker?.recovery?.maxRecoveryDurationMs ?? 15 * 60 * 1e3,
+    initialRetryDelayMs: options?.recovery?.initialRetryDelayMs ?? config.worker?.recovery?.initialRetryDelayMs ?? 1e3,
+    maxRetryDelayMs: options?.recovery?.maxRetryDelayMs ?? config.worker?.recovery?.maxRetryDelayMs ?? 3e4,
+    backoffFactor: options?.recovery?.backoffFactor ?? config.worker?.recovery?.backoffFactor ?? 2
+  };
+  const signalHandlers = /* @__PURE__ */ new Map();
+  let stopRequested = false;
+  let activeIndexer = null;
+  const stopSignalPromise = new Promise((resolve) => {
     for (const signal of signals) {
       const handler = () => {
-        void handleStop();
+        stopRequested = true;
+        resolve();
       };
       signalHandlers.set(signal, handler);
       runtime.process.on(signal, handler);
     }
   });
+  const cleanup = () => {
+    for (const signal of signals) {
+      const handler = signalHandlers.get(signal);
+      if (handler !== void 0) {
+        runtime.process.off(signal, handler);
+      }
+    }
+  };
+  const stopActiveIndexer = async () => {
+    if (activeIndexer !== null) {
+      try {
+        await activeIndexer.stop();
+      } finally {
+        activeIndexer = null;
+      }
+    }
+  };
+  let firstFailureAt = null;
+  let attempts = 0;
+  try {
+    while (!stopRequested) {
+      activeIndexer = create(config);
+      try {
+        await activeIndexer.start();
+        await Promise.race([stopSignalPromise, activeIndexer.waitForExit()]);
+        if (stopRequested) {
+          break;
+        }
+        throw new Error("Indexer worker exited unexpectedly");
+      } catch (error) {
+        if (stopRequested) {
+          break;
+        }
+        attempts += 1;
+        firstFailureAt = firstFailureAt ?? Date.now();
+        const recoveryDurationMs = Date.now() - firstFailureAt;
+        await stopActiveIndexer();
+        if (!recovery.enabled || recoveryDurationMs >= recovery.maxRecoveryDurationMs) {
+          if (onRecoveryFailure) {
+            await onRecoveryFailure({
+              attempts,
+              recoveryDurationMs,
+              error,
+              timestamp: (/* @__PURE__ */ new Date()).toISOString()
+            });
+          }
+          throw error;
+        }
+        const delay = Math.min(
+          recovery.initialRetryDelayMs * recovery.backoffFactor ** Math.max(attempts - 1, 0),
+          recovery.maxRetryDelayMs
+        );
+        console.error(
+          `Indexer worker crashed, retrying in ${delay}ms (attempt ${attempts})`,
+          error
+        );
+        await effect.Effect.runPromise(effect.Effect.sleep(effect.Duration.millis(delay)));
+      }
+    }
+  } finally {
+    await stopActiveIndexer();
+    cleanup();
+  }
 };
 var Indexer = {
   create: createIndexer
@@ -1349,8 +1515,10 @@ exports.Storage = Storage;
 exports.StorageLive = StorageLive;
 exports.computeSnapshot = computeSnapshot;
 exports.createIndexer = createIndexer;
+exports.createWebhookNotifier = createWebhookNotifier;
 exports.defineIndexerConfig = defineIndexerConfig;
 exports.resolveConfig = resolveConfig;
+exports.resolveConfigEffect = resolveConfigEffect;
 exports.resolveIndexerConfigFromEnv = resolveIndexerConfigFromEnv;
 exports.runIndexerWorker = runIndexerWorker;
 //# sourceMappingURL=index.cjs.map