effective-indexer 0.2.2 → 0.2.4

package/LICENSE

@@ -1,21 +1,7 @@
-MIT License
-
 Copyright (c) 2026 Aleksandr Shenshin
+All rights reserved.
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+This software and associated documentation files are proprietary and confidential.
+No part of this software may be copied, modified, distributed, sublicensed,
+published, sold, or used in any form without prior written permission from
+the copyright holder.

package/README.md

@@ -1,7 +1,160 @@
 # Effective Indexer
 
+EVM event indexing without hosted lock-in.
+
+`effective-indexer` runs as your own worker, writes directly to your SQLite database, and gives you a typed query API.
+
+## Why this approach
+
+- **Own your data**: events are stored in your DB, not in a third-party service.
+- **Simple operations**: one worker process, one config file, no subgraph deployment pipeline.
+- **Production-safe behavior**: checkpoint resume, reorg detection, retry/backoff, live polling.
+- **Fast backfill**: parallel `eth_getLogs` with deterministic chunk ordering.
+- **Typed DX**: TypeScript-first config and query surface.
+
+## Install
+
+```bash
+npm install effective-indexer effect
+```
+
+`effect` is a peer dependency.
+
+## 5-minute setup
+
+### 1) Create `indexer.config.ts`
+
+```ts
+import { defineIndexerConfig } from "effective-indexer"
+import type { Abi } from "viem"
+
+const transferAbi: Abi = [
+	{
+		type: "event",
+		name: "Transfer",
+		inputs: [
+			{ indexed: true, name: "from", type: "address" },
+			{ indexed: true, name: "to", type: "address" },
+			{ indexed: false, name: "value", type: "uint256" },
+		],
+	},
+]
+
+export default defineIndexerConfig({
+	rpcUrl: "https://rpc.mainnet.rootstock.io/{{EVM_RPC_API_KEY}}",
+	dbPath: "./data/events.db",
+	contracts: [
+		{
+			name: "Token",
+			address: "0xYourContractAddress",
+			abi: transferAbi,
+			events: ["Transfer"],
+			startBlock: 0n,
+		},
+	],
+	network: {
+		logs: {
+			chunkSize: 2000,
+			parallelRequests: 3,
+		},
+	},
+})
+```
+
+### 2) Create `scripts/indexer.ts`
+
+```ts
+import config from "../indexer.config"
+import { resolveIndexerConfigFromEnv, runIndexerWorker } from "effective-indexer"
+
+const resolvedConfig = resolveIndexerConfigFromEnv(config)
+
+runIndexerWorker(resolvedConfig).catch(error => {
+	console.error("Indexer worker failed:", error)
+	process.exit(1)
+})
+```
+
+### 3) Add env and run
+
+`.env`:
+
+```bash
+EVM_RPC_API_KEY=your-rpc-api-key
+# Optional full URL override:
+# EVM_RPC_URL=https://rpc.mainnet.rootstock.io/<API_KEY>
+```
+
+Run:
+
+```bash
+node --import tsx ./scripts/indexer.ts
+```
+
+## Query data
+
+```ts
+import config from "../indexer.config"
+import { Indexer, resolveIndexerConfigFromEnv } from "effective-indexer"
+
+const indexer = Indexer.create(resolveIndexerConfigFromEnv(config))
+
+const events = await indexer.query({
+	contractName: "Token",
+	eventName: "Transfer",
+	order: "desc",
+	limit: 50,
+})
+
+console.log(events.length)
+await indexer.stop()
+```
+
+## Public API
+
+- `defineIndexerConfig(config)`  
+  Identity helper for typed config files (Hardhat-style).
+- `resolveIndexerConfigFromEnv(config, options?)`  
+  Resolves `{{ENV_VAR}}` placeholders and optional RPC URL override.
+- `runIndexerWorker(config, options?)`  
+  Runs long-lived worker with built-in DB directory creation and graceful shutdown.
+- `Indexer.create(config)`  
+  Returns handle: `start()`, `stop()`, `query()`, `count()`.
+
+## Config essentials
+
+- `rpcUrl`: RPC endpoint URL (supports placeholders like `{{EVM_RPC_API_KEY}}`)
+- `dbPath`: SQLite path (default `./indexer.db`)
+- `contracts`: non-empty list of contracts and events to index
+- `network.polling`: block polling interval and confirmations
+- `network.logs`: chunk size, retries, parallel requests
+- `network.reorg.depth`: reorg buffer depth
+- `telemetry.progress`: CLI progress rendering
+- `logLevel`, `logFormat`, `enableTelemetry`
+
+## Operational notes
+
+- Run a single writer process per SQLite file.
+- Keep DB on persistent storage.
+- Worker resumes from checkpoint after restart.
+- RPC must support `eth_getLogs`.
+
+## Development
+
+```bash
+npm run build
+npm run typecheck
+npm run test
+npm run check
+```
+
+Repository: [github.com/cybervoid0/effective-indexer](https://github.com/cybervoid0/effective-indexer)
+# Effective Indexer
+
 Lightweight EVM smart contract event indexer built with [Effect](https://effect.website).
 
+Index EVM events to your own database in minutes — no hosted lock-in, no PhD required.
+
 Repository: [github.com/cybervoid0/effective-indexer](https://github.com/cybervoid0/effective-indexer)
 
 Indexes smart contract events into SQLite with:
@@ -87,6 +240,21 @@ Returns `IndexerHandle`:
 - `query(q?: EventQuery): Promise<ParsedEvent[]>`
 - `count(q?: EventQuery): Promise<number>`
 
+### `defineIndexerConfig(config)`
+
+Identity helper for a typed config file (Hardhat-style DX).
+
+### `resolveIndexerConfigFromEnv(config, options?)`
+
+Resolves `{{ENV_VAR}}` placeholders in `rpcUrl` and supports optional RPC override from env (`EVM_RPC_URL` by default).
+
+### `runIndexerWorker(config, options?)`
+
+Runs a long-lived worker with built-in:
+- SQLite directory creation
+- graceful shutdown on `SIGINT` / `SIGTERM`
+- keep-alive process loop
+
 ### `IndexerConfig`
 
 | Field | Type | Default | Description |
@@ -159,7 +327,21 @@ Run the indexer as a dedicated long-lived worker process (not in request handler
 Create `scripts/indexer.ts`:
 
 ```ts
-import { Indexer } from "effective-indexer"
+import config from "../indexer.config"
+import { resolveIndexerConfigFromEnv, runIndexerWorker } from "effective-indexer"
+
+const resolvedConfig = resolveIndexerConfigFromEnv(config)
+
+runIndexerWorker(resolvedConfig).catch(error => {
+	console.error("Indexer worker failed:", error)
+	process.exit(1)
+})
+```
+
+Create `indexer.config.ts`:
+
+```ts
+import { defineIndexerConfig } from "effective-indexer"
 import type { Abi } from "viem"
 
 const transferAbi: Abi = [
@@ -174,104 +356,27 @@ const transferAbi: Abi = [
 	},
 ]
 
-const indexer = Indexer.create({
-	rpcUrl: process.env.EVM_RPC_URL!,
-	dbPath: process.env.INDEXER_DB_PATH ?? "./data/events.db",
+export default defineIndexerConfig({
+	rpcUrl: "https://rpc.mainnet.rootstock.io/{{EVM_RPC_API_KEY}}",
+	dbPath: "./data/events.db",
 	contracts: [
 		{
-			name: process.env.INDEXER_CONTRACT_NAME ?? "Token",
-			address: process.env.INDEXER_CONTRACT_ADDRESS!,
+			name: "Token",
+			address: "0xYourContractAddress",
 			abi: transferAbi,
 			events: ["Transfer"],
-			startBlock: BigInt(process.env.INDEXER_START_BLOCK ?? "0"),
+			startBlock: 0n,
 		},
 	],
-	network: {
-		polling: {
-			intervalMs: Number(process.env.INDEXER_POLL_INTERVAL_MS ?? "12000"),
-			confirmations: Number(process.env.INDEXER_CONFIRMATIONS ?? "1"),
-		},
-		logs: {
-			chunkSize: Number(process.env.INDEXER_CHUNK_SIZE ?? "2000"),
-			parallelRequests: Number(process.env.INDEXER_PARALLEL_REQUESTS ?? "1"),
-			maxRetries: Number(process.env.INDEXER_MAX_RETRIES ?? "5"),
-			retry: {
-				baseDelayMs: Number(process.env.INDEXER_RETRY_BASE_MS ?? "1000"),
-				maxDelayMs: Number(process.env.INDEXER_RETRY_MAX_MS ?? "30000"),
-			},
-		},
-		reorg: {
-			depth: Number(process.env.INDEXER_REORG_DEPTH ?? "20"),
-		},
-	},
-	telemetry: {
-		progress: {
-			enabled: process.env.INDEXER_PROGRESS_ENABLED !== "false",
-			intervalMs: Number(process.env.INDEXER_PROGRESS_INTERVAL_MS ?? "3000"),
-		},
-	},
-	logLevel: (process.env.INDEXER_LOG_LEVEL ?? "info") as
-		| "trace"
-		| "debug"
-		| "info"
-		| "warning"
-		| "error"
-		| "none",
-	logFormat: (process.env.INDEXER_LOG_FORMAT ?? "pretty") as
-		| "pretty"
-		| "json"
-		| "structured",
-	enableTelemetry: process.env.INDEXER_TELEMETRY !== "false",
-})
-
-const start = async (): Promise<void> => {
-	await indexer.start()
-	console.log("Indexer worker started")
-
-	// Keep the process alive while indexing in background.
-	const keepAlive = setInterval(() => undefined, 60_000)
-
-	const stop = async (): Promise<void> => {
-		clearInterval(keepAlive)
-		await indexer.stop()
-		process.exit(0)
-	}
-
-	process.on("SIGINT", () => {
-		void stop()
-	})
-	process.on("SIGTERM", () => {
-		void stop()
-	})
-}
-
-start().catch(error => {
-	console.error("Indexer worker failed:", error)
-	process.exit(1)
 })
 ```
 
 Create `.env`:
 
 ```bash
-EVM_RPC_URL=https://your-rpc-url
-INDEXER_DB_PATH=./data/events.db
-INDEXER_CONTRACT_NAME=Token
-INDEXER_CONTRACT_ADDRESS=0xYourContractAddress
-INDEXER_START_BLOCK=0
-INDEXER_POLL_INTERVAL_MS=12000
-INDEXER_CONFIRMATIONS=1
-INDEXER_CHUNK_SIZE=2000
-INDEXER_PARALLEL_REQUESTS=1
-INDEXER_MAX_RETRIES=5
-INDEXER_RETRY_BASE_MS=1000
-INDEXER_RETRY_MAX_MS=30000
-INDEXER_REORG_DEPTH=20
-INDEXER_PROGRESS_ENABLED=true
-INDEXER_PROGRESS_INTERVAL_MS=3000
-INDEXER_LOG_LEVEL=info
-INDEXER_LOG_FORMAT=pretty
-INDEXER_TELEMETRY=true
+EVM_RPC_API_KEY=your-rpc-api-key
+# Optional full RPC URL override:
+# EVM_RPC_URL=https://rpc.mainnet.rootstock.io/<API_KEY>
 ```
 
 Add scripts (with `tsx` installed):

package/dist/index.cjs

@@ -1,5 +1,7 @@
 'use strict';
 
+var promises = require('fs/promises');
+var path = require('path');
 var sqlSqliteNode = require('@effect/sql-sqlite-node');
 var effect = require('effect');
 var viem = require('viem');
@@ -20,6 +22,7 @@ var ConfigError = class extends effect.Data.TaggedError("ConfigError") {
 };
 
 // src/config.ts
+var defineIndexerConfig = (config) => config;
 var resolveNetwork = (config) => {
   const n = config.network;
   return {
@@ -135,6 +138,7 @@ var RpcProviderLive = effect.Layer.effect(
       })
     });
     const getLogs = (params) => effect.Effect.tryPromise({
+      // Use raw request to pass topics exactly as eth_getLogs expects.
       try: () => client.request({
         method: "eth_getLogs",
         params: [
@@ -712,6 +716,7 @@ var fetchLogs = (params) => effect.Stream.unwrap(
             fromBlock: chunk.from,
             toBlock: chunk.to
           }).pipe(
+            // Retry policy is exponential and bounded by maxDelayMs.
             effect.Effect.tapError(
               (e) => effect.Effect.gen(function* () {
                 const n = yield* effect.Ref.getAndUpdate(attempt, (a) => a + 1);
@@ -910,11 +915,14 @@ var indexContract = (contract) => effect.Stream.unwrap(
             );
           }
           const chunkSize = BigInt(config.network.logs.chunkSize);
-          const lastProcessed = yield* effect.Ref.modify(processedBlocksRef, (current) => {
-            const advanced = current + chunkSize;
-            const next = advanced > totalBackfillBlocks ? totalBackfillBlocks : advanced;
-            return [next, next];
-          });
+          const lastProcessed = yield* effect.Ref.modify(
+            processedBlocksRef,
+            (current) => {
+              const advanced = current + chunkSize;
+              const next = advanced > totalBackfillBlocks ? totalBackfillBlocks : advanced;
+              return [next, next];
+            }
+          );
           yield* progress.update(
             contract.name,
             lastProcessed,
@@ -1097,6 +1105,64 @@ var QueryApiLive = effect.Layer.effect(
     return { getEvents, getEventCount, getLatestBlock };
   })
 );
+var toConfigMap = (env) => {
+  const map = /* @__PURE__ */ new Map();
+  for (const [key, value] of Object.entries(env)) {
+    if (typeof value === "string") {
+      map.set(key, value);
+    }
+  }
+  return map;
+};
+var getProvider = (env) => effect.ConfigProvider.fromMap(toConfigMap(env ?? process.env));
+var readOptionalString = (name, provider) => effect.Effect.runSync(
+  effect.Effect.withConfigProvider(provider)(
+    effect.Config.option(effect.Config.nonEmptyString(name))
+  )
+);
+var readRequiredString = (name, provider) => {
+  try {
+    return effect.Effect.runSync(
+      effect.Effect.withConfigProvider(provider)(effect.Config.nonEmptyString(name))
+    );
+  } catch {
+    throw new Error(`Missing required env var: ${name}`);
+  }
+};
+var readRequiredRedactedString = (name, provider) => {
+  try {
+    const redacted = effect.Effect.runSync(
+      effect.Effect.withConfigProvider(provider)(
+        effect.Config.redacted(effect.Config.nonEmptyString(name))
+      )
+    );
+    return effect.Redacted.value(redacted);
+  } catch {
+    throw new Error(`Missing required env var: ${name}`);
+  }
+};
+var resolveRpcUrl = (template, provider, sensitiveEnvNames) => template.replace(
+  /\{\{([A-Z0-9_]+)\}\}/g,
+  (_input, envName) => sensitiveEnvNames.has(envName) ? readRequiredRedactedString(envName, provider) : readRequiredString(envName, provider)
+);
+var resolveIndexerConfigFromEnv = (config, options) => {
+  const provider = getProvider(options?.env);
+  const rpcUrlOverrideEnv = options?.rpcUrlOverrideEnv ?? "EVM_RPC_URL";
+  const rpcUrlOverride = readOptionalString(rpcUrlOverrideEnv, provider);
+  const sensitiveEnvNames = new Set(
+    options?.sensitiveEnvNames ?? ["EVM_RPC_API_KEY"]
+  );
+  if (effect.Option.isSome(rpcUrlOverride)) {
+    return {
+      ...config,
+      rpcUrl: rpcUrlOverride.value
+    };
+  }
+  return {
+    ...config,
+    rpcUrl: resolveRpcUrl(config.rpcUrl, provider, sensitiveEnvNames)
+  };
+};
 
 // src/index.ts
 var buildLayers = (config) => {
@@ -1194,6 +1260,67 @@ var createIndexer = (config) => {
     }
   };
 };
+var defaultWorkerRuntime = {
+  process,
+  setInterval,
+  clearInterval
+};
+var resolveDbPath = (config) => config.dbPath ?? "./indexer.db";
+var ensureDbDirectory = async (config) => {
+  const dbPath = resolveDbPath(config);
+  if (dbPath === ":memory:") {
+    return;
+  }
+  await promises.mkdir(path.dirname(dbPath), { recursive: true });
+};
+var runIndexerWorker = async (config, options) => {
+  if (options?.ensureDbDirectory ?? true) {
+    await ensureDbDirectory(config);
+  }
+  const runtime = options?.runtime ?? defaultWorkerRuntime;
+  const create = options?.createIndexer ?? createIndexer;
+  const signals = options?.shutdownSignals ?? ["SIGINT", "SIGTERM"];
+  const keepAliveIntervalMs = options?.keepAliveIntervalMs ?? 6e4;
+  const indexer = create(config);
+  await indexer.start();
+  await new Promise((resolve, reject) => {
+    const keepAliveTimer = runtime.setInterval(
+      () => void 0,
+      keepAliveIntervalMs
+    );
+    let stopping = false;
+    const signalHandlers = /* @__PURE__ */ new Map();
+    const cleanup = () => {
+      runtime.clearInterval(keepAliveTimer);
+      for (const signal of signals) {
+        const handler = signalHandlers.get(signal);
+        if (handler !== void 0) {
+          runtime.process.off(signal, handler);
+        }
+      }
+    };
+    const handleStop = async () => {
+      if (stopping) {
+        return;
+      }
+      stopping = true;
+      cleanup();
+      try {
+        await indexer.stop();
+        resolve();
+      } catch (error) {
+        reject(error);
+      }
+    };
+    for (const signal of signals) {
+      const handler = () => {
+        void handleStop();
+      };
+      signalHandlers.set(signal, handler);
+      runtime.process.on(signal, handler);
+    }
+  });
+};
 var Indexer = {
   create: createIndexer
 };
@@ -1222,6 +1349,9 @@ exports.Storage = Storage;
 exports.StorageLive = StorageLive;
 exports.computeSnapshot = computeSnapshot;
 exports.createIndexer = createIndexer;
+exports.defineIndexerConfig = defineIndexerConfig;
 exports.resolveConfig = resolveConfig;
+exports.resolveIndexerConfigFromEnv = resolveIndexerConfigFromEnv;
+exports.runIndexerWorker = runIndexerWorker;
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map