envio 3.0.0-alpha.10 → 3.0.0-alpha.11

package/index.d.ts

@@ -518,6 +518,14 @@ type AddressRegistration = {
 type ConfigEntities<Config extends IndexerConfigTypes> =
   Config["entities"] extends Record<string, object> ? Config["entities"] : {};
 
+/** Entity operations available on test indexer for direct entity manipulation. */
+type EntityOps<Entity> = {
+  /** Get an entity by ID. Returns undefined if not found. */
+  readonly get: (id: string) => Promise<Entity | undefined>;
+  /** Set (create or update) an entity. */
+  readonly set: (entity: Entity) => void;
+};
+
 /** A single change representing entity modifications at a specific block. */
 type EntityChange<Config extends IndexerConfigTypes> = {
   /** The block where the changes occurred. */
@@ -581,4 +589,9 @@ export type TestIndexerFromConfig<Config extends IndexerConfigTypes> = {
     /** Changes happened during the processing. */
     readonly changes: readonly EntityChange<Config>[];
   }>;
+} & {
+  /** Entity operations for direct manipulation outside of handlers. */
+  readonly [K in keyof ConfigEntities<Config>]: EntityOps<
+    ConfigEntities<Config>[K]
+  >;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v3.0.0-alpha.10",
+  "version": "v3.0.0-alpha.11",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
@@ -29,10 +29,10 @@
     "node": ">=22.0.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "v3.0.0-alpha.10",
-    "envio-linux-arm64": "v3.0.0-alpha.10",
-    "envio-darwin-x64": "v3.0.0-alpha.10",
-    "envio-darwin-arm64": "v3.0.0-alpha.10"
+    "envio-linux-x64": "v3.0.0-alpha.11",
+    "envio-linux-arm64": "v3.0.0-alpha.11",
+    "envio-darwin-x64": "v3.0.0-alpha.11",
+    "envio-darwin-arm64": "v3.0.0-alpha.11"
   },
   "dependencies": {
     "@clickhouse/client": "1.12.1",

package/src/ChainManager.res

@@ -11,7 +11,9 @@ type t = {
 // A chain is in reorg threshold when progressBlockNumber > sourceBlockNumber - maxReorgDepth.
 // This matches the logic in InternalTable.Checkpoints.makeGetReorgCheckpointsQuery.
 let isProgressInReorgThreshold = (~progressBlockNumber, ~sourceBlockNumber, ~maxReorgDepth) => {
-  maxReorgDepth > 0 && progressBlockNumber > sourceBlockNumber - maxReorgDepth
+  maxReorgDepth > 0 &&
+  sourceBlockNumber > 0 &&
+  progressBlockNumber > sourceBlockNumber - maxReorgDepth
 }
 
 let calculateTargetBufferSize = (~activeChainsCount, ~config: Config.t) => {

package/src/ChainManager.res.mjs

@@ -11,7 +11,7 @@ import * as ChainFetcher from "./ChainFetcher.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
 function isProgressInReorgThreshold(progressBlockNumber, sourceBlockNumber, maxReorgDepth) {
-  if (maxReorgDepth > 0) {
+  if (maxReorgDepth > 0 && sourceBlockNumber > 0) {
     return progressBlockNumber > (sourceBlockNumber - maxReorgDepth | 0);
   } else {
     return false;

package/src/Env.res

@@ -153,6 +153,7 @@ module Db = {
     //the SSL modes should be provided as string otherwise as 'require' | 'allow' | 'prefer' | 'verify-full'
     ~devFallback=Bool(false),
   )
+  let maxConnections = envSafe->EnvSafe.get("ENVIO_PG_MAX_CONNECTIONS", S.int, ~fallback=2)
 }
 
 module ClickHouseSink = {

package/src/Env.res.mjs

@@ -139,6 +139,8 @@ var publicSchema = EnvSafe.get(envSafe, "ENVIO_PG_PUBLIC_SCHEMA", S$RescriptSche
 
 var ssl = EnvSafe.get(envSafe, "ENVIO_PG_SSL_MODE", Postgres.sslOptionsSchema, undefined, undefined, false, undefined);
 
+var maxConnections = EnvSafe.get(envSafe, "ENVIO_PG_MAX_CONNECTIONS", S$RescriptSchema.$$int, undefined, 2, undefined, undefined);
+
 var Db = {
   host: host,
   port: port,
@@ -146,7 +148,8 @@ var Db = {
   password: password,
   database: database,
   publicSchema: publicSchema,
-  ssl: ssl
+  ssl: ssl,
+  maxConnections: maxConnections
 };
 
 var host$1 = EnvSafe.get(envSafe, "ENVIO_CLICKHOUSE_SINK_HOST", S$RescriptSchema.option(S$RescriptSchema.string), undefined, undefined, undefined, undefined);

package/src/PgStorage.res

@@ -19,7 +19,7 @@ let makeClient = () => {
           : Some(_str => ())
       ),
       transform: {undefined: Null},
-      max: 2,
+      max: Env.Db.maxConnections,
       // debug: (~connection, ~query, ~params as _, ~types as _) => Js.log2(connection, query),
     },
   )

package/src/PgStorage.res.mjs

@@ -37,7 +37,7 @@ function makeClient() {
               username: Env.Db.user,
               password: Env.Db.password,
               ssl: Env.Db.ssl,
-              max: 2,
+              max: Env.Db.maxConnections,
               onnotice: Caml_obj.equal(Env.userLogLevel, "warn") || Caml_obj.equal(Env.userLogLevel, "error") ? undefined : (function (_str) {
                     
                   }),

package/src/Prometheus.res

@@ -416,6 +416,18 @@ let sourceLabelsSchema = S.schema(s =>
   }
 )
 
+module SourceRequestCount = {
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_source_request_count",
+    ~help="The number of requests made to data sources.",
+    ~labelSchema=sourceLabelsSchema,
+  )
+
+  let increment = (~sourceName, ~chainId) => {
+    counter->SafeCounter.increment(~labels={"source": sourceName, "chainId": chainId})
+  }
+}
+
 module SourceHeight = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_source_height",

package/src/Prometheus.res.mjs

@@ -513,6 +513,20 @@ var sourceLabelsSchema = S$RescriptSchema.schema(function (s) {
             };
     });
 
+var counter$3 = makeOrThrow("envio_source_request_count", "The number of requests made to data sources.", sourceLabelsSchema);
+
+function increment$2(sourceName, chainId) {
+  increment(counter$3, {
+        source: sourceName,
+        chainId: chainId
+      });
+}
+
+var SourceRequestCount = {
+  counter: counter$3,
+  increment: increment$2
+};
+
 var gauge$11 = makeOrThrow$1("envio_source_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", sourceLabelsSchema);
 
 function set$11(sourceName, chainId, blockNumber) {
@@ -540,13 +554,13 @@ var SourceGetHeightDuration = {
 
 var gauge$12 = makeOrThrow$1("envio_reorg_count", "Total number of reorgs detected", chainIdLabelsSchema);
 
-function increment$2(chain) {
+function increment$3(chain) {
   increment$1(gauge$12, chain);
 }
 
 var ReorgCount = {
   gauge: gauge$12,
-  increment: increment$2
+  increment: increment$3
 };
 
 var gauge$13 = makeOrThrow$1("envio_reorg_detection_block_number", "The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.", chainIdLabelsSchema);
@@ -593,7 +607,7 @@ var timeCounter = new PromClient.Counter({
       help: "Rollback on reorg total time in milliseconds"
     });
 
-var counter$3 = new PromClient.Counter({
+var counter$4 = new PromClient.Counter({
       name: "envio_rollback_count",
       help: "Number of successful rollbacks on reorg"
     });
@@ -603,17 +617,17 @@ var eventsCounter = new PromClient.Counter({
       help: "Number of events rollbacked on reorg"
     });
 
-function increment$3(timeMillis, rollbackedProcessedEvents) {
+function increment$4(timeMillis, rollbackedProcessedEvents) {
   timeCounter.inc(Hrtime.intFromMillis(timeMillis));
-  counter$3.inc();
+  counter$4.inc();
   eventsCounter.inc(rollbackedProcessedEvents);
 }
 
 var RollbackSuccess = {
   timeCounter: timeCounter,
-  counter: counter$3,
+  counter: counter$4,
   eventsCounter: eventsCounter,
-  increment: increment$3
+  increment: increment$4
 };
 
 var entityNameLabelsSchema = S$RescriptSchema.object(function (s) {
@@ -622,18 +636,18 @@ var entityNameLabelsSchema = S$RescriptSchema.object(function (s) {
 
 var timeCounter$1 = makeOrThrow("envio_rollback_history_prune_time", "The total time spent pruning entity history which is not in the reorg threshold. (milliseconds)", entityNameLabelsSchema);
 
-var counter$4 = makeOrThrow("envio_rollback_history_prune_count", "Number of successful entity history prunes", entityNameLabelsSchema);
+var counter$5 = makeOrThrow("envio_rollback_history_prune_count", "Number of successful entity history prunes", entityNameLabelsSchema);
 
-function increment$4(timeMillis, entityName) {
+function increment$5(timeMillis, entityName) {
   handleInt(timeCounter$1, entityName, Hrtime.intFromMillis(timeMillis));
-  increment(counter$4, entityName);
+  increment(counter$5, entityName);
 }
 
 var RollbackHistoryPrune = {
   entityNameLabelsSchema: entityNameLabelsSchema,
   timeCounter: timeCounter$1,
-  counter: counter$4,
-  increment: increment$4
+  counter: counter$5,
+  increment: increment$5
 };
 
 var gauge$16 = makeOrThrow$1("envio_rollback_target_block_number", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
@@ -693,18 +707,18 @@ var ProgressEventsCount = {
   set: set$18
 };
 
-var counter$5 = new PromClient.Counter({
+var counter$6 = new PromClient.Counter({
       name: "envio_progress_batches_count",
       help: "The number of batches processed and reflected in the database."
     });
 
-function increment$5() {
-  counter$5.inc();
+function increment$6() {
+  counter$6.inc();
 }
 
 var ProgressBatchCount = {
-  counter: counter$5,
-  increment: increment$5
+  counter: counter$6,
+  increment: increment$6
 };
 
 var gauge$20 = makeOrThrow$1("envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", chainIdLabelsSchema);
@@ -748,15 +762,15 @@ var EffectCacheCount = {
   set: set$20
 };
 
-var counter$6 = makeOrThrow("envio_effect_cache_invalidations_count", "The number of effect cache invalidations.", effectLabelsSchema);
+var counter$7 = makeOrThrow("envio_effect_cache_invalidations_count", "The number of effect cache invalidations.", effectLabelsSchema);
 
-function increment$6(effectName) {
-  increment(counter$6, effectName);
+function increment$7(effectName) {
+  increment(counter$7, effectName);
 }
 
 var EffectCacheInvalidationsCount = {
-  counter: counter$6,
-  increment: increment$6
+  counter: counter$7,
+  increment: increment$7
 };
 
 var gauge$22 = makeOrThrow$1("envio_effect_queue_count", "The number of effect calls waiting in the rate limit queue.", effectLabelsSchema);
@@ -781,7 +795,7 @@ var timeCounter$4 = makeOrThrow("envio_storage_load_time", "Processing time take
 
 var sumTimeCounter$1 = makeOrThrow("envio_storage_load_sum_time", "Cumulative time spent loading data from storage during the indexing process. (milliseconds)", operationLabelsSchema);
 
-var counter$7 = makeOrThrow("envio_storage_load_count", "Cumulative number of successful storage load operations during the indexing process.", operationLabelsSchema);
+var counter$8 = makeOrThrow("envio_storage_load_count", "Cumulative number of successful storage load operations during the indexing process.", operationLabelsSchema);
 
 var whereSizeCounter = makeOrThrow("envio_storage_load_where_size", "Cumulative number of filter conditions ('where' items) used in storage load operations during the indexing process.", operationLabelsSchema);
 
@@ -810,7 +824,7 @@ function endOperation(timerRef, operation, whereSize, size) {
     Utils.Dict.deleteInPlace(operations, operation);
   }
   handleInt(sumTimeCounter$1, operation, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timerRef))));
-  increment(counter$7, operation);
+  increment(counter$8, operation);
   handleInt(whereSizeCounter, operation, whereSize);
   handleInt(sizeCounter, operation, size);
 }
@@ -819,7 +833,7 @@ var StorageLoad = {
   operationLabelsSchema: operationLabelsSchema,
   timeCounter: timeCounter$4,
   sumTimeCounter: sumTimeCounter$1,
-  counter: counter$7,
+  counter: counter$8,
   whereSizeCounter: whereSizeCounter,
   sizeCounter: sizeCounter,
   operations: operations,
@@ -833,18 +847,18 @@ var sinkLabelsSchema = S$RescriptSchema.object(function (s) {
 
 var timeCounter$5 = makeOrThrow("envio_sink_write_time", "Processing time taken to write data to sink. (milliseconds)", sinkLabelsSchema);
 
-var counter$8 = makeOrThrow("envio_sink_write_count", "Cumulative number of successful sink write operations during the indexing process.", sinkLabelsSchema);
+var counter$9 = makeOrThrow("envio_sink_write_count", "Cumulative number of successful sink write operations during the indexing process.", sinkLabelsSchema);
 
-function increment$7(sinkName, timeMillis) {
+function increment$8(sinkName, timeMillis) {
   handleInt(timeCounter$5, sinkName, timeMillis);
-  increment(counter$8, sinkName);
+  increment(counter$9, sinkName);
 }
 
 var SinkWrite = {
   sinkLabelsSchema: sinkLabelsSchema,
   timeCounter: timeCounter$5,
-  counter: counter$8,
-  increment: increment$7
+  counter: counter$9,
+  increment: increment$8
 };
 
 export {
@@ -884,6 +898,7 @@ export {
   IndexingBufferBlockNumber ,
   IndexingEndBlock ,
   sourceLabelsSchema ,
+  SourceRequestCount ,
   SourceHeight ,
   SourceGetHeightDuration ,
   ReorgCount ,

package/src/TestIndexer.res

@@ -324,6 +324,51 @@ let validateBlockRange = (
   }
 }
 
+// Entity operations for direct manipulation outside of handlers
+let makeEntityGet = (
+  ~state: testIndexerState,
+  ~entityConfig: Internal.entityConfig,
+): (string => promise<option<Internal.entity>>) => {
+  entityId => {
+    if state.processInProgress {
+      Js.Exn.raiseError(
+        `Cannot call ${entityConfig.name}.get() while indexer.process() is running. ` ++
+        "Wait for process() to complete before accessing entities directly.",
+      )
+    }
+    let entityDict =
+      state.entities->Js.Dict.get(entityConfig.name)->Option.getWithDefault(Js.Dict.empty())
+    Promise.resolve(entityDict->Js.Dict.get(entityId))
+  }
+}
+
+let makeEntitySet = (
+  ~state: testIndexerState,
+  ~entityConfig: Internal.entityConfig,
+): (Internal.entity => unit) => {
+  entity => {
+    if state.processInProgress {
+      Js.Exn.raiseError(
+        `Cannot call ${entityConfig.name}.set() while indexer.process() is running. ` ++
+        "Wait for process() to complete before modifying entities directly.",
+      )
+    }
+    let entityDict = switch state.entities->Js.Dict.get(entityConfig.name) {
+    | Some(dict) => dict
+    | None =>
+      let dict = Js.Dict.empty()
+      state.entities->Js.Dict.set(entityConfig.name, dict)
+      dict
+    }
+    entityDict->Js.Dict.set(entity.id, entity)
+  }
+}
+
+type entityOps = {
+  get: string => promise<option<Internal.entity>>,
+  set: Internal.entity => unit,
+}
+
 let makeCreateTestIndexer = (
   ~config: Config.t,
   ~workerPath: string,
@@ -343,8 +388,33 @@ let makeCreateTestIndexer = (
       entityConfigs,
       processChanges: [],
     }
-    {
-      process: processConfig => {
+
+    // Build entity operations for each user entity
+    let entityOpsDict: Js.Dict.t<entityOps> = Js.Dict.empty()
+    allEntities->Array.forEach(entityConfig => {
+      // Only create ops for user entities (not internal tables like dynamic_contract_registry)
+      if entityConfig.name !== InternalTable.DynamicContractRegistry.name {
+        entityOpsDict->Js.Dict.set(
+          entityConfig.name,
+          {
+            get: makeEntityGet(~state, ~entityConfig),
+            set: makeEntitySet(~state, ~entityConfig),
+          },
+        )
+      }
+    })
+
+    // Build the result object with process + entity operations
+    let result: Js.Dict.t<unknown> = Js.Dict.empty()
+    entityOpsDict
+    ->Js.Dict.entries
+    ->Array.forEach(((name, ops)) => {
+      result->Js.Dict.set(name, ops->(Utils.magic: entityOps => unknown))
+    })
+
+    result->Js.Dict.set(
+      "process",
+      (processConfig => {
         // Check if already processing
         if state.processInProgress {
           Js.Exn.raiseError(
@@ -491,8 +561,10 @@ let makeCreateTestIndexer = (
             }
           })
         })
-      },
-    }
+      })->(Utils.magic: ('a => promise<processResult>) => unknown),
+    )
+
+    result->(Utils.magic: Js.Dict.t<unknown> => t<'processConfig>)
   }
 }