@sentio/runtime 2.40.0-rc.4 → 2.40.0-rc.40

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.40.0-rc.4",
+  "version": "2.40.0-rc.40",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {

package/src/db-context.ts

@@ -8,55 +8,33 @@ import {
   ProcessStreamResponse
 } from '@sentio/protos'
 import * as process from 'node:process'
-import { Attributes, Counter, metrics } from '@opentelemetry/api'
-
+import { dbMetrics } from './metrics.js'
+const {
+  request_errors,
+  unsolved_requests,
+  request_times,
+  batched_request_count,
+  batched_total_count,
+  send_counts,
+  recv_counts
+} = dbMetrics
 const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
 const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
+const STORE_UPSERT_NO_WAIT = process.env['STORE_UPSERT_NO_WAIT'] === 'true'
 
 type Request = Omit<DBRequest, 'opId'>
 type RequestType = keyof Request
 
-const meter = metrics.getMeter('processor_store')
-const send_counts: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_count'),
-  upsert: meter.createCounter('store_upsert_count'),
-  list: meter.createCounter('store_list_count'),
-  delete: meter.createCounter('store_delete_count')
-}
-const recv_counts: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_count'),
-  upsert: meter.createCounter('store_upsert_count'),
-  list: meter.createCounter('store_list_count'),
-  delete: meter.createCounter('store_delete_count')
-}
-const request_times: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_time'),
-  upsert: meter.createCounter('store_upsert_time'),
-  list: meter.createCounter('store_list_time'),
-  delete: meter.createCounter('store_delete_time')
-}
-const request_errors: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_error'),
-  upsert: meter.createCounter('store_upsert_error'),
-  list: meter.createCounter('store_list_error'),
-  delete: meter.createCounter('store_delete_error')
-}
-
-const batched_total_count = meter.createCounter('batched_total_count')
-const batched_request_count = meter.createCounter('batched_request_count')
-
-const unsolved_requests = meter.createGauge('store_unsolved_requests')
-
-export const timeoutError = Symbol()
+export const timeoutError = new Error('timeout')
 
 export class StoreContext {
   private static opCounter = 0n
-
   private defers = new Map<
     bigint,
     { resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
   >()
   private statsInterval: NodeJS.Timeout | undefined
+  private pendings: Promise<unknown>[] = []
 
   constructor(
     readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
@@ -82,7 +60,7 @@ export class StoreContext {
 
     const start = Date.now()
     const promises = [promise]
-    console.debug('sending db request ', opId, request)
+    // console.debug('sending db request ', opId, request)
     let timer: NodeJS.Timeout | undefined
     if (timeoutSecs) {
       const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
@@ -99,9 +77,18 @@ export class StoreContext {
 
     send_counts[requestType]?.add(1)
 
+    if (requestType === 'upsert' && STORE_UPSERT_NO_WAIT) {
+      this.pendings.push(promise)
+      return Promise.resolve({
+        opId,
+      } as DBResponse)
+    }
+
     return Promise.race(promises)
       .then((result: DBResponse) => {
-        console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
+        if (timeoutSecs) {
+          console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
+        }
         request_times[requestType]?.add(Date.now() - start)
         return result
       })
@@ -122,7 +109,7 @@ export class StoreContext {
   result(dbResult: DBResponse) {
     const opId = dbResult.opId
     const defer = this.defers.get(opId)
-    console.debug('received db result ', opId, dbResult)
+    // console.debug('received db result ', opId, dbResult)
     if (defer) {
       if (defer.requestType) {
         recv_counts[defer.requestType]?.add(1)
@@ -152,8 +139,8 @@ export class StoreContext {
 
   close() {
     for (const [opId, defer] of this.defers) {
-      console.warn('context closed before db response', opId)
-      defer.reject(new Error('context closed'))
+      // console.warn('context closed before db response', opId)
+      defer.reject(new Error('context closed before db response, processId: ' + this.processId + ' opId: ' + opId))
     }
     this.defers.clear()
     if (this.statsInterval) {
@@ -180,29 +167,45 @@ export class StoreContext {
       if (request.entity.length >= STORE_BATCH_SIZE) {
         this.sendBatch()
       }
+      if (STORE_UPSERT_NO_WAIT) {
+        return {
+          opId: this.upsertBatch.opId
+        }
+      }
+
       return promise
     } else {
       const opId = StoreContext.opCounter++
-      const promise = this.newPromise<DBResponse>(opId, 'upsert')
       const timeout = setTimeout(() => {
         this.sendBatch()
       }, STORE_BATCH_IDLE)
+      const start = Date.now()
+      const promise = this.newPromise<DBResponse>(opId, 'upsert').finally(() => {
+        request_times['upsert'].add(Date.now() - start)
+      })
 
       this.upsertBatch = {
         opId,
         request: req,
         promise,
-        timer: timeout
+        timer: timeout,
       }
 
-      return promise
+      if (STORE_UPSERT_NO_WAIT) {
+        this.pendings.push(promise)
+        return {
+          opId: this.upsertBatch.opId
+        }
+      } else {
+        return promise
+      }
     }
   }
 
   private sendBatch() {
     if (this.upsertBatch) {
       const { request, opId, timer } = this.upsertBatch
-      console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
+      // console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
       clearTimeout(timer)
       this.upsertBatch = undefined
       this.subject.next({
@@ -217,4 +220,8 @@ export class StoreContext {
       batched_total_count.add(request.entity.length)
     }
   }
+
+  async awaitPendings() {
+    await Promise.all(this.pendings)
+  }
 }

package/src/gen/processor/protos/processor.ts

@@ -3,7 +3,7 @@ import Long from "long";
 import type { CallContext, CallOptions } from "nice-grpc-common";
 import _m0 from "protobufjs/minimal.js";
 import { Empty } from "../../google/protobuf/empty.js";
-import { ListValue, Struct, Value } from "../../google/protobuf/struct.js";
+import { ListValue, Struct } from "../../google/protobuf/struct.js";
 import { Timestamp } from "../../google/protobuf/timestamp.js";
 import { BigInteger, CoinID, RichStruct, RichStructList, RichValueList } from "../../service/common/protos/common.js";
 
@@ -1069,11 +1069,14 @@ export interface ProcessResult {
 }
 
 export interface EthCallParam {
+  context: EthCallContext | undefined;
+  calldata: string;
+}
+
+export interface EthCallContext {
   chainId: string;
   address: string;
-  function: string;
-  signature: string;
-  args: any[];
+  blockTag: string;
 }
 
 export interface PreprocessResult {
@@ -1081,12 +1084,12 @@ export interface PreprocessResult {
 }
 
 export interface PreparedData {
-  ethCallResults: { [key: string]: Array<any> | undefined };
+  ethCallResults: { [key: string]: string };
 }
 
 export interface PreparedData_EthCallResultsEntry {
   key: string;
-  value: Array<any> | undefined;
+  value: string;
 }
 
 export interface RecordMetaData {
@@ -9337,25 +9340,16 @@ export const ProcessResult = {
 };
 
 function createBaseEthCallParam(): EthCallParam {
-  return { chainId: "", address: "", function: "", signature: "", args: [] };
+  return { context: undefined, calldata: "" };
 }
 
 export const EthCallParam = {
   encode(message: EthCallParam, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
-    if (message.chainId !== "") {
-      writer.uint32(10).string(message.chainId);
+    if (message.context !== undefined) {
+      EthCallContext.encode(message.context, writer.uint32(10).fork()).ldelim();
     }
-    if (message.address !== "") {
-      writer.uint32(18).string(message.address);
-    }
-    if (message.function !== "") {
-      writer.uint32(26).string(message.function);
-    }
-    if (message.signature !== "") {
-      writer.uint32(34).string(message.signature);
-    }
-    for (const v of message.args) {
-      Value.encode(Value.wrap(v!), writer.uint32(42).fork()).ldelim();
+    if (message.calldata !== "") {
+      writer.uint32(18).string(message.calldata);
     }
     return writer;
   },
@@ -9372,35 +9366,100 @@ export const EthCallParam = {
             break;
           }
 
-          message.chainId = reader.string();
+          message.context = EthCallContext.decode(reader, reader.uint32());
           continue;
         case 2:
           if (tag !== 18) {
             break;
           }
 
-          message.address = reader.string();
+          message.calldata = reader.string();
           continue;
-        case 3:
-          if (tag !== 26) {
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): EthCallParam {
+    return {
+      context: isSet(object.context) ? EthCallContext.fromJSON(object.context) : undefined,
+      calldata: isSet(object.calldata) ? globalThis.String(object.calldata) : "",
+    };
+  },
+
+  toJSON(message: EthCallParam): unknown {
+    const obj: any = {};
+    if (message.context !== undefined) {
+      obj.context = EthCallContext.toJSON(message.context);
+    }
+    if (message.calldata !== "") {
+      obj.calldata = message.calldata;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<EthCallParam>): EthCallParam {
+    return EthCallParam.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<EthCallParam>): EthCallParam {
+    const message = createBaseEthCallParam();
+    message.context = (object.context !== undefined && object.context !== null)
+      ? EthCallContext.fromPartial(object.context)
+      : undefined;
+    message.calldata = object.calldata ?? "";
+    return message;
+  },
+};
+
+function createBaseEthCallContext(): EthCallContext {
+  return { chainId: "", address: "", blockTag: "" };
+}
+
+export const EthCallContext = {
+  encode(message: EthCallContext, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.chainId !== "") {
+      writer.uint32(10).string(message.chainId);
+    }
+    if (message.address !== "") {
+      writer.uint32(18).string(message.address);
+    }
+    if (message.blockTag !== "") {
+      writer.uint32(26).string(message.blockTag);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): EthCallContext {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseEthCallContext();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
             break;
           }
 
-          message.function = reader.string();
+          message.chainId = reader.string();
           continue;
-        case 4:
-          if (tag !== 34) {
+        case 2:
+          if (tag !== 18) {
             break;
           }
 
-          message.signature = reader.string();
+          message.address = reader.string();
           continue;
-        case 5:
-          if (tag !== 42) {
+        case 3:
+          if (tag !== 26) {
             break;
           }
 
-          message.args.push(Value.unwrap(Value.decode(reader, reader.uint32())));
+          message.blockTag = reader.string();
           continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
@@ -9411,17 +9470,15 @@ export const EthCallParam = {
     return message;
   },
 
-  fromJSON(object: any): EthCallParam {
+  fromJSON(object: any): EthCallContext {
     return {
       chainId: isSet(object.chainId) ? globalThis.String(object.chainId) : "",
       address: isSet(object.address) ? globalThis.String(object.address) : "",
-      function: isSet(object.function) ? globalThis.String(object.function) : "",
-      signature: isSet(object.signature) ? globalThis.String(object.signature) : "",
-      args: globalThis.Array.isArray(object?.args) ? [...object.args] : [],
+      blockTag: isSet(object.blockTag) ? globalThis.String(object.blockTag) : "",
     };
   },
 
-  toJSON(message: EthCallParam): unknown {
+  toJSON(message: EthCallContext): unknown {
     const obj: any = {};
     if (message.chainId !== "") {
       obj.chainId = message.chainId;
@@ -9429,28 +9486,20 @@ export const EthCallParam = {
     if (message.address !== "") {
       obj.address = message.address;
     }
-    if (message.function !== "") {
-      obj.function = message.function;
-    }
-    if (message.signature !== "") {
-      obj.signature = message.signature;
-    }
-    if (message.args?.length) {
-      obj.args = message.args;
+    if (message.blockTag !== "") {
+      obj.blockTag = message.blockTag;
     }
     return obj;
   },
 
-  create(base?: DeepPartial<EthCallParam>): EthCallParam {
-    return EthCallParam.fromPartial(base ?? {});
+  create(base?: DeepPartial<EthCallContext>): EthCallContext {
+    return EthCallContext.fromPartial(base ?? {});
   },
-  fromPartial(object: DeepPartial<EthCallParam>): EthCallParam {
-    const message = createBaseEthCallParam();
+  fromPartial(object: DeepPartial<EthCallContext>): EthCallContext {
+    const message = createBaseEthCallContext();
     message.chainId = object.chainId ?? "";
     message.address = object.address ?? "";
-    message.function = object.function ?? "";
-    message.signature = object.signature ?? "";
-    message.args = object.args?.map((e) => e) || [];
+    message.blockTag = object.blockTag ?? "";
     return message;
   },
 };
@@ -9523,9 +9572,7 @@ function createBasePreparedData(): PreparedData {
 export const PreparedData = {
   encode(message: PreparedData, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
     Object.entries(message.ethCallResults).forEach(([key, value]) => {
-      if (value !== undefined) {
-        PreparedData_EthCallResultsEntry.encode({ key: key as any, value }, writer.uint32(10).fork()).ldelim();
-      }
+      PreparedData_EthCallResultsEntry.encode({ key: key as any, value }, writer.uint32(10).fork()).ldelim();
     });
     return writer;
   },
@@ -9559,13 +9606,10 @@ export const PreparedData = {
   fromJSON(object: any): PreparedData {
     return {
       ethCallResults: isObject(object.ethCallResults)
-        ? Object.entries(object.ethCallResults).reduce<{ [key: string]: Array<any> | undefined }>(
-          (acc, [key, value]) => {
-            acc[key] = value as Array<any> | undefined;
-            return acc;
-          },
-          {},
-        )
+        ? Object.entries(object.ethCallResults).reduce<{ [key: string]: string }>((acc, [key, value]) => {
+          acc[key] = String(value);
+          return acc;
+        }, {})
         : {},
     };
   },
@@ -9589,20 +9633,21 @@ export const PreparedData = {
   },
   fromPartial(object: DeepPartial<PreparedData>): PreparedData {
     const message = createBasePreparedData();
-    message.ethCallResults = Object.entries(object.ethCallResults ?? {}).reduce<
-      { [key: string]: Array<any> | undefined }
-    >((acc, [key, value]) => {
-      if (value !== undefined) {
-        acc[key] = value;
-      }
-      return acc;
-    }, {});
+    message.ethCallResults = Object.entries(object.ethCallResults ?? {}).reduce<{ [key: string]: string }>(
+      (acc, [key, value]) => {
+        if (value !== undefined) {
+          acc[key] = globalThis.String(value);
+        }
+        return acc;
+      },
+      {},
+    );
     return message;
   },
 };
 
 function createBasePreparedData_EthCallResultsEntry(): PreparedData_EthCallResultsEntry {
-  return { key: "", value: undefined };
+  return { key: "", value: "" };
 }
 
 export const PreparedData_EthCallResultsEntry = {
@@ -9610,8 +9655,8 @@ export const PreparedData_EthCallResultsEntry = {
     if (message.key !== "") {
       writer.uint32(10).string(message.key);
     }
-    if (message.value !== undefined) {
-      ListValue.encode(ListValue.wrap(message.value), writer.uint32(18).fork()).ldelim();
+    if (message.value !== "") {
+      writer.uint32(18).string(message.value);
     }
     return writer;
   },
@@ -9635,7 +9680,7 @@ export const PreparedData_EthCallResultsEntry = {
             break;
           }
 
-          message.value = ListValue.unwrap(ListValue.decode(reader, reader.uint32()));
+          message.value = reader.string();
           continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
@@ -9649,7 +9694,7 @@ export const PreparedData_EthCallResultsEntry = {
   fromJSON(object: any): PreparedData_EthCallResultsEntry {
     return {
       key: isSet(object.key) ? globalThis.String(object.key) : "",
-      value: globalThis.Array.isArray(object.value) ? [...object.value] : undefined,
+      value: isSet(object.value) ? globalThis.String(object.value) : "",
     };
   },
 
@@ -9658,7 +9703,7 @@ export const PreparedData_EthCallResultsEntry = {
     if (message.key !== "") {
       obj.key = message.key;
     }
-    if (message.value !== undefined) {
+    if (message.value !== "") {
       obj.value = message.value;
     }
     return obj;
@@ -9670,7 +9715,7 @@ export const PreparedData_EthCallResultsEntry = {
   fromPartial(object: DeepPartial<PreparedData_EthCallResultsEntry>): PreparedData_EthCallResultsEntry {
     const message = createBasePreparedData_EthCallResultsEntry();
     message.key = object.key ?? "";
-    message.value = object.value ?? undefined;
+    message.value = object.value ?? "";
     return message;
   },
 };

package/src/index.ts

@@ -6,3 +6,4 @@ export * from './chain-config.js'
 export * from './service.js'
 export { GLOBAL_CONFIG, type GlobalConfig } from './global-config.js'
 export * from './db-context.js'
+export * from './provider.js'

package/src/metrics.ts

@@ -0,0 +1,138 @@
+import { Attributes, Counter, metrics, Gauge } from '@opentelemetry/api'
+
+const meter = metrics.getMeter('processor')
+
+class C {
+  private counter: Counter<Attributes>
+  private value: number = 0
+
+  constructor(name: string) {
+    this.counter = meter.createCounter(name)
+  }
+
+  add(value: number, attributes?: Attributes) {
+    this.counter.add(value, attributes)
+    this.value += value
+  }
+
+  get() {
+    return this.value
+  }
+}
+
+class G {
+  private gauge: Gauge<Attributes>
+  private value: number = 0
+
+  constructor(name: string) {
+    this.gauge = meter.createGauge(name)
+  }
+
+  record(value: number, attributes?: Attributes) {
+    this.gauge.record(value, attributes)
+    this.value = value
+  }
+
+  get() {
+    return this.value
+  }
+}
+
+export const dbMetrics = {
+  send_counts: {
+    get: new C('store_get_send'),
+    upsert: new C('store_upsert_send'),
+    list: new C('store_list_send'),
+    delete: new C('store_delete_send')
+  },
+  recv_counts: {
+    get: new C('store_get_recv'),
+    upsert: new C('store_upsert_recv'),
+    list: new C('store_list_recv'),
+    delete: new C('store_delete_recv')
+  },
+  request_times: {
+    get: new C('store_get_time'),
+    upsert: new C('store_upsert_time'),
+    list: new C('store_list_time'),
+    delete: new C('store_delete_time')
+  },
+  request_errors: {
+    get: new C('store_get_error'),
+    upsert: new C('store_upsert_error'),
+    list: new C('store_list_error'),
+    delete: new C('store_delete_error')
+  },
+  batched_total_count: new C('batched_total_count'),
+  batched_request_count: new C('batched_request_count'),
+  unsolved_requests: new G('store_unsolved_requests'),
+
+  stats() {
+    return {
+      send_counts: {
+        get: this.send_counts.get.get(),
+        upsert: this.send_counts.upsert.get(),
+        list: this.send_counts.list.get(),
+        delete: this.send_counts.delete.get()
+      },
+      recv_counts: {
+        get: this.recv_counts.get.get(),
+        upsert: this.recv_counts.upsert.get(),
+        list: this.recv_counts.list.get(),
+        delete: this.recv_counts.delete.get()
+      },
+      request_times: {
+        get: this.request_times.get.get(),
+        upsert: this.request_times.upsert.get(),
+        list: this.request_times.list.get(),
+        delete: this.request_times.delete.get()
+      },
+      request_errors: {
+        get: this.request_errors.get.get(),
+        upsert: this.request_errors.upsert.get(),
+        list: this.request_errors.list.get(),
+        delete: this.request_errors.delete.get()
+      },
+      batched_total_count: this.batched_total_count.get(),
+      batched_request_count: this.batched_request_count.get(),
+      unsolved_requests: this.unsolved_requests.get(),
+      average_request_time: {
+        get: this.request_times.get.get() / this.send_counts.get.get(),
+        upsert: this.request_times.upsert.get() / this.send_counts.upsert.get(),
+        list: this.request_times.list.get() / this.send_counts.list.get()
+      }
+    }
+  }
+}
+
+export const providerMetrics = {
+  hit_count: new C('provider_hit_count'),
+  miss_count: new C('provider_miss_count'),
+  queue_size: new G('provider_queue_size'),
+  total_duration: new C('provider_total_duration'),
+  total_queued: new C('provider_total_queued'),
+  stats() {
+    return {
+      hit_count: this.hit_count.get(),
+      miss_count: this.miss_count.get(),
+      queue_size: this.queue_size.get(),
+      total_duration: this.total_duration.get(),
+      total_queued: this.total_queued.get(),
+      average_queue_time: this.total_queued.get() / (this.hit_count.get() + this.miss_count.get()),
+      average_duration: this.total_duration.get() / (this.hit_count.get() + this.miss_count.get())
+    }
+  }
+}
+
+export const processMetrics = {
+  process_binding_count: new C('process_binding_count'),
+  process_binding_time: new C('process_binding_time'),
+  process_binding_error: new C('process_binding_error'),
+  stats() {
+    return {
+      process_binding_count: this.process_binding_count.get(),
+      process_binding_time: this.process_binding_time.get(),
+      process_binding_error: this.process_binding_error.get()
+    }
+  }
+}