@sentio/runtime 2.39.7-rc.8 → 2.40.0-rc.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.39.7-rc.8",
+  "version": "2.40.0-rc.1",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {
@@ -14,46 +14,26 @@
     "!**/*.test.{js,ts}",
     "!{lib,src}/tests"
   ],
-  "dependencies": {
-    "@grpc/grpc-js": "^1.9.14",
-    "command-line-args": "^5.2.1",
-    "command-line-usage": "^7.0.1",
-    "fs-extra": "^11.2.0",
-    "google-protobuf": "^3.21.2",
-    "ix": "^5.0.0",
-    "long": "^5.2.3",
-    "nice-grpc": "^2.1.7",
-    "nice-grpc-client-middleware-retry": "^3.1.6",
-    "nice-grpc-common": "^2.0.2",
-    "nice-grpc-error-details": "^0.2.4",
-    "nice-grpc-prometheus": "^0.2.2",
-    "prom-client": "^14.2.0",
-    "protobufjs": "^7.2.6",
-    "rxjs": "^7.8.1",
-    "utility-types": "^3.11.0",
-    "winston": "^3.11.0",
-    "@sentio/protos": "2.39.7-rc.8"
-  },
+  "dependencies": {},
   "devDependencies": {
     "@types/command-line-args": "^5.2.3",
     "@types/command-line-usage": "^5.0.4",
     "@types/fs-extra": "^11.0.4",
     "@types/google-protobuf": "^3.15.12",
-    "jest": "^29.7.0",
-    "ts-node": "^10.9.2",
     "tslib": "^2.6.2"
   },
   "engines": {
     "node": ">=16"
   },
   "scripts": {
-    "build": "pnpm compile",
+    "build": "pnpm bundle",
     "build:all": "pnpm --filter=$(node -p \"require('./package.json').name\")... build",
+    "bundle": "tsup --config src/tsup.config.ts",
     "compile": "tsc",
     "run": "tsx src/processor-runner.ts --log-format=json",
     "run-benchmark": "tsx src/decode-benchmark.ts",
     "start_js": "tsx ./lib/processor-runner.js $PWD/../../debug/dist/lib.js",
     "start_ts": "tsx ./lib/processor-runner.js --log-format=json $PWD/../../debug/src/processor.ts",
-    "test": "NODE_OPTIONS=--experimental-vm-modules pnpm jest"
+    "test": "glob -c 'tsx --test' '**/*.test.ts'"
   }
 }

package/src/db-context.ts

@@ -1,28 +1,94 @@
 import { Subject } from 'rxjs'
-import { DBRequest, DBResponse, DeepPartial, ProcessResult, ProcessStreamResponse } from '@sentio/protos'
+import {
+  DBRequest,
+  DBRequest_DBUpsert,
+  DBResponse,
+  DeepPartial,
+  ProcessResult,
+  ProcessStreamResponse
+} from '@sentio/protos'
+import * as process from 'node:process'
+import { Attributes, Counter, metrics } from '@opentelemetry/api'
+
+const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
+const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
 
 type Request = Omit<DBRequest, 'opId'>
+type RequestType = keyof Request
+
+const meter = metrics.getMeter('processor_store')
+const send_counts: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_count'),
+  upsert: meter.createCounter('store_upsert_count'),
+  list: meter.createCounter('store_list_count'),
+  delete: meter.createCounter('store_delete_count')
+}
+const recv_counts: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_count'),
+  upsert: meter.createCounter('store_upsert_count'),
+  list: meter.createCounter('store_list_count'),
+  delete: meter.createCounter('store_delete_count')
+}
+const request_times: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_time'),
+  upsert: meter.createCounter('store_upsert_time'),
+  list: meter.createCounter('store_list_time'),
+  delete: meter.createCounter('store_delete_time')
+}
+const request_errors: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_error'),
+  upsert: meter.createCounter('store_upsert_error'),
+  list: meter.createCounter('store_list_error'),
+  delete: meter.createCounter('store_delete_error')
+}
+
+const batched_total_count = meter.createCounter('batched_total_count')
+const batched_request_count = meter.createCounter('batched_request_count')
+
+const unsolved_requests = meter.createGauge('store_unsolved_requests')
+
+export const timeoutError = Symbol()
 
 export class StoreContext {
   private static opCounter = 0n
 
-  private defers = new Map<bigint, { resolve: (value: any) => void; reject: (reason?: any) => void }>()
+  private defers = new Map<
+    bigint,
+    { resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
+  >()
+  private statsInterval: NodeJS.Timeout | undefined
 
   constructor(
     readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
     readonly processId: number
   ) {}
 
-  newPromise<T>(opId: bigint) {
+  newPromise<T>(opId: bigint, requestType?: RequestType) {
     return new Promise<T>((resolve, reject) => {
-      this.defers.set(opId, { resolve, reject })
+      this.defers.set(opId, { resolve, reject, requestType })
+      unsolved_requests.record(this.defers.size, { processId: this.processId })
     })
   }
 
-  sendRequest(request: DeepPartial<Request>) {
+  sendRequest(request: DeepPartial<Request>, timeoutSecs?: number): Promise<DBResponse> {
+    if (STORE_BATCH_IDLE > 0 && STORE_BATCH_SIZE > 1 && request.upsert) {
+      // batch upsert if possible
+      return this.sendUpsertInBatch(request.upsert as DBRequest_DBUpsert)
+    }
+
+    const requestType = Object.keys(request)[0] as RequestType
     const opId = StoreContext.opCounter++
-    const promise = this.newPromise(opId)
+    const promise = this.newPromise(opId, requestType)
+
+    const start = Date.now()
+    const promises = [promise]
     console.debug('sending db request ', opId, request)
+    let timer: NodeJS.Timeout | undefined
+    if (timeoutSecs) {
+      const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
+      promises.push(timeoutPromise)
+    }
+
     this.subject.next({
       dbRequest: {
         ...request,
@@ -30,7 +96,27 @@ export class StoreContext {
       },
       processId: this.processId
     })
-    return promise
+
+    send_counts[requestType]?.add(1)
+
+    return Promise.race(promises)
+      .then((result: DBResponse) => {
+        console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
+        request_times[requestType]?.add(Date.now() - start)
+        return result
+      })
+      .catch((e) => {
+        if (e === timeoutError) {
+          console.error('db request', requestType, 'op:', opId, ' timeout')
+        }
+        request_errors[requestType]?.add(1)
+        throw e
+      })
+      .finally(() => {
+        if (timer) {
+          clearTimeout(timer)
+        }
+      })
   }
 
   result(dbResult: DBResponse) {
@@ -38,6 +124,9 @@ export class StoreContext {
     const defer = this.defers.get(opId)
     console.debug('received db result ', opId, dbResult)
     if (defer) {
+      if (defer.requestType) {
+        recv_counts[defer.requestType]?.add(1)
+      }
       if (dbResult.error) {
         defer.reject(new Error(dbResult.error))
       } else {
@@ -45,6 +134,7 @@ export class StoreContext {
       }
       this.defers.delete(opId)
     }
+    unsolved_requests.record(this.defers.size, { processId: this.processId })
   }
 
   error(processId: number, e: any) {
@@ -59,4 +149,72 @@ export class StoreContext {
       processId
     })
   }
+
+  close() {
+    for (const [opId, defer] of this.defers) {
+      console.warn('context closed before db response', opId)
+      defer.reject(new Error('context closed'))
+    }
+    this.defers.clear()
+    if (this.statsInterval) {
+      clearInterval(this.statsInterval)
+    }
+  }
+
+  upsertBatch:
+    | {
+        opId: bigint
+        request: DBRequest_DBUpsert
+        promise: Promise<DBResponse>
+        timer: NodeJS.Timeout
+      }
+    | undefined = undefined
+
+  private async sendUpsertInBatch(req: DBRequest_DBUpsert): Promise<DBResponse> {
+    if (this.upsertBatch) {
+      // merge the upserts
+      const { request, promise } = this.upsertBatch
+      request.entity = this.upsertBatch.request.entity.concat(req.entity)
+      request.entityData = this.upsertBatch.request.entityData.concat(req.entityData)
+      request.id = this.upsertBatch.request.id.concat(req.id)
+      if (request.entity.length >= STORE_BATCH_SIZE) {
+        this.sendBatch()
+      }
+      return promise
+    } else {
+      const opId = StoreContext.opCounter++
+      const promise = this.newPromise<DBResponse>(opId, 'upsert')
+      const timeout = setTimeout(() => {
+        this.sendBatch()
+      }, STORE_BATCH_IDLE)
+
+      this.upsertBatch = {
+        opId,
+        request: req,
+        promise,
+        timer: timeout
+      }
+
+      return promise
+    }
+  }
+
+  private sendBatch() {
+    if (this.upsertBatch) {
+      const { request, opId, timer } = this.upsertBatch
+      console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
+      clearTimeout(timer)
+      this.upsertBatch = undefined
+      this.subject.next({
+        dbRequest: {
+          upsert: request,
+          opId
+        },
+        processId: this.processId
+      })
+      send_counts['upsert']?.add(1)
+      batched_request_count.add(1)
+      batched_total_count.add(request.entity.length)
+    }
+  }
 }

package/src/gen/processor/protos/processor.ts

@@ -3,7 +3,7 @@ import Long from "long";
 import type { CallContext, CallOptions } from "nice-grpc-common";
 import _m0 from "protobufjs/minimal.js";
 import { Empty } from "../../google/protobuf/empty.js";
-import { ListValue, Struct } from "../../google/protobuf/struct.js";
+import { ListValue, Struct, Value } from "../../google/protobuf/struct.js";
 import { Timestamp } from "../../google/protobuf/timestamp.js";
 import { BigInteger, CoinID, RichStruct, RichStructList, RichValueList } from "../../service/common/protos/common.js";
 
@@ -1030,6 +1030,27 @@ export interface ProcessResult {
   states: StateResult | undefined;
 }
 
+export interface EthCallParam {
+  chainId: string;
+  address: string;
+  function: string;
+  signature: string;
+  args: any[];
+}
+
+export interface PreprocessResult {
+  ethCallParams: EthCallParam[];
+}
+
+export interface PreparedData {
+  ethCallResults: { [key: string]: Array<any> | undefined };
+}
+
+export interface PreparedData_EthCallResultsEntry {
+  key: string;
+  value: Array<any> | undefined;
+}
+
 export interface RecordMetaData {
   address: string;
   contractName: string;
@@ -8786,6 +8807,345 @@ export const ProcessResult = {
   },
 };
 
+function createBaseEthCallParam(): EthCallParam {
+  return { chainId: "", address: "", function: "", signature: "", args: [] };
+}
+
+export const EthCallParam = {
+  encode(message: EthCallParam, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.chainId !== "") {
+      writer.uint32(10).string(message.chainId);
+    }
+    if (message.address !== "") {
+      writer.uint32(18).string(message.address);
+    }
+    if (message.function !== "") {
+      writer.uint32(26).string(message.function);
+    }
+    if (message.signature !== "") {
+      writer.uint32(34).string(message.signature);
+    }
+    for (const v of message.args) {
+      Value.encode(Value.wrap(v!), writer.uint32(42).fork()).ldelim();
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): EthCallParam {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseEthCallParam();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.chainId = reader.string();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.address = reader.string();
+          continue;
+        case 3:
+          if (tag !== 26) {
+            break;
+          }
+
+          message.function = reader.string();
+          continue;
+        case 4:
+          if (tag !== 34) {
+            break;
+          }
+
+          message.signature = reader.string();
+          continue;
+        case 5:
+          if (tag !== 42) {
+            break;
+          }
+
+          message.args.push(Value.unwrap(Value.decode(reader, reader.uint32())));
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): EthCallParam {
+    return {
+      chainId: isSet(object.chainId) ? globalThis.String(object.chainId) : "",
+      address: isSet(object.address) ? globalThis.String(object.address) : "",
+      function: isSet(object.function) ? globalThis.String(object.function) : "",
+      signature: isSet(object.signature) ? globalThis.String(object.signature) : "",
+      args: globalThis.Array.isArray(object?.args) ? [...object.args] : [],
+    };
+  },
+
+  toJSON(message: EthCallParam): unknown {
+    const obj: any = {};
+    if (message.chainId !== "") {
+      obj.chainId = message.chainId;
+    }
+    if (message.address !== "") {
+      obj.address = message.address;
+    }
+    if (message.function !== "") {
+      obj.function = message.function;
+    }
+    if (message.signature !== "") {
+      obj.signature = message.signature;
+    }
+    if (message.args?.length) {
+      obj.args = message.args;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<EthCallParam>): EthCallParam {
+    return EthCallParam.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<EthCallParam>): EthCallParam {
+    const message = createBaseEthCallParam();
+    message.chainId = object.chainId ?? "";
+    message.address = object.address ?? "";
+    message.function = object.function ?? "";
+    message.signature = object.signature ?? "";
+    message.args = object.args?.map((e) => e) || [];
+    return message;
+  },
+};
+
+function createBasePreprocessResult(): PreprocessResult {
+  return { ethCallParams: [] };
+}
+
+export const PreprocessResult = {
+  encode(message: PreprocessResult, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    for (const v of message.ethCallParams) {
+      EthCallParam.encode(v!, writer.uint32(10).fork()).ldelim();
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): PreprocessResult {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBasePreprocessResult();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.ethCallParams.push(EthCallParam.decode(reader, reader.uint32()));
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): PreprocessResult {
+    return {
+      ethCallParams: globalThis.Array.isArray(object?.ethCallParams)
+        ? object.ethCallParams.map((e: any) => EthCallParam.fromJSON(e))
+        : [],
+    };
+  },
+
+  toJSON(message: PreprocessResult): unknown {
+    const obj: any = {};
+    if (message.ethCallParams?.length) {
+      obj.ethCallParams = message.ethCallParams.map((e) => EthCallParam.toJSON(e));
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<PreprocessResult>): PreprocessResult {
+    return PreprocessResult.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<PreprocessResult>): PreprocessResult {
+    const message = createBasePreprocessResult();
+    message.ethCallParams = object.ethCallParams?.map((e) => EthCallParam.fromPartial(e)) || [];
+    return message;
+  },
+};
+
+function createBasePreparedData(): PreparedData {
+  return { ethCallResults: {} };
+}
+
+export const PreparedData = {
+  encode(message: PreparedData, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    Object.entries(message.ethCallResults).forEach(([key, value]) => {
+      if (value !== undefined) {
+        PreparedData_EthCallResultsEntry.encode({ key: key as any, value }, writer.uint32(10).fork()).ldelim();
+      }
+    });
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): PreparedData {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBasePreparedData();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          const entry1 = PreparedData_EthCallResultsEntry.decode(reader, reader.uint32());
+          if (entry1.value !== undefined) {
+            message.ethCallResults[entry1.key] = entry1.value;
+          }
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): PreparedData {
+    return {
+      ethCallResults: isObject(object.ethCallResults)
+        ? Object.entries(object.ethCallResults).reduce<{ [key: string]: Array<any> | undefined }>(
+          (acc, [key, value]) => {
+            acc[key] = value as Array<any> | undefined;
+            return acc;
+          },
+          {},
+        )
+        : {},
+    };
+  },
+
+  toJSON(message: PreparedData): unknown {
+    const obj: any = {};
+    if (message.ethCallResults) {
+      const entries = Object.entries(message.ethCallResults);
+      if (entries.length > 0) {
+        obj.ethCallResults = {};
+        entries.forEach(([k, v]) => {
+          obj.ethCallResults[k] = v;
+        });
+      }
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<PreparedData>): PreparedData {
+    return PreparedData.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<PreparedData>): PreparedData {
+    const message = createBasePreparedData();
+    message.ethCallResults = Object.entries(object.ethCallResults ?? {}).reduce<
+      { [key: string]: Array<any> | undefined }
+    >((acc, [key, value]) => {
+      if (value !== undefined) {
+        acc[key] = value;
+      }
+      return acc;
+    }, {});
+    return message;
+  },
+};
+
+function createBasePreparedData_EthCallResultsEntry(): PreparedData_EthCallResultsEntry {
+  return { key: "", value: undefined };
+}
+
+export const PreparedData_EthCallResultsEntry = {
+  encode(message: PreparedData_EthCallResultsEntry, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.key !== "") {
+      writer.uint32(10).string(message.key);
+    }
+    if (message.value !== undefined) {
+      ListValue.encode(ListValue.wrap(message.value), writer.uint32(18).fork()).ldelim();
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): PreparedData_EthCallResultsEntry {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBasePreparedData_EthCallResultsEntry();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.key = reader.string();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.value = ListValue.unwrap(ListValue.decode(reader, reader.uint32()));
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): PreparedData_EthCallResultsEntry {
+    return {
+      key: isSet(object.key) ? globalThis.String(object.key) : "",
+      value: globalThis.Array.isArray(object.value) ? [...object.value] : undefined,
+    };
+  },
+
+  toJSON(message: PreparedData_EthCallResultsEntry): unknown {
+    const obj: any = {};
+    if (message.key !== "") {
+      obj.key = message.key;
+    }
+    if (message.value !== undefined) {
+      obj.value = message.value;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<PreparedData_EthCallResultsEntry>): PreparedData_EthCallResultsEntry {
+    return PreparedData_EthCallResultsEntry.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<PreparedData_EthCallResultsEntry>): PreparedData_EthCallResultsEntry {
+    const message = createBasePreparedData_EthCallResultsEntry();
+    message.key = object.key ?? "";
+    message.value = object.value ?? undefined;
+    return message;
+  },
+};
+
 function createBaseRecordMetaData(): RecordMetaData {
   return {
     address: "",