@sentio/runtime 2.39.7-rc.9 → 2.40.0-rc.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.39.7-rc.9",
+  "version": "2.40.0-rc.2",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {
@@ -14,46 +14,26 @@
     "!**/*.test.{js,ts}",
     "!{lib,src}/tests"
   ],
-  "dependencies": {
-    "@grpc/grpc-js": "^1.9.14",
-    "command-line-args": "^5.2.1",
-    "command-line-usage": "^7.0.1",
-    "fs-extra": "^11.2.0",
-    "google-protobuf": "^3.21.2",
-    "ix": "^5.0.0",
-    "long": "^5.2.3",
-    "nice-grpc": "^2.1.7",
-    "nice-grpc-client-middleware-retry": "^3.1.6",
-    "nice-grpc-common": "^2.0.2",
-    "nice-grpc-error-details": "^0.2.4",
-    "nice-grpc-prometheus": "^0.2.2",
-    "prom-client": "^14.2.0",
-    "protobufjs": "^7.2.6",
-    "rxjs": "^7.8.1",
-    "utility-types": "^3.11.0",
-    "winston": "^3.11.0",
-    "@sentio/protos": "2.39.7-rc.9"
-  },
+  "dependencies": {},
   "devDependencies": {
     "@types/command-line-args": "^5.2.3",
     "@types/command-line-usage": "^5.0.4",
     "@types/fs-extra": "^11.0.4",
     "@types/google-protobuf": "^3.15.12",
-    "jest": "^29.7.0",
-    "ts-node": "^10.9.2",
     "tslib": "^2.6.2"
   },
   "engines": {
     "node": ">=16"
   },
   "scripts": {
-    "build": "pnpm compile",
+    "build": "pnpm bundle",
     "build:all": "pnpm --filter=$(node -p \"require('./package.json').name\")... build",
+    "bundle": "tsup --config src/tsup.config.ts",
     "compile": "tsc",
     "run": "tsx src/processor-runner.ts --log-format=json",
     "run-benchmark": "tsx src/decode-benchmark.ts",
     "start_js": "tsx ./lib/processor-runner.js $PWD/../../debug/dist/lib.js",
     "start_ts": "tsx ./lib/processor-runner.js --log-format=json $PWD/../../debug/src/processor.ts",
-    "test": "NODE_OPTIONS=--experimental-vm-modules pnpm jest"
+    "test": "glob -c 'tsx --test' '**/*.test.ts'"
   }
 }

package/src/db-context.ts

@@ -1,28 +1,94 @@
 import { Subject } from 'rxjs'
-import { DBRequest, DBResponse, DeepPartial, ProcessResult, ProcessStreamResponse } from '@sentio/protos'
+import {
+  DBRequest,
+  DBRequest_DBUpsert,
+  DBResponse,
+  DeepPartial,
+  ProcessResult,
+  ProcessStreamResponse
+} from '@sentio/protos'
+import * as process from 'node:process'
+import { Attributes, Counter, metrics } from '@opentelemetry/api'
+
+const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
+const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
 
 type Request = Omit<DBRequest, 'opId'>
+type RequestType = keyof Request
+
+const meter = metrics.getMeter('processor_store')
+const send_counts: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_count'),
+  upsert: meter.createCounter('store_upsert_count'),
+  list: meter.createCounter('store_list_count'),
+  delete: meter.createCounter('store_delete_count')
+}
+const recv_counts: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_count'),
+  upsert: meter.createCounter('store_upsert_count'),
+  list: meter.createCounter('store_list_count'),
+  delete: meter.createCounter('store_delete_count')
+}
+const request_times: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_time'),
+  upsert: meter.createCounter('store_upsert_time'),
+  list: meter.createCounter('store_list_time'),
+  delete: meter.createCounter('store_delete_time')
+}
+const request_errors: Record<RequestType, Counter<Attributes>> = {
+  get: meter.createCounter('store_get_error'),
+  upsert: meter.createCounter('store_upsert_error'),
+  list: meter.createCounter('store_list_error'),
+  delete: meter.createCounter('store_delete_error')
+}
+
+const batched_total_count = meter.createCounter('batched_total_count')
+const batched_request_count = meter.createCounter('batched_request_count')
+
+const unsolved_requests = meter.createGauge('store_unsolved_requests')
+
+export const timeoutError = Symbol()
 
 export class StoreContext {
   private static opCounter = 0n
 
-  private defers = new Map<bigint, { resolve: (value: any) => void; reject: (reason?: any) => void }>()
+  private defers = new Map<
+    bigint,
+    { resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
+  >()
+  private statsInterval: NodeJS.Timeout | undefined
 
   constructor(
     readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
     readonly processId: number
   ) {}
 
-  newPromise<T>(opId: bigint) {
+  newPromise<T>(opId: bigint, requestType?: RequestType) {
     return new Promise<T>((resolve, reject) => {
-      this.defers.set(opId, { resolve, reject })
+      this.defers.set(opId, { resolve, reject, requestType })
+      unsolved_requests.record(this.defers.size, { processId: this.processId })
     })
   }
 
-  sendRequest(request: DeepPartial<Request>) {
+  sendRequest(request: DeepPartial<Request>, timeoutSecs?: number): Promise<DBResponse> {
+    if (STORE_BATCH_IDLE > 0 && STORE_BATCH_SIZE > 1 && request.upsert) {
+      // batch upsert if possible
+      return this.sendUpsertInBatch(request.upsert as DBRequest_DBUpsert)
+    }
+
+    const requestType = Object.keys(request)[0] as RequestType
     const opId = StoreContext.opCounter++
-    const promise = this.newPromise(opId)
+    const promise = this.newPromise(opId, requestType)
+
+    const start = Date.now()
+    const promises = [promise]
     console.debug('sending db request ', opId, request)
+    let timer: NodeJS.Timeout | undefined
+    if (timeoutSecs) {
+      const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
+      promises.push(timeoutPromise)
+    }
+
     this.subject.next({
       dbRequest: {
         ...request,
@@ -30,7 +96,27 @@ export class StoreContext {
       },
       processId: this.processId
     })
-    return promise
+
+    send_counts[requestType]?.add(1)
+
+    return Promise.race(promises)
+      .then((result: DBResponse) => {
+        console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
+        request_times[requestType]?.add(Date.now() - start)
+        return result
+      })
+      .catch((e) => {
+        if (e === timeoutError) {
+          console.error('db request', requestType, 'op:', opId, ' timeout')
+        }
+        request_errors[requestType]?.add(1)
+        throw e
+      })
+      .finally(() => {
+        if (timer) {
+          clearTimeout(timer)
+        }
+      })
   }
 
   result(dbResult: DBResponse) {
@@ -38,6 +124,9 @@ export class StoreContext {
     const defer = this.defers.get(opId)
     console.debug('received db result ', opId, dbResult)
     if (defer) {
+      if (defer.requestType) {
+        recv_counts[defer.requestType]?.add(1)
+      }
       if (dbResult.error) {
         defer.reject(new Error(dbResult.error))
       } else {
@@ -45,6 +134,7 @@ export class StoreContext {
       }
       this.defers.delete(opId)
     }
+    unsolved_requests.record(this.defers.size, { processId: this.processId })
   }
 
   error(processId: number, e: any) {
@@ -66,5 +156,65 @@ export class StoreContext {
       defer.reject(new Error('context closed'))
     }
     this.defers.clear()
+    if (this.statsInterval) {
+      clearInterval(this.statsInterval)
+    }
+  }
+
+  upsertBatch:
+    | {
+        opId: bigint
+        request: DBRequest_DBUpsert
+        promise: Promise<DBResponse>
+        timer: NodeJS.Timeout
+      }
+    | undefined = undefined
+
+  private async sendUpsertInBatch(req: DBRequest_DBUpsert): Promise<DBResponse> {
+    if (this.upsertBatch) {
+      // merge the upserts
+      const { request, promise } = this.upsertBatch
+      request.entity = this.upsertBatch.request.entity.concat(req.entity)
+      request.entityData = this.upsertBatch.request.entityData.concat(req.entityData)
+      request.id = this.upsertBatch.request.id.concat(req.id)
+      if (request.entity.length >= STORE_BATCH_SIZE) {
+        this.sendBatch()
+      }
+      return promise
+    } else {
+      const opId = StoreContext.opCounter++
+      const promise = this.newPromise<DBResponse>(opId, 'upsert')
+      const timeout = setTimeout(() => {
+        this.sendBatch()
+      }, STORE_BATCH_IDLE)
+
+      this.upsertBatch = {
+        opId,
+        request: req,
+        promise,
+        timer: timeout
+      }
+
+      return promise
+    }
+  }
+
+  private sendBatch() {
+    if (this.upsertBatch) {
+      const { request, opId, timer } = this.upsertBatch
+      console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
+      clearTimeout(timer)
+      this.upsertBatch = undefined
+      this.subject.next({
+        dbRequest: {
+          upsert: request,
+          opId
+        },
+        processId: this.processId
+      })
+      send_counts['upsert']?.add(1)
+      batched_request_count.add(1)
+      batched_total_count.add(request.entity.length)
+    }
   }
 }

package/src/full-service.ts

@@ -6,6 +6,7 @@ const require = createRequire(import.meta.url)
 import {
   DataBinding,
   HandlerType,
+  PreprocessStreamRequest,
   ProcessBindingsRequest,
   ProcessStreamRequest,
   ProcessConfigRequest,
@@ -111,6 +112,10 @@ export class FullProcessorServiceImpl implements ProcessorServiceImplementation
     yield* this.instance.processBindingsStream(requests, context)
   }
 
+  async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
+    yield* this.instance.preprocessBindingsStream(requests, context)
+  }
+
   private adjustResult(res: ProcessResult): void {}
 
   private adjustDataBinding(dataBinding: DataBinding): void {