@sentio/runtime 2.40.0-rc.9 → 2.40.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.40.0-rc.9",
+  "version": "2.40.0",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {
@@ -26,7 +26,7 @@
     "node": ">=16"
   },
   "scripts": {
-    "build": "pnpm bundle",
+    "build": "pnpm tsc --noEmit && pnpm bundle",
     "build:all": "pnpm --filter=$(node -p \"require('./package.json').name\")... build",
     "bundle": "tsup --config src/tsup.config.ts",
     "compile": "tsc",

package/src/db-context.ts

@@ -8,55 +8,33 @@ import {
   ProcessStreamResponse
 } from '@sentio/protos'
 import * as process from 'node:process'
-import { Attributes, Counter, metrics } from '@opentelemetry/api'
-
+import { dbMetrics } from './metrics.js'
+const {
+  request_errors,
+  unsolved_requests,
+  request_times,
+  batched_request_count,
+  batched_total_count,
+  send_counts,
+  recv_counts
+} = dbMetrics
 const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
 const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
+const STORE_UPSERT_NO_WAIT = process.env['STORE_UPSERT_NO_WAIT'] === 'true'
 
 type Request = Omit<DBRequest, 'opId'>
 type RequestType = keyof Request
 
-const meter = metrics.getMeter('processor_store')
-const send_counts: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_count'),
-  upsert: meter.createCounter('store_upsert_count'),
-  list: meter.createCounter('store_list_count'),
-  delete: meter.createCounter('store_delete_count')
-}
-const recv_counts: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_count'),
-  upsert: meter.createCounter('store_upsert_count'),
-  list: meter.createCounter('store_list_count'),
-  delete: meter.createCounter('store_delete_count')
-}
-const request_times: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_time'),
-  upsert: meter.createCounter('store_upsert_time'),
-  list: meter.createCounter('store_list_time'),
-  delete: meter.createCounter('store_delete_time')
-}
-const request_errors: Record<RequestType, Counter<Attributes>> = {
-  get: meter.createCounter('store_get_error'),
-  upsert: meter.createCounter('store_upsert_error'),
-  list: meter.createCounter('store_list_error'),
-  delete: meter.createCounter('store_delete_error')
-}
-
-const batched_total_count = meter.createCounter('batched_total_count')
-const batched_request_count = meter.createCounter('batched_request_count')
-
-const unsolved_requests = meter.createGauge('store_unsolved_requests')
-
-export const timeoutError = Symbol()
+export const timeoutError = new Error('timeout')
 
 export class StoreContext {
   private static opCounter = 0n
-
   private defers = new Map<
     bigint,
     { resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
   >()
   private statsInterval: NodeJS.Timeout | undefined
+  private pendings: Promise<unknown>[] = []
 
   constructor(
     readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
@@ -82,7 +60,7 @@ export class StoreContext {
 
     const start = Date.now()
     const promises = [promise]
-    console.debug('sending db request ', opId, request)
+    // console.debug('sending db request ', opId, request)
     let timer: NodeJS.Timeout | undefined
     if (timeoutSecs) {
       const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
@@ -99,9 +77,18 @@ export class StoreContext {
 
     send_counts[requestType]?.add(1)
 
+    if (requestType === 'upsert' && STORE_UPSERT_NO_WAIT) {
+      this.pendings.push(promise)
+      return Promise.resolve({
+        opId,
+      } as DBResponse)
+    }
+
     return Promise.race(promises)
       .then((result: DBResponse) => {
-        console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
+        if (timeoutSecs) {
+          console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
+        }
         request_times[requestType]?.add(Date.now() - start)
         return result
       })
@@ -122,7 +109,7 @@ export class StoreContext {
   result(dbResult: DBResponse) {
     const opId = dbResult.opId
     const defer = this.defers.get(opId)
-    console.debug('received db result ', opId, dbResult)
+    // console.debug('received db result ', opId, dbResult)
     if (defer) {
       if (defer.requestType) {
         recv_counts[defer.requestType]?.add(1)
@@ -152,8 +139,8 @@ export class StoreContext {
 
   close() {
     for (const [opId, defer] of this.defers) {
-      console.warn('context closed before db response', opId)
-      defer.reject(new Error('context closed'))
+      // console.warn('context closed before db response', opId)
+      defer.reject(new Error('context closed before db response, processId: ' + this.processId + ' opId: ' + opId))
     }
     this.defers.clear()
     if (this.statsInterval) {
@@ -180,29 +167,45 @@ export class StoreContext {
       if (request.entity.length >= STORE_BATCH_SIZE) {
         this.sendBatch()
       }
+      if (STORE_UPSERT_NO_WAIT) {
+        return {
+          opId: this.upsertBatch.opId
+        }
+      }
+
       return promise
     } else {
       const opId = StoreContext.opCounter++
-      const promise = this.newPromise<DBResponse>(opId, 'upsert')
       const timeout = setTimeout(() => {
         this.sendBatch()
       }, STORE_BATCH_IDLE)
+      const start = Date.now()
+      const promise = this.newPromise<DBResponse>(opId, 'upsert').finally(() => {
+        request_times['upsert'].add(Date.now() - start)
+      })
 
       this.upsertBatch = {
         opId,
         request: req,
         promise,
-        timer: timeout
+        timer: timeout,
       }
 
-      return promise
+      if (STORE_UPSERT_NO_WAIT) {
+        this.pendings.push(promise)
+        return {
+          opId: this.upsertBatch.opId
+        }
+      } else {
+        return promise
+      }
     }
   }
 
   private sendBatch() {
     if (this.upsertBatch) {
       const { request, opId, timer } = this.upsertBatch
-      console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
+      // console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
       clearTimeout(timer)
       this.upsertBatch = undefined
       this.subject.next({
@@ -217,4 +220,8 @@ export class StoreContext {
       batched_total_count.add(request.entity.length)
     }
   }
+
+  async awaitPendings() {
+    await Promise.all(this.pendings)
+  }
 }

package/src/index.ts

@@ -6,3 +6,4 @@ export * from './chain-config.js'
 export * from './service.js'
 export { GLOBAL_CONFIG, type GlobalConfig } from './global-config.js'
 export * from './db-context.js'
+export * from './provider.js'

package/src/metrics.ts

@@ -0,0 +1,138 @@
+import { Attributes, Counter, metrics, Gauge } from '@opentelemetry/api'
+
+const meter = metrics.getMeter('processor')
+
+class C {
+  private counter: Counter<Attributes>
+  private value: number = 0
+
+  constructor(name: string) {
+    this.counter = meter.createCounter(name)
+  }
+
+  add(value: number, attributes?: Attributes) {
+    this.counter.add(value, attributes)
+    this.value += value
+  }
+
+  get() {
+    return this.value
+  }
+}
+
+class G {
+  private gauge: Gauge<Attributes>
+  private value: number = 0
+
+  constructor(name: string) {
+    this.gauge = meter.createGauge(name)
+  }
+
+  record(value: number, attributes?: Attributes) {
+    this.gauge.record(value, attributes)
+    this.value = value
+  }
+
+  get() {
+    return this.value
+  }
+}
+
+export const dbMetrics = {
+  send_counts: {
+    get: new C('store_get_send'),
+    upsert: new C('store_upsert_send'),
+    list: new C('store_list_send'),
+    delete: new C('store_delete_send')
+  },
+  recv_counts: {
+    get: new C('store_get_recv'),
+    upsert: new C('store_upsert_recv'),
+    list: new C('store_list_recv'),
+    delete: new C('store_delete_recv')
+  },
+  request_times: {
+    get: new C('store_get_time'),
+    upsert: new C('store_upsert_time'),
+    list: new C('store_list_time'),
+    delete: new C('store_delete_time')
+  },
+  request_errors: {
+    get: new C('store_get_error'),
+    upsert: new C('store_upsert_error'),
+    list: new C('store_list_error'),
+    delete: new C('store_delete_error')
+  },
+  batched_total_count: new C('batched_total_count'),
+  batched_request_count: new C('batched_request_count'),
+  unsolved_requests: new G('store_unsolved_requests'),
+
+  stats() {
+    return {
+      send_counts: {
+        get: this.send_counts.get.get(),
+        upsert: this.send_counts.upsert.get(),
+        list: this.send_counts.list.get(),
+        delete: this.send_counts.delete.get()
+      },
+      recv_counts: {
+        get: this.recv_counts.get.get(),
+        upsert: this.recv_counts.upsert.get(),
+        list: this.recv_counts.list.get(),
+        delete: this.recv_counts.delete.get()
+      },
+      request_times: {
+        get: this.request_times.get.get(),
+        upsert: this.request_times.upsert.get(),
+        list: this.request_times.list.get(),
+        delete: this.request_times.delete.get()
+      },
+      request_errors: {
+        get: this.request_errors.get.get(),
+        upsert: this.request_errors.upsert.get(),
+        list: this.request_errors.list.get(),
+        delete: this.request_errors.delete.get()
+      },
+      batched_total_count: this.batched_total_count.get(),
+      batched_request_count: this.batched_request_count.get(),
+      unsolved_requests: this.unsolved_requests.get(),
+      average_request_time: {
+        get: this.request_times.get.get() / this.send_counts.get.get(),
+        upsert: this.request_times.upsert.get() / this.send_counts.upsert.get(),
+        list: this.request_times.list.get() / this.send_counts.list.get()
+      }
+    }
+  }
+}
+
+export const providerMetrics = {
+  hit_count: new C('provider_hit_count'),
+  miss_count: new C('provider_miss_count'),
+  queue_size: new G('provider_queue_size'),
+  total_duration: new C('provider_total_duration'),
+  total_queued: new C('provider_total_queued'),
+  stats() {
+    return {
+      hit_count: this.hit_count.get(),
+      miss_count: this.miss_count.get(),
+      queue_size: this.queue_size.get(),
+      total_duration: this.total_duration.get(),
+      total_queued: this.total_queued.get(),
+      average_queue_time: this.total_queued.get() / (this.hit_count.get() + this.miss_count.get()),
+      average_duration: this.total_duration.get() / (this.hit_count.get() + this.miss_count.get())
+    }
+  }
+}
+
+export const processMetrics = {
+  process_binding_count: new C('process_binding_count'),
+  process_binding_time: new C('process_binding_time'),
+  process_binding_error: new C('process_binding_error'),
+  stats() {
+    return {
+      process_binding_count: this.process_binding_count.get(),
+      process_binding_time: this.process_binding_time.get(),
+      process_binding_error: this.process_binding_error.get()
+    }
+  }
+}