@sentio/runtime 2.39.7-rc.3 → 2.39.7-rc.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/lib/chunk-FIFFMJK7.js +58496 -0
  2. package/lib/index.d.ts +302 -9
  3. package/lib/index.js +92 -9
  4. package/lib/processor-runner.d.ts +0 -2
  5. package/lib/processor-runner.js +51457 -129
  6. package/package.json +5 -25
  7. package/src/db-context.ts +165 -7
  8. package/src/gen/processor/protos/processor.ts +41 -1
  9. package/src/processor-runner.ts +26 -2
  10. package/src/service.ts +61 -24
  11. package/src/tsup.config.ts +14 -0
  12. package/lib/chain-config.d.ts +0 -6
  13. package/lib/chain-config.d.ts.map +0 -1
  14. package/lib/chain-config.js +0 -2
  15. package/lib/chain-config.js.map +0 -1
  16. package/lib/db-context.d.ts +0 -16
  17. package/lib/db-context.d.ts.map +0 -1
  18. package/lib/db-context.js +0 -56
  19. package/lib/db-context.js.map +0 -1
  20. package/lib/decode-benchmark.d.ts +0 -3
  21. package/lib/decode-benchmark.d.ts.map +0 -1
  22. package/lib/decode-benchmark.js +0 -20
  23. package/lib/decode-benchmark.js.map +0 -1
  24. package/lib/endpoints.d.ts +0 -9
  25. package/lib/endpoints.d.ts.map +0 -1
  26. package/lib/endpoints.js +0 -9
  27. package/lib/endpoints.js.map +0 -1
  28. package/lib/full-service.d.ts +0 -654
  29. package/lib/full-service.d.ts.map +0 -1
  30. package/lib/full-service.js +0 -137
  31. package/lib/full-service.js.map +0 -1
  32. package/lib/gen/google/protobuf/empty.d.ts +0 -17
  33. package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
  34. package/lib/gen/google/protobuf/empty.js +0 -40
  35. package/lib/gen/google/protobuf/empty.js.map +0 -1
  36. package/lib/gen/google/protobuf/struct.d.ts +0 -77
  37. package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
  38. package/lib/gen/google/protobuf/struct.js +0 -429
  39. package/lib/gen/google/protobuf/struct.js.map +0 -1
  40. package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
  41. package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
  42. package/lib/gen/google/protobuf/timestamp.js +0 -83
  43. package/lib/gen/google/protobuf/timestamp.js.map +0 -1
  44. package/lib/gen/processor/protos/processor.d.ts +0 -1465
  45. package/lib/gen/processor/protos/processor.d.ts.map +0 -1
  46. package/lib/gen/processor/protos/processor.js +0 -8474
  47. package/lib/gen/processor/protos/processor.js.map +0 -1
  48. package/lib/gen/service/common/protos/common.d.ts +0 -1698
  49. package/lib/gen/service/common/protos/common.d.ts.map +0 -1
  50. package/lib/gen/service/common/protos/common.js +0 -11383
  51. package/lib/gen/service/common/protos/common.js.map +0 -1
  52. package/lib/global-config.d.ts +0 -8
  53. package/lib/global-config.d.ts.map +0 -1
  54. package/lib/global-config.js +0 -23
  55. package/lib/global-config.js.map +0 -1
  56. package/lib/global-config.test.d.ts +0 -2
  57. package/lib/global-config.test.d.ts.map +0 -1
  58. package/lib/global-config.test.js.map +0 -1
  59. package/lib/index.d.ts.map +0 -1
  60. package/lib/index.js.map +0 -1
  61. package/lib/logger.d.ts +0 -2
  62. package/lib/logger.d.ts.map +0 -1
  63. package/lib/logger.js +0 -39
  64. package/lib/logger.js.map +0 -1
  65. package/lib/logger.test.d.ts +0 -2
  66. package/lib/logger.test.d.ts.map +0 -1
  67. package/lib/logger.test.js.map +0 -1
  68. package/lib/plugin.d.ts +0 -29
  69. package/lib/plugin.d.ts.map +0 -1
  70. package/lib/plugin.js +0 -58
  71. package/lib/plugin.js.map +0 -1
  72. package/lib/processor-runner.d.ts.map +0 -1
  73. package/lib/processor-runner.js.map +0 -1
  74. package/lib/seq-mode.test.d.ts +0 -3
  75. package/lib/seq-mode.test.d.ts.map +0 -1
  76. package/lib/seq-mode.test.js.map +0 -1
  77. package/lib/service.d.ts +0 -178
  78. package/lib/service.d.ts.map +0 -1
  79. package/lib/service.js +0 -179
  80. package/lib/service.js.map +0 -1
  81. package/lib/service.test.d.ts +0 -3
  82. package/lib/service.test.d.ts.map +0 -1
  83. package/lib/service.test.js.map +0 -1
  84. package/lib/state-storage.test.d.ts +0 -2
  85. package/lib/state-storage.test.d.ts.map +0 -1
  86. package/lib/state-storage.test.js.map +0 -1
  87. package/lib/state.d.ts +0 -23
  88. package/lib/state.d.ts.map +0 -1
  89. package/lib/state.js +0 -61
  90. package/lib/state.js.map +0 -1
  91. package/lib/utils.d.ts +0 -6
  92. package/lib/utils.d.ts.map +0 -1
  93. package/lib/utils.js +0 -23
  94. package/lib/utils.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sentio/runtime",
3
- "version": "2.39.7-rc.3",
3
+ "version": "2.39.7-rc.30",
4
4
  "license": "Apache-2.0",
5
5
  "type": "module",
6
6
  "exports": {
@@ -14,46 +14,26 @@
14
14
  "!**/*.test.{js,ts}",
15
15
  "!{lib,src}/tests"
16
16
  ],
17
- "dependencies": {
18
- "@grpc/grpc-js": "^1.9.14",
19
- "command-line-args": "^5.2.1",
20
- "command-line-usage": "^7.0.1",
21
- "fs-extra": "^11.2.0",
22
- "google-protobuf": "^3.21.2",
23
- "ix": "^5.0.0",
24
- "long": "^5.2.3",
25
- "nice-grpc": "^2.1.7",
26
- "nice-grpc-client-middleware-retry": "^3.1.6",
27
- "nice-grpc-common": "^2.0.2",
28
- "nice-grpc-error-details": "^0.2.4",
29
- "nice-grpc-prometheus": "^0.2.2",
30
- "prom-client": "^14.2.0",
31
- "protobufjs": "^7.2.6",
32
- "rxjs": "^7.8.1",
33
- "utility-types": "^3.11.0",
34
- "winston": "^3.11.0",
35
- "@sentio/protos": "2.39.7-rc.3"
36
- },
17
+ "dependencies": {},
37
18
  "devDependencies": {
38
19
  "@types/command-line-args": "^5.2.3",
39
20
  "@types/command-line-usage": "^5.0.4",
40
21
  "@types/fs-extra": "^11.0.4",
41
22
  "@types/google-protobuf": "^3.15.12",
42
- "jest": "^29.7.0",
43
- "ts-node": "^10.9.2",
44
23
  "tslib": "^2.6.2"
45
24
  },
46
25
  "engines": {
47
26
  "node": ">=16"
48
27
  },
49
28
  "scripts": {
50
- "build": "pnpm compile",
29
+ "build": "pnpm bundle",
51
30
  "build:all": "pnpm --filter=$(node -p \"require('./package.json').name\")... build",
31
+ "bundle": "tsup --config src/tsup.config.ts",
52
32
  "compile": "tsc",
53
33
  "run": "tsx src/processor-runner.ts --log-format=json",
54
34
  "run-benchmark": "tsx src/decode-benchmark.ts",
55
35
  "start_js": "tsx ./lib/processor-runner.js $PWD/../../debug/dist/lib.js",
56
36
  "start_ts": "tsx ./lib/processor-runner.js --log-format=json $PWD/../../debug/src/processor.ts",
57
- "test": "NODE_OPTIONS=--experimental-vm-modules pnpm jest"
37
+ "test": "glob -c 'tsx --test' '**/*.test.ts'"
58
38
  }
59
39
  }
package/src/db-context.ts CHANGED
@@ -1,28 +1,94 @@
1
1
  import { Subject } from 'rxjs'
2
- import { DBRequest, DBResponse, DeepPartial, ProcessResult, ProcessStreamResponse } from '@sentio/protos'
2
+ import {
3
+ DBRequest,
4
+ DBRequest_DBUpsert,
5
+ DBResponse,
6
+ DeepPartial,
7
+ ProcessResult,
8
+ ProcessStreamResponse
9
+ } from '@sentio/protos'
10
+ import * as process from 'node:process'
11
+ import { Attributes, Counter, metrics } from '@opentelemetry/api'
12
+
13
+ const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
14
+ const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
3
15
 
4
16
  type Request = Omit<DBRequest, 'opId'>
17
+ type RequestType = keyof Request
18
+
19
+ const meter = metrics.getMeter('processor_store')
20
+ const send_counts: Record<RequestType, Counter<Attributes>> = {
21
+ get: meter.createCounter('store_get_count'),
22
+ upsert: meter.createCounter('store_upsert_count'),
23
+ list: meter.createCounter('store_list_count'),
24
+ delete: meter.createCounter('store_delete_count')
25
+ }
26
+ const recv_counts: Record<RequestType, Counter<Attributes>> = {
27
+ get: meter.createCounter('store_get_count'),
28
+ upsert: meter.createCounter('store_upsert_count'),
29
+ list: meter.createCounter('store_list_count'),
30
+ delete: meter.createCounter('store_delete_count')
31
+ }
32
+ const request_times: Record<RequestType, Counter<Attributes>> = {
33
+ get: meter.createCounter('store_get_time'),
34
+ upsert: meter.createCounter('store_upsert_time'),
35
+ list: meter.createCounter('store_list_time'),
36
+ delete: meter.createCounter('store_delete_time')
37
+ }
38
+ const request_errors: Record<RequestType, Counter<Attributes>> = {
39
+ get: meter.createCounter('store_get_error'),
40
+ upsert: meter.createCounter('store_upsert_error'),
41
+ list: meter.createCounter('store_list_error'),
42
+ delete: meter.createCounter('store_delete_error')
43
+ }
44
+
45
+ const batched_total_count = meter.createCounter('batched_total_count')
46
+ const batched_request_count = meter.createCounter('batched_request_count')
47
+
48
+ const unsolved_requests = meter.createGauge('store_unsolved_requests')
49
+
50
+ export const timeoutError = Symbol()
5
51
 
6
52
  export class StoreContext {
7
53
  private static opCounter = 0n
8
54
 
9
- private defers = new Map<bigint, { resolve: (value: any) => void; reject: (reason?: any) => void }>()
55
+ private defers = new Map<
56
+ bigint,
57
+ { resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
58
+ >()
59
+ private statsInterval: NodeJS.Timeout | undefined
10
60
 
11
61
  constructor(
12
62
  readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
13
63
  readonly processId: number
14
64
  ) {}
15
65
 
16
- newPromise<T>(opId: bigint) {
66
+ newPromise<T>(opId: bigint, requestType?: RequestType) {
17
67
  return new Promise<T>((resolve, reject) => {
18
- this.defers.set(opId, { resolve, reject })
68
+ this.defers.set(opId, { resolve, reject, requestType })
69
+ unsolved_requests.record(this.defers.size, { processId: this.processId })
19
70
  })
20
71
  }
21
72
 
22
- sendRequest(request: DeepPartial<Request>) {
73
+ sendRequest(request: DeepPartial<Request>, timeoutSecs?: number): Promise<DBResponse> {
74
+ if (STORE_BATCH_IDLE > 0 && STORE_BATCH_SIZE > 1 && request.upsert) {
75
+ // batch upsert if possible
76
+ return this.sendUpsertInBatch(request.upsert as DBRequest_DBUpsert)
77
+ }
78
+
79
+ const requestType = Object.keys(request)[0] as RequestType
23
80
  const opId = StoreContext.opCounter++
24
- const promise = this.newPromise(opId)
81
+ const promise = this.newPromise(opId, requestType)
82
+
83
+ const start = Date.now()
84
+ const promises = [promise]
25
85
  console.debug('sending db request ', opId, request)
86
+ let timer: NodeJS.Timeout | undefined
87
+ if (timeoutSecs) {
88
+ const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
89
+ promises.push(timeoutPromise)
90
+ }
91
+
26
92
  this.subject.next({
27
93
  dbRequest: {
28
94
  ...request,
@@ -30,7 +96,27 @@ export class StoreContext {
30
96
  },
31
97
  processId: this.processId
32
98
  })
33
- return promise
99
+
100
+ send_counts[requestType]?.add(1)
101
+
102
+ return Promise.race(promises)
103
+ .then((result: DBResponse) => {
104
+ console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
105
+ request_times[requestType]?.add(Date.now() - start)
106
+ return result
107
+ })
108
+ .catch((e) => {
109
+ if (e === timeoutError) {
110
+ console.error('db request', requestType, 'op:', opId, ' timeout')
111
+ }
112
+ request_errors[requestType]?.add(1)
113
+ throw e
114
+ })
115
+ .finally(() => {
116
+ if (timer) {
117
+ clearTimeout(timer)
118
+ }
119
+ })
34
120
  }
35
121
 
36
122
  result(dbResult: DBResponse) {
@@ -38,6 +124,9 @@ export class StoreContext {
38
124
  const defer = this.defers.get(opId)
39
125
  console.debug('received db result ', opId, dbResult)
40
126
  if (defer) {
127
+ if (defer.requestType) {
128
+ recv_counts[defer.requestType]?.add(1)
129
+ }
41
130
  if (dbResult.error) {
42
131
  defer.reject(new Error(dbResult.error))
43
132
  } else {
@@ -45,6 +134,7 @@ export class StoreContext {
45
134
  }
46
135
  this.defers.delete(opId)
47
136
  }
137
+ unsolved_requests.record(this.defers.size, { processId: this.processId })
48
138
  }
49
139
 
50
140
  error(processId: number, e: any) {
@@ -59,4 +149,72 @@ export class StoreContext {
59
149
  processId
60
150
  })
61
151
  }
152
+
153
+ close() {
154
+ for (const [opId, defer] of this.defers) {
155
+ console.warn('context closed before db response', opId)
156
+ defer.reject(new Error('context closed'))
157
+ }
158
+ this.defers.clear()
159
+ if (this.statsInterval) {
160
+ clearInterval(this.statsInterval)
161
+ }
162
+ }
163
+
164
+ upsertBatch:
165
+ | {
166
+ opId: bigint
167
+ request: DBRequest_DBUpsert
168
+ promise: Promise<DBResponse>
169
+ timer: NodeJS.Timeout
170
+ }
171
+ | undefined = undefined
172
+
173
+ private async sendUpsertInBatch(req: DBRequest_DBUpsert): Promise<DBResponse> {
174
+ if (this.upsertBatch) {
175
+ // merge the upserts
176
+ const { request, promise } = this.upsertBatch
177
+ request.entity = this.upsertBatch.request.entity.concat(req.entity)
178
+ request.entityData = this.upsertBatch.request.entityData.concat(req.entityData)
179
+ request.id = this.upsertBatch.request.id.concat(req.id)
180
+ if (request.entity.length >= STORE_BATCH_SIZE) {
181
+ this.sendBatch()
182
+ }
183
+ return promise
184
+ } else {
185
+ const opId = StoreContext.opCounter++
186
+ const promise = this.newPromise<DBResponse>(opId, 'upsert')
187
+ const timeout = setTimeout(() => {
188
+ this.sendBatch()
189
+ }, STORE_BATCH_IDLE)
190
+
191
+ this.upsertBatch = {
192
+ opId,
193
+ request: req,
194
+ promise,
195
+ timer: timeout
196
+ }
197
+
198
+ return promise
199
+ }
200
+ }
201
+
202
+ private sendBatch() {
203
+ if (this.upsertBatch) {
204
+ const { request, opId, timer } = this.upsertBatch
205
+ console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
206
+ clearTimeout(timer)
207
+ this.upsertBatch = undefined
208
+ this.subject.next({
209
+ dbRequest: {
210
+ upsert: request,
211
+ opId
212
+ },
213
+ processId: this.processId
214
+ })
215
+ send_counts['upsert']?.add(1)
216
+ batched_request_count.add(1)
217
+ batched_total_count.add(request.entity.length)
218
+ }
219
+ }
62
220
  }
@@ -790,6 +790,10 @@ export enum DBRequest_DBOperator {
790
790
  LE = 5,
791
791
  IN = 6,
792
792
  NOT_IN = 7,
793
+ LIKE = 8,
794
+ NOT_LIKE = 9,
795
+ HAS_ALL = 10,
796
+ HAS_ANY = 11,
793
797
  UNRECOGNIZED = -1,
794
798
  }
795
799
 
@@ -819,6 +823,18 @@ export function dBRequest_DBOperatorFromJSON(object: any): DBRequest_DBOperator
819
823
  case 7:
820
824
  case "NOT_IN":
821
825
  return DBRequest_DBOperator.NOT_IN;
826
+ case 8:
827
+ case "LIKE":
828
+ return DBRequest_DBOperator.LIKE;
829
+ case 9:
830
+ case "NOT_LIKE":
831
+ return DBRequest_DBOperator.NOT_LIKE;
832
+ case 10:
833
+ case "HAS_ALL":
834
+ return DBRequest_DBOperator.HAS_ALL;
835
+ case 11:
836
+ case "HAS_ANY":
837
+ return DBRequest_DBOperator.HAS_ANY;
822
838
  case -1:
823
839
  case "UNRECOGNIZED":
824
840
  default:
@@ -844,6 +860,14 @@ export function dBRequest_DBOperatorToJSON(object: DBRequest_DBOperator): string
844
860
  return "IN";
845
861
  case DBRequest_DBOperator.NOT_IN:
846
862
  return "NOT_IN";
863
+ case DBRequest_DBOperator.LIKE:
864
+ return "LIKE";
865
+ case DBRequest_DBOperator.NOT_LIKE:
866
+ return "NOT_LIKE";
867
+ case DBRequest_DBOperator.HAS_ALL:
868
+ return "HAS_ALL";
869
+ case DBRequest_DBOperator.HAS_ANY:
870
+ return "HAS_ANY";
847
871
  case DBRequest_DBOperator.UNRECOGNIZED:
848
872
  default:
849
873
  return "UNRECOGNIZED";
@@ -859,6 +883,7 @@ export interface DBRequest_DBList {
859
883
  entity: string;
860
884
  filters: DBRequest_DBFilter[];
861
885
  cursor: string;
886
+ pageSize?: number | undefined;
862
887
  }
863
888
 
864
889
  export interface DBRequest_DBUpsert {
@@ -6442,7 +6467,7 @@ export const DBRequest_DBGet = {
6442
6467
  };
6443
6468
 
6444
6469
  function createBaseDBRequest_DBList(): DBRequest_DBList {
6445
- return { entity: "", filters: [], cursor: "" };
6470
+ return { entity: "", filters: [], cursor: "", pageSize: undefined };
6446
6471
  }
6447
6472
 
6448
6473
  export const DBRequest_DBList = {
@@ -6456,6 +6481,9 @@ export const DBRequest_DBList = {
6456
6481
  if (message.cursor !== "") {
6457
6482
  writer.uint32(42).string(message.cursor);
6458
6483
  }
6484
+ if (message.pageSize !== undefined) {
6485
+ writer.uint32(48).uint32(message.pageSize);
6486
+ }
6459
6487
  return writer;
6460
6488
  },
6461
6489
 
@@ -6487,6 +6515,13 @@ export const DBRequest_DBList = {
6487
6515
 
6488
6516
  message.cursor = reader.string();
6489
6517
  continue;
6518
+ case 6:
6519
+ if (tag !== 48) {
6520
+ break;
6521
+ }
6522
+
6523
+ message.pageSize = reader.uint32();
6524
+ continue;
6490
6525
  }
6491
6526
  if ((tag & 7) === 4 || tag === 0) {
6492
6527
  break;
@@ -6503,6 +6538,7 @@ export const DBRequest_DBList = {
6503
6538
  ? object.filters.map((e: any) => DBRequest_DBFilter.fromJSON(e))
6504
6539
  : [],
6505
6540
  cursor: isSet(object.cursor) ? globalThis.String(object.cursor) : "",
6541
+ pageSize: isSet(object.pageSize) ? globalThis.Number(object.pageSize) : undefined,
6506
6542
  };
6507
6543
  },
6508
6544
 
@@ -6517,6 +6553,9 @@ export const DBRequest_DBList = {
6517
6553
  if (message.cursor !== "") {
6518
6554
  obj.cursor = message.cursor;
6519
6555
  }
6556
+ if (message.pageSize !== undefined) {
6557
+ obj.pageSize = Math.round(message.pageSize);
6558
+ }
6520
6559
  return obj;
6521
6560
  },
6522
6561
 
@@ -6528,6 +6567,7 @@ export const DBRequest_DBList = {
6528
6567
  message.entity = object.entity ?? "";
6529
6568
  message.filters = object.filters?.map((e) => DBRequest_DBFilter.fromPartial(e)) || [];
6530
6569
  message.cursor = object.cursor ?? "";
6570
+ message.pageSize = object.pageSize ?? undefined;
6531
6571
  return message;
6532
6572
  },
6533
6573
  };
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
8
8
  import { createServer } from 'nice-grpc'
9
9
  import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
10
10
  import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
11
+ import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
11
12
  import { register as globalRegistry, Registry } from 'prom-client'
12
13
  import http from 'http'
13
14
  // @ts-ignore inspector promises is not included in @type/node
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
20
21
  import { ChainConfig } from './chain-config.js'
21
22
  import { setupLogger } from './logger.js'
22
23
 
24
+ import { NodeSDK } from '@opentelemetry/sdk-node'
25
+ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
26
+ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
27
+ import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
28
+
29
+ const sdk = new NodeSDK({
30
+ traceExporter: new OTLPTraceExporter(),
31
+ metricReader: new PeriodicExportingMetricReader({
32
+ exporter: new OTLPMetricExporter()
33
+ })
34
+ // instrumentations: [getNodeAutoInstrumentations()],
35
+ })
36
+
37
+ sdk.start()
38
+
23
39
  const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
24
40
 
25
41
  const optionDefinitions = [
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
50
66
  const fullPath = path.resolve(options['chains-config'])
51
67
  const chainsConfig = fs.readJsonSync(fullPath)
52
68
 
53
- Endpoints.INSTANCE.concurrency = options.concurrency
54
- Endpoints.INSTANCE.batchCount = options['batch-count']
69
+ const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
70
+ ? parseInt(process.env['OVERRIDE_CONCURRENCY'])
71
+ : undefined
72
+ const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
73
+ ? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
74
+ : undefined
75
+
76
+ Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
77
+ Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
55
78
  Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
56
79
  Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
57
80
 
@@ -77,6 +100,7 @@ const server = createServer({
77
100
  'grpc.default_compression_algorithm': compressionAlgorithms.gzip
78
101
  })
79
102
  .use(prometheusServerMiddleware())
103
+ .use(openTelemetryServerMiddleware())
80
104
  .use(errorDetailsServerMiddleware)
81
105
  const baseService = new ProcessorServiceImpl(async () => {
82
106
  const m = await import(options.target)
package/src/service.ts CHANGED
@@ -25,6 +25,13 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
25
25
 
26
26
  import { StoreContext } from './db-context.js'
27
27
  import { Subject } from 'rxjs'
28
+ import { metrics } from '@opentelemetry/api'
29
+
30
+ const meter = metrics.getMeter('processor_service')
31
+ const process_binding_count = meter.createCounter('process_binding_count')
32
+ const process_binding_time = meter.createCounter('process_binding_time')
33
+ const process_binding_error = meter.createCounter('process_binding_error')
34
+
28
35
  ;(BigInt.prototype as any).toJSON = function () {
29
36
  return this.toString()
30
37
  }
@@ -181,33 +188,43 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
181
188
  requests: AsyncIterable<ProcessStreamRequest>,
182
189
  subject: Subject<DeepPartial<ProcessStreamResponse>>
183
190
  ) {
184
- const contexts: Record<number, StoreContext> = {}
191
+ const contexts = new Contexts()
185
192
 
186
193
  for await (const request of requests) {
187
- console.debug('received request:', request)
188
- if (request.binding) {
189
- const binding = request.binding
190
- const dbContext = new StoreContext(subject, request.processId)
191
- contexts[request.processId] = dbContext
192
- PluginManager.INSTANCE.processBinding(binding, dbContext)
193
- .then((result) => {
194
- subject.next({
195
- result,
196
- processId: request.processId
194
+ try {
195
+ console.debug('received request:', request)
196
+ if (request.binding) {
197
+ process_binding_count.add(1)
198
+ const binding = request.binding
199
+ const dbContext = contexts.new(request.processId, subject)
200
+ const start = Date.now()
201
+ PluginManager.INSTANCE.processBinding(binding, dbContext)
202
+ .then((result) => {
203
+ subject.next({
204
+ result,
205
+ processId: request.processId
206
+ })
207
+ recordRuntimeInfo(result, binding.handlerType)
197
208
  })
198
- recordRuntimeInfo(result, binding.handlerType)
199
- })
200
- .catch((e) => {
201
- console.error(e)
202
- dbContext.error(request.processId, e)
203
- })
204
- .finally(() => {
205
- delete contexts[request.processId]
206
- })
207
- }
208
- if (request.dbResult) {
209
- const dbContext = contexts[request.processId]
210
- dbContext?.result(request.dbResult)
209
+ .catch((e) => {
210
+ console.debug(e)
211
+ dbContext.error(request.processId, e)
212
+ process_binding_error.add(1)
213
+ })
214
+ .finally(() => {
215
+ const cost = Date.now() - start
216
+ console.debug('processBinding', request.processId, ' took', cost, 'ms')
217
+ process_binding_time.add(cost)
218
+ contexts.delete(request.processId)
219
+ })
220
+ }
221
+ if (request.dbResult) {
222
+ const dbContext = contexts.get(request.processId)
223
+ dbContext?.result(request.dbResult)
224
+ }
225
+ } catch (e) {
226
+ // should not happen
227
+ console.error('unexpect error during handle loop', e)
211
228
  }
212
229
  }
213
230
  }
@@ -222,3 +239,23 @@ function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
222
239
  })
223
240
  }
224
241
  }
242
+
243
+ class Contexts {
244
+ private contexts: Map<number, StoreContext> = new Map()
245
+
246
+ get(processId: number) {
247
+ return this.contexts.get(processId)
248
+ }
249
+
250
+ new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
251
+ const context = new StoreContext(subject, processId)
252
+ this.contexts.set(processId, context)
253
+ return context
254
+ }
255
+
256
+ delete(processId: number) {
257
+ const context = this.get(processId)
258
+ context?.close()
259
+ this.contexts.delete(processId)
260
+ }
261
+ }
@@ -0,0 +1,14 @@
1
+ import { defineConfig } from 'tsup'
2
+
3
+ export default defineConfig({
4
+ esbuildOptions: (options) => {
5
+ options.banner = {
6
+ js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url);`
7
+ }
8
+ },
9
+ entry: ['src/index.ts', 'src/processor-runner.ts'],
10
+ outDir: 'lib',
11
+ clean: true,
12
+ dts: true,
13
+ format: 'esm'
14
+ })
@@ -1,6 +0,0 @@
1
- export interface ChainConfig {
2
- ChainID: string;
3
- Https?: string[];
4
- ChainServer?: string;
5
- }
6
- //# sourceMappingURL=chain-config.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=chain-config.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
@@ -1,16 +0,0 @@
1
- import { Subject } from 'rxjs';
2
- import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
3
- type Request = Omit<DBRequest, 'opId'>;
4
- export declare class StoreContext {
5
- readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
6
- readonly processId: number;
7
- private static opCounter;
8
- private defers;
9
- constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
10
- newPromise<T>(opId: bigint): Promise<T>;
11
- sendRequest(request: DeepPartial<Request>): Promise<unknown>;
12
- result(dbResult: DBResponse): void;
13
- error(processId: number, e: any): void;
14
- }
15
- export {};
16
- //# sourceMappingURL=db-context.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;CAYhC"}
package/lib/db-context.js DELETED
@@ -1,56 +0,0 @@
1
- import { ProcessResult } from '@sentio/protos';
2
- export class StoreContext {
3
- subject;
4
- processId;
5
- static opCounter = 0n;
6
- defers = new Map();
7
- constructor(subject, processId) {
8
- this.subject = subject;
9
- this.processId = processId;
10
- }
11
- newPromise(opId) {
12
- return new Promise((resolve, reject) => {
13
- this.defers.set(opId, { resolve, reject });
14
- });
15
- }
16
- sendRequest(request) {
17
- const opId = StoreContext.opCounter++;
18
- const promise = this.newPromise(opId);
19
- console.debug('sending db request ', opId, request);
20
- this.subject.next({
21
- dbRequest: {
22
- ...request,
23
- opId
24
- },
25
- processId: this.processId
26
- });
27
- return promise;
28
- }
29
- result(dbResult) {
30
- const opId = dbResult.opId;
31
- const defer = this.defers.get(opId);
32
- console.debug('received db result ', opId, dbResult);
33
- if (defer) {
34
- if (dbResult.error) {
35
- defer.reject(new Error(dbResult.error));
36
- }
37
- else {
38
- defer.resolve(dbResult);
39
- }
40
- this.defers.delete(opId);
41
- }
42
- }
43
- error(processId, e) {
44
- console.error('process error', processId, e);
45
- const errorResult = ProcessResult.create({
46
- states: {
47
- error: e?.toString()
48
- }
49
- });
50
- this.subject.next({
51
- result: errorResult,
52
- processId
53
- });
54
- }
55
- }
56
- //# sourceMappingURL=db-context.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAAsC,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAIzG,MAAM,OAAO,YAAY;IAMZ;IACA;IANH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAA6E,CAAA;IAErG,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY;QACxB,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC"}
@@ -1,3 +0,0 @@
1
- #!/usr/bin/env node
2
- export {};
3
- //# sourceMappingURL=decode-benchmark.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"decode-benchmark.d.ts","sourceRoot":"","sources":["../src/decode-benchmark.ts"],"names":[],"mappings":""}