@sentio/runtime 2.39.7-rc.9 → 2.40.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/lib/chunk-WDKQZPD5.js +78804 -0
  2. package/lib/index.d.ts +369 -9
  3. package/lib/index.js +92 -9
  4. package/lib/processor-runner.d.ts +0 -2
  5. package/lib/processor-runner.js +51460 -129
  6. package/package.json +5 -25
  7. package/src/db-context.ts +157 -7
  8. package/src/full-service.ts +5 -0
  9. package/src/gen/processor/protos/processor.ts +995 -90
  10. package/src/plugin.ts +30 -4
  11. package/src/processor-runner.ts +26 -2
  12. package/src/provider.ts +166 -0
  13. package/src/service.ts +197 -27
  14. package/src/tsup.config.ts +14 -0
  15. package/lib/chain-config.d.ts +0 -6
  16. package/lib/chain-config.d.ts.map +0 -1
  17. package/lib/chain-config.js +0 -2
  18. package/lib/chain-config.js.map +0 -1
  19. package/lib/db-context.d.ts +0 -17
  20. package/lib/db-context.d.ts.map +0 -1
  21. package/lib/db-context.js +0 -63
  22. package/lib/db-context.js.map +0 -1
  23. package/lib/decode-benchmark.d.ts +0 -3
  24. package/lib/decode-benchmark.d.ts.map +0 -1
  25. package/lib/decode-benchmark.js +0 -20
  26. package/lib/decode-benchmark.js.map +0 -1
  27. package/lib/endpoints.d.ts +0 -9
  28. package/lib/endpoints.d.ts.map +0 -1
  29. package/lib/endpoints.js +0 -9
  30. package/lib/endpoints.js.map +0 -1
  31. package/lib/full-service.d.ts +0 -655
  32. package/lib/full-service.d.ts.map +0 -1
  33. package/lib/full-service.js +0 -137
  34. package/lib/full-service.js.map +0 -1
  35. package/lib/gen/google/protobuf/empty.d.ts +0 -17
  36. package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
  37. package/lib/gen/google/protobuf/empty.js +0 -40
  38. package/lib/gen/google/protobuf/empty.js.map +0 -1
  39. package/lib/gen/google/protobuf/struct.d.ts +0 -77
  40. package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
  41. package/lib/gen/google/protobuf/struct.js +0 -429
  42. package/lib/gen/google/protobuf/struct.js.map +0 -1
  43. package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
  44. package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
  45. package/lib/gen/google/protobuf/timestamp.js +0 -83
  46. package/lib/gen/google/protobuf/timestamp.js.map +0 -1
  47. package/lib/gen/processor/protos/processor.d.ts +0 -1470
  48. package/lib/gen/processor/protos/processor.d.ts.map +0 -1
  49. package/lib/gen/processor/protos/processor.js +0 -8512
  50. package/lib/gen/processor/protos/processor.js.map +0 -1
  51. package/lib/gen/service/common/protos/common.d.ts +0 -1698
  52. package/lib/gen/service/common/protos/common.d.ts.map +0 -1
  53. package/lib/gen/service/common/protos/common.js +0 -11383
  54. package/lib/gen/service/common/protos/common.js.map +0 -1
  55. package/lib/global-config.d.ts +0 -8
  56. package/lib/global-config.d.ts.map +0 -1
  57. package/lib/global-config.js +0 -23
  58. package/lib/global-config.js.map +0 -1
  59. package/lib/global-config.test.d.ts +0 -2
  60. package/lib/global-config.test.d.ts.map +0 -1
  61. package/lib/global-config.test.js.map +0 -1
  62. package/lib/index.d.ts.map +0 -1
  63. package/lib/index.js.map +0 -1
  64. package/lib/logger.d.ts +0 -2
  65. package/lib/logger.d.ts.map +0 -1
  66. package/lib/logger.js +0 -39
  67. package/lib/logger.js.map +0 -1
  68. package/lib/logger.test.d.ts +0 -2
  69. package/lib/logger.test.d.ts.map +0 -1
  70. package/lib/logger.test.js.map +0 -1
  71. package/lib/plugin.d.ts +0 -29
  72. package/lib/plugin.d.ts.map +0 -1
  73. package/lib/plugin.js +0 -58
  74. package/lib/plugin.js.map +0 -1
  75. package/lib/processor-runner.d.ts.map +0 -1
  76. package/lib/processor-runner.js.map +0 -1
  77. package/lib/seq-mode.test.d.ts +0 -3
  78. package/lib/seq-mode.test.d.ts.map +0 -1
  79. package/lib/seq-mode.test.js.map +0 -1
  80. package/lib/service.d.ts +0 -179
  81. package/lib/service.d.ts.map +0 -1
  82. package/lib/service.js +0 -194
  83. package/lib/service.js.map +0 -1
  84. package/lib/service.test.d.ts +0 -3
  85. package/lib/service.test.d.ts.map +0 -1
  86. package/lib/service.test.js.map +0 -1
  87. package/lib/state-storage.test.d.ts +0 -2
  88. package/lib/state-storage.test.d.ts.map +0 -1
  89. package/lib/state-storage.test.js.map +0 -1
  90. package/lib/state.d.ts +0 -23
  91. package/lib/state.d.ts.map +0 -1
  92. package/lib/state.js +0 -61
  93. package/lib/state.js.map +0 -1
  94. package/lib/utils.d.ts +0 -6
  95. package/lib/utils.d.ts.map +0 -1
  96. package/lib/utils.js +0 -23
  97. package/lib/utils.js.map +0 -1
package/src/plugin.ts CHANGED
@@ -1,4 +1,12 @@
1
- import { DataBinding, HandlerType, ProcessConfigResponse, ProcessResult, StartRequest } from '@sentio/protos'
1
+ import {
2
+ DataBinding,
3
+ HandlerType,
4
+ PreparedData,
5
+ PreprocessResult,
6
+ ProcessConfigResponse,
7
+ ProcessResult,
8
+ StartRequest
9
+ } from '@sentio/protos'
2
10
  import { StoreContext } from './db-context.js'
3
11
  import { AsyncLocalStorage } from 'node:async_hooks'
4
12
 
@@ -16,9 +24,13 @@ export abstract class Plugin {
16
24
  return false
17
25
  }
18
26
 
19
- async processBinding(request: DataBinding): Promise<ProcessResult> {
27
+ async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
20
28
  return ProcessResult.create()
21
29
  }
30
+
31
+ async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
32
+ return PreprocessResult.create()
33
+ }
22
34
  }
23
35
 
24
36
  export class PluginManager {
@@ -58,13 +70,27 @@ export class PluginManager {
58
70
  return this.plugins.some((plugin) => plugin.stateDiff(config))
59
71
  }
60
72
 
61
- processBinding(request: DataBinding, dbContext?: StoreContext): Promise<ProcessResult> {
73
+ processBinding(
74
+ request: DataBinding,
75
+ preparedData: PreparedData | undefined,
76
+ dbContext?: StoreContext
77
+ ): Promise<ProcessResult> {
78
+ const plugin = this.typesToPlugin.get(request.handlerType)
79
+ if (!plugin) {
80
+ throw new Error(`No plugin for ${request.handlerType}`)
81
+ }
82
+ return this.dbContextLocalStorage.run(dbContext, () => {
83
+ return plugin.processBinding(request, preparedData)
84
+ })
85
+ }
86
+
87
+ preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
62
88
  const plugin = this.typesToPlugin.get(request.handlerType)
63
89
  if (!plugin) {
64
90
  throw new Error(`No plugin for ${request.handlerType}`)
65
91
  }
66
92
  return this.dbContextLocalStorage.run(dbContext, () => {
67
- return plugin.processBinding(request)
93
+ return plugin.preprocessBinding(request)
68
94
  })
69
95
  }
70
96
  }
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
8
8
  import { createServer } from 'nice-grpc'
9
9
  import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
10
10
  import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
11
+ import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
11
12
  import { register as globalRegistry, Registry } from 'prom-client'
12
13
  import http from 'http'
13
14
  // @ts-ignore inspector promises is not included in @type/node
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
20
21
  import { ChainConfig } from './chain-config.js'
21
22
  import { setupLogger } from './logger.js'
22
23
 
24
+ import { NodeSDK } from '@opentelemetry/sdk-node'
25
+ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
26
+ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
27
+ import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
28
+
29
+ const sdk = new NodeSDK({
30
+ traceExporter: new OTLPTraceExporter(),
31
+ metricReader: new PeriodicExportingMetricReader({
32
+ exporter: new OTLPMetricExporter()
33
+ })
34
+ // instrumentations: [getNodeAutoInstrumentations()],
35
+ })
36
+
37
+ sdk.start()
38
+
23
39
  const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
24
40
 
25
41
  const optionDefinitions = [
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
50
66
  const fullPath = path.resolve(options['chains-config'])
51
67
  const chainsConfig = fs.readJsonSync(fullPath)
52
68
 
53
- Endpoints.INSTANCE.concurrency = options.concurrency
54
- Endpoints.INSTANCE.batchCount = options['batch-count']
69
+ const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
70
+ ? parseInt(process.env['OVERRIDE_CONCURRENCY'])
71
+ : undefined
72
+ const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
73
+ ? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
74
+ : undefined
75
+
76
+ Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
77
+ Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
55
78
  Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
56
79
  Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
57
80
 
@@ -77,6 +100,7 @@ const server = createServer({
77
100
  'grpc.default_compression_algorithm': compressionAlgorithms.gzip
78
101
  })
79
102
  .use(prometheusServerMiddleware())
103
+ .use(openTelemetryServerMiddleware())
80
104
  .use(errorDetailsServerMiddleware)
81
105
  const baseService = new ProcessorServiceImpl(async () => {
82
106
  const m = await import(options.target)
@@ -0,0 +1,166 @@
1
+ import { JsonRpcProvider, Network, Provider } from 'ethers'
2
+
3
+ import PQueue from 'p-queue'
4
+ import { Endpoints } from './endpoints.js'
5
+ import { EthChainId } from '@sentio/chain'
6
+ import { LRUCache } from 'lru-cache'
7
+ import { metrics } from '@opentelemetry/api'
8
+
9
+ export const DummyProvider = new JsonRpcProvider('', Network.from(1))
10
+
11
+ const meter = metrics.getMeter('processor_provider')
12
+ const hit_count = meter.createCounter('provider_hit_count')
13
+ const miss_count = meter.createCounter('provider_miss_count')
14
+ const queue_size = meter.createGauge('provider_queue_size')
15
+ const total_duration = meter.createCounter('provider_total_duration')
16
+ const total_queued = meter.createCounter('provider_total_queued')
17
+
18
+ const providers = new Map<string, JsonRpcProvider>()
19
+
20
+ // export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
21
+ // if (!networkish) {
22
+ // networkish = EthChainId.ETHEREUM
23
+ // }
24
+ // if (networkish instanceof BaseContext) {
25
+ // networkish = networkish.getChainId()
26
+ // }
27
+ // return networkish
28
+ // }
29
+
30
+ export function getProvider(chainId?: EthChainId): Provider {
31
+ // const network = getNetworkFromCtxOrNetworkish(networkish)
32
+ if (!chainId) {
33
+ chainId = EthChainId.ETHEREUM
34
+ }
35
+ const network = Network.from(parseInt(chainId))
36
+ // TODO check if other key needed
37
+
38
+ const address = Endpoints.INSTANCE.chainServer.get(chainId)
39
+ const key = network.chainId.toString() + '-' + address
40
+ let provider = providers.get(key)
41
+
42
+ if (provider) {
43
+ return provider
44
+ }
45
+
46
+ if (address === undefined) {
47
+ throw Error(
48
+ 'Provider not found for chain ' +
49
+ network.chainId +
50
+ ', configured chains: ' +
51
+ [...Endpoints.INSTANCE.chainServer.keys()].join(' ')
52
+ )
53
+ }
54
+ console.log(
55
+ `init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
56
+ )
57
+ provider = new QueuedStaticJsonRpcProvider(
58
+ address,
59
+ network,
60
+ Endpoints.INSTANCE.concurrency,
61
+ Endpoints.INSTANCE.batchCount
62
+ )
63
+ providers.set(key, provider)
64
+ return provider
65
+ }
66
+
67
+ function getTag(prefix: string, value: any): string {
68
+ return (
69
+ prefix +
70
+ ':' +
71
+ JSON.stringify(value, (k, v) => {
72
+ if (v == null) {
73
+ return 'null'
74
+ }
75
+ if (typeof v === 'bigint') {
76
+ return `bigint:${v.toString()}`
77
+ }
78
+ if (typeof v === 'string') {
79
+ return v.toLowerCase()
80
+ }
81
+
82
+ // Sort object keys
83
+ if (typeof v === 'object' && !Array.isArray(v)) {
84
+ const keys = Object.keys(v)
85
+ keys.sort()
86
+ return keys.reduce(
87
+ (accum, key) => {
88
+ accum[key] = v[key]
89
+ return accum
90
+ },
91
+ <any>{}
92
+ )
93
+ }
94
+
95
+ return v
96
+ })
97
+ )
98
+ }
99
+
100
+ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
101
+ executor: PQueue
102
+ #performCache = new LRUCache<string, Promise<any>>({
103
+ max: 300000 // 300k items
104
+ // maxSize: 300 * 1024 * 1024, // 300mb for cache
105
+ // ttl: 1000 * 60 * 60, // 1 hour no ttl for better performance
106
+ // sizeCalculation: (value: any) => {
107
+ // assume each item is 1kb for simplicity
108
+ // return 1024
109
+ // }
110
+ })
111
+
112
+ constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
113
+ // TODO re-enable match when possible
114
+ super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
115
+ this.executor = new PQueue({ concurrency: concurrency })
116
+ }
117
+
118
+ async send(method: string, params: Array<any>): Promise<any> {
119
+ if (method !== 'eth_call') {
120
+ return await this.executor.add(() => super.send(method, params))
121
+ }
122
+ const tag = getTag(method, params)
123
+ const block = params[params.length - 1]
124
+ let perform = this.#performCache.get(tag)
125
+ if (!perform) {
126
+ miss_count.add(1)
127
+ const queued: number = Date.now()
128
+ perform = this.executor.add(() => {
129
+ const started = Date.now()
130
+ total_queued.add(started - queued)
131
+
132
+ return super.send(method, params).finally(() => {
133
+ total_duration.add(Date.now() - started)
134
+ })
135
+ })
136
+ perform.catch((e) => {
137
+ // if (e.code !== 'CALL_EXCEPTION' && e.code !== 'BAD_DATA') {
138
+ setTimeout(() => {
139
+ if (this.#performCache.get(tag) === perform) {
140
+ this.#performCache.delete(tag)
141
+ }
142
+ }, 1000)
143
+ })
144
+
145
+ queue_size.record(this.executor.size)
146
+
147
+ this.#performCache.set(tag, perform)
148
+ // For non latest block call, we cache permanently, otherwise we cache for one minute
149
+ if (block === 'latest') {
150
+ setTimeout(() => {
151
+ if (this.#performCache.get(tag) === perform) {
152
+ this.#performCache.delete(tag)
153
+ }
154
+ }, 60 * 1000)
155
+ }
156
+ } else {
157
+ hit_count.add(1)
158
+ }
159
+
160
+ const result = await perform
161
+ if (!result) {
162
+ throw Error('Unexpected null response')
163
+ }
164
+ return result
165
+ }
166
+ }
package/src/service.ts CHANGED
@@ -7,7 +7,12 @@ import {
7
7
  DataBinding,
8
8
  DeepPartial,
9
9
  Empty,
10
+ EthCallParam,
10
11
  HandlerType,
12
+ PreparedData,
13
+ PreprocessResult,
14
+ PreprocessStreamRequest,
15
+ PreprocessStreamResponse,
11
16
  ProcessBindingResponse,
12
17
  ProcessBindingsRequest,
13
18
  ProcessConfigRequest,
@@ -25,6 +30,16 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
25
30
 
26
31
  import { StoreContext } from './db-context.js'
27
32
  import { Subject } from 'rxjs'
33
+ import { metrics } from '@opentelemetry/api'
34
+ import { getProvider } from './provider.js'
35
+ import { EthChainId } from '@sentio/chain'
36
+ import { Provider, Interface } from 'ethers'
37
+
38
+ const meter = metrics.getMeter('processor_service')
39
+ const process_binding_count = meter.createCounter('process_binding_count')
40
+ const process_binding_time = meter.createCounter('process_binding_time')
41
+ const process_binding_error = meter.createCounter('process_binding_error')
42
+
28
43
  ;(BigInt.prototype as any).toJSON = function () {
29
44
  return this.toString()
30
45
  }
@@ -111,10 +126,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
111
126
  }
112
127
 
113
128
  async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
114
- const promises = []
129
+ const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
115
130
 
131
+ const promises = []
116
132
  for (const binding of request.bindings) {
117
- const promise = this.processBinding(binding)
133
+ const promise = this.processBinding(binding, { ethCallResults })
118
134
  if (GLOBAL_CONFIG.execution.sequential) {
119
135
  await promise
120
136
  }
@@ -139,7 +155,92 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
139
155
  }
140
156
  }
141
157
 
142
- async processBinding(request: DataBinding, options?: CallContext): Promise<ProcessResult> {
158
+ async preprocessBindings(
159
+ bindings: DataBinding[],
160
+ dbContext?: StoreContext,
161
+ options?: CallContext
162
+ ): Promise<{ [calldata: string]: any[] }> {
163
+ console.log('preprocessBindings start')
164
+ const promises = []
165
+ for (const binding of bindings) {
166
+ promises.push(this.preprocessBinding(binding, dbContext, options))
167
+ }
168
+ let preprocessResults: PreprocessResult[]
169
+ try {
170
+ preprocessResults = await Promise.all(promises)
171
+ } catch (e) {
172
+ throw e
173
+ }
174
+ console.log(
175
+ 'ethCallParams: ',
176
+ preprocessResults.map((r) => r.ethCallParams)
177
+ )
178
+ const groupedRequests = new Map<string, EthCallParam[]>()
179
+ const providers = new Map<string, Provider>()
180
+ for (const result of preprocessResults) {
181
+ for (const param of result.ethCallParams) {
182
+ if (!providers.has(param.chainId)) {
183
+ providers.set(param.chainId, getProvider(param.chainId as EthChainId))
184
+ }
185
+ const key = param.chainId + '|' + param.address
186
+ if (!groupedRequests.has(key)) {
187
+ groupedRequests.set(key, [])
188
+ }
189
+ groupedRequests.get(key)!.push(param)
190
+ }
191
+ }
192
+
193
+ const start = Date.now()
194
+ const callPromises = []
195
+ for (const params of groupedRequests.values()) {
196
+ console.log(`chain: ${params[0].chainId}, address: ${params[0].address}, totalCalls: ${params.length}`)
197
+ for (const param of params) {
198
+ const frag = new Interface(param.signature)
199
+ const calldata = frag.encodeFunctionData(param.function, param.args)
200
+ callPromises.push(
201
+ providers
202
+ .get(param.chainId)!
203
+ .call({
204
+ to: param.address,
205
+ data: calldata
206
+ })
207
+ .then((ret) => [calldata, frag.decodeFunctionResult(param.function, ret).toArray()] as [string, any[]])
208
+ )
209
+ }
210
+ }
211
+ const results = Object.fromEntries(await Promise.all(callPromises))
212
+ console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
213
+ return results
214
+ }
215
+
216
+ async preprocessBinding(
217
+ request: DataBinding,
218
+ dbContext?: StoreContext,
219
+ options?: CallContext
220
+ ): Promise<PreprocessResult> {
221
+ if (!this.started) {
222
+ throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
223
+ }
224
+ if (this.unhandled) {
225
+ throw new RichServerError(
226
+ Status.UNAVAILABLE,
227
+ 'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
228
+ [
229
+ DebugInfo.fromPartial({
230
+ detail: this.unhandled.message,
231
+ stackEntries: this.unhandled.stack?.split('\n')
232
+ })
233
+ ]
234
+ )
235
+ }
236
+ return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
237
+ }
238
+
239
+ async processBinding(
240
+ request: DataBinding,
241
+ preparedData: PreparedData | undefined,
242
+ options?: CallContext
243
+ ): Promise<ProcessResult> {
143
244
  if (!this.started) {
144
245
  throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
145
246
  }
@@ -155,7 +256,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
155
256
  ]
156
257
  )
157
258
  }
158
- const result = await PluginManager.INSTANCE.processBinding(request)
259
+ const result = await PluginManager.INSTANCE.processBinding(request, preparedData)
159
260
  recordRuntimeInfo(result, request.handlerType)
160
261
  return result
161
262
  }
@@ -177,6 +278,65 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
177
278
  yield* from(subject).pipe(withAbort(context.signal))
178
279
  }
179
280
 
281
+ async handlePreprocessRequests(
282
+ requests: AsyncIterable<PreprocessStreamRequest>,
283
+ subject: Subject<DeepPartial<PreprocessStreamResponse>>
284
+ ) {
285
+ const contexts = new Contexts()
286
+
287
+ for await (const request of requests) {
288
+ try {
289
+ console.debug('received request:', request)
290
+ if (request.bindings) {
291
+ const bindings = request.bindings.bindings
292
+ const dbContext = contexts.new(request.processId, subject)
293
+ const start = Date.now()
294
+ this.preprocessBindings(bindings, dbContext)
295
+ .then(() => {
296
+ subject.next({
297
+ processId: request.processId
298
+ })
299
+ })
300
+ .catch((e) => {
301
+ console.debug(e)
302
+ dbContext.error(request.processId, e)
303
+ process_binding_error.add(1)
304
+ })
305
+ .finally(() => {
306
+ const cost = Date.now() - start
307
+ console.debug('preprocessBinding', request.processId, ' took', cost, 'ms')
308
+ process_binding_time.add(cost)
309
+ contexts.delete(request.processId)
310
+ })
311
+ }
312
+ if (request.dbResult) {
313
+ const dbContext = contexts.get(request.processId)
314
+ dbContext?.result(request.dbResult)
315
+ }
316
+ } catch (e) {
317
+ // should not happen
318
+ console.error('unexpect error during handle loop', e)
319
+ }
320
+ }
321
+ }
322
+
323
+ async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
324
+ if (!this.started) {
325
+ throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
326
+ }
327
+
328
+ const subject = new Subject<DeepPartial<PreprocessStreamResponse>>()
329
+ this.handlePreprocessRequests(requests, subject)
330
+ .then(() => {
331
+ subject.complete()
332
+ })
333
+ .catch((e) => {
334
+ console.error(e)
335
+ subject.error(e)
336
+ })
337
+ yield* from(subject).pipe(withAbort(context.signal))
338
+ }
339
+
180
340
  private async handleRequests(
181
341
  requests: AsyncIterable<ProcessStreamRequest>,
182
342
  subject: Subject<DeepPartial<ProcessStreamResponse>>
@@ -184,30 +344,40 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
184
344
  const contexts = new Contexts()
185
345
 
186
346
  for await (const request of requests) {
187
- console.debug('received request:', request)
188
- if (request.binding) {
189
- const binding = request.binding
190
- const dbContext = contexts.new(request.processId, subject)
191
-
192
- PluginManager.INSTANCE.processBinding(binding, dbContext)
193
- .then((result) => {
194
- subject.next({
195
- result,
196
- processId: request.processId
347
+ try {
348
+ console.debug('received request:', request)
349
+ if (request.binding) {
350
+ process_binding_count.add(1)
351
+ const binding = request.binding
352
+ const dbContext = contexts.new(request.processId, subject)
353
+ const start = Date.now()
354
+ PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
355
+ .then((result) => {
356
+ subject.next({
357
+ result,
358
+ processId: request.processId
359
+ })
360
+ recordRuntimeInfo(result, binding.handlerType)
197
361
  })
198
- recordRuntimeInfo(result, binding.handlerType)
199
- })
200
- .catch((e) => {
201
- console.error(e)
202
- dbContext.error(request.processId, e)
203
- })
204
- .finally(() => {
205
- contexts.delete(request.processId)
206
- })
207
- }
208
- if (request.dbResult) {
209
- const dbContext = contexts.get(request.processId)
210
- dbContext?.result(request.dbResult)
362
+ .catch((e) => {
363
+ console.debug(e)
364
+ dbContext.error(request.processId, e)
365
+ process_binding_error.add(1)
366
+ })
367
+ .finally(() => {
368
+ const cost = Date.now() - start
369
+ console.debug('processBinding', request.processId, ' took', cost, 'ms')
370
+ process_binding_time.add(cost)
371
+ contexts.delete(request.processId)
372
+ })
373
+ }
374
+ if (request.dbResult) {
375
+ const dbContext = contexts.get(request.processId)
376
+ dbContext?.result(request.dbResult)
377
+ }
378
+ } catch (e) {
379
+ // should not happen
380
+ console.error('unexpect error during handle loop', e)
211
381
  }
212
382
  }
213
383
  }
@@ -0,0 +1,14 @@
1
+ import { defineConfig } from 'tsup'
2
+
3
+ export default defineConfig({
4
+ esbuildOptions: (options) => {
5
+ options.banner = {
6
+ js: `import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);`
7
+ }
8
+ },
9
+ entry: ['src/index.ts', 'src/processor-runner.ts'],
10
+ outDir: 'lib',
11
+ clean: true,
12
+ dts: true,
13
+ format: 'esm'
14
+ })
@@ -1,6 +0,0 @@
1
- export interface ChainConfig {
2
- ChainID: string;
3
- Https?: string[];
4
- ChainServer?: string;
5
- }
6
- //# sourceMappingURL=chain-config.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=chain-config.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
@@ -1,17 +0,0 @@
1
- import { Subject } from 'rxjs';
2
- import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
3
- type Request = Omit<DBRequest, 'opId'>;
4
- export declare class StoreContext {
5
- readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
6
- readonly processId: number;
7
- private static opCounter;
8
- private defers;
9
- constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
10
- newPromise<T>(opId: bigint): Promise<T>;
11
- sendRequest(request: DeepPartial<Request>): Promise<unknown>;
12
- result(dbResult: DBResponse): void;
13
- error(processId: number, e: any): void;
14
- close(): void;
15
- }
16
- export {};
17
- //# sourceMappingURL=db-context.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;IAa/B,KAAK;CAON"}
package/lib/db-context.js DELETED
@@ -1,63 +0,0 @@
1
- import { ProcessResult } from '@sentio/protos';
2
- export class StoreContext {
3
- subject;
4
- processId;
5
- static opCounter = 0n;
6
- defers = new Map();
7
- constructor(subject, processId) {
8
- this.subject = subject;
9
- this.processId = processId;
10
- }
11
- newPromise(opId) {
12
- return new Promise((resolve, reject) => {
13
- this.defers.set(opId, { resolve, reject });
14
- });
15
- }
16
- sendRequest(request) {
17
- const opId = StoreContext.opCounter++;
18
- const promise = this.newPromise(opId);
19
- console.debug('sending db request ', opId, request);
20
- this.subject.next({
21
- dbRequest: {
22
- ...request,
23
- opId
24
- },
25
- processId: this.processId
26
- });
27
- return promise;
28
- }
29
- result(dbResult) {
30
- const opId = dbResult.opId;
31
- const defer = this.defers.get(opId);
32
- console.debug('received db result ', opId, dbResult);
33
- if (defer) {
34
- if (dbResult.error) {
35
- defer.reject(new Error(dbResult.error));
36
- }
37
- else {
38
- defer.resolve(dbResult);
39
- }
40
- this.defers.delete(opId);
41
- }
42
- }
43
- error(processId, e) {
44
- console.error('process error', processId, e);
45
- const errorResult = ProcessResult.create({
46
- states: {
47
- error: e?.toString()
48
- }
49
- });
50
- this.subject.next({
51
- result: errorResult,
52
- processId
53
- });
54
- }
55
- close() {
56
- for (const [opId, defer] of this.defers) {
57
- console.warn('context closed before db response', opId);
58
- defer.reject(new Error('context closed'));
59
- }
60
- this.defers.clear();
61
- }
62
- }
63
- //# sourceMappingURL=db-context.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAAsC,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAIzG,MAAM,OAAO,YAAY;IAMZ;IACA;IANH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAA6E,CAAA;IAErG,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY;QACxB,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC;IAED,KAAK;QACH,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,CAAA;YACvD,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC3C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC"}
@@ -1,3 +0,0 @@
1
- #!/usr/bin/env node
2
- export {};
3
- //# sourceMappingURL=decode-benchmark.d.ts.map