@sentio/runtime 2.39.7-rc.9 → 2.40.0-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/lib/chunk-FFU5RYDX.js +78856 -0
  2. package/lib/index.d.ts +371 -9
  3. package/lib/index.js +94 -9
  4. package/lib/processor-runner.d.ts +0 -2
  5. package/lib/processor-runner.js +51460 -129
  6. package/package.json +5 -25
  7. package/src/db-context.ts +157 -7
  8. package/src/full-service.ts +5 -0
  9. package/src/gen/processor/protos/processor.ts +2294 -1344
  10. package/src/plugin.ts +30 -4
  11. package/src/processor-runner.ts +26 -2
  12. package/src/provider.ts +168 -0
  13. package/src/service.ts +205 -28
  14. package/src/tsup.config.ts +14 -0
  15. package/src/utils.ts +11 -3
  16. package/lib/chain-config.d.ts +0 -6
  17. package/lib/chain-config.d.ts.map +0 -1
  18. package/lib/chain-config.js +0 -2
  19. package/lib/chain-config.js.map +0 -1
  20. package/lib/db-context.d.ts +0 -17
  21. package/lib/db-context.d.ts.map +0 -1
  22. package/lib/db-context.js +0 -63
  23. package/lib/db-context.js.map +0 -1
  24. package/lib/decode-benchmark.d.ts +0 -3
  25. package/lib/decode-benchmark.d.ts.map +0 -1
  26. package/lib/decode-benchmark.js +0 -20
  27. package/lib/decode-benchmark.js.map +0 -1
  28. package/lib/endpoints.d.ts +0 -9
  29. package/lib/endpoints.d.ts.map +0 -1
  30. package/lib/endpoints.js +0 -9
  31. package/lib/endpoints.js.map +0 -1
  32. package/lib/full-service.d.ts +0 -655
  33. package/lib/full-service.d.ts.map +0 -1
  34. package/lib/full-service.js +0 -137
  35. package/lib/full-service.js.map +0 -1
  36. package/lib/gen/google/protobuf/empty.d.ts +0 -17
  37. package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
  38. package/lib/gen/google/protobuf/empty.js +0 -40
  39. package/lib/gen/google/protobuf/empty.js.map +0 -1
  40. package/lib/gen/google/protobuf/struct.d.ts +0 -77
  41. package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
  42. package/lib/gen/google/protobuf/struct.js +0 -429
  43. package/lib/gen/google/protobuf/struct.js.map +0 -1
  44. package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
  45. package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
  46. package/lib/gen/google/protobuf/timestamp.js +0 -83
  47. package/lib/gen/google/protobuf/timestamp.js.map +0 -1
  48. package/lib/gen/processor/protos/processor.d.ts +0 -1470
  49. package/lib/gen/processor/protos/processor.d.ts.map +0 -1
  50. package/lib/gen/processor/protos/processor.js +0 -8512
  51. package/lib/gen/processor/protos/processor.js.map +0 -1
  52. package/lib/gen/service/common/protos/common.d.ts +0 -1698
  53. package/lib/gen/service/common/protos/common.d.ts.map +0 -1
  54. package/lib/gen/service/common/protos/common.js +0 -11383
  55. package/lib/gen/service/common/protos/common.js.map +0 -1
  56. package/lib/global-config.d.ts +0 -8
  57. package/lib/global-config.d.ts.map +0 -1
  58. package/lib/global-config.js +0 -23
  59. package/lib/global-config.js.map +0 -1
  60. package/lib/global-config.test.d.ts +0 -2
  61. package/lib/global-config.test.d.ts.map +0 -1
  62. package/lib/global-config.test.js.map +0 -1
  63. package/lib/index.d.ts.map +0 -1
  64. package/lib/index.js.map +0 -1
  65. package/lib/logger.d.ts +0 -2
  66. package/lib/logger.d.ts.map +0 -1
  67. package/lib/logger.js +0 -39
  68. package/lib/logger.js.map +0 -1
  69. package/lib/logger.test.d.ts +0 -2
  70. package/lib/logger.test.d.ts.map +0 -1
  71. package/lib/logger.test.js.map +0 -1
  72. package/lib/plugin.d.ts +0 -29
  73. package/lib/plugin.d.ts.map +0 -1
  74. package/lib/plugin.js +0 -58
  75. package/lib/plugin.js.map +0 -1
  76. package/lib/processor-runner.d.ts.map +0 -1
  77. package/lib/processor-runner.js.map +0 -1
  78. package/lib/seq-mode.test.d.ts +0 -3
  79. package/lib/seq-mode.test.d.ts.map +0 -1
  80. package/lib/seq-mode.test.js.map +0 -1
  81. package/lib/service.d.ts +0 -179
  82. package/lib/service.d.ts.map +0 -1
  83. package/lib/service.js +0 -194
  84. package/lib/service.js.map +0 -1
  85. package/lib/service.test.d.ts +0 -3
  86. package/lib/service.test.d.ts.map +0 -1
  87. package/lib/service.test.js.map +0 -1
  88. package/lib/state-storage.test.d.ts +0 -2
  89. package/lib/state-storage.test.d.ts.map +0 -1
  90. package/lib/state-storage.test.js.map +0 -1
  91. package/lib/state.d.ts +0 -23
  92. package/lib/state.d.ts.map +0 -1
  93. package/lib/state.js +0 -61
  94. package/lib/state.js.map +0 -1
  95. package/lib/utils.d.ts +0 -6
  96. package/lib/utils.d.ts.map +0 -1
  97. package/lib/utils.js +0 -23
  98. package/lib/utils.js.map +0 -1
package/src/plugin.ts CHANGED
@@ -1,4 +1,12 @@
1
- import { DataBinding, HandlerType, ProcessConfigResponse, ProcessResult, StartRequest } from '@sentio/protos'
1
+ import {
2
+ DataBinding,
3
+ HandlerType,
4
+ PreparedData,
5
+ PreprocessResult,
6
+ ProcessConfigResponse,
7
+ ProcessResult,
8
+ StartRequest
9
+ } from '@sentio/protos'
2
10
  import { StoreContext } from './db-context.js'
3
11
  import { AsyncLocalStorage } from 'node:async_hooks'
4
12
 
@@ -16,9 +24,13 @@ export abstract class Plugin {
16
24
  return false
17
25
  }
18
26
 
19
- async processBinding(request: DataBinding): Promise<ProcessResult> {
27
+ async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
20
28
  return ProcessResult.create()
21
29
  }
30
+
31
+ async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
32
+ return PreprocessResult.create()
33
+ }
22
34
  }
23
35
 
24
36
  export class PluginManager {
@@ -58,13 +70,27 @@ export class PluginManager {
58
70
  return this.plugins.some((plugin) => plugin.stateDiff(config))
59
71
  }
60
72
 
61
- processBinding(request: DataBinding, dbContext?: StoreContext): Promise<ProcessResult> {
73
+ processBinding(
74
+ request: DataBinding,
75
+ preparedData: PreparedData | undefined,
76
+ dbContext?: StoreContext
77
+ ): Promise<ProcessResult> {
78
+ const plugin = this.typesToPlugin.get(request.handlerType)
79
+ if (!plugin) {
80
+ throw new Error(`No plugin for ${request.handlerType}`)
81
+ }
82
+ return this.dbContextLocalStorage.run(dbContext, () => {
83
+ return plugin.processBinding(request, preparedData)
84
+ })
85
+ }
86
+
87
+ preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
62
88
  const plugin = this.typesToPlugin.get(request.handlerType)
63
89
  if (!plugin) {
64
90
  throw new Error(`No plugin for ${request.handlerType}`)
65
91
  }
66
92
  return this.dbContextLocalStorage.run(dbContext, () => {
67
- return plugin.processBinding(request)
93
+ return plugin.preprocessBinding(request)
68
94
  })
69
95
  }
70
96
  }
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
8
8
  import { createServer } from 'nice-grpc'
9
9
  import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
10
10
  import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
11
+ import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
11
12
  import { register as globalRegistry, Registry } from 'prom-client'
12
13
  import http from 'http'
13
14
  // @ts-ignore inspector promises is not included in @type/node
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
20
21
  import { ChainConfig } from './chain-config.js'
21
22
  import { setupLogger } from './logger.js'
22
23
 
24
+ import { NodeSDK } from '@opentelemetry/sdk-node'
25
+ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
26
+ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
27
+ import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
28
+
29
+ const sdk = new NodeSDK({
30
+ traceExporter: new OTLPTraceExporter(),
31
+ metricReader: new PeriodicExportingMetricReader({
32
+ exporter: new OTLPMetricExporter()
33
+ })
34
+ // instrumentations: [getNodeAutoInstrumentations()],
35
+ })
36
+
37
+ sdk.start()
38
+
23
39
  const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
24
40
 
25
41
  const optionDefinitions = [
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
50
66
  const fullPath = path.resolve(options['chains-config'])
51
67
  const chainsConfig = fs.readJsonSync(fullPath)
52
68
 
53
- Endpoints.INSTANCE.concurrency = options.concurrency
54
- Endpoints.INSTANCE.batchCount = options['batch-count']
69
+ const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
70
+ ? parseInt(process.env['OVERRIDE_CONCURRENCY'])
71
+ : undefined
72
+ const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
73
+ ? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
74
+ : undefined
75
+
76
+ Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
77
+ Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
55
78
  Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
56
79
  Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
57
80
 
@@ -77,6 +100,7 @@ const server = createServer({
77
100
  'grpc.default_compression_algorithm': compressionAlgorithms.gzip
78
101
  })
79
102
  .use(prometheusServerMiddleware())
103
+ .use(openTelemetryServerMiddleware())
80
104
  .use(errorDetailsServerMiddleware)
81
105
  const baseService = new ProcessorServiceImpl(async () => {
82
106
  const m = await import(options.target)
@@ -0,0 +1,168 @@
1
+ import { JsonRpcProvider, Network, Provider } from 'ethers'
2
+
3
+ import PQueue from 'p-queue'
4
+ import { Endpoints } from './endpoints.js'
5
+ import { EthChainId } from '@sentio/chain'
6
+ import { LRUCache } from 'lru-cache'
7
+ import { metrics } from '@opentelemetry/api'
8
+
9
+ export const DummyProvider = new JsonRpcProvider('', Network.from(1))
10
+
11
+ const meter = metrics.getMeter('processor_provider')
12
+ const hit_count = meter.createCounter('provider_hit_count')
13
+ const miss_count = meter.createCounter('provider_miss_count')
14
+ const queue_size = meter.createGauge('provider_queue_size')
15
+ const total_duration = meter.createCounter('provider_total_duration')
16
+ const total_queued = meter.createCounter('provider_total_queued')
17
+
18
+ const providers = new Map<string, JsonRpcProvider>()
19
+
20
+ // export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
21
+ // if (!networkish) {
22
+ // networkish = EthChainId.ETHEREUM
23
+ // }
24
+ // if (networkish instanceof BaseContext) {
25
+ // networkish = networkish.getChainId()
26
+ // }
27
+ // return networkish
28
+ // }
29
+
30
+ export function getProvider(chainId?: EthChainId): Provider {
31
+ // const network = getNetworkFromCtxOrNetworkish(networkish)
32
+ if (!chainId) {
33
+ chainId = EthChainId.ETHEREUM
34
+ }
35
+ const network = Network.from(parseInt(chainId))
36
+ // TODO check if other key needed
37
+
38
+ const address = Endpoints.INSTANCE.chainServer.get(chainId)
39
+ const key = network.chainId.toString() + '-' + address
40
+
41
+ console.debug(`init provider for ${chainId}, address: ${address}`)
42
+ let provider = providers.get(key)
43
+
44
+ if (provider) {
45
+ return provider
46
+ }
47
+
48
+ if (address === undefined) {
49
+ throw Error(
50
+ 'Provider not found for chain ' +
51
+ network.chainId +
52
+ ', configured chains: ' +
53
+ [...Endpoints.INSTANCE.chainServer.keys()].join(' ')
54
+ )
55
+ }
56
+ console.log(
57
+ `init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
58
+ )
59
+ provider = new QueuedStaticJsonRpcProvider(
60
+ address,
61
+ network,
62
+ Endpoints.INSTANCE.concurrency,
63
+ Endpoints.INSTANCE.batchCount
64
+ )
65
+ providers.set(key, provider)
66
+ return provider
67
+ }
68
+
69
+ function getTag(prefix: string, value: any): string {
70
+ return (
71
+ prefix +
72
+ ':' +
73
+ JSON.stringify(value, (k, v) => {
74
+ if (v == null) {
75
+ return 'null'
76
+ }
77
+ if (typeof v === 'bigint') {
78
+ return `bigint:${v.toString()}`
79
+ }
80
+ if (typeof v === 'string') {
81
+ return v.toLowerCase()
82
+ }
83
+
84
+ // Sort object keys
85
+ if (typeof v === 'object' && !Array.isArray(v)) {
86
+ const keys = Object.keys(v)
87
+ keys.sort()
88
+ return keys.reduce(
89
+ (accum, key) => {
90
+ accum[key] = v[key]
91
+ return accum
92
+ },
93
+ <any>{}
94
+ )
95
+ }
96
+
97
+ return v
98
+ })
99
+ )
100
+ }
101
+
102
+ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
103
+ executor: PQueue
104
+ #performCache = new LRUCache<string, Promise<any>>({
105
+ max: 300000 // 300k items
106
+ // maxSize: 300 * 1024 * 1024, // 300mb for cache
107
+ // ttl: 1000 * 60 * 60, // 1 hour no ttl for better performance
108
+ // sizeCalculation: (value: any) => {
109
+ // assume each item is 1kb for simplicity
110
+ // return 1024
111
+ // }
112
+ })
113
+
114
+ constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
115
+ // TODO re-enable match when possible
116
+ super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
117
+ this.executor = new PQueue({ concurrency: concurrency })
118
+ }
119
+
120
+ async send(method: string, params: Array<any>): Promise<any> {
121
+ if (method !== 'eth_call') {
122
+ return await this.executor.add(() => super.send(method, params))
123
+ }
124
+ const tag = getTag(method, params)
125
+ const block = params[params.length - 1]
126
+ let perform = this.#performCache.get(tag)
127
+ if (!perform) {
128
+ miss_count.add(1)
129
+ const queued: number = Date.now()
130
+ perform = this.executor.add(() => {
131
+ const started = Date.now()
132
+ total_queued.add(started - queued)
133
+
134
+ return super.send(method, params).finally(() => {
135
+ total_duration.add(Date.now() - started)
136
+ })
137
+ })
138
+ perform.catch((e) => {
139
+ // if (e.code !== 'CALL_EXCEPTION' && e.code !== 'BAD_DATA') {
140
+ setTimeout(() => {
141
+ if (this.#performCache.get(tag) === perform) {
142
+ this.#performCache.delete(tag)
143
+ }
144
+ }, 1000)
145
+ })
146
+
147
+ queue_size.record(this.executor.size)
148
+
149
+ this.#performCache.set(tag, perform)
150
+ // For non latest block call, we cache permanently, otherwise we cache for one minute
151
+ if (block === 'latest') {
152
+ setTimeout(() => {
153
+ if (this.#performCache.get(tag) === perform) {
154
+ this.#performCache.delete(tag)
155
+ }
156
+ }, 60 * 1000)
157
+ }
158
+ } else {
159
+ hit_count.add(1)
160
+ }
161
+
162
+ const result = await perform
163
+ if (!result) {
164
+ throw Error('Unexpected null response')
165
+ }
166
+ return result
167
+ }
168
+ }
package/src/service.ts CHANGED
@@ -7,7 +7,12 @@ import {
7
7
  DataBinding,
8
8
  DeepPartial,
9
9
  Empty,
10
+ EthCallParam,
10
11
  HandlerType,
12
+ PreparedData,
13
+ PreprocessResult,
14
+ PreprocessStreamRequest,
15
+ PreprocessStreamResponse,
11
16
  ProcessBindingResponse,
12
17
  ProcessBindingsRequest,
13
18
  ProcessConfigRequest,
@@ -20,11 +25,21 @@ import {
20
25
  } from '@sentio/protos'
21
26
 
22
27
  import { PluginManager } from './plugin.js'
23
- import { errorString, mergeProcessResults } from './utils.js'
28
+ import { errorString, makeEthCallKey, mergeProcessResults } from './utils.js'
24
29
  import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
25
30
 
26
31
  import { StoreContext } from './db-context.js'
27
32
  import { Subject } from 'rxjs'
33
+ import { metrics } from '@opentelemetry/api'
34
+ import { getProvider } from './provider.js'
35
+ import { EthChainId } from '@sentio/chain'
36
+ import { Provider } from 'ethers'
37
+
38
+ const meter = metrics.getMeter('processor_service')
39
+ const process_binding_count = meter.createCounter('process_binding_count')
40
+ const process_binding_time = meter.createCounter('process_binding_time')
41
+ const process_binding_error = meter.createCounter('process_binding_error')
42
+
28
43
  ;(BigInt.prototype as any).toJSON = function () {
29
44
  return this.toString()
30
45
  }
@@ -39,6 +54,8 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
39
54
 
40
55
  private readonly shutdownHandler?: () => void
41
56
 
57
+ private readonly preprocessedEthCalls: { [calldata: string]: any[] }
58
+
42
59
  constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
43
60
  this.loader = loader
44
61
  this.shutdownHandler = shutdownHandler
@@ -111,10 +128,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
111
128
  }
112
129
 
113
130
  async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
114
- const promises = []
131
+ const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
115
132
 
133
+ const promises = []
116
134
  for (const binding of request.bindings) {
117
- const promise = this.processBinding(binding)
135
+ const promise = this.processBinding(binding, { ethCallResults })
118
136
  if (GLOBAL_CONFIG.execution.sequential) {
119
137
  await promise
120
138
  }
@@ -139,7 +157,99 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
139
157
  }
140
158
  }
141
159
 
142
- async processBinding(request: DataBinding, options?: CallContext): Promise<ProcessResult> {
160
+ async preprocessBindings(
161
+ bindings: DataBinding[],
162
+ dbContext?: StoreContext,
163
+ options?: CallContext
164
+ ): Promise<{ [ethCallKey: string]: string }> {
165
+ console.debug(`preprocessBindings start, bindings: ${bindings.length}`)
166
+ const promises = []
167
+ for (const binding of bindings) {
168
+ promises.push(this.preprocessBinding(binding, dbContext, options))
169
+ }
170
+ let preprocessResults: PreprocessResult[]
171
+ try {
172
+ preprocessResults = await Promise.all(promises)
173
+ } catch (e) {
174
+ throw e
175
+ }
176
+ console.debug(
177
+ 'ethCallParams: ',
178
+ preprocessResults.map((r) => r.ethCallParams)
179
+ )
180
+ const groupedRequests = new Map<string, EthCallParam[]>()
181
+ const providers = new Map<string, Provider>()
182
+ for (const result of preprocessResults) {
183
+ for (const param of result.ethCallParams) {
184
+ const { chainId, address, blockTag } = param.context!
185
+ if (!providers.has(chainId)) {
186
+ providers.set(chainId, getProvider(chainId as EthChainId))
187
+ }
188
+ const key = [chainId, address, blockTag].join('|')
189
+ if (!groupedRequests.has(key)) {
190
+ groupedRequests.set(key, [])
191
+ }
192
+ groupedRequests.get(key)!.push(param)
193
+ }
194
+ }
195
+
196
+ const start = Date.now()
197
+ const callPromises = []
198
+ for (const params of groupedRequests.values()) {
199
+ const { chainId, address, blockTag } = params[0].context!
200
+ console.log(`chain: ${chainId}, address: ${address}, blockTag: ${blockTag}, totalCalls: ${params.length}`)
201
+ // TODO multicall
202
+ for (const param of params) {
203
+ callPromises.push(
204
+ providers
205
+ .get(chainId)!
206
+ .call({
207
+ to: address,
208
+ data: param.calldata,
209
+ blockTag
210
+ })
211
+ .then((result) => [makeEthCallKey(param), result])
212
+ )
213
+ }
214
+ }
215
+ let results = {}
216
+ try {
217
+ results = Object.fromEntries(await Promise.all(callPromises))
218
+ } catch (e) {
219
+ console.error(`eth call error: ${e}`)
220
+ }
221
+ console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
222
+ return results
223
+ }
224
+
225
+ async preprocessBinding(
226
+ request: DataBinding,
227
+ dbContext?: StoreContext,
228
+ options?: CallContext
229
+ ): Promise<PreprocessResult> {
230
+ if (!this.started) {
231
+ throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
232
+ }
233
+ if (this.unhandled) {
234
+ throw new RichServerError(
235
+ Status.UNAVAILABLE,
236
+ 'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
237
+ [
238
+ DebugInfo.fromPartial({
239
+ detail: this.unhandled.message,
240
+ stackEntries: this.unhandled.stack?.split('\n')
241
+ })
242
+ ]
243
+ )
244
+ }
245
+ return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
246
+ }
247
+
248
+ async processBinding(
249
+ request: DataBinding,
250
+ preparedData: PreparedData | undefined,
251
+ options?: CallContext
252
+ ): Promise<ProcessResult> {
143
253
  if (!this.started) {
144
254
  throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
145
255
  }
@@ -155,7 +265,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
155
265
  ]
156
266
  )
157
267
  }
158
- const result = await PluginManager.INSTANCE.processBinding(request)
268
+ const result = await PluginManager.INSTANCE.processBinding(request, preparedData)
159
269
  recordRuntimeInfo(result, request.handlerType)
160
270
  return result
161
271
  }
@@ -177,6 +287,63 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
177
287
  yield* from(subject).pipe(withAbort(context.signal))
178
288
  }
179
289
 
290
+ async handlePreprocessRequests(
291
+ requests: AsyncIterable<PreprocessStreamRequest>,
292
+ subject: Subject<DeepPartial<PreprocessStreamResponse>>
293
+ ) {
294
+ const contexts = new Contexts()
295
+
296
+ for await (const request of requests) {
297
+ try {
298
+ console.debug('received request:', request)
299
+ if (request.bindings) {
300
+ const bindings = request.bindings.bindings
301
+ const dbContext = contexts.new(request.processId, subject)
302
+ const start = Date.now()
303
+ this.preprocessBindings(bindings, dbContext)
304
+ .then(() => {
305
+ subject.next({
306
+ processId: request.processId
307
+ })
308
+ })
309
+ .catch((e) => {
310
+ console.debug(e)
311
+ dbContext.error(request.processId, e)
312
+ })
313
+ .finally(() => {
314
+ const cost = Date.now() - start
315
+ console.debug('preprocessBinding', request.processId, ' took', cost, 'ms')
316
+ contexts.delete(request.processId)
317
+ })
318
+ }
319
+ if (request.dbResult) {
320
+ const dbContext = contexts.get(request.processId)
321
+ dbContext?.result(request.dbResult)
322
+ }
323
+ } catch (e) {
324
+ // should not happen
325
+ console.error('unexpect error during handle loop', e)
326
+ }
327
+ }
328
+ }
329
+
330
+ async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
331
+ if (!this.started) {
332
+ throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
333
+ }
334
+
335
+ const subject = new Subject<DeepPartial<PreprocessStreamResponse>>()
336
+ this.handlePreprocessRequests(requests, subject)
337
+ .then(() => {
338
+ subject.complete()
339
+ })
340
+ .catch((e) => {
341
+ console.error(e)
342
+ subject.error(e)
343
+ })
344
+ yield* from(subject).pipe(withAbort(context.signal))
345
+ }
346
+
180
347
  private async handleRequests(
181
348
  requests: AsyncIterable<ProcessStreamRequest>,
182
349
  subject: Subject<DeepPartial<ProcessStreamResponse>>
@@ -184,30 +351,40 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
184
351
  const contexts = new Contexts()
185
352
 
186
353
  for await (const request of requests) {
187
- console.debug('received request:', request)
188
- if (request.binding) {
189
- const binding = request.binding
190
- const dbContext = contexts.new(request.processId, subject)
191
-
192
- PluginManager.INSTANCE.processBinding(binding, dbContext)
193
- .then((result) => {
194
- subject.next({
195
- result,
196
- processId: request.processId
354
+ try {
355
+ console.debug('received request:', request)
356
+ if (request.binding) {
357
+ process_binding_count.add(1)
358
+ const binding = request.binding
359
+ const dbContext = contexts.new(request.processId, subject)
360
+ const start = Date.now()
361
+ PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
362
+ .then((result) => {
363
+ subject.next({
364
+ result,
365
+ processId: request.processId
366
+ })
367
+ recordRuntimeInfo(result, binding.handlerType)
197
368
  })
198
- recordRuntimeInfo(result, binding.handlerType)
199
- })
200
- .catch((e) => {
201
- console.error(e)
202
- dbContext.error(request.processId, e)
203
- })
204
- .finally(() => {
205
- contexts.delete(request.processId)
206
- })
207
- }
208
- if (request.dbResult) {
209
- const dbContext = contexts.get(request.processId)
210
- dbContext?.result(request.dbResult)
369
+ .catch((e) => {
370
+ console.debug(e)
371
+ dbContext.error(request.processId, e)
372
+ process_binding_error.add(1)
373
+ })
374
+ .finally(() => {
375
+ const cost = Date.now() - start
376
+ console.debug('processBinding', request.processId, ' took', cost, 'ms')
377
+ process_binding_time.add(cost)
378
+ contexts.delete(request.processId)
379
+ })
380
+ }
381
+ if (request.dbResult) {
382
+ const dbContext = contexts.get(request.processId)
383
+ dbContext?.result(request.dbResult)
384
+ }
385
+ } catch (e) {
386
+ // should not happen
387
+ console.error('unexpect error during handle loop', e)
211
388
  }
212
389
  }
213
390
  }
@@ -0,0 +1,14 @@
1
+ import { defineConfig } from 'tsup'
2
+
3
+ export default defineConfig({
4
+ esbuildOptions: (options) => {
5
+ options.banner = {
6
+ js: `import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);`
7
+ }
8
+ },
9
+ entry: ['src/index.ts', 'src/processor-runner.ts'],
10
+ outDir: 'lib',
11
+ clean: true,
12
+ dts: true,
13
+ format: 'esm'
14
+ })
package/src/utils.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { ProcessResult } from '@sentio/protos'
1
+ import { EthCallParam, ProcessResult } from '@sentio/protos'
2
2
 
3
3
  // TODO better handling this, because old proto doesn't have this
4
4
  import { StateResult, ProcessResult as ProcessResultFull } from './gen/processor/protos/processor.js'
@@ -8,7 +8,7 @@ import { Required } from 'utility-types'
8
8
  export function mergeProcessResults(results: ProcessResult[]): Required<ProcessResult, 'states'> {
9
9
  const res = {
10
10
  ...ProcessResultFull.create(),
11
- states: StateResult.create(),
11
+ states: StateResult.create()
12
12
  }
13
13
 
14
14
  for (const r of results) {
@@ -17,7 +17,7 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
17
17
  res.events = res.events.concat(r.events)
18
18
  res.exports = res.exports.concat(r.exports)
19
19
  res.states = {
20
- configUpdated: res.states?.configUpdated || r.states?.configUpdated || false,
20
+ configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
21
21
  }
22
22
  }
23
23
  return res
@@ -28,3 +28,11 @@ export function errorString(e: Error): string {
28
28
  }
29
29
 
30
30
  export const USER_PROCESSOR = 'user_processor'
31
+
32
+ export function makeEthCallKey(param: EthCallParam) {
33
+ if (!param.context) {
34
+ throw new Error('null context for eth call')
35
+ }
36
+ const { chainId, address, blockTag } = param.context
37
+ return `${chainId}|${address}|${blockTag}|${param.calldata}`
38
+ }
@@ -1,6 +0,0 @@
1
- export interface ChainConfig {
2
- ChainID: string;
3
- Https?: string[];
4
- ChainServer?: string;
5
- }
6
- //# sourceMappingURL=chain-config.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=chain-config.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
@@ -1,17 +0,0 @@
1
- import { Subject } from 'rxjs';
2
- import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
3
- type Request = Omit<DBRequest, 'opId'>;
4
- export declare class StoreContext {
5
- readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
6
- readonly processId: number;
7
- private static opCounter;
8
- private defers;
9
- constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
10
- newPromise<T>(opId: bigint): Promise<T>;
11
- sendRequest(request: DeepPartial<Request>): Promise<unknown>;
12
- result(dbResult: DBResponse): void;
13
- error(processId: number, e: any): void;
14
- close(): void;
15
- }
16
- export {};
17
- //# sourceMappingURL=db-context.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;IAa/B,KAAK;CAON"}