@sentio/runtime 2.39.7-rc.8 → 2.40.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/lib/chunk-LMUCLUM5.js +78161 -0
  2. package/lib/index.d.ts +308 -9
  3. package/lib/index.js +92 -9
  4. package/lib/processor-runner.d.ts +0 -2
  5. package/lib/processor-runner.js +51457 -129
  6. package/package.json +5 -25
  7. package/src/db-context.ts +165 -7
  8. package/src/gen/processor/protos/processor.ts +361 -1
  9. package/src/plugin.ts +30 -4
  10. package/src/processor-runner.ts +26 -2
  11. package/src/provider.ts +166 -0
  12. package/src/service.ts +147 -28
  13. package/src/tsup.config.ts +14 -0
  14. package/lib/chain-config.d.ts +0 -6
  15. package/lib/chain-config.d.ts.map +0 -1
  16. package/lib/chain-config.js +0 -2
  17. package/lib/chain-config.js.map +0 -1
  18. package/lib/db-context.d.ts +0 -16
  19. package/lib/db-context.d.ts.map +0 -1
  20. package/lib/db-context.js +0 -56
  21. package/lib/db-context.js.map +0 -1
  22. package/lib/decode-benchmark.d.ts +0 -3
  23. package/lib/decode-benchmark.d.ts.map +0 -1
  24. package/lib/decode-benchmark.js +0 -20
  25. package/lib/decode-benchmark.js.map +0 -1
  26. package/lib/endpoints.d.ts +0 -9
  27. package/lib/endpoints.d.ts.map +0 -1
  28. package/lib/endpoints.js +0 -9
  29. package/lib/endpoints.js.map +0 -1
  30. package/lib/full-service.d.ts +0 -655
  31. package/lib/full-service.d.ts.map +0 -1
  32. package/lib/full-service.js +0 -137
  33. package/lib/full-service.js.map +0 -1
  34. package/lib/gen/google/protobuf/empty.d.ts +0 -17
  35. package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
  36. package/lib/gen/google/protobuf/empty.js +0 -40
  37. package/lib/gen/google/protobuf/empty.js.map +0 -1
  38. package/lib/gen/google/protobuf/struct.d.ts +0 -77
  39. package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
  40. package/lib/gen/google/protobuf/struct.js +0 -429
  41. package/lib/gen/google/protobuf/struct.js.map +0 -1
  42. package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
  43. package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
  44. package/lib/gen/google/protobuf/timestamp.js +0 -83
  45. package/lib/gen/google/protobuf/timestamp.js.map +0 -1
  46. package/lib/gen/processor/protos/processor.d.ts +0 -1470
  47. package/lib/gen/processor/protos/processor.d.ts.map +0 -1
  48. package/lib/gen/processor/protos/processor.js +0 -8512
  49. package/lib/gen/processor/protos/processor.js.map +0 -1
  50. package/lib/gen/service/common/protos/common.d.ts +0 -1698
  51. package/lib/gen/service/common/protos/common.d.ts.map +0 -1
  52. package/lib/gen/service/common/protos/common.js +0 -11383
  53. package/lib/gen/service/common/protos/common.js.map +0 -1
  54. package/lib/global-config.d.ts +0 -8
  55. package/lib/global-config.d.ts.map +0 -1
  56. package/lib/global-config.js +0 -23
  57. package/lib/global-config.js.map +0 -1
  58. package/lib/global-config.test.d.ts +0 -2
  59. package/lib/global-config.test.d.ts.map +0 -1
  60. package/lib/global-config.test.js.map +0 -1
  61. package/lib/index.d.ts.map +0 -1
  62. package/lib/index.js.map +0 -1
  63. package/lib/logger.d.ts +0 -2
  64. package/lib/logger.d.ts.map +0 -1
  65. package/lib/logger.js +0 -39
  66. package/lib/logger.js.map +0 -1
  67. package/lib/logger.test.d.ts +0 -2
  68. package/lib/logger.test.d.ts.map +0 -1
  69. package/lib/logger.test.js.map +0 -1
  70. package/lib/plugin.d.ts +0 -29
  71. package/lib/plugin.d.ts.map +0 -1
  72. package/lib/plugin.js +0 -58
  73. package/lib/plugin.js.map +0 -1
  74. package/lib/processor-runner.d.ts.map +0 -1
  75. package/lib/processor-runner.js.map +0 -1
  76. package/lib/seq-mode.test.d.ts +0 -3
  77. package/lib/seq-mode.test.d.ts.map +0 -1
  78. package/lib/seq-mode.test.js.map +0 -1
  79. package/lib/service.d.ts +0 -179
  80. package/lib/service.d.ts.map +0 -1
  81. package/lib/service.js +0 -179
  82. package/lib/service.js.map +0 -1
  83. package/lib/service.test.d.ts +0 -3
  84. package/lib/service.test.d.ts.map +0 -1
  85. package/lib/service.test.js.map +0 -1
  86. package/lib/state-storage.test.d.ts +0 -2
  87. package/lib/state-storage.test.d.ts.map +0 -1
  88. package/lib/state-storage.test.js.map +0 -1
  89. package/lib/state.d.ts +0 -23
  90. package/lib/state.d.ts.map +0 -1
  91. package/lib/state.js +0 -61
  92. package/lib/state.js.map +0 -1
  93. package/lib/utils.d.ts +0 -6
  94. package/lib/utils.d.ts.map +0 -1
  95. package/lib/utils.js +0 -23
  96. package/lib/utils.js.map +0 -1
package/src/plugin.ts CHANGED
@@ -1,4 +1,12 @@
1
- import { DataBinding, HandlerType, ProcessConfigResponse, ProcessResult, StartRequest } from '@sentio/protos'
1
+ import {
2
+ DataBinding,
3
+ HandlerType,
4
+ PreparedData,
5
+ PreprocessResult,
6
+ ProcessConfigResponse,
7
+ ProcessResult,
8
+ StartRequest
9
+ } from '@sentio/protos'
2
10
  import { StoreContext } from './db-context.js'
3
11
  import { AsyncLocalStorage } from 'node:async_hooks'
4
12
 
@@ -16,9 +24,13 @@ export abstract class Plugin {
16
24
  return false
17
25
  }
18
26
 
19
- async processBinding(request: DataBinding): Promise<ProcessResult> {
27
+ async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
20
28
  return ProcessResult.create()
21
29
  }
30
+
31
+ async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
32
+ return PreprocessResult.create()
33
+ }
22
34
  }
23
35
 
24
36
  export class PluginManager {
@@ -58,13 +70,27 @@ export class PluginManager {
58
70
  return this.plugins.some((plugin) => plugin.stateDiff(config))
59
71
  }
60
72
 
61
- processBinding(request: DataBinding, dbContext?: StoreContext): Promise<ProcessResult> {
73
+ processBinding(
74
+ request: DataBinding,
75
+ preparedData: PreparedData | undefined,
76
+ dbContext?: StoreContext
77
+ ): Promise<ProcessResult> {
78
+ const plugin = this.typesToPlugin.get(request.handlerType)
79
+ if (!plugin) {
80
+ throw new Error(`No plugin for ${request.handlerType}`)
81
+ }
82
+ return this.dbContextLocalStorage.run(dbContext, () => {
83
+ return plugin.processBinding(request, preparedData)
84
+ })
85
+ }
86
+
87
+ preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
62
88
  const plugin = this.typesToPlugin.get(request.handlerType)
63
89
  if (!plugin) {
64
90
  throw new Error(`No plugin for ${request.handlerType}`)
65
91
  }
66
92
  return this.dbContextLocalStorage.run(dbContext, () => {
67
- return plugin.processBinding(request)
93
+ return plugin.preprocessBinding(request)
68
94
  })
69
95
  }
70
96
  }
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
8
8
  import { createServer } from 'nice-grpc'
9
9
  import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
10
10
  import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
11
+ import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
11
12
  import { register as globalRegistry, Registry } from 'prom-client'
12
13
  import http from 'http'
13
14
  // @ts-ignore inspector promises is not included in @type/node
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
20
21
  import { ChainConfig } from './chain-config.js'
21
22
  import { setupLogger } from './logger.js'
22
23
 
24
+ import { NodeSDK } from '@opentelemetry/sdk-node'
25
+ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
26
+ import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
27
+ import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
28
+
29
+ const sdk = new NodeSDK({
30
+ traceExporter: new OTLPTraceExporter(),
31
+ metricReader: new PeriodicExportingMetricReader({
32
+ exporter: new OTLPMetricExporter()
33
+ })
34
+ // instrumentations: [getNodeAutoInstrumentations()],
35
+ })
36
+
37
+ sdk.start()
38
+
23
39
  const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
24
40
 
25
41
  const optionDefinitions = [
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
50
66
  const fullPath = path.resolve(options['chains-config'])
51
67
  const chainsConfig = fs.readJsonSync(fullPath)
52
68
 
53
- Endpoints.INSTANCE.concurrency = options.concurrency
54
- Endpoints.INSTANCE.batchCount = options['batch-count']
69
+ const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
70
+ ? parseInt(process.env['OVERRIDE_CONCURRENCY'])
71
+ : undefined
72
+ const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
73
+ ? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
74
+ : undefined
75
+
76
+ Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
77
+ Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
55
78
  Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
56
79
  Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
57
80
 
@@ -77,6 +100,7 @@ const server = createServer({
77
100
  'grpc.default_compression_algorithm': compressionAlgorithms.gzip
78
101
  })
79
102
  .use(prometheusServerMiddleware())
103
+ .use(openTelemetryServerMiddleware())
80
104
  .use(errorDetailsServerMiddleware)
81
105
  const baseService = new ProcessorServiceImpl(async () => {
82
106
  const m = await import(options.target)
@@ -0,0 +1,166 @@
1
+ import { JsonRpcProvider, Network, Provider } from 'ethers'
2
+
3
+ import PQueue from 'p-queue'
4
+ import { Endpoints } from './endpoints.js'
5
+ import { EthChainId } from '@sentio/chain'
6
+ import { LRUCache } from 'lru-cache'
7
+ import { metrics } from '@opentelemetry/api'
8
+
9
+ export const DummyProvider = new JsonRpcProvider('', Network.from(1))
10
+
11
+ const meter = metrics.getMeter('processor_provider')
12
+ const hit_count = meter.createCounter('provider_hit_count')
13
+ const miss_count = meter.createCounter('provider_miss_count')
14
+ const queue_size = meter.createGauge('provider_queue_size')
15
+ const total_duration = meter.createCounter('provider_total_duration')
16
+ const total_queued = meter.createCounter('provider_total_queued')
17
+
18
+ const providers = new Map<string, JsonRpcProvider>()
19
+
20
+ // export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
21
+ // if (!networkish) {
22
+ // networkish = EthChainId.ETHEREUM
23
+ // }
24
+ // if (networkish instanceof BaseContext) {
25
+ // networkish = networkish.getChainId()
26
+ // }
27
+ // return networkish
28
+ // }
29
+
30
+ export function getProvider(chainId?: EthChainId): Provider {
31
+ // const network = getNetworkFromCtxOrNetworkish(networkish)
32
+ if (!chainId) {
33
+ chainId = EthChainId.ETHEREUM
34
+ }
35
+ const network = Network.from(parseInt(chainId))
36
+ // TODO check if other key needed
37
+
38
+ const address = Endpoints.INSTANCE.chainServer.get(chainId)
39
+ const key = network.chainId.toString() + '-' + address
40
+ let provider = providers.get(key)
41
+
42
+ if (provider) {
43
+ return provider
44
+ }
45
+
46
+ if (address === undefined) {
47
+ throw Error(
48
+ 'Provider not found for chain ' +
49
+ network.chainId +
50
+ ', configured chains: ' +
51
+ [...Endpoints.INSTANCE.chainServer.keys()].join(' ')
52
+ )
53
+ }
54
+ console.log(
55
+ `init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
56
+ )
57
+ provider = new QueuedStaticJsonRpcProvider(
58
+ address,
59
+ network,
60
+ Endpoints.INSTANCE.concurrency,
61
+ Endpoints.INSTANCE.batchCount
62
+ )
63
+ providers.set(key, provider)
64
+ return provider
65
+ }
66
+
67
+ function getTag(prefix: string, value: any): string {
68
+ return (
69
+ prefix +
70
+ ':' +
71
+ JSON.stringify(value, (k, v) => {
72
+ if (v == null) {
73
+ return 'null'
74
+ }
75
+ if (typeof v === 'bigint') {
76
+ return `bigint:${v.toString()}`
77
+ }
78
+ if (typeof v === 'string') {
79
+ return v.toLowerCase()
80
+ }
81
+
82
+ // Sort object keys
83
+ if (typeof v === 'object' && !Array.isArray(v)) {
84
+ const keys = Object.keys(v)
85
+ keys.sort()
86
+ return keys.reduce(
87
+ (accum, key) => {
88
+ accum[key] = v[key]
89
+ return accum
90
+ },
91
+ <any>{}
92
+ )
93
+ }
94
+
95
+ return v
96
+ })
97
+ )
98
+ }
99
+
100
+ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
101
+ executor: PQueue
102
+ #performCache = new LRUCache<string, Promise<any>>({
103
+ max: 300000 // 300k items
104
+ // maxSize: 300 * 1024 * 1024, // 300mb for cache
105
+ // ttl: 1000 * 60 * 60, // 1 hour no ttl for better performance
106
+ // sizeCalculation: (value: any) => {
107
+ // assume each item is 1kb for simplicity
108
+ // return 1024
109
+ // }
110
+ })
111
+
112
+ constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
113
+ // TODO re-enable match when possible
114
+ super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
115
+ this.executor = new PQueue({ concurrency: concurrency })
116
+ }
117
+
118
+ async send(method: string, params: Array<any>): Promise<any> {
119
+ if (method !== 'eth_call') {
120
+ return await this.executor.add(() => super.send(method, params))
121
+ }
122
+ const tag = getTag(method, params)
123
+ const block = params[params.length - 1]
124
+ let perform = this.#performCache.get(tag)
125
+ if (!perform) {
126
+ miss_count.add(1)
127
+ const queued: number = Date.now()
128
+ perform = this.executor.add(() => {
129
+ const started = Date.now()
130
+ total_queued.add(started - queued)
131
+
132
+ return super.send(method, params).finally(() => {
133
+ total_duration.add(Date.now() - started)
134
+ })
135
+ })
136
+ perform.catch((e) => {
137
+ // if (e.code !== 'CALL_EXCEPTION' && e.code !== 'BAD_DATA') {
138
+ setTimeout(() => {
139
+ if (this.#performCache.get(tag) === perform) {
140
+ this.#performCache.delete(tag)
141
+ }
142
+ }, 1000)
143
+ })
144
+
145
+ queue_size.record(this.executor.size)
146
+
147
+ this.#performCache.set(tag, perform)
148
+ // For non latest block call, we cache permanently, otherwise we cache for one minute
149
+ if (block === 'latest') {
150
+ setTimeout(() => {
151
+ if (this.#performCache.get(tag) === perform) {
152
+ this.#performCache.delete(tag)
153
+ }
154
+ }, 60 * 1000)
155
+ }
156
+ } else {
157
+ hit_count.add(1)
158
+ }
159
+
160
+ const result = await perform
161
+ if (!result) {
162
+ throw Error('Unexpected null response')
163
+ }
164
+ return result
165
+ }
166
+ }
package/src/service.ts CHANGED
@@ -7,7 +7,10 @@ import {
7
7
  DataBinding,
8
8
  DeepPartial,
9
9
  Empty,
10
+ EthCallParam,
10
11
  HandlerType,
12
+ PreparedData,
13
+ PreprocessResult,
11
14
  ProcessBindingResponse,
12
15
  ProcessBindingsRequest,
13
16
  ProcessConfigRequest,
@@ -25,6 +28,16 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
25
28
 
26
29
  import { StoreContext } from './db-context.js'
27
30
  import { Subject } from 'rxjs'
31
+ import { metrics } from '@opentelemetry/api'
32
+ import { getProvider } from './provider.js'
33
+ import { EthChainId } from '@sentio/chain'
34
+ import { Provider, Interface } from 'ethers'
35
+
36
+ const meter = metrics.getMeter('processor_service')
37
+ const process_binding_count = meter.createCounter('process_binding_count')
38
+ const process_binding_time = meter.createCounter('process_binding_time')
39
+ const process_binding_error = meter.createCounter('process_binding_error')
40
+
28
41
  ;(BigInt.prototype as any).toJSON = function () {
29
42
  return this.toString()
30
43
  }
@@ -111,10 +124,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
111
124
  }
112
125
 
113
126
  async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
114
- const promises = []
127
+ const ethCallResults = await this.preprocessBindings(request, options)
115
128
 
129
+ const promises = []
116
130
  for (const binding of request.bindings) {
117
- const promise = this.processBinding(binding)
131
+ const promise = this.processBinding(binding, { ethCallResults })
118
132
  if (GLOBAL_CONFIG.execution.sequential) {
119
133
  await promise
120
134
  }
@@ -139,7 +153,82 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
139
153
  }
140
154
  }
141
155
 
142
- async processBinding(request: DataBinding, options?: CallContext): Promise<ProcessResult> {
156
+ async preprocessBindings(
157
+ request: ProcessBindingsRequest,
158
+ options?: CallContext
159
+ ): Promise<{ [calldata: string]: any[] }> {
160
+ const promises = []
161
+ for (const binding of request.bindings) {
162
+ promises.push(this.preprocessBinding(binding, options))
163
+ }
164
+ let preprocessResults: PreprocessResult[]
165
+ try {
166
+ preprocessResults = await Promise.all(promises)
167
+ } catch (e) {
168
+ throw e
169
+ }
170
+ const groupedRequests = new Map<string, EthCallParam[]>()
171
+ const providers = new Map<string, Provider>()
172
+ for (const result of preprocessResults) {
173
+ for (const param of result.ethCallParams) {
174
+ if (!providers.has(param.chainId)) {
175
+ providers.set(param.chainId, getProvider(param.chainId as EthChainId))
176
+ }
177
+ const key = param.chainId + '|' + param.address
178
+ if (!groupedRequests.has(key)) {
179
+ groupedRequests.set(key, [])
180
+ }
181
+ groupedRequests.get(key)!.push(param)
182
+ }
183
+ }
184
+
185
+ const start = Date.now()
186
+ const callPromises = []
187
+ for (const params of groupedRequests.values()) {
188
+ console.log(`chain: ${params[0].chainId}, address: ${params[0].address}, totalCalls: ${params.length}`)
189
+ for (const param of params) {
190
+ const frag = new Interface(param.signature)
191
+ const calldata = frag.encodeFunctionData(param.function, param.args)
192
+ callPromises.push(
193
+ providers
194
+ .get(param.chainId)!
195
+ .call({
196
+ to: param.address,
197
+ data: calldata
198
+ })
199
+ .then((ret) => [calldata, frag.decodeFunctionResult(param.function, ret).toArray()] as [string, any[]])
200
+ )
201
+ }
202
+ }
203
+ const results = Object.fromEntries(await Promise.all(callPromises))
204
+ console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
205
+ return results
206
+ }
207
+
208
+ async preprocessBinding(request: DataBinding, options?: CallContext): Promise<PreprocessResult> {
209
+ if (!this.started) {
210
+ throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
211
+ }
212
+ if (this.unhandled) {
213
+ throw new RichServerError(
214
+ Status.UNAVAILABLE,
215
+ 'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
216
+ [
217
+ DebugInfo.fromPartial({
218
+ detail: this.unhandled.message,
219
+ stackEntries: this.unhandled.stack?.split('\n')
220
+ })
221
+ ]
222
+ )
223
+ }
224
+ return await PluginManager.INSTANCE.preprocessBinding(request)
225
+ }
226
+
227
+ async processBinding(
228
+ request: DataBinding,
229
+ preparedData: PreparedData | undefined,
230
+ options?: CallContext
231
+ ): Promise<ProcessResult> {
143
232
  if (!this.started) {
144
233
  throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
145
234
  }
@@ -155,7 +244,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
155
244
  ]
156
245
  )
157
246
  }
158
- const result = await PluginManager.INSTANCE.processBinding(request)
247
+ const result = await PluginManager.INSTANCE.processBinding(request, preparedData)
159
248
  recordRuntimeInfo(result, request.handlerType)
160
249
  return result
161
250
  }
@@ -181,33 +270,43 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
181
270
  requests: AsyncIterable<ProcessStreamRequest>,
182
271
  subject: Subject<DeepPartial<ProcessStreamResponse>>
183
272
  ) {
184
- const contexts: Record<number, StoreContext> = {}
273
+ const contexts = new Contexts()
185
274
 
186
275
  for await (const request of requests) {
187
- console.debug('received request:', request)
188
- if (request.binding) {
189
- const binding = request.binding
190
- const dbContext = new StoreContext(subject, request.processId)
191
- contexts[request.processId] = dbContext
192
- PluginManager.INSTANCE.processBinding(binding, dbContext)
193
- .then((result) => {
194
- subject.next({
195
- result,
196
- processId: request.processId
276
+ try {
277
+ console.debug('received request:', request)
278
+ if (request.binding) {
279
+ process_binding_count.add(1)
280
+ const binding = request.binding
281
+ const dbContext = contexts.new(request.processId, subject)
282
+ const start = Date.now()
283
+ PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
284
+ .then((result) => {
285
+ subject.next({
286
+ result,
287
+ processId: request.processId
288
+ })
289
+ recordRuntimeInfo(result, binding.handlerType)
197
290
  })
198
- recordRuntimeInfo(result, binding.handlerType)
199
- })
200
- .catch((e) => {
201
- console.error(e)
202
- dbContext.error(request.processId, e)
203
- })
204
- .finally(() => {
205
- delete contexts[request.processId]
206
- })
207
- }
208
- if (request.dbResult) {
209
- const dbContext = contexts[request.processId]
210
- dbContext?.result(request.dbResult)
291
+ .catch((e) => {
292
+ console.debug(e)
293
+ dbContext.error(request.processId, e)
294
+ process_binding_error.add(1)
295
+ })
296
+ .finally(() => {
297
+ const cost = Date.now() - start
298
+ console.debug('processBinding', request.processId, ' took', cost, 'ms')
299
+ process_binding_time.add(cost)
300
+ contexts.delete(request.processId)
301
+ })
302
+ }
303
+ if (request.dbResult) {
304
+ const dbContext = contexts.get(request.processId)
305
+ dbContext?.result(request.dbResult)
306
+ }
307
+ } catch (e) {
308
+ // should not happen
309
+ console.error('unexpect error during handle loop', e)
211
310
  }
212
311
  }
213
312
  }
@@ -222,3 +321,23 @@ function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
222
321
  })
223
322
  }
224
323
  }
324
+
325
+ class Contexts {
326
+ private contexts: Map<number, StoreContext> = new Map()
327
+
328
+ get(processId: number) {
329
+ return this.contexts.get(processId)
330
+ }
331
+
332
+ new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
333
+ const context = new StoreContext(subject, processId)
334
+ this.contexts.set(processId, context)
335
+ return context
336
+ }
337
+
338
+ delete(processId: number) {
339
+ const context = this.get(processId)
340
+ context?.close()
341
+ this.contexts.delete(processId)
342
+ }
343
+ }
@@ -0,0 +1,14 @@
1
+ import { defineConfig } from 'tsup'
2
+
3
+ export default defineConfig({
4
+ esbuildOptions: (options) => {
5
+ options.banner = {
6
+ js: `import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);`
7
+ }
8
+ },
9
+ entry: ['src/index.ts', 'src/processor-runner.ts'],
10
+ outDir: 'lib',
11
+ clean: true,
12
+ dts: true,
13
+ format: 'esm'
14
+ })
@@ -1,6 +0,0 @@
1
- export interface ChainConfig {
2
- ChainID: string;
3
- Https?: string[];
4
- ChainServer?: string;
5
- }
6
- //# sourceMappingURL=chain-config.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=chain-config.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
@@ -1,16 +0,0 @@
1
- import { Subject } from 'rxjs';
2
- import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
3
- type Request = Omit<DBRequest, 'opId'>;
4
- export declare class StoreContext {
5
- readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
6
- readonly processId: number;
7
- private static opCounter;
8
- private defers;
9
- constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
10
- newPromise<T>(opId: bigint): Promise<T>;
11
- sendRequest(request: DeepPartial<Request>): Promise<unknown>;
12
- result(dbResult: DBResponse): void;
13
- error(processId: number, e: any): void;
14
- }
15
- export {};
16
- //# sourceMappingURL=db-context.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;CAYhC"}
package/lib/db-context.js DELETED
@@ -1,56 +0,0 @@
1
- import { ProcessResult } from '@sentio/protos';
2
- export class StoreContext {
3
- subject;
4
- processId;
5
- static opCounter = 0n;
6
- defers = new Map();
7
- constructor(subject, processId) {
8
- this.subject = subject;
9
- this.processId = processId;
10
- }
11
- newPromise(opId) {
12
- return new Promise((resolve, reject) => {
13
- this.defers.set(opId, { resolve, reject });
14
- });
15
- }
16
- sendRequest(request) {
17
- const opId = StoreContext.opCounter++;
18
- const promise = this.newPromise(opId);
19
- console.debug('sending db request ', opId, request);
20
- this.subject.next({
21
- dbRequest: {
22
- ...request,
23
- opId
24
- },
25
- processId: this.processId
26
- });
27
- return promise;
28
- }
29
- result(dbResult) {
30
- const opId = dbResult.opId;
31
- const defer = this.defers.get(opId);
32
- console.debug('received db result ', opId, dbResult);
33
- if (defer) {
34
- if (dbResult.error) {
35
- defer.reject(new Error(dbResult.error));
36
- }
37
- else {
38
- defer.resolve(dbResult);
39
- }
40
- this.defers.delete(opId);
41
- }
42
- }
43
- error(processId, e) {
44
- console.error('process error', processId, e);
45
- const errorResult = ProcessResult.create({
46
- states: {
47
- error: e?.toString()
48
- }
49
- });
50
- this.subject.next({
51
- result: errorResult,
52
- processId
53
- });
54
- }
55
- }
56
- //# sourceMappingURL=db-context.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAAsC,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAIzG,MAAM,OAAO,YAAY;IAMZ;IACA;IANH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAA6E,CAAA;IAErG,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY;QACxB,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC"}
@@ -1,3 +0,0 @@
1
- #!/usr/bin/env node
2
- export {};
3
- //# sourceMappingURL=decode-benchmark.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"decode-benchmark.d.ts","sourceRoot":"","sources":["../src/decode-benchmark.ts"],"names":[],"mappings":""}
@@ -1,20 +0,0 @@
1
- #!/usr/bin/env node
2
- import fs from 'fs';
3
- const fileContent = fs.readFileSync('../../../chain/aptos/testdata/block-0-9999.json', { encoding: 'utf-8' });
4
- // console.log(fileContent)
5
- const contents = JSON.parse(fileContent);
6
- console.log(contents.length);
7
- const raws = [];
8
- for (const content of contents) {
9
- raws.push(new TextEncoder().encode(JSON.stringify(content)));
10
- }
11
- console.log('test begin');
12
- const start = Date.now();
13
- for (let i = 0; i < 10; i++) {
14
- for (const raw of raws) {
15
- const x = JSON.parse(new TextDecoder().decode(raw));
16
- }
17
- }
18
- const end = Date.now();
19
- console.log((end - start) / 1000.0, 'seconds');
20
- //# sourceMappingURL=decode-benchmark.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"decode-benchmark.js","sourceRoot":"","sources":["../src/decode-benchmark.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,iDAAiD,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;AAC7G,2BAA2B;AAE3B,MAAM,QAAQ,GAAa,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;AAElD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;AAE5B,MAAM,IAAI,GAAiB,EAAE,CAAA;AAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;IAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;AAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;AAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACrD,CAAC;AACH,CAAC;AAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;AACtB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,MAAM,EAAE,SAAS,CAAC,CAAA"}
@@ -1,9 +0,0 @@
1
- export declare class Endpoints {
2
- static INSTANCE: Endpoints;
3
- concurrency: number;
4
- chainQueryAPI: string;
5
- priceFeedAPI: string;
6
- chainServer: Map<string, string>;
7
- batchCount: number;
8
- }
9
- //# sourceMappingURL=endpoints.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"endpoints.d.ts","sourceRoot":"","sources":["../src/endpoints.ts"],"names":[],"mappings":"AAAA,qBAAa,SAAS;IACpB,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAkB;IAE5C,WAAW,SAAI;IACf,aAAa,SAAK;IAClB,YAAY,SAAK;IAEjB,WAAW,sBAA4B;IAEvC,UAAU,SAAI;CACf"}