@sentio/runtime 2.39.7-rc.9 → 2.40.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/chunk-FFU5RYDX.js +78856 -0
- package/lib/index.d.ts +371 -9
- package/lib/index.js +94 -9
- package/lib/processor-runner.d.ts +0 -2
- package/lib/processor-runner.js +51460 -129
- package/package.json +5 -25
- package/src/db-context.ts +157 -7
- package/src/full-service.ts +5 -0
- package/src/gen/processor/protos/processor.ts +2294 -1344
- package/src/plugin.ts +30 -4
- package/src/processor-runner.ts +26 -2
- package/src/provider.ts +168 -0
- package/src/service.ts +205 -28
- package/src/tsup.config.ts +14 -0
- package/src/utils.ts +11 -3
- package/lib/chain-config.d.ts +0 -6
- package/lib/chain-config.d.ts.map +0 -1
- package/lib/chain-config.js +0 -2
- package/lib/chain-config.js.map +0 -1
- package/lib/db-context.d.ts +0 -17
- package/lib/db-context.d.ts.map +0 -1
- package/lib/db-context.js +0 -63
- package/lib/db-context.js.map +0 -1
- package/lib/decode-benchmark.d.ts +0 -3
- package/lib/decode-benchmark.d.ts.map +0 -1
- package/lib/decode-benchmark.js +0 -20
- package/lib/decode-benchmark.js.map +0 -1
- package/lib/endpoints.d.ts +0 -9
- package/lib/endpoints.d.ts.map +0 -1
- package/lib/endpoints.js +0 -9
- package/lib/endpoints.js.map +0 -1
- package/lib/full-service.d.ts +0 -655
- package/lib/full-service.d.ts.map +0 -1
- package/lib/full-service.js +0 -137
- package/lib/full-service.js.map +0 -1
- package/lib/gen/google/protobuf/empty.d.ts +0 -17
- package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
- package/lib/gen/google/protobuf/empty.js +0 -40
- package/lib/gen/google/protobuf/empty.js.map +0 -1
- package/lib/gen/google/protobuf/struct.d.ts +0 -77
- package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
- package/lib/gen/google/protobuf/struct.js +0 -429
- package/lib/gen/google/protobuf/struct.js.map +0 -1
- package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
- package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
- package/lib/gen/google/protobuf/timestamp.js +0 -83
- package/lib/gen/google/protobuf/timestamp.js.map +0 -1
- package/lib/gen/processor/protos/processor.d.ts +0 -1470
- package/lib/gen/processor/protos/processor.d.ts.map +0 -1
- package/lib/gen/processor/protos/processor.js +0 -8512
- package/lib/gen/processor/protos/processor.js.map +0 -1
- package/lib/gen/service/common/protos/common.d.ts +0 -1698
- package/lib/gen/service/common/protos/common.d.ts.map +0 -1
- package/lib/gen/service/common/protos/common.js +0 -11383
- package/lib/gen/service/common/protos/common.js.map +0 -1
- package/lib/global-config.d.ts +0 -8
- package/lib/global-config.d.ts.map +0 -1
- package/lib/global-config.js +0 -23
- package/lib/global-config.js.map +0 -1
- package/lib/global-config.test.d.ts +0 -2
- package/lib/global-config.test.d.ts.map +0 -1
- package/lib/global-config.test.js.map +0 -1
- package/lib/index.d.ts.map +0 -1
- package/lib/index.js.map +0 -1
- package/lib/logger.d.ts +0 -2
- package/lib/logger.d.ts.map +0 -1
- package/lib/logger.js +0 -39
- package/lib/logger.js.map +0 -1
- package/lib/logger.test.d.ts +0 -2
- package/lib/logger.test.d.ts.map +0 -1
- package/lib/logger.test.js.map +0 -1
- package/lib/plugin.d.ts +0 -29
- package/lib/plugin.d.ts.map +0 -1
- package/lib/plugin.js +0 -58
- package/lib/plugin.js.map +0 -1
- package/lib/processor-runner.d.ts.map +0 -1
- package/lib/processor-runner.js.map +0 -1
- package/lib/seq-mode.test.d.ts +0 -3
- package/lib/seq-mode.test.d.ts.map +0 -1
- package/lib/seq-mode.test.js.map +0 -1
- package/lib/service.d.ts +0 -179
- package/lib/service.d.ts.map +0 -1
- package/lib/service.js +0 -194
- package/lib/service.js.map +0 -1
- package/lib/service.test.d.ts +0 -3
- package/lib/service.test.d.ts.map +0 -1
- package/lib/service.test.js.map +0 -1
- package/lib/state-storage.test.d.ts +0 -2
- package/lib/state-storage.test.d.ts.map +0 -1
- package/lib/state-storage.test.js.map +0 -1
- package/lib/state.d.ts +0 -23
- package/lib/state.d.ts.map +0 -1
- package/lib/state.js +0 -61
- package/lib/state.js.map +0 -1
- package/lib/utils.d.ts +0 -6
- package/lib/utils.d.ts.map +0 -1
- package/lib/utils.js +0 -23
- package/lib/utils.js.map +0 -1
package/src/plugin.ts
CHANGED
@@ -1,4 +1,12 @@
|
|
1
|
-
import {
|
1
|
+
import {
|
2
|
+
DataBinding,
|
3
|
+
HandlerType,
|
4
|
+
PreparedData,
|
5
|
+
PreprocessResult,
|
6
|
+
ProcessConfigResponse,
|
7
|
+
ProcessResult,
|
8
|
+
StartRequest
|
9
|
+
} from '@sentio/protos'
|
2
10
|
import { StoreContext } from './db-context.js'
|
3
11
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
4
12
|
|
@@ -16,9 +24,13 @@ export abstract class Plugin {
|
|
16
24
|
return false
|
17
25
|
}
|
18
26
|
|
19
|
-
async processBinding(request: DataBinding): Promise<ProcessResult> {
|
27
|
+
async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
|
20
28
|
return ProcessResult.create()
|
21
29
|
}
|
30
|
+
|
31
|
+
async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
|
32
|
+
return PreprocessResult.create()
|
33
|
+
}
|
22
34
|
}
|
23
35
|
|
24
36
|
export class PluginManager {
|
@@ -58,13 +70,27 @@ export class PluginManager {
|
|
58
70
|
return this.plugins.some((plugin) => plugin.stateDiff(config))
|
59
71
|
}
|
60
72
|
|
61
|
-
processBinding(
|
73
|
+
processBinding(
|
74
|
+
request: DataBinding,
|
75
|
+
preparedData: PreparedData | undefined,
|
76
|
+
dbContext?: StoreContext
|
77
|
+
): Promise<ProcessResult> {
|
78
|
+
const plugin = this.typesToPlugin.get(request.handlerType)
|
79
|
+
if (!plugin) {
|
80
|
+
throw new Error(`No plugin for ${request.handlerType}`)
|
81
|
+
}
|
82
|
+
return this.dbContextLocalStorage.run(dbContext, () => {
|
83
|
+
return plugin.processBinding(request, preparedData)
|
84
|
+
})
|
85
|
+
}
|
86
|
+
|
87
|
+
preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
|
62
88
|
const plugin = this.typesToPlugin.get(request.handlerType)
|
63
89
|
if (!plugin) {
|
64
90
|
throw new Error(`No plugin for ${request.handlerType}`)
|
65
91
|
}
|
66
92
|
return this.dbContextLocalStorage.run(dbContext, () => {
|
67
|
-
return plugin.
|
93
|
+
return plugin.preprocessBinding(request)
|
68
94
|
})
|
69
95
|
}
|
70
96
|
}
|
package/src/processor-runner.ts
CHANGED
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
|
|
8
8
|
import { createServer } from 'nice-grpc'
|
9
9
|
import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
|
10
10
|
import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
|
11
|
+
import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
|
11
12
|
import { register as globalRegistry, Registry } from 'prom-client'
|
12
13
|
import http from 'http'
|
13
14
|
// @ts-ignore inspector promises is not included in @type/node
|
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
|
|
20
21
|
import { ChainConfig } from './chain-config.js'
|
21
22
|
import { setupLogger } from './logger.js'
|
22
23
|
|
24
|
+
import { NodeSDK } from '@opentelemetry/sdk-node'
|
25
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
|
26
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
|
27
|
+
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
|
28
|
+
|
29
|
+
const sdk = new NodeSDK({
|
30
|
+
traceExporter: new OTLPTraceExporter(),
|
31
|
+
metricReader: new PeriodicExportingMetricReader({
|
32
|
+
exporter: new OTLPMetricExporter()
|
33
|
+
})
|
34
|
+
// instrumentations: [getNodeAutoInstrumentations()],
|
35
|
+
})
|
36
|
+
|
37
|
+
sdk.start()
|
38
|
+
|
23
39
|
const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
|
24
40
|
|
25
41
|
const optionDefinitions = [
|
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
|
|
50
66
|
const fullPath = path.resolve(options['chains-config'])
|
51
67
|
const chainsConfig = fs.readJsonSync(fullPath)
|
52
68
|
|
53
|
-
|
54
|
-
|
69
|
+
const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
|
70
|
+
? parseInt(process.env['OVERRIDE_CONCURRENCY'])
|
71
|
+
: undefined
|
72
|
+
const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
|
73
|
+
? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
|
74
|
+
: undefined
|
75
|
+
|
76
|
+
Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
|
77
|
+
Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
|
55
78
|
Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
|
56
79
|
Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
|
57
80
|
|
@@ -77,6 +100,7 @@ const server = createServer({
|
|
77
100
|
'grpc.default_compression_algorithm': compressionAlgorithms.gzip
|
78
101
|
})
|
79
102
|
.use(prometheusServerMiddleware())
|
103
|
+
.use(openTelemetryServerMiddleware())
|
80
104
|
.use(errorDetailsServerMiddleware)
|
81
105
|
const baseService = new ProcessorServiceImpl(async () => {
|
82
106
|
const m = await import(options.target)
|
package/src/provider.ts
ADDED
@@ -0,0 +1,168 @@
|
|
1
|
+
import { JsonRpcProvider, Network, Provider } from 'ethers'
|
2
|
+
|
3
|
+
import PQueue from 'p-queue'
|
4
|
+
import { Endpoints } from './endpoints.js'
|
5
|
+
import { EthChainId } from '@sentio/chain'
|
6
|
+
import { LRUCache } from 'lru-cache'
|
7
|
+
import { metrics } from '@opentelemetry/api'
|
8
|
+
|
9
|
+
export const DummyProvider = new JsonRpcProvider('', Network.from(1))
|
10
|
+
|
11
|
+
const meter = metrics.getMeter('processor_provider')
|
12
|
+
const hit_count = meter.createCounter('provider_hit_count')
|
13
|
+
const miss_count = meter.createCounter('provider_miss_count')
|
14
|
+
const queue_size = meter.createGauge('provider_queue_size')
|
15
|
+
const total_duration = meter.createCounter('provider_total_duration')
|
16
|
+
const total_queued = meter.createCounter('provider_total_queued')
|
17
|
+
|
18
|
+
const providers = new Map<string, JsonRpcProvider>()
|
19
|
+
|
20
|
+
// export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
|
21
|
+
// if (!networkish) {
|
22
|
+
// networkish = EthChainId.ETHEREUM
|
23
|
+
// }
|
24
|
+
// if (networkish instanceof BaseContext) {
|
25
|
+
// networkish = networkish.getChainId()
|
26
|
+
// }
|
27
|
+
// return networkish
|
28
|
+
// }
|
29
|
+
|
30
|
+
export function getProvider(chainId?: EthChainId): Provider {
|
31
|
+
// const network = getNetworkFromCtxOrNetworkish(networkish)
|
32
|
+
if (!chainId) {
|
33
|
+
chainId = EthChainId.ETHEREUM
|
34
|
+
}
|
35
|
+
const network = Network.from(parseInt(chainId))
|
36
|
+
// TODO check if other key needed
|
37
|
+
|
38
|
+
const address = Endpoints.INSTANCE.chainServer.get(chainId)
|
39
|
+
const key = network.chainId.toString() + '-' + address
|
40
|
+
|
41
|
+
console.debug(`init provider for ${chainId}, address: ${address}`)
|
42
|
+
let provider = providers.get(key)
|
43
|
+
|
44
|
+
if (provider) {
|
45
|
+
return provider
|
46
|
+
}
|
47
|
+
|
48
|
+
if (address === undefined) {
|
49
|
+
throw Error(
|
50
|
+
'Provider not found for chain ' +
|
51
|
+
network.chainId +
|
52
|
+
', configured chains: ' +
|
53
|
+
[...Endpoints.INSTANCE.chainServer.keys()].join(' ')
|
54
|
+
)
|
55
|
+
}
|
56
|
+
console.log(
|
57
|
+
`init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
|
58
|
+
)
|
59
|
+
provider = new QueuedStaticJsonRpcProvider(
|
60
|
+
address,
|
61
|
+
network,
|
62
|
+
Endpoints.INSTANCE.concurrency,
|
63
|
+
Endpoints.INSTANCE.batchCount
|
64
|
+
)
|
65
|
+
providers.set(key, provider)
|
66
|
+
return provider
|
67
|
+
}
|
68
|
+
|
69
|
+
function getTag(prefix: string, value: any): string {
|
70
|
+
return (
|
71
|
+
prefix +
|
72
|
+
':' +
|
73
|
+
JSON.stringify(value, (k, v) => {
|
74
|
+
if (v == null) {
|
75
|
+
return 'null'
|
76
|
+
}
|
77
|
+
if (typeof v === 'bigint') {
|
78
|
+
return `bigint:${v.toString()}`
|
79
|
+
}
|
80
|
+
if (typeof v === 'string') {
|
81
|
+
return v.toLowerCase()
|
82
|
+
}
|
83
|
+
|
84
|
+
// Sort object keys
|
85
|
+
if (typeof v === 'object' && !Array.isArray(v)) {
|
86
|
+
const keys = Object.keys(v)
|
87
|
+
keys.sort()
|
88
|
+
return keys.reduce(
|
89
|
+
(accum, key) => {
|
90
|
+
accum[key] = v[key]
|
91
|
+
return accum
|
92
|
+
},
|
93
|
+
<any>{}
|
94
|
+
)
|
95
|
+
}
|
96
|
+
|
97
|
+
return v
|
98
|
+
})
|
99
|
+
)
|
100
|
+
}
|
101
|
+
|
102
|
+
class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
|
103
|
+
executor: PQueue
|
104
|
+
#performCache = new LRUCache<string, Promise<any>>({
|
105
|
+
max: 300000 // 300k items
|
106
|
+
// maxSize: 300 * 1024 * 1024, // 300mb for cache
|
107
|
+
// ttl: 1000 * 60 * 60, // 1 hour no ttl for better performance
|
108
|
+
// sizeCalculation: (value: any) => {
|
109
|
+
// assume each item is 1kb for simplicity
|
110
|
+
// return 1024
|
111
|
+
// }
|
112
|
+
})
|
113
|
+
|
114
|
+
constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
|
115
|
+
// TODO re-enable match when possible
|
116
|
+
super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
|
117
|
+
this.executor = new PQueue({ concurrency: concurrency })
|
118
|
+
}
|
119
|
+
|
120
|
+
async send(method: string, params: Array<any>): Promise<any> {
|
121
|
+
if (method !== 'eth_call') {
|
122
|
+
return await this.executor.add(() => super.send(method, params))
|
123
|
+
}
|
124
|
+
const tag = getTag(method, params)
|
125
|
+
const block = params[params.length - 1]
|
126
|
+
let perform = this.#performCache.get(tag)
|
127
|
+
if (!perform) {
|
128
|
+
miss_count.add(1)
|
129
|
+
const queued: number = Date.now()
|
130
|
+
perform = this.executor.add(() => {
|
131
|
+
const started = Date.now()
|
132
|
+
total_queued.add(started - queued)
|
133
|
+
|
134
|
+
return super.send(method, params).finally(() => {
|
135
|
+
total_duration.add(Date.now() - started)
|
136
|
+
})
|
137
|
+
})
|
138
|
+
perform.catch((e) => {
|
139
|
+
// if (e.code !== 'CALL_EXCEPTION' && e.code !== 'BAD_DATA') {
|
140
|
+
setTimeout(() => {
|
141
|
+
if (this.#performCache.get(tag) === perform) {
|
142
|
+
this.#performCache.delete(tag)
|
143
|
+
}
|
144
|
+
}, 1000)
|
145
|
+
})
|
146
|
+
|
147
|
+
queue_size.record(this.executor.size)
|
148
|
+
|
149
|
+
this.#performCache.set(tag, perform)
|
150
|
+
// For non latest block call, we cache permanently, otherwise we cache for one minute
|
151
|
+
if (block === 'latest') {
|
152
|
+
setTimeout(() => {
|
153
|
+
if (this.#performCache.get(tag) === perform) {
|
154
|
+
this.#performCache.delete(tag)
|
155
|
+
}
|
156
|
+
}, 60 * 1000)
|
157
|
+
}
|
158
|
+
} else {
|
159
|
+
hit_count.add(1)
|
160
|
+
}
|
161
|
+
|
162
|
+
const result = await perform
|
163
|
+
if (!result) {
|
164
|
+
throw Error('Unexpected null response')
|
165
|
+
}
|
166
|
+
return result
|
167
|
+
}
|
168
|
+
}
|
package/src/service.ts
CHANGED
@@ -7,7 +7,12 @@ import {
|
|
7
7
|
DataBinding,
|
8
8
|
DeepPartial,
|
9
9
|
Empty,
|
10
|
+
EthCallParam,
|
10
11
|
HandlerType,
|
12
|
+
PreparedData,
|
13
|
+
PreprocessResult,
|
14
|
+
PreprocessStreamRequest,
|
15
|
+
PreprocessStreamResponse,
|
11
16
|
ProcessBindingResponse,
|
12
17
|
ProcessBindingsRequest,
|
13
18
|
ProcessConfigRequest,
|
@@ -20,11 +25,21 @@ import {
|
|
20
25
|
} from '@sentio/protos'
|
21
26
|
|
22
27
|
import { PluginManager } from './plugin.js'
|
23
|
-
import { errorString, mergeProcessResults } from './utils.js'
|
28
|
+
import { errorString, makeEthCallKey, mergeProcessResults } from './utils.js'
|
24
29
|
import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
|
25
30
|
|
26
31
|
import { StoreContext } from './db-context.js'
|
27
32
|
import { Subject } from 'rxjs'
|
33
|
+
import { metrics } from '@opentelemetry/api'
|
34
|
+
import { getProvider } from './provider.js'
|
35
|
+
import { EthChainId } from '@sentio/chain'
|
36
|
+
import { Provider } from 'ethers'
|
37
|
+
|
38
|
+
const meter = metrics.getMeter('processor_service')
|
39
|
+
const process_binding_count = meter.createCounter('process_binding_count')
|
40
|
+
const process_binding_time = meter.createCounter('process_binding_time')
|
41
|
+
const process_binding_error = meter.createCounter('process_binding_error')
|
42
|
+
|
28
43
|
;(BigInt.prototype as any).toJSON = function () {
|
29
44
|
return this.toString()
|
30
45
|
}
|
@@ -39,6 +54,8 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
39
54
|
|
40
55
|
private readonly shutdownHandler?: () => void
|
41
56
|
|
57
|
+
private readonly preprocessedEthCalls: { [calldata: string]: any[] }
|
58
|
+
|
42
59
|
constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
|
43
60
|
this.loader = loader
|
44
61
|
this.shutdownHandler = shutdownHandler
|
@@ -111,10 +128,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
111
128
|
}
|
112
129
|
|
113
130
|
async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
|
114
|
-
const
|
131
|
+
const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
|
115
132
|
|
133
|
+
const promises = []
|
116
134
|
for (const binding of request.bindings) {
|
117
|
-
const promise = this.processBinding(binding)
|
135
|
+
const promise = this.processBinding(binding, { ethCallResults })
|
118
136
|
if (GLOBAL_CONFIG.execution.sequential) {
|
119
137
|
await promise
|
120
138
|
}
|
@@ -139,7 +157,99 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
139
157
|
}
|
140
158
|
}
|
141
159
|
|
142
|
-
async
|
160
|
+
async preprocessBindings(
|
161
|
+
bindings: DataBinding[],
|
162
|
+
dbContext?: StoreContext,
|
163
|
+
options?: CallContext
|
164
|
+
): Promise<{ [ethCallKey: string]: string }> {
|
165
|
+
console.debug(`preprocessBindings start, bindings: ${bindings.length}`)
|
166
|
+
const promises = []
|
167
|
+
for (const binding of bindings) {
|
168
|
+
promises.push(this.preprocessBinding(binding, dbContext, options))
|
169
|
+
}
|
170
|
+
let preprocessResults: PreprocessResult[]
|
171
|
+
try {
|
172
|
+
preprocessResults = await Promise.all(promises)
|
173
|
+
} catch (e) {
|
174
|
+
throw e
|
175
|
+
}
|
176
|
+
console.debug(
|
177
|
+
'ethCallParams: ',
|
178
|
+
preprocessResults.map((r) => r.ethCallParams)
|
179
|
+
)
|
180
|
+
const groupedRequests = new Map<string, EthCallParam[]>()
|
181
|
+
const providers = new Map<string, Provider>()
|
182
|
+
for (const result of preprocessResults) {
|
183
|
+
for (const param of result.ethCallParams) {
|
184
|
+
const { chainId, address, blockTag } = param.context!
|
185
|
+
if (!providers.has(chainId)) {
|
186
|
+
providers.set(chainId, getProvider(chainId as EthChainId))
|
187
|
+
}
|
188
|
+
const key = [chainId, address, blockTag].join('|')
|
189
|
+
if (!groupedRequests.has(key)) {
|
190
|
+
groupedRequests.set(key, [])
|
191
|
+
}
|
192
|
+
groupedRequests.get(key)!.push(param)
|
193
|
+
}
|
194
|
+
}
|
195
|
+
|
196
|
+
const start = Date.now()
|
197
|
+
const callPromises = []
|
198
|
+
for (const params of groupedRequests.values()) {
|
199
|
+
const { chainId, address, blockTag } = params[0].context!
|
200
|
+
console.log(`chain: ${chainId}, address: ${address}, blockTag: ${blockTag}, totalCalls: ${params.length}`)
|
201
|
+
// TODO multicall
|
202
|
+
for (const param of params) {
|
203
|
+
callPromises.push(
|
204
|
+
providers
|
205
|
+
.get(chainId)!
|
206
|
+
.call({
|
207
|
+
to: address,
|
208
|
+
data: param.calldata,
|
209
|
+
blockTag
|
210
|
+
})
|
211
|
+
.then((result) => [makeEthCallKey(param), result])
|
212
|
+
)
|
213
|
+
}
|
214
|
+
}
|
215
|
+
let results = {}
|
216
|
+
try {
|
217
|
+
results = Object.fromEntries(await Promise.all(callPromises))
|
218
|
+
} catch (e) {
|
219
|
+
console.error(`eth call error: ${e}`)
|
220
|
+
}
|
221
|
+
console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
|
222
|
+
return results
|
223
|
+
}
|
224
|
+
|
225
|
+
async preprocessBinding(
|
226
|
+
request: DataBinding,
|
227
|
+
dbContext?: StoreContext,
|
228
|
+
options?: CallContext
|
229
|
+
): Promise<PreprocessResult> {
|
230
|
+
if (!this.started) {
|
231
|
+
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
232
|
+
}
|
233
|
+
if (this.unhandled) {
|
234
|
+
throw new RichServerError(
|
235
|
+
Status.UNAVAILABLE,
|
236
|
+
'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
|
237
|
+
[
|
238
|
+
DebugInfo.fromPartial({
|
239
|
+
detail: this.unhandled.message,
|
240
|
+
stackEntries: this.unhandled.stack?.split('\n')
|
241
|
+
})
|
242
|
+
]
|
243
|
+
)
|
244
|
+
}
|
245
|
+
return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
|
246
|
+
}
|
247
|
+
|
248
|
+
async processBinding(
|
249
|
+
request: DataBinding,
|
250
|
+
preparedData: PreparedData | undefined,
|
251
|
+
options?: CallContext
|
252
|
+
): Promise<ProcessResult> {
|
143
253
|
if (!this.started) {
|
144
254
|
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
145
255
|
}
|
@@ -155,7 +265,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
155
265
|
]
|
156
266
|
)
|
157
267
|
}
|
158
|
-
const result = await PluginManager.INSTANCE.processBinding(request)
|
268
|
+
const result = await PluginManager.INSTANCE.processBinding(request, preparedData)
|
159
269
|
recordRuntimeInfo(result, request.handlerType)
|
160
270
|
return result
|
161
271
|
}
|
@@ -177,6 +287,63 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
177
287
|
yield* from(subject).pipe(withAbort(context.signal))
|
178
288
|
}
|
179
289
|
|
290
|
+
async handlePreprocessRequests(
|
291
|
+
requests: AsyncIterable<PreprocessStreamRequest>,
|
292
|
+
subject: Subject<DeepPartial<PreprocessStreamResponse>>
|
293
|
+
) {
|
294
|
+
const contexts = new Contexts()
|
295
|
+
|
296
|
+
for await (const request of requests) {
|
297
|
+
try {
|
298
|
+
console.debug('received request:', request)
|
299
|
+
if (request.bindings) {
|
300
|
+
const bindings = request.bindings.bindings
|
301
|
+
const dbContext = contexts.new(request.processId, subject)
|
302
|
+
const start = Date.now()
|
303
|
+
this.preprocessBindings(bindings, dbContext)
|
304
|
+
.then(() => {
|
305
|
+
subject.next({
|
306
|
+
processId: request.processId
|
307
|
+
})
|
308
|
+
})
|
309
|
+
.catch((e) => {
|
310
|
+
console.debug(e)
|
311
|
+
dbContext.error(request.processId, e)
|
312
|
+
})
|
313
|
+
.finally(() => {
|
314
|
+
const cost = Date.now() - start
|
315
|
+
console.debug('preprocessBinding', request.processId, ' took', cost, 'ms')
|
316
|
+
contexts.delete(request.processId)
|
317
|
+
})
|
318
|
+
}
|
319
|
+
if (request.dbResult) {
|
320
|
+
const dbContext = contexts.get(request.processId)
|
321
|
+
dbContext?.result(request.dbResult)
|
322
|
+
}
|
323
|
+
} catch (e) {
|
324
|
+
// should not happen
|
325
|
+
console.error('unexpect error during handle loop', e)
|
326
|
+
}
|
327
|
+
}
|
328
|
+
}
|
329
|
+
|
330
|
+
async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
|
331
|
+
if (!this.started) {
|
332
|
+
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
333
|
+
}
|
334
|
+
|
335
|
+
const subject = new Subject<DeepPartial<PreprocessStreamResponse>>()
|
336
|
+
this.handlePreprocessRequests(requests, subject)
|
337
|
+
.then(() => {
|
338
|
+
subject.complete()
|
339
|
+
})
|
340
|
+
.catch((e) => {
|
341
|
+
console.error(e)
|
342
|
+
subject.error(e)
|
343
|
+
})
|
344
|
+
yield* from(subject).pipe(withAbort(context.signal))
|
345
|
+
}
|
346
|
+
|
180
347
|
private async handleRequests(
|
181
348
|
requests: AsyncIterable<ProcessStreamRequest>,
|
182
349
|
subject: Subject<DeepPartial<ProcessStreamResponse>>
|
@@ -184,30 +351,40 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
184
351
|
const contexts = new Contexts()
|
185
352
|
|
186
353
|
for await (const request of requests) {
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
.
|
194
|
-
|
195
|
-
|
196
|
-
|
354
|
+
try {
|
355
|
+
console.debug('received request:', request)
|
356
|
+
if (request.binding) {
|
357
|
+
process_binding_count.add(1)
|
358
|
+
const binding = request.binding
|
359
|
+
const dbContext = contexts.new(request.processId, subject)
|
360
|
+
const start = Date.now()
|
361
|
+
PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
|
362
|
+
.then((result) => {
|
363
|
+
subject.next({
|
364
|
+
result,
|
365
|
+
processId: request.processId
|
366
|
+
})
|
367
|
+
recordRuntimeInfo(result, binding.handlerType)
|
197
368
|
})
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
369
|
+
.catch((e) => {
|
370
|
+
console.debug(e)
|
371
|
+
dbContext.error(request.processId, e)
|
372
|
+
process_binding_error.add(1)
|
373
|
+
})
|
374
|
+
.finally(() => {
|
375
|
+
const cost = Date.now() - start
|
376
|
+
console.debug('processBinding', request.processId, ' took', cost, 'ms')
|
377
|
+
process_binding_time.add(cost)
|
378
|
+
contexts.delete(request.processId)
|
379
|
+
})
|
380
|
+
}
|
381
|
+
if (request.dbResult) {
|
382
|
+
const dbContext = contexts.get(request.processId)
|
383
|
+
dbContext?.result(request.dbResult)
|
384
|
+
}
|
385
|
+
} catch (e) {
|
386
|
+
// should not happen
|
387
|
+
console.error('unexpect error during handle loop', e)
|
211
388
|
}
|
212
389
|
}
|
213
390
|
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
import { defineConfig } from 'tsup'
|
2
|
+
|
3
|
+
export default defineConfig({
|
4
|
+
esbuildOptions: (options) => {
|
5
|
+
options.banner = {
|
6
|
+
js: `import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);`
|
7
|
+
}
|
8
|
+
},
|
9
|
+
entry: ['src/index.ts', 'src/processor-runner.ts'],
|
10
|
+
outDir: 'lib',
|
11
|
+
clean: true,
|
12
|
+
dts: true,
|
13
|
+
format: 'esm'
|
14
|
+
})
|
package/src/utils.ts
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
import { ProcessResult } from '@sentio/protos'
|
1
|
+
import { EthCallParam, ProcessResult } from '@sentio/protos'
|
2
2
|
|
3
3
|
// TODO better handling this, because old proto doesn't have this
|
4
4
|
import { StateResult, ProcessResult as ProcessResultFull } from './gen/processor/protos/processor.js'
|
@@ -8,7 +8,7 @@ import { Required } from 'utility-types'
|
|
8
8
|
export function mergeProcessResults(results: ProcessResult[]): Required<ProcessResult, 'states'> {
|
9
9
|
const res = {
|
10
10
|
...ProcessResultFull.create(),
|
11
|
-
states: StateResult.create()
|
11
|
+
states: StateResult.create()
|
12
12
|
}
|
13
13
|
|
14
14
|
for (const r of results) {
|
@@ -17,7 +17,7 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
|
|
17
17
|
res.events = res.events.concat(r.events)
|
18
18
|
res.exports = res.exports.concat(r.exports)
|
19
19
|
res.states = {
|
20
|
-
configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
|
20
|
+
configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
|
21
21
|
}
|
22
22
|
}
|
23
23
|
return res
|
@@ -28,3 +28,11 @@ export function errorString(e: Error): string {
|
|
28
28
|
}
|
29
29
|
|
30
30
|
export const USER_PROCESSOR = 'user_processor'
|
31
|
+
|
32
|
+
export function makeEthCallKey(param: EthCallParam) {
|
33
|
+
if (!param.context) {
|
34
|
+
throw new Error('null context for eth call')
|
35
|
+
}
|
36
|
+
const { chainId, address, blockTag } = param.context
|
37
|
+
return `${chainId}|${address}|${blockTag}|${param.calldata}`
|
38
|
+
}
|
package/lib/chain-config.d.ts
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
|
package/lib/chain-config.js
DELETED
package/lib/chain-config.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
|
package/lib/db-context.d.ts
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
import { Subject } from 'rxjs';
|
2
|
-
import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
|
3
|
-
type Request = Omit<DBRequest, 'opId'>;
|
4
|
-
export declare class StoreContext {
|
5
|
-
readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
|
6
|
-
readonly processId: number;
|
7
|
-
private static opCounter;
|
8
|
-
private defers;
|
9
|
-
constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
|
10
|
-
newPromise<T>(opId: bigint): Promise<T>;
|
11
|
-
sendRequest(request: DeepPartial<Request>): Promise<unknown>;
|
12
|
-
result(dbResult: DBResponse): void;
|
13
|
-
error(processId: number, e: any): void;
|
14
|
-
close(): void;
|
15
|
-
}
|
16
|
-
export {};
|
17
|
-
//# sourceMappingURL=db-context.d.ts.map
|
package/lib/db-context.d.ts.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;IAa/B,KAAK;CAON"}
|