@sentio/runtime 2.39.7-rc.9 → 2.40.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/chunk-LMUCLUM5.js +78161 -0
- package/lib/index.d.ts +308 -9
- package/lib/index.js +92 -9
- package/lib/processor-runner.d.ts +0 -2
- package/lib/processor-runner.js +51457 -129
- package/package.json +5 -25
- package/src/db-context.ts +157 -7
- package/src/gen/processor/protos/processor.ts +361 -1
- package/src/plugin.ts +30 -4
- package/src/processor-runner.ts +26 -2
- package/src/provider.ts +166 -0
- package/src/service.ts +126 -27
- package/src/tsup.config.ts +14 -0
- package/lib/chain-config.d.ts +0 -6
- package/lib/chain-config.d.ts.map +0 -1
- package/lib/chain-config.js +0 -2
- package/lib/chain-config.js.map +0 -1
- package/lib/db-context.d.ts +0 -17
- package/lib/db-context.d.ts.map +0 -1
- package/lib/db-context.js +0 -63
- package/lib/db-context.js.map +0 -1
- package/lib/decode-benchmark.d.ts +0 -3
- package/lib/decode-benchmark.d.ts.map +0 -1
- package/lib/decode-benchmark.js +0 -20
- package/lib/decode-benchmark.js.map +0 -1
- package/lib/endpoints.d.ts +0 -9
- package/lib/endpoints.d.ts.map +0 -1
- package/lib/endpoints.js +0 -9
- package/lib/endpoints.js.map +0 -1
- package/lib/full-service.d.ts +0 -655
- package/lib/full-service.d.ts.map +0 -1
- package/lib/full-service.js +0 -137
- package/lib/full-service.js.map +0 -1
- package/lib/gen/google/protobuf/empty.d.ts +0 -17
- package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
- package/lib/gen/google/protobuf/empty.js +0 -40
- package/lib/gen/google/protobuf/empty.js.map +0 -1
- package/lib/gen/google/protobuf/struct.d.ts +0 -77
- package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
- package/lib/gen/google/protobuf/struct.js +0 -429
- package/lib/gen/google/protobuf/struct.js.map +0 -1
- package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
- package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
- package/lib/gen/google/protobuf/timestamp.js +0 -83
- package/lib/gen/google/protobuf/timestamp.js.map +0 -1
- package/lib/gen/processor/protos/processor.d.ts +0 -1470
- package/lib/gen/processor/protos/processor.d.ts.map +0 -1
- package/lib/gen/processor/protos/processor.js +0 -8512
- package/lib/gen/processor/protos/processor.js.map +0 -1
- package/lib/gen/service/common/protos/common.d.ts +0 -1698
- package/lib/gen/service/common/protos/common.d.ts.map +0 -1
- package/lib/gen/service/common/protos/common.js +0 -11383
- package/lib/gen/service/common/protos/common.js.map +0 -1
- package/lib/global-config.d.ts +0 -8
- package/lib/global-config.d.ts.map +0 -1
- package/lib/global-config.js +0 -23
- package/lib/global-config.js.map +0 -1
- package/lib/global-config.test.d.ts +0 -2
- package/lib/global-config.test.d.ts.map +0 -1
- package/lib/global-config.test.js.map +0 -1
- package/lib/index.d.ts.map +0 -1
- package/lib/index.js.map +0 -1
- package/lib/logger.d.ts +0 -2
- package/lib/logger.d.ts.map +0 -1
- package/lib/logger.js +0 -39
- package/lib/logger.js.map +0 -1
- package/lib/logger.test.d.ts +0 -2
- package/lib/logger.test.d.ts.map +0 -1
- package/lib/logger.test.js.map +0 -1
- package/lib/plugin.d.ts +0 -29
- package/lib/plugin.d.ts.map +0 -1
- package/lib/plugin.js +0 -58
- package/lib/plugin.js.map +0 -1
- package/lib/processor-runner.d.ts.map +0 -1
- package/lib/processor-runner.js.map +0 -1
- package/lib/seq-mode.test.d.ts +0 -3
- package/lib/seq-mode.test.d.ts.map +0 -1
- package/lib/seq-mode.test.js.map +0 -1
- package/lib/service.d.ts +0 -179
- package/lib/service.d.ts.map +0 -1
- package/lib/service.js +0 -194
- package/lib/service.js.map +0 -1
- package/lib/service.test.d.ts +0 -3
- package/lib/service.test.d.ts.map +0 -1
- package/lib/service.test.js.map +0 -1
- package/lib/state-storage.test.d.ts +0 -2
- package/lib/state-storage.test.d.ts.map +0 -1
- package/lib/state-storage.test.js.map +0 -1
- package/lib/state.d.ts +0 -23
- package/lib/state.d.ts.map +0 -1
- package/lib/state.js +0 -61
- package/lib/state.js.map +0 -1
- package/lib/utils.d.ts +0 -6
- package/lib/utils.d.ts.map +0 -1
- package/lib/utils.js +0 -23
- package/lib/utils.js.map +0 -1
package/src/plugin.ts
CHANGED
@@ -1,4 +1,12 @@
|
|
1
|
-
import {
|
1
|
+
import {
|
2
|
+
DataBinding,
|
3
|
+
HandlerType,
|
4
|
+
PreparedData,
|
5
|
+
PreprocessResult,
|
6
|
+
ProcessConfigResponse,
|
7
|
+
ProcessResult,
|
8
|
+
StartRequest
|
9
|
+
} from '@sentio/protos'
|
2
10
|
import { StoreContext } from './db-context.js'
|
3
11
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
4
12
|
|
@@ -16,9 +24,13 @@ export abstract class Plugin {
|
|
16
24
|
return false
|
17
25
|
}
|
18
26
|
|
19
|
-
async processBinding(request: DataBinding): Promise<ProcessResult> {
|
27
|
+
async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
|
20
28
|
return ProcessResult.create()
|
21
29
|
}
|
30
|
+
|
31
|
+
async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
|
32
|
+
return PreprocessResult.create()
|
33
|
+
}
|
22
34
|
}
|
23
35
|
|
24
36
|
export class PluginManager {
|
@@ -58,13 +70,27 @@ export class PluginManager {
|
|
58
70
|
return this.plugins.some((plugin) => plugin.stateDiff(config))
|
59
71
|
}
|
60
72
|
|
61
|
-
processBinding(
|
73
|
+
processBinding(
|
74
|
+
request: DataBinding,
|
75
|
+
preparedData: PreparedData | undefined,
|
76
|
+
dbContext?: StoreContext
|
77
|
+
): Promise<ProcessResult> {
|
78
|
+
const plugin = this.typesToPlugin.get(request.handlerType)
|
79
|
+
if (!plugin) {
|
80
|
+
throw new Error(`No plugin for ${request.handlerType}`)
|
81
|
+
}
|
82
|
+
return this.dbContextLocalStorage.run(dbContext, () => {
|
83
|
+
return plugin.processBinding(request, preparedData)
|
84
|
+
})
|
85
|
+
}
|
86
|
+
|
87
|
+
preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
|
62
88
|
const plugin = this.typesToPlugin.get(request.handlerType)
|
63
89
|
if (!plugin) {
|
64
90
|
throw new Error(`No plugin for ${request.handlerType}`)
|
65
91
|
}
|
66
92
|
return this.dbContextLocalStorage.run(dbContext, () => {
|
67
|
-
return plugin.
|
93
|
+
return plugin.preprocessBinding(request)
|
68
94
|
})
|
69
95
|
}
|
70
96
|
}
|
package/src/processor-runner.ts
CHANGED
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
|
|
8
8
|
import { createServer } from 'nice-grpc'
|
9
9
|
import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
|
10
10
|
import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
|
11
|
+
import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
|
11
12
|
import { register as globalRegistry, Registry } from 'prom-client'
|
12
13
|
import http from 'http'
|
13
14
|
// @ts-ignore inspector promises is not included in @type/node
|
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
|
|
20
21
|
import { ChainConfig } from './chain-config.js'
|
21
22
|
import { setupLogger } from './logger.js'
|
22
23
|
|
24
|
+
import { NodeSDK } from '@opentelemetry/sdk-node'
|
25
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
|
26
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
|
27
|
+
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
|
28
|
+
|
29
|
+
const sdk = new NodeSDK({
|
30
|
+
traceExporter: new OTLPTraceExporter(),
|
31
|
+
metricReader: new PeriodicExportingMetricReader({
|
32
|
+
exporter: new OTLPMetricExporter()
|
33
|
+
})
|
34
|
+
// instrumentations: [getNodeAutoInstrumentations()],
|
35
|
+
})
|
36
|
+
|
37
|
+
sdk.start()
|
38
|
+
|
23
39
|
const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
|
24
40
|
|
25
41
|
const optionDefinitions = [
|
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
|
|
50
66
|
const fullPath = path.resolve(options['chains-config'])
|
51
67
|
const chainsConfig = fs.readJsonSync(fullPath)
|
52
68
|
|
53
|
-
|
54
|
-
|
69
|
+
const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
|
70
|
+
? parseInt(process.env['OVERRIDE_CONCURRENCY'])
|
71
|
+
: undefined
|
72
|
+
const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
|
73
|
+
? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
|
74
|
+
: undefined
|
75
|
+
|
76
|
+
Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
|
77
|
+
Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
|
55
78
|
Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
|
56
79
|
Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
|
57
80
|
|
@@ -77,6 +100,7 @@ const server = createServer({
|
|
77
100
|
'grpc.default_compression_algorithm': compressionAlgorithms.gzip
|
78
101
|
})
|
79
102
|
.use(prometheusServerMiddleware())
|
103
|
+
.use(openTelemetryServerMiddleware())
|
80
104
|
.use(errorDetailsServerMiddleware)
|
81
105
|
const baseService = new ProcessorServiceImpl(async () => {
|
82
106
|
const m = await import(options.target)
|
package/src/provider.ts
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
import { JsonRpcProvider, Network, Provider } from 'ethers'
|
2
|
+
|
3
|
+
import PQueue from 'p-queue'
|
4
|
+
import { Endpoints } from './endpoints.js'
|
5
|
+
import { EthChainId } from '@sentio/chain'
|
6
|
+
import { LRUCache } from 'lru-cache'
|
7
|
+
import { metrics } from '@opentelemetry/api'
|
8
|
+
|
9
|
+
export const DummyProvider = new JsonRpcProvider('', Network.from(1))
|
10
|
+
|
11
|
+
const meter = metrics.getMeter('processor_provider')
|
12
|
+
const hit_count = meter.createCounter('provider_hit_count')
|
13
|
+
const miss_count = meter.createCounter('provider_miss_count')
|
14
|
+
const queue_size = meter.createGauge('provider_queue_size')
|
15
|
+
const total_duration = meter.createCounter('provider_total_duration')
|
16
|
+
const total_queued = meter.createCounter('provider_total_queued')
|
17
|
+
|
18
|
+
const providers = new Map<string, JsonRpcProvider>()
|
19
|
+
|
20
|
+
// export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
|
21
|
+
// if (!networkish) {
|
22
|
+
// networkish = EthChainId.ETHEREUM
|
23
|
+
// }
|
24
|
+
// if (networkish instanceof BaseContext) {
|
25
|
+
// networkish = networkish.getChainId()
|
26
|
+
// }
|
27
|
+
// return networkish
|
28
|
+
// }
|
29
|
+
|
30
|
+
export function getProvider(chainId?: EthChainId): Provider {
|
31
|
+
// const network = getNetworkFromCtxOrNetworkish(networkish)
|
32
|
+
if (!chainId) {
|
33
|
+
chainId = EthChainId.ETHEREUM
|
34
|
+
}
|
35
|
+
const network = Network.from(parseInt(chainId))
|
36
|
+
// TODO check if other key needed
|
37
|
+
|
38
|
+
const address = Endpoints.INSTANCE.chainServer.get(chainId)
|
39
|
+
const key = network.chainId.toString() + '-' + address
|
40
|
+
let provider = providers.get(key)
|
41
|
+
|
42
|
+
if (provider) {
|
43
|
+
return provider
|
44
|
+
}
|
45
|
+
|
46
|
+
if (address === undefined) {
|
47
|
+
throw Error(
|
48
|
+
'Provider not found for chain ' +
|
49
|
+
network.chainId +
|
50
|
+
', configured chains: ' +
|
51
|
+
[...Endpoints.INSTANCE.chainServer.keys()].join(' ')
|
52
|
+
)
|
53
|
+
}
|
54
|
+
console.log(
|
55
|
+
`init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
|
56
|
+
)
|
57
|
+
provider = new QueuedStaticJsonRpcProvider(
|
58
|
+
address,
|
59
|
+
network,
|
60
|
+
Endpoints.INSTANCE.concurrency,
|
61
|
+
Endpoints.INSTANCE.batchCount
|
62
|
+
)
|
63
|
+
providers.set(key, provider)
|
64
|
+
return provider
|
65
|
+
}
|
66
|
+
|
67
|
+
function getTag(prefix: string, value: any): string {
|
68
|
+
return (
|
69
|
+
prefix +
|
70
|
+
':' +
|
71
|
+
JSON.stringify(value, (k, v) => {
|
72
|
+
if (v == null) {
|
73
|
+
return 'null'
|
74
|
+
}
|
75
|
+
if (typeof v === 'bigint') {
|
76
|
+
return `bigint:${v.toString()}`
|
77
|
+
}
|
78
|
+
if (typeof v === 'string') {
|
79
|
+
return v.toLowerCase()
|
80
|
+
}
|
81
|
+
|
82
|
+
// Sort object keys
|
83
|
+
if (typeof v === 'object' && !Array.isArray(v)) {
|
84
|
+
const keys = Object.keys(v)
|
85
|
+
keys.sort()
|
86
|
+
return keys.reduce(
|
87
|
+
(accum, key) => {
|
88
|
+
accum[key] = v[key]
|
89
|
+
return accum
|
90
|
+
},
|
91
|
+
<any>{}
|
92
|
+
)
|
93
|
+
}
|
94
|
+
|
95
|
+
return v
|
96
|
+
})
|
97
|
+
)
|
98
|
+
}
|
99
|
+
|
100
|
+
class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
|
101
|
+
executor: PQueue
|
102
|
+
#performCache = new LRUCache<string, Promise<any>>({
|
103
|
+
max: 300000 // 300k items
|
104
|
+
// maxSize: 300 * 1024 * 1024, // 300mb for cache
|
105
|
+
// ttl: 1000 * 60 * 60, // 1 hour no ttl for better performance
|
106
|
+
// sizeCalculation: (value: any) => {
|
107
|
+
// assume each item is 1kb for simplicity
|
108
|
+
// return 1024
|
109
|
+
// }
|
110
|
+
})
|
111
|
+
|
112
|
+
constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
|
113
|
+
// TODO re-enable match when possible
|
114
|
+
super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
|
115
|
+
this.executor = new PQueue({ concurrency: concurrency })
|
116
|
+
}
|
117
|
+
|
118
|
+
async send(method: string, params: Array<any>): Promise<any> {
|
119
|
+
if (method !== 'eth_call') {
|
120
|
+
return await this.executor.add(() => super.send(method, params))
|
121
|
+
}
|
122
|
+
const tag = getTag(method, params)
|
123
|
+
const block = params[params.length - 1]
|
124
|
+
let perform = this.#performCache.get(tag)
|
125
|
+
if (!perform) {
|
126
|
+
miss_count.add(1)
|
127
|
+
const queued: number = Date.now()
|
128
|
+
perform = this.executor.add(() => {
|
129
|
+
const started = Date.now()
|
130
|
+
total_queued.add(started - queued)
|
131
|
+
|
132
|
+
return super.send(method, params).finally(() => {
|
133
|
+
total_duration.add(Date.now() - started)
|
134
|
+
})
|
135
|
+
})
|
136
|
+
perform.catch((e) => {
|
137
|
+
// if (e.code !== 'CALL_EXCEPTION' && e.code !== 'BAD_DATA') {
|
138
|
+
setTimeout(() => {
|
139
|
+
if (this.#performCache.get(tag) === perform) {
|
140
|
+
this.#performCache.delete(tag)
|
141
|
+
}
|
142
|
+
}, 1000)
|
143
|
+
})
|
144
|
+
|
145
|
+
queue_size.record(this.executor.size)
|
146
|
+
|
147
|
+
this.#performCache.set(tag, perform)
|
148
|
+
// For non latest block call, we cache permanently, otherwise we cache for one minute
|
149
|
+
if (block === 'latest') {
|
150
|
+
setTimeout(() => {
|
151
|
+
if (this.#performCache.get(tag) === perform) {
|
152
|
+
this.#performCache.delete(tag)
|
153
|
+
}
|
154
|
+
}, 60 * 1000)
|
155
|
+
}
|
156
|
+
} else {
|
157
|
+
hit_count.add(1)
|
158
|
+
}
|
159
|
+
|
160
|
+
const result = await perform
|
161
|
+
if (!result) {
|
162
|
+
throw Error('Unexpected null response')
|
163
|
+
}
|
164
|
+
return result
|
165
|
+
}
|
166
|
+
}
|
package/src/service.ts
CHANGED
@@ -7,7 +7,10 @@ import {
|
|
7
7
|
DataBinding,
|
8
8
|
DeepPartial,
|
9
9
|
Empty,
|
10
|
+
EthCallParam,
|
10
11
|
HandlerType,
|
12
|
+
PreparedData,
|
13
|
+
PreprocessResult,
|
11
14
|
ProcessBindingResponse,
|
12
15
|
ProcessBindingsRequest,
|
13
16
|
ProcessConfigRequest,
|
@@ -25,6 +28,16 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
|
|
25
28
|
|
26
29
|
import { StoreContext } from './db-context.js'
|
27
30
|
import { Subject } from 'rxjs'
|
31
|
+
import { metrics } from '@opentelemetry/api'
|
32
|
+
import { getProvider } from './provider.js'
|
33
|
+
import { EthChainId } from '@sentio/chain'
|
34
|
+
import { Provider, Interface } from 'ethers'
|
35
|
+
|
36
|
+
const meter = metrics.getMeter('processor_service')
|
37
|
+
const process_binding_count = meter.createCounter('process_binding_count')
|
38
|
+
const process_binding_time = meter.createCounter('process_binding_time')
|
39
|
+
const process_binding_error = meter.createCounter('process_binding_error')
|
40
|
+
|
28
41
|
;(BigInt.prototype as any).toJSON = function () {
|
29
42
|
return this.toString()
|
30
43
|
}
|
@@ -111,10 +124,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
111
124
|
}
|
112
125
|
|
113
126
|
async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
|
114
|
-
const
|
127
|
+
const ethCallResults = await this.preprocessBindings(request, options)
|
115
128
|
|
129
|
+
const promises = []
|
116
130
|
for (const binding of request.bindings) {
|
117
|
-
const promise = this.processBinding(binding)
|
131
|
+
const promise = this.processBinding(binding, { ethCallResults })
|
118
132
|
if (GLOBAL_CONFIG.execution.sequential) {
|
119
133
|
await promise
|
120
134
|
}
|
@@ -139,7 +153,82 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
139
153
|
}
|
140
154
|
}
|
141
155
|
|
142
|
-
async
|
156
|
+
async preprocessBindings(
|
157
|
+
request: ProcessBindingsRequest,
|
158
|
+
options?: CallContext
|
159
|
+
): Promise<{ [calldata: string]: any[] }> {
|
160
|
+
const promises = []
|
161
|
+
for (const binding of request.bindings) {
|
162
|
+
promises.push(this.preprocessBinding(binding, options))
|
163
|
+
}
|
164
|
+
let preprocessResults: PreprocessResult[]
|
165
|
+
try {
|
166
|
+
preprocessResults = await Promise.all(promises)
|
167
|
+
} catch (e) {
|
168
|
+
throw e
|
169
|
+
}
|
170
|
+
const groupedRequests = new Map<string, EthCallParam[]>()
|
171
|
+
const providers = new Map<string, Provider>()
|
172
|
+
for (const result of preprocessResults) {
|
173
|
+
for (const param of result.ethCallParams) {
|
174
|
+
if (!providers.has(param.chainId)) {
|
175
|
+
providers.set(param.chainId, getProvider(param.chainId as EthChainId))
|
176
|
+
}
|
177
|
+
const key = param.chainId + '|' + param.address
|
178
|
+
if (!groupedRequests.has(key)) {
|
179
|
+
groupedRequests.set(key, [])
|
180
|
+
}
|
181
|
+
groupedRequests.get(key)!.push(param)
|
182
|
+
}
|
183
|
+
}
|
184
|
+
|
185
|
+
const start = Date.now()
|
186
|
+
const callPromises = []
|
187
|
+
for (const params of groupedRequests.values()) {
|
188
|
+
console.log(`chain: ${params[0].chainId}, address: ${params[0].address}, totalCalls: ${params.length}`)
|
189
|
+
for (const param of params) {
|
190
|
+
const frag = new Interface(param.signature)
|
191
|
+
const calldata = frag.encodeFunctionData(param.function, param.args)
|
192
|
+
callPromises.push(
|
193
|
+
providers
|
194
|
+
.get(param.chainId)!
|
195
|
+
.call({
|
196
|
+
to: param.address,
|
197
|
+
data: calldata
|
198
|
+
})
|
199
|
+
.then((ret) => [calldata, frag.decodeFunctionResult(param.function, ret).toArray()] as [string, any[]])
|
200
|
+
)
|
201
|
+
}
|
202
|
+
}
|
203
|
+
const results = Object.fromEntries(await Promise.all(callPromises))
|
204
|
+
console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
|
205
|
+
return results
|
206
|
+
}
|
207
|
+
|
208
|
+
async preprocessBinding(request: DataBinding, options?: CallContext): Promise<PreprocessResult> {
|
209
|
+
if (!this.started) {
|
210
|
+
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
211
|
+
}
|
212
|
+
if (this.unhandled) {
|
213
|
+
throw new RichServerError(
|
214
|
+
Status.UNAVAILABLE,
|
215
|
+
'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
|
216
|
+
[
|
217
|
+
DebugInfo.fromPartial({
|
218
|
+
detail: this.unhandled.message,
|
219
|
+
stackEntries: this.unhandled.stack?.split('\n')
|
220
|
+
})
|
221
|
+
]
|
222
|
+
)
|
223
|
+
}
|
224
|
+
return await PluginManager.INSTANCE.preprocessBinding(request)
|
225
|
+
}
|
226
|
+
|
227
|
+
async processBinding(
|
228
|
+
request: DataBinding,
|
229
|
+
preparedData: PreparedData | undefined,
|
230
|
+
options?: CallContext
|
231
|
+
): Promise<ProcessResult> {
|
143
232
|
if (!this.started) {
|
144
233
|
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
145
234
|
}
|
@@ -155,7 +244,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
155
244
|
]
|
156
245
|
)
|
157
246
|
}
|
158
|
-
const result = await PluginManager.INSTANCE.processBinding(request)
|
247
|
+
const result = await PluginManager.INSTANCE.processBinding(request, preparedData)
|
159
248
|
recordRuntimeInfo(result, request.handlerType)
|
160
249
|
return result
|
161
250
|
}
|
@@ -184,30 +273,40 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
184
273
|
const contexts = new Contexts()
|
185
274
|
|
186
275
|
for await (const request of requests) {
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
.
|
194
|
-
|
195
|
-
|
196
|
-
|
276
|
+
try {
|
277
|
+
console.debug('received request:', request)
|
278
|
+
if (request.binding) {
|
279
|
+
process_binding_count.add(1)
|
280
|
+
const binding = request.binding
|
281
|
+
const dbContext = contexts.new(request.processId, subject)
|
282
|
+
const start = Date.now()
|
283
|
+
PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
|
284
|
+
.then((result) => {
|
285
|
+
subject.next({
|
286
|
+
result,
|
287
|
+
processId: request.processId
|
288
|
+
})
|
289
|
+
recordRuntimeInfo(result, binding.handlerType)
|
197
290
|
})
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
291
|
+
.catch((e) => {
|
292
|
+
console.debug(e)
|
293
|
+
dbContext.error(request.processId, e)
|
294
|
+
process_binding_error.add(1)
|
295
|
+
})
|
296
|
+
.finally(() => {
|
297
|
+
const cost = Date.now() - start
|
298
|
+
console.debug('processBinding', request.processId, ' took', cost, 'ms')
|
299
|
+
process_binding_time.add(cost)
|
300
|
+
contexts.delete(request.processId)
|
301
|
+
})
|
302
|
+
}
|
303
|
+
if (request.dbResult) {
|
304
|
+
const dbContext = contexts.get(request.processId)
|
305
|
+
dbContext?.result(request.dbResult)
|
306
|
+
}
|
307
|
+
} catch (e) {
|
308
|
+
// should not happen
|
309
|
+
console.error('unexpect error during handle loop', e)
|
211
310
|
}
|
212
311
|
}
|
213
312
|
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
import { defineConfig } from 'tsup'
|
2
|
+
|
3
|
+
export default defineConfig({
|
4
|
+
esbuildOptions: (options) => {
|
5
|
+
options.banner = {
|
6
|
+
js: `import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);`
|
7
|
+
}
|
8
|
+
},
|
9
|
+
entry: ['src/index.ts', 'src/processor-runner.ts'],
|
10
|
+
outDir: 'lib',
|
11
|
+
clean: true,
|
12
|
+
dts: true,
|
13
|
+
format: 'esm'
|
14
|
+
})
|
package/lib/chain-config.d.ts
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
|
package/lib/chain-config.js
DELETED
package/lib/chain-config.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
|
package/lib/db-context.d.ts
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
import { Subject } from 'rxjs';
|
2
|
-
import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
|
3
|
-
type Request = Omit<DBRequest, 'opId'>;
|
4
|
-
export declare class StoreContext {
|
5
|
-
readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
|
6
|
-
readonly processId: number;
|
7
|
-
private static opCounter;
|
8
|
-
private defers;
|
9
|
-
constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
|
10
|
-
newPromise<T>(opId: bigint): Promise<T>;
|
11
|
-
sendRequest(request: DeepPartial<Request>): Promise<unknown>;
|
12
|
-
result(dbResult: DBResponse): void;
|
13
|
-
error(processId: number, e: any): void;
|
14
|
-
close(): void;
|
15
|
-
}
|
16
|
-
export {};
|
17
|
-
//# sourceMappingURL=db-context.d.ts.map
|
package/lib/db-context.d.ts.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;IAa/B,KAAK;CAON"}
|
package/lib/db-context.js
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
import { ProcessResult } from '@sentio/protos';
|
2
|
-
export class StoreContext {
|
3
|
-
subject;
|
4
|
-
processId;
|
5
|
-
static opCounter = 0n;
|
6
|
-
defers = new Map();
|
7
|
-
constructor(subject, processId) {
|
8
|
-
this.subject = subject;
|
9
|
-
this.processId = processId;
|
10
|
-
}
|
11
|
-
newPromise(opId) {
|
12
|
-
return new Promise((resolve, reject) => {
|
13
|
-
this.defers.set(opId, { resolve, reject });
|
14
|
-
});
|
15
|
-
}
|
16
|
-
sendRequest(request) {
|
17
|
-
const opId = StoreContext.opCounter++;
|
18
|
-
const promise = this.newPromise(opId);
|
19
|
-
console.debug('sending db request ', opId, request);
|
20
|
-
this.subject.next({
|
21
|
-
dbRequest: {
|
22
|
-
...request,
|
23
|
-
opId
|
24
|
-
},
|
25
|
-
processId: this.processId
|
26
|
-
});
|
27
|
-
return promise;
|
28
|
-
}
|
29
|
-
result(dbResult) {
|
30
|
-
const opId = dbResult.opId;
|
31
|
-
const defer = this.defers.get(opId);
|
32
|
-
console.debug('received db result ', opId, dbResult);
|
33
|
-
if (defer) {
|
34
|
-
if (dbResult.error) {
|
35
|
-
defer.reject(new Error(dbResult.error));
|
36
|
-
}
|
37
|
-
else {
|
38
|
-
defer.resolve(dbResult);
|
39
|
-
}
|
40
|
-
this.defers.delete(opId);
|
41
|
-
}
|
42
|
-
}
|
43
|
-
error(processId, e) {
|
44
|
-
console.error('process error', processId, e);
|
45
|
-
const errorResult = ProcessResult.create({
|
46
|
-
states: {
|
47
|
-
error: e?.toString()
|
48
|
-
}
|
49
|
-
});
|
50
|
-
this.subject.next({
|
51
|
-
result: errorResult,
|
52
|
-
processId
|
53
|
-
});
|
54
|
-
}
|
55
|
-
close() {
|
56
|
-
for (const [opId, defer] of this.defers) {
|
57
|
-
console.warn('context closed before db response', opId);
|
58
|
-
defer.reject(new Error('context closed'));
|
59
|
-
}
|
60
|
-
this.defers.clear();
|
61
|
-
}
|
62
|
-
}
|
63
|
-
//# sourceMappingURL=db-context.js.map
|
package/lib/db-context.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAAsC,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAIzG,MAAM,OAAO,YAAY;IAMZ;IACA;IANH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAA6E,CAAA;IAErG,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY;QACxB,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC;IAED,KAAK;QACH,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,CAAA;YACvD,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC3C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC"}
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"decode-benchmark.d.ts","sourceRoot":"","sources":["../src/decode-benchmark.ts"],"names":[],"mappings":""}
|
package/lib/decode-benchmark.js
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
#!/usr/bin/env node
|
2
|
-
import fs from 'fs';
|
3
|
-
const fileContent = fs.readFileSync('../../../chain/aptos/testdata/block-0-9999.json', { encoding: 'utf-8' });
|
4
|
-
// console.log(fileContent)
|
5
|
-
const contents = JSON.parse(fileContent);
|
6
|
-
console.log(contents.length);
|
7
|
-
const raws = [];
|
8
|
-
for (const content of contents) {
|
9
|
-
raws.push(new TextEncoder().encode(JSON.stringify(content)));
|
10
|
-
}
|
11
|
-
console.log('test begin');
|
12
|
-
const start = Date.now();
|
13
|
-
for (let i = 0; i < 10; i++) {
|
14
|
-
for (const raw of raws) {
|
15
|
-
const x = JSON.parse(new TextDecoder().decode(raw));
|
16
|
-
}
|
17
|
-
}
|
18
|
-
const end = Date.now();
|
19
|
-
console.log((end - start) / 1000.0, 'seconds');
|
20
|
-
//# sourceMappingURL=decode-benchmark.js.map
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"decode-benchmark.js","sourceRoot":"","sources":["../src/decode-benchmark.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,iDAAiD,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;AAC7G,2BAA2B;AAE3B,MAAM,QAAQ,GAAa,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;AAElD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;AAE5B,MAAM,IAAI,GAAiB,EAAE,CAAA;AAC7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;IAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;AAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;AAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;IAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACrD,CAAC;AACH,CAAC;AAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;AACtB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,MAAM,EAAE,SAAS,CAAC,CAAA"}
|
package/lib/endpoints.d.ts
DELETED
package/lib/endpoints.d.ts.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"endpoints.d.ts","sourceRoot":"","sources":["../src/endpoints.ts"],"names":[],"mappings":"AAAA,qBAAa,SAAS;IACpB,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAkB;IAE5C,WAAW,SAAI;IACf,aAAa,SAAK;IAClB,YAAY,SAAK;IAEjB,WAAW,sBAA4B;IAEvC,UAAU,SAAI;CACf"}
|
package/lib/endpoints.js
DELETED
package/lib/endpoints.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"endpoints.js","sourceRoot":"","sources":["../src/endpoints.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,SAAS;IACpB,MAAM,CAAC,QAAQ,GAAc,IAAI,SAAS,EAAE,CAAA;IAE5C,WAAW,GAAG,CAAC,CAAA;IACf,aAAa,GAAG,EAAE,CAAA;IAClB,YAAY,GAAG,EAAE,CAAA;IAEjB,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEvC,UAAU,GAAG,CAAC,CAAA"}
|