@sentio/runtime 2.39.7-rc.9 → 2.40.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/chunk-WDKQZPD5.js +78804 -0
- package/lib/index.d.ts +369 -9
- package/lib/index.js +92 -9
- package/lib/processor-runner.d.ts +0 -2
- package/lib/processor-runner.js +51460 -129
- package/package.json +5 -25
- package/src/db-context.ts +157 -7
- package/src/full-service.ts +5 -0
- package/src/gen/processor/protos/processor.ts +995 -90
- package/src/plugin.ts +30 -4
- package/src/processor-runner.ts +26 -2
- package/src/provider.ts +166 -0
- package/src/service.ts +197 -27
- package/src/tsup.config.ts +14 -0
- package/lib/chain-config.d.ts +0 -6
- package/lib/chain-config.d.ts.map +0 -1
- package/lib/chain-config.js +0 -2
- package/lib/chain-config.js.map +0 -1
- package/lib/db-context.d.ts +0 -17
- package/lib/db-context.d.ts.map +0 -1
- package/lib/db-context.js +0 -63
- package/lib/db-context.js.map +0 -1
- package/lib/decode-benchmark.d.ts +0 -3
- package/lib/decode-benchmark.d.ts.map +0 -1
- package/lib/decode-benchmark.js +0 -20
- package/lib/decode-benchmark.js.map +0 -1
- package/lib/endpoints.d.ts +0 -9
- package/lib/endpoints.d.ts.map +0 -1
- package/lib/endpoints.js +0 -9
- package/lib/endpoints.js.map +0 -1
- package/lib/full-service.d.ts +0 -655
- package/lib/full-service.d.ts.map +0 -1
- package/lib/full-service.js +0 -137
- package/lib/full-service.js.map +0 -1
- package/lib/gen/google/protobuf/empty.d.ts +0 -17
- package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
- package/lib/gen/google/protobuf/empty.js +0 -40
- package/lib/gen/google/protobuf/empty.js.map +0 -1
- package/lib/gen/google/protobuf/struct.d.ts +0 -77
- package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
- package/lib/gen/google/protobuf/struct.js +0 -429
- package/lib/gen/google/protobuf/struct.js.map +0 -1
- package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
- package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
- package/lib/gen/google/protobuf/timestamp.js +0 -83
- package/lib/gen/google/protobuf/timestamp.js.map +0 -1
- package/lib/gen/processor/protos/processor.d.ts +0 -1470
- package/lib/gen/processor/protos/processor.d.ts.map +0 -1
- package/lib/gen/processor/protos/processor.js +0 -8512
- package/lib/gen/processor/protos/processor.js.map +0 -1
- package/lib/gen/service/common/protos/common.d.ts +0 -1698
- package/lib/gen/service/common/protos/common.d.ts.map +0 -1
- package/lib/gen/service/common/protos/common.js +0 -11383
- package/lib/gen/service/common/protos/common.js.map +0 -1
- package/lib/global-config.d.ts +0 -8
- package/lib/global-config.d.ts.map +0 -1
- package/lib/global-config.js +0 -23
- package/lib/global-config.js.map +0 -1
- package/lib/global-config.test.d.ts +0 -2
- package/lib/global-config.test.d.ts.map +0 -1
- package/lib/global-config.test.js.map +0 -1
- package/lib/index.d.ts.map +0 -1
- package/lib/index.js.map +0 -1
- package/lib/logger.d.ts +0 -2
- package/lib/logger.d.ts.map +0 -1
- package/lib/logger.js +0 -39
- package/lib/logger.js.map +0 -1
- package/lib/logger.test.d.ts +0 -2
- package/lib/logger.test.d.ts.map +0 -1
- package/lib/logger.test.js.map +0 -1
- package/lib/plugin.d.ts +0 -29
- package/lib/plugin.d.ts.map +0 -1
- package/lib/plugin.js +0 -58
- package/lib/plugin.js.map +0 -1
- package/lib/processor-runner.d.ts.map +0 -1
- package/lib/processor-runner.js.map +0 -1
- package/lib/seq-mode.test.d.ts +0 -3
- package/lib/seq-mode.test.d.ts.map +0 -1
- package/lib/seq-mode.test.js.map +0 -1
- package/lib/service.d.ts +0 -179
- package/lib/service.d.ts.map +0 -1
- package/lib/service.js +0 -194
- package/lib/service.js.map +0 -1
- package/lib/service.test.d.ts +0 -3
- package/lib/service.test.d.ts.map +0 -1
- package/lib/service.test.js.map +0 -1
- package/lib/state-storage.test.d.ts +0 -2
- package/lib/state-storage.test.d.ts.map +0 -1
- package/lib/state-storage.test.js.map +0 -1
- package/lib/state.d.ts +0 -23
- package/lib/state.d.ts.map +0 -1
- package/lib/state.js +0 -61
- package/lib/state.js.map +0 -1
- package/lib/utils.d.ts +0 -6
- package/lib/utils.d.ts.map +0 -1
- package/lib/utils.js +0 -23
- package/lib/utils.js.map +0 -1
package/src/plugin.ts
CHANGED
@@ -1,4 +1,12 @@
|
|
1
|
-
import {
|
1
|
+
import {
|
2
|
+
DataBinding,
|
3
|
+
HandlerType,
|
4
|
+
PreparedData,
|
5
|
+
PreprocessResult,
|
6
|
+
ProcessConfigResponse,
|
7
|
+
ProcessResult,
|
8
|
+
StartRequest
|
9
|
+
} from '@sentio/protos'
|
2
10
|
import { StoreContext } from './db-context.js'
|
3
11
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
4
12
|
|
@@ -16,9 +24,13 @@ export abstract class Plugin {
|
|
16
24
|
return false
|
17
25
|
}
|
18
26
|
|
19
|
-
async processBinding(request: DataBinding): Promise<ProcessResult> {
|
27
|
+
async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
|
20
28
|
return ProcessResult.create()
|
21
29
|
}
|
30
|
+
|
31
|
+
async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
|
32
|
+
return PreprocessResult.create()
|
33
|
+
}
|
22
34
|
}
|
23
35
|
|
24
36
|
export class PluginManager {
|
@@ -58,13 +70,27 @@ export class PluginManager {
|
|
58
70
|
return this.plugins.some((plugin) => plugin.stateDiff(config))
|
59
71
|
}
|
60
72
|
|
61
|
-
processBinding(
|
73
|
+
processBinding(
|
74
|
+
request: DataBinding,
|
75
|
+
preparedData: PreparedData | undefined,
|
76
|
+
dbContext?: StoreContext
|
77
|
+
): Promise<ProcessResult> {
|
78
|
+
const plugin = this.typesToPlugin.get(request.handlerType)
|
79
|
+
if (!plugin) {
|
80
|
+
throw new Error(`No plugin for ${request.handlerType}`)
|
81
|
+
}
|
82
|
+
return this.dbContextLocalStorage.run(dbContext, () => {
|
83
|
+
return plugin.processBinding(request, preparedData)
|
84
|
+
})
|
85
|
+
}
|
86
|
+
|
87
|
+
preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
|
62
88
|
const plugin = this.typesToPlugin.get(request.handlerType)
|
63
89
|
if (!plugin) {
|
64
90
|
throw new Error(`No plugin for ${request.handlerType}`)
|
65
91
|
}
|
66
92
|
return this.dbContextLocalStorage.run(dbContext, () => {
|
67
|
-
return plugin.
|
93
|
+
return plugin.preprocessBinding(request)
|
68
94
|
})
|
69
95
|
}
|
70
96
|
}
|
package/src/processor-runner.ts
CHANGED
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
|
|
8
8
|
import { createServer } from 'nice-grpc'
|
9
9
|
import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
|
10
10
|
import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
|
11
|
+
import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
|
11
12
|
import { register as globalRegistry, Registry } from 'prom-client'
|
12
13
|
import http from 'http'
|
13
14
|
// @ts-ignore inspector promises is not included in @type/node
|
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
|
|
20
21
|
import { ChainConfig } from './chain-config.js'
|
21
22
|
import { setupLogger } from './logger.js'
|
22
23
|
|
24
|
+
import { NodeSDK } from '@opentelemetry/sdk-node'
|
25
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
|
26
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
|
27
|
+
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
|
28
|
+
|
29
|
+
const sdk = new NodeSDK({
|
30
|
+
traceExporter: new OTLPTraceExporter(),
|
31
|
+
metricReader: new PeriodicExportingMetricReader({
|
32
|
+
exporter: new OTLPMetricExporter()
|
33
|
+
})
|
34
|
+
// instrumentations: [getNodeAutoInstrumentations()],
|
35
|
+
})
|
36
|
+
|
37
|
+
sdk.start()
|
38
|
+
|
23
39
|
const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
|
24
40
|
|
25
41
|
const optionDefinitions = [
|
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
|
|
50
66
|
const fullPath = path.resolve(options['chains-config'])
|
51
67
|
const chainsConfig = fs.readJsonSync(fullPath)
|
52
68
|
|
53
|
-
|
54
|
-
|
69
|
+
const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
|
70
|
+
? parseInt(process.env['OVERRIDE_CONCURRENCY'])
|
71
|
+
: undefined
|
72
|
+
const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
|
73
|
+
? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
|
74
|
+
: undefined
|
75
|
+
|
76
|
+
Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
|
77
|
+
Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
|
55
78
|
Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
|
56
79
|
Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
|
57
80
|
|
@@ -77,6 +100,7 @@ const server = createServer({
|
|
77
100
|
'grpc.default_compression_algorithm': compressionAlgorithms.gzip
|
78
101
|
})
|
79
102
|
.use(prometheusServerMiddleware())
|
103
|
+
.use(openTelemetryServerMiddleware())
|
80
104
|
.use(errorDetailsServerMiddleware)
|
81
105
|
const baseService = new ProcessorServiceImpl(async () => {
|
82
106
|
const m = await import(options.target)
|
package/src/provider.ts
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
import { JsonRpcProvider, Network, Provider } from 'ethers'
|
2
|
+
|
3
|
+
import PQueue from 'p-queue'
|
4
|
+
import { Endpoints } from './endpoints.js'
|
5
|
+
import { EthChainId } from '@sentio/chain'
|
6
|
+
import { LRUCache } from 'lru-cache'
|
7
|
+
import { metrics } from '@opentelemetry/api'
|
8
|
+
|
9
|
+
export const DummyProvider = new JsonRpcProvider('', Network.from(1))
|
10
|
+
|
11
|
+
const meter = metrics.getMeter('processor_provider')
|
12
|
+
const hit_count = meter.createCounter('provider_hit_count')
|
13
|
+
const miss_count = meter.createCounter('provider_miss_count')
|
14
|
+
const queue_size = meter.createGauge('provider_queue_size')
|
15
|
+
const total_duration = meter.createCounter('provider_total_duration')
|
16
|
+
const total_queued = meter.createCounter('provider_total_queued')
|
17
|
+
|
18
|
+
const providers = new Map<string, JsonRpcProvider>()
|
19
|
+
|
20
|
+
// export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
|
21
|
+
// if (!networkish) {
|
22
|
+
// networkish = EthChainId.ETHEREUM
|
23
|
+
// }
|
24
|
+
// if (networkish instanceof BaseContext) {
|
25
|
+
// networkish = networkish.getChainId()
|
26
|
+
// }
|
27
|
+
// return networkish
|
28
|
+
// }
|
29
|
+
|
30
|
+
export function getProvider(chainId?: EthChainId): Provider {
|
31
|
+
// const network = getNetworkFromCtxOrNetworkish(networkish)
|
32
|
+
if (!chainId) {
|
33
|
+
chainId = EthChainId.ETHEREUM
|
34
|
+
}
|
35
|
+
const network = Network.from(parseInt(chainId))
|
36
|
+
// TODO check if other key needed
|
37
|
+
|
38
|
+
const address = Endpoints.INSTANCE.chainServer.get(chainId)
|
39
|
+
const key = network.chainId.toString() + '-' + address
|
40
|
+
let provider = providers.get(key)
|
41
|
+
|
42
|
+
if (provider) {
|
43
|
+
return provider
|
44
|
+
}
|
45
|
+
|
46
|
+
if (address === undefined) {
|
47
|
+
throw Error(
|
48
|
+
'Provider not found for chain ' +
|
49
|
+
network.chainId +
|
50
|
+
', configured chains: ' +
|
51
|
+
[...Endpoints.INSTANCE.chainServer.keys()].join(' ')
|
52
|
+
)
|
53
|
+
}
|
54
|
+
console.log(
|
55
|
+
`init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
|
56
|
+
)
|
57
|
+
provider = new QueuedStaticJsonRpcProvider(
|
58
|
+
address,
|
59
|
+
network,
|
60
|
+
Endpoints.INSTANCE.concurrency,
|
61
|
+
Endpoints.INSTANCE.batchCount
|
62
|
+
)
|
63
|
+
providers.set(key, provider)
|
64
|
+
return provider
|
65
|
+
}
|
66
|
+
|
67
|
+
function getTag(prefix: string, value: any): string {
|
68
|
+
return (
|
69
|
+
prefix +
|
70
|
+
':' +
|
71
|
+
JSON.stringify(value, (k, v) => {
|
72
|
+
if (v == null) {
|
73
|
+
return 'null'
|
74
|
+
}
|
75
|
+
if (typeof v === 'bigint') {
|
76
|
+
return `bigint:${v.toString()}`
|
77
|
+
}
|
78
|
+
if (typeof v === 'string') {
|
79
|
+
return v.toLowerCase()
|
80
|
+
}
|
81
|
+
|
82
|
+
// Sort object keys
|
83
|
+
if (typeof v === 'object' && !Array.isArray(v)) {
|
84
|
+
const keys = Object.keys(v)
|
85
|
+
keys.sort()
|
86
|
+
return keys.reduce(
|
87
|
+
(accum, key) => {
|
88
|
+
accum[key] = v[key]
|
89
|
+
return accum
|
90
|
+
},
|
91
|
+
<any>{}
|
92
|
+
)
|
93
|
+
}
|
94
|
+
|
95
|
+
return v
|
96
|
+
})
|
97
|
+
)
|
98
|
+
}
|
99
|
+
|
100
|
+
class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
|
101
|
+
executor: PQueue
|
102
|
+
#performCache = new LRUCache<string, Promise<any>>({
|
103
|
+
max: 300000 // 300k items
|
104
|
+
// maxSize: 300 * 1024 * 1024, // 300mb for cache
|
105
|
+
// ttl: 1000 * 60 * 60, // 1 hour no ttl for better performance
|
106
|
+
// sizeCalculation: (value: any) => {
|
107
|
+
// assume each item is 1kb for simplicity
|
108
|
+
// return 1024
|
109
|
+
// }
|
110
|
+
})
|
111
|
+
|
112
|
+
constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
|
113
|
+
// TODO re-enable match when possible
|
114
|
+
super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
|
115
|
+
this.executor = new PQueue({ concurrency: concurrency })
|
116
|
+
}
|
117
|
+
|
118
|
+
async send(method: string, params: Array<any>): Promise<any> {
|
119
|
+
if (method !== 'eth_call') {
|
120
|
+
return await this.executor.add(() => super.send(method, params))
|
121
|
+
}
|
122
|
+
const tag = getTag(method, params)
|
123
|
+
const block = params[params.length - 1]
|
124
|
+
let perform = this.#performCache.get(tag)
|
125
|
+
if (!perform) {
|
126
|
+
miss_count.add(1)
|
127
|
+
const queued: number = Date.now()
|
128
|
+
perform = this.executor.add(() => {
|
129
|
+
const started = Date.now()
|
130
|
+
total_queued.add(started - queued)
|
131
|
+
|
132
|
+
return super.send(method, params).finally(() => {
|
133
|
+
total_duration.add(Date.now() - started)
|
134
|
+
})
|
135
|
+
})
|
136
|
+
perform.catch((e) => {
|
137
|
+
// if (e.code !== 'CALL_EXCEPTION' && e.code !== 'BAD_DATA') {
|
138
|
+
setTimeout(() => {
|
139
|
+
if (this.#performCache.get(tag) === perform) {
|
140
|
+
this.#performCache.delete(tag)
|
141
|
+
}
|
142
|
+
}, 1000)
|
143
|
+
})
|
144
|
+
|
145
|
+
queue_size.record(this.executor.size)
|
146
|
+
|
147
|
+
this.#performCache.set(tag, perform)
|
148
|
+
// For non latest block call, we cache permanently, otherwise we cache for one minute
|
149
|
+
if (block === 'latest') {
|
150
|
+
setTimeout(() => {
|
151
|
+
if (this.#performCache.get(tag) === perform) {
|
152
|
+
this.#performCache.delete(tag)
|
153
|
+
}
|
154
|
+
}, 60 * 1000)
|
155
|
+
}
|
156
|
+
} else {
|
157
|
+
hit_count.add(1)
|
158
|
+
}
|
159
|
+
|
160
|
+
const result = await perform
|
161
|
+
if (!result) {
|
162
|
+
throw Error('Unexpected null response')
|
163
|
+
}
|
164
|
+
return result
|
165
|
+
}
|
166
|
+
}
|
package/src/service.ts
CHANGED
@@ -7,7 +7,12 @@ import {
|
|
7
7
|
DataBinding,
|
8
8
|
DeepPartial,
|
9
9
|
Empty,
|
10
|
+
EthCallParam,
|
10
11
|
HandlerType,
|
12
|
+
PreparedData,
|
13
|
+
PreprocessResult,
|
14
|
+
PreprocessStreamRequest,
|
15
|
+
PreprocessStreamResponse,
|
11
16
|
ProcessBindingResponse,
|
12
17
|
ProcessBindingsRequest,
|
13
18
|
ProcessConfigRequest,
|
@@ -25,6 +30,16 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
|
|
25
30
|
|
26
31
|
import { StoreContext } from './db-context.js'
|
27
32
|
import { Subject } from 'rxjs'
|
33
|
+
import { metrics } from '@opentelemetry/api'
|
34
|
+
import { getProvider } from './provider.js'
|
35
|
+
import { EthChainId } from '@sentio/chain'
|
36
|
+
import { Provider, Interface } from 'ethers'
|
37
|
+
|
38
|
+
const meter = metrics.getMeter('processor_service')
|
39
|
+
const process_binding_count = meter.createCounter('process_binding_count')
|
40
|
+
const process_binding_time = meter.createCounter('process_binding_time')
|
41
|
+
const process_binding_error = meter.createCounter('process_binding_error')
|
42
|
+
|
28
43
|
;(BigInt.prototype as any).toJSON = function () {
|
29
44
|
return this.toString()
|
30
45
|
}
|
@@ -111,10 +126,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
111
126
|
}
|
112
127
|
|
113
128
|
async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
|
114
|
-
const
|
129
|
+
const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
|
115
130
|
|
131
|
+
const promises = []
|
116
132
|
for (const binding of request.bindings) {
|
117
|
-
const promise = this.processBinding(binding)
|
133
|
+
const promise = this.processBinding(binding, { ethCallResults })
|
118
134
|
if (GLOBAL_CONFIG.execution.sequential) {
|
119
135
|
await promise
|
120
136
|
}
|
@@ -139,7 +155,92 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
139
155
|
}
|
140
156
|
}
|
141
157
|
|
142
|
-
async
|
158
|
+
async preprocessBindings(
|
159
|
+
bindings: DataBinding[],
|
160
|
+
dbContext?: StoreContext,
|
161
|
+
options?: CallContext
|
162
|
+
): Promise<{ [calldata: string]: any[] }> {
|
163
|
+
console.log('preprocessBindings start')
|
164
|
+
const promises = []
|
165
|
+
for (const binding of bindings) {
|
166
|
+
promises.push(this.preprocessBinding(binding, dbContext, options))
|
167
|
+
}
|
168
|
+
let preprocessResults: PreprocessResult[]
|
169
|
+
try {
|
170
|
+
preprocessResults = await Promise.all(promises)
|
171
|
+
} catch (e) {
|
172
|
+
throw e
|
173
|
+
}
|
174
|
+
console.log(
|
175
|
+
'ethCallParams: ',
|
176
|
+
preprocessResults.map((r) => r.ethCallParams)
|
177
|
+
)
|
178
|
+
const groupedRequests = new Map<string, EthCallParam[]>()
|
179
|
+
const providers = new Map<string, Provider>()
|
180
|
+
for (const result of preprocessResults) {
|
181
|
+
for (const param of result.ethCallParams) {
|
182
|
+
if (!providers.has(param.chainId)) {
|
183
|
+
providers.set(param.chainId, getProvider(param.chainId as EthChainId))
|
184
|
+
}
|
185
|
+
const key = param.chainId + '|' + param.address
|
186
|
+
if (!groupedRequests.has(key)) {
|
187
|
+
groupedRequests.set(key, [])
|
188
|
+
}
|
189
|
+
groupedRequests.get(key)!.push(param)
|
190
|
+
}
|
191
|
+
}
|
192
|
+
|
193
|
+
const start = Date.now()
|
194
|
+
const callPromises = []
|
195
|
+
for (const params of groupedRequests.values()) {
|
196
|
+
console.log(`chain: ${params[0].chainId}, address: ${params[0].address}, totalCalls: ${params.length}`)
|
197
|
+
for (const param of params) {
|
198
|
+
const frag = new Interface(param.signature)
|
199
|
+
const calldata = frag.encodeFunctionData(param.function, param.args)
|
200
|
+
callPromises.push(
|
201
|
+
providers
|
202
|
+
.get(param.chainId)!
|
203
|
+
.call({
|
204
|
+
to: param.address,
|
205
|
+
data: calldata
|
206
|
+
})
|
207
|
+
.then((ret) => [calldata, frag.decodeFunctionResult(param.function, ret).toArray()] as [string, any[]])
|
208
|
+
)
|
209
|
+
}
|
210
|
+
}
|
211
|
+
const results = Object.fromEntries(await Promise.all(callPromises))
|
212
|
+
console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
|
213
|
+
return results
|
214
|
+
}
|
215
|
+
|
216
|
+
async preprocessBinding(
|
217
|
+
request: DataBinding,
|
218
|
+
dbContext?: StoreContext,
|
219
|
+
options?: CallContext
|
220
|
+
): Promise<PreprocessResult> {
|
221
|
+
if (!this.started) {
|
222
|
+
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
223
|
+
}
|
224
|
+
if (this.unhandled) {
|
225
|
+
throw new RichServerError(
|
226
|
+
Status.UNAVAILABLE,
|
227
|
+
'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
|
228
|
+
[
|
229
|
+
DebugInfo.fromPartial({
|
230
|
+
detail: this.unhandled.message,
|
231
|
+
stackEntries: this.unhandled.stack?.split('\n')
|
232
|
+
})
|
233
|
+
]
|
234
|
+
)
|
235
|
+
}
|
236
|
+
return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
|
237
|
+
}
|
238
|
+
|
239
|
+
async processBinding(
|
240
|
+
request: DataBinding,
|
241
|
+
preparedData: PreparedData | undefined,
|
242
|
+
options?: CallContext
|
243
|
+
): Promise<ProcessResult> {
|
143
244
|
if (!this.started) {
|
144
245
|
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
145
246
|
}
|
@@ -155,7 +256,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
155
256
|
]
|
156
257
|
)
|
157
258
|
}
|
158
|
-
const result = await PluginManager.INSTANCE.processBinding(request)
|
259
|
+
const result = await PluginManager.INSTANCE.processBinding(request, preparedData)
|
159
260
|
recordRuntimeInfo(result, request.handlerType)
|
160
261
|
return result
|
161
262
|
}
|
@@ -177,6 +278,65 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
177
278
|
yield* from(subject).pipe(withAbort(context.signal))
|
178
279
|
}
|
179
280
|
|
281
|
+
async handlePreprocessRequests(
|
282
|
+
requests: AsyncIterable<PreprocessStreamRequest>,
|
283
|
+
subject: Subject<DeepPartial<PreprocessStreamResponse>>
|
284
|
+
) {
|
285
|
+
const contexts = new Contexts()
|
286
|
+
|
287
|
+
for await (const request of requests) {
|
288
|
+
try {
|
289
|
+
console.debug('received request:', request)
|
290
|
+
if (request.bindings) {
|
291
|
+
const bindings = request.bindings.bindings
|
292
|
+
const dbContext = contexts.new(request.processId, subject)
|
293
|
+
const start = Date.now()
|
294
|
+
this.preprocessBindings(bindings, dbContext)
|
295
|
+
.then(() => {
|
296
|
+
subject.next({
|
297
|
+
processId: request.processId
|
298
|
+
})
|
299
|
+
})
|
300
|
+
.catch((e) => {
|
301
|
+
console.debug(e)
|
302
|
+
dbContext.error(request.processId, e)
|
303
|
+
process_binding_error.add(1)
|
304
|
+
})
|
305
|
+
.finally(() => {
|
306
|
+
const cost = Date.now() - start
|
307
|
+
console.debug('preprocessBinding', request.processId, ' took', cost, 'ms')
|
308
|
+
process_binding_time.add(cost)
|
309
|
+
contexts.delete(request.processId)
|
310
|
+
})
|
311
|
+
}
|
312
|
+
if (request.dbResult) {
|
313
|
+
const dbContext = contexts.get(request.processId)
|
314
|
+
dbContext?.result(request.dbResult)
|
315
|
+
}
|
316
|
+
} catch (e) {
|
317
|
+
// should not happen
|
318
|
+
console.error('unexpect error during handle loop', e)
|
319
|
+
}
|
320
|
+
}
|
321
|
+
}
|
322
|
+
|
323
|
+
async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
|
324
|
+
if (!this.started) {
|
325
|
+
throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
|
326
|
+
}
|
327
|
+
|
328
|
+
const subject = new Subject<DeepPartial<PreprocessStreamResponse>>()
|
329
|
+
this.handlePreprocessRequests(requests, subject)
|
330
|
+
.then(() => {
|
331
|
+
subject.complete()
|
332
|
+
})
|
333
|
+
.catch((e) => {
|
334
|
+
console.error(e)
|
335
|
+
subject.error(e)
|
336
|
+
})
|
337
|
+
yield* from(subject).pipe(withAbort(context.signal))
|
338
|
+
}
|
339
|
+
|
180
340
|
private async handleRequests(
|
181
341
|
requests: AsyncIterable<ProcessStreamRequest>,
|
182
342
|
subject: Subject<DeepPartial<ProcessStreamResponse>>
|
@@ -184,30 +344,40 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
184
344
|
const contexts = new Contexts()
|
185
345
|
|
186
346
|
for await (const request of requests) {
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
.
|
194
|
-
|
195
|
-
|
196
|
-
|
347
|
+
try {
|
348
|
+
console.debug('received request:', request)
|
349
|
+
if (request.binding) {
|
350
|
+
process_binding_count.add(1)
|
351
|
+
const binding = request.binding
|
352
|
+
const dbContext = contexts.new(request.processId, subject)
|
353
|
+
const start = Date.now()
|
354
|
+
PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
|
355
|
+
.then((result) => {
|
356
|
+
subject.next({
|
357
|
+
result,
|
358
|
+
processId: request.processId
|
359
|
+
})
|
360
|
+
recordRuntimeInfo(result, binding.handlerType)
|
197
361
|
})
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
362
|
+
.catch((e) => {
|
363
|
+
console.debug(e)
|
364
|
+
dbContext.error(request.processId, e)
|
365
|
+
process_binding_error.add(1)
|
366
|
+
})
|
367
|
+
.finally(() => {
|
368
|
+
const cost = Date.now() - start
|
369
|
+
console.debug('processBinding', request.processId, ' took', cost, 'ms')
|
370
|
+
process_binding_time.add(cost)
|
371
|
+
contexts.delete(request.processId)
|
372
|
+
})
|
373
|
+
}
|
374
|
+
if (request.dbResult) {
|
375
|
+
const dbContext = contexts.get(request.processId)
|
376
|
+
dbContext?.result(request.dbResult)
|
377
|
+
}
|
378
|
+
} catch (e) {
|
379
|
+
// should not happen
|
380
|
+
console.error('unexpect error during handle loop', e)
|
211
381
|
}
|
212
382
|
}
|
213
383
|
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
import { defineConfig } from 'tsup'
|
2
|
+
|
3
|
+
export default defineConfig({
|
4
|
+
esbuildOptions: (options) => {
|
5
|
+
options.banner = {
|
6
|
+
js: `import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);`
|
7
|
+
}
|
8
|
+
},
|
9
|
+
entry: ['src/index.ts', 'src/processor-runner.ts'],
|
10
|
+
outDir: 'lib',
|
11
|
+
clean: true,
|
12
|
+
dts: true,
|
13
|
+
format: 'esm'
|
14
|
+
})
|
package/lib/chain-config.d.ts
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
|
package/lib/chain-config.js
DELETED
package/lib/chain-config.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
|
package/lib/db-context.d.ts
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
import { Subject } from 'rxjs';
|
2
|
-
import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
|
3
|
-
type Request = Omit<DBRequest, 'opId'>;
|
4
|
-
export declare class StoreContext {
|
5
|
-
readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
|
6
|
-
readonly processId: number;
|
7
|
-
private static opCounter;
|
8
|
-
private defers;
|
9
|
-
constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
|
10
|
-
newPromise<T>(opId: bigint): Promise<T>;
|
11
|
-
sendRequest(request: DeepPartial<Request>): Promise<unknown>;
|
12
|
-
result(dbResult: DBResponse): void;
|
13
|
-
error(processId: number, e: any): void;
|
14
|
-
close(): void;
|
15
|
-
}
|
16
|
-
export {};
|
17
|
-
//# sourceMappingURL=db-context.d.ts.map
|
package/lib/db-context.d.ts.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;IAa/B,KAAK;CAON"}
|
package/lib/db-context.js
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
import { ProcessResult } from '@sentio/protos';
|
2
|
-
export class StoreContext {
|
3
|
-
subject;
|
4
|
-
processId;
|
5
|
-
static opCounter = 0n;
|
6
|
-
defers = new Map();
|
7
|
-
constructor(subject, processId) {
|
8
|
-
this.subject = subject;
|
9
|
-
this.processId = processId;
|
10
|
-
}
|
11
|
-
newPromise(opId) {
|
12
|
-
return new Promise((resolve, reject) => {
|
13
|
-
this.defers.set(opId, { resolve, reject });
|
14
|
-
});
|
15
|
-
}
|
16
|
-
sendRequest(request) {
|
17
|
-
const opId = StoreContext.opCounter++;
|
18
|
-
const promise = this.newPromise(opId);
|
19
|
-
console.debug('sending db request ', opId, request);
|
20
|
-
this.subject.next({
|
21
|
-
dbRequest: {
|
22
|
-
...request,
|
23
|
-
opId
|
24
|
-
},
|
25
|
-
processId: this.processId
|
26
|
-
});
|
27
|
-
return promise;
|
28
|
-
}
|
29
|
-
result(dbResult) {
|
30
|
-
const opId = dbResult.opId;
|
31
|
-
const defer = this.defers.get(opId);
|
32
|
-
console.debug('received db result ', opId, dbResult);
|
33
|
-
if (defer) {
|
34
|
-
if (dbResult.error) {
|
35
|
-
defer.reject(new Error(dbResult.error));
|
36
|
-
}
|
37
|
-
else {
|
38
|
-
defer.resolve(dbResult);
|
39
|
-
}
|
40
|
-
this.defers.delete(opId);
|
41
|
-
}
|
42
|
-
}
|
43
|
-
error(processId, e) {
|
44
|
-
console.error('process error', processId, e);
|
45
|
-
const errorResult = ProcessResult.create({
|
46
|
-
states: {
|
47
|
-
error: e?.toString()
|
48
|
-
}
|
49
|
-
});
|
50
|
-
this.subject.next({
|
51
|
-
result: errorResult,
|
52
|
-
processId
|
53
|
-
});
|
54
|
-
}
|
55
|
-
close() {
|
56
|
-
for (const [opId, defer] of this.defers) {
|
57
|
-
console.warn('context closed before db response', opId);
|
58
|
-
defer.reject(new Error('context closed'));
|
59
|
-
}
|
60
|
-
this.defers.clear();
|
61
|
-
}
|
62
|
-
}
|
63
|
-
//# sourceMappingURL=db-context.js.map
|
package/lib/db-context.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAAsC,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAIzG,MAAM,OAAO,YAAY;IAMZ;IACA;IANH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAA6E,CAAA;IAErG,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY;QACxB,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC;IAED,KAAK;QACH,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,CAAA;YACvD,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC3C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC"}
|