@sentio/runtime 2.39.7-rc.3 → 2.39.7-rc.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/chunk-FIFFMJK7.js +58496 -0
- package/lib/index.d.ts +302 -9
- package/lib/index.js +92 -9
- package/lib/processor-runner.d.ts +0 -2
- package/lib/processor-runner.js +51457 -129
- package/package.json +5 -25
- package/src/db-context.ts +165 -7
- package/src/gen/processor/protos/processor.ts +41 -1
- package/src/processor-runner.ts +26 -2
- package/src/service.ts +61 -24
- package/src/tsup.config.ts +14 -0
- package/lib/chain-config.d.ts +0 -6
- package/lib/chain-config.d.ts.map +0 -1
- package/lib/chain-config.js +0 -2
- package/lib/chain-config.js.map +0 -1
- package/lib/db-context.d.ts +0 -16
- package/lib/db-context.d.ts.map +0 -1
- package/lib/db-context.js +0 -56
- package/lib/db-context.js.map +0 -1
- package/lib/decode-benchmark.d.ts +0 -3
- package/lib/decode-benchmark.d.ts.map +0 -1
- package/lib/decode-benchmark.js +0 -20
- package/lib/decode-benchmark.js.map +0 -1
- package/lib/endpoints.d.ts +0 -9
- package/lib/endpoints.d.ts.map +0 -1
- package/lib/endpoints.js +0 -9
- package/lib/endpoints.js.map +0 -1
- package/lib/full-service.d.ts +0 -654
- package/lib/full-service.d.ts.map +0 -1
- package/lib/full-service.js +0 -137
- package/lib/full-service.js.map +0 -1
- package/lib/gen/google/protobuf/empty.d.ts +0 -17
- package/lib/gen/google/protobuf/empty.d.ts.map +0 -1
- package/lib/gen/google/protobuf/empty.js +0 -40
- package/lib/gen/google/protobuf/empty.js.map +0 -1
- package/lib/gen/google/protobuf/struct.d.ts +0 -77
- package/lib/gen/google/protobuf/struct.d.ts.map +0 -1
- package/lib/gen/google/protobuf/struct.js +0 -429
- package/lib/gen/google/protobuf/struct.js.map +0 -1
- package/lib/gen/google/protobuf/timestamp.d.ts +0 -19
- package/lib/gen/google/protobuf/timestamp.d.ts.map +0 -1
- package/lib/gen/google/protobuf/timestamp.js +0 -83
- package/lib/gen/google/protobuf/timestamp.js.map +0 -1
- package/lib/gen/processor/protos/processor.d.ts +0 -1465
- package/lib/gen/processor/protos/processor.d.ts.map +0 -1
- package/lib/gen/processor/protos/processor.js +0 -8474
- package/lib/gen/processor/protos/processor.js.map +0 -1
- package/lib/gen/service/common/protos/common.d.ts +0 -1698
- package/lib/gen/service/common/protos/common.d.ts.map +0 -1
- package/lib/gen/service/common/protos/common.js +0 -11383
- package/lib/gen/service/common/protos/common.js.map +0 -1
- package/lib/global-config.d.ts +0 -8
- package/lib/global-config.d.ts.map +0 -1
- package/lib/global-config.js +0 -23
- package/lib/global-config.js.map +0 -1
- package/lib/global-config.test.d.ts +0 -2
- package/lib/global-config.test.d.ts.map +0 -1
- package/lib/global-config.test.js.map +0 -1
- package/lib/index.d.ts.map +0 -1
- package/lib/index.js.map +0 -1
- package/lib/logger.d.ts +0 -2
- package/lib/logger.d.ts.map +0 -1
- package/lib/logger.js +0 -39
- package/lib/logger.js.map +0 -1
- package/lib/logger.test.d.ts +0 -2
- package/lib/logger.test.d.ts.map +0 -1
- package/lib/logger.test.js.map +0 -1
- package/lib/plugin.d.ts +0 -29
- package/lib/plugin.d.ts.map +0 -1
- package/lib/plugin.js +0 -58
- package/lib/plugin.js.map +0 -1
- package/lib/processor-runner.d.ts.map +0 -1
- package/lib/processor-runner.js.map +0 -1
- package/lib/seq-mode.test.d.ts +0 -3
- package/lib/seq-mode.test.d.ts.map +0 -1
- package/lib/seq-mode.test.js.map +0 -1
- package/lib/service.d.ts +0 -178
- package/lib/service.d.ts.map +0 -1
- package/lib/service.js +0 -179
- package/lib/service.js.map +0 -1
- package/lib/service.test.d.ts +0 -3
- package/lib/service.test.d.ts.map +0 -1
- package/lib/service.test.js.map +0 -1
- package/lib/state-storage.test.d.ts +0 -2
- package/lib/state-storage.test.d.ts.map +0 -1
- package/lib/state-storage.test.js.map +0 -1
- package/lib/state.d.ts +0 -23
- package/lib/state.d.ts.map +0 -1
- package/lib/state.js +0 -61
- package/lib/state.js.map +0 -1
- package/lib/utils.d.ts +0 -6
- package/lib/utils.d.ts.map +0 -1
- package/lib/utils.js +0 -23
- package/lib/utils.js.map +0 -1
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@sentio/runtime",
|
3
|
-
"version": "2.39.7-rc.
|
3
|
+
"version": "2.39.7-rc.30",
|
4
4
|
"license": "Apache-2.0",
|
5
5
|
"type": "module",
|
6
6
|
"exports": {
|
@@ -14,46 +14,26 @@
|
|
14
14
|
"!**/*.test.{js,ts}",
|
15
15
|
"!{lib,src}/tests"
|
16
16
|
],
|
17
|
-
"dependencies": {
|
18
|
-
"@grpc/grpc-js": "^1.9.14",
|
19
|
-
"command-line-args": "^5.2.1",
|
20
|
-
"command-line-usage": "^7.0.1",
|
21
|
-
"fs-extra": "^11.2.0",
|
22
|
-
"google-protobuf": "^3.21.2",
|
23
|
-
"ix": "^5.0.0",
|
24
|
-
"long": "^5.2.3",
|
25
|
-
"nice-grpc": "^2.1.7",
|
26
|
-
"nice-grpc-client-middleware-retry": "^3.1.6",
|
27
|
-
"nice-grpc-common": "^2.0.2",
|
28
|
-
"nice-grpc-error-details": "^0.2.4",
|
29
|
-
"nice-grpc-prometheus": "^0.2.2",
|
30
|
-
"prom-client": "^14.2.0",
|
31
|
-
"protobufjs": "^7.2.6",
|
32
|
-
"rxjs": "^7.8.1",
|
33
|
-
"utility-types": "^3.11.0",
|
34
|
-
"winston": "^3.11.0",
|
35
|
-
"@sentio/protos": "2.39.7-rc.3"
|
36
|
-
},
|
17
|
+
"dependencies": {},
|
37
18
|
"devDependencies": {
|
38
19
|
"@types/command-line-args": "^5.2.3",
|
39
20
|
"@types/command-line-usage": "^5.0.4",
|
40
21
|
"@types/fs-extra": "^11.0.4",
|
41
22
|
"@types/google-protobuf": "^3.15.12",
|
42
|
-
"jest": "^29.7.0",
|
43
|
-
"ts-node": "^10.9.2",
|
44
23
|
"tslib": "^2.6.2"
|
45
24
|
},
|
46
25
|
"engines": {
|
47
26
|
"node": ">=16"
|
48
27
|
},
|
49
28
|
"scripts": {
|
50
|
-
"build": "pnpm
|
29
|
+
"build": "pnpm bundle",
|
51
30
|
"build:all": "pnpm --filter=$(node -p \"require('./package.json').name\")... build",
|
31
|
+
"bundle": "tsup --config src/tsup.config.ts",
|
52
32
|
"compile": "tsc",
|
53
33
|
"run": "tsx src/processor-runner.ts --log-format=json",
|
54
34
|
"run-benchmark": "tsx src/decode-benchmark.ts",
|
55
35
|
"start_js": "tsx ./lib/processor-runner.js $PWD/../../debug/dist/lib.js",
|
56
36
|
"start_ts": "tsx ./lib/processor-runner.js --log-format=json $PWD/../../debug/src/processor.ts",
|
57
|
-
"test": "
|
37
|
+
"test": "glob -c 'tsx --test' '**/*.test.ts'"
|
58
38
|
}
|
59
39
|
}
|
package/src/db-context.ts
CHANGED
@@ -1,28 +1,94 @@
|
|
1
1
|
import { Subject } from 'rxjs'
|
2
|
-
import {
|
2
|
+
import {
|
3
|
+
DBRequest,
|
4
|
+
DBRequest_DBUpsert,
|
5
|
+
DBResponse,
|
6
|
+
DeepPartial,
|
7
|
+
ProcessResult,
|
8
|
+
ProcessStreamResponse
|
9
|
+
} from '@sentio/protos'
|
10
|
+
import * as process from 'node:process'
|
11
|
+
import { Attributes, Counter, metrics } from '@opentelemetry/api'
|
12
|
+
|
13
|
+
const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
|
14
|
+
const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
|
3
15
|
|
4
16
|
type Request = Omit<DBRequest, 'opId'>
|
17
|
+
type RequestType = keyof Request
|
18
|
+
|
19
|
+
const meter = metrics.getMeter('processor_store')
|
20
|
+
const send_counts: Record<RequestType, Counter<Attributes>> = {
|
21
|
+
get: meter.createCounter('store_get_count'),
|
22
|
+
upsert: meter.createCounter('store_upsert_count'),
|
23
|
+
list: meter.createCounter('store_list_count'),
|
24
|
+
delete: meter.createCounter('store_delete_count')
|
25
|
+
}
|
26
|
+
const recv_counts: Record<RequestType, Counter<Attributes>> = {
|
27
|
+
get: meter.createCounter('store_get_count'),
|
28
|
+
upsert: meter.createCounter('store_upsert_count'),
|
29
|
+
list: meter.createCounter('store_list_count'),
|
30
|
+
delete: meter.createCounter('store_delete_count')
|
31
|
+
}
|
32
|
+
const request_times: Record<RequestType, Counter<Attributes>> = {
|
33
|
+
get: meter.createCounter('store_get_time'),
|
34
|
+
upsert: meter.createCounter('store_upsert_time'),
|
35
|
+
list: meter.createCounter('store_list_time'),
|
36
|
+
delete: meter.createCounter('store_delete_time')
|
37
|
+
}
|
38
|
+
const request_errors: Record<RequestType, Counter<Attributes>> = {
|
39
|
+
get: meter.createCounter('store_get_error'),
|
40
|
+
upsert: meter.createCounter('store_upsert_error'),
|
41
|
+
list: meter.createCounter('store_list_error'),
|
42
|
+
delete: meter.createCounter('store_delete_error')
|
43
|
+
}
|
44
|
+
|
45
|
+
const batched_total_count = meter.createCounter('batched_total_count')
|
46
|
+
const batched_request_count = meter.createCounter('batched_request_count')
|
47
|
+
|
48
|
+
const unsolved_requests = meter.createGauge('store_unsolved_requests')
|
49
|
+
|
50
|
+
export const timeoutError = Symbol()
|
5
51
|
|
6
52
|
export class StoreContext {
|
7
53
|
private static opCounter = 0n
|
8
54
|
|
9
|
-
private defers = new Map<
|
55
|
+
private defers = new Map<
|
56
|
+
bigint,
|
57
|
+
{ resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
|
58
|
+
>()
|
59
|
+
private statsInterval: NodeJS.Timeout | undefined
|
10
60
|
|
11
61
|
constructor(
|
12
62
|
readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
|
13
63
|
readonly processId: number
|
14
64
|
) {}
|
15
65
|
|
16
|
-
newPromise<T>(opId: bigint) {
|
66
|
+
newPromise<T>(opId: bigint, requestType?: RequestType) {
|
17
67
|
return new Promise<T>((resolve, reject) => {
|
18
|
-
this.defers.set(opId, { resolve, reject })
|
68
|
+
this.defers.set(opId, { resolve, reject, requestType })
|
69
|
+
unsolved_requests.record(this.defers.size, { processId: this.processId })
|
19
70
|
})
|
20
71
|
}
|
21
72
|
|
22
|
-
sendRequest(request: DeepPartial<Request
|
73
|
+
sendRequest(request: DeepPartial<Request>, timeoutSecs?: number): Promise<DBResponse> {
|
74
|
+
if (STORE_BATCH_IDLE > 0 && STORE_BATCH_SIZE > 1 && request.upsert) {
|
75
|
+
// batch upsert if possible
|
76
|
+
return this.sendUpsertInBatch(request.upsert as DBRequest_DBUpsert)
|
77
|
+
}
|
78
|
+
|
79
|
+
const requestType = Object.keys(request)[0] as RequestType
|
23
80
|
const opId = StoreContext.opCounter++
|
24
|
-
const promise = this.newPromise(opId)
|
81
|
+
const promise = this.newPromise(opId, requestType)
|
82
|
+
|
83
|
+
const start = Date.now()
|
84
|
+
const promises = [promise]
|
25
85
|
console.debug('sending db request ', opId, request)
|
86
|
+
let timer: NodeJS.Timeout | undefined
|
87
|
+
if (timeoutSecs) {
|
88
|
+
const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
|
89
|
+
promises.push(timeoutPromise)
|
90
|
+
}
|
91
|
+
|
26
92
|
this.subject.next({
|
27
93
|
dbRequest: {
|
28
94
|
...request,
|
@@ -30,7 +96,27 @@ export class StoreContext {
|
|
30
96
|
},
|
31
97
|
processId: this.processId
|
32
98
|
})
|
33
|
-
|
99
|
+
|
100
|
+
send_counts[requestType]?.add(1)
|
101
|
+
|
102
|
+
return Promise.race(promises)
|
103
|
+
.then((result: DBResponse) => {
|
104
|
+
console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
|
105
|
+
request_times[requestType]?.add(Date.now() - start)
|
106
|
+
return result
|
107
|
+
})
|
108
|
+
.catch((e) => {
|
109
|
+
if (e === timeoutError) {
|
110
|
+
console.error('db request', requestType, 'op:', opId, ' timeout')
|
111
|
+
}
|
112
|
+
request_errors[requestType]?.add(1)
|
113
|
+
throw e
|
114
|
+
})
|
115
|
+
.finally(() => {
|
116
|
+
if (timer) {
|
117
|
+
clearTimeout(timer)
|
118
|
+
}
|
119
|
+
})
|
34
120
|
}
|
35
121
|
|
36
122
|
result(dbResult: DBResponse) {
|
@@ -38,6 +124,9 @@ export class StoreContext {
|
|
38
124
|
const defer = this.defers.get(opId)
|
39
125
|
console.debug('received db result ', opId, dbResult)
|
40
126
|
if (defer) {
|
127
|
+
if (defer.requestType) {
|
128
|
+
recv_counts[defer.requestType]?.add(1)
|
129
|
+
}
|
41
130
|
if (dbResult.error) {
|
42
131
|
defer.reject(new Error(dbResult.error))
|
43
132
|
} else {
|
@@ -45,6 +134,7 @@ export class StoreContext {
|
|
45
134
|
}
|
46
135
|
this.defers.delete(opId)
|
47
136
|
}
|
137
|
+
unsolved_requests.record(this.defers.size, { processId: this.processId })
|
48
138
|
}
|
49
139
|
|
50
140
|
error(processId: number, e: any) {
|
@@ -59,4 +149,72 @@ export class StoreContext {
|
|
59
149
|
processId
|
60
150
|
})
|
61
151
|
}
|
152
|
+
|
153
|
+
close() {
|
154
|
+
for (const [opId, defer] of this.defers) {
|
155
|
+
console.warn('context closed before db response', opId)
|
156
|
+
defer.reject(new Error('context closed'))
|
157
|
+
}
|
158
|
+
this.defers.clear()
|
159
|
+
if (this.statsInterval) {
|
160
|
+
clearInterval(this.statsInterval)
|
161
|
+
}
|
162
|
+
}
|
163
|
+
|
164
|
+
upsertBatch:
|
165
|
+
| {
|
166
|
+
opId: bigint
|
167
|
+
request: DBRequest_DBUpsert
|
168
|
+
promise: Promise<DBResponse>
|
169
|
+
timer: NodeJS.Timeout
|
170
|
+
}
|
171
|
+
| undefined = undefined
|
172
|
+
|
173
|
+
private async sendUpsertInBatch(req: DBRequest_DBUpsert): Promise<DBResponse> {
|
174
|
+
if (this.upsertBatch) {
|
175
|
+
// merge the upserts
|
176
|
+
const { request, promise } = this.upsertBatch
|
177
|
+
request.entity = this.upsertBatch.request.entity.concat(req.entity)
|
178
|
+
request.entityData = this.upsertBatch.request.entityData.concat(req.entityData)
|
179
|
+
request.id = this.upsertBatch.request.id.concat(req.id)
|
180
|
+
if (request.entity.length >= STORE_BATCH_SIZE) {
|
181
|
+
this.sendBatch()
|
182
|
+
}
|
183
|
+
return promise
|
184
|
+
} else {
|
185
|
+
const opId = StoreContext.opCounter++
|
186
|
+
const promise = this.newPromise<DBResponse>(opId, 'upsert')
|
187
|
+
const timeout = setTimeout(() => {
|
188
|
+
this.sendBatch()
|
189
|
+
}, STORE_BATCH_IDLE)
|
190
|
+
|
191
|
+
this.upsertBatch = {
|
192
|
+
opId,
|
193
|
+
request: req,
|
194
|
+
promise,
|
195
|
+
timer: timeout
|
196
|
+
}
|
197
|
+
|
198
|
+
return promise
|
199
|
+
}
|
200
|
+
}
|
201
|
+
|
202
|
+
private sendBatch() {
|
203
|
+
if (this.upsertBatch) {
|
204
|
+
const { request, opId, timer } = this.upsertBatch
|
205
|
+
console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
|
206
|
+
clearTimeout(timer)
|
207
|
+
this.upsertBatch = undefined
|
208
|
+
this.subject.next({
|
209
|
+
dbRequest: {
|
210
|
+
upsert: request,
|
211
|
+
opId
|
212
|
+
},
|
213
|
+
processId: this.processId
|
214
|
+
})
|
215
|
+
send_counts['upsert']?.add(1)
|
216
|
+
batched_request_count.add(1)
|
217
|
+
batched_total_count.add(request.entity.length)
|
218
|
+
}
|
219
|
+
}
|
62
220
|
}
|
@@ -790,6 +790,10 @@ export enum DBRequest_DBOperator {
|
|
790
790
|
LE = 5,
|
791
791
|
IN = 6,
|
792
792
|
NOT_IN = 7,
|
793
|
+
LIKE = 8,
|
794
|
+
NOT_LIKE = 9,
|
795
|
+
HAS_ALL = 10,
|
796
|
+
HAS_ANY = 11,
|
793
797
|
UNRECOGNIZED = -1,
|
794
798
|
}
|
795
799
|
|
@@ -819,6 +823,18 @@ export function dBRequest_DBOperatorFromJSON(object: any): DBRequest_DBOperator
|
|
819
823
|
case 7:
|
820
824
|
case "NOT_IN":
|
821
825
|
return DBRequest_DBOperator.NOT_IN;
|
826
|
+
case 8:
|
827
|
+
case "LIKE":
|
828
|
+
return DBRequest_DBOperator.LIKE;
|
829
|
+
case 9:
|
830
|
+
case "NOT_LIKE":
|
831
|
+
return DBRequest_DBOperator.NOT_LIKE;
|
832
|
+
case 10:
|
833
|
+
case "HAS_ALL":
|
834
|
+
return DBRequest_DBOperator.HAS_ALL;
|
835
|
+
case 11:
|
836
|
+
case "HAS_ANY":
|
837
|
+
return DBRequest_DBOperator.HAS_ANY;
|
822
838
|
case -1:
|
823
839
|
case "UNRECOGNIZED":
|
824
840
|
default:
|
@@ -844,6 +860,14 @@ export function dBRequest_DBOperatorToJSON(object: DBRequest_DBOperator): string
|
|
844
860
|
return "IN";
|
845
861
|
case DBRequest_DBOperator.NOT_IN:
|
846
862
|
return "NOT_IN";
|
863
|
+
case DBRequest_DBOperator.LIKE:
|
864
|
+
return "LIKE";
|
865
|
+
case DBRequest_DBOperator.NOT_LIKE:
|
866
|
+
return "NOT_LIKE";
|
867
|
+
case DBRequest_DBOperator.HAS_ALL:
|
868
|
+
return "HAS_ALL";
|
869
|
+
case DBRequest_DBOperator.HAS_ANY:
|
870
|
+
return "HAS_ANY";
|
847
871
|
case DBRequest_DBOperator.UNRECOGNIZED:
|
848
872
|
default:
|
849
873
|
return "UNRECOGNIZED";
|
@@ -859,6 +883,7 @@ export interface DBRequest_DBList {
|
|
859
883
|
entity: string;
|
860
884
|
filters: DBRequest_DBFilter[];
|
861
885
|
cursor: string;
|
886
|
+
pageSize?: number | undefined;
|
862
887
|
}
|
863
888
|
|
864
889
|
export interface DBRequest_DBUpsert {
|
@@ -6442,7 +6467,7 @@ export const DBRequest_DBGet = {
|
|
6442
6467
|
};
|
6443
6468
|
|
6444
6469
|
function createBaseDBRequest_DBList(): DBRequest_DBList {
|
6445
|
-
return { entity: "", filters: [], cursor: "" };
|
6470
|
+
return { entity: "", filters: [], cursor: "", pageSize: undefined };
|
6446
6471
|
}
|
6447
6472
|
|
6448
6473
|
export const DBRequest_DBList = {
|
@@ -6456,6 +6481,9 @@ export const DBRequest_DBList = {
|
|
6456
6481
|
if (message.cursor !== "") {
|
6457
6482
|
writer.uint32(42).string(message.cursor);
|
6458
6483
|
}
|
6484
|
+
if (message.pageSize !== undefined) {
|
6485
|
+
writer.uint32(48).uint32(message.pageSize);
|
6486
|
+
}
|
6459
6487
|
return writer;
|
6460
6488
|
},
|
6461
6489
|
|
@@ -6487,6 +6515,13 @@ export const DBRequest_DBList = {
|
|
6487
6515
|
|
6488
6516
|
message.cursor = reader.string();
|
6489
6517
|
continue;
|
6518
|
+
case 6:
|
6519
|
+
if (tag !== 48) {
|
6520
|
+
break;
|
6521
|
+
}
|
6522
|
+
|
6523
|
+
message.pageSize = reader.uint32();
|
6524
|
+
continue;
|
6490
6525
|
}
|
6491
6526
|
if ((tag & 7) === 4 || tag === 0) {
|
6492
6527
|
break;
|
@@ -6503,6 +6538,7 @@ export const DBRequest_DBList = {
|
|
6503
6538
|
? object.filters.map((e: any) => DBRequest_DBFilter.fromJSON(e))
|
6504
6539
|
: [],
|
6505
6540
|
cursor: isSet(object.cursor) ? globalThis.String(object.cursor) : "",
|
6541
|
+
pageSize: isSet(object.pageSize) ? globalThis.Number(object.pageSize) : undefined,
|
6506
6542
|
};
|
6507
6543
|
},
|
6508
6544
|
|
@@ -6517,6 +6553,9 @@ export const DBRequest_DBList = {
|
|
6517
6553
|
if (message.cursor !== "") {
|
6518
6554
|
obj.cursor = message.cursor;
|
6519
6555
|
}
|
6556
|
+
if (message.pageSize !== undefined) {
|
6557
|
+
obj.pageSize = Math.round(message.pageSize);
|
6558
|
+
}
|
6520
6559
|
return obj;
|
6521
6560
|
},
|
6522
6561
|
|
@@ -6528,6 +6567,7 @@ export const DBRequest_DBList = {
|
|
6528
6567
|
message.entity = object.entity ?? "";
|
6529
6568
|
message.filters = object.filters?.map((e) => DBRequest_DBFilter.fromPartial(e)) || [];
|
6530
6569
|
message.cursor = object.cursor ?? "";
|
6570
|
+
message.pageSize = object.pageSize ?? undefined;
|
6531
6571
|
return message;
|
6532
6572
|
},
|
6533
6573
|
};
|
package/src/processor-runner.ts
CHANGED
@@ -8,6 +8,7 @@ import commandLineArgs from 'command-line-args'
|
|
8
8
|
import { createServer } from 'nice-grpc'
|
9
9
|
import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
|
10
10
|
import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
|
11
|
+
import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
|
11
12
|
import { register as globalRegistry, Registry } from 'prom-client'
|
12
13
|
import http from 'http'
|
13
14
|
// @ts-ignore inspector promises is not included in @type/node
|
@@ -20,6 +21,21 @@ import { FullProcessorServiceImpl } from './full-service.js'
|
|
20
21
|
import { ChainConfig } from './chain-config.js'
|
21
22
|
import { setupLogger } from './logger.js'
|
22
23
|
|
24
|
+
import { NodeSDK } from '@opentelemetry/sdk-node'
|
25
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
|
26
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
|
27
|
+
import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
|
28
|
+
|
29
|
+
const sdk = new NodeSDK({
|
30
|
+
traceExporter: new OTLPTraceExporter(),
|
31
|
+
metricReader: new PeriodicExportingMetricReader({
|
32
|
+
exporter: new OTLPMetricExporter()
|
33
|
+
})
|
34
|
+
// instrumentations: [getNodeAutoInstrumentations()],
|
35
|
+
})
|
36
|
+
|
37
|
+
sdk.start()
|
38
|
+
|
23
39
|
const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
|
24
40
|
|
25
41
|
const optionDefinitions = [
|
@@ -50,8 +66,15 @@ Error.stackTraceLimit = 20
|
|
50
66
|
const fullPath = path.resolve(options['chains-config'])
|
51
67
|
const chainsConfig = fs.readJsonSync(fullPath)
|
52
68
|
|
53
|
-
|
54
|
-
|
69
|
+
const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
|
70
|
+
? parseInt(process.env['OVERRIDE_CONCURRENCY'])
|
71
|
+
: undefined
|
72
|
+
const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
|
73
|
+
? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
|
74
|
+
: undefined
|
75
|
+
|
76
|
+
Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
|
77
|
+
Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
|
55
78
|
Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
|
56
79
|
Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
|
57
80
|
|
@@ -77,6 +100,7 @@ const server = createServer({
|
|
77
100
|
'grpc.default_compression_algorithm': compressionAlgorithms.gzip
|
78
101
|
})
|
79
102
|
.use(prometheusServerMiddleware())
|
103
|
+
.use(openTelemetryServerMiddleware())
|
80
104
|
.use(errorDetailsServerMiddleware)
|
81
105
|
const baseService = new ProcessorServiceImpl(async () => {
|
82
106
|
const m = await import(options.target)
|
package/src/service.ts
CHANGED
@@ -25,6 +25,13 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
|
|
25
25
|
|
26
26
|
import { StoreContext } from './db-context.js'
|
27
27
|
import { Subject } from 'rxjs'
|
28
|
+
import { metrics } from '@opentelemetry/api'
|
29
|
+
|
30
|
+
const meter = metrics.getMeter('processor_service')
|
31
|
+
const process_binding_count = meter.createCounter('process_binding_count')
|
32
|
+
const process_binding_time = meter.createCounter('process_binding_time')
|
33
|
+
const process_binding_error = meter.createCounter('process_binding_error')
|
34
|
+
|
28
35
|
;(BigInt.prototype as any).toJSON = function () {
|
29
36
|
return this.toString()
|
30
37
|
}
|
@@ -181,33 +188,43 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
|
|
181
188
|
requests: AsyncIterable<ProcessStreamRequest>,
|
182
189
|
subject: Subject<DeepPartial<ProcessStreamResponse>>
|
183
190
|
) {
|
184
|
-
const contexts
|
191
|
+
const contexts = new Contexts()
|
185
192
|
|
186
193
|
for await (const request of requests) {
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
.
|
194
|
-
|
195
|
-
|
196
|
-
|
194
|
+
try {
|
195
|
+
console.debug('received request:', request)
|
196
|
+
if (request.binding) {
|
197
|
+
process_binding_count.add(1)
|
198
|
+
const binding = request.binding
|
199
|
+
const dbContext = contexts.new(request.processId, subject)
|
200
|
+
const start = Date.now()
|
201
|
+
PluginManager.INSTANCE.processBinding(binding, dbContext)
|
202
|
+
.then((result) => {
|
203
|
+
subject.next({
|
204
|
+
result,
|
205
|
+
processId: request.processId
|
206
|
+
})
|
207
|
+
recordRuntimeInfo(result, binding.handlerType)
|
197
208
|
})
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
209
|
+
.catch((e) => {
|
210
|
+
console.debug(e)
|
211
|
+
dbContext.error(request.processId, e)
|
212
|
+
process_binding_error.add(1)
|
213
|
+
})
|
214
|
+
.finally(() => {
|
215
|
+
const cost = Date.now() - start
|
216
|
+
console.debug('processBinding', request.processId, ' took', cost, 'ms')
|
217
|
+
process_binding_time.add(cost)
|
218
|
+
contexts.delete(request.processId)
|
219
|
+
})
|
220
|
+
}
|
221
|
+
if (request.dbResult) {
|
222
|
+
const dbContext = contexts.get(request.processId)
|
223
|
+
dbContext?.result(request.dbResult)
|
224
|
+
}
|
225
|
+
} catch (e) {
|
226
|
+
// should not happen
|
227
|
+
console.error('unexpect error during handle loop', e)
|
211
228
|
}
|
212
229
|
}
|
213
230
|
}
|
@@ -222,3 +239,23 @@ function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
|
|
222
239
|
})
|
223
240
|
}
|
224
241
|
}
|
242
|
+
|
243
|
+
class Contexts {
|
244
|
+
private contexts: Map<number, StoreContext> = new Map()
|
245
|
+
|
246
|
+
get(processId: number) {
|
247
|
+
return this.contexts.get(processId)
|
248
|
+
}
|
249
|
+
|
250
|
+
new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
|
251
|
+
const context = new StoreContext(subject, processId)
|
252
|
+
this.contexts.set(processId, context)
|
253
|
+
return context
|
254
|
+
}
|
255
|
+
|
256
|
+
delete(processId: number) {
|
257
|
+
const context = this.get(processId)
|
258
|
+
context?.close()
|
259
|
+
this.contexts.delete(processId)
|
260
|
+
}
|
261
|
+
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
import { defineConfig } from 'tsup'
|
2
|
+
|
3
|
+
export default defineConfig({
|
4
|
+
esbuildOptions: (options) => {
|
5
|
+
options.banner = {
|
6
|
+
js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url);`
|
7
|
+
}
|
8
|
+
},
|
9
|
+
entry: ['src/index.ts', 'src/processor-runner.ts'],
|
10
|
+
outDir: 'lib',
|
11
|
+
clean: true,
|
12
|
+
dts: true,
|
13
|
+
format: 'esm'
|
14
|
+
})
|
package/lib/chain-config.d.ts
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.d.ts","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB"}
|
package/lib/chain-config.js
DELETED
package/lib/chain-config.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"chain-config.js","sourceRoot":"","sources":["../src/chain-config.ts"],"names":[],"mappings":""}
|
package/lib/db-context.d.ts
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
import { Subject } from 'rxjs';
|
2
|
-
import { DBRequest, DBResponse, DeepPartial, ProcessStreamResponse } from '@sentio/protos';
|
3
|
-
type Request = Omit<DBRequest, 'opId'>;
|
4
|
-
export declare class StoreContext {
|
5
|
-
readonly subject: Subject<DeepPartial<ProcessStreamResponse>>;
|
6
|
-
readonly processId: number;
|
7
|
-
private static opCounter;
|
8
|
-
private defers;
|
9
|
-
constructor(subject: Subject<DeepPartial<ProcessStreamResponse>>, processId: number);
|
10
|
-
newPromise<T>(opId: bigint): Promise<T>;
|
11
|
-
sendRequest(request: DeepPartial<Request>): Promise<unknown>;
|
12
|
-
result(dbResult: DBResponse): void;
|
13
|
-
error(processId: number, e: any): void;
|
14
|
-
}
|
15
|
-
export {};
|
16
|
-
//# sourceMappingURL=db-context.d.ts.map
|
package/lib/db-context.d.ts.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAiB,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AAEzG,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AAEtC,qBAAa,YAAY;IAMrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAN5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAAuF;gBAG1F,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM;IAM1B,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC;IAczC,MAAM,CAAC,QAAQ,EAAE,UAAU;IAc3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;CAYhC"}
|
package/lib/db-context.js
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
import { ProcessResult } from '@sentio/protos';
|
2
|
-
export class StoreContext {
|
3
|
-
subject;
|
4
|
-
processId;
|
5
|
-
static opCounter = 0n;
|
6
|
-
defers = new Map();
|
7
|
-
constructor(subject, processId) {
|
8
|
-
this.subject = subject;
|
9
|
-
this.processId = processId;
|
10
|
-
}
|
11
|
-
newPromise(opId) {
|
12
|
-
return new Promise((resolve, reject) => {
|
13
|
-
this.defers.set(opId, { resolve, reject });
|
14
|
-
});
|
15
|
-
}
|
16
|
-
sendRequest(request) {
|
17
|
-
const opId = StoreContext.opCounter++;
|
18
|
-
const promise = this.newPromise(opId);
|
19
|
-
console.debug('sending db request ', opId, request);
|
20
|
-
this.subject.next({
|
21
|
-
dbRequest: {
|
22
|
-
...request,
|
23
|
-
opId
|
24
|
-
},
|
25
|
-
processId: this.processId
|
26
|
-
});
|
27
|
-
return promise;
|
28
|
-
}
|
29
|
-
result(dbResult) {
|
30
|
-
const opId = dbResult.opId;
|
31
|
-
const defer = this.defers.get(opId);
|
32
|
-
console.debug('received db result ', opId, dbResult);
|
33
|
-
if (defer) {
|
34
|
-
if (dbResult.error) {
|
35
|
-
defer.reject(new Error(dbResult.error));
|
36
|
-
}
|
37
|
-
else {
|
38
|
-
defer.resolve(dbResult);
|
39
|
-
}
|
40
|
-
this.defers.delete(opId);
|
41
|
-
}
|
42
|
-
}
|
43
|
-
error(processId, e) {
|
44
|
-
console.error('process error', processId, e);
|
45
|
-
const errorResult = ProcessResult.create({
|
46
|
-
states: {
|
47
|
-
error: e?.toString()
|
48
|
-
}
|
49
|
-
});
|
50
|
-
this.subject.next({
|
51
|
-
result: errorResult,
|
52
|
-
processId
|
53
|
-
});
|
54
|
-
}
|
55
|
-
}
|
56
|
-
//# sourceMappingURL=db-context.js.map
|
package/lib/db-context.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAAsC,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAIzG,MAAM,OAAO,YAAY;IAMZ;IACA;IANH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAA6E,CAAA;IAErG,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY;QACxB,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QAC5C,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B;QACvC,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACrC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC"}
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"decode-benchmark.d.ts","sourceRoot":"","sources":["../src/decode-benchmark.ts"],"names":[],"mappings":""}
|