@sentio/runtime 2.40.0-rc.9 → 2.40.1-rc.1
Sign up to get free protection for your applications and to get access to all the features.
- package/lib/chunk-N7A3MJFE.js +131 -0
- package/lib/chunk-N7A3MJFE.js.map +1 -0
- package/lib/index.d.ts +22 -9
- package/lib/index.js +2 -93
- package/lib/index.js.map +1 -0
- package/lib/processor-runner.d.ts +3 -0
- package/lib/processor-runner.js +58 -51468
- package/lib/processor-runner.js.map +1 -0
- package/package.json +2 -2
- package/src/db-context.ts +51 -44
- package/src/index.ts +1 -0
- package/src/metrics.ts +138 -0
- package/src/multicall.ts +1615 -0
- package/src/otlp.ts +50 -0
- package/src/plugin.ts +7 -3
- package/src/processor-runner.ts +64 -21
- package/src/provider.ts +3 -9
- package/src/service.ts +98 -37
- package/src/tsup.config.ts +2 -0
- package/src/utils.ts +1 -1
- package/lib/chunk-FFU5RYDX.js +0 -78856
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@sentio/runtime",
|
3
|
-
"version": "2.40.
|
3
|
+
"version": "2.40.1-rc.1",
|
4
4
|
"license": "Apache-2.0",
|
5
5
|
"type": "module",
|
6
6
|
"exports": {
|
@@ -26,7 +26,7 @@
|
|
26
26
|
"node": ">=16"
|
27
27
|
},
|
28
28
|
"scripts": {
|
29
|
-
"build": "pnpm bundle",
|
29
|
+
"build": "pnpm tsc --noEmit && pnpm bundle",
|
30
30
|
"build:all": "pnpm --filter=$(node -p \"require('./package.json').name\")... build",
|
31
31
|
"bundle": "tsup --config src/tsup.config.ts",
|
32
32
|
"compile": "tsc",
|
package/src/db-context.ts
CHANGED
@@ -8,55 +8,33 @@ import {
|
|
8
8
|
ProcessStreamResponse
|
9
9
|
} from '@sentio/protos'
|
10
10
|
import * as process from 'node:process'
|
11
|
-
import {
|
12
|
-
|
11
|
+
import { dbMetrics } from './metrics.js'
|
12
|
+
const {
|
13
|
+
request_errors,
|
14
|
+
unsolved_requests,
|
15
|
+
request_times,
|
16
|
+
batched_request_count,
|
17
|
+
batched_total_count,
|
18
|
+
send_counts,
|
19
|
+
recv_counts
|
20
|
+
} = dbMetrics
|
13
21
|
const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
|
14
22
|
const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
|
23
|
+
const STORE_UPSERT_NO_WAIT = process.env['STORE_UPSERT_NO_WAIT'] === 'true'
|
15
24
|
|
16
25
|
type Request = Omit<DBRequest, 'opId'>
|
17
26
|
type RequestType = keyof Request
|
18
27
|
|
19
|
-
const
|
20
|
-
const send_counts: Record<RequestType, Counter<Attributes>> = {
|
21
|
-
get: meter.createCounter('store_get_count'),
|
22
|
-
upsert: meter.createCounter('store_upsert_count'),
|
23
|
-
list: meter.createCounter('store_list_count'),
|
24
|
-
delete: meter.createCounter('store_delete_count')
|
25
|
-
}
|
26
|
-
const recv_counts: Record<RequestType, Counter<Attributes>> = {
|
27
|
-
get: meter.createCounter('store_get_count'),
|
28
|
-
upsert: meter.createCounter('store_upsert_count'),
|
29
|
-
list: meter.createCounter('store_list_count'),
|
30
|
-
delete: meter.createCounter('store_delete_count')
|
31
|
-
}
|
32
|
-
const request_times: Record<RequestType, Counter<Attributes>> = {
|
33
|
-
get: meter.createCounter('store_get_time'),
|
34
|
-
upsert: meter.createCounter('store_upsert_time'),
|
35
|
-
list: meter.createCounter('store_list_time'),
|
36
|
-
delete: meter.createCounter('store_delete_time')
|
37
|
-
}
|
38
|
-
const request_errors: Record<RequestType, Counter<Attributes>> = {
|
39
|
-
get: meter.createCounter('store_get_error'),
|
40
|
-
upsert: meter.createCounter('store_upsert_error'),
|
41
|
-
list: meter.createCounter('store_list_error'),
|
42
|
-
delete: meter.createCounter('store_delete_error')
|
43
|
-
}
|
44
|
-
|
45
|
-
const batched_total_count = meter.createCounter('batched_total_count')
|
46
|
-
const batched_request_count = meter.createCounter('batched_request_count')
|
47
|
-
|
48
|
-
const unsolved_requests = meter.createGauge('store_unsolved_requests')
|
49
|
-
|
50
|
-
export const timeoutError = Symbol()
|
28
|
+
export const timeoutError = new Error('timeout')
|
51
29
|
|
52
30
|
export class StoreContext {
|
53
31
|
private static opCounter = 0n
|
54
|
-
|
55
32
|
private defers = new Map<
|
56
33
|
bigint,
|
57
34
|
{ resolve: (value: any) => void; reject: (reason?: any) => void; requestType?: RequestType }
|
58
35
|
>()
|
59
36
|
private statsInterval: NodeJS.Timeout | undefined
|
37
|
+
private pendings: Promise<unknown>[] = []
|
60
38
|
|
61
39
|
constructor(
|
62
40
|
readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
|
@@ -82,7 +60,7 @@ export class StoreContext {
|
|
82
60
|
|
83
61
|
const start = Date.now()
|
84
62
|
const promises = [promise]
|
85
|
-
console.debug('sending db request ', opId, request)
|
63
|
+
// console.debug('sending db request ', opId, request)
|
86
64
|
let timer: NodeJS.Timeout | undefined
|
87
65
|
if (timeoutSecs) {
|
88
66
|
const timeoutPromise = new Promise((_r, rej) => (timer = setTimeout(rej, timeoutSecs * 1000, timeoutError)))
|
@@ -99,9 +77,18 @@ export class StoreContext {
|
|
99
77
|
|
100
78
|
send_counts[requestType]?.add(1)
|
101
79
|
|
80
|
+
if (requestType === 'upsert' && STORE_UPSERT_NO_WAIT) {
|
81
|
+
this.pendings.push(promise)
|
82
|
+
return Promise.resolve({
|
83
|
+
opId,
|
84
|
+
} as DBResponse)
|
85
|
+
}
|
86
|
+
|
102
87
|
return Promise.race(promises)
|
103
88
|
.then((result: DBResponse) => {
|
104
|
-
|
89
|
+
if (timeoutSecs) {
|
90
|
+
console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
|
91
|
+
}
|
105
92
|
request_times[requestType]?.add(Date.now() - start)
|
106
93
|
return result
|
107
94
|
})
|
@@ -122,7 +109,7 @@ export class StoreContext {
|
|
122
109
|
result(dbResult: DBResponse) {
|
123
110
|
const opId = dbResult.opId
|
124
111
|
const defer = this.defers.get(opId)
|
125
|
-
console.debug('received db result ', opId, dbResult)
|
112
|
+
// console.debug('received db result ', opId, dbResult)
|
126
113
|
if (defer) {
|
127
114
|
if (defer.requestType) {
|
128
115
|
recv_counts[defer.requestType]?.add(1)
|
@@ -152,8 +139,8 @@ export class StoreContext {
|
|
152
139
|
|
153
140
|
close() {
|
154
141
|
for (const [opId, defer] of this.defers) {
|
155
|
-
console.warn('context closed before db response', opId)
|
156
|
-
defer.reject(new Error('context closed'))
|
142
|
+
// console.warn('context closed before db response', opId)
|
143
|
+
defer.reject(new Error('context closed before db response, processId: ' + this.processId + ' opId: ' + opId))
|
157
144
|
}
|
158
145
|
this.defers.clear()
|
159
146
|
if (this.statsInterval) {
|
@@ -180,29 +167,45 @@ export class StoreContext {
|
|
180
167
|
if (request.entity.length >= STORE_BATCH_SIZE) {
|
181
168
|
this.sendBatch()
|
182
169
|
}
|
170
|
+
if (STORE_UPSERT_NO_WAIT) {
|
171
|
+
return {
|
172
|
+
opId: this.upsertBatch.opId
|
173
|
+
}
|
174
|
+
}
|
175
|
+
|
183
176
|
return promise
|
184
177
|
} else {
|
185
178
|
const opId = StoreContext.opCounter++
|
186
|
-
const promise = this.newPromise<DBResponse>(opId, 'upsert')
|
187
179
|
const timeout = setTimeout(() => {
|
188
180
|
this.sendBatch()
|
189
181
|
}, STORE_BATCH_IDLE)
|
182
|
+
const start = Date.now()
|
183
|
+
const promise = this.newPromise<DBResponse>(opId, 'upsert').finally(() => {
|
184
|
+
request_times['upsert'].add(Date.now() - start)
|
185
|
+
})
|
190
186
|
|
191
187
|
this.upsertBatch = {
|
192
188
|
opId,
|
193
189
|
request: req,
|
194
190
|
promise,
|
195
|
-
timer: timeout
|
191
|
+
timer: timeout,
|
196
192
|
}
|
197
193
|
|
198
|
-
|
194
|
+
if (STORE_UPSERT_NO_WAIT) {
|
195
|
+
this.pendings.push(promise)
|
196
|
+
return {
|
197
|
+
opId: this.upsertBatch.opId
|
198
|
+
}
|
199
|
+
} else {
|
200
|
+
return promise
|
201
|
+
}
|
199
202
|
}
|
200
203
|
}
|
201
204
|
|
202
205
|
private sendBatch() {
|
203
206
|
if (this.upsertBatch) {
|
204
207
|
const { request, opId, timer } = this.upsertBatch
|
205
|
-
console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
|
208
|
+
// console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
|
206
209
|
clearTimeout(timer)
|
207
210
|
this.upsertBatch = undefined
|
208
211
|
this.subject.next({
|
@@ -217,4 +220,8 @@ export class StoreContext {
|
|
217
220
|
batched_total_count.add(request.entity.length)
|
218
221
|
}
|
219
222
|
}
|
223
|
+
|
224
|
+
async awaitPendings() {
|
225
|
+
await Promise.all(this.pendings)
|
226
|
+
}
|
220
227
|
}
|
package/src/index.ts
CHANGED
package/src/metrics.ts
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
import { Attributes, Counter, metrics, Gauge } from '@opentelemetry/api'
|
2
|
+
|
3
|
+
const meter = metrics.getMeter('processor')
|
4
|
+
|
5
|
+
class C {
|
6
|
+
private counter: Counter<Attributes>
|
7
|
+
private value: number = 0
|
8
|
+
|
9
|
+
constructor(name: string) {
|
10
|
+
this.counter = meter.createCounter(name)
|
11
|
+
}
|
12
|
+
|
13
|
+
add(value: number, attributes?: Attributes) {
|
14
|
+
this.counter.add(value, attributes)
|
15
|
+
this.value += value
|
16
|
+
}
|
17
|
+
|
18
|
+
get() {
|
19
|
+
return this.value
|
20
|
+
}
|
21
|
+
}
|
22
|
+
|
23
|
+
class G {
|
24
|
+
private gauge: Gauge<Attributes>
|
25
|
+
private value: number = 0
|
26
|
+
|
27
|
+
constructor(name: string) {
|
28
|
+
this.gauge = meter.createGauge(name)
|
29
|
+
}
|
30
|
+
|
31
|
+
record(value: number, attributes?: Attributes) {
|
32
|
+
this.gauge.record(value, attributes)
|
33
|
+
this.value = value
|
34
|
+
}
|
35
|
+
|
36
|
+
get() {
|
37
|
+
return this.value
|
38
|
+
}
|
39
|
+
}
|
40
|
+
|
41
|
+
export const dbMetrics = {
|
42
|
+
send_counts: {
|
43
|
+
get: new C('store_get_send'),
|
44
|
+
upsert: new C('store_upsert_send'),
|
45
|
+
list: new C('store_list_send'),
|
46
|
+
delete: new C('store_delete_send')
|
47
|
+
},
|
48
|
+
recv_counts: {
|
49
|
+
get: new C('store_get_recv'),
|
50
|
+
upsert: new C('store_upsert_recv'),
|
51
|
+
list: new C('store_list_recv'),
|
52
|
+
delete: new C('store_delete_recv')
|
53
|
+
},
|
54
|
+
request_times: {
|
55
|
+
get: new C('store_get_time'),
|
56
|
+
upsert: new C('store_upsert_time'),
|
57
|
+
list: new C('store_list_time'),
|
58
|
+
delete: new C('store_delete_time')
|
59
|
+
},
|
60
|
+
request_errors: {
|
61
|
+
get: new C('store_get_error'),
|
62
|
+
upsert: new C('store_upsert_error'),
|
63
|
+
list: new C('store_list_error'),
|
64
|
+
delete: new C('store_delete_error')
|
65
|
+
},
|
66
|
+
batched_total_count: new C('batched_total_count'),
|
67
|
+
batched_request_count: new C('batched_request_count'),
|
68
|
+
unsolved_requests: new G('store_unsolved_requests'),
|
69
|
+
|
70
|
+
stats() {
|
71
|
+
return {
|
72
|
+
send_counts: {
|
73
|
+
get: this.send_counts.get.get(),
|
74
|
+
upsert: this.send_counts.upsert.get(),
|
75
|
+
list: this.send_counts.list.get(),
|
76
|
+
delete: this.send_counts.delete.get()
|
77
|
+
},
|
78
|
+
recv_counts: {
|
79
|
+
get: this.recv_counts.get.get(),
|
80
|
+
upsert: this.recv_counts.upsert.get(),
|
81
|
+
list: this.recv_counts.list.get(),
|
82
|
+
delete: this.recv_counts.delete.get()
|
83
|
+
},
|
84
|
+
request_times: {
|
85
|
+
get: this.request_times.get.get(),
|
86
|
+
upsert: this.request_times.upsert.get(),
|
87
|
+
list: this.request_times.list.get(),
|
88
|
+
delete: this.request_times.delete.get()
|
89
|
+
},
|
90
|
+
request_errors: {
|
91
|
+
get: this.request_errors.get.get(),
|
92
|
+
upsert: this.request_errors.upsert.get(),
|
93
|
+
list: this.request_errors.list.get(),
|
94
|
+
delete: this.request_errors.delete.get()
|
95
|
+
},
|
96
|
+
batched_total_count: this.batched_total_count.get(),
|
97
|
+
batched_request_count: this.batched_request_count.get(),
|
98
|
+
unsolved_requests: this.unsolved_requests.get(),
|
99
|
+
average_request_time: {
|
100
|
+
get: this.request_times.get.get() / this.send_counts.get.get(),
|
101
|
+
upsert: this.request_times.upsert.get() / this.send_counts.upsert.get(),
|
102
|
+
list: this.request_times.list.get() / this.send_counts.list.get()
|
103
|
+
}
|
104
|
+
}
|
105
|
+
}
|
106
|
+
}
|
107
|
+
|
108
|
+
export const providerMetrics = {
|
109
|
+
hit_count: new C('provider_hit_count'),
|
110
|
+
miss_count: new C('provider_miss_count'),
|
111
|
+
queue_size: new G('provider_queue_size'),
|
112
|
+
total_duration: new C('provider_total_duration'),
|
113
|
+
total_queued: new C('provider_total_queued'),
|
114
|
+
stats() {
|
115
|
+
return {
|
116
|
+
hit_count: this.hit_count.get(),
|
117
|
+
miss_count: this.miss_count.get(),
|
118
|
+
queue_size: this.queue_size.get(),
|
119
|
+
total_duration: this.total_duration.get(),
|
120
|
+
total_queued: this.total_queued.get(),
|
121
|
+
average_queue_time: this.total_queued.get() / (this.hit_count.get() + this.miss_count.get()),
|
122
|
+
average_duration: this.total_duration.get() / (this.hit_count.get() + this.miss_count.get())
|
123
|
+
}
|
124
|
+
}
|
125
|
+
}
|
126
|
+
|
127
|
+
export const processMetrics = {
|
128
|
+
process_binding_count: new C('process_binding_count'),
|
129
|
+
process_binding_time: new C('process_binding_time'),
|
130
|
+
process_binding_error: new C('process_binding_error'),
|
131
|
+
stats() {
|
132
|
+
return {
|
133
|
+
process_binding_count: this.process_binding_count.get(),
|
134
|
+
process_binding_time: this.process_binding_time.get(),
|
135
|
+
process_binding_error: this.process_binding_error.get()
|
136
|
+
}
|
137
|
+
}
|
138
|
+
}
|