@sentio/runtime 2.39.7-rc.23 → 2.39.7-rc.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/db-context.d.ts +8 -3
- package/lib/db-context.d.ts.map +1 -1
- package/lib/db-context.js +34 -22
- package/lib/db-context.js.map +1 -1
- package/lib/processor-runner.js +8 -2
- package/lib/processor-runner.js.map +1 -1
- package/package.json +2 -2
- package/src/db-context.ts +44 -25
- package/src/processor-runner.ts +9 -2
package/lib/db-context.d.ts
CHANGED
|
@@ -15,9 +15,14 @@ export declare class StoreContext {
|
|
|
15
15
|
result(dbResult: DBResponse): void;
|
|
16
16
|
error(processId: number, e: any): void;
|
|
17
17
|
close(): void;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
upsertBatch: {
|
|
19
|
+
opId: bigint;
|
|
20
|
+
request: DBRequest_DBUpsert;
|
|
21
|
+
promise: Promise<DBResponse>;
|
|
22
|
+
timer: NodeJS.Timeout;
|
|
23
|
+
} | undefined;
|
|
24
|
+
private sendUpsertInBatch;
|
|
25
|
+
private sendBatch;
|
|
21
26
|
}
|
|
22
27
|
export {};
|
|
23
28
|
//# sourceMappingURL=db-context.d.ts.map
|
package/lib/db-context.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,UAAU,EACV,WAAW,EAEX,qBAAqB,EACtB,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"db-context.d.ts","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,UAAU,EACV,WAAW,EAEX,qBAAqB,EACtB,MAAM,gBAAgB,CAAA;AAOvB,KAAK,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;AACtC,KAAK,WAAW,GAAG,MAAM,OAAO,CAAA;AA8BhC,eAAO,MAAM,YAAY,eAAW,CAAA;AAEpC,qBAAa,YAAY;IAUrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,MAAM;IAV5B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAK;IAE7B,OAAO,CAAC,MAAM,CAGX;IACH,OAAO,CAAC,aAAa,CAA4B;gBAGtC,OAAO,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,EACpD,SAAS,EAAE,MAAM;IAG5B,UAAU,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,WAAW;IAOrD,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAiDrF,MAAM,CAAC,QAAQ,EAAE,UAAU;IAkB3B,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG;IAa/B,KAAK;IAWL,WAAW,EACP;QACE,IAAI,EAAE,MAAM,CAAA;QACZ,OAAO,EAAE,kBAAkB,CAAA;QAC3B,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;QAC5B,KAAK,EAAE,MAAM,CAAC,OAAO,CAAA;KACtB,GACD,SAAS,CAAY;YAEX,iBAAiB;IA6B/B,OAAO,CAAC,SAAS;CAgBlB"}
|
package/lib/db-context.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { ProcessResult } from '@sentio/protos';
|
|
2
2
|
import * as process from 'node:process';
|
|
3
3
|
import { metrics } from '@opentelemetry/api';
|
|
4
|
-
const STORE_BATCH_IDLE = process.env['
|
|
4
|
+
const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1;
|
|
5
|
+
const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10;
|
|
5
6
|
const meter = metrics.getMeter('store');
|
|
6
7
|
const send_counts = {
|
|
7
8
|
get: meter.createCounter('store_get_count'),
|
|
@@ -46,9 +47,9 @@ export class StoreContext {
|
|
|
46
47
|
});
|
|
47
48
|
}
|
|
48
49
|
sendRequest(request, timeoutSecs) {
|
|
49
|
-
if (STORE_BATCH_IDLE > 0 && request.upsert) {
|
|
50
|
+
if (STORE_BATCH_IDLE > 0 && STORE_BATCH_SIZE > 1 && request.upsert) {
|
|
50
51
|
// batch upsert if possible
|
|
51
|
-
return this.
|
|
52
|
+
return this.sendUpsertInBatch(request.upsert);
|
|
52
53
|
}
|
|
53
54
|
const requestType = Object.keys(request)[0];
|
|
54
55
|
const opId = StoreContext.opCounter++;
|
|
@@ -128,38 +129,49 @@ export class StoreContext {
|
|
|
128
129
|
clearInterval(this.statsInterval);
|
|
129
130
|
}
|
|
130
131
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
if (this.queuedUpsert && this.queuedUpsertPromise) {
|
|
132
|
+
upsertBatch = undefined;
|
|
133
|
+
async sendUpsertInBatch(req) {
|
|
134
|
+
if (this.upsertBatch) {
|
|
135
135
|
// merge the upserts
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
136
|
+
const { request } = this.upsertBatch;
|
|
137
|
+
request.entity = this.upsertBatch.request.entity.concat(req.entity);
|
|
138
|
+
request.entityData = this.upsertBatch.request.entityData.concat(req.entityData);
|
|
139
|
+
request.id = this.upsertBatch.request.id.concat(req.id);
|
|
140
|
+
if (request.entity.length >= STORE_BATCH_SIZE) {
|
|
141
|
+
this.sendBatch();
|
|
142
|
+
}
|
|
143
|
+
return this.upsertBatch.promise;
|
|
140
144
|
}
|
|
141
145
|
else {
|
|
142
|
-
this.queuedUpsert = req;
|
|
143
146
|
const opId = StoreContext.opCounter++;
|
|
144
147
|
const promise = this.newPromise(opId, 'upsert');
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
this.
|
|
149
|
-
|
|
148
|
+
const timeout = setTimeout(() => {
|
|
149
|
+
this.sendBatch();
|
|
150
|
+
}, STORE_BATCH_IDLE);
|
|
151
|
+
this.upsertBatch = {
|
|
152
|
+
opId,
|
|
153
|
+
request: req,
|
|
154
|
+
promise,
|
|
155
|
+
timer: timeout
|
|
156
|
+
};
|
|
157
|
+
return promise;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
sendBatch() {
|
|
161
|
+
if (this.upsertBatch) {
|
|
162
|
+
const { request, opId, timer } = this.upsertBatch;
|
|
163
|
+
console.debug('sending batch upsert', opId, 'batch size', request?.entity.length);
|
|
164
|
+
clearTimeout(timer);
|
|
165
|
+
this.upsertBatch = undefined;
|
|
150
166
|
this.subject.next({
|
|
151
167
|
dbRequest: {
|
|
152
|
-
upsert:
|
|
168
|
+
upsert: request,
|
|
153
169
|
opId
|
|
154
170
|
},
|
|
155
171
|
processId: this.processId
|
|
156
172
|
});
|
|
157
173
|
send_counts['upsert']?.add(1);
|
|
158
|
-
return promise;
|
|
159
174
|
}
|
|
160
175
|
}
|
|
161
176
|
}
|
|
162
|
-
function delay(ms) {
|
|
163
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
164
|
-
}
|
|
165
177
|
//# sourceMappingURL=db-context.js.map
|
package/lib/db-context.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAKL,aAAa,EAEd,MAAM,gBAAgB,CAAA;AACvB,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AACvC,OAAO,EAAuB,OAAO,EAAE,MAAM,oBAAoB,CAAA;AAEjE,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,
|
|
1
|
+
{"version":3,"file":"db-context.js","sourceRoot":"","sources":["../src/db-context.ts"],"names":[],"mappings":"AACA,OAAO,EAKL,aAAa,EAEd,MAAM,gBAAgB,CAAA;AACvB,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AACvC,OAAO,EAAuB,OAAO,EAAE,MAAM,oBAAoB,CAAA;AAEjE,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAChH,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAKzG,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAA;AACvC,MAAM,WAAW,GAA6C;IAC5D,GAAG,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC;IAC3C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC;IACjD,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,kBAAkB,CAAC;IAC7C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC;CAClD,CAAA;AACD,MAAM,WAAW,GAA6C;IAC5D,GAAG,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC;IAC3C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC;IACjD,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,kBAAkB,CAAC;IAC7C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC;CAClD,CAAA;AACD,MAAM,aAAa,GAA6C;IAC9D,GAAG,EAAE,KAAK,CAAC,aAAa,CAAC,gBAAgB,CAAC;IAC1C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,mBAAmB,CAAC;IAChD,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC;IAC5C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,mBAAmB,CAAC;CACjD,CAAA;AACD,MAAM,cAAc,GAA6C;IAC/D,GAAG,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC;IAC3C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC;IACjD,IAAI,EAAE,KAAK,CAAC,aAAa,CAAC,kBAAkB,CAAC;IAC7C,MAAM,EAAE,KAAK,CAAC,aAAa,CAAC,oBAAoB,CAAC;CAClD,CAAA;AAED,MAAM,iBAAiB,GAAG,KAAK,CAAC,WAAW,CAAC,yBAAyB,CAAC,CAAA;AAEtE,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,EAAE,CAAA;AAEpC,MAAM,OAAO,YAAY;IAUZ;IACA;IAVH,MAAM,CAAC,SAAS,GAAG,EAAE,CAAA;IAErB,MAAM,GAAG,IAAI,GAAG,EAGrB,CAAA;IACK,aAAa,CAA4B;IAEjD,YACW,OAAoD,EACpD,SAAiB;QADjB,YAAO,GAAP,OAAO,CAA6C;QACpD,cAAS,GAAT,SAAS,CAAQ;IACzB,CAAC;IAEJ,UAAU,CAAI,IAAY,EAAE,WAAyB;QACnD,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;YACvD,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;QAC3E,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,WAAW,CAAC,OAA6B,EAAE,WAAoB;QAC7D,IAAI,gBAAgB,GAAG,CAAC,IAAI,gBAAgB,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnE,2BAA2B;YAC3B,OAAO,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,MAA4B,CAAC,CAAA;QACrE,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAgB,CAAA;QAC1D,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;QAElD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,QAAQ,GAAG,CAAC,OAAO,CAAC,CAAA;QAC1B,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;QACnD,IAAI,KAAiC,CAAA;QACrC,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC,CAAA;YAC5G,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAC/B,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,SAAS,EAAE;gBACT,GAAG,OAAO;gBACV,IAAI;aACL;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAA;QAEF,WAAW,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;QAEhC,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC1B,IAAI,CAAC,CAAC,MAAkB,EAAE,EAAE;YAC3B,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,IAAI,CAAC,CAAA;YACvF,aAAa,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAA;YACnD,OAAO,MAAM,CAAA;QACf,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACX,IAAI,CAAC,KAAK,YAAY,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAAA;YACnE,CAAC;YACD,cAAc,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACnC,MAAM,CAAC,CAAA;QACT,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,KAAK,CAAC,CAAA;YACrB,CAAC;QACH,CAAC,CAAC,CAAA;IACN,CAAC;IAED,MAAM,CAAC,QAAoB;QACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QACpD,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;gBACtB,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACxC,CAAC;YACD,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBACnB,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC;QACD,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;IAC3E,CAAC;IAED,KAAK,CAAC,SAAiB,EAAE,CAAM;QAC7B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE;aACrB;SACF,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;YAChB,MAAM,EAAE,WAAW;YACnB,SAAS;SACV,CAAC,CAAA;IACJ,CAAC;IAED,KAAK;QACH,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,mCAAmC,EAAE,IAAI,CAAC,CAAA;YACvD,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC3C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;QACnB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QACnC,CAAC;IACH,CAAC;IAED,WAAW,GAOK,SAAS,CAAA;IAEjB,KAAK,CAAC,iBAAiB,CAAC,GAAuB;QACrD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,oBAAoB;YACpB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,WAAW,CAAA;YACpC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACnE,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YAC/E,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YACvD,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;gBAC9C,IAAI,CAAC,SAAS,EAAE,CAAA;YAClB,CAAC;YACD,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,YAAY,CAAC,SAAS,EAAE,CAAA;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAa,IAAI,EAAE,QAAQ,CAAC,CAAA;YAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC9B,IAAI,CAAC,SAAS,EAAE,CAAA;YAClB,CAAC,EAAE,gBAAgB,CAAC,CAAA;YAEpB,IAAI,CAAC,WAAW,GAAG;gBACjB,IAAI;gBACJ,OAAO,EAAE,GAAG;gBACZ,OAAO;gBACP,KAAK,EAAE,OAAO;aACf,CAAA;YAED,OAAO,OAAO,CAAA;QAChB,CAAC;IACH,CAAC;IAEO,SAAS;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,WAAW,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YACjF,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;YAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,SAAS,EAAE;oBACT,MAAM,EAAE,OAAO;oBACf,IAAI;iBACL;gBACD,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAA;YACF,WAAW,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;QAC/B,CAAC;IACH,CAAC"}
|
package/lib/processor-runner.js
CHANGED
|
@@ -52,8 +52,14 @@ console.debug('Starting with', options.target);
|
|
|
52
52
|
Error.stackTraceLimit = 20;
|
|
53
53
|
const fullPath = path.resolve(options['chains-config']);
|
|
54
54
|
const chainsConfig = fs.readJsonSync(fullPath);
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
|
|
56
|
+
? parseInt(process.env['OVERRIDE_CONCURRENCY'])
|
|
57
|
+
: undefined;
|
|
58
|
+
const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
|
|
59
|
+
? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
|
|
60
|
+
: undefined;
|
|
61
|
+
Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency;
|
|
62
|
+
Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count'];
|
|
57
63
|
Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server'];
|
|
58
64
|
Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server'];
|
|
59
65
|
for (const [id, config] of Object.entries(chainsConfig)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"processor-runner.js","sourceRoot":"","sources":["../src/processor-runner.ts"],"names":[],"mappings":";AAEA,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,MAAM,UAAU,CAAA;AAEzB,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,eAAe,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAA;AAC/F,OAAO,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAA;AACvE,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,8DAA8D;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AAEjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAA;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAA;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAA;AAC3E,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAA;AAE1E,MAAM,GAAG,GAAG,IAAI,OAAO,CAAC;IACtB,aAAa,EAAE,IAAI,iBAAiB,EAAE;IACtC,YAAY,EAAE,IAAI,6BAA6B,CAAC;QAC9C,QAAQ,EAAE,IAAI,kBAAkB,EAAE;KACnC,CAAC;IACF,qDAAqD;CACtD,CAAC,CAAA;AAEF,GAAG,CAAC,KAAK,EAAE,CAAA;AAEX,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAA;AAEzE,MAAM,iBAAiB,GAAG;IACxB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE;IACrD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE;IAChE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE;IACtD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE;IACtD,mEAAmE;IACnE;QACE,IAAI,EAAE,eAAe;QACrB,KAAK,EAAE,GAAG;QACV,IAAI,EAAE,MAAM;QACZ,YAAY,EAAE,oBAAoB;KACnC;IACD,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE;IAC7D,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE;IAC5D,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE;IAC7D,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE;CACtD,CAAA;AAED,MAAM,OAAO,GAAG,eAAe,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;AAErE,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5D,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;AAE9C,KAAK,CAAC,eAAe,GAAG,EAAE,CAAA;AAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAA;AACvD,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;AAE9C,SAAS,CAAC,QAAQ,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"processor-runner.js","sourceRoot":"","sources":["../src/processor-runner.ts"],"names":[],"mappings":";AAEA,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,MAAM,UAAU,CAAA;AAEzB,OAAO,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA;AACrD,OAAO,eAAe,MAAM,mBAAmB,CAAA;AAC/C,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AACxC,OAAO,EAAE,4BAA4B,EAAE,MAAM,yBAAyB,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAA;AAC/F,OAAO,EAAE,6BAA6B,EAAE,MAAM,yBAAyB,CAAA;AACvE,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,8DAA8D;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AAEjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAA;AACzE,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAC1C,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2CAA2C,CAAA;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,yCAAyC,CAAA;AAC3E,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAA;AAE1E,MAAM,GAAG,GAAG,IAAI,OAAO,CAAC;IACtB,aAAa,EAAE,IAAI,iBAAiB,EAAE;IACtC,YAAY,EAAE,IAAI,6BAA6B,CAAC;QAC9C,QAAQ,EAAE,IAAI,kBAAkB,EAAE;KACnC,CAAC;IACF,qDAAqD;CACtD,CAAC,CAAA;AAEF,GAAG,CAAC,KAAK,EAAE,CAAA;AAEX,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAA;AAEzE,MAAM,iBAAiB,GAAG;IACxB,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE;IACrD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE;IAChE,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE;IACtD,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE;IACtD,mEAAmE;IACnE;QACE,IAAI,EAAE,eAAe;QACrB,KAAK,EAAE,GAAG;QACV,IAAI,EAAE,MAAM;QACZ,YAAY,EAAE,oBAAoB;KACnC;IACD,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE;IAC7D,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE;IAC5D,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE;IAC7D,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE;CACtD,CAAA;AAED,MAAM,OAAO,GAAG,eAAe,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;AAErE,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;AAC5D,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;AAE9C,KAAK,CAAC,eAAe,GAAG,EAAE,CAAA;AAE1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAA;AACvD,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;AAE9C,MAAM,mBAAmB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC7D,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAC/C,CAAC,CAAC,SAAS,CAAA;AACb,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IAC5D,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAC/C,CAAC,CAAC,SAAS,CAAA;AAEb,SAAS,CAAC,QAAQ,CAAC,WAAW,GAAG,mBAAmB,IAAI,OAAO,CAAC,WAAW,CAAA;AAC3E,SAAS,CAAC,QAAQ,CAAC,UAAU,GAAG,kBAAkB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAA;AAC5E,SAAS,CAAC,QAAQ,CAAC,aAAa,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;AAC/D,SAAS,CAAC,QAAQ,CAAC,YAAY,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;AAE7D,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;IACxD,MAAM,WAAW,GAAG,MAAqB,CAAA;IACzC,IAAI,WAAW,CAAC,WAAW,EAAE,CAAC;QAC5B,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,CAAC,WAAW,CAAC,CAAA;IACjE,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACnC,IAAI,IAAI,EAAE,CAAC;YACT,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;QAC9C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAA;QACjD,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;AAEzC,MAAM,MAAM,GAAG,YAAY,CAAC;IAC1B,8BAA8B,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI;IACjD,iCAAiC,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI;IACpD,oCAAoC,EAAE,qBAAqB,CAAC,IAAI;CACjE,CAAC;KACC,GAAG,CAAC,0BAA0B,EAAE,CAAC;KACjC,GAAG,CAAC,6BAA6B,EAAE,CAAC;KACpC,GAAG,CAAC,4BAA4B,CAAC,CAAA;AACpC,MAAM,WAAW,GAAG,IAAI,oBAAoB,CAAC,KAAK,IAAI,EAAE;IACtD,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IACtC,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,CAAA;IACjC,OAAO,CAAC,CAAA;AACV,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;AACnB,MAAM,OAAO,GAAG,IAAI,wBAAwB,CAAC,WAAW,CAAC,CAAA;AAEzD,MAAM,CAAC,GAAG,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAA;AAExC,MAAM,CAAC,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;AAExC,OAAO,CAAC,GAAG,CAAC,8BAA8B,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;AAEzD,MAAM,WAAW,GAAG,IAAI,CAAA;AACxB,MAAM,UAAU,GAAG,IAAI;KACpB,YAAY,CAAC,KAAK,WAAW,GAAG,EAAE,GAAG;IACpC,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,MAAM,OAAO,GAAG,MAAM,CAAC,YAAY,CAAA;QACnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxB,KAAK,UAAU;gBACb,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,CAAA;gBAC9C,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;gBAClB,MAAK;YACP,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC;oBACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,EAAE,EAAE,CAAC,IAAI,IAAI,CAAA;oBACpE,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;oBAC7B,OAAO,CAAC,OAAO,EAAE,CAAA;oBAEjB,MAAM,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;oBACrC,MAAM,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;oBAEpC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAA;oBAChE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;oBAEvD,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAA;oBAC1D,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAA;oBAClC,OAAO,CAAC,UAAU,EAAE,CAAA;gBACtB,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;gBACpB,CAAC;gBACD,MAAK;YACP,CAAC;YACD;gBACE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IACpB,CAAC;IACD,GAAG,CAAC,GAAG,EAAE,CAAA;AACX,CAAC,CAAC;KACD,MAAM,CAAC,WAAW,CAAC,CAAA;AAEtB,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,WAAW,CAAC,CAAA;AAErD,OAAO;KACJ,EAAE,CAAC,QAAQ,EAAE;IACZ,eAAe,CAAC,CAAC,CAAC,CAAA;AACpB,CAAC,CAAC;KACD,EAAE,CAAC,mBAAmB,EAAE,CAAC,GAAG,EAAE,EAAE;IAC/B,OAAO,CAAC,KAAK,CAAC,+DAA+D,EAAE,GAAG,CAAC,CAAA;IACnF,WAAW,CAAC,SAAS,GAAG,GAAG,CAAA;IAC3B,qBAAqB;AACvB,CAAC,CAAC;KACD,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;IACtC,sCAAsC;IACtC,IAAI,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,6CAA6C,CAAC,EAAE,CAAC;QAC9E,OAAM;IACR,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,2DAA2D,EAAE,MAAM,CAAC,CAAA;IAClF,WAAW,CAAC,SAAS,GAAG,MAAe,CAAA;IACvC,qBAAqB;AACvB,CAAC,CAAC,CAAA;AAEJ,SAAS,eAAe,CAAC,QAAgB;IACvC,MAAM,CAAC,aAAa,EAAE,CAAA;IACtB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IAEnC,UAAU,CAAC,KAAK,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;QACpC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sentio/runtime",
|
|
3
|
-
"version": "2.39.7-rc.
|
|
3
|
+
"version": "2.39.7-rc.25",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
"rxjs": "^7.8.1",
|
|
36
36
|
"utility-types": "^3.11.0",
|
|
37
37
|
"winston": "^3.11.0",
|
|
38
|
-
"@sentio/protos": "2.39.7-rc.
|
|
38
|
+
"@sentio/protos": "2.39.7-rc.25"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@types/command-line-args": "^5.2.3",
|
package/src/db-context.ts
CHANGED
|
@@ -10,7 +10,8 @@ import {
|
|
|
10
10
|
import * as process from 'node:process'
|
|
11
11
|
import { Attributes, Counter, metrics } from '@opentelemetry/api'
|
|
12
12
|
|
|
13
|
-
const STORE_BATCH_IDLE = process.env['
|
|
13
|
+
const STORE_BATCH_IDLE = process.env['STORE_BATCH_MAX_IDLE'] ? parseInt(process.env['STORE_BATCH_MAX_IDLE']) : 1
|
|
14
|
+
const STORE_BATCH_SIZE = process.env['STORE_BATCH_SIZE'] ? parseInt(process.env['STORE_BATCH_SIZE']) : 10
|
|
14
15
|
|
|
15
16
|
type Request = Omit<DBRequest, 'opId'>
|
|
16
17
|
type RequestType = keyof Request
|
|
@@ -67,9 +68,9 @@ export class StoreContext {
|
|
|
67
68
|
}
|
|
68
69
|
|
|
69
70
|
sendRequest(request: DeepPartial<Request>, timeoutSecs?: number): Promise<DBResponse> {
|
|
70
|
-
if (STORE_BATCH_IDLE > 0 && request.upsert) {
|
|
71
|
+
if (STORE_BATCH_IDLE > 0 && STORE_BATCH_SIZE > 1 && request.upsert) {
|
|
71
72
|
// batch upsert if possible
|
|
72
|
-
return this.
|
|
73
|
+
return this.sendUpsertInBatch(request.upsert as DBRequest_DBUpsert)
|
|
73
74
|
}
|
|
74
75
|
|
|
75
76
|
const requestType = Object.keys(request)[0] as RequestType
|
|
@@ -157,40 +158,58 @@ export class StoreContext {
|
|
|
157
158
|
}
|
|
158
159
|
}
|
|
159
160
|
|
|
160
|
-
|
|
161
|
-
|
|
161
|
+
upsertBatch:
|
|
162
|
+
| {
|
|
163
|
+
opId: bigint
|
|
164
|
+
request: DBRequest_DBUpsert
|
|
165
|
+
promise: Promise<DBResponse>
|
|
166
|
+
timer: NodeJS.Timeout
|
|
167
|
+
}
|
|
168
|
+
| undefined = undefined
|
|
162
169
|
|
|
163
|
-
private async
|
|
164
|
-
if (this.
|
|
170
|
+
private async sendUpsertInBatch(req: DBRequest_DBUpsert): Promise<DBResponse> {
|
|
171
|
+
if (this.upsertBatch) {
|
|
165
172
|
// merge the upserts
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
173
|
+
const { request } = this.upsertBatch
|
|
174
|
+
request.entity = this.upsertBatch.request.entity.concat(req.entity)
|
|
175
|
+
request.entityData = this.upsertBatch.request.entityData.concat(req.entityData)
|
|
176
|
+
request.id = this.upsertBatch.request.id.concat(req.id)
|
|
177
|
+
if (request.entity.length >= STORE_BATCH_SIZE) {
|
|
178
|
+
this.sendBatch()
|
|
179
|
+
}
|
|
180
|
+
return this.upsertBatch.promise
|
|
171
181
|
} else {
|
|
172
|
-
this.queuedUpsert = req
|
|
173
182
|
const opId = StoreContext.opCounter++
|
|
174
183
|
const promise = this.newPromise<DBResponse>(opId, 'upsert')
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
const timeout = setTimeout(() => {
|
|
185
|
+
this.sendBatch()
|
|
186
|
+
}, STORE_BATCH_IDLE)
|
|
187
|
+
|
|
188
|
+
this.upsertBatch = {
|
|
189
|
+
opId,
|
|
190
|
+
request: req,
|
|
191
|
+
promise,
|
|
192
|
+
timer: timeout
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return promise
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
private sendBatch() {
|
|
200
|
+
if (this.upsertBatch) {
|
|
201
|
+
const { request, opId, timer } = this.upsertBatch
|
|
202
|
+
console.debug('sending batch upsert', opId, 'batch size', request?.entity.length)
|
|
203
|
+
clearTimeout(timer)
|
|
204
|
+
this.upsertBatch = undefined
|
|
180
205
|
this.subject.next({
|
|
181
206
|
dbRequest: {
|
|
182
|
-
upsert:
|
|
207
|
+
upsert: request,
|
|
183
208
|
opId
|
|
184
209
|
},
|
|
185
210
|
processId: this.processId
|
|
186
211
|
})
|
|
187
212
|
send_counts['upsert']?.add(1)
|
|
188
|
-
|
|
189
|
-
return promise
|
|
190
213
|
}
|
|
191
214
|
}
|
|
192
215
|
}
|
|
193
|
-
|
|
194
|
-
function delay(ms: number) {
|
|
195
|
-
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
196
|
-
}
|
package/src/processor-runner.ts
CHANGED
|
@@ -66,8 +66,15 @@ Error.stackTraceLimit = 20
|
|
|
66
66
|
const fullPath = path.resolve(options['chains-config'])
|
|
67
67
|
const chainsConfig = fs.readJsonSync(fullPath)
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
|
|
70
|
+
? parseInt(process.env['OVERRIDE_CONCURRENCY'])
|
|
71
|
+
: undefined
|
|
72
|
+
const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
|
|
73
|
+
? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
|
|
74
|
+
: undefined
|
|
75
|
+
|
|
76
|
+
Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
|
|
77
|
+
Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
|
|
71
78
|
Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
|
|
72
79
|
Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
|
|
73
80
|
|