@subsquid/batch-processor 0.1.0-portal-api.d0738f → 0.2.0-portal-api.b88e99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/run.d.ts +6 -1
- package/lib/run.d.ts.map +1 -1
- package/lib/run.js +26 -40
- package/lib/run.js.map +1 -1
- package/package.json +5 -5
- package/src/run.ts +47 -24
- package/lib/metrics.d.ts +0 -21
- package/lib/metrics.d.ts.map +0 -1
- package/lib/metrics.js +0 -92
- package/lib/metrics.js.map +0 -1
- package/src/metrics.ts +0 -111
package/lib/run.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { Database } from './database';
|
|
2
2
|
import { DataSource, BlockRef } from '@subsquid/util-internal-data-source';
|
|
3
|
+
import { PrometheusServer } from '@subsquid/util-internal-processor-tools';
|
|
4
|
+
export { PrometheusServer };
|
|
3
5
|
export interface DataHandlerContext<Block, Store> {
|
|
4
6
|
/**
|
|
5
7
|
* Storage interface provided by the database
|
|
@@ -17,6 +19,9 @@ export interface DataHandlerContext<Block, Store> {
|
|
|
17
19
|
export interface BlockBase {
|
|
18
20
|
header: BlockRef;
|
|
19
21
|
}
|
|
22
|
+
interface RunOptions {
|
|
23
|
+
prometheus?: PrometheusServer;
|
|
24
|
+
}
|
|
20
25
|
/**
|
|
21
26
|
* Run data processing.
|
|
22
27
|
*
|
|
@@ -31,5 +36,5 @@ export interface BlockBase {
|
|
|
31
36
|
*
|
|
32
37
|
* @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
|
|
33
38
|
*/
|
|
34
|
-
export declare function run<Block extends BlockBase, Store>(src: DataSource<Block>, db: Database<Store>, dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void
|
|
39
|
+
export declare function run<Block extends BlockBase, Store>(src: DataSource<Block>, db: Database<Store>, dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>, opts?: RunOptions): void;
|
|
35
40
|
//# sourceMappingURL=run.d.ts.map
|
package/lib/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgB,QAAQ,EAAC,MAAM,YAAY,CAAA;AAClD,OAAO,EAAC,UAAU,EAAmB,QAAQ,EAAkB,MAAM,qCAAqC,CAAA;AAE1G,OAAO,EAAC,gBAAgB,EAAgB,MAAM,yCAAyC,CAAA;AAIvF,OAAO,EAAC,gBAAgB,EAAC,CAAA;AAKzB,MAAM,WAAW,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAC5C;;OAEG;IACH,KAAK,EAAE,KAAK,CAAA;IACZ;;OAEG;IACH,MAAM,EAAE,KAAK,EAAE,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,OAAO,CAAA;CAClB;AAGD,MAAM,WAAW,SAAS;IACtB,MAAM,EAAE,QAAQ,CAAA;CACnB;AASD,UAAU,UAAU;IAChB,UAAU,CAAC,EAAE,gBAAgB,CAAA;CAChC;AAGD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,GAAG,CAAC,KAAK,SAAS,SAAS,EAAE,KAAK,EAC9C,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,EACtB,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EACnB,WAAW,EAAE,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,EACrE,IAAI,CAAC,EAAE,UAAU,GAClB,IAAI,CAMN"}
|
package/lib/run.js
CHANGED
|
@@ -1,40 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
4
|
};
|
|
28
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.PrometheusServer = void 0;
|
|
29
7
|
exports.run = run;
|
|
30
8
|
const logger_1 = require("@subsquid/logger");
|
|
31
9
|
const util_internal_1 = require("@subsquid/util-internal");
|
|
32
|
-
const util_internal_prometheus_server_1 = require("@subsquid/util-internal-prometheus-server");
|
|
33
|
-
const prom = __importStar(require("prom-client"));
|
|
34
|
-
const metrics_1 = require("./metrics");
|
|
35
10
|
const util_internal_data_source_1 = require("@subsquid/util-internal-data-source");
|
|
36
|
-
const util_1 = require("./util");
|
|
37
11
|
const assert_1 = __importDefault(require("assert"));
|
|
12
|
+
const util_internal_processor_tools_1 = require("@subsquid/util-internal-processor-tools");
|
|
13
|
+
Object.defineProperty(exports, "PrometheusServer", { enumerable: true, get: function () { return util_internal_processor_tools_1.PrometheusServer; } });
|
|
14
|
+
const util_1 = require("./util");
|
|
38
15
|
const log = (0, logger_1.createLogger)('sqd:batch-processor');
|
|
39
16
|
/**
|
|
40
17
|
* Run data processing.
|
|
@@ -50,20 +27,21 @@ const log = (0, logger_1.createLogger)('sqd:batch-processor');
|
|
|
50
27
|
*
|
|
51
28
|
* @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
|
|
52
29
|
*/
|
|
53
|
-
function run(src, db, dataHandler) {
|
|
30
|
+
function run(src, db, dataHandler, opts) {
|
|
54
31
|
(0, util_internal_1.runProgram)(() => {
|
|
55
|
-
return new Processor(src, db, dataHandler).run();
|
|
56
|
-
},
|
|
32
|
+
return new Processor(src, db, dataHandler, opts).run();
|
|
33
|
+
}, err => {
|
|
57
34
|
log.fatal(err);
|
|
58
35
|
});
|
|
59
36
|
}
|
|
60
37
|
class Processor {
|
|
61
|
-
constructor(src, db, handler) {
|
|
38
|
+
constructor(src, db, handler, opts) {
|
|
62
39
|
this.src = src;
|
|
63
40
|
this.db = db;
|
|
64
41
|
this.handler = handler;
|
|
65
|
-
this.
|
|
42
|
+
this.opts = opts;
|
|
66
43
|
this.hasStatusNews = false;
|
|
44
|
+
this.metrics = new util_internal_processor_tools_1.RunnerMetrics(src.getBlocksCountInRange?.bind(src) ?? ((range) => Math.max(0, range.to - range.from + 1)));
|
|
67
45
|
}
|
|
68
46
|
async run() {
|
|
69
47
|
let getHead = this.db.supportsHotBlocks ? this.src.getHead.bind(this.src) : this.src.getFinalizedHead.bind(this.src);
|
|
@@ -122,19 +100,24 @@ class Processor {
|
|
|
122
100
|
async initMetrics(state, chainHeight) {
|
|
123
101
|
this.updateProgressMetrics(chainHeight, state);
|
|
124
102
|
let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT;
|
|
125
|
-
|
|
103
|
+
let prometheusServer;
|
|
104
|
+
if (this.opts?.prometheus != null) {
|
|
105
|
+
prometheusServer = this.opts.prometheus;
|
|
106
|
+
}
|
|
107
|
+
else if (port != null) {
|
|
108
|
+
prometheusServer = new util_internal_processor_tools_1.PrometheusServer();
|
|
109
|
+
prometheusServer.setPort(port);
|
|
110
|
+
}
|
|
111
|
+
if (prometheusServer == null)
|
|
126
112
|
return;
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
log.info(`prometheus metrics are served on port ${server.port}`);
|
|
113
|
+
prometheusServer.addRunnerMetrics(this.metrics);
|
|
114
|
+
let listening = await prometheusServer.serve();
|
|
115
|
+
log.info(`prometheus metrics are served on port ${listening.port}`);
|
|
131
116
|
}
|
|
132
117
|
updateProgressMetrics(chainHeight, indexerHeight, time) {
|
|
133
118
|
this.metrics.setChainHeight(chainHeight);
|
|
134
119
|
this.metrics.setLastProcessedBlock(indexerHeight);
|
|
135
|
-
|
|
136
|
-
let processed = this.metrics.getLastProcessedBlock();
|
|
137
|
-
this.metrics.updateProgress(processed, left, time);
|
|
120
|
+
this.metrics.updateProgress(time);
|
|
138
121
|
}
|
|
139
122
|
async processBatch(state, data, chainHeight) {
|
|
140
123
|
let { blocks, finalizedHead: finalizedHeadData } = data;
|
|
@@ -145,6 +128,9 @@ class Processor {
|
|
|
145
128
|
if (prevHead && prevHead.number >= blocks[0].header.number) {
|
|
146
129
|
throw new Error('Data is not continuous');
|
|
147
130
|
}
|
|
131
|
+
if (finalizedHeadData != null && state.finalizedHead != null && finalizedHeadData.number <= state.finalizedHead.number) {
|
|
132
|
+
finalizedHeadData = state.finalizedHead;
|
|
133
|
+
}
|
|
148
134
|
let unfinalizedIndex = 0;
|
|
149
135
|
if (finalizedHeadData != null) {
|
|
150
136
|
unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHeadData.number);
|
package/lib/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;;;;AA4DA,kBAWC;AAvED,6CAA6C;AAC7C,2DAA8E;AAE9E,mFAA0G;AAC1G,oDAA2B;AAC3B,2FAAuF;AAI/E,iGAJA,gDAAgB,OAIA;AAHxB,iCAAgD;AAMhD,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AAkC/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE,EACrE,IAAiB;IAEjB,IAAA,0BAAU,EAAC,GAAG,EAAE;QACR,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAA;IAC9D,CAAC,EAAE,GAAG,CAAC,EAAE;QACD,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC,CAAC,CAAA;AACN,CAAC;AAED,MAAM,SAAS;IAKX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD,EAChD,IAAiB;QAH1B,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAChD,SAAI,GAAJ,IAAI,CAAa;QAN9B,kBAAa,GAAG,KAAK,CAAA;QAQzB,IAAI,CAAC,OAAO,GAAG,IAAI,6CAAa,CAC5B,GAAG,CAAC,qBAAqB,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAC9F,CAAA;IACL,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACpH,IAAI,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,KAAM,CAAC,CAAA;QAEtF,IAAI,KAAK,GAAmB;YACxB,aAAa,EAAE,SAAS;YACxB,gBAAgB,EAAE,EAAE;SACvB,CAAA;QACD,IAAI,IAAI,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC;YAC5B,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACrC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;YAC1E,KAAK,CAAC,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QACxD,CAAC;aAAM,CAAC;YACJ,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;YACrC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;QAC9E,CAAC;QAED,IAAI,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACf,GAAG,CAAC,IAAI,CAAC,4BAA4B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACvD,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;QAEnE,OAAO,IAAI,EAAE,CAAC;YACV,IAAI,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,iBAAiB;gBACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBACnC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEhD,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;YAC1B,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,SAAS,CAAC,EAAC,KAAK,EAAE,IAAI,EAAC,CAAC,EAAE,CAAC;oBAC9C,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;gBACzE,CAAC;gBACD,MAAK,CAAC,2CAA2C;YACrD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAI,CAAC,IAAA,2CAAe,EAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,iBAAiB;oBAAE,MAAM,CAAC,CAAA;gBAE9D,yCAAyC;gBACzC,IAAI,KAAK,GAAG,KAAK,CAAC,aAAa;oBAC3B,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;oBAClD,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAA;gBAC5B,IAAI,aAAa,GAAG,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,cAAc,CAAC,CAAA;gBAC9D,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;oBACvB,IAAI,KAAK,CAAC,aAAa,IAAI,IAAI;wBAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;oBAC1E,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAA;gBAC/B,CAAC;qBAAM,CAAC;oBACJ,MAAM,YAAY,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;oBACzC,GAAG,CAAC,IAAI,CAAC,sCAAsC,IAAA,iBAAU,EAAC,YAAY,CAAC,EAAE,CAAC,CAAA;oBAE1E,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,GAAG,CAAC,CAAC,CAAA;gBAC9D,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAa,EAAE,WAAmB;QACxD,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;QAC9C,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAA;QAE/E,IAAI,gBAA8C,CAAA;QAClD,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;YAChC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAA;QAC3C,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,gBAAgB,GAAG,IAAI,gDAAgB,EAAE,CAAA;YACzC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAClC,CAAC;QACD,IAAI,gBAAgB,IAAI,IAAI;YAAE,OAAM;QAEpC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/C,IAAI,SAAS,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,CAAA;QAC9C,GAAG,CAAC,IAAI,CAAC,yCAAyC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IACvE,CAAC;IAEO,qBAAqB,CAAC,WAAmB,EAAE,aAAqB,EAAE,IAAa;QACnF,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAA;QACjD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,YAAY,CACtB,KAAqB,EACrB,IAAmB,EACnB,WAAmB;QAEnB,IAAI,EAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAC,GAAG,IAAI,CAAA;QAErD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QAErC,IAAI,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAA;QAElC,2BAA2B;QAC3B,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;QAC7C,CAAC;QAED,IAAI,iBAAiB,IAAI,IAAI,IAAI,KAAK,CAAC,aAAa,IAAI,IAAI,IAAI,iBAAiB,CAAC,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YACrH,iBAAiB,GAAG,KAAK,CAAC,aAAa,CAAA;QAC3C,CAAC;QAED,IAAI,gBAAgB,GAAG,CAAC,CAAA;QACxB,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAC5B,gBAAgB,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,iBAAkB,CAAC,MAAM,CAAC,CAAA;QAC3F,CAAC;QAED,IAAI,QAAQ,GAAG,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAA;QAClC,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,IAAI,WAAW,CAAA;QAE5C,IAAI,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9C,+BAA+B;QAC/B,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAA;YACrD,KAAK,CAAC,aAAa,GAAG,EAAC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC,CAAA;YAC5E,KAAK,CAAC,gBAAgB,GAAG,EAAE,CAAA;YAE3B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAClB;gBACI,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAC,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAC;gBACzE,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC;gBACnC,OAAO;aACV,EACD,CAAC,KAAK,EAAE,EAAE;gBACN,OAAO,IAAI,CAAC,OAAO,CAAC;oBAChB,KAAK;oBACL,MAAM;oBACN,MAAM,EAAE,OAAO;iBAClB,CAAC,CAAA;YACN,CAAC,CACJ,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,IAAA,gBAAM,EAAC,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAA;YAEjC,IAAI,YAAY,GAAyB,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,MAAM,CAAA;YAC7E,IAAI,iBAAiB,EAAE,IAAI,IAAI,iBAAiB,EAAE,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC/D,YAAY,GAAG,EAAC,IAAI,EAAE,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAC,CAAA;YACnF,CAAC;YACD,KAAK,CAAC,aAAa,GAAG,YAAY,IAAI,KAAK,CAAC,aAAa,CAAA;YAEzD,2DAA2D;YAC3D,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;gBACtB,IAAI,aAAa,GAAG,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,aAAc,CAAC,MAAM,CAAC,CAAA;gBACnG,KAAK,CAAC,gBAAgB,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;YACjG,CAAC;YAED,6BAA6B;YAC7B,KAAK,IAAI,CAAC,GAAG,gBAAgB,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpD,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAC,CAAC,CAAA;YAC/F,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,CACtB;gBACI,aAAa,EAAE,eAAe,CAAC,iBAAiB,IAAI,KAAK,CAAC,aAAa,CAAC;gBACxE,QAAQ,EAAE,eAAe,CAAC,QAAQ,IAAI,KAAK,CAAC,aAAa,CAAC;gBAC1D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;aAC1D,EACD,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAClB,OAAO,IAAI,CAAC,OAAO,CAAC;oBAChB,KAAK;oBACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;oBAChC,MAAM,EAAE,OAAO;iBAClB,CAAC,CAAA;YACN,CAAC,CACJ,CAAA;QACL,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE5C,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;QACxE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,IAAA,oBAAa,EAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAA;QAElG,IAAI,CAAC,YAAY,EAAE,CAAA;QAEnB,OAAO,KAAK,CAAA;IAChB,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAA;gBAClC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;oBAC1B,IAAI,CAAC,YAAY,EAAE,CAAA;gBACvB,CAAC;YACL,CAAC,EAAE,IAAK,CAAC,CAAA;QACb,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC7B,CAAC;IACL,CAAC;IAEO,iBAAiB;QACrB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;IACL,CAAC;CACJ;AAED,SAAS,iBAAiB,CAAC,YAAwB,EAAE,SAAqB;IACtE,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,eAAe,GAAG,CAAC,CAAC,CAAA;IAExB,OAAO,YAAY,GAAG,YAAY,CAAC,MAAM,IAAI,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;QACxE,MAAM,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC,CAAA;QAC/C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;QAEtC,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YACzC,SAAS,EAAE,CAAA;YACX,SAAQ;QACZ,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YACzC,YAAY,EAAE,CAAA;YACd,SAAQ;QACZ,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACvC,OAAO,eAAe,CAAA;QAC1B,CAAC;QAED,eAAe,GAAG,YAAY,CAAA;QAC9B,YAAY,EAAE,CAAA;QACd,SAAS,EAAE,CAAA;IACf,CAAC;IAED,OAAO,eAAe,CAAA;AAC1B,CAAC;AAED,SAAS,eAAe,CAAC,GAAa;IAClC,OAAO,EAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAC,CAAA;AAC/C,CAAC;AAED,SAAS,UAAU,CAAC,aAA4B;IAC5C,OAAO,EAAC,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,IAAI,EAAC,CAAA;AACnE,CAAC;AAED,SAAS,YAAY,CAAC,KAAqB;IACvC,OAAO,IAAA,yBAAS,EAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAA;AACnE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@subsquid/batch-processor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0-portal-api.b88e99",
|
|
4
4
|
"description": "ETL processor",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
|
-
"repository": "git@github.com:subsquid/squid.git",
|
|
6
|
+
"repository": "git@github.com:subsquid/squid-sdk.git",
|
|
7
7
|
"publishConfig": {
|
|
8
8
|
"access": "public",
|
|
9
9
|
"registry": "https://registry.npmjs.org/"
|
|
@@ -14,13 +14,13 @@
|
|
|
14
14
|
],
|
|
15
15
|
"main": "lib/index.js",
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@subsquid/logger": "^1.
|
|
17
|
+
"@subsquid/logger": "^1.5.0",
|
|
18
18
|
"@subsquid/util-internal": "^3.2.0",
|
|
19
|
+
"@subsquid/util-internal-processor-tools": "^4.4.0",
|
|
19
20
|
"@subsquid/util-internal-counters": "^1.3.2",
|
|
20
21
|
"@subsquid/util-internal-prometheus-server": "^1.3.0",
|
|
21
22
|
"@subsquid/util-internal-range": "^0.3.0",
|
|
22
|
-
"
|
|
23
|
-
"@subsquid/util-internal-data-source": "0.0.1-portal-api.d0738f"
|
|
23
|
+
"@subsquid/util-internal-data-source": "0.0.1-portal-api.b88e99"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@types/node": "^18.18.14",
|
package/src/run.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import {createLogger} from '@subsquid/logger'
|
|
2
2
|
import {last, maybeLast, runProgram, Throttler} from '@subsquid/util-internal'
|
|
3
|
-
import {
|
|
4
|
-
import * as prom from 'prom-client'
|
|
5
|
-
import {HashAndHeight, Database, HotDatabaseState} from './database'
|
|
6
|
-
import {Metrics} from './metrics'
|
|
3
|
+
import {HashAndHeight, Database} from './database'
|
|
7
4
|
import {DataSource, isForkException, BlockRef, type BlockBatch} from '@subsquid/util-internal-data-source'
|
|
8
|
-
import {formatHead, getItemsCount} from './util'
|
|
9
5
|
import assert from 'assert'
|
|
6
|
+
import {PrometheusServer, RunnerMetrics} from '@subsquid/util-internal-processor-tools'
|
|
7
|
+
import {formatHead, getItemsCount} from './util'
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
export {PrometheusServer}
|
|
11
|
+
|
|
10
12
|
|
|
11
13
|
const log = createLogger('sqd:batch-processor')
|
|
12
14
|
|
|
@@ -25,15 +27,23 @@ export interface DataHandlerContext<Block, Store> {
|
|
|
25
27
|
isHead: boolean
|
|
26
28
|
}
|
|
27
29
|
|
|
30
|
+
|
|
28
31
|
export interface BlockBase {
|
|
29
32
|
header: BlockRef
|
|
30
33
|
}
|
|
31
34
|
|
|
35
|
+
|
|
32
36
|
interface ProcessorState {
|
|
33
37
|
finalizedHead: BlockRef | undefined
|
|
34
38
|
unfinalizedHeads: BlockRef[]
|
|
35
39
|
}
|
|
36
40
|
|
|
41
|
+
|
|
42
|
+
interface RunOptions {
|
|
43
|
+
prometheus?: PrometheusServer
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
37
47
|
/**
|
|
38
48
|
* Run data processing.
|
|
39
49
|
*
|
|
@@ -51,28 +61,31 @@ interface ProcessorState {
|
|
|
51
61
|
export function run<Block extends BlockBase, Store>(
|
|
52
62
|
src: DataSource<Block>,
|
|
53
63
|
db: Database<Store>,
|
|
54
|
-
dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void
|
|
64
|
+
dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>,
|
|
65
|
+
opts?: RunOptions
|
|
55
66
|
): void {
|
|
56
|
-
runProgram(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
},
|
|
60
|
-
(err) => {
|
|
67
|
+
runProgram(() => {
|
|
68
|
+
return new Processor(src, db, dataHandler, opts).run()
|
|
69
|
+
}, err => {
|
|
61
70
|
log.fatal(err)
|
|
62
|
-
|
|
63
|
-
)
|
|
71
|
+
})
|
|
64
72
|
}
|
|
65
73
|
|
|
66
74
|
class Processor<B extends BlockBase, S> {
|
|
67
|
-
private metrics
|
|
75
|
+
private metrics: RunnerMetrics
|
|
68
76
|
private statusReportTimer?: any
|
|
69
77
|
private hasStatusNews = false
|
|
70
78
|
|
|
71
79
|
constructor(
|
|
72
80
|
private src: DataSource<B>,
|
|
73
81
|
private db: Database<S>,
|
|
74
|
-
private handler: (ctx: DataHandlerContext<B, S>) => Promise<void
|
|
75
|
-
|
|
82
|
+
private handler: (ctx: DataHandlerContext<B, S>) => Promise<void>,
|
|
83
|
+
private readonly opts?: RunOptions
|
|
84
|
+
) {
|
|
85
|
+
this.metrics = new RunnerMetrics(
|
|
86
|
+
src.getBlocksCountInRange?.bind(src) ?? ((range) => Math.max(0, range.to - range.from + 1)),
|
|
87
|
+
)
|
|
88
|
+
}
|
|
76
89
|
|
|
77
90
|
async run(): Promise<void> {
|
|
78
91
|
let getHead = this.db.supportsHotBlocks ? this.src.getHead.bind(this.src) : this.src.getFinalizedHead.bind(this.src)
|
|
@@ -134,19 +147,25 @@ class Processor<B extends BlockBase, S> {
|
|
|
134
147
|
private async initMetrics(state: number, chainHeight: number): Promise<void> {
|
|
135
148
|
this.updateProgressMetrics(chainHeight, state)
|
|
136
149
|
let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
this.
|
|
140
|
-
|
|
141
|
-
|
|
150
|
+
|
|
151
|
+
let prometheusServer: PrometheusServer | undefined
|
|
152
|
+
if (this.opts?.prometheus != null) {
|
|
153
|
+
prometheusServer = this.opts.prometheus
|
|
154
|
+
} else if (port != null) {
|
|
155
|
+
prometheusServer = new PrometheusServer()
|
|
156
|
+
prometheusServer.setPort(port)
|
|
157
|
+
}
|
|
158
|
+
if (prometheusServer == null) return
|
|
159
|
+
|
|
160
|
+
prometheusServer.addRunnerMetrics(this.metrics)
|
|
161
|
+
let listening = await prometheusServer.serve()
|
|
162
|
+
log.info(`prometheus metrics are served on port ${listening.port}`)
|
|
142
163
|
}
|
|
143
164
|
|
|
144
165
|
private updateProgressMetrics(chainHeight: number, indexerHeight: number, time?: bigint): void {
|
|
145
166
|
this.metrics.setChainHeight(chainHeight)
|
|
146
167
|
this.metrics.setLastProcessedBlock(indexerHeight)
|
|
147
|
-
|
|
148
|
-
let processed = this.metrics.getLastProcessedBlock()
|
|
149
|
-
this.metrics.updateProgress(processed, left, time)
|
|
168
|
+
this.metrics.updateProgress(time)
|
|
150
169
|
}
|
|
151
170
|
|
|
152
171
|
private async processBatch(
|
|
@@ -165,6 +184,10 @@ class Processor<B extends BlockBase, S> {
|
|
|
165
184
|
throw new Error('Data is not continuous')
|
|
166
185
|
}
|
|
167
186
|
|
|
187
|
+
if (finalizedHeadData != null && state.finalizedHead != null && finalizedHeadData.number <= state.finalizedHead.number) {
|
|
188
|
+
finalizedHeadData = state.finalizedHead
|
|
189
|
+
}
|
|
190
|
+
|
|
168
191
|
let unfinalizedIndex = 0
|
|
169
192
|
if (finalizedHeadData != null) {
|
|
170
193
|
unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHeadData!.number)
|
package/lib/metrics.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
export declare class Metrics {
|
|
2
|
-
private chainHeight;
|
|
3
|
-
private lastBlock;
|
|
4
|
-
private mappingSpeed;
|
|
5
|
-
private mappingItemSpeed;
|
|
6
|
-
private blockProgress;
|
|
7
|
-
setChainHeight(height: number): void;
|
|
8
|
-
setLastProcessedBlock(height: number): void;
|
|
9
|
-
updateProgress(processed: number, left: number, time?: bigint): void;
|
|
10
|
-
registerBatch(batchSize: number, batchItemSize: number, batchMappingStartTime: bigint, batchMappingEndTime: bigint): void;
|
|
11
|
-
getChainHeight(): number;
|
|
12
|
-
getLastProcessedBlock(): number;
|
|
13
|
-
getSyncSpeed(): number;
|
|
14
|
-
getSyncEtaSeconds(): number;
|
|
15
|
-
getSyncRatio(): number;
|
|
16
|
-
getMappingSpeed(): number;
|
|
17
|
-
getMappingItemSpeed(): number;
|
|
18
|
-
getStatusLine(): string;
|
|
19
|
-
install(): void;
|
|
20
|
-
}
|
|
21
|
-
//# sourceMappingURL=metrics.d.ts.map
|
package/lib/metrics.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAKA,qBAAa,OAAO;IAChB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,aAAa,CAAkD;IAEvE,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIpC,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK3C,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI;IAKpE,aAAa,CACT,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,qBAAqB,EAAE,MAAM,EAC7B,mBAAmB,EAAE,MAAM,GAC5B,IAAI;IAKP,cAAc,IAAI,MAAM;IAIxB,qBAAqB,IAAI,MAAM;IAI/B,YAAY,IAAI,MAAM;IAItB,iBAAiB,IAAI,MAAM;IAI3B,YAAY,IAAI,MAAM;IAItB,eAAe,IAAI,MAAM;IAIzB,mBAAmB,IAAI,MAAM;IAI7B,aAAa,IAAI,MAAM;IAQvB,OAAO,IAAI,IAAI;CA+BlB"}
|
package/lib/metrics.js
DELETED
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Metrics = void 0;
|
|
4
|
-
const util_internal_counters_1 = require("@subsquid/util-internal-counters");
|
|
5
|
-
const prom_client_1 = require("prom-client");
|
|
6
|
-
const util_1 = require("./util");
|
|
7
|
-
class Metrics {
|
|
8
|
-
constructor() {
|
|
9
|
-
this.chainHeight = -1;
|
|
10
|
-
this.lastBlock = -1;
|
|
11
|
-
this.mappingSpeed = new util_internal_counters_1.Speed({ windowSize: 5 });
|
|
12
|
-
this.mappingItemSpeed = new util_internal_counters_1.Speed({ windowSize: 5 });
|
|
13
|
-
this.blockProgress = new util_internal_counters_1.Progress({ initialValue: 0, windowSize: 20 });
|
|
14
|
-
}
|
|
15
|
-
setChainHeight(height) {
|
|
16
|
-
this.chainHeight = Math.max(height, this.lastBlock);
|
|
17
|
-
}
|
|
18
|
-
setLastProcessedBlock(height) {
|
|
19
|
-
this.lastBlock = height;
|
|
20
|
-
this.chainHeight = Math.max(this.chainHeight, this.lastBlock);
|
|
21
|
-
}
|
|
22
|
-
updateProgress(processed, left, time) {
|
|
23
|
-
this.blockProgress.setTargetValue(processed + left);
|
|
24
|
-
this.blockProgress.setCurrentValue(processed, time);
|
|
25
|
-
}
|
|
26
|
-
registerBatch(batchSize, batchItemSize, batchMappingStartTime, batchMappingEndTime) {
|
|
27
|
-
this.mappingSpeed.push(batchSize, batchMappingStartTime, batchMappingEndTime);
|
|
28
|
-
this.mappingItemSpeed.push(batchItemSize || 1, batchMappingStartTime, batchMappingEndTime);
|
|
29
|
-
}
|
|
30
|
-
getChainHeight() {
|
|
31
|
-
return this.chainHeight;
|
|
32
|
-
}
|
|
33
|
-
getLastProcessedBlock() {
|
|
34
|
-
return this.lastBlock;
|
|
35
|
-
}
|
|
36
|
-
getSyncSpeed() {
|
|
37
|
-
return this.blockProgress.speed();
|
|
38
|
-
}
|
|
39
|
-
getSyncEtaSeconds() {
|
|
40
|
-
return this.blockProgress.eta();
|
|
41
|
-
}
|
|
42
|
-
getSyncRatio() {
|
|
43
|
-
return this.blockProgress.ratio();
|
|
44
|
-
}
|
|
45
|
-
getMappingSpeed() {
|
|
46
|
-
return this.mappingSpeed.speed();
|
|
47
|
-
}
|
|
48
|
-
getMappingItemSpeed() {
|
|
49
|
-
return this.mappingItemSpeed.speed();
|
|
50
|
-
}
|
|
51
|
-
getStatusLine() {
|
|
52
|
-
return `${this.lastBlock} / ${this.chainHeight}, ` +
|
|
53
|
-
`rate: ${Math.round(this.getSyncSpeed())} blocks/sec, ` +
|
|
54
|
-
`mapping: ${Math.round(this.getMappingSpeed())} blocks/sec, ` +
|
|
55
|
-
`${Math.round(this.getMappingItemSpeed())} items/sec, ` +
|
|
56
|
-
`eta: ${(0, util_1.timeInterval)(this.getSyncEtaSeconds())}`;
|
|
57
|
-
}
|
|
58
|
-
install() {
|
|
59
|
-
new prom_client_1.Gauge({
|
|
60
|
-
name: 'sqd_processor_chain_height',
|
|
61
|
-
help: 'Chain height of the data source',
|
|
62
|
-
collect: collect(() => this.getChainHeight())
|
|
63
|
-
});
|
|
64
|
-
new prom_client_1.Gauge({
|
|
65
|
-
name: 'sqd_processor_last_block',
|
|
66
|
-
help: 'Last processed block',
|
|
67
|
-
collect: collect(() => this.getLastProcessedBlock())
|
|
68
|
-
});
|
|
69
|
-
new prom_client_1.Gauge({
|
|
70
|
-
name: 'sqd_processor_mapping_blocks_per_second',
|
|
71
|
-
help: 'Mapping performance',
|
|
72
|
-
collect: collect(() => this.getMappingSpeed())
|
|
73
|
-
});
|
|
74
|
-
new prom_client_1.Gauge({
|
|
75
|
-
name: 'sqd_processor_sync_eta_seconds',
|
|
76
|
-
help: 'Estimated time until all required blocks will be processed or until the chain height will be reached',
|
|
77
|
-
collect: collect(() => this.getSyncEtaSeconds())
|
|
78
|
-
});
|
|
79
|
-
new prom_client_1.Gauge({
|
|
80
|
-
name: 'sqd_processor_sync_ratio',
|
|
81
|
-
help: 'Percentage of processed blocks',
|
|
82
|
-
collect: collect(() => this.getSyncRatio())
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
exports.Metrics = Metrics;
|
|
87
|
-
function collect(fn) {
|
|
88
|
-
return function () {
|
|
89
|
-
this.set(fn());
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
//# sourceMappingURL=metrics.js.map
|
package/lib/metrics.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":";;;AAAA,6EAAgE;AAChE,6CAAiC;AACjC,iCAAmC;AAGnC,MAAa,OAAO;IAApB;QACY,gBAAW,GAAG,CAAC,CAAC,CAAA;QAChB,cAAS,GAAG,CAAC,CAAC,CAAA;QACd,iBAAY,GAAG,IAAI,8BAAK,CAAC,EAAC,UAAU,EAAE,CAAC,EAAC,CAAC,CAAA;QACzC,qBAAgB,GAAG,IAAI,8BAAK,CAAC,EAAC,UAAU,EAAE,CAAC,EAAC,CAAC,CAAA;QAC7C,kBAAa,GAAG,IAAI,iCAAQ,CAAC,EAAC,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAC,CAAC,CAAA;IA6F3E,CAAC;IA3FG,cAAc,CAAC,MAAc;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACvD,CAAC;IAED,qBAAqB,CAAC,MAAc;QAChC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAA;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACjE,CAAC;IAED,cAAc,CAAC,SAAiB,EAAE,IAAY,EAAE,IAAa;QACzD,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,SAAS,GAAG,IAAI,CAAC,CAAA;QACnD,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACvD,CAAC;IAED,aAAa,CACT,SAAiB,EACjB,aAAqB,EACrB,qBAA6B,EAC7B,mBAA2B;QAE3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,CAAA;QAC7E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,CAAA;IAC9F,CAAC;IAED,cAAc;QACV,OAAO,IAAI,CAAC,WAAW,CAAA;IAC3B,CAAC;IAED,qBAAqB;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;IACzB,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,iBAAiB;QACb,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAA;IACnC,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,eAAe;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;IACpC,CAAC;IAED,mBAAmB;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAA;IACxC,CAAC;IAED,aAAa;QACT,OAAO,GAAG,IAAI,CAAC,SAAS,MAAM,IAAI,CAAC,WAAW,IAAI;YAClD,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,eAAe;YACvD,YAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe;YAC7D,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,cAAc;YACvD,QAAQ,IAAA,mBAAY,EAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpD,CAAC;IAED,OAAO;QACH,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,4BAA4B;YAClC,IAAI,EAAE,iCAAiC;YACvC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;SAChD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,sBAAsB;YAC5B,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;SACvD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;SACjD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,gCAAgC;YACtC,IAAI,EAAE,sGAAsG;YAC5G,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;SACnD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,gCAAgC;YACtC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9C,CAAC,CAAA;IACN,CAAC;CACJ;AAlGD,0BAkGC;AAGD,SAAS,OAAO,CAAC,EAAgB;IAC7B,OAAO;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAA;IAClB,CAAC,CAAA;AACL,CAAC"}
|
package/src/metrics.ts
DELETED
|
@@ -1,111 +0,0 @@
|
|
|
1
|
-
import {Progress, Speed} from '@subsquid/util-internal-counters'
|
|
2
|
-
import {Gauge} from 'prom-client'
|
|
3
|
-
import {timeInterval} from './util'
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
export class Metrics {
|
|
7
|
-
private chainHeight = -1
|
|
8
|
-
private lastBlock = -1
|
|
9
|
-
private mappingSpeed = new Speed({windowSize: 5})
|
|
10
|
-
private mappingItemSpeed = new Speed({windowSize: 5})
|
|
11
|
-
private blockProgress = new Progress({initialValue: 0, windowSize: 20})
|
|
12
|
-
|
|
13
|
-
setChainHeight(height: number): void {
|
|
14
|
-
this.chainHeight = Math.max(height, this.lastBlock)
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
setLastProcessedBlock(height: number): void {
|
|
18
|
-
this.lastBlock = height
|
|
19
|
-
this.chainHeight = Math.max(this.chainHeight, this.lastBlock)
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
updateProgress(processed: number, left: number, time?: bigint): void {
|
|
23
|
-
this.blockProgress.setTargetValue(processed + left)
|
|
24
|
-
this.blockProgress.setCurrentValue(processed, time)
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
registerBatch(
|
|
28
|
-
batchSize: number,
|
|
29
|
-
batchItemSize: number,
|
|
30
|
-
batchMappingStartTime: bigint,
|
|
31
|
-
batchMappingEndTime: bigint,
|
|
32
|
-
): void {
|
|
33
|
-
this.mappingSpeed.push(batchSize, batchMappingStartTime, batchMappingEndTime)
|
|
34
|
-
this.mappingItemSpeed.push(batchItemSize || 1, batchMappingStartTime, batchMappingEndTime)
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
getChainHeight(): number {
|
|
38
|
-
return this.chainHeight
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
getLastProcessedBlock(): number {
|
|
42
|
-
return this.lastBlock
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
getSyncSpeed(): number {
|
|
46
|
-
return this.blockProgress.speed()
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
getSyncEtaSeconds(): number {
|
|
50
|
-
return this.blockProgress.eta()
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
getSyncRatio(): number {
|
|
54
|
-
return this.blockProgress.ratio()
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
getMappingSpeed(): number {
|
|
58
|
-
return this.mappingSpeed.speed()
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
getMappingItemSpeed(): number {
|
|
62
|
-
return this.mappingItemSpeed.speed()
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
getStatusLine(): string {
|
|
66
|
-
return `${this.lastBlock} / ${this.chainHeight}, ` +
|
|
67
|
-
`rate: ${Math.round(this.getSyncSpeed())} blocks/sec, ` +
|
|
68
|
-
`mapping: ${Math.round(this.getMappingSpeed())} blocks/sec, ` +
|
|
69
|
-
`${Math.round(this.getMappingItemSpeed())} items/sec, ` +
|
|
70
|
-
`eta: ${timeInterval(this.getSyncEtaSeconds())}`
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
install(): void {
|
|
74
|
-
new Gauge({
|
|
75
|
-
name: 'sqd_processor_chain_height',
|
|
76
|
-
help: 'Chain height of the data source',
|
|
77
|
-
collect: collect(() => this.getChainHeight())
|
|
78
|
-
})
|
|
79
|
-
|
|
80
|
-
new Gauge({
|
|
81
|
-
name: 'sqd_processor_last_block',
|
|
82
|
-
help: 'Last processed block',
|
|
83
|
-
collect: collect(() => this.getLastProcessedBlock())
|
|
84
|
-
})
|
|
85
|
-
|
|
86
|
-
new Gauge({
|
|
87
|
-
name: 'sqd_processor_mapping_blocks_per_second',
|
|
88
|
-
help: 'Mapping performance',
|
|
89
|
-
collect: collect(() => this.getMappingSpeed())
|
|
90
|
-
})
|
|
91
|
-
|
|
92
|
-
new Gauge({
|
|
93
|
-
name: 'sqd_processor_sync_eta_seconds',
|
|
94
|
-
help: 'Estimated time until all required blocks will be processed or until the chain height will be reached',
|
|
95
|
-
collect: collect(() => this.getSyncEtaSeconds())
|
|
96
|
-
})
|
|
97
|
-
|
|
98
|
-
new Gauge({
|
|
99
|
-
name: 'sqd_processor_sync_ratio',
|
|
100
|
-
help: 'Percentage of processed blocks',
|
|
101
|
-
collect: collect(() => this.getSyncRatio())
|
|
102
|
-
})
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
function collect(fn: () => number) {
|
|
108
|
-
return function(this: Gauge<string>) {
|
|
109
|
-
this.set(fn())
|
|
110
|
-
}
|
|
111
|
-
}
|