@subsquid/batch-processor 0.1.0-portal-api.d0738f → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/database.d.ts +6 -4
- package/lib/database.d.ts.map +1 -1
- package/lib/datasource.d.ts +8 -0
- package/lib/datasource.d.ts.map +1 -0
- package/lib/datasource.js +3 -0
- package/lib/datasource.js.map +1 -0
- package/lib/index.d.ts +1 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -0
- package/lib/index.js.map +1 -1
- package/lib/run.d.ts +10 -5
- package/lib/run.d.ts.map +1 -1
- package/lib/run.js +62 -190
- package/lib/run.js.map +1 -1
- package/lib/util.d.ts +2 -2
- package/lib/util.d.ts.map +1 -1
- package/lib/util.js +1 -1
- package/lib/util.js.map +1 -1
- package/package.json +5 -7
- package/src/database.ts +7 -6
- package/src/datasource.ts +9 -0
- package/src/index.ts +1 -0
- package/src/run.ts +92 -204
- package/src/util.ts +2 -3
- package/lib/metrics.d.ts +0 -21
- package/lib/metrics.d.ts.map +0 -1
- package/lib/metrics.js +0 -92
- package/lib/metrics.js.map +0 -1
- package/src/metrics.ts +0 -111
package/lib/database.d.ts
CHANGED
|
@@ -10,11 +10,9 @@ export interface FinalTxInfo {
|
|
|
10
10
|
}
|
|
11
11
|
export interface FinalDatabase<S> {
|
|
12
12
|
supportsHotBlocks?: false;
|
|
13
|
-
connect(): Promise<
|
|
13
|
+
connect(): Promise<HashAndHeight>;
|
|
14
14
|
transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>;
|
|
15
15
|
}
|
|
16
|
-
export interface FinalDatabaseState extends HashAndHeight {
|
|
17
|
-
}
|
|
18
16
|
export interface HotTxInfo {
|
|
19
17
|
finalizedHead: HashAndHeight;
|
|
20
18
|
baseHead: HashAndHeight;
|
|
@@ -24,7 +22,11 @@ export interface HotDatabase<S> {
|
|
|
24
22
|
supportsHotBlocks: true;
|
|
25
23
|
connect(): Promise<HotDatabaseState>;
|
|
26
24
|
transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>;
|
|
27
|
-
|
|
25
|
+
/**
|
|
26
|
+
* @deprecated
|
|
27
|
+
*/
|
|
28
|
+
transactHot(info: HotTxInfo, cb: (store: S, block: HashAndHeight) => Promise<void>): Promise<void>;
|
|
29
|
+
transactHot2?(info: HotTxInfo, cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>): Promise<void>;
|
|
28
30
|
}
|
|
29
31
|
export interface HotDatabaseState extends HashAndHeight {
|
|
30
32
|
top: HashAndHeight[];
|
package/lib/database.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;AAG3D,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;IACvB,OAAO,EAAE,OAAO,CAAA;CACnB;AAGD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC5B,iBAAiB,CAAC,EAAE,KAAK,CAAA;IACzB,OAAO,IAAI,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../src/database.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,MAAM,QAAQ,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;AAG3D,MAAM,WAAW,WAAW;IACxB,QAAQ,EAAE,aAAa,CAAA;IACvB,QAAQ,EAAE,aAAa,CAAA;IACvB,OAAO,EAAE,OAAO,CAAA;CACnB;AAGD,MAAM,WAAW,aAAa,CAAC,CAAC;IAC5B,iBAAiB,CAAC,EAAE,KAAK,CAAA;IACzB,OAAO,IAAI,OAAO,CAAC,aAAa,CAAC,CAAA;IACjC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9E;AAGD,MAAM,WAAW,SAAS;IACtB,aAAa,EAAE,aAAa,CAAA;IAC5B,QAAQ,EAAE,aAAa,CAAA;IACvB,SAAS,EAAE,aAAa,EAAE,CAAA;CAC7B;AAGD,MAAM,WAAW,WAAW,CAAC,CAAC;IAC1B,iBAAiB,EAAE,IAAI,CAAA;IACvB,OAAO,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAA;IACpC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC3E;;OAEG;IACH,WAAW,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAElG,YAAY,CAAC,CACT,IAAI,EAAE,SAAS,EACf,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAChF,OAAO,CAAC,IAAI,CAAC,CAAA;CACnB;AAGD,MAAM,WAAW,gBAAiB,SAAQ,aAAa;IACnD,GAAG,EAAE,aAAa,EAAE,CAAA;CACvB;AAGD,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACf"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { FiniteRange } from '@subsquid/util-internal-range';
|
|
2
|
+
export interface DataSource<B> {
|
|
3
|
+
getFinalizedHeight(): Promise<number>;
|
|
4
|
+
getBlockHash(height: number): Promise<string | undefined>;
|
|
5
|
+
getBlockStream(fromBlock?: number): AsyncIterable<B[]>;
|
|
6
|
+
getBlocksCountInRange?(range: FiniteRange): number;
|
|
7
|
+
}
|
|
8
|
+
//# sourceMappingURL=datasource.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"datasource.d.ts","sourceRoot":"","sources":["../src/datasource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAA;AAG9D,MAAM,WAAW,UAAU,CAAC,CAAC;IACzB,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IACrC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;IACzD,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,CAAC,EAAE,CAAC,CAAA;IACtD,qBAAqB,CAAC,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAA;CACrD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"datasource.js","sourceRoot":"","sources":["../src/datasource.ts"],"names":[],"mappings":""}
|
package/lib/index.d.ts
CHANGED
package/lib/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,OAAO,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAA;AAC1B,cAAc,cAAc,CAAA;AAC5B,cAAc,OAAO,CAAA"}
|
package/lib/index.js
CHANGED
|
@@ -15,5 +15,6 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
__exportStar(require("./database"), exports);
|
|
18
|
+
__exportStar(require("./datasource"), exports);
|
|
18
19
|
__exportStar(require("./run"), exports);
|
|
19
20
|
//# sourceMappingURL=index.js.map
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA0B;AAC1B,wCAAqB"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA0B;AAC1B,+CAA4B;AAC5B,wCAAqB"}
|
package/lib/run.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { PrometheusServer } from '@subsquid/util-internal-processor-tools';
|
|
2
|
+
import { Database, HashAndHeight } from './database';
|
|
3
|
+
import { DataSource } from './datasource';
|
|
4
|
+
export { PrometheusServer };
|
|
3
5
|
export interface DataHandlerContext<Block, Store> {
|
|
4
6
|
/**
|
|
5
7
|
* Storage interface provided by the database
|
|
@@ -14,8 +16,11 @@ export interface DataHandlerContext<Block, Store> {
|
|
|
14
16
|
*/
|
|
15
17
|
isHead: boolean;
|
|
16
18
|
}
|
|
17
|
-
|
|
18
|
-
header:
|
|
19
|
+
interface BlockBase {
|
|
20
|
+
header: HashAndHeight;
|
|
21
|
+
}
|
|
22
|
+
interface RunOptions {
|
|
23
|
+
prometheus?: PrometheusServer;
|
|
19
24
|
}
|
|
20
25
|
/**
|
|
21
26
|
* Run data processing.
|
|
@@ -31,5 +36,5 @@ export interface BlockBase {
|
|
|
31
36
|
*
|
|
32
37
|
* @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
|
|
33
38
|
*/
|
|
34
|
-
export declare function run<Block extends BlockBase, Store>(src: DataSource<Block>, db: Database<Store>, dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void
|
|
39
|
+
export declare function run<Block extends BlockBase, Store>(src: DataSource<Block>, db: Database<Store>, dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>, opts?: RunOptions): void;
|
|
35
40
|
//# sourceMappingURL=run.d.ts.map
|
package/lib/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,gBAAgB,EAAgB,MAAM,yCAAyC,CAAA;AACvF,OAAO,EAAC,QAAQ,EAAE,aAAa,EAAC,MAAM,YAAY,CAAA;AAClD,OAAO,EAAC,UAAU,EAAC,MAAM,cAAc,CAAA;AAGvC,OAAO,EAAC,gBAAgB,EAAC,CAAA;AAMzB,MAAM,WAAW,kBAAkB,CAAC,KAAK,EAAE,KAAK;IAC5C;;OAEG;IACH,KAAK,EAAE,KAAK,CAAA;IACZ;;OAEG;IACH,MAAM,EAAE,KAAK,EAAE,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,OAAO,CAAA;CAClB;AAGD,UAAU,SAAS;IACf,MAAM,EAAE,aAAa,CAAA;CACxB;AAED,UAAU,UAAU;IAChB,UAAU,CAAC,EAAE,gBAAgB,CAAA;CAChC;AAGD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,GAAG,CAAC,KAAK,SAAS,SAAS,EAAE,KAAK,EAC9C,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,EACtB,EAAE,EAAE,QAAQ,CAAC,KAAK,CAAC,EACnB,WAAW,EAAE,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,EACrE,IAAI,CAAC,EAAE,UAAU,GAClB,IAAI,CAMN"}
|
package/lib/run.js
CHANGED
|
@@ -1,40 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PrometheusServer = void 0;
|
|
29
4
|
exports.run = run;
|
|
30
5
|
const logger_1 = require("@subsquid/logger");
|
|
31
6
|
const util_internal_1 = require("@subsquid/util-internal");
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
const metrics_1 = require("./metrics");
|
|
35
|
-
const util_internal_data_source_1 = require("@subsquid/util-internal-data-source");
|
|
7
|
+
const util_internal_processor_tools_1 = require("@subsquid/util-internal-processor-tools");
|
|
8
|
+
Object.defineProperty(exports, "PrometheusServer", { enumerable: true, get: function () { return util_internal_processor_tools_1.PrometheusServer; } });
|
|
36
9
|
const util_1 = require("./util");
|
|
37
|
-
const assert_1 = __importDefault(require("assert"));
|
|
38
10
|
const log = (0, logger_1.createLogger)('sqd:batch-processor');
|
|
39
11
|
/**
|
|
40
12
|
* Run data processing.
|
|
@@ -50,158 +22,91 @@ const log = (0, logger_1.createLogger)('sqd:batch-processor');
|
|
|
50
22
|
*
|
|
51
23
|
* @param dataHandler - The data handler, see {@link DataHandlerContext} for an API available to the handler.
|
|
52
24
|
*/
|
|
53
|
-
function run(src, db, dataHandler) {
|
|
25
|
+
function run(src, db, dataHandler, opts) {
|
|
54
26
|
(0, util_internal_1.runProgram)(() => {
|
|
55
|
-
return new Processor(src, db, dataHandler).run();
|
|
56
|
-
},
|
|
27
|
+
return new Processor(src, db, dataHandler, opts).run();
|
|
28
|
+
}, err => {
|
|
57
29
|
log.fatal(err);
|
|
58
30
|
});
|
|
59
31
|
}
|
|
60
32
|
class Processor {
|
|
61
|
-
constructor(src, db, handler) {
|
|
33
|
+
constructor(src, db, handler, opts) {
|
|
62
34
|
this.src = src;
|
|
63
35
|
this.db = db;
|
|
64
36
|
this.handler = handler;
|
|
65
|
-
this.
|
|
37
|
+
this.opts = opts;
|
|
66
38
|
this.hasStatusNews = false;
|
|
39
|
+
this.metrics = new util_internal_processor_tools_1.RunnerMetrics(src.getBlocksCountInRange?.bind(src) ?? ((range) => Math.max(0, range.to - range.from + 1)));
|
|
40
|
+
this.chainHeight = new util_internal_1.Throttler(() => this.src.getFinalizedHeight(), 30000);
|
|
67
41
|
}
|
|
68
42
|
async run() {
|
|
69
|
-
let
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
finalizedHead: undefined,
|
|
73
|
-
unfinalizedHeads: [],
|
|
74
|
-
};
|
|
75
|
-
if (this.db.supportsHotBlocks) {
|
|
76
|
-
let dbState = await this.db.connect();
|
|
77
|
-
state.finalizedHead = dbState.height < 0 ? undefined : toBlockRef(dbState);
|
|
78
|
-
state.unfinalizedHeads = dbState.top.map(toBlockRef);
|
|
79
|
-
}
|
|
80
|
-
else {
|
|
81
|
-
let dbState = await this.db.connect();
|
|
82
|
-
state.finalizedHead = dbState.height < 0 ? undefined : toBlockRef(dbState);
|
|
43
|
+
let state = await this.db.connect();
|
|
44
|
+
if (state.height >= 0) {
|
|
45
|
+
log.info(`last processed final block was ${state.height}`);
|
|
83
46
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
while (true) {
|
|
90
|
-
let getStream = this.db.supportsHotBlocks
|
|
91
|
-
? this.src.getStream.bind(this.src)
|
|
92
|
-
: this.src.getFinalizedStream.bind(this.src);
|
|
93
|
-
head = getStateHead(state);
|
|
94
|
-
try {
|
|
95
|
-
for await (let data of getStream({ after: head })) {
|
|
96
|
-
state = await this.processBatch(state, data, await chainHeight.get());
|
|
97
|
-
}
|
|
98
|
-
break; // Stream completed successfully, exit loop
|
|
99
|
-
}
|
|
100
|
-
catch (e) {
|
|
101
|
-
if (!(0, util_internal_data_source_1.isForkException)(e) || !this.db.supportsHotBlocks)
|
|
102
|
-
throw e;
|
|
103
|
-
// Handle fork and continue loop to retry
|
|
104
|
-
let chain = state.finalizedHead
|
|
105
|
-
? [state.finalizedHead, ...state.unfinalizedHeads]
|
|
106
|
-
: state.unfinalizedHeads;
|
|
107
|
-
let rollbackIndex = findRollbackIndex(chain, e.previousBlocks);
|
|
108
|
-
if (rollbackIndex === -1) {
|
|
109
|
-
if (state.finalizedHead != null)
|
|
110
|
-
throw new Error('Unable to process fork');
|
|
111
|
-
state.unfinalizedHeads = [];
|
|
112
|
-
}
|
|
113
|
-
else {
|
|
114
|
-
const rollbackHead = chain[rollbackIndex];
|
|
115
|
-
log.info(`navigating a fork on a common base ${(0, util_1.formatHead)(rollbackHead)}`);
|
|
116
|
-
state.unfinalizedHeads = chain.slice(1, rollbackIndex + 1);
|
|
117
|
-
}
|
|
47
|
+
await this.assertWeAreOnTheSameChain(state);
|
|
48
|
+
await this.initMetrics(state);
|
|
49
|
+
for await (let blocks of this.src.getBlockStream(state.height + 1)) {
|
|
50
|
+
if (blocks.length > 0) {
|
|
51
|
+
state = await this.processBatch(state, blocks);
|
|
118
52
|
}
|
|
119
53
|
}
|
|
120
54
|
this.reportFinalStatus();
|
|
121
55
|
}
|
|
122
|
-
async
|
|
123
|
-
|
|
56
|
+
async assertWeAreOnTheSameChain(state) {
|
|
57
|
+
if (state.height < 0)
|
|
58
|
+
return;
|
|
59
|
+
let hash = await this.src.getBlockHash(state.height);
|
|
60
|
+
if (state.hash === hash)
|
|
61
|
+
return;
|
|
62
|
+
throw new Error(`already indexed block ${(0, util_1.formatHead)(state)} was not found on chain`);
|
|
63
|
+
}
|
|
64
|
+
async initMetrics(state) {
|
|
65
|
+
this.updateProgressMetrics(await this.chainHeight.get(), state);
|
|
124
66
|
let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT;
|
|
125
|
-
|
|
67
|
+
let prometheusServer;
|
|
68
|
+
if (this.opts?.prometheus != null) {
|
|
69
|
+
prometheusServer = this.opts.prometheus;
|
|
70
|
+
}
|
|
71
|
+
else if (port != null) {
|
|
72
|
+
prometheusServer = new util_internal_processor_tools_1.PrometheusServer();
|
|
73
|
+
prometheusServer.setPort(port);
|
|
74
|
+
}
|
|
75
|
+
if (prometheusServer == null)
|
|
126
76
|
return;
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
log.info(`prometheus metrics are served on port ${server.port}`);
|
|
77
|
+
prometheusServer.addRunnerMetrics(this.metrics);
|
|
78
|
+
let listening = await prometheusServer.serve();
|
|
79
|
+
log.info(`prometheus metrics are served on port ${listening.port}`);
|
|
131
80
|
}
|
|
132
|
-
updateProgressMetrics(chainHeight,
|
|
81
|
+
updateProgressMetrics(chainHeight, state, time) {
|
|
133
82
|
this.metrics.setChainHeight(chainHeight);
|
|
134
|
-
this.metrics.setLastProcessedBlock(
|
|
135
|
-
|
|
136
|
-
let processed = this.metrics.getLastProcessedBlock();
|
|
137
|
-
this.metrics.updateProgress(processed, left, time);
|
|
83
|
+
this.metrics.setLastProcessedBlock(state.height);
|
|
84
|
+
this.metrics.updateProgress(time);
|
|
138
85
|
}
|
|
139
|
-
async processBatch(
|
|
140
|
-
let
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
throw new Error('Data is not continuous');
|
|
147
|
-
}
|
|
148
|
-
let unfinalizedIndex = 0;
|
|
149
|
-
if (finalizedHeadData != null) {
|
|
150
|
-
unfinalizedIndex = blocks.findIndex((b) => b.header.number > finalizedHeadData.number);
|
|
151
|
-
}
|
|
152
|
-
let nextHead = (0, util_internal_1.last)(blocks).header;
|
|
153
|
-
let isOnTop = nextHead.number >= chainHeight;
|
|
86
|
+
async processBatch(prevHead, blocks) {
|
|
87
|
+
let chainHeight = await this.chainHeight.get();
|
|
88
|
+
let nextHead = {
|
|
89
|
+
hash: (0, util_internal_1.last)(blocks).header.hash,
|
|
90
|
+
height: (0, util_internal_1.last)(blocks).header.height
|
|
91
|
+
};
|
|
92
|
+
let isOnTop = nextHead.height >= chainHeight;
|
|
154
93
|
let mappingStartTime = process.hrtime.bigint();
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
isOnTop
|
|
164
|
-
}, (store) => {
|
|
165
|
-
return this.handler({
|
|
166
|
-
store,
|
|
167
|
-
blocks,
|
|
168
|
-
isHead: isOnTop,
|
|
169
|
-
});
|
|
170
|
-
});
|
|
171
|
-
}
|
|
172
|
-
else {
|
|
173
|
-
(0, assert_1.default)(this.db.supportsHotBlocks);
|
|
174
|
-
let finalizedRef = blocks[unfinalizedIndex - 1]?.header;
|
|
175
|
-
if (finalizedHeadData?.hash && finalizedHeadData?.number != null) {
|
|
176
|
-
finalizedRef = { hash: finalizedHeadData.hash, number: finalizedHeadData.number };
|
|
177
|
-
}
|
|
178
|
-
state.finalizedHead = finalizedRef ?? state.finalizedHead;
|
|
179
|
-
// Finalize all hot heads that are older than the cold head
|
|
180
|
-
if (state.finalizedHead) {
|
|
181
|
-
let finalizeIndex = state.unfinalizedHeads.findIndex((h) => h.number > state.finalizedHead.number);
|
|
182
|
-
state.unfinalizedHeads = finalizeIndex < 0 ? [] : state.unfinalizedHeads.slice(finalizeIndex);
|
|
183
|
-
}
|
|
184
|
-
// Process unfinalized blocks
|
|
185
|
-
for (let i = unfinalizedIndex; i < blocks.length; i++) {
|
|
186
|
-
state.unfinalizedHeads.push({ number: blocks[i].header.number, hash: blocks[i].header.hash });
|
|
187
|
-
}
|
|
188
|
-
await this.db.transactHot2({
|
|
189
|
-
finalizedHead: toHashAndHeight(finalizedHeadData ?? state.finalizedHead),
|
|
190
|
-
baseHead: toHashAndHeight(prevHead ?? state.finalizedHead),
|
|
191
|
-
newBlocks: blocks.map((b) => toHashAndHeight(b.header)),
|
|
192
|
-
}, (store, start, end) => {
|
|
193
|
-
return this.handler({
|
|
194
|
-
store,
|
|
195
|
-
blocks: blocks.slice(start, end),
|
|
196
|
-
isHead: isOnTop,
|
|
197
|
-
});
|
|
94
|
+
await this.db.transact({
|
|
95
|
+
prevHead,
|
|
96
|
+
nextHead,
|
|
97
|
+
isOnTop
|
|
98
|
+
}, store => {
|
|
99
|
+
return this.handler({
|
|
100
|
+
store,
|
|
101
|
+
blocks,
|
|
102
|
+
isHead: isOnTop
|
|
198
103
|
});
|
|
199
|
-
}
|
|
104
|
+
});
|
|
200
105
|
let mappingEndTime = process.hrtime.bigint();
|
|
201
|
-
this.updateProgressMetrics(chainHeight, nextHead
|
|
106
|
+
this.updateProgressMetrics(chainHeight, nextHead, mappingEndTime);
|
|
202
107
|
this.metrics.registerBatch(blocks.length, (0, util_1.getItemsCount)(blocks), mappingStartTime, mappingEndTime);
|
|
203
108
|
this.reportStatus();
|
|
204
|
-
return
|
|
109
|
+
return nextHead;
|
|
205
110
|
}
|
|
206
111
|
reportStatus() {
|
|
207
112
|
if (this.statusReportTimer == null) {
|
|
@@ -228,37 +133,4 @@ class Processor {
|
|
|
228
133
|
}
|
|
229
134
|
}
|
|
230
135
|
}
|
|
231
|
-
function findRollbackIndex(currentChain, forkChain) {
|
|
232
|
-
let currentIndex = 0;
|
|
233
|
-
let forkIndex = 0;
|
|
234
|
-
let lastCommonIndex = -1;
|
|
235
|
-
while (currentIndex < currentChain.length && forkIndex < forkChain.length) {
|
|
236
|
-
const currentBlock = currentChain[currentIndex];
|
|
237
|
-
const forkBlock = forkChain[forkIndex];
|
|
238
|
-
if (currentBlock.number > forkBlock.number) {
|
|
239
|
-
forkIndex++;
|
|
240
|
-
continue;
|
|
241
|
-
}
|
|
242
|
-
if (currentBlock.number < forkBlock.number) {
|
|
243
|
-
currentIndex++;
|
|
244
|
-
continue;
|
|
245
|
-
}
|
|
246
|
-
if (currentBlock.hash !== forkBlock.hash) {
|
|
247
|
-
return lastCommonIndex;
|
|
248
|
-
}
|
|
249
|
-
lastCommonIndex = currentIndex;
|
|
250
|
-
currentIndex++;
|
|
251
|
-
forkIndex++;
|
|
252
|
-
}
|
|
253
|
-
return lastCommonIndex;
|
|
254
|
-
}
|
|
255
|
-
function toHashAndHeight(ref) {
|
|
256
|
-
return { height: ref.number, hash: ref.hash };
|
|
257
|
-
}
|
|
258
|
-
function toBlockRef(hashAndHeight) {
|
|
259
|
-
return { number: hashAndHeight.height, hash: hashAndHeight.hash };
|
|
260
|
-
}
|
|
261
|
-
function getStateHead(state) {
|
|
262
|
-
return (0, util_internal_1.maybeLast)(state.unfinalizedHeads) ?? state.finalizedHead;
|
|
263
|
-
}
|
|
264
136
|
//# sourceMappingURL=run.js.map
|
package/lib/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":";;;AAoDA,kBAWC;AA/DD,6CAA6C;AAC7C,2DAAmE;AACnE,2FAAuF;AAK/E,iGALA,gDAAgB,OAKA;AAFxB,iCAAgD;AAKhD,MAAM,GAAG,GAAG,IAAA,qBAAY,EAAC,qBAAqB,CAAC,CAAA;AA4B/C;;;;;;;;;;;;;GAaG;AACH,SAAgB,GAAG,CACf,GAAsB,EACtB,EAAmB,EACnB,WAAqE,EACrE,IAAiB;IAEjB,IAAA,0BAAU,EAAC,GAAG,EAAE;QACR,OAAO,IAAI,SAAS,CAAC,GAAG,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAA;IAC9D,CAAC,EAAE,GAAG,CAAC,EAAE;QACD,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC,CAAC,CAAA;AACN,CAAC;AAGD,MAAM,SAAS;IAMX,YACY,GAAkB,EAClB,EAAe,EACf,OAAyD,EAChD,IAAiB;QAH1B,QAAG,GAAH,GAAG,CAAe;QAClB,OAAE,GAAF,EAAE,CAAa;QACf,YAAO,GAAP,OAAO,CAAkD;QAChD,SAAI,GAAJ,IAAI,CAAa;QAN9B,kBAAa,GAAG,KAAK,CAAA;QAQzB,IAAI,CAAC,OAAO,GAAG,IAAI,6CAAa,CAC5B,GAAG,CAAC,qBAAqB,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAC9F,CAAA;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,yBAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,EAAE,EAAE,KAAM,CAAC,CAAA;IACjF,CAAC;IAED,KAAK,CAAC,GAAG;QACL,IAAI,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAA;QACnC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,kCAAkC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,IAAI,CAAC,yBAAyB,CAAC,KAAK,CAAC,CAAA;QAC3C,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAE7B,IAAI,KAAK,EAAE,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;YACjE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YAClD,CAAC;QACL,CAAC;QAED,IAAI,CAAC,iBAAiB,EAAE,CAAA;IAC5B,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAoB;QACxD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAM;QAC5B,IAAI,IAAI,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpD,IAAI,KAAK,CAAC,IAAI,KAAK,IAAI;YAAE,OAAM;QAC/B,MAAM,IAAI,KAAK,CACX,yBAAyB,IAAA,iBAAU,EAAC,KAAK,CAAC,yBAAyB,CACtE,CAAA;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,KAAoB;QAC1C,IAAI,CAAC,qBAAqB,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;QAC/D,IAAI,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAA;QAE/E,IAAI,gBAA8C,CAAA;QAClD,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,IAAI,IAAI,EAAE,CAAC;YAChC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAA;QAC3C,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACtB,gBAAgB,GAAG,IAAI,gDAAgB,EAAE,CAAA;YACzC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAClC,CAAC;QACD,IAAI,gBAAgB,IAAI,IAAI;YAAE,OAAM;QAEpC,gBAAgB,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/C,IAAI,SAAS,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,CAAA;QAC9C,GAAG,CAAC,IAAI,CAAC,yCAAyC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IACvE,CAAC;IAEO,qBAAqB,CAAC,WAAmB,EAAE,KAAoB,EAAE,IAAa;QAClF,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAChD,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAuB,EAAE,MAAW;QAC3D,IAAI,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,CAAA;QAE9C,IAAI,QAAQ,GAAG;YACX,IAAI,EAAE,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI;YAC9B,MAAM,EAAE,IAAA,oBAAI,EAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM;SACrC,CAAA;QAED,IAAI,OAAO,GAAG,QAAQ,CAAC,MAAM,IAAI,WAAW,CAAA;QAE5C,IAAI,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9C,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;YACf,QAAQ;YACR,QAAQ;YACZ,OAAO;SACV,EAAE,KAAK,CAAC,EAAE;YACH,OAAO,IAAI,CAAC,OAAO,CAAC;gBAChB,KAAK;gBACL,MAAM;gBACV,MAAM,EAAE,OAAO;aACd,CAAC,CAAA;QACV,CAAC,CAAC,CAAA;QAEF,IAAI,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QAE5C,IAAI,CAAC,qBAAqB,CAAC,WAAW,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAA;QACjE,IAAI,CAAC,OAAO,CAAC,aAAa,CACtB,MAAM,CAAC,MAAM,EACb,IAAA,oBAAa,EAAC,MAAM,CAAC,EACrB,gBAAgB,EAChB,cAAc,CACjB,CAAA;QAED,IAAI,CAAC,YAAY,EAAE,CAAA;QAEnB,OAAO,QAAQ,CAAA;IACnB,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;YACtC,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC,GAAG,EAAE;gBACrC,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAA;gBAClC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;oBAC1B,IAAI,CAAC,YAAY,EAAE,CAAA;gBACvB,CAAC;YACL,CAAC,EAAE,IAAI,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAA;QAC7B,CAAC;IACL,CAAC;IAEO,iBAAiB;QACrB,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACjC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QACxC,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,GAAG,KAAK,CAAA;YAC1B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAA;QAC1C,CAAC;IACL,CAAC;CACJ"}
|
package/lib/util.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { HashAndHeight } from './database';
|
|
2
2
|
export declare function timeInterval(seconds: number): string;
|
|
3
3
|
export declare function getItemsCount(blocks: any[]): number;
|
|
4
|
-
export declare function formatHead(head:
|
|
4
|
+
export declare function formatHead(head: HashAndHeight): string;
|
|
5
5
|
export declare function shortHash(hash: string): string;
|
|
6
6
|
//# sourceMappingURL=util.d.ts.map
|
package/lib/util.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,YAAY,CAAA;AAGxC,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAWpD;AAGD,wBAAgB,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAWnD;AAGD,wBAAgB,UAAU,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAEtD;AAGD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAM9C"}
|
package/lib/util.js
CHANGED
package/lib/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;AAGA,oCAWC;AAGD,sCAWC;AAGD,gCAEC;AAGD,8BAMC;AAvCD,SAAgB,YAAY,CAAC,OAAe;IACxC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,GAAG,CAAA;IACpC,CAAC;IACD,IAAI,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,GAAC,EAAE,CAAC,CAAA;IACnC,IAAI,OAAO,GAAG,EAAE,EAAE,CAAC;QACf,OAAQ,OAAO,GAAC,GAAG,CAAA;IACvB,CAAC;IACD,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAA;IACpC,OAAO,GAAG,OAAO,GAAG,KAAK,GAAG,EAAE,CAAA;IAC9B,OAAO,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,GAAG,CAAA;AACvC,CAAC;AAGD,SAAgB,aAAa,CAAC,MAAa;IACvC,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACvB,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;YACpB,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAA;YACpB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,IAAI,GAAG,CAAC,MAAM,CAAA;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAA;AAChB,CAAC;AAGD,SAAgB,UAAU,CAAC,IAAmB;IAC1C,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAA;AACnD,CAAC;AAGD,SAAgB,SAAS,CAAC,IAAY;IAClC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;SAAM,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC3B,CAAC;AACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@subsquid/batch-processor",
|
|
3
|
-
"version": "0.1.0
|
|
3
|
+
"version": "0.1.0",
|
|
4
4
|
"description": "ETL processor",
|
|
5
5
|
"license": "GPL-3.0-or-later",
|
|
6
|
-
"repository": "git@github.com:subsquid/squid.git",
|
|
6
|
+
"repository": "git@github.com:subsquid/squid-sdk.git",
|
|
7
7
|
"publishConfig": {
|
|
8
8
|
"access": "public",
|
|
9
9
|
"registry": "https://registry.npmjs.org/"
|
|
@@ -14,13 +14,11 @@
|
|
|
14
14
|
],
|
|
15
15
|
"main": "lib/index.js",
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@subsquid/logger": "^1.
|
|
17
|
+
"@subsquid/logger": "^1.5.0",
|
|
18
18
|
"@subsquid/util-internal": "^3.2.0",
|
|
19
|
+
"@subsquid/util-internal-processor-tools": "^4.4.0",
|
|
19
20
|
"@subsquid/util-internal-counters": "^1.3.2",
|
|
20
|
-
"@subsquid/util-internal-
|
|
21
|
-
"@subsquid/util-internal-range": "^0.3.0",
|
|
22
|
-
"prom-client": "^14.2.0",
|
|
23
|
-
"@subsquid/util-internal-data-source": "0.0.1-portal-api.d0738f"
|
|
21
|
+
"@subsquid/util-internal-range": "^0.3.0"
|
|
24
22
|
},
|
|
25
23
|
"devDependencies": {
|
|
26
24
|
"@types/node": "^18.18.14",
|
package/src/database.ts
CHANGED
|
@@ -14,14 +14,11 @@ export interface FinalTxInfo {
|
|
|
14
14
|
|
|
15
15
|
export interface FinalDatabase<S> {
|
|
16
16
|
supportsHotBlocks?: false
|
|
17
|
-
connect(): Promise<
|
|
17
|
+
connect(): Promise<HashAndHeight>
|
|
18
18
|
transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
export interface FinalDatabaseState extends HashAndHeight {}
|
|
23
|
-
|
|
24
|
-
|
|
25
22
|
export interface HotTxInfo {
|
|
26
23
|
finalizedHead: HashAndHeight
|
|
27
24
|
baseHead: HashAndHeight
|
|
@@ -33,8 +30,12 @@ export interface HotDatabase<S> {
|
|
|
33
30
|
supportsHotBlocks: true
|
|
34
31
|
connect(): Promise<HotDatabaseState>
|
|
35
32
|
transact(info: FinalTxInfo, cb: (store: S) => Promise<void>): Promise<void>
|
|
33
|
+
/**
|
|
34
|
+
* @deprecated
|
|
35
|
+
*/
|
|
36
|
+
transactHot(info: HotTxInfo, cb: (store: S, block: HashAndHeight) => Promise<void>): Promise<void>
|
|
36
37
|
|
|
37
|
-
transactHot2(
|
|
38
|
+
transactHot2?(
|
|
38
39
|
info: HotTxInfo,
|
|
39
40
|
cb: (store: S, blockSliceStart: number, blockSliceEnd: number) => Promise<void>
|
|
40
41
|
): Promise<void>
|
|
@@ -49,4 +50,4 @@ export interface HotDatabaseState extends HashAndHeight {
|
|
|
49
50
|
export interface HashAndHeight {
|
|
50
51
|
height: number
|
|
51
52
|
hash: string
|
|
52
|
-
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type {FiniteRange} from '@subsquid/util-internal-range'
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
export interface DataSource<B> {
|
|
5
|
+
getFinalizedHeight(): Promise<number>
|
|
6
|
+
getBlockHash(height: number): Promise<string | undefined>
|
|
7
|
+
getBlockStream(fromBlock?: number): AsyncIterable<B[]>
|
|
8
|
+
getBlocksCountInRange?(range: FiniteRange): number
|
|
9
|
+
}
|
package/src/index.ts
CHANGED
package/src/run.ts
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import {createLogger} from '@subsquid/logger'
|
|
2
|
-
import {last,
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import {
|
|
6
|
-
import {Metrics} from './metrics'
|
|
7
|
-
import {DataSource, isForkException, BlockRef, type BlockBatch} from '@subsquid/util-internal-data-source'
|
|
2
|
+
import {last, runProgram, Throttler} from '@subsquid/util-internal'
|
|
3
|
+
import {PrometheusServer, RunnerMetrics} from '@subsquid/util-internal-processor-tools'
|
|
4
|
+
import {Database, HashAndHeight} from './database'
|
|
5
|
+
import {DataSource} from './datasource'
|
|
8
6
|
import {formatHead, getItemsCount} from './util'
|
|
9
|
-
|
|
7
|
+
|
|
8
|
+
export {PrometheusServer}
|
|
9
|
+
|
|
10
10
|
|
|
11
11
|
const log = createLogger('sqd:batch-processor')
|
|
12
12
|
|
|
13
|
+
|
|
13
14
|
export interface DataHandlerContext<Block, Store> {
|
|
14
15
|
/**
|
|
15
16
|
* Storage interface provided by the database
|
|
@@ -25,15 +26,16 @@ export interface DataHandlerContext<Block, Store> {
|
|
|
25
26
|
isHead: boolean
|
|
26
27
|
}
|
|
27
28
|
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
|
|
30
|
+
interface BlockBase {
|
|
31
|
+
header: HashAndHeight
|
|
30
32
|
}
|
|
31
33
|
|
|
32
|
-
interface
|
|
33
|
-
|
|
34
|
-
unfinalizedHeads: BlockRef[]
|
|
34
|
+
interface RunOptions {
|
|
35
|
+
prometheus?: PrometheusServer
|
|
35
36
|
}
|
|
36
37
|
|
|
38
|
+
|
|
37
39
|
/**
|
|
38
40
|
* Run data processing.
|
|
39
41
|
*
|
|
@@ -51,194 +53,123 @@ interface ProcessorState {
|
|
|
51
53
|
export function run<Block extends BlockBase, Store>(
|
|
52
54
|
src: DataSource<Block>,
|
|
53
55
|
db: Database<Store>,
|
|
54
|
-
dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void
|
|
56
|
+
dataHandler: (ctx: DataHandlerContext<Block, Store>) => Promise<void>,
|
|
57
|
+
opts?: RunOptions
|
|
55
58
|
): void {
|
|
56
|
-
runProgram(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
},
|
|
60
|
-
(err) => {
|
|
59
|
+
runProgram(() => {
|
|
60
|
+
return new Processor(src, db, dataHandler, opts).run()
|
|
61
|
+
}, err => {
|
|
61
62
|
log.fatal(err)
|
|
62
|
-
|
|
63
|
-
)
|
|
63
|
+
})
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
|
|
66
67
|
class Processor<B extends BlockBase, S> {
|
|
67
|
-
private metrics
|
|
68
|
+
private metrics: RunnerMetrics
|
|
69
|
+
private chainHeight: Throttler<number>
|
|
68
70
|
private statusReportTimer?: any
|
|
69
71
|
private hasStatusNews = false
|
|
70
72
|
|
|
71
73
|
constructor(
|
|
72
74
|
private src: DataSource<B>,
|
|
73
75
|
private db: Database<S>,
|
|
74
|
-
private handler: (ctx: DataHandlerContext<B, S>) => Promise<void
|
|
75
|
-
|
|
76
|
+
private handler: (ctx: DataHandlerContext<B, S>) => Promise<void>,
|
|
77
|
+
private readonly opts?: RunOptions
|
|
78
|
+
) {
|
|
79
|
+
this.metrics = new RunnerMetrics(
|
|
80
|
+
src.getBlocksCountInRange?.bind(src) ?? ((range) => Math.max(0, range.to - range.from + 1)),
|
|
81
|
+
)
|
|
82
|
+
this.chainHeight = new Throttler(() => this.src.getFinalizedHeight(), 30_000)
|
|
83
|
+
}
|
|
76
84
|
|
|
77
85
|
async run(): Promise<void> {
|
|
78
|
-
let
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
let state: ProcessorState = {
|
|
82
|
-
finalizedHead: undefined,
|
|
83
|
-
unfinalizedHeads: [],
|
|
84
|
-
}
|
|
85
|
-
if (this.db.supportsHotBlocks) {
|
|
86
|
-
let dbState = await this.db.connect()
|
|
87
|
-
state.finalizedHead = dbState.height < 0 ? undefined : toBlockRef(dbState)
|
|
88
|
-
state.unfinalizedHeads = dbState.top.map(toBlockRef)
|
|
89
|
-
} else {
|
|
90
|
-
let dbState = await this.db.connect()
|
|
91
|
-
state.finalizedHead = dbState.height < 0 ? undefined : toBlockRef(dbState)
|
|
86
|
+
let state = await this.db.connect()
|
|
87
|
+
if (state.height >= 0) {
|
|
88
|
+
log.info(`last processed final block was ${state.height}`)
|
|
92
89
|
}
|
|
93
90
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
log.info(`last processed block was ${head.number}`)
|
|
97
|
-
}
|
|
98
|
-
await this.initMetrics(head?.number ?? -1, await chainHeight.get())
|
|
99
|
-
|
|
100
|
-
while (true) {
|
|
101
|
-
let getStream = this.db.supportsHotBlocks
|
|
102
|
-
? this.src.getStream.bind(this.src)
|
|
103
|
-
: this.src.getFinalizedStream.bind(this.src)
|
|
91
|
+
await this.assertWeAreOnTheSameChain(state)
|
|
92
|
+
await this.initMetrics(state)
|
|
104
93
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
state = await this.processBatch(state, data, await chainHeight.get())
|
|
109
|
-
}
|
|
110
|
-
break // Stream completed successfully, exit loop
|
|
111
|
-
} catch (e) {
|
|
112
|
-
if (!isForkException(e) || !this.db.supportsHotBlocks) throw e
|
|
113
|
-
|
|
114
|
-
// Handle fork and continue loop to retry
|
|
115
|
-
let chain = state.finalizedHead
|
|
116
|
-
? [state.finalizedHead, ...state.unfinalizedHeads]
|
|
117
|
-
: state.unfinalizedHeads
|
|
118
|
-
let rollbackIndex = findRollbackIndex(chain, e.previousBlocks)
|
|
119
|
-
if (rollbackIndex === -1) {
|
|
120
|
-
if (state.finalizedHead != null) throw new Error('Unable to process fork')
|
|
121
|
-
state.unfinalizedHeads = []
|
|
122
|
-
} else {
|
|
123
|
-
const rollbackHead = chain[rollbackIndex]
|
|
124
|
-
log.info(`navigating a fork on a common base ${formatHead(rollbackHead)}`)
|
|
125
|
-
|
|
126
|
-
state.unfinalizedHeads = chain.slice(1, rollbackIndex + 1)
|
|
127
|
-
}
|
|
94
|
+
for await (let blocks of this.src.getBlockStream(state.height + 1)) {
|
|
95
|
+
if (blocks.length > 0) {
|
|
96
|
+
state = await this.processBatch(state, blocks)
|
|
128
97
|
}
|
|
129
98
|
}
|
|
130
99
|
|
|
131
100
|
this.reportFinalStatus()
|
|
132
101
|
}
|
|
133
102
|
|
|
134
|
-
private async
|
|
135
|
-
|
|
136
|
-
let
|
|
137
|
-
if (
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
log.info(`prometheus metrics are served on port ${server.port}`)
|
|
103
|
+
private async assertWeAreOnTheSameChain(state: HashAndHeight): Promise<void> {
|
|
104
|
+
if (state.height < 0) return
|
|
105
|
+
let hash = await this.src.getBlockHash(state.height)
|
|
106
|
+
if (state.hash === hash) return
|
|
107
|
+
throw new Error(
|
|
108
|
+
`already indexed block ${formatHead(state)} was not found on chain`
|
|
109
|
+
)
|
|
142
110
|
}
|
|
143
111
|
|
|
144
|
-
private
|
|
145
|
-
this.
|
|
146
|
-
|
|
147
|
-
let left = this.metrics.getChainHeight() - this.metrics.getLastProcessedBlock()
|
|
148
|
-
let processed = this.metrics.getLastProcessedBlock()
|
|
149
|
-
this.metrics.updateProgress(processed, left, time)
|
|
150
|
-
}
|
|
112
|
+
private async initMetrics(state: HashAndHeight): Promise<void> {
|
|
113
|
+
this.updateProgressMetrics(await this.chainHeight.get(), state)
|
|
114
|
+
let port = process.env.PROCESSOR_PROMETHEUS_PORT || process.env.PROMETHEUS_PORT
|
|
151
115
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
116
|
+
let prometheusServer: PrometheusServer | undefined
|
|
117
|
+
if (this.opts?.prometheus != null) {
|
|
118
|
+
prometheusServer = this.opts.prometheus
|
|
119
|
+
} else if (port != null) {
|
|
120
|
+
prometheusServer = new PrometheusServer()
|
|
121
|
+
prometheusServer.setPort(port)
|
|
122
|
+
}
|
|
123
|
+
if (prometheusServer == null) return
|
|
158
124
|
|
|
159
|
-
|
|
125
|
+
prometheusServer.addRunnerMetrics(this.metrics)
|
|
126
|
+
let listening = await prometheusServer.serve()
|
|
127
|
+
log.info(`prometheus metrics are served on port ${listening.port}`)
|
|
128
|
+
}
|
|
160
129
|
|
|
161
|
-
|
|
130
|
+
private updateProgressMetrics(chainHeight: number, state: HashAndHeight, time?: bigint): void {
|
|
131
|
+
this.metrics.setChainHeight(chainHeight)
|
|
132
|
+
this.metrics.setLastProcessedBlock(state.height)
|
|
133
|
+
this.metrics.updateProgress(time)
|
|
134
|
+
}
|
|
162
135
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
throw new Error('Data is not continuous')
|
|
166
|
-
}
|
|
136
|
+
private async processBatch(prevHead: HashAndHeight, blocks: B[]): Promise<HashAndHeight> {
|
|
137
|
+
let chainHeight = await this.chainHeight.get()
|
|
167
138
|
|
|
168
|
-
let
|
|
169
|
-
|
|
170
|
-
|
|
139
|
+
let nextHead = {
|
|
140
|
+
hash: last(blocks).header.hash,
|
|
141
|
+
height: last(blocks).header.height
|
|
171
142
|
}
|
|
172
143
|
|
|
173
|
-
let
|
|
174
|
-
let isOnTop = nextHead.number >= chainHeight
|
|
144
|
+
let isOnTop = nextHead.height >= chainHeight
|
|
175
145
|
|
|
176
146
|
let mappingStartTime = process.hrtime.bigint()
|
|
177
147
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
},
|
|
190
|
-
(store) => {
|
|
191
|
-
return this.handler({
|
|
192
|
-
store,
|
|
193
|
-
blocks,
|
|
194
|
-
isHead: isOnTop,
|
|
195
|
-
})
|
|
196
|
-
}
|
|
197
|
-
)
|
|
198
|
-
} else {
|
|
199
|
-
assert(this.db.supportsHotBlocks)
|
|
200
|
-
|
|
201
|
-
let finalizedRef: BlockRef | undefined = blocks[unfinalizedIndex - 1]?.header
|
|
202
|
-
if (finalizedHeadData?.hash && finalizedHeadData?.number != null) {
|
|
203
|
-
finalizedRef = {hash: finalizedHeadData.hash, number: finalizedHeadData.number}
|
|
204
|
-
}
|
|
205
|
-
state.finalizedHead = finalizedRef ?? state.finalizedHead
|
|
206
|
-
|
|
207
|
-
// Finalize all hot heads that are older than the cold head
|
|
208
|
-
if (state.finalizedHead) {
|
|
209
|
-
let finalizeIndex = state.unfinalizedHeads.findIndex((h) => h.number > state.finalizedHead!.number)
|
|
210
|
-
state.unfinalizedHeads = finalizeIndex < 0 ? [] : state.unfinalizedHeads.slice(finalizeIndex)
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
// Process unfinalized blocks
|
|
214
|
-
for (let i = unfinalizedIndex; i < blocks.length; i++) {
|
|
215
|
-
state.unfinalizedHeads.push({number: blocks[i].header.number, hash: blocks[i].header.hash})
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
await this.db.transactHot2(
|
|
219
|
-
{
|
|
220
|
-
finalizedHead: toHashAndHeight(finalizedHeadData ?? state.finalizedHead),
|
|
221
|
-
baseHead: toHashAndHeight(prevHead ?? state.finalizedHead),
|
|
222
|
-
newBlocks: blocks.map((b) => toHashAndHeight(b.header)),
|
|
223
|
-
},
|
|
224
|
-
(store, start, end) => {
|
|
225
|
-
return this.handler({
|
|
226
|
-
store,
|
|
227
|
-
blocks: blocks.slice(start, end),
|
|
228
|
-
isHead: isOnTop,
|
|
229
|
-
})
|
|
230
|
-
}
|
|
231
|
-
)
|
|
232
|
-
}
|
|
148
|
+
await this.db.transact({
|
|
149
|
+
prevHead,
|
|
150
|
+
nextHead,
|
|
151
|
+
isOnTop
|
|
152
|
+
}, store => {
|
|
153
|
+
return this.handler({
|
|
154
|
+
store,
|
|
155
|
+
blocks,
|
|
156
|
+
isHead: isOnTop
|
|
157
|
+
})
|
|
158
|
+
})
|
|
233
159
|
|
|
234
160
|
let mappingEndTime = process.hrtime.bigint()
|
|
235
161
|
|
|
236
|
-
this.updateProgressMetrics(chainHeight, nextHead
|
|
237
|
-
this.metrics.registerBatch(
|
|
162
|
+
this.updateProgressMetrics(chainHeight, nextHead, mappingEndTime)
|
|
163
|
+
this.metrics.registerBatch(
|
|
164
|
+
blocks.length,
|
|
165
|
+
getItemsCount(blocks),
|
|
166
|
+
mappingStartTime,
|
|
167
|
+
mappingEndTime
|
|
168
|
+
)
|
|
238
169
|
|
|
239
170
|
this.reportStatus()
|
|
240
171
|
|
|
241
|
-
return
|
|
172
|
+
return nextHead
|
|
242
173
|
}
|
|
243
174
|
|
|
244
175
|
private reportStatus(): void {
|
|
@@ -250,7 +181,7 @@ class Processor<B extends BlockBase, S> {
|
|
|
250
181
|
this.hasStatusNews = false
|
|
251
182
|
this.reportStatus()
|
|
252
183
|
}
|
|
253
|
-
},
|
|
184
|
+
}, 5000)
|
|
254
185
|
} else {
|
|
255
186
|
this.hasStatusNews = true
|
|
256
187
|
}
|
|
@@ -266,46 +197,3 @@ class Processor<B extends BlockBase, S> {
|
|
|
266
197
|
}
|
|
267
198
|
}
|
|
268
199
|
}
|
|
269
|
-
|
|
270
|
-
function findRollbackIndex(currentChain: BlockRef[], forkChain: BlockRef[]): number {
|
|
271
|
-
let currentIndex = 0
|
|
272
|
-
let forkIndex = 0
|
|
273
|
-
let lastCommonIndex = -1
|
|
274
|
-
|
|
275
|
-
while (currentIndex < currentChain.length && forkIndex < forkChain.length) {
|
|
276
|
-
const currentBlock = currentChain[currentIndex]
|
|
277
|
-
const forkBlock = forkChain[forkIndex]
|
|
278
|
-
|
|
279
|
-
if (currentBlock.number > forkBlock.number) {
|
|
280
|
-
forkIndex++
|
|
281
|
-
continue
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
if (currentBlock.number < forkBlock.number) {
|
|
285
|
-
currentIndex++
|
|
286
|
-
continue
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
if (currentBlock.hash !== forkBlock.hash) {
|
|
290
|
-
return lastCommonIndex
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
lastCommonIndex = currentIndex
|
|
294
|
-
currentIndex++
|
|
295
|
-
forkIndex++
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
return lastCommonIndex
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
function toHashAndHeight(ref: BlockRef): HashAndHeight {
|
|
302
|
-
return {height: ref.number, hash: ref.hash}
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
function toBlockRef(hashAndHeight: HashAndHeight): BlockRef {
|
|
306
|
-
return {number: hashAndHeight.height, hash: hashAndHeight.hash}
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
function getStateHead(state: ProcessorState): BlockRef | undefined {
|
|
310
|
-
return maybeLast(state.unfinalizedHeads) ?? state.finalizedHead
|
|
311
|
-
}
|
package/src/util.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type {BlockRef} from '@subsquid/util-internal-data-source'
|
|
2
1
|
import {HashAndHeight} from './database'
|
|
3
2
|
|
|
4
3
|
|
|
@@ -30,8 +29,8 @@ export function getItemsCount(blocks: any[]): number {
|
|
|
30
29
|
}
|
|
31
30
|
|
|
32
31
|
|
|
33
|
-
export function formatHead(head:
|
|
34
|
-
return `${head.
|
|
32
|
+
export function formatHead(head: HashAndHeight): string {
|
|
33
|
+
return `${head.height}#${shortHash(head.hash)}`
|
|
35
34
|
}
|
|
36
35
|
|
|
37
36
|
|
package/lib/metrics.d.ts
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
export declare class Metrics {
|
|
2
|
-
private chainHeight;
|
|
3
|
-
private lastBlock;
|
|
4
|
-
private mappingSpeed;
|
|
5
|
-
private mappingItemSpeed;
|
|
6
|
-
private blockProgress;
|
|
7
|
-
setChainHeight(height: number): void;
|
|
8
|
-
setLastProcessedBlock(height: number): void;
|
|
9
|
-
updateProgress(processed: number, left: number, time?: bigint): void;
|
|
10
|
-
registerBatch(batchSize: number, batchItemSize: number, batchMappingStartTime: bigint, batchMappingEndTime: bigint): void;
|
|
11
|
-
getChainHeight(): number;
|
|
12
|
-
getLastProcessedBlock(): number;
|
|
13
|
-
getSyncSpeed(): number;
|
|
14
|
-
getSyncEtaSeconds(): number;
|
|
15
|
-
getSyncRatio(): number;
|
|
16
|
-
getMappingSpeed(): number;
|
|
17
|
-
getMappingItemSpeed(): number;
|
|
18
|
-
getStatusLine(): string;
|
|
19
|
-
install(): void;
|
|
20
|
-
}
|
|
21
|
-
//# sourceMappingURL=metrics.d.ts.map
|
package/lib/metrics.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAKA,qBAAa,OAAO;IAChB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,aAAa,CAAkD;IAEvE,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIpC,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAK3C,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI;IAKpE,aAAa,CACT,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,qBAAqB,EAAE,MAAM,EAC7B,mBAAmB,EAAE,MAAM,GAC5B,IAAI;IAKP,cAAc,IAAI,MAAM;IAIxB,qBAAqB,IAAI,MAAM;IAI/B,YAAY,IAAI,MAAM;IAItB,iBAAiB,IAAI,MAAM;IAI3B,YAAY,IAAI,MAAM;IAItB,eAAe,IAAI,MAAM;IAIzB,mBAAmB,IAAI,MAAM;IAI7B,aAAa,IAAI,MAAM;IAQvB,OAAO,IAAI,IAAI;CA+BlB"}
|
package/lib/metrics.js
DELETED
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Metrics = void 0;
|
|
4
|
-
const util_internal_counters_1 = require("@subsquid/util-internal-counters");
|
|
5
|
-
const prom_client_1 = require("prom-client");
|
|
6
|
-
const util_1 = require("./util");
|
|
7
|
-
class Metrics {
|
|
8
|
-
constructor() {
|
|
9
|
-
this.chainHeight = -1;
|
|
10
|
-
this.lastBlock = -1;
|
|
11
|
-
this.mappingSpeed = new util_internal_counters_1.Speed({ windowSize: 5 });
|
|
12
|
-
this.mappingItemSpeed = new util_internal_counters_1.Speed({ windowSize: 5 });
|
|
13
|
-
this.blockProgress = new util_internal_counters_1.Progress({ initialValue: 0, windowSize: 20 });
|
|
14
|
-
}
|
|
15
|
-
setChainHeight(height) {
|
|
16
|
-
this.chainHeight = Math.max(height, this.lastBlock);
|
|
17
|
-
}
|
|
18
|
-
setLastProcessedBlock(height) {
|
|
19
|
-
this.lastBlock = height;
|
|
20
|
-
this.chainHeight = Math.max(this.chainHeight, this.lastBlock);
|
|
21
|
-
}
|
|
22
|
-
updateProgress(processed, left, time) {
|
|
23
|
-
this.blockProgress.setTargetValue(processed + left);
|
|
24
|
-
this.blockProgress.setCurrentValue(processed, time);
|
|
25
|
-
}
|
|
26
|
-
registerBatch(batchSize, batchItemSize, batchMappingStartTime, batchMappingEndTime) {
|
|
27
|
-
this.mappingSpeed.push(batchSize, batchMappingStartTime, batchMappingEndTime);
|
|
28
|
-
this.mappingItemSpeed.push(batchItemSize || 1, batchMappingStartTime, batchMappingEndTime);
|
|
29
|
-
}
|
|
30
|
-
getChainHeight() {
|
|
31
|
-
return this.chainHeight;
|
|
32
|
-
}
|
|
33
|
-
getLastProcessedBlock() {
|
|
34
|
-
return this.lastBlock;
|
|
35
|
-
}
|
|
36
|
-
getSyncSpeed() {
|
|
37
|
-
return this.blockProgress.speed();
|
|
38
|
-
}
|
|
39
|
-
getSyncEtaSeconds() {
|
|
40
|
-
return this.blockProgress.eta();
|
|
41
|
-
}
|
|
42
|
-
getSyncRatio() {
|
|
43
|
-
return this.blockProgress.ratio();
|
|
44
|
-
}
|
|
45
|
-
getMappingSpeed() {
|
|
46
|
-
return this.mappingSpeed.speed();
|
|
47
|
-
}
|
|
48
|
-
getMappingItemSpeed() {
|
|
49
|
-
return this.mappingItemSpeed.speed();
|
|
50
|
-
}
|
|
51
|
-
getStatusLine() {
|
|
52
|
-
return `${this.lastBlock} / ${this.chainHeight}, ` +
|
|
53
|
-
`rate: ${Math.round(this.getSyncSpeed())} blocks/sec, ` +
|
|
54
|
-
`mapping: ${Math.round(this.getMappingSpeed())} blocks/sec, ` +
|
|
55
|
-
`${Math.round(this.getMappingItemSpeed())} items/sec, ` +
|
|
56
|
-
`eta: ${(0, util_1.timeInterval)(this.getSyncEtaSeconds())}`;
|
|
57
|
-
}
|
|
58
|
-
install() {
|
|
59
|
-
new prom_client_1.Gauge({
|
|
60
|
-
name: 'sqd_processor_chain_height',
|
|
61
|
-
help: 'Chain height of the data source',
|
|
62
|
-
collect: collect(() => this.getChainHeight())
|
|
63
|
-
});
|
|
64
|
-
new prom_client_1.Gauge({
|
|
65
|
-
name: 'sqd_processor_last_block',
|
|
66
|
-
help: 'Last processed block',
|
|
67
|
-
collect: collect(() => this.getLastProcessedBlock())
|
|
68
|
-
});
|
|
69
|
-
new prom_client_1.Gauge({
|
|
70
|
-
name: 'sqd_processor_mapping_blocks_per_second',
|
|
71
|
-
help: 'Mapping performance',
|
|
72
|
-
collect: collect(() => this.getMappingSpeed())
|
|
73
|
-
});
|
|
74
|
-
new prom_client_1.Gauge({
|
|
75
|
-
name: 'sqd_processor_sync_eta_seconds',
|
|
76
|
-
help: 'Estimated time until all required blocks will be processed or until the chain height will be reached',
|
|
77
|
-
collect: collect(() => this.getSyncEtaSeconds())
|
|
78
|
-
});
|
|
79
|
-
new prom_client_1.Gauge({
|
|
80
|
-
name: 'sqd_processor_sync_ratio',
|
|
81
|
-
help: 'Percentage of processed blocks',
|
|
82
|
-
collect: collect(() => this.getSyncRatio())
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
exports.Metrics = Metrics;
|
|
87
|
-
function collect(fn) {
|
|
88
|
-
return function () {
|
|
89
|
-
this.set(fn());
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
//# sourceMappingURL=metrics.js.map
|
package/lib/metrics.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":";;;AAAA,6EAAgE;AAChE,6CAAiC;AACjC,iCAAmC;AAGnC,MAAa,OAAO;IAApB;QACY,gBAAW,GAAG,CAAC,CAAC,CAAA;QAChB,cAAS,GAAG,CAAC,CAAC,CAAA;QACd,iBAAY,GAAG,IAAI,8BAAK,CAAC,EAAC,UAAU,EAAE,CAAC,EAAC,CAAC,CAAA;QACzC,qBAAgB,GAAG,IAAI,8BAAK,CAAC,EAAC,UAAU,EAAE,CAAC,EAAC,CAAC,CAAA;QAC7C,kBAAa,GAAG,IAAI,iCAAQ,CAAC,EAAC,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,EAAC,CAAC,CAAA;IA6F3E,CAAC;IA3FG,cAAc,CAAC,MAAc;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACvD,CAAC;IAED,qBAAqB,CAAC,MAAc;QAChC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAA;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACjE,CAAC;IAED,cAAc,CAAC,SAAiB,EAAE,IAAY,EAAE,IAAa;QACzD,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,SAAS,GAAG,IAAI,CAAC,CAAA;QACnD,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACvD,CAAC;IAED,aAAa,CACT,SAAiB,EACjB,aAAqB,EACrB,qBAA6B,EAC7B,mBAA2B;QAE3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,CAAA;QAC7E,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,qBAAqB,EAAE,mBAAmB,CAAC,CAAA;IAC9F,CAAC;IAED,cAAc;QACV,OAAO,IAAI,CAAC,WAAW,CAAA;IAC3B,CAAC;IAED,qBAAqB;QACjB,OAAO,IAAI,CAAC,SAAS,CAAA;IACzB,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,iBAAiB;QACb,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAA;IACnC,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAA;IACrC,CAAC;IAED,eAAe;QACX,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;IACpC,CAAC;IAED,mBAAmB;QACf,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAA;IACxC,CAAC;IAED,aAAa;QACT,OAAO,GAAG,IAAI,CAAC,SAAS,MAAM,IAAI,CAAC,WAAW,IAAI;YAClD,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,eAAe;YACvD,YAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe;YAC7D,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,cAAc;YACvD,QAAQ,IAAA,mBAAY,EAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpD,CAAC;IAED,OAAO;QACH,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,4BAA4B;YAClC,IAAI,EAAE,iCAAiC;YACvC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;SAChD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,sBAAsB;YAC5B,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;SACvD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,yCAAyC;YAC/C,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;SACjD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,gCAAgC;YACtC,IAAI,EAAE,sGAAsG;YAC5G,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;SACnD,CAAC,CAAA;QAEF,IAAI,mBAAK,CAAC;YACN,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE,gCAAgC;YACtC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;SAC9C,CAAC,CAAA;IACN,CAAC;CACJ;AAlGD,0BAkGC;AAGD,SAAS,OAAO,CAAC,EAAgB;IAC7B,OAAO;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAA;IAClB,CAAC,CAAA;AACL,CAAC"}
|
package/src/metrics.ts
DELETED
|
@@ -1,111 +0,0 @@
|
|
|
1
|
-
import {Progress, Speed} from '@subsquid/util-internal-counters'
|
|
2
|
-
import {Gauge} from 'prom-client'
|
|
3
|
-
import {timeInterval} from './util'
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
export class Metrics {
|
|
7
|
-
private chainHeight = -1
|
|
8
|
-
private lastBlock = -1
|
|
9
|
-
private mappingSpeed = new Speed({windowSize: 5})
|
|
10
|
-
private mappingItemSpeed = new Speed({windowSize: 5})
|
|
11
|
-
private blockProgress = new Progress({initialValue: 0, windowSize: 20})
|
|
12
|
-
|
|
13
|
-
setChainHeight(height: number): void {
|
|
14
|
-
this.chainHeight = Math.max(height, this.lastBlock)
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
setLastProcessedBlock(height: number): void {
|
|
18
|
-
this.lastBlock = height
|
|
19
|
-
this.chainHeight = Math.max(this.chainHeight, this.lastBlock)
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
updateProgress(processed: number, left: number, time?: bigint): void {
|
|
23
|
-
this.blockProgress.setTargetValue(processed + left)
|
|
24
|
-
this.blockProgress.setCurrentValue(processed, time)
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
registerBatch(
|
|
28
|
-
batchSize: number,
|
|
29
|
-
batchItemSize: number,
|
|
30
|
-
batchMappingStartTime: bigint,
|
|
31
|
-
batchMappingEndTime: bigint,
|
|
32
|
-
): void {
|
|
33
|
-
this.mappingSpeed.push(batchSize, batchMappingStartTime, batchMappingEndTime)
|
|
34
|
-
this.mappingItemSpeed.push(batchItemSize || 1, batchMappingStartTime, batchMappingEndTime)
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
getChainHeight(): number {
|
|
38
|
-
return this.chainHeight
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
getLastProcessedBlock(): number {
|
|
42
|
-
return this.lastBlock
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
getSyncSpeed(): number {
|
|
46
|
-
return this.blockProgress.speed()
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
getSyncEtaSeconds(): number {
|
|
50
|
-
return this.blockProgress.eta()
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
getSyncRatio(): number {
|
|
54
|
-
return this.blockProgress.ratio()
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
getMappingSpeed(): number {
|
|
58
|
-
return this.mappingSpeed.speed()
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
getMappingItemSpeed(): number {
|
|
62
|
-
return this.mappingItemSpeed.speed()
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
getStatusLine(): string {
|
|
66
|
-
return `${this.lastBlock} / ${this.chainHeight}, ` +
|
|
67
|
-
`rate: ${Math.round(this.getSyncSpeed())} blocks/sec, ` +
|
|
68
|
-
`mapping: ${Math.round(this.getMappingSpeed())} blocks/sec, ` +
|
|
69
|
-
`${Math.round(this.getMappingItemSpeed())} items/sec, ` +
|
|
70
|
-
`eta: ${timeInterval(this.getSyncEtaSeconds())}`
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
install(): void {
|
|
74
|
-
new Gauge({
|
|
75
|
-
name: 'sqd_processor_chain_height',
|
|
76
|
-
help: 'Chain height of the data source',
|
|
77
|
-
collect: collect(() => this.getChainHeight())
|
|
78
|
-
})
|
|
79
|
-
|
|
80
|
-
new Gauge({
|
|
81
|
-
name: 'sqd_processor_last_block',
|
|
82
|
-
help: 'Last processed block',
|
|
83
|
-
collect: collect(() => this.getLastProcessedBlock())
|
|
84
|
-
})
|
|
85
|
-
|
|
86
|
-
new Gauge({
|
|
87
|
-
name: 'sqd_processor_mapping_blocks_per_second',
|
|
88
|
-
help: 'Mapping performance',
|
|
89
|
-
collect: collect(() => this.getMappingSpeed())
|
|
90
|
-
})
|
|
91
|
-
|
|
92
|
-
new Gauge({
|
|
93
|
-
name: 'sqd_processor_sync_eta_seconds',
|
|
94
|
-
help: 'Estimated time until all required blocks will be processed or until the chain height will be reached',
|
|
95
|
-
collect: collect(() => this.getSyncEtaSeconds())
|
|
96
|
-
})
|
|
97
|
-
|
|
98
|
-
new Gauge({
|
|
99
|
-
name: 'sqd_processor_sync_ratio',
|
|
100
|
-
help: 'Percentage of processed blocks',
|
|
101
|
-
collect: collect(() => this.getSyncRatio())
|
|
102
|
-
})
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
function collect(fn: () => number) {
|
|
108
|
-
return function(this: Gauge<string>) {
|
|
109
|
-
this.set(fn())
|
|
110
|
-
}
|
|
111
|
-
}
|