envio 3.0.0-alpha.2 → 3.0.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +164 -30
- package/bin.mjs +49 -0
- package/evm.schema.json +79 -169
- package/fuel.schema.json +50 -21
- package/index.d.ts +497 -1
- package/index.js +4 -0
- package/package.json +42 -31
- package/rescript.json +4 -1
- package/src/Batch.res +11 -8
- package/src/Batch.res.mjs +11 -9
- package/src/ChainFetcher.res +531 -0
- package/src/ChainFetcher.res.mjs +339 -0
- package/src/ChainManager.res +190 -0
- package/src/ChainManager.res.mjs +166 -0
- package/src/Change.res +3 -3
- package/src/Config.gen.ts +19 -0
- package/src/Config.res +737 -22
- package/src/Config.res.mjs +703 -26
- package/src/{Indexer.res → Ctx.res} +1 -1
- package/src/Ecosystem.res +9 -124
- package/src/Ecosystem.res.mjs +19 -160
- package/src/Env.res +30 -74
- package/src/Env.res.mjs +25 -87
- package/src/Envio.gen.ts +3 -1
- package/src/Envio.res +20 -9
- package/src/EventProcessing.res +469 -0
- package/src/EventProcessing.res.mjs +337 -0
- package/src/EvmTypes.gen.ts +6 -0
- package/src/EvmTypes.res +1 -0
- package/src/FetchState.res +1256 -639
- package/src/FetchState.res.mjs +1135 -612
- package/src/GlobalState.res +1190 -0
- package/src/GlobalState.res.mjs +1183 -0
- package/src/GlobalStateManager.res +68 -0
- package/src/GlobalStateManager.res.mjs +75 -0
- package/src/GlobalStateManager.resi +7 -0
- package/src/HandlerLoader.res +89 -0
- package/src/HandlerLoader.res.mjs +79 -0
- package/src/HandlerRegister.res +357 -0
- package/src/HandlerRegister.res.mjs +299 -0
- package/src/{EventRegister.resi → HandlerRegister.resi} +13 -13
- package/src/Hasura.res +111 -175
- package/src/Hasura.res.mjs +88 -150
- package/src/InMemoryStore.res +1 -1
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/InMemoryTable.res +1 -1
- package/src/InMemoryTable.res.mjs +1 -1
- package/src/Internal.gen.ts +4 -0
- package/src/Internal.res +230 -12
- package/src/Internal.res.mjs +115 -1
- package/src/LoadLayer.res +444 -0
- package/src/LoadLayer.res.mjs +296 -0
- package/src/LoadLayer.resi +32 -0
- package/src/LogSelection.res +33 -27
- package/src/LogSelection.res.mjs +6 -0
- package/src/Logging.res +21 -7
- package/src/Logging.res.mjs +16 -8
- package/src/Main.res +377 -0
- package/src/Main.res.mjs +339 -0
- package/src/Persistence.res +7 -21
- package/src/Persistence.res.mjs +3 -3
- package/src/PgStorage.gen.ts +10 -0
- package/src/PgStorage.res +116 -69
- package/src/PgStorage.res.d.mts +5 -0
- package/src/PgStorage.res.mjs +93 -50
- package/src/Prometheus.res +294 -224
- package/src/Prometheus.res.mjs +353 -340
- package/src/ReorgDetection.res +6 -10
- package/src/ReorgDetection.res.mjs +6 -6
- package/src/SafeCheckpointTracking.res +4 -4
- package/src/SafeCheckpointTracking.res.mjs +2 -2
- package/src/Sink.res +4 -2
- package/src/Sink.res.mjs +2 -1
- package/src/TableIndices.res +0 -1
- package/src/TestIndexer.res +692 -0
- package/src/TestIndexer.res.mjs +527 -0
- package/src/TestIndexerProxyStorage.res +205 -0
- package/src/TestIndexerProxyStorage.res.mjs +151 -0
- package/src/TopicFilter.res +1 -1
- package/src/Types.ts +1 -1
- package/src/UserContext.res +424 -0
- package/src/UserContext.res.mjs +279 -0
- package/src/Utils.res +97 -26
- package/src/Utils.res.mjs +91 -44
- package/src/bindings/BigInt.res +10 -0
- package/src/bindings/BigInt.res.mjs +15 -0
- package/src/bindings/ClickHouse.res +120 -23
- package/src/bindings/ClickHouse.res.mjs +118 -28
- package/src/bindings/DateFns.res +74 -0
- package/src/bindings/DateFns.res.mjs +22 -0
- package/src/bindings/EventSource.res +8 -1
- package/src/bindings/EventSource.res.mjs +8 -1
- package/src/bindings/Express.res +1 -0
- package/src/bindings/Hrtime.res +14 -1
- package/src/bindings/Hrtime.res.mjs +22 -2
- package/src/bindings/Hrtime.resi +4 -0
- package/src/bindings/Lodash.res +0 -1
- package/src/bindings/NodeJs.res +49 -3
- package/src/bindings/NodeJs.res.mjs +11 -3
- package/src/bindings/Pino.res +24 -10
- package/src/bindings/Pino.res.mjs +14 -8
- package/src/bindings/Postgres.gen.ts +8 -0
- package/src/bindings/Postgres.res +5 -1
- package/src/bindings/Postgres.res.d.mts +5 -0
- package/src/bindings/PromClient.res +0 -10
- package/src/bindings/PromClient.res.mjs +0 -3
- package/src/bindings/Vitest.res +142 -0
- package/src/bindings/Vitest.res.mjs +9 -0
- package/src/bindings/WebSocket.res +27 -0
- package/src/bindings/WebSocket.res.mjs +2 -0
- package/src/bindings/Yargs.res +8 -0
- package/src/bindings/Yargs.res.mjs +2 -0
- package/src/db/EntityHistory.res +7 -7
- package/src/db/EntityHistory.res.mjs +9 -9
- package/src/db/InternalTable.res +59 -111
- package/src/db/InternalTable.res.mjs +73 -104
- package/src/db/Table.res +27 -8
- package/src/db/Table.res.mjs +25 -14
- package/src/sources/Evm.res +84 -0
- package/src/sources/Evm.res.mjs +105 -0
- package/src/sources/EvmChain.res +94 -0
- package/src/sources/EvmChain.res.mjs +60 -0
- package/src/sources/Fuel.res +19 -34
- package/src/sources/Fuel.res.mjs +34 -16
- package/src/sources/FuelSDK.res +38 -0
- package/src/sources/FuelSDK.res.mjs +29 -0
- package/src/sources/HyperFuel.res +2 -2
- package/src/sources/HyperFuel.resi +1 -1
- package/src/sources/HyperFuelClient.res +2 -2
- package/src/sources/HyperFuelSource.res +33 -13
- package/src/sources/HyperFuelSource.res.mjs +24 -16
- package/src/sources/HyperSync.res +36 -6
- package/src/sources/HyperSync.res.mjs +9 -7
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncClient.res +1 -1
- package/src/sources/HyperSyncHeightStream.res +47 -116
- package/src/sources/HyperSyncHeightStream.res.mjs +46 -73
- package/src/sources/HyperSyncSource.res +118 -139
- package/src/sources/HyperSyncSource.res.mjs +104 -121
- package/src/sources/Rpc.res +86 -14
- package/src/sources/Rpc.res.mjs +101 -9
- package/src/sources/RpcSource.res +621 -364
- package/src/sources/RpcSource.res.mjs +843 -410
- package/src/sources/RpcWebSocketHeightStream.res +181 -0
- package/src/sources/RpcWebSocketHeightStream.res.mjs +196 -0
- package/src/sources/Source.res +7 -5
- package/src/sources/SourceManager.res +325 -225
- package/src/sources/SourceManager.res.mjs +314 -171
- package/src/sources/SourceManager.resi +17 -6
- package/src/sources/Svm.res +81 -0
- package/src/sources/Svm.res.mjs +90 -0
- package/src/tui/Tui.res +247 -0
- package/src/tui/Tui.res.mjs +337 -0
- package/src/tui/bindings/Ink.res +371 -0
- package/src/tui/bindings/Ink.res.mjs +72 -0
- package/src/tui/bindings/Style.res +123 -0
- package/src/tui/bindings/Style.res.mjs +2 -0
- package/src/tui/components/BufferedProgressBar.res +40 -0
- package/src/tui/components/BufferedProgressBar.res.mjs +57 -0
- package/src/tui/components/CustomHooks.res +122 -0
- package/src/tui/components/CustomHooks.res.mjs +179 -0
- package/src/tui/components/Messages.res +41 -0
- package/src/tui/components/Messages.res.mjs +75 -0
- package/src/tui/components/SyncETA.res +174 -0
- package/src/tui/components/SyncETA.res.mjs +263 -0
- package/src/tui/components/TuiData.res +47 -0
- package/src/tui/components/TuiData.res.mjs +34 -0
- package/svm.schema.json +112 -0
- package/bin.js +0 -48
- package/src/EventRegister.res +0 -241
- package/src/EventRegister.res.mjs +0 -240
- package/src/bindings/Ethers.gen.ts +0 -14
- package/src/bindings/Ethers.res +0 -204
- package/src/bindings/Ethers.res.mjs +0 -130
- /package/src/{Indexer.res.mjs → Ctx.res.mjs} +0 -0
|
@@ -0,0 +1,1183 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Env from "./Env.res.mjs";
|
|
4
|
+
import * as Batch from "./Batch.res.mjs";
|
|
5
|
+
import * as Utils from "./Utils.res.mjs";
|
|
6
|
+
import * as $$BigInt from "./bindings/BigInt.res.mjs";
|
|
7
|
+
import * as Config from "./Config.res.mjs";
|
|
8
|
+
import * as Hrtime from "./bindings/Hrtime.res.mjs";
|
|
9
|
+
import * as Js_exn from "rescript/lib/es6/js_exn.js";
|
|
10
|
+
import * as Js_null from "rescript/lib/es6/js_null.js";
|
|
11
|
+
import * as Logging from "./Logging.res.mjs";
|
|
12
|
+
import * as Process from "process";
|
|
13
|
+
import * as Caml_obj from "rescript/lib/es6/caml_obj.js";
|
|
14
|
+
import * as ChainMap from "./ChainMap.res.mjs";
|
|
15
|
+
import * as Throttler from "./Throttler.res.mjs";
|
|
16
|
+
import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
|
|
17
|
+
import * as Belt_Float from "rescript/lib/es6/belt_Float.js";
|
|
18
|
+
import * as FetchState from "./FetchState.res.mjs";
|
|
19
|
+
import * as Prometheus from "./Prometheus.res.mjs";
|
|
20
|
+
import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
|
|
21
|
+
import * as Caml_option from "rescript/lib/es6/caml_option.js";
|
|
22
|
+
import * as LoadManager from "./LoadManager.res.mjs";
|
|
23
|
+
import * as Persistence from "./Persistence.res.mjs";
|
|
24
|
+
import * as ChainFetcher from "./ChainFetcher.res.mjs";
|
|
25
|
+
import * as ChainManager from "./ChainManager.res.mjs";
|
|
26
|
+
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
27
|
+
import * as InMemoryStore from "./InMemoryStore.res.mjs";
|
|
28
|
+
import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
29
|
+
import * as ReorgDetection from "./ReorgDetection.res.mjs";
|
|
30
|
+
import * as EventProcessing from "./EventProcessing.res.mjs";
|
|
31
|
+
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
32
|
+
import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
|
|
33
|
+
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
34
|
+
|
|
35
|
+
function make() {
|
|
36
|
+
var intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
|
|
37
|
+
var logger = Logging.createChild({
|
|
38
|
+
context: "Throttler for chain metadata writes",
|
|
39
|
+
intervalMillis: intervalMillis
|
|
40
|
+
});
|
|
41
|
+
var chainMetaData = Throttler.make(intervalMillis, logger);
|
|
42
|
+
var intervalMillis$1 = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
|
|
43
|
+
var logger$1 = Logging.createChild({
|
|
44
|
+
context: "Throttler for pruning stale entity history data",
|
|
45
|
+
intervalMillis: intervalMillis$1
|
|
46
|
+
});
|
|
47
|
+
var pruneStaleEntityHistory = Throttler.make(intervalMillis$1, logger$1);
|
|
48
|
+
return {
|
|
49
|
+
chainMetaData: chainMetaData,
|
|
50
|
+
pruneStaleEntityHistory: pruneStaleEntityHistory
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
var WriteThrottlers = {
|
|
55
|
+
make: make
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
function make$1(ctx, chainManager, isDevelopmentModeOpt, shouldUseTuiOpt) {
|
|
59
|
+
var isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
|
|
60
|
+
var shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
|
|
61
|
+
return {
|
|
62
|
+
ctx: ctx,
|
|
63
|
+
chainManager: chainManager,
|
|
64
|
+
processedBatches: 0,
|
|
65
|
+
currentlyProcessingBatch: false,
|
|
66
|
+
rollbackState: "NoRollback",
|
|
67
|
+
indexerStartTime: new Date(),
|
|
68
|
+
writeThrottlers: make(),
|
|
69
|
+
loadManager: LoadManager.make(),
|
|
70
|
+
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
71
|
+
id: 0
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function getId(self) {
|
|
76
|
+
return self.id;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function setChainManager(self, chainManager) {
|
|
80
|
+
return {
|
|
81
|
+
ctx: self.ctx,
|
|
82
|
+
chainManager: chainManager,
|
|
83
|
+
processedBatches: self.processedBatches,
|
|
84
|
+
currentlyProcessingBatch: self.currentlyProcessingBatch,
|
|
85
|
+
rollbackState: self.rollbackState,
|
|
86
|
+
indexerStartTime: self.indexerStartTime,
|
|
87
|
+
writeThrottlers: self.writeThrottlers,
|
|
88
|
+
loadManager: self.loadManager,
|
|
89
|
+
keepProcessAlive: self.keepProcessAlive,
|
|
90
|
+
id: self.id
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function isPreparingRollback(state) {
|
|
95
|
+
var match = state.rollbackState;
|
|
96
|
+
if (typeof match !== "object") {
|
|
97
|
+
if (match === "NoRollback") {
|
|
98
|
+
return false;
|
|
99
|
+
} else {
|
|
100
|
+
return true;
|
|
101
|
+
}
|
|
102
|
+
} else if (match.TAG === "RollbackReady") {
|
|
103
|
+
return false;
|
|
104
|
+
} else {
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function updateChainMetadataTable(cm, persistence, throttler) {
|
|
110
|
+
var chainsData = {};
|
|
111
|
+
Belt_Array.forEach(ChainMap.values(cm.chainFetchers), (function (cf) {
|
|
112
|
+
chainsData[String(cf.chainConfig.id)] = {
|
|
113
|
+
first_event_block: Js_null.fromOption(cf.fetchState.firstEventBlock),
|
|
114
|
+
buffer_block: FetchState.bufferBlockNumber(cf.fetchState),
|
|
115
|
+
ready_at: Js_null.fromOption(cf.timestampCaughtUpToHeadOrEndblock),
|
|
116
|
+
_is_hyper_sync: SourceManager.getActiveSource(cf.sourceManager).poweredByHyperSync
|
|
117
|
+
};
|
|
118
|
+
}));
|
|
119
|
+
Throttler.schedule(throttler, (function () {
|
|
120
|
+
return persistence.storage.setChainMeta(chainsData);
|
|
121
|
+
}));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function updateProgressedChains(chainManager, batch, ctx) {
|
|
125
|
+
var nextQueueItemIsNone = ChainManager.nextItemIsNone(chainManager);
|
|
126
|
+
var allChainsAtHead = ChainManager.isProgressAtHead(chainManager);
|
|
127
|
+
var allChainsReady = {
|
|
128
|
+
contents: true
|
|
129
|
+
};
|
|
130
|
+
var chainFetchers = ChainMap.map(chainManager.chainFetchers, (function (prev) {
|
|
131
|
+
var chain = ChainMap.Chain.makeUnsafe(prev.chainConfig.id);
|
|
132
|
+
var maybeChainAfterBatch = batch.progressedChainsById[chain];
|
|
133
|
+
var cf;
|
|
134
|
+
if (maybeChainAfterBatch !== undefined) {
|
|
135
|
+
if (prev.committedProgressBlockNumber !== maybeChainAfterBatch.progressBlockNumber) {
|
|
136
|
+
Prometheus.ProgressBlockNumber.set(maybeChainAfterBatch.progressBlockNumber, chain);
|
|
137
|
+
}
|
|
138
|
+
if (prev.numEventsProcessed !== maybeChainAfterBatch.totalEventsProcessed) {
|
|
139
|
+
Prometheus.ProgressEventsCount.set(maybeChainAfterBatch.totalEventsProcessed, chain);
|
|
140
|
+
}
|
|
141
|
+
var eventItem = Batch.findLastEventItem(batch, chain);
|
|
142
|
+
if (eventItem !== undefined) {
|
|
143
|
+
var blockTimestamp = ctx.config.ecosystem.getTimestamp(eventItem.event.block);
|
|
144
|
+
var currentTimeMs = Date.now() | 0;
|
|
145
|
+
var blockTimestampMs = Math.imul(blockTimestamp, 1000);
|
|
146
|
+
var latencyMs = currentTimeMs - blockTimestampMs | 0;
|
|
147
|
+
Prometheus.ProgressLatency.set(latencyMs, chain);
|
|
148
|
+
}
|
|
149
|
+
var match = prev.fetchState.firstEventBlock;
|
|
150
|
+
var tmp;
|
|
151
|
+
if (match !== undefined) {
|
|
152
|
+
tmp = prev.fetchState;
|
|
153
|
+
} else {
|
|
154
|
+
var firstEventBlock = Batch.findFirstEventBlockNumber(batch, chain);
|
|
155
|
+
if (firstEventBlock !== undefined) {
|
|
156
|
+
var init = prev.fetchState;
|
|
157
|
+
tmp = {
|
|
158
|
+
optimizedPartitions: init.optimizedPartitions,
|
|
159
|
+
startBlock: init.startBlock,
|
|
160
|
+
endBlock: init.endBlock,
|
|
161
|
+
normalSelection: init.normalSelection,
|
|
162
|
+
indexingContracts: init.indexingContracts,
|
|
163
|
+
contractConfigs: init.contractConfigs,
|
|
164
|
+
chainId: init.chainId,
|
|
165
|
+
latestOnBlockBlockNumber: init.latestOnBlockBlockNumber,
|
|
166
|
+
blockLag: init.blockLag,
|
|
167
|
+
buffer: init.buffer,
|
|
168
|
+
targetBufferSize: init.targetBufferSize,
|
|
169
|
+
onBlockConfigs: init.onBlockConfigs,
|
|
170
|
+
knownHeight: init.knownHeight,
|
|
171
|
+
firstEventBlock: firstEventBlock
|
|
172
|
+
};
|
|
173
|
+
} else {
|
|
174
|
+
tmp = prev.fetchState;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
var safeCheckpointTracking = prev.safeCheckpointTracking;
|
|
178
|
+
cf = {
|
|
179
|
+
logger: prev.logger,
|
|
180
|
+
fetchState: tmp,
|
|
181
|
+
sourceManager: prev.sourceManager,
|
|
182
|
+
chainConfig: prev.chainConfig,
|
|
183
|
+
isProgressAtHead: prev.isProgressAtHead || maybeChainAfterBatch.isProgressAtHeadWhenBatchCreated,
|
|
184
|
+
timestampCaughtUpToHeadOrEndblock: prev.timestampCaughtUpToHeadOrEndblock,
|
|
185
|
+
committedProgressBlockNumber: maybeChainAfterBatch.progressBlockNumber,
|
|
186
|
+
numEventsProcessed: maybeChainAfterBatch.totalEventsProcessed,
|
|
187
|
+
reorgDetection: prev.reorgDetection,
|
|
188
|
+
safeCheckpointTracking: safeCheckpointTracking !== undefined ? SafeCheckpointTracking.updateOnNewBatch(safeCheckpointTracking, prev.fetchState.knownHeight, chain, batch.checkpointIds, batch.checkpointBlockNumbers, batch.checkpointChainIds) : undefined
|
|
189
|
+
};
|
|
190
|
+
} else {
|
|
191
|
+
cf = prev;
|
|
192
|
+
}
|
|
193
|
+
var cf$1;
|
|
194
|
+
if (ChainFetcher.hasProcessedToEndblock(cf)) {
|
|
195
|
+
var timestampCaughtUpToHeadOrEndblock = ChainFetcher.isReady(cf) ? cf.timestampCaughtUpToHeadOrEndblock : Caml_option.some(new Date());
|
|
196
|
+
cf$1 = {
|
|
197
|
+
logger: cf.logger,
|
|
198
|
+
fetchState: cf.fetchState,
|
|
199
|
+
sourceManager: cf.sourceManager,
|
|
200
|
+
chainConfig: cf.chainConfig,
|
|
201
|
+
isProgressAtHead: cf.isProgressAtHead,
|
|
202
|
+
timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
|
|
203
|
+
committedProgressBlockNumber: cf.committedProgressBlockNumber,
|
|
204
|
+
numEventsProcessed: cf.numEventsProcessed,
|
|
205
|
+
reorgDetection: cf.reorgDetection,
|
|
206
|
+
safeCheckpointTracking: cf.safeCheckpointTracking
|
|
207
|
+
};
|
|
208
|
+
} else if (!ChainFetcher.isReady(cf) && cf.isProgressAtHead) {
|
|
209
|
+
if (nextQueueItemIsNone && allChainsAtHead) {
|
|
210
|
+
cf$1 = {
|
|
211
|
+
logger: cf.logger,
|
|
212
|
+
fetchState: cf.fetchState,
|
|
213
|
+
sourceManager: cf.sourceManager,
|
|
214
|
+
chainConfig: cf.chainConfig,
|
|
215
|
+
isProgressAtHead: cf.isProgressAtHead,
|
|
216
|
+
timestampCaughtUpToHeadOrEndblock: Caml_option.some(new Date()),
|
|
217
|
+
committedProgressBlockNumber: cf.committedProgressBlockNumber,
|
|
218
|
+
numEventsProcessed: cf.numEventsProcessed,
|
|
219
|
+
reorgDetection: cf.reorgDetection,
|
|
220
|
+
safeCheckpointTracking: cf.safeCheckpointTracking
|
|
221
|
+
};
|
|
222
|
+
} else {
|
|
223
|
+
var hasNoMoreEventsToProcess = ChainFetcher.hasNoMoreEventsToProcess(cf);
|
|
224
|
+
cf$1 = hasNoMoreEventsToProcess ? ({
|
|
225
|
+
logger: cf.logger,
|
|
226
|
+
fetchState: cf.fetchState,
|
|
227
|
+
sourceManager: cf.sourceManager,
|
|
228
|
+
chainConfig: cf.chainConfig,
|
|
229
|
+
isProgressAtHead: cf.isProgressAtHead,
|
|
230
|
+
timestampCaughtUpToHeadOrEndblock: Caml_option.some(new Date()),
|
|
231
|
+
committedProgressBlockNumber: cf.committedProgressBlockNumber,
|
|
232
|
+
numEventsProcessed: cf.numEventsProcessed,
|
|
233
|
+
reorgDetection: cf.reorgDetection,
|
|
234
|
+
safeCheckpointTracking: cf.safeCheckpointTracking
|
|
235
|
+
}) : cf;
|
|
236
|
+
}
|
|
237
|
+
} else {
|
|
238
|
+
cf$1 = cf;
|
|
239
|
+
}
|
|
240
|
+
if (ChainFetcher.isReady(cf$1)) {
|
|
241
|
+
if (!ChainFetcher.isReady(prev)) {
|
|
242
|
+
Prometheus.ProgressReady.set(chain);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
} else {
|
|
246
|
+
allChainsReady.contents = false;
|
|
247
|
+
}
|
|
248
|
+
return cf$1;
|
|
249
|
+
}));
|
|
250
|
+
if (allChainsReady.contents) {
|
|
251
|
+
Prometheus.ProgressReady.setAllReady();
|
|
252
|
+
}
|
|
253
|
+
var checkpointId = Utils.$$Array.last(batch.checkpointIds);
|
|
254
|
+
return {
|
|
255
|
+
committedCheckpointId: checkpointId !== undefined ? checkpointId : chainManager.committedCheckpointId,
|
|
256
|
+
chainFetchers: chainFetchers,
|
|
257
|
+
multichain: chainManager.multichain,
|
|
258
|
+
isInReorgThreshold: chainManager.isInReorgThreshold
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function validatePartitionQueryResponse(state, partitionQueryResponse) {
|
|
263
|
+
var response = partitionQueryResponse.response;
|
|
264
|
+
var chain = partitionQueryResponse.chain;
|
|
265
|
+
var chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
|
|
266
|
+
var stats = response.stats;
|
|
267
|
+
var knownHeight = response.knownHeight;
|
|
268
|
+
if (knownHeight > chainFetcher.fetchState.knownHeight) {
|
|
269
|
+
Prometheus.SourceHeight.set(SourceManager.getActiveSource(chainFetcher.sourceManager).name, chainFetcher.chainConfig.id, knownHeight);
|
|
270
|
+
}
|
|
271
|
+
Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Belt_Option.getWithDefault(stats["parsing time (s)"], 0), response.parsedQueueItems.length, (response.latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
|
|
272
|
+
var match = ReorgDetection.registerReorgGuard(chainFetcher.reorgDetection, response.reorgGuard, knownHeight);
|
|
273
|
+
var reorgResult = match[1];
|
|
274
|
+
var updatedChainFetcher_logger = chainFetcher.logger;
|
|
275
|
+
var updatedChainFetcher_fetchState = chainFetcher.fetchState;
|
|
276
|
+
var updatedChainFetcher_sourceManager = chainFetcher.sourceManager;
|
|
277
|
+
var updatedChainFetcher_chainConfig = chainFetcher.chainConfig;
|
|
278
|
+
var updatedChainFetcher_isProgressAtHead = chainFetcher.isProgressAtHead;
|
|
279
|
+
var updatedChainFetcher_timestampCaughtUpToHeadOrEndblock = chainFetcher.timestampCaughtUpToHeadOrEndblock;
|
|
280
|
+
var updatedChainFetcher_committedProgressBlockNumber = chainFetcher.committedProgressBlockNumber;
|
|
281
|
+
var updatedChainFetcher_numEventsProcessed = chainFetcher.numEventsProcessed;
|
|
282
|
+
var updatedChainFetcher_reorgDetection = match[0];
|
|
283
|
+
var updatedChainFetcher_safeCheckpointTracking = chainFetcher.safeCheckpointTracking;
|
|
284
|
+
var updatedChainFetcher = {
|
|
285
|
+
logger: updatedChainFetcher_logger,
|
|
286
|
+
fetchState: updatedChainFetcher_fetchState,
|
|
287
|
+
sourceManager: updatedChainFetcher_sourceManager,
|
|
288
|
+
chainConfig: updatedChainFetcher_chainConfig,
|
|
289
|
+
isProgressAtHead: updatedChainFetcher_isProgressAtHead,
|
|
290
|
+
timestampCaughtUpToHeadOrEndblock: updatedChainFetcher_timestampCaughtUpToHeadOrEndblock,
|
|
291
|
+
committedProgressBlockNumber: updatedChainFetcher_committedProgressBlockNumber,
|
|
292
|
+
numEventsProcessed: updatedChainFetcher_numEventsProcessed,
|
|
293
|
+
reorgDetection: updatedChainFetcher_reorgDetection,
|
|
294
|
+
safeCheckpointTracking: updatedChainFetcher_safeCheckpointTracking
|
|
295
|
+
};
|
|
296
|
+
var init = state.chainManager;
|
|
297
|
+
var nextState_ctx = state.ctx;
|
|
298
|
+
var nextState_chainManager = {
|
|
299
|
+
committedCheckpointId: init.committedCheckpointId,
|
|
300
|
+
chainFetchers: ChainMap.set(state.chainManager.chainFetchers, chain, updatedChainFetcher),
|
|
301
|
+
multichain: init.multichain,
|
|
302
|
+
isInReorgThreshold: init.isInReorgThreshold
|
|
303
|
+
};
|
|
304
|
+
var nextState_processedBatches = state.processedBatches;
|
|
305
|
+
var nextState_currentlyProcessingBatch = state.currentlyProcessingBatch;
|
|
306
|
+
var nextState_rollbackState = state.rollbackState;
|
|
307
|
+
var nextState_indexerStartTime = state.indexerStartTime;
|
|
308
|
+
var nextState_writeThrottlers = state.writeThrottlers;
|
|
309
|
+
var nextState_loadManager = state.loadManager;
|
|
310
|
+
var nextState_keepProcessAlive = state.keepProcessAlive;
|
|
311
|
+
var nextState_id = state.id;
|
|
312
|
+
var nextState = {
|
|
313
|
+
ctx: nextState_ctx,
|
|
314
|
+
chainManager: nextState_chainManager,
|
|
315
|
+
processedBatches: nextState_processedBatches,
|
|
316
|
+
currentlyProcessingBatch: nextState_currentlyProcessingBatch,
|
|
317
|
+
rollbackState: nextState_rollbackState,
|
|
318
|
+
indexerStartTime: nextState_indexerStartTime,
|
|
319
|
+
writeThrottlers: nextState_writeThrottlers,
|
|
320
|
+
loadManager: nextState_loadManager,
|
|
321
|
+
keepProcessAlive: nextState_keepProcessAlive,
|
|
322
|
+
id: nextState_id
|
|
323
|
+
};
|
|
324
|
+
var rollbackWithReorgDetectedBlockNumber;
|
|
325
|
+
if (typeof reorgResult !== "object") {
|
|
326
|
+
rollbackWithReorgDetectedBlockNumber = undefined;
|
|
327
|
+
} else {
|
|
328
|
+
var reorgDetected = reorgResult._0;
|
|
329
|
+
Logging.childInfo(chainFetcher.logger, ReorgDetection.reorgDetectedToLogParams(reorgDetected, state.ctx.config.shouldRollbackOnReorg));
|
|
330
|
+
Prometheus.ReorgCount.increment(chain);
|
|
331
|
+
Prometheus.ReorgDetectionBlockNumber.set(reorgDetected.scannedBlock.blockNumber, chain);
|
|
332
|
+
rollbackWithReorgDetectedBlockNumber = state.ctx.config.shouldRollbackOnReorg ? reorgDetected.scannedBlock.blockNumber : undefined;
|
|
333
|
+
}
|
|
334
|
+
if (rollbackWithReorgDetectedBlockNumber === undefined) {
|
|
335
|
+
return [
|
|
336
|
+
nextState,
|
|
337
|
+
[{
|
|
338
|
+
TAG: "ProcessPartitionQueryResponse",
|
|
339
|
+
_0: partitionQueryResponse
|
|
340
|
+
}]
|
|
341
|
+
];
|
|
342
|
+
}
|
|
343
|
+
var match$1 = state.rollbackState;
|
|
344
|
+
var chainManager;
|
|
345
|
+
if (typeof match$1 !== "object" || match$1.TAG !== "RollbackReady") {
|
|
346
|
+
chainManager = state.chainManager;
|
|
347
|
+
} else {
|
|
348
|
+
var eventsProcessedDiffByChain = match$1.eventsProcessedDiffByChain;
|
|
349
|
+
var init$1 = state.chainManager;
|
|
350
|
+
chainManager = {
|
|
351
|
+
committedCheckpointId: init$1.committedCheckpointId,
|
|
352
|
+
chainFetchers: ChainMap.mapWithKey(state.chainManager.chainFetchers, (function (c, chainFetcher) {
|
|
353
|
+
var eventsProcessedDiff = eventsProcessedDiffByChain[c];
|
|
354
|
+
if (eventsProcessedDiff !== undefined) {
|
|
355
|
+
return {
|
|
356
|
+
logger: chainFetcher.logger,
|
|
357
|
+
fetchState: chainFetcher.fetchState,
|
|
358
|
+
sourceManager: chainFetcher.sourceManager,
|
|
359
|
+
chainConfig: chainFetcher.chainConfig,
|
|
360
|
+
isProgressAtHead: chainFetcher.isProgressAtHead,
|
|
361
|
+
timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
|
|
362
|
+
committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
|
|
363
|
+
numEventsProcessed: chainFetcher.numEventsProcessed + eventsProcessedDiff,
|
|
364
|
+
reorgDetection: chainFetcher.reorgDetection,
|
|
365
|
+
safeCheckpointTracking: chainFetcher.safeCheckpointTracking
|
|
366
|
+
};
|
|
367
|
+
} else {
|
|
368
|
+
return chainFetcher;
|
|
369
|
+
}
|
|
370
|
+
})),
|
|
371
|
+
multichain: init$1.multichain,
|
|
372
|
+
isInReorgThreshold: init$1.isInReorgThreshold
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
return [
|
|
376
|
+
{
|
|
377
|
+
ctx: nextState_ctx,
|
|
378
|
+
chainManager: {
|
|
379
|
+
committedCheckpointId: chainManager.committedCheckpointId,
|
|
380
|
+
chainFetchers: ChainMap.map(chainManager.chainFetchers, (function (chainFetcher) {
|
|
381
|
+
return {
|
|
382
|
+
logger: chainFetcher.logger,
|
|
383
|
+
fetchState: FetchState.resetPendingQueries(chainFetcher.fetchState),
|
|
384
|
+
sourceManager: chainFetcher.sourceManager,
|
|
385
|
+
chainConfig: chainFetcher.chainConfig,
|
|
386
|
+
isProgressAtHead: chainFetcher.isProgressAtHead,
|
|
387
|
+
timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
|
|
388
|
+
committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
|
|
389
|
+
numEventsProcessed: chainFetcher.numEventsProcessed,
|
|
390
|
+
reorgDetection: chainFetcher.reorgDetection,
|
|
391
|
+
safeCheckpointTracking: chainFetcher.safeCheckpointTracking
|
|
392
|
+
};
|
|
393
|
+
})),
|
|
394
|
+
multichain: chainManager.multichain,
|
|
395
|
+
isInReorgThreshold: chainManager.isInReorgThreshold
|
|
396
|
+
},
|
|
397
|
+
processedBatches: nextState_processedBatches,
|
|
398
|
+
currentlyProcessingBatch: nextState_currentlyProcessingBatch,
|
|
399
|
+
rollbackState: {
|
|
400
|
+
TAG: "ReorgDetected",
|
|
401
|
+
chain: chain,
|
|
402
|
+
blockNumber: rollbackWithReorgDetectedBlockNumber
|
|
403
|
+
},
|
|
404
|
+
indexerStartTime: nextState_indexerStartTime,
|
|
405
|
+
writeThrottlers: nextState_writeThrottlers,
|
|
406
|
+
loadManager: nextState_loadManager,
|
|
407
|
+
keepProcessAlive: nextState_keepProcessAlive,
|
|
408
|
+
id: nextState_id + 1 | 0
|
|
409
|
+
},
|
|
410
|
+
["Rollback"]
|
|
411
|
+
];
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function submitPartitionQueryResponse(state, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, chain) {
|
|
415
|
+
var chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
|
|
416
|
+
var updatedChainFetcher = ChainFetcher.handleQueryResult(chainFetcher, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight);
|
|
417
|
+
if (!chainFetcher.isProgressAtHead && updatedChainFetcher.isProgressAtHead) {
|
|
418
|
+
Logging.childInfo(updatedChainFetcher.logger, "All events have been fetched");
|
|
419
|
+
}
|
|
420
|
+
var init = state.chainManager;
|
|
421
|
+
var nextState_ctx = state.ctx;
|
|
422
|
+
var nextState_chainManager = {
|
|
423
|
+
committedCheckpointId: init.committedCheckpointId,
|
|
424
|
+
chainFetchers: ChainMap.set(state.chainManager.chainFetchers, chain, updatedChainFetcher),
|
|
425
|
+
multichain: init.multichain,
|
|
426
|
+
isInReorgThreshold: init.isInReorgThreshold
|
|
427
|
+
};
|
|
428
|
+
var nextState_processedBatches = state.processedBatches;
|
|
429
|
+
var nextState_currentlyProcessingBatch = state.currentlyProcessingBatch;
|
|
430
|
+
var nextState_rollbackState = state.rollbackState;
|
|
431
|
+
var nextState_indexerStartTime = state.indexerStartTime;
|
|
432
|
+
var nextState_writeThrottlers = state.writeThrottlers;
|
|
433
|
+
var nextState_loadManager = state.loadManager;
|
|
434
|
+
var nextState_keepProcessAlive = state.keepProcessAlive;
|
|
435
|
+
var nextState_id = state.id;
|
|
436
|
+
var nextState = {
|
|
437
|
+
ctx: nextState_ctx,
|
|
438
|
+
chainManager: nextState_chainManager,
|
|
439
|
+
processedBatches: nextState_processedBatches,
|
|
440
|
+
currentlyProcessingBatch: nextState_currentlyProcessingBatch,
|
|
441
|
+
rollbackState: nextState_rollbackState,
|
|
442
|
+
indexerStartTime: nextState_indexerStartTime,
|
|
443
|
+
writeThrottlers: nextState_writeThrottlers,
|
|
444
|
+
loadManager: nextState_loadManager,
|
|
445
|
+
keepProcessAlive: nextState_keepProcessAlive,
|
|
446
|
+
id: nextState_id
|
|
447
|
+
};
|
|
448
|
+
return [
|
|
449
|
+
nextState,
|
|
450
|
+
[
|
|
451
|
+
{
|
|
452
|
+
TAG: "UpdateChainMetaDataAndCheckForExit",
|
|
453
|
+
_0: "NoExit"
|
|
454
|
+
},
|
|
455
|
+
"ProcessEventBatch",
|
|
456
|
+
{
|
|
457
|
+
TAG: "NextQuery",
|
|
458
|
+
_0: {
|
|
459
|
+
TAG: "Chain",
|
|
460
|
+
_0: chain
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
]
|
|
464
|
+
];
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
async function processPartitionQueryResponse(state, param, dispatchAction) {
|
|
468
|
+
var response = param.response;
|
|
469
|
+
var chain = param.chain;
|
|
470
|
+
var parsedQueueItems = response.parsedQueueItems;
|
|
471
|
+
var itemsWithContractRegister = [];
|
|
472
|
+
var newItems = [];
|
|
473
|
+
for(var idx = 0 ,idx_finish = parsedQueueItems.length; idx < idx_finish; ++idx){
|
|
474
|
+
var item = parsedQueueItems[idx];
|
|
475
|
+
if (item.eventConfig.contractRegister !== undefined) {
|
|
476
|
+
itemsWithContractRegister.push(item);
|
|
477
|
+
}
|
|
478
|
+
newItems.push(item);
|
|
479
|
+
}
|
|
480
|
+
var newItemsWithDcs = itemsWithContractRegister.length !== 0 ? await ChainFetcher.runContractRegistersOrThrow(itemsWithContractRegister, chain, state.ctx.config) : itemsWithContractRegister;
|
|
481
|
+
return dispatchAction({
|
|
482
|
+
TAG: "SubmitPartitionQueryResponse",
|
|
483
|
+
newItems: newItems,
|
|
484
|
+
newItemsWithDcs: newItemsWithDcs,
|
|
485
|
+
knownHeight: response.knownHeight,
|
|
486
|
+
latestFetchedBlock: {
|
|
487
|
+
blockNumber: response.latestFetchedBlockNumber,
|
|
488
|
+
blockTimestamp: response.latestFetchedBlockTimestamp
|
|
489
|
+
},
|
|
490
|
+
query: param.query,
|
|
491
|
+
chain: chain
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
function updateChainFetcher(chainFetcherUpdate, state, chain) {
|
|
496
|
+
var init = state.chainManager;
|
|
497
|
+
return [
|
|
498
|
+
{
|
|
499
|
+
ctx: state.ctx,
|
|
500
|
+
chainManager: {
|
|
501
|
+
committedCheckpointId: init.committedCheckpointId,
|
|
502
|
+
chainFetchers: ChainMap.update(state.chainManager.chainFetchers, chain, chainFetcherUpdate),
|
|
503
|
+
multichain: init.multichain,
|
|
504
|
+
isInReorgThreshold: init.isInReorgThreshold
|
|
505
|
+
},
|
|
506
|
+
processedBatches: state.processedBatches,
|
|
507
|
+
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
508
|
+
rollbackState: state.rollbackState,
|
|
509
|
+
indexerStartTime: state.indexerStartTime,
|
|
510
|
+
writeThrottlers: state.writeThrottlers,
|
|
511
|
+
loadManager: state.loadManager,
|
|
512
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
513
|
+
id: state.id
|
|
514
|
+
},
|
|
515
|
+
[]
|
|
516
|
+
];
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function onEnterReorgThreshold(state) {
|
|
520
|
+
Logging.info("Reorg threshold reached");
|
|
521
|
+
Prometheus.ReorgThreshold.set(true);
|
|
522
|
+
var chainFetchers = ChainMap.map(state.chainManager.chainFetchers, (function (chainFetcher) {
|
|
523
|
+
return {
|
|
524
|
+
logger: chainFetcher.logger,
|
|
525
|
+
fetchState: FetchState.updateInternal(chainFetcher.fetchState, undefined, undefined, undefined, chainFetcher.chainConfig.blockLag, undefined),
|
|
526
|
+
sourceManager: chainFetcher.sourceManager,
|
|
527
|
+
chainConfig: chainFetcher.chainConfig,
|
|
528
|
+
isProgressAtHead: chainFetcher.isProgressAtHead,
|
|
529
|
+
timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
|
|
530
|
+
committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
|
|
531
|
+
numEventsProcessed: chainFetcher.numEventsProcessed,
|
|
532
|
+
reorgDetection: chainFetcher.reorgDetection,
|
|
533
|
+
safeCheckpointTracking: chainFetcher.safeCheckpointTracking
|
|
534
|
+
};
|
|
535
|
+
}));
|
|
536
|
+
var init = state.chainManager;
|
|
537
|
+
return {
|
|
538
|
+
ctx: state.ctx,
|
|
539
|
+
chainManager: {
|
|
540
|
+
committedCheckpointId: init.committedCheckpointId,
|
|
541
|
+
chainFetchers: chainFetchers,
|
|
542
|
+
multichain: init.multichain,
|
|
543
|
+
isInReorgThreshold: true
|
|
544
|
+
},
|
|
545
|
+
processedBatches: state.processedBatches,
|
|
546
|
+
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
547
|
+
rollbackState: state.rollbackState,
|
|
548
|
+
indexerStartTime: state.indexerStartTime,
|
|
549
|
+
writeThrottlers: state.writeThrottlers,
|
|
550
|
+
loadManager: state.loadManager,
|
|
551
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
552
|
+
id: state.id
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function actionReducer(state, action) {
|
|
557
|
+
if (typeof action !== "object") {
|
|
558
|
+
switch (action) {
|
|
559
|
+
case "StartProcessingBatch" :
|
|
560
|
+
return [
|
|
561
|
+
{
|
|
562
|
+
ctx: state.ctx,
|
|
563
|
+
chainManager: state.chainManager,
|
|
564
|
+
processedBatches: state.processedBatches,
|
|
565
|
+
currentlyProcessingBatch: true,
|
|
566
|
+
rollbackState: state.rollbackState,
|
|
567
|
+
indexerStartTime: state.indexerStartTime,
|
|
568
|
+
writeThrottlers: state.writeThrottlers,
|
|
569
|
+
loadManager: state.loadManager,
|
|
570
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
571
|
+
id: state.id
|
|
572
|
+
},
|
|
573
|
+
[]
|
|
574
|
+
];
|
|
575
|
+
case "StartFindingReorgDepth" :
|
|
576
|
+
return [
|
|
577
|
+
{
|
|
578
|
+
ctx: state.ctx,
|
|
579
|
+
chainManager: state.chainManager,
|
|
580
|
+
processedBatches: state.processedBatches,
|
|
581
|
+
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
582
|
+
rollbackState: "FindingReorgDepth",
|
|
583
|
+
indexerStartTime: state.indexerStartTime,
|
|
584
|
+
writeThrottlers: state.writeThrottlers,
|
|
585
|
+
loadManager: state.loadManager,
|
|
586
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
587
|
+
id: state.id
|
|
588
|
+
},
|
|
589
|
+
[]
|
|
590
|
+
];
|
|
591
|
+
case "EnterReorgThreshold" :
|
|
592
|
+
return [
|
|
593
|
+
onEnterReorgThreshold(state),
|
|
594
|
+
[{
|
|
595
|
+
TAG: "NextQuery",
|
|
596
|
+
_0: "CheckAllChains"
|
|
597
|
+
}]
|
|
598
|
+
];
|
|
599
|
+
case "SuccessExit" :
|
|
600
|
+
Logging.info("Exiting with success");
|
|
601
|
+
Process.exit(0);
|
|
602
|
+
return [
|
|
603
|
+
state,
|
|
604
|
+
[]
|
|
605
|
+
];
|
|
606
|
+
|
|
607
|
+
}
|
|
608
|
+
} else {
|
|
609
|
+
switch (action.TAG) {
|
|
610
|
+
case "ValidatePartitionQueryResponse" :
|
|
611
|
+
return validatePartitionQueryResponse(state, action._0);
|
|
612
|
+
case "SubmitPartitionQueryResponse" :
|
|
613
|
+
return submitPartitionQueryResponse(state, action.newItems, action.newItemsWithDcs, action.knownHeight, action.latestFetchedBlock, action.query, action.chain);
|
|
614
|
+
case "FinishWaitingForNewBlock" :
|
|
615
|
+
var knownHeight = action.knownHeight;
|
|
616
|
+
var chain = action.chain;
|
|
617
|
+
var updatedChainFetchers = ChainMap.update(state.chainManager.chainFetchers, chain, (function (chainFetcher) {
|
|
618
|
+
var updatedFetchState = FetchState.updateKnownHeight(chainFetcher.fetchState, knownHeight);
|
|
619
|
+
if (updatedFetchState !== chainFetcher.fetchState) {
|
|
620
|
+
return {
|
|
621
|
+
logger: chainFetcher.logger,
|
|
622
|
+
fetchState: updatedFetchState,
|
|
623
|
+
sourceManager: chainFetcher.sourceManager,
|
|
624
|
+
chainConfig: chainFetcher.chainConfig,
|
|
625
|
+
isProgressAtHead: chainFetcher.isProgressAtHead,
|
|
626
|
+
timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
|
|
627
|
+
committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
|
|
628
|
+
numEventsProcessed: chainFetcher.numEventsProcessed,
|
|
629
|
+
reorgDetection: chainFetcher.reorgDetection,
|
|
630
|
+
safeCheckpointTracking: chainFetcher.safeCheckpointTracking
|
|
631
|
+
};
|
|
632
|
+
} else {
|
|
633
|
+
return chainFetcher;
|
|
634
|
+
}
|
|
635
|
+
}));
|
|
636
|
+
var isBelowReorgThreshold = !state.chainManager.isInReorgThreshold && state.ctx.config.shouldRollbackOnReorg;
|
|
637
|
+
var shouldEnterReorgThreshold = isBelowReorgThreshold && Belt_Array.every(ChainMap.values(updatedChainFetchers), (function (chainFetcher) {
|
|
638
|
+
return FetchState.isReadyToEnterReorgThreshold(chainFetcher.fetchState);
|
|
639
|
+
}));
|
|
640
|
+
var init = state.chainManager;
|
|
641
|
+
var state_ctx = state.ctx;
|
|
642
|
+
var state_chainManager = {
|
|
643
|
+
committedCheckpointId: init.committedCheckpointId,
|
|
644
|
+
chainFetchers: updatedChainFetchers,
|
|
645
|
+
multichain: init.multichain,
|
|
646
|
+
isInReorgThreshold: init.isInReorgThreshold
|
|
647
|
+
};
|
|
648
|
+
var state_processedBatches = state.processedBatches;
|
|
649
|
+
var state_currentlyProcessingBatch = state.currentlyProcessingBatch;
|
|
650
|
+
var state_rollbackState = state.rollbackState;
|
|
651
|
+
var state_indexerStartTime = state.indexerStartTime;
|
|
652
|
+
var state_writeThrottlers = state.writeThrottlers;
|
|
653
|
+
var state_loadManager = state.loadManager;
|
|
654
|
+
var state_keepProcessAlive = state.keepProcessAlive;
|
|
655
|
+
var state_id = state.id;
|
|
656
|
+
var state$1 = {
|
|
657
|
+
ctx: state_ctx,
|
|
658
|
+
chainManager: state_chainManager,
|
|
659
|
+
processedBatches: state_processedBatches,
|
|
660
|
+
currentlyProcessingBatch: state_currentlyProcessingBatch,
|
|
661
|
+
rollbackState: state_rollbackState,
|
|
662
|
+
indexerStartTime: state_indexerStartTime,
|
|
663
|
+
writeThrottlers: state_writeThrottlers,
|
|
664
|
+
loadManager: state_loadManager,
|
|
665
|
+
keepProcessAlive: state_keepProcessAlive,
|
|
666
|
+
id: state_id
|
|
667
|
+
};
|
|
668
|
+
if (shouldEnterReorgThreshold) {
|
|
669
|
+
return [
|
|
670
|
+
onEnterReorgThreshold(state$1),
|
|
671
|
+
[
|
|
672
|
+
{
|
|
673
|
+
TAG: "NextQuery",
|
|
674
|
+
_0: "CheckAllChains"
|
|
675
|
+
},
|
|
676
|
+
"ProcessEventBatch"
|
|
677
|
+
]
|
|
678
|
+
];
|
|
679
|
+
} else {
|
|
680
|
+
return [
|
|
681
|
+
state$1,
|
|
682
|
+
[
|
|
683
|
+
{
|
|
684
|
+
TAG: "NextQuery",
|
|
685
|
+
_0: {
|
|
686
|
+
TAG: "Chain",
|
|
687
|
+
_0: chain
|
|
688
|
+
}
|
|
689
|
+
},
|
|
690
|
+
"ProcessEventBatch"
|
|
691
|
+
]
|
|
692
|
+
];
|
|
693
|
+
}
|
|
694
|
+
case "EventBatchProcessed" :
|
|
695
|
+
var maybePruneEntityHistory = Config.shouldPruneHistory(state.ctx.config, state.chainManager.isInReorgThreshold) ? ["PruneStaleEntityHistory"] : [];
|
|
696
|
+
var state_ctx$1 = state.ctx;
|
|
697
|
+
var state_chainManager$1 = updateProgressedChains(state.chainManager, action.batch, state.ctx);
|
|
698
|
+
var state_processedBatches$1 = state.processedBatches + 1 | 0;
|
|
699
|
+
var state_indexerStartTime$1 = state.indexerStartTime;
|
|
700
|
+
var state_writeThrottlers$1 = state.writeThrottlers;
|
|
701
|
+
var state_loadManager$1 = state.loadManager;
|
|
702
|
+
var state_keepProcessAlive$1 = state.keepProcessAlive;
|
|
703
|
+
var state_id$1 = state.id;
|
|
704
|
+
var state$2 = {
|
|
705
|
+
ctx: state_ctx$1,
|
|
706
|
+
chainManager: state_chainManager$1,
|
|
707
|
+
processedBatches: state_processedBatches$1,
|
|
708
|
+
currentlyProcessingBatch: false,
|
|
709
|
+
rollbackState: "NoRollback",
|
|
710
|
+
indexerStartTime: state_indexerStartTime$1,
|
|
711
|
+
writeThrottlers: state_writeThrottlers$1,
|
|
712
|
+
loadManager: state_loadManager$1,
|
|
713
|
+
keepProcessAlive: state_keepProcessAlive$1,
|
|
714
|
+
id: state_id$1
|
|
715
|
+
};
|
|
716
|
+
var shouldExit = EventProcessing.allChainsEventsProcessedToEndblock(state_chainManager$1.chainFetchers) ? (Logging.info("All chains are caught up to end blocks."), state_keepProcessAlive$1 ? "NoExit" : "ExitWithSuccess") : "NoExit";
|
|
717
|
+
return [
|
|
718
|
+
state$2,
|
|
719
|
+
Belt_Array.concat([
|
|
720
|
+
{
|
|
721
|
+
TAG: "UpdateChainMetaDataAndCheckForExit",
|
|
722
|
+
_0: shouldExit
|
|
723
|
+
},
|
|
724
|
+
"ProcessEventBatch"
|
|
725
|
+
], maybePruneEntityHistory)
|
|
726
|
+
];
|
|
727
|
+
case "FindReorgDepth" :
|
|
728
|
+
return [
|
|
729
|
+
{
|
|
730
|
+
ctx: state.ctx,
|
|
731
|
+
chainManager: state.chainManager,
|
|
732
|
+
processedBatches: state.processedBatches,
|
|
733
|
+
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
734
|
+
rollbackState: {
|
|
735
|
+
TAG: "FoundReorgDepth",
|
|
736
|
+
chain: action.chain,
|
|
737
|
+
rollbackTargetBlockNumber: action.rollbackTargetBlockNumber
|
|
738
|
+
},
|
|
739
|
+
indexerStartTime: state.indexerStartTime,
|
|
740
|
+
writeThrottlers: state.writeThrottlers,
|
|
741
|
+
loadManager: state.loadManager,
|
|
742
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
743
|
+
id: state.id
|
|
744
|
+
},
|
|
745
|
+
["Rollback"]
|
|
746
|
+
];
|
|
747
|
+
case "UpdateQueues" :
|
|
748
|
+
var shouldEnterReorgThreshold$1 = action.shouldEnterReorgThreshold;
|
|
749
|
+
var progressedChainsById = action.progressedChainsById;
|
|
750
|
+
var chainFetchers = ChainMap.mapWithKey(state.chainManager.chainFetchers, (function (chain, cf) {
|
|
751
|
+
var chainAfterBatch = progressedChainsById[chain];
|
|
752
|
+
var fs = chainAfterBatch !== undefined ? chainAfterBatch.fetchState : cf.fetchState;
|
|
753
|
+
return {
|
|
754
|
+
logger: cf.logger,
|
|
755
|
+
fetchState: shouldEnterReorgThreshold$1 ? FetchState.updateInternal(fs, undefined, undefined, undefined, cf.chainConfig.blockLag, undefined) : fs,
|
|
756
|
+
sourceManager: cf.sourceManager,
|
|
757
|
+
chainConfig: cf.chainConfig,
|
|
758
|
+
isProgressAtHead: cf.isProgressAtHead,
|
|
759
|
+
timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock,
|
|
760
|
+
committedProgressBlockNumber: cf.committedProgressBlockNumber,
|
|
761
|
+
numEventsProcessed: cf.numEventsProcessed,
|
|
762
|
+
reorgDetection: cf.reorgDetection,
|
|
763
|
+
safeCheckpointTracking: cf.safeCheckpointTracking
|
|
764
|
+
};
|
|
765
|
+
}));
|
|
766
|
+
var init$1 = state.chainManager;
|
|
767
|
+
var chainManager_committedCheckpointId = init$1.committedCheckpointId;
|
|
768
|
+
var chainManager_multichain = init$1.multichain;
|
|
769
|
+
var chainManager_isInReorgThreshold = init$1.isInReorgThreshold;
|
|
770
|
+
var chainManager = {
|
|
771
|
+
committedCheckpointId: chainManager_committedCheckpointId,
|
|
772
|
+
chainFetchers: chainFetchers,
|
|
773
|
+
multichain: chainManager_multichain,
|
|
774
|
+
isInReorgThreshold: chainManager_isInReorgThreshold
|
|
775
|
+
};
|
|
776
|
+
return [
|
|
777
|
+
{
|
|
778
|
+
ctx: state.ctx,
|
|
779
|
+
chainManager: chainManager,
|
|
780
|
+
processedBatches: state.processedBatches,
|
|
781
|
+
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
782
|
+
rollbackState: state.rollbackState,
|
|
783
|
+
indexerStartTime: state.indexerStartTime,
|
|
784
|
+
writeThrottlers: state.writeThrottlers,
|
|
785
|
+
loadManager: state.loadManager,
|
|
786
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
787
|
+
id: state.id
|
|
788
|
+
},
|
|
789
|
+
[{
|
|
790
|
+
TAG: "NextQuery",
|
|
791
|
+
_0: "CheckAllChains"
|
|
792
|
+
}]
|
|
793
|
+
];
|
|
794
|
+
case "ErrorExit" :
|
|
795
|
+
ErrorHandling.log(action._0);
|
|
796
|
+
Process.exit(1);
|
|
797
|
+
return [
|
|
798
|
+
state,
|
|
799
|
+
[]
|
|
800
|
+
];
|
|
801
|
+
case "SetRollbackState" :
|
|
802
|
+
return [
|
|
803
|
+
{
|
|
804
|
+
ctx: state.ctx,
|
|
805
|
+
chainManager: action.rollbackedChainManager,
|
|
806
|
+
processedBatches: state.processedBatches,
|
|
807
|
+
currentlyProcessingBatch: state.currentlyProcessingBatch,
|
|
808
|
+
rollbackState: {
|
|
809
|
+
TAG: "RollbackReady",
|
|
810
|
+
diffInMemoryStore: action.diffInMemoryStore,
|
|
811
|
+
eventsProcessedDiffByChain: action.eventsProcessedDiffByChain
|
|
812
|
+
},
|
|
813
|
+
indexerStartTime: state.indexerStartTime,
|
|
814
|
+
writeThrottlers: state.writeThrottlers,
|
|
815
|
+
loadManager: state.loadManager,
|
|
816
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
817
|
+
id: state.id
|
|
818
|
+
},
|
|
819
|
+
[
|
|
820
|
+
{
|
|
821
|
+
TAG: "NextQuery",
|
|
822
|
+
_0: "CheckAllChains"
|
|
823
|
+
},
|
|
824
|
+
"ProcessEventBatch"
|
|
825
|
+
]
|
|
826
|
+
];
|
|
827
|
+
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
function invalidatedActionReducer(state, action) {
|
|
833
|
+
if (typeof action === "object") {
|
|
834
|
+
switch (action.TAG) {
|
|
835
|
+
case "EventBatchProcessed" :
|
|
836
|
+
if (isPreparingRollback(state)) {
|
|
837
|
+
Logging.info("Finished processing batch before rollback, actioning rollback");
|
|
838
|
+
return [
|
|
839
|
+
{
|
|
840
|
+
ctx: state.ctx,
|
|
841
|
+
chainManager: updateProgressedChains(state.chainManager, action.batch, state.ctx),
|
|
842
|
+
processedBatches: state.processedBatches + 1 | 0,
|
|
843
|
+
currentlyProcessingBatch: false,
|
|
844
|
+
rollbackState: state.rollbackState,
|
|
845
|
+
indexerStartTime: state.indexerStartTime,
|
|
846
|
+
writeThrottlers: state.writeThrottlers,
|
|
847
|
+
loadManager: state.loadManager,
|
|
848
|
+
keepProcessAlive: state.keepProcessAlive,
|
|
849
|
+
id: state.id
|
|
850
|
+
},
|
|
851
|
+
["Rollback"]
|
|
852
|
+
];
|
|
853
|
+
}
|
|
854
|
+
break;
|
|
855
|
+
case "ErrorExit" :
|
|
856
|
+
return actionReducer(state, action);
|
|
857
|
+
default:
|
|
858
|
+
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
Logging.trace({
|
|
862
|
+
msg: "Invalidated action discarded",
|
|
863
|
+
action: S$RescriptSchema.convertOrThrow(action, Utils.Schema.variantTag)
|
|
864
|
+
});
|
|
865
|
+
return [
|
|
866
|
+
state,
|
|
867
|
+
[]
|
|
868
|
+
];
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
function checkAndFetchForChain(waitForNewBlock, executeQuery, state, dispatchAction) {
|
|
872
|
+
return async function (chain) {
|
|
873
|
+
var chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
|
|
874
|
+
if (isPreparingRollback(state)) {
|
|
875
|
+
return ;
|
|
876
|
+
}
|
|
877
|
+
var fetchState = chainFetcher.fetchState;
|
|
878
|
+
return await SourceManager.fetchNext(chainFetcher.sourceManager, fetchState, (async function (query) {
|
|
879
|
+
try {
|
|
880
|
+
var response = await executeQuery(chainFetcher.sourceManager, query, fetchState.knownHeight, ChainFetcher.isReady(chainFetcher));
|
|
881
|
+
return dispatchAction({
|
|
882
|
+
TAG: "ValidatePartitionQueryResponse",
|
|
883
|
+
_0: {
|
|
884
|
+
chain: chain,
|
|
885
|
+
response: response,
|
|
886
|
+
query: query
|
|
887
|
+
}
|
|
888
|
+
});
|
|
889
|
+
}
|
|
890
|
+
catch (raw_exn){
|
|
891
|
+
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
892
|
+
return dispatchAction({
|
|
893
|
+
TAG: "ErrorExit",
|
|
894
|
+
_0: ErrorHandling.make(exn, undefined, undefined)
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
}), (function (knownHeight) {
|
|
898
|
+
return waitForNewBlock(chainFetcher.sourceManager, knownHeight, ChainFetcher.isReady(chainFetcher));
|
|
899
|
+
}), (function (knownHeight) {
|
|
900
|
+
dispatchAction({
|
|
901
|
+
TAG: "FinishWaitingForNewBlock",
|
|
902
|
+
chain: chain,
|
|
903
|
+
knownHeight: knownHeight
|
|
904
|
+
});
|
|
905
|
+
}), state.id);
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
function injectedTaskReducer(waitForNewBlock, executeQuery, getLastKnownValidBlock) {
|
|
910
|
+
return async function (state, task, dispatchAction) {
|
|
911
|
+
if (typeof task !== "object") {
|
|
912
|
+
switch (task) {
|
|
913
|
+
case "ProcessEventBatch" :
|
|
914
|
+
if (!(!state.currentlyProcessingBatch && !isPreparingRollback(state))) {
|
|
915
|
+
return ;
|
|
916
|
+
}
|
|
917
|
+
var match = state.rollbackState;
|
|
918
|
+
var rollbackInMemStore;
|
|
919
|
+
rollbackInMemStore = typeof match !== "object" || match.TAG !== "RollbackReady" ? undefined : match.diffInMemoryStore;
|
|
920
|
+
var batch = ChainManager.createBatch(state.chainManager, state.ctx.config.batchSize, rollbackInMemStore !== undefined);
|
|
921
|
+
var progressedChainsById = batch.progressedChainsById;
|
|
922
|
+
var isInReorgThreshold = state.chainManager.isInReorgThreshold;
|
|
923
|
+
var shouldSaveHistory = Config.shouldSaveHistory(state.ctx.config, isInReorgThreshold);
|
|
924
|
+
var isBelowReorgThreshold = !state.chainManager.isInReorgThreshold && state.ctx.config.shouldRollbackOnReorg;
|
|
925
|
+
var shouldEnterReorgThreshold = isBelowReorgThreshold && Belt_Array.every(ChainMap.values(state.chainManager.chainFetchers), (function (chainFetcher) {
|
|
926
|
+
var chainAfterBatch = progressedChainsById[chainFetcher.fetchState.chainId];
|
|
927
|
+
var fetchState = chainAfterBatch !== undefined ? chainAfterBatch.fetchState : chainFetcher.fetchState;
|
|
928
|
+
return FetchState.isReadyToEnterReorgThreshold(fetchState);
|
|
929
|
+
}));
|
|
930
|
+
if (shouldEnterReorgThreshold) {
|
|
931
|
+
dispatchAction("EnterReorgThreshold");
|
|
932
|
+
}
|
|
933
|
+
if (Utils.Dict.isEmpty(progressedChainsById)) {
|
|
934
|
+
if (EventProcessing.allChainsEventsProcessedToEndblock(state.chainManager.chainFetchers)) {
|
|
935
|
+
Logging.info("All chains are caught up to end blocks.");
|
|
936
|
+
if (!state.keepProcessAlive) {
|
|
937
|
+
updateChainMetadataTable(state.chainManager, state.ctx.persistence, state.writeThrottlers.chainMetaData);
|
|
938
|
+
return dispatchAction("SuccessExit");
|
|
939
|
+
} else {
|
|
940
|
+
return ;
|
|
941
|
+
}
|
|
942
|
+
} else {
|
|
943
|
+
return ;
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
dispatchAction("StartProcessingBatch");
|
|
947
|
+
dispatchAction({
|
|
948
|
+
TAG: "UpdateQueues",
|
|
949
|
+
progressedChainsById: progressedChainsById,
|
|
950
|
+
shouldEnterReorgThreshold: shouldEnterReorgThreshold
|
|
951
|
+
});
|
|
952
|
+
var inMemoryStore = Belt_Option.getWithDefault(rollbackInMemStore, InMemoryStore.make(state.ctx.persistence.allEntities, undefined));
|
|
953
|
+
InMemoryStore.setBatchDcs(inMemoryStore, batch, shouldSaveHistory);
|
|
954
|
+
var res;
|
|
955
|
+
try {
|
|
956
|
+
res = await EventProcessing.processEventBatch(batch, inMemoryStore, isInReorgThreshold, state.loadManager, state.ctx, state.chainManager.chainFetchers);
|
|
957
|
+
}
|
|
958
|
+
catch (raw_exn){
|
|
959
|
+
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
960
|
+
var errHandler = ErrorHandling.make(exn, undefined, "A top level unexpected error occurred during processing");
|
|
961
|
+
return dispatchAction({
|
|
962
|
+
TAG: "ErrorExit",
|
|
963
|
+
_0: errHandler
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
if (res.TAG === "Ok") {
|
|
967
|
+
return dispatchAction({
|
|
968
|
+
TAG: "EventBatchProcessed",
|
|
969
|
+
batch: batch
|
|
970
|
+
});
|
|
971
|
+
} else {
|
|
972
|
+
return dispatchAction({
|
|
973
|
+
TAG: "ErrorExit",
|
|
974
|
+
_0: res._0
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
break;
|
|
978
|
+
case "Rollback" :
|
|
979
|
+
var match$1 = state.rollbackState;
|
|
980
|
+
if (typeof match$1 !== "object") {
|
|
981
|
+
if (match$1 === "NoRollback") {
|
|
982
|
+
return Js_exn.raiseError("Internal error: Rollback initiated with invalid state");
|
|
983
|
+
} else {
|
|
984
|
+
return ;
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
switch (match$1.TAG) {
|
|
988
|
+
case "ReorgDetected" :
|
|
989
|
+
var chain = match$1.chain;
|
|
990
|
+
var chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
|
|
991
|
+
dispatchAction("StartFindingReorgDepth");
|
|
992
|
+
var rollbackTargetBlockNumber = await getLastKnownValidBlock(chainFetcher, match$1.blockNumber);
|
|
993
|
+
return dispatchAction({
|
|
994
|
+
TAG: "FindReorgDepth",
|
|
995
|
+
chain: chain,
|
|
996
|
+
rollbackTargetBlockNumber: rollbackTargetBlockNumber
|
|
997
|
+
});
|
|
998
|
+
case "FoundReorgDepth" :
|
|
999
|
+
if (state.currentlyProcessingBatch) {
|
|
1000
|
+
return Logging.info("Waiting for batch to finish processing before executing rollback");
|
|
1001
|
+
}
|
|
1002
|
+
var match$2 = state.rollbackState;
|
|
1003
|
+
var rollbackTargetBlockNumber$1 = match$2.rollbackTargetBlockNumber;
|
|
1004
|
+
var reorgChain = match$2.chain;
|
|
1005
|
+
var startTime = Hrtime.makeTimer();
|
|
1006
|
+
var chainFetcher$1 = ChainMap.get(state.chainManager.chainFetchers, reorgChain);
|
|
1007
|
+
var logger = Logging.createChildFrom(chainFetcher$1.logger, {
|
|
1008
|
+
action: "Rollback",
|
|
1009
|
+
reorgChain: reorgChain,
|
|
1010
|
+
targetBlockNumber: rollbackTargetBlockNumber$1
|
|
1011
|
+
});
|
|
1012
|
+
Logging.childInfo(logger, "Started rollback on reorg");
|
|
1013
|
+
Prometheus.RollbackTargetBlockNumber.set(rollbackTargetBlockNumber$1, reorgChain);
|
|
1014
|
+
var checkpointId = await state.ctx.persistence.storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber$1);
|
|
1015
|
+
var rollbackTargetCheckpointId = checkpointId !== undefined ? checkpointId : 0n;
|
|
1016
|
+
var eventsProcessedDiffByChain = {};
|
|
1017
|
+
var newProgressBlockNumberPerChain = {};
|
|
1018
|
+
var rollbackedProcessedEvents = 0;
|
|
1019
|
+
var rollbackProgressDiff = await state.ctx.persistence.storage.getRollbackProgressDiff(rollbackTargetCheckpointId);
|
|
1020
|
+
for(var idx = 0 ,idx_finish = rollbackProgressDiff.length; idx < idx_finish; ++idx){
|
|
1021
|
+
var diff = rollbackProgressDiff[idx];
|
|
1022
|
+
var eventsProcessedDiff = Belt_Option.getExn(Belt_Float.fromString(diff.events_processed_diff));
|
|
1023
|
+
rollbackedProcessedEvents = rollbackedProcessedEvents + eventsProcessedDiff;
|
|
1024
|
+
eventsProcessedDiffByChain[diff.chain_id] = eventsProcessedDiff;
|
|
1025
|
+
newProgressBlockNumberPerChain[diff.chain_id] = rollbackTargetCheckpointId === 0n && diff.chain_id === reorgChain && diff.new_progress_block_number >= rollbackTargetBlockNumber$1 ? rollbackTargetBlockNumber$1 : diff.new_progress_block_number;
|
|
1026
|
+
}
|
|
1027
|
+
var chainFetchers = ChainMap.mapWithKey(state.chainManager.chainFetchers, (function (chain, cf) {
|
|
1028
|
+
var newProgressBlockNumber = newProgressBlockNumberPerChain[chain];
|
|
1029
|
+
if (newProgressBlockNumber === undefined) {
|
|
1030
|
+
if (Caml_obj.equal(chain, reorgChain)) {
|
|
1031
|
+
return {
|
|
1032
|
+
logger: cf.logger,
|
|
1033
|
+
fetchState: FetchState.rollback(cf.fetchState, rollbackTargetBlockNumber$1),
|
|
1034
|
+
sourceManager: cf.sourceManager,
|
|
1035
|
+
chainConfig: cf.chainConfig,
|
|
1036
|
+
isProgressAtHead: cf.isProgressAtHead,
|
|
1037
|
+
timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock,
|
|
1038
|
+
committedProgressBlockNumber: cf.committedProgressBlockNumber,
|
|
1039
|
+
numEventsProcessed: cf.numEventsProcessed,
|
|
1040
|
+
reorgDetection: ReorgDetection.rollbackToValidBlockNumber(cf.reorgDetection, rollbackTargetBlockNumber$1),
|
|
1041
|
+
safeCheckpointTracking: cf.safeCheckpointTracking
|
|
1042
|
+
};
|
|
1043
|
+
} else {
|
|
1044
|
+
return cf;
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
var fetchState = FetchState.rollback(cf.fetchState, newProgressBlockNumber);
|
|
1048
|
+
var newTotalEventsProcessed = cf.numEventsProcessed - eventsProcessedDiffByChain[chain];
|
|
1049
|
+
if (cf.committedProgressBlockNumber !== newProgressBlockNumber) {
|
|
1050
|
+
Prometheus.ProgressBlockNumber.set(newProgressBlockNumber, chain);
|
|
1051
|
+
}
|
|
1052
|
+
if (cf.numEventsProcessed !== newTotalEventsProcessed) {
|
|
1053
|
+
Prometheus.ProgressEventsCount.set(newTotalEventsProcessed, chain);
|
|
1054
|
+
}
|
|
1055
|
+
var safeCheckpointTracking = cf.safeCheckpointTracking;
|
|
1056
|
+
return {
|
|
1057
|
+
logger: cf.logger,
|
|
1058
|
+
fetchState: fetchState,
|
|
1059
|
+
sourceManager: cf.sourceManager,
|
|
1060
|
+
chainConfig: cf.chainConfig,
|
|
1061
|
+
isProgressAtHead: cf.isProgressAtHead,
|
|
1062
|
+
timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock,
|
|
1063
|
+
committedProgressBlockNumber: newProgressBlockNumber,
|
|
1064
|
+
numEventsProcessed: newTotalEventsProcessed,
|
|
1065
|
+
reorgDetection: Caml_obj.equal(chain, reorgChain) ? ReorgDetection.rollbackToValidBlockNumber(cf.reorgDetection, rollbackTargetBlockNumber$1) : cf.reorgDetection,
|
|
1066
|
+
safeCheckpointTracking: safeCheckpointTracking !== undefined ? SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber) : undefined
|
|
1067
|
+
};
|
|
1068
|
+
}));
|
|
1069
|
+
var diff$1 = await Persistence.prepareRollbackDiff(state.ctx.persistence, rollbackTargetCheckpointId, $$BigInt.add(state.chainManager.committedCheckpointId, 1n));
|
|
1070
|
+
var init = state.chainManager;
|
|
1071
|
+
var chainManager_committedCheckpointId = init.committedCheckpointId;
|
|
1072
|
+
var chainManager_multichain = init.multichain;
|
|
1073
|
+
var chainManager_isInReorgThreshold = init.isInReorgThreshold;
|
|
1074
|
+
var chainManager = {
|
|
1075
|
+
committedCheckpointId: chainManager_committedCheckpointId,
|
|
1076
|
+
chainFetchers: chainFetchers,
|
|
1077
|
+
multichain: chainManager_multichain,
|
|
1078
|
+
isInReorgThreshold: chainManager_isInReorgThreshold
|
|
1079
|
+
};
|
|
1080
|
+
Logging.childTrace(logger, {
|
|
1081
|
+
msg: "Finished rollback on reorg",
|
|
1082
|
+
entityChanges: {
|
|
1083
|
+
deleted: diff$1.deletedEntities,
|
|
1084
|
+
upserted: diff$1.setEntities
|
|
1085
|
+
},
|
|
1086
|
+
rollbackedEvents: rollbackedProcessedEvents,
|
|
1087
|
+
beforeCheckpointId: state.chainManager.committedCheckpointId,
|
|
1088
|
+
targetCheckpointId: rollbackTargetCheckpointId
|
|
1089
|
+
});
|
|
1090
|
+
Prometheus.RollbackSuccess.increment(Hrtime.toSecondsFloat(Hrtime.timeSince(startTime)), rollbackedProcessedEvents);
|
|
1091
|
+
return dispatchAction({
|
|
1092
|
+
TAG: "SetRollbackState",
|
|
1093
|
+
diffInMemoryStore: diff$1.inMemStore,
|
|
1094
|
+
rollbackedChainManager: chainManager,
|
|
1095
|
+
eventsProcessedDiffByChain: eventsProcessedDiffByChain
|
|
1096
|
+
});
|
|
1097
|
+
case "RollbackReady" :
|
|
1098
|
+
return Js_exn.raiseError("Internal error: Rollback initiated with invalid state");
|
|
1099
|
+
|
|
1100
|
+
}
|
|
1101
|
+
case "PruneStaleEntityHistory" :
|
|
1102
|
+
var runPrune = async function () {
|
|
1103
|
+
var safeCheckpointId = ChainManager.getSafeCheckpointId(state.chainManager);
|
|
1104
|
+
if (safeCheckpointId === undefined) {
|
|
1105
|
+
return ;
|
|
1106
|
+
}
|
|
1107
|
+
await state.ctx.persistence.storage.pruneStaleCheckpoints(safeCheckpointId);
|
|
1108
|
+
for(var idx = 0 ,idx_finish = state.ctx.persistence.allEntities.length; idx < idx_finish; ++idx){
|
|
1109
|
+
if (idx !== 0) {
|
|
1110
|
+
await Utils.delay(1000);
|
|
1111
|
+
}
|
|
1112
|
+
var entityConfig = state.ctx.persistence.allEntities[idx];
|
|
1113
|
+
var timeRef = Hrtime.makeTimer();
|
|
1114
|
+
try {
|
|
1115
|
+
await state.ctx.persistence.storage.pruneStaleEntityHistory(entityConfig.name, entityConfig.index, safeCheckpointId);
|
|
1116
|
+
}
|
|
1117
|
+
catch (raw_exn){
|
|
1118
|
+
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
1119
|
+
ErrorHandling.mkLogAndRaise(Logging.createChild({
|
|
1120
|
+
entityName: entityConfig.name,
|
|
1121
|
+
safeCheckpointId: safeCheckpointId
|
|
1122
|
+
}), "Failed to prune stale entity history", exn);
|
|
1123
|
+
}
|
|
1124
|
+
Prometheus.RollbackHistoryPrune.increment(Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef)), entityConfig.name);
|
|
1125
|
+
}
|
|
1126
|
+
};
|
|
1127
|
+
return Throttler.schedule(state.writeThrottlers.pruneStaleEntityHistory, runPrune);
|
|
1128
|
+
|
|
1129
|
+
}
|
|
1130
|
+
} else {
|
|
1131
|
+
switch (task.TAG) {
|
|
1132
|
+
case "NextQuery" :
|
|
1133
|
+
var chainCheck = task._0;
|
|
1134
|
+
var fetchForChain = checkAndFetchForChain(waitForNewBlock, executeQuery, state, dispatchAction);
|
|
1135
|
+
if (typeof chainCheck === "object") {
|
|
1136
|
+
return await fetchForChain(chainCheck._0);
|
|
1137
|
+
}
|
|
1138
|
+
await Promise.all(Belt_Array.map(ChainMap.keys(state.chainManager.chainFetchers), (function (__x) {
|
|
1139
|
+
return fetchForChain(__x);
|
|
1140
|
+
})));
|
|
1141
|
+
return ;
|
|
1142
|
+
case "ProcessPartitionQueryResponse" :
|
|
1143
|
+
processPartitionQueryResponse(state, task._0, dispatchAction);
|
|
1144
|
+
return ;
|
|
1145
|
+
case "UpdateChainMetaDataAndCheckForExit" :
|
|
1146
|
+
var writeThrottlers = state.writeThrottlers;
|
|
1147
|
+
var chainManager$1 = state.chainManager;
|
|
1148
|
+
if (task._0 === "ExitWithSuccess") {
|
|
1149
|
+
updateChainMetadataTable(chainManager$1, state.ctx.persistence, writeThrottlers.chainMetaData);
|
|
1150
|
+
return dispatchAction("SuccessExit");
|
|
1151
|
+
}
|
|
1152
|
+
updateChainMetadataTable(chainManager$1, state.ctx.persistence, writeThrottlers.chainMetaData);
|
|
1153
|
+
return ;
|
|
1154
|
+
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
var taskReducer = injectedTaskReducer(SourceManager.waitForNewBlock, SourceManager.executeQuery, (function (chainFetcher, reorgBlockNumber) {
|
|
1161
|
+
return ChainFetcher.getLastKnownValidBlock(chainFetcher, reorgBlockNumber, undefined);
|
|
1162
|
+
}));
|
|
1163
|
+
|
|
1164
|
+
export {
|
|
1165
|
+
WriteThrottlers ,
|
|
1166
|
+
make$1 as make,
|
|
1167
|
+
getId ,
|
|
1168
|
+
setChainManager ,
|
|
1169
|
+
isPreparingRollback ,
|
|
1170
|
+
updateChainMetadataTable ,
|
|
1171
|
+
updateProgressedChains ,
|
|
1172
|
+
validatePartitionQueryResponse ,
|
|
1173
|
+
submitPartitionQueryResponse ,
|
|
1174
|
+
processPartitionQueryResponse ,
|
|
1175
|
+
updateChainFetcher ,
|
|
1176
|
+
onEnterReorgThreshold ,
|
|
1177
|
+
actionReducer ,
|
|
1178
|
+
invalidatedActionReducer ,
|
|
1179
|
+
checkAndFetchForChain ,
|
|
1180
|
+
injectedTaskReducer ,
|
|
1181
|
+
taskReducer ,
|
|
1182
|
+
}
|
|
1183
|
+
/* taskReducer Not a pure module */
|