envio 3.2.1 → 3.3.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -7
- package/src/Batch.res +12 -15
- package/src/Batch.res.mjs +4 -5
- package/src/BatchProcessing.res +199 -0
- package/src/BatchProcessing.res.mjs +125 -0
- package/src/ChainFetching.res +375 -0
- package/src/ChainFetching.res.mjs +206 -0
- package/src/ChainMetadata.res +27 -0
- package/src/ChainMetadata.res.mjs +27 -0
- package/src/ChainState.res +618 -0
- package/src/ChainState.res.mjs +461 -0
- package/src/ChainState.resi +85 -0
- package/src/Config.res +13 -4
- package/src/Config.res.mjs +8 -3
- package/src/ContractRegisterContext.res +106 -0
- package/src/ContractRegisterContext.res.mjs +76 -0
- package/src/Core.res +3 -0
- package/src/CrossChainState.res +269 -0
- package/src/CrossChainState.res.mjs +197 -0
- package/src/CrossChainState.resi +47 -0
- package/src/Ecosystem.res +58 -0
- package/src/Ecosystem.res.mjs +43 -1
- package/src/Env.res +7 -2
- package/src/Env.res.mjs +5 -2
- package/src/EventConfigBuilder.res +2 -2
- package/src/EventProcessing.res +60 -46
- package/src/EventProcessing.res.mjs +33 -32
- package/src/EventUtils.res +0 -4
- package/src/EventUtils.res.mjs +0 -4
- package/src/ExitOnCaughtUp.res +10 -0
- package/src/ExitOnCaughtUp.res.mjs +22 -0
- package/src/FetchState.res +124 -62
- package/src/FetchState.res.mjs +159 -93
- package/src/InMemoryStore.res +36 -451
- package/src/InMemoryStore.res.mjs +20 -369
- package/src/IndexerLoop.res +44 -0
- package/src/IndexerLoop.res.mjs +46 -0
- package/src/IndexerState.res +544 -0
- package/src/IndexerState.res.mjs +547 -0
- package/src/IndexerState.resi +132 -0
- package/src/Internal.res +29 -8
- package/src/LoadLayer.res +20 -18
- package/src/LoadLayer.res.mjs +14 -12
- package/src/LoadLayer.resi +6 -3
- package/src/Logging.res +11 -44
- package/src/Logging.res.mjs +10 -42
- package/src/Main.res +54 -79
- package/src/Main.res.mjs +44 -56
- package/src/PgStorage.res +8 -71
- package/src/PgStorage.res.mjs +3 -47
- package/src/Prometheus.res +7 -8
- package/src/Prometheus.res.mjs +7 -4
- package/src/PruneStaleHistory.res +45 -0
- package/src/PruneStaleHistory.res.mjs +46 -0
- package/src/RawEvent.res +73 -0
- package/src/RawEvent.res.mjs +51 -0
- package/src/Rollback.res +204 -0
- package/src/Rollback.res.mjs +118 -0
- package/src/SimulateItems.res +12 -10
- package/src/SimulateItems.res.mjs +1 -1
- package/src/UserContext.res +26 -114
- package/src/UserContext.res.mjs +15 -78
- package/src/Writing.res +242 -0
- package/src/Writing.res.mjs +215 -0
- package/src/db/InternalTable.res +14 -27
- package/src/sources/Evm.res +40 -1
- package/src/sources/Evm.res.mjs +94 -84
- package/src/sources/Fuel.res +40 -1
- package/src/sources/Fuel.res.mjs +36 -26
- package/src/sources/HyperFuel.res +41 -120
- package/src/sources/HyperFuel.res.mjs +42 -80
- package/src/sources/HyperFuel.resi +1 -24
- package/src/sources/HyperFuelClient.res +16 -297
- package/src/sources/HyperFuelClient.res.mjs +5 -4
- package/src/sources/HyperFuelSource.res +33 -8
- package/src/sources/HyperFuelSource.res.mjs +56 -10
- package/src/sources/HyperSyncSource.res +5 -3
- package/src/sources/HyperSyncSource.res.mjs +1 -1
- package/src/sources/RpcSource.res +2 -2
- package/src/sources/RpcSource.res.mjs +1 -1
- package/src/sources/SourceManager.res +8 -18
- package/src/sources/SourceManager.res.mjs +8 -5
- package/src/sources/SourceManager.resi +4 -1
- package/src/sources/Svm.res +18 -1
- package/src/sources/Svm.res.mjs +30 -22
- package/src/sources/SvmHyperSyncSource.res +1 -1
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/src/tui/Tui.res +38 -36
- package/src/tui/Tui.res.mjs +51 -51
- package/src/ChainFetcher.res +0 -519
- package/src/ChainFetcher.res.mjs +0 -314
- package/src/ChainManager.res +0 -358
- package/src/ChainManager.res.mjs +0 -291
- package/src/Ctx.res +0 -6
- package/src/Ctx.res.mjs +0 -2
- package/src/GlobalState.res +0 -1056
- package/src/GlobalState.res.mjs +0 -1086
- package/src/GlobalStateManager.res +0 -57
- package/src/GlobalStateManager.res.mjs +0 -68
- package/src/GlobalStateManager.resi +0 -7
- package/src/Ports.res +0 -5
- package/src/Ports.res.mjs +0 -9
- package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
- package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
- package/src/sources/EnvioApiClient.res +0 -15
- package/src/sources/EnvioApiClient.res.mjs +0 -24
|
@@ -0,0 +1,547 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Env from "./Env.res.mjs";
|
|
4
|
+
import * as Utils from "./Utils.res.mjs";
|
|
5
|
+
import * as Config from "./Config.res.mjs";
|
|
6
|
+
import * as Logging from "./Logging.res.mjs";
|
|
7
|
+
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
|
+
import * as Internal from "./Internal.res.mjs";
|
|
9
|
+
import * as Throttler from "./Throttler.res.mjs";
|
|
10
|
+
import * as ChainState from "./ChainState.res.mjs";
|
|
11
|
+
import * as Prometheus from "./Prometheus.res.mjs";
|
|
12
|
+
import * as LoadManager from "./LoadManager.res.mjs";
|
|
13
|
+
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
14
|
+
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
15
|
+
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
16
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
17
|
+
import * as CrossChainState from "./CrossChainState.res.mjs";
|
|
18
|
+
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
19
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
20
|
+
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
21
|
+
|
|
22
|
+
function make() {
|
|
23
|
+
let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
|
|
24
|
+
let logger = Logging.createChild({
|
|
25
|
+
context: "Throttler for pruning stale entity history data",
|
|
26
|
+
intervalMillis: intervalMillis
|
|
27
|
+
});
|
|
28
|
+
let pruneStaleEntityHistory = Throttler.make(intervalMillis, logger);
|
|
29
|
+
return {
|
|
30
|
+
pruneStaleEntityHistory: pruneStaleEntityHistory
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("IndexerState.EntityTables.UndefinedEntity");
|
|
35
|
+
|
|
36
|
+
function make$1(entities) {
|
|
37
|
+
let init = {};
|
|
38
|
+
entities.forEach(entityConfig => {
|
|
39
|
+
init[entityConfig.name] = InMemoryTable.Entity.make();
|
|
40
|
+
});
|
|
41
|
+
return init;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function get(self, entityName) {
|
|
45
|
+
let table = self[entityName];
|
|
46
|
+
if (table !== undefined) {
|
|
47
|
+
return table;
|
|
48
|
+
} else {
|
|
49
|
+
return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
|
|
50
|
+
RE_EXN_ID: UndefinedEntity,
|
|
51
|
+
entityName: entityName
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrencyOpt, maxRealtimeConcurrencyOpt, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
|
|
57
|
+
let maxBackfillConcurrency = maxBackfillConcurrencyOpt !== undefined ? maxBackfillConcurrencyOpt : Env.maxBackfillConcurrency;
|
|
58
|
+
let maxRealtimeConcurrency = maxRealtimeConcurrencyOpt !== undefined ? maxRealtimeConcurrencyOpt : Env.maxRealtimeConcurrency;
|
|
59
|
+
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
|
|
60
|
+
let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
|
|
61
|
+
let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
|
|
62
|
+
let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
|
|
63
|
+
let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
|
|
64
|
+
let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
|
|
65
|
+
let chainMetaThrottler = Throttler.make(intervalMillis, Logging.createChild({
|
|
66
|
+
context: "Throttler for chain metadata writes",
|
|
67
|
+
intervalMillis: intervalMillis
|
|
68
|
+
}));
|
|
69
|
+
return {
|
|
70
|
+
config: config,
|
|
71
|
+
persistence: persistence,
|
|
72
|
+
allEntities: persistence.allEntities,
|
|
73
|
+
entities: make$1(persistence.allEntities),
|
|
74
|
+
effects: {},
|
|
75
|
+
rollback: undefined,
|
|
76
|
+
committedCheckpointId: committedCheckpointId,
|
|
77
|
+
processedCheckpointId: committedCheckpointId,
|
|
78
|
+
processedBatches: [],
|
|
79
|
+
processedBatchesCount: 0,
|
|
80
|
+
writeFiber: undefined,
|
|
81
|
+
hasFailedWrite: false,
|
|
82
|
+
commitWaiters: [],
|
|
83
|
+
chainMeta: {},
|
|
84
|
+
chainMetaDirty: false,
|
|
85
|
+
chainMetaThrottler: chainMetaThrottler,
|
|
86
|
+
isProcessing: false,
|
|
87
|
+
crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrency, maxRealtimeConcurrency, targetBufferSize),
|
|
88
|
+
rollbackState: "NoRollback",
|
|
89
|
+
indexerStartTime: new Date(),
|
|
90
|
+
writeThrottlers: make(),
|
|
91
|
+
loadManager: LoadManager.make(),
|
|
92
|
+
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
93
|
+
exitAfterFirstEventBlock: exitAfterFirstEventBlock,
|
|
94
|
+
onError: onError,
|
|
95
|
+
isStopped: false,
|
|
96
|
+
epoch: 0
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function makeFromDbState(config, persistence, initialState, registrations, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, reducedPollingInterval, maxBackfillConcurrencyOpt, maxRealtimeConcurrencyOpt, targetBufferSizeOpt, onError) {
|
|
101
|
+
let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
|
|
102
|
+
let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
|
|
103
|
+
let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
|
|
104
|
+
let maxBackfillConcurrency = maxBackfillConcurrencyOpt !== undefined ? maxBackfillConcurrencyOpt : Env.maxBackfillConcurrency;
|
|
105
|
+
let maxRealtimeConcurrency = maxRealtimeConcurrencyOpt !== undefined ? maxRealtimeConcurrencyOpt : Env.maxRealtimeConcurrency;
|
|
106
|
+
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
|
|
107
|
+
let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(chain => {
|
|
108
|
+
let progressBlockNumber = chain.progressBlockNumber;
|
|
109
|
+
let sourceBlockNumber = chain.sourceBlockNumber;
|
|
110
|
+
let maxReorgDepth = chain.maxReorgDepth;
|
|
111
|
+
if (maxReorgDepth > 0 && sourceBlockNumber > 0) {
|
|
112
|
+
return progressBlockNumber > (sourceBlockNumber - maxReorgDepth | 0);
|
|
113
|
+
} else {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
Prometheus.ProcessingMaxBatchSize.set(config.batchSize);
|
|
118
|
+
Prometheus.ReorgThreshold.set(isInReorgThreshold);
|
|
119
|
+
Utils.Dict.forEach(initialState.cache, param => {
|
|
120
|
+
let count = param.count;
|
|
121
|
+
Prometheus.EffectCacheCount.set(count, param.effectName);
|
|
122
|
+
});
|
|
123
|
+
let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
|
|
124
|
+
let chainStates = {};
|
|
125
|
+
initialState.chains.forEach(resumedChainState => {
|
|
126
|
+
let chain = Config.getChain(config, resumedChainState.id);
|
|
127
|
+
let chainConfig = ChainMap.get(config.chainMap, chain);
|
|
128
|
+
chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, reducedPollingInterval);
|
|
129
|
+
});
|
|
130
|
+
let allChainsReady = {
|
|
131
|
+
contents: initialState.chains.length !== 0
|
|
132
|
+
};
|
|
133
|
+
Utils.Dict.forEach(chainStates, cs => {
|
|
134
|
+
let chainId = ChainState.chainConfig(cs).id;
|
|
135
|
+
Prometheus.ProgressBlockNumber.set(ChainState.committedProgressBlockNumber(cs), chainId);
|
|
136
|
+
Prometheus.ProgressReady.init(chainId);
|
|
137
|
+
if (ChainState.isReady(cs)) {
|
|
138
|
+
return Prometheus.ProgressReady.set(chainId);
|
|
139
|
+
} else {
|
|
140
|
+
allChainsReady.contents = false;
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
});
|
|
144
|
+
if (allChainsReady.contents) {
|
|
145
|
+
Prometheus.ProgressReady.setAllReady();
|
|
146
|
+
}
|
|
147
|
+
return make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime, maxBackfillConcurrency, maxRealtimeConcurrency, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function isStale(state, stateId) {
|
|
151
|
+
if (state.isStopped) {
|
|
152
|
+
return true;
|
|
153
|
+
} else {
|
|
154
|
+
return stateId !== state.epoch;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function isResolvingReorg(state) {
|
|
159
|
+
let match = state.rollbackState;
|
|
160
|
+
if (typeof match !== "object") {
|
|
161
|
+
return match !== "NoRollback";
|
|
162
|
+
} else {
|
|
163
|
+
return match.TAG !== "RollbackReady";
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function errorExit(state, errHandler) {
|
|
168
|
+
if (!state.isStopped) {
|
|
169
|
+
state.isStopped = true;
|
|
170
|
+
return state.onError(errHandler);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function stop(state) {
|
|
175
|
+
state.isStopped = true;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function getChainState(state, chain) {
|
|
179
|
+
let cs = CrossChainState.chainStates(state.crossChainState)[chain];
|
|
180
|
+
if (cs !== undefined) {
|
|
181
|
+
return Primitive_option.valFromOption(cs);
|
|
182
|
+
} else {
|
|
183
|
+
return Stdlib_JsError.throwWithMessage("No chain with id " + ChainMap.Chain.toString(chain) + " found in chain states");
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function getSafeCheckpointId(state) {
|
|
188
|
+
return CrossChainState.getSafeCheckpointId(state.crossChainState);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function createBatch(state, processedCheckpointId, batchSizeTarget, isRollback) {
|
|
192
|
+
return CrossChainState.createBatch(state.crossChainState, processedCheckpointId, batchSizeTarget, isRollback);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function enterReorgThreshold(state) {
|
|
196
|
+
CrossChainState.enterReorgThreshold(state.crossChainState);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function beginReorg(state, chain, blockNumber) {
|
|
200
|
+
state.epoch = state.epoch + 1 | 0;
|
|
201
|
+
state.rollbackState = {
|
|
202
|
+
TAG: "ReorgDetected",
|
|
203
|
+
chain: chain,
|
|
204
|
+
blockNumber: blockNumber
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function enterFindingReorgDepth(state) {
|
|
209
|
+
state.rollbackState = "FindingReorgDepth";
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function foundReorgDepth(state, chain, rollbackTargetBlockNumber) {
|
|
213
|
+
state.rollbackState = {
|
|
214
|
+
TAG: "FoundReorgDepth",
|
|
215
|
+
chain: chain,
|
|
216
|
+
rollbackTargetBlockNumber: rollbackTargetBlockNumber
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function completeRollback(state, eventsProcessedDiffByChain) {
|
|
221
|
+
state.rollbackState = {
|
|
222
|
+
TAG: "RollbackReady",
|
|
223
|
+
eventsProcessedDiffByChain: eventsProcessedDiffByChain
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function clearRollback(state) {
|
|
228
|
+
state.rollbackState = "NoRollback";
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function invalidateInflight(state) {
|
|
232
|
+
state.epoch = state.epoch + 1 | 0;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function applyBatchProgress(state, batch) {
|
|
236
|
+
CrossChainState.applyBatchProgress(state.crossChainState, batch);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function isProcessing(state) {
|
|
240
|
+
return state.isProcessing;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function beginProcessing(state) {
|
|
244
|
+
state.isProcessing = true;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function endProcessing(state) {
|
|
248
|
+
state.isProcessing = false;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function recordProcessedBatch(state) {
|
|
252
|
+
state.processedBatchesCount = state.processedBatchesCount + 1 | 0;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function config(state) {
|
|
256
|
+
return state.config;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function persistence(state) {
|
|
260
|
+
return state.persistence;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function allEntities(state) {
|
|
264
|
+
return state.allEntities;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function entities(state) {
|
|
268
|
+
return state.entities;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function effects(state) {
|
|
272
|
+
return state.effects;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function committedCheckpointId(state) {
|
|
276
|
+
return state.committedCheckpointId;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function processedCheckpointId(state) {
|
|
280
|
+
return state.processedCheckpointId;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function processedBatches(state) {
|
|
284
|
+
return state.processedBatches;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function processedBatchesCount(state) {
|
|
288
|
+
return state.processedBatchesCount;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function writeFiber(state) {
|
|
292
|
+
return state.writeFiber;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function hasFailedWrite(state) {
|
|
296
|
+
return state.hasFailedWrite;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function chainMetaDirty(state) {
|
|
300
|
+
return state.chainMetaDirty;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
function chainMetaThrottler(state) {
|
|
304
|
+
return state.chainMetaThrottler;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function crossChainState(state) {
|
|
308
|
+
return state.crossChainState;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function chainStates(state) {
|
|
312
|
+
return CrossChainState.chainStates(state.crossChainState);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function isInReorgThreshold(state) {
|
|
316
|
+
return CrossChainState.isInReorgThreshold(state.crossChainState);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
function isRealtime(state) {
|
|
320
|
+
return CrossChainState.isRealtime(state.crossChainState);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function rollbackState(state) {
|
|
324
|
+
return state.rollbackState;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function indexerStartTime(state) {
|
|
328
|
+
return state.indexerStartTime;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function loadManager(state) {
|
|
332
|
+
return state.loadManager;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function keepProcessAlive(state) {
|
|
336
|
+
return state.keepProcessAlive;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function exitAfterFirstEventBlock(state) {
|
|
340
|
+
return state.exitAfterFirstEventBlock;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function isStopped(state) {
|
|
344
|
+
return state.isStopped;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function epoch(state) {
|
|
348
|
+
return state.epoch;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
function pruneStaleEntityHistoryThrottler(state) {
|
|
352
|
+
return state.writeThrottlers.pruneStaleEntityHistory;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
function queueProcessedBatch(state, batch) {
|
|
356
|
+
state.processedBatches.push(batch);
|
|
357
|
+
let checkpointId = Utils.$$Array.last(batch.checkpointIds);
|
|
358
|
+
if (checkpointId !== undefined) {
|
|
359
|
+
state.processedCheckpointId = checkpointId;
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
function drainBatchRun(state) {
|
|
365
|
+
let all = state.processedBatches;
|
|
366
|
+
let isInReorgThreshold = all[0].isInReorgThreshold;
|
|
367
|
+
let rest = [];
|
|
368
|
+
let progressedChainsById = {};
|
|
369
|
+
let totalBatchSize = {
|
|
370
|
+
contents: 0
|
|
371
|
+
};
|
|
372
|
+
let items = [];
|
|
373
|
+
let checkpointIds = [];
|
|
374
|
+
let checkpointChainIds = [];
|
|
375
|
+
let checkpointBlockNumbers = [];
|
|
376
|
+
let checkpointBlockHashes = [];
|
|
377
|
+
let checkpointEventsProcessed = [];
|
|
378
|
+
all.forEach(batch => {
|
|
379
|
+
if (Utils.$$Array.isEmpty(rest) && batch.isInReorgThreshold === isInReorgThreshold) {
|
|
380
|
+
Utils.Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, key) => {
|
|
381
|
+
progressedChainsById[key] = chainAfterBatch;
|
|
382
|
+
});
|
|
383
|
+
totalBatchSize.contents = totalBatchSize.contents + batch.totalBatchSize | 0;
|
|
384
|
+
items.push(...batch.items);
|
|
385
|
+
checkpointIds.push(...batch.checkpointIds);
|
|
386
|
+
checkpointChainIds.push(...batch.checkpointChainIds);
|
|
387
|
+
checkpointBlockNumbers.push(...batch.checkpointBlockNumbers);
|
|
388
|
+
checkpointBlockHashes.push(...batch.checkpointBlockHashes);
|
|
389
|
+
checkpointEventsProcessed.push(...batch.checkpointEventsProcessed);
|
|
390
|
+
} else {
|
|
391
|
+
rest.push(batch);
|
|
392
|
+
}
|
|
393
|
+
});
|
|
394
|
+
state.processedBatches = rest;
|
|
395
|
+
return {
|
|
396
|
+
totalBatchSize: totalBatchSize.contents,
|
|
397
|
+
items: items,
|
|
398
|
+
progressedChainsById: progressedChainsById,
|
|
399
|
+
isInReorgThreshold: isInReorgThreshold,
|
|
400
|
+
checkpointIds: checkpointIds,
|
|
401
|
+
checkpointChainIds: checkpointChainIds,
|
|
402
|
+
checkpointBlockNumbers: checkpointBlockNumbers,
|
|
403
|
+
checkpointBlockHashes: checkpointBlockHashes,
|
|
404
|
+
checkpointEventsProcessed: checkpointEventsProcessed
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function takeRollback(state) {
|
|
409
|
+
let rollback = state.rollback;
|
|
410
|
+
state.rollback = undefined;
|
|
411
|
+
return rollback;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function markCommitted(state, upToCheckpointId) {
|
|
415
|
+
state.committedCheckpointId = upToCheckpointId;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
|
|
419
|
+
state.entities = make$1(state.allEntities);
|
|
420
|
+
state.effects = {};
|
|
421
|
+
state.rollback = {
|
|
422
|
+
targetCheckpointId: targetCheckpointId,
|
|
423
|
+
diffCheckpointId: diffCheckpointId,
|
|
424
|
+
progressBlockNumberByChainId: progressBlockNumberByChainId
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function recordWriteFailure(state, exn) {
|
|
429
|
+
state.hasFailedWrite = true;
|
|
430
|
+
state.onError(ErrorHandling.make(exn, undefined, "Failed writing batch to the database"));
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function beginWriteFiber(state, fiber) {
|
|
434
|
+
state.writeFiber = fiber;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
function endWriteFiber(state) {
|
|
438
|
+
state.writeFiber = undefined;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function wakeCommitWaiters(state) {
|
|
442
|
+
let waiters = state.commitWaiters;
|
|
443
|
+
state.commitWaiters = [];
|
|
444
|
+
waiters.forEach(resolve => resolve());
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function addCommitWaiter(state, resolve) {
|
|
448
|
+
state.commitWaiters.push(resolve);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function metaFieldsEqual(a, b) {
|
|
452
|
+
if (Primitive_object.equal(a.first_event_block, b.first_event_block) && a.buffer_block === b.buffer_block && a._is_hyper_sync === b._is_hyper_sync) {
|
|
453
|
+
return Primitive_object.equal(Stdlib_Option.map(Primitive_option.fromNull(a.ready_at), prim => prim.getTime()), Stdlib_Option.map(Primitive_option.fromNull(b.ready_at), prim => prim.getTime()));
|
|
454
|
+
} else {
|
|
455
|
+
return false;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
function stageChainMeta(state, chainsData) {
|
|
460
|
+
Utils.Dict.forEachWithKey(chainsData, (meta, chainId) => {
|
|
461
|
+
let prev = state.chainMeta[chainId];
|
|
462
|
+
let changed = prev !== undefined ? !metaFieldsEqual(meta, prev) : true;
|
|
463
|
+
if (changed) {
|
|
464
|
+
state.chainMeta[chainId] = meta;
|
|
465
|
+
state.chainMetaDirty = true;
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function takeChainMetaSnapshot(state) {
|
|
472
|
+
if (state.chainMetaDirty) {
|
|
473
|
+
state.chainMetaDirty = false;
|
|
474
|
+
return Utils.Dict.shallowCopy(state.chainMeta);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
let EntityTables = {
|
|
479
|
+
make: make$1,
|
|
480
|
+
get: get
|
|
481
|
+
};
|
|
482
|
+
|
|
483
|
+
let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
|
|
484
|
+
|
|
485
|
+
export {
|
|
486
|
+
EntityTables,
|
|
487
|
+
make$2 as make,
|
|
488
|
+
makeFromDbState,
|
|
489
|
+
unexpectedErrorMsg,
|
|
490
|
+
isStale,
|
|
491
|
+
isResolvingReorg,
|
|
492
|
+
errorExit,
|
|
493
|
+
stop,
|
|
494
|
+
getChainState,
|
|
495
|
+
enterReorgThreshold,
|
|
496
|
+
beginReorg,
|
|
497
|
+
enterFindingReorgDepth,
|
|
498
|
+
foundReorgDepth,
|
|
499
|
+
completeRollback,
|
|
500
|
+
clearRollback,
|
|
501
|
+
invalidateInflight,
|
|
502
|
+
applyBatchProgress,
|
|
503
|
+
isProcessing,
|
|
504
|
+
beginProcessing,
|
|
505
|
+
endProcessing,
|
|
506
|
+
recordProcessedBatch,
|
|
507
|
+
createBatch,
|
|
508
|
+
getSafeCheckpointId,
|
|
509
|
+
config,
|
|
510
|
+
persistence,
|
|
511
|
+
allEntities,
|
|
512
|
+
entities,
|
|
513
|
+
effects,
|
|
514
|
+
committedCheckpointId,
|
|
515
|
+
processedCheckpointId,
|
|
516
|
+
processedBatches,
|
|
517
|
+
processedBatchesCount,
|
|
518
|
+
writeFiber,
|
|
519
|
+
hasFailedWrite,
|
|
520
|
+
chainMetaDirty,
|
|
521
|
+
chainMetaThrottler,
|
|
522
|
+
crossChainState,
|
|
523
|
+
chainStates,
|
|
524
|
+
isInReorgThreshold,
|
|
525
|
+
isRealtime,
|
|
526
|
+
rollbackState,
|
|
527
|
+
indexerStartTime,
|
|
528
|
+
loadManager,
|
|
529
|
+
keepProcessAlive,
|
|
530
|
+
exitAfterFirstEventBlock,
|
|
531
|
+
isStopped,
|
|
532
|
+
epoch,
|
|
533
|
+
pruneStaleEntityHistoryThrottler,
|
|
534
|
+
queueProcessedBatch,
|
|
535
|
+
drainBatchRun,
|
|
536
|
+
takeRollback,
|
|
537
|
+
markCommitted,
|
|
538
|
+
beginRollbackDiff,
|
|
539
|
+
recordWriteFailure,
|
|
540
|
+
beginWriteFiber,
|
|
541
|
+
endWriteFiber,
|
|
542
|
+
wakeCommitWaiters,
|
|
543
|
+
addCommitWaiter,
|
|
544
|
+
stageChainMeta,
|
|
545
|
+
takeChainMetaSnapshot,
|
|
546
|
+
}
|
|
547
|
+
/* Env Not a pure module */
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// The indexer state. `t` is opaque: other modules read it through the accessors
|
|
2
|
+
// and change it only through the transitions and setters exposed here.
|
|
3
|
+
|
|
4
|
+
type chain = ChainMap.Chain.t
|
|
5
|
+
|
|
6
|
+
type rollbackState =
|
|
7
|
+
| NoRollback
|
|
8
|
+
| ReorgDetected({chain: chain, blockNumber: int})
|
|
9
|
+
| FindingReorgDepth
|
|
10
|
+
| FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
|
|
11
|
+
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
12
|
+
|
|
13
|
+
module EntityTables: {
|
|
14
|
+
type t = dict<InMemoryTable.Entity.t>
|
|
15
|
+
let make: array<Internal.entityConfig> => t
|
|
16
|
+
let get: (t, ~entityName: string) => InMemoryTable.Entity.t
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
type effectCacheInMemTable = {
|
|
20
|
+
mutable idsToStore: array<string>,
|
|
21
|
+
mutable invalidationsCount: int,
|
|
22
|
+
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
23
|
+
mutable changesCount: float,
|
|
24
|
+
effect: Internal.effect,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
type t
|
|
28
|
+
|
|
29
|
+
let make: (
|
|
30
|
+
~config: Config.t,
|
|
31
|
+
~persistence: Persistence.t,
|
|
32
|
+
~chainStates: dict<ChainState.t>,
|
|
33
|
+
~isInReorgThreshold: bool,
|
|
34
|
+
~isRealtime: bool,
|
|
35
|
+
~maxBackfillConcurrency: int=?,
|
|
36
|
+
~maxRealtimeConcurrency: int=?,
|
|
37
|
+
~targetBufferSize: int=?,
|
|
38
|
+
~committedCheckpointId: Internal.checkpointId=?,
|
|
39
|
+
~isDevelopmentMode: bool=?,
|
|
40
|
+
~shouldUseTui: bool=?,
|
|
41
|
+
~exitAfterFirstEventBlock: bool=?,
|
|
42
|
+
~onError: ErrorHandling.t => unit,
|
|
43
|
+
) => t
|
|
44
|
+
|
|
45
|
+
let makeFromDbState: (
|
|
46
|
+
~config: Config.t,
|
|
47
|
+
~persistence: Persistence.t,
|
|
48
|
+
~initialState: Persistence.initialState,
|
|
49
|
+
~registrations: HandlerRegister.registrations,
|
|
50
|
+
~isDevelopmentMode: bool=?,
|
|
51
|
+
~shouldUseTui: bool=?,
|
|
52
|
+
~exitAfterFirstEventBlock: bool=?,
|
|
53
|
+
~reducedPollingInterval: int=?,
|
|
54
|
+
~maxBackfillConcurrency: int=?,
|
|
55
|
+
~maxRealtimeConcurrency: int=?,
|
|
56
|
+
~targetBufferSize: int=?,
|
|
57
|
+
~onError: ErrorHandling.t => unit,
|
|
58
|
+
) => t
|
|
59
|
+
|
|
60
|
+
let unexpectedErrorMsg: string
|
|
61
|
+
|
|
62
|
+
// Read guards / transitions.
|
|
63
|
+
let isStale: (t, ~stateId: int) => bool
|
|
64
|
+
let isResolvingReorg: t => bool
|
|
65
|
+
let errorExit: (t, ErrorHandling.t) => unit
|
|
66
|
+
let stop: t => unit
|
|
67
|
+
let getChainState: (t, ~chain: chain) => ChainState.t
|
|
68
|
+
let enterReorgThreshold: t => unit
|
|
69
|
+
let beginReorg: (t, ~chain: chain, ~blockNumber: int) => unit
|
|
70
|
+
let enterFindingReorgDepth: t => unit
|
|
71
|
+
let foundReorgDepth: (t, ~chain: chain, ~rollbackTargetBlockNumber: int) => unit
|
|
72
|
+
let completeRollback: (t, ~eventsProcessedDiffByChain: dict<float>) => unit
|
|
73
|
+
let clearRollback: t => unit
|
|
74
|
+
let invalidateInflight: t => unit
|
|
75
|
+
let applyBatchProgress: (t, ~batch: Batch.t) => unit
|
|
76
|
+
let isProcessing: t => bool
|
|
77
|
+
let beginProcessing: t => unit
|
|
78
|
+
let endProcessing: t => unit
|
|
79
|
+
let recordProcessedBatch: t => unit
|
|
80
|
+
let createBatch: (
|
|
81
|
+
t,
|
|
82
|
+
~processedCheckpointId: Internal.checkpointId,
|
|
83
|
+
~batchSizeTarget: int,
|
|
84
|
+
~isRollback: bool,
|
|
85
|
+
) => Batch.t
|
|
86
|
+
let getSafeCheckpointId: t => option<Internal.checkpointId>
|
|
87
|
+
|
|
88
|
+
// Accessors.
|
|
89
|
+
let config: t => Config.t
|
|
90
|
+
let persistence: t => Persistence.t
|
|
91
|
+
let allEntities: t => array<Internal.entityConfig>
|
|
92
|
+
let entities: t => EntityTables.t
|
|
93
|
+
let effects: t => dict<effectCacheInMemTable>
|
|
94
|
+
let committedCheckpointId: t => Internal.checkpointId
|
|
95
|
+
let processedCheckpointId: t => Internal.checkpointId
|
|
96
|
+
let processedBatches: t => array<Batch.t>
|
|
97
|
+
let processedBatchesCount: t => int
|
|
98
|
+
let writeFiber: t => option<promise<unit>>
|
|
99
|
+
let hasFailedWrite: t => bool
|
|
100
|
+
let chainMetaDirty: t => bool
|
|
101
|
+
let chainMetaThrottler: t => Throttler.t
|
|
102
|
+
let crossChainState: t => CrossChainState.t
|
|
103
|
+
let chainStates: t => dict<ChainState.t>
|
|
104
|
+
let isInReorgThreshold: t => bool
|
|
105
|
+
let isRealtime: t => bool
|
|
106
|
+
let rollbackState: t => rollbackState
|
|
107
|
+
let indexerStartTime: t => Date.t
|
|
108
|
+
let loadManager: t => LoadManager.t
|
|
109
|
+
let keepProcessAlive: t => bool
|
|
110
|
+
let exitAfterFirstEventBlock: t => bool
|
|
111
|
+
let isStopped: t => bool
|
|
112
|
+
let epoch: t => int
|
|
113
|
+
let pruneStaleEntityHistoryThrottler: t => Throttler.t
|
|
114
|
+
|
|
115
|
+
// Store domain operations.
|
|
116
|
+
let queueProcessedBatch: (t, ~batch: Batch.t) => unit
|
|
117
|
+
let drainBatchRun: t => Batch.t
|
|
118
|
+
let takeRollback: t => option<Persistence.rollback>
|
|
119
|
+
let markCommitted: (t, ~upToCheckpointId: Internal.checkpointId) => unit
|
|
120
|
+
let beginRollbackDiff: (
|
|
121
|
+
t,
|
|
122
|
+
~targetCheckpointId: Internal.checkpointId,
|
|
123
|
+
~diffCheckpointId: Internal.checkpointId,
|
|
124
|
+
~progressBlockNumberByChainId: dict<int>,
|
|
125
|
+
) => unit
|
|
126
|
+
let recordWriteFailure: (t, exn) => unit
|
|
127
|
+
let beginWriteFiber: (t, promise<unit>) => unit
|
|
128
|
+
let endWriteFiber: t => unit
|
|
129
|
+
let wakeCommitWaiters: t => unit
|
|
130
|
+
let addCommitWaiter: (t, unit => unit) => unit
|
|
131
|
+
let stageChainMeta: (t, dict<InternalTable.Chains.metaFields>) => unit
|
|
132
|
+
let takeChainMetaSnapshot: t => option<dict<InternalTable.Chains.metaFields>>
|