envio 3.3.0-alpha.1 → 3.3.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +10 -0
- package/index.d.ts +45 -25
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +10 -9
- package/src/Address.res +5 -2
- package/src/Address.res.mjs +3 -1
- package/src/Api.res +1 -1
- package/src/Api.res.mjs +1 -1
- package/src/Batch.res +0 -6
- package/src/Batch.res.mjs +1 -12
- package/src/BatchProcessing.res +7 -12
- package/src/BatchProcessing.res.mjs +8 -7
- package/src/Bin.res +3 -0
- package/src/Bin.res.mjs +4 -0
- package/src/ChainFetching.res +53 -16
- package/src/ChainFetching.res.mjs +37 -19
- package/src/ChainMetadata.res +1 -11
- package/src/ChainMetadata.res.mjs +1 -10
- package/src/ChainState.res +630 -161
- package/src/ChainState.res.mjs +438 -101
- package/src/ChainState.resi +77 -4
- package/src/Config.res +70 -38
- package/src/Config.res.mjs +52 -12
- package/src/ContractRegisterContext.res +2 -12
- package/src/ContractRegisterContext.res.mjs +3 -5
- package/src/Core.res +34 -3
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +130 -89
- package/src/CrossChainState.res.mjs +66 -61
- package/src/CrossChainState.resi +1 -1
- package/src/Ecosystem.res +5 -6
- package/src/Ecosystem.res.mjs +3 -3
- package/src/Envio.res +17 -11
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +191 -127
- package/src/EventConfigBuilder.res.mjs +116 -66
- package/src/EventProcessing.res +77 -28
- package/src/EventProcessing.res.mjs +56 -29
- package/src/ExitOnCaughtUp.res +10 -2
- package/src/ExitOnCaughtUp.res.mjs +10 -1
- package/src/FetchState.res +684 -415
- package/src/FetchState.res.mjs +505 -486
- package/src/HandlerLoader.res +8 -104
- package/src/HandlerLoader.res.mjs +2 -81
- package/src/HandlerRegister.res +520 -144
- package/src/HandlerRegister.res.mjs +286 -133
- package/src/HandlerRegister.resi +24 -8
- package/src/IndexerState.res +10 -3
- package/src/IndexerState.res.mjs +11 -4
- package/src/IndexerState.resi +2 -1
- package/src/IndexingAddresses.res +108 -0
- package/src/IndexingAddresses.res.mjs +101 -0
- package/src/IndexingAddresses.resi +34 -0
- package/src/Internal.res +166 -48
- package/src/Internal.res.mjs +30 -2
- package/src/LoadLayer.res +5 -5
- package/src/LoadLayer.res.mjs +6 -6
- package/src/LogSelection.res +92 -217
- package/src/LogSelection.res.mjs +95 -184
- package/src/Main.res +71 -236
- package/src/Main.res.mjs +60 -145
- package/src/Metrics.res +74 -0
- package/src/Metrics.res.mjs +75 -0
- package/src/PgStorage.res +4 -4
- package/src/PgStorage.res.mjs +5 -5
- package/src/Prometheus.res +10 -63
- package/src/Prometheus.res.mjs +124 -171
- package/src/PruneStaleHistory.res +2 -2
- package/src/PruneStaleHistory.res.mjs +3 -3
- package/src/RawEvent.res +7 -2
- package/src/RawEvent.res.mjs +4 -4
- package/src/Rollback.res +3 -3
- package/src/Rollback.res.mjs +4 -4
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateDeadInputTracker.res +85 -0
- package/src/SimulateDeadInputTracker.res.mjs +73 -0
- package/src/SimulateDeadInputTracker.resi +12 -0
- package/src/SimulateItems.res +128 -28
- package/src/SimulateItems.res.mjs +82 -40
- package/src/TestIndexer.res +55 -41
- package/src/TestIndexer.res.mjs +39 -32
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/bindings/Performance.res +7 -0
- package/src/bindings/Performance.res.mjs +21 -0
- package/src/bindings/Performance.resi +7 -0
- package/src/bindings/Viem.res +0 -46
- package/src/bindings/Viem.res.mjs +1 -43
- package/src/sources/BlockStore.res +46 -0
- package/src/sources/BlockStore.res.mjs +24 -0
- package/src/sources/EventRouter.res +31 -49
- package/src/sources/EventRouter.res.mjs +12 -25
- package/src/sources/Evm.res +83 -72
- package/src/sources/Evm.res.mjs +53 -69
- package/src/sources/EvmChain.res +7 -34
- package/src/sources/EvmChain.res.mjs +6 -27
- package/src/sources/EvmRpcClient.res +71 -5
- package/src/sources/EvmRpcClient.res.mjs +13 -4
- package/src/sources/FieldMask.res +39 -0
- package/src/sources/FieldMask.res.mjs +42 -0
- package/src/sources/Fuel.res +9 -6
- package/src/sources/Fuel.res.mjs +5 -10
- package/src/sources/HyperFuelSource.res +46 -50
- package/src/sources/HyperFuelSource.res.mjs +64 -54
- package/src/sources/HyperSync.res +45 -42
- package/src/sources/HyperSync.res.mjs +68 -60
- package/src/sources/HyperSync.resi +14 -4
- package/src/sources/HyperSyncClient.res +107 -119
- package/src/sources/HyperSyncClient.res.mjs +36 -31
- package/src/sources/HyperSyncHeightStream.res +1 -8
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
- package/src/sources/HyperSyncSource.res +60 -254
- package/src/sources/HyperSyncSource.res.mjs +57 -180
- package/src/sources/Rpc.res +15 -79
- package/src/sources/Rpc.res.mjs +25 -101
- package/src/sources/RpcSource.res +221 -514
- package/src/sources/RpcSource.res.mjs +216 -411
- package/src/sources/RpcWebSocketHeightStream.res +0 -5
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
- package/src/sources/SimulateSource.res +44 -20
- package/src/sources/SimulateSource.res.mjs +41 -14
- package/src/sources/Source.res +34 -6
- package/src/sources/SourceManager.res +77 -19
- package/src/sources/SourceManager.res.mjs +63 -21
- package/src/sources/SourceManager.resi +10 -0
- package/src/sources/Svm.res +31 -16
- package/src/sources/Svm.res.mjs +36 -13
- package/src/sources/SvmHyperSyncClient.res +5 -32
- package/src/sources/SvmHyperSyncSource.res +159 -134
- package/src/sources/SvmHyperSyncSource.res.mjs +164 -129
- package/src/sources/TransactionStore.res +49 -0
- package/src/sources/TransactionStore.res.mjs +30 -0
- package/src/tui/Tui.res +13 -16
- package/src/tui/Tui.res.mjs +12 -14
- package/svm.schema.json +49 -37
- package/src/bindings/Hrtime.res +0 -58
- package/src/bindings/Hrtime.res.mjs +0 -90
- package/src/bindings/Hrtime.resi +0 -30
package/src/Main.res.mjs
CHANGED
|
@@ -6,13 +6,14 @@ import * as Envio from "./Envio.res.mjs";
|
|
|
6
6
|
import * as Utils from "./Utils.res.mjs";
|
|
7
7
|
import * as Config from "./Config.res.mjs";
|
|
8
8
|
import * as Logging from "./Logging.res.mjs";
|
|
9
|
+
import * as Metrics from "./Metrics.res.mjs";
|
|
9
10
|
import Express from "express";
|
|
10
11
|
import * as Process from "process";
|
|
11
12
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
12
13
|
import * as PgStorage from "./PgStorage.res.mjs";
|
|
13
14
|
import * as ChainState from "./ChainState.res.mjs";
|
|
14
|
-
import * as FetchState from "./FetchState.res.mjs";
|
|
15
15
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
16
|
+
import * as EnvioGlobal from "./EnvioGlobal.res.mjs";
|
|
16
17
|
import * as IndexerLoop from "./IndexerLoop.res.mjs";
|
|
17
18
|
import * as Persistence from "./Persistence.res.mjs";
|
|
18
19
|
import * as PromClient from "prom-client";
|
|
@@ -20,8 +21,6 @@ import Yargs from "yargs/yargs";
|
|
|
20
21
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
21
22
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
22
23
|
import * as HandlerLoader from "./HandlerLoader.res.mjs";
|
|
23
|
-
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
24
|
-
import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
25
24
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
26
25
|
import * as Helpers from "yargs/helpers";
|
|
27
26
|
import * as RollbackCommit from "./RollbackCommit.res.mjs";
|
|
@@ -63,28 +62,24 @@ let stateSchema = S$RescriptSchema.union([
|
|
|
63
62
|
}))
|
|
64
63
|
]);
|
|
65
64
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
_every: s.f("_every", S$RescriptSchema.Option.getOr(S$RescriptSchema.option(S$RescriptSchema.intMin(S$RescriptSchema.int, 1, undefined)), 1))
|
|
70
|
-
})));
|
|
65
|
+
function getIndexerState() {
|
|
66
|
+
return EnvioGlobal.value.indexerState;
|
|
67
|
+
}
|
|
71
68
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
_every: 1
|
|
76
|
-
};
|
|
69
|
+
function setIndexerState(state) {
|
|
70
|
+
EnvioGlobal.value.indexerState = Primitive_option.some(state);
|
|
71
|
+
}
|
|
77
72
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
73
|
+
function getGlobalPersistence() {
|
|
74
|
+
return EnvioGlobal.value.persistence;
|
|
75
|
+
}
|
|
81
76
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
}
|
|
77
|
+
function setGlobalPersistence(persistence) {
|
|
78
|
+
EnvioGlobal.value.persistence = Primitive_option.some(persistence);
|
|
79
|
+
}
|
|
85
80
|
|
|
86
81
|
function getInitialChainState(chainId) {
|
|
87
|
-
let persistence =
|
|
82
|
+
let persistence = EnvioGlobal.value.persistence;
|
|
88
83
|
if (persistence === undefined) {
|
|
89
84
|
return;
|
|
90
85
|
}
|
|
@@ -136,7 +131,7 @@ function buildChainsObject(config) {
|
|
|
136
131
|
}), "isRealtime", {
|
|
137
132
|
enumerable: true,
|
|
138
133
|
get: () => {
|
|
139
|
-
let state =
|
|
134
|
+
let state = EnvioGlobal.value.indexerState;
|
|
140
135
|
if (state !== undefined) {
|
|
141
136
|
return IndexerState.isRealtime(Primitive_option.valFromOption(state));
|
|
142
137
|
} else if (Env.updateSyncTimeOnRestart) {
|
|
@@ -164,33 +159,24 @@ function buildChainsObject(config) {
|
|
|
164
159
|
}), "addresses", {
|
|
165
160
|
enumerable: true,
|
|
166
161
|
get: () => {
|
|
167
|
-
let state =
|
|
162
|
+
let state = EnvioGlobal.value.indexerState;
|
|
168
163
|
if (state !== undefined) {
|
|
169
164
|
let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
|
|
170
165
|
let chainState = IndexerState.getChainState(Primitive_option.valFromOption(state), chain);
|
|
171
|
-
|
|
172
|
-
let addresses = [];
|
|
173
|
-
let values = Object.values(indexingAddresses);
|
|
174
|
-
for (let idx = 0, idx_finish = values.length; idx < idx_finish; ++idx) {
|
|
175
|
-
let indexingContract = values[idx];
|
|
176
|
-
if (indexingContract.contractName === contract.name) {
|
|
177
|
-
addresses.push(indexingContract.address);
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
return addresses;
|
|
166
|
+
return ChainState.contractAddresses(chainState, contract.name);
|
|
181
167
|
}
|
|
182
168
|
let chainState$1 = getInitialChainState(chainConfig.id);
|
|
183
169
|
if (chainState$1 === undefined) {
|
|
184
170
|
return contract.addresses;
|
|
185
171
|
}
|
|
186
|
-
let addresses
|
|
172
|
+
let addresses = contract.addresses.slice();
|
|
187
173
|
chainState$1.indexingAddresses.forEach(dc => {
|
|
188
174
|
if (dc.contractName === contract.name) {
|
|
189
|
-
addresses
|
|
175
|
+
addresses.push(dc.address);
|
|
190
176
|
return;
|
|
191
177
|
}
|
|
192
178
|
});
|
|
193
|
-
return addresses
|
|
179
|
+
return addresses;
|
|
194
180
|
}
|
|
195
181
|
});
|
|
196
182
|
Object.defineProperty(chainObj, contract.name, {
|
|
@@ -289,69 +275,9 @@ function getGlobalIndexer() {
|
|
|
289
275
|
HandlerRegister.throwIfFinishedRegistration("~internalAndWillBeRemovedSoon_onRollbackCommit");
|
|
290
276
|
RollbackCommit.register(callback);
|
|
291
277
|
};
|
|
292
|
-
let extractRange = (filter, name, ecosystem) => {
|
|
293
|
-
try {
|
|
294
|
-
let inner = S$RescriptSchema.parseOrThrow(filter, ecosystem.onBlockFilterSchema);
|
|
295
|
-
if (inner !== undefined) {
|
|
296
|
-
return S$RescriptSchema.parseOrThrow(Primitive_option.valFromOption(inner), blockRangeSchema);
|
|
297
|
-
} else {
|
|
298
|
-
return defaultBlockRange;
|
|
299
|
-
}
|
|
300
|
-
} catch (raw_exn) {
|
|
301
|
-
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
302
|
-
if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
|
|
303
|
-
return Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` returned an invalid filter: ` + Utils.prettifyExn(exn._1));
|
|
304
|
-
}
|
|
305
|
-
throw exn;
|
|
306
|
-
}
|
|
307
|
-
};
|
|
308
278
|
let onBlockFn = (rawOptions, handler) => {
|
|
309
279
|
HandlerRegister.throwIfFinishedRegistration("onBlock");
|
|
310
|
-
|
|
311
|
-
let ecosystem = config.ecosystem;
|
|
312
|
-
let match = buildChainsObject(config);
|
|
313
|
-
let chains = match[0];
|
|
314
|
-
let name = rawOptions.name;
|
|
315
|
-
let logger = Logging.createChild({
|
|
316
|
-
onBlock: name
|
|
317
|
-
});
|
|
318
|
-
let w = rawOptions.where;
|
|
319
|
-
let where = w === undefined || w === null ? undefined : (
|
|
320
|
-
typeof w === "function" ? rawOptions.where : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` expected \`where\` to be a function or omitted, but got ` + typeof w + `.`)
|
|
321
|
-
);
|
|
322
|
-
let matchedAny = {
|
|
323
|
-
contents: false
|
|
324
|
-
};
|
|
325
|
-
ChainMap.values(config.chainMap).forEach(chainConfig => {
|
|
326
|
-
let chainId = chainConfig.id;
|
|
327
|
-
let chainObj = chains[chainId.toString()];
|
|
328
|
-
let result = where !== undefined ? where({
|
|
329
|
-
chain: chainObj
|
|
330
|
-
}) : true;
|
|
331
|
-
let match = result === true ? [
|
|
332
|
-
true,
|
|
333
|
-
defaultBlockRange
|
|
334
|
-
] : (
|
|
335
|
-
result === false ? [
|
|
336
|
-
false,
|
|
337
|
-
defaultBlockRange
|
|
338
|
-
] : (
|
|
339
|
-
typeof result === "object" && !Array.isArray(result) && result !== null ? [
|
|
340
|
-
true,
|
|
341
|
-
extractRange(result, name, ecosystem)
|
|
342
|
-
] : Stdlib_JsError.throwWithMessage(`\`indexer.` + ecosystem.onBlockMethodName + `("` + name + `")\` \`where\` predicate returned an invalid value of type ` + typeof result + `. Expected boolean or a filter object.`)
|
|
343
|
-
)
|
|
344
|
-
);
|
|
345
|
-
if (!match[0]) {
|
|
346
|
-
return;
|
|
347
|
-
}
|
|
348
|
-
let range = match[1];
|
|
349
|
-
matchedAny.contents = true;
|
|
350
|
-
HandlerRegister.registerOnBlock(name, chainId, range._every, range._gte, range._lte, handler);
|
|
351
|
-
});
|
|
352
|
-
if (!matchedAny.contents) {
|
|
353
|
-
return Logging.childWarn(logger, `\`indexer.` + ecosystem.onBlockMethodName + `\` matched 0 chains. Check the \`where\` predicate.`);
|
|
354
|
-
}
|
|
280
|
+
HandlerRegister.registerOnBlock(rawOptions.name, rawOptions.where, handler, config => buildChainsObject(config)[0]);
|
|
355
281
|
};
|
|
356
282
|
let keysMemo = {
|
|
357
283
|
contents: undefined
|
|
@@ -361,7 +287,7 @@ function getGlobalIndexer() {
|
|
|
361
287
|
if (k !== undefined) {
|
|
362
288
|
return k;
|
|
363
289
|
}
|
|
364
|
-
let match = Config.
|
|
290
|
+
let match = Config.load().ecosystem.name;
|
|
365
291
|
let keys;
|
|
366
292
|
let exit = 0;
|
|
367
293
|
switch (match) {
|
|
@@ -399,15 +325,15 @@ function getGlobalIndexer() {
|
|
|
399
325
|
let get = prop => {
|
|
400
326
|
switch (prop) {
|
|
401
327
|
case "chainIds" :
|
|
402
|
-
return buildChainsObject(Config.
|
|
328
|
+
return buildChainsObject(Config.load())[1];
|
|
403
329
|
case "chains" :
|
|
404
|
-
return buildChainsObject(Config.
|
|
330
|
+
return buildChainsObject(Config.load())[0];
|
|
405
331
|
case "contractRegister" :
|
|
406
332
|
return contractRegisterFn;
|
|
407
333
|
case "description" :
|
|
408
|
-
return Config.
|
|
334
|
+
return Config.load().description;
|
|
409
335
|
case "name" :
|
|
410
|
-
return Config.
|
|
336
|
+
return Config.load().name;
|
|
411
337
|
case "onEvent" :
|
|
412
338
|
return onEventFn;
|
|
413
339
|
case "onInstruction" :
|
|
@@ -489,7 +415,7 @@ function startServer(getState, persistence, isDevelopmentMode) {
|
|
|
489
415
|
});
|
|
490
416
|
app.get("/metrics", (_req, res) => {
|
|
491
417
|
res.set("Content-Type", PromClient.register.contentType);
|
|
492
|
-
|
|
418
|
+
Metrics.collect(EnvioGlobal.value.indexerState).then(metrics => res.end(metrics));
|
|
493
419
|
});
|
|
494
420
|
app.get("/metrics/runtime", (_req, res) => {
|
|
495
421
|
res.set("Content-Type", runtimeRegistry.contentType);
|
|
@@ -512,19 +438,21 @@ function getEnvioInfo() {
|
|
|
512
438
|
}
|
|
513
439
|
|
|
514
440
|
async function migrate(reset) {
|
|
515
|
-
let config = Config.
|
|
441
|
+
let config = Config.load();
|
|
516
442
|
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
|
|
517
443
|
await Persistence.init(persistence, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", undefined, reset);
|
|
518
444
|
return await persistence.storage.close();
|
|
519
445
|
}
|
|
520
446
|
|
|
521
447
|
async function dropSchema() {
|
|
522
|
-
let config = Config.
|
|
448
|
+
let config = Config.load();
|
|
523
449
|
let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
|
|
524
450
|
await persistence.storage.reset();
|
|
525
451
|
return await persistence.storage.close();
|
|
526
452
|
}
|
|
527
453
|
|
|
454
|
+
let FatalError = /* @__PURE__ */Primitive_exceptions.create("Main.FatalError");
|
|
455
|
+
|
|
528
456
|
async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockOpt, patchConfig) {
|
|
529
457
|
let reset = resetOpt !== undefined ? resetOpt : false;
|
|
530
458
|
let isTest = isTestOpt !== undefined ? isTestOpt : false;
|
|
@@ -535,14 +463,12 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
535
463
|
let shouldUseTui = isTest ? false : (
|
|
536
464
|
explicitTui !== undefined ? explicitTui : !Envio.isNonInteractive()
|
|
537
465
|
);
|
|
538
|
-
let
|
|
539
|
-
let isDevelopmentMode = !isTest &&
|
|
540
|
-
let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(
|
|
541
|
-
|
|
542
|
-
await Persistence.init(persistence$1, ChainMap.values(
|
|
543
|
-
let
|
|
544
|
-
let registrations = match[1];
|
|
545
|
-
let config = match[0];
|
|
466
|
+
let config = Config.load();
|
|
467
|
+
let isDevelopmentMode = !isTest && config.isDev;
|
|
468
|
+
let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(config, undefined);
|
|
469
|
+
EnvioGlobal.value.persistence = Primitive_option.some(persistence$1);
|
|
470
|
+
await Persistence.init(persistence$1, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset);
|
|
471
|
+
let registrationsByChainId = await HandlerLoader.registerAllHandlers(config);
|
|
546
472
|
let config$1 = isTest ? ({
|
|
547
473
|
name: config.name,
|
|
548
474
|
description: config.description,
|
|
@@ -564,14 +490,19 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
564
490
|
allEntities: config.allEntities,
|
|
565
491
|
allEnums: config.allEnums
|
|
566
492
|
}) : config;
|
|
567
|
-
let config$2 = patchConfig !== undefined ? patchConfig(config$1,
|
|
493
|
+
let config$2 = patchConfig !== undefined ? patchConfig(config$1, registrationsByChainId) : config$1;
|
|
494
|
+
let onErrorReject = {
|
|
495
|
+
contents: undefined
|
|
496
|
+
};
|
|
497
|
+
let runUntilFatalError = new Promise((_resolve, reject) => {
|
|
498
|
+
onErrorReject.contents = reject;
|
|
499
|
+
});
|
|
568
500
|
let onError = errHandler => {
|
|
569
501
|
ErrorHandling.log(errHandler);
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
}
|
|
502
|
+
onErrorReject.contents({
|
|
503
|
+
RE_EXN_ID: FatalError,
|
|
504
|
+
_1: Utils.prettifyExn(errHandler.exn)
|
|
505
|
+
});
|
|
575
506
|
};
|
|
576
507
|
let envioVersion = Utils.EnvioPackage.value.version;
|
|
577
508
|
Prometheus.Info.set(envioVersion);
|
|
@@ -579,32 +510,14 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
579
510
|
Prometheus.RollbackEnabled.set(config$2.shouldRollbackOnReorg);
|
|
580
511
|
if (!isTest) {
|
|
581
512
|
startServer(() => {
|
|
582
|
-
let state =
|
|
513
|
+
let state = EnvioGlobal.value.indexerState;
|
|
583
514
|
if (state === undefined) {
|
|
584
515
|
return {
|
|
585
516
|
status: "initializing"
|
|
586
517
|
};
|
|
587
518
|
}
|
|
588
519
|
let state$1 = Primitive_option.valFromOption(state);
|
|
589
|
-
let chains = Object.values(IndexerState.chainStates(state$1)).map(
|
|
590
|
-
let fetchState = ChainState.fetchState(cs);
|
|
591
|
-
let latestFetchedBlockNumber = Primitive_int.max(FetchState.bufferBlockNumber(fetchState), 0);
|
|
592
|
-
let knownHeight = ChainState.hasProcessedToEndblock(cs) ? Stdlib_Option.getOr(fetchState.endBlock, fetchState.knownHeight) : fetchState.knownHeight;
|
|
593
|
-
return {
|
|
594
|
-
chainId: ChainState.chainConfig(cs).id,
|
|
595
|
-
poweredByHyperSync: SourceManager.getActiveSource(ChainState.sourceManager(cs)).poweredByHyperSync,
|
|
596
|
-
firstEventBlockNumber: fetchState.firstEventBlock,
|
|
597
|
-
latestProcessedBlock: ChainState.committedProgressBlockNumber(cs) === -1 ? undefined : ChainState.committedProgressBlockNumber(cs),
|
|
598
|
-
timestampCaughtUpToHeadOrEndblock: ChainState.timestampCaughtUpToHeadOrEndblock(cs),
|
|
599
|
-
numEventsProcessed: ChainState.numEventsProcessed(cs),
|
|
600
|
-
latestFetchedBlockNumber: latestFetchedBlockNumber,
|
|
601
|
-
currentBlockHeight: knownHeight,
|
|
602
|
-
numBatchesFetched: 0,
|
|
603
|
-
startBlock: fetchState.startBlock,
|
|
604
|
-
endBlock: fetchState.endBlock,
|
|
605
|
-
numAddresses: FetchState.numAddresses(fetchState)
|
|
606
|
-
};
|
|
607
|
-
});
|
|
520
|
+
let chains = Object.values(IndexerState.chainStates(state$1)).map(ChainState.toChainData);
|
|
608
521
|
return {
|
|
609
522
|
status: "active",
|
|
610
523
|
envioVersion: envioVersion,
|
|
@@ -615,21 +528,22 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
|
|
|
615
528
|
};
|
|
616
529
|
}, persistence$1, isDevelopmentMode);
|
|
617
530
|
}
|
|
618
|
-
let state = IndexerState.makeFromDbState(config$2, persistence$1, Persistence.getInitializedState(persistence$1),
|
|
531
|
+
let state = IndexerState.makeFromDbState(config$2, persistence$1, Persistence.getInitializedState(persistence$1), registrationsByChainId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, undefined, undefined, onError);
|
|
619
532
|
if (shouldUseTui) {
|
|
620
533
|
Tui.start(() => state);
|
|
621
534
|
}
|
|
622
|
-
|
|
623
|
-
|
|
535
|
+
EnvioGlobal.value.indexerState = Primitive_option.some(state);
|
|
536
|
+
IndexerLoop.start(state);
|
|
537
|
+
return await runUntilFatalError;
|
|
624
538
|
}
|
|
625
539
|
|
|
626
540
|
export {
|
|
627
541
|
chainDataSchema,
|
|
628
542
|
stateSchema,
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
543
|
+
getIndexerState,
|
|
544
|
+
setIndexerState,
|
|
545
|
+
getGlobalPersistence,
|
|
546
|
+
setGlobalPersistence,
|
|
633
547
|
getInitialChainState,
|
|
634
548
|
buildChainsObject,
|
|
635
549
|
getGlobalIndexer,
|
|
@@ -637,6 +551,7 @@ export {
|
|
|
637
551
|
getEnvioInfo,
|
|
638
552
|
migrate,
|
|
639
553
|
dropSchema,
|
|
554
|
+
FatalError,
|
|
640
555
|
start,
|
|
641
556
|
}
|
|
642
557
|
/* chainDataSchema Not a pure module */
|
package/src/Metrics.res
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// Pull-based metrics computed from live indexer state at scrape time, merged with
|
|
2
|
+
// the imperatively-updated prom-client default registry. Used because the address
|
|
3
|
+
// index is mutated in place (no single update site to fire a gauge from).
|
|
4
|
+
|
|
5
|
+
let indexingAddressesName = "envio_indexing_addresses"
|
|
6
|
+
let indexingAddressesHelp = "The number of addresses indexed on chain. Includes both static and dynamic addresses."
|
|
7
|
+
|
|
8
|
+
// Render a gauge straight from the per-chain dict, one sample per key, without
|
|
9
|
+
// materialising an intermediate samples array. Accumulate into one string rather
|
|
10
|
+
// than building a lines array to join: `++` compiles to JS `+=`, which V8 grows
|
|
11
|
+
// as a ConsString instead of recopying.
|
|
12
|
+
let renderGauge = (~name, ~help, ~chains: dict<'a>, ~value: 'a => int) => {
|
|
13
|
+
let out = ref(`# HELP ${name} ${help}\n# TYPE ${name} gauge`)
|
|
14
|
+
let prefix = `\n${name}{chainId="`
|
|
15
|
+
chains->Utils.Dict.forEachWithKey((chain, chainId) => {
|
|
16
|
+
out := out.contents ++ prefix ++ chainId ++ `"} ` ++ value(chain)->Int.toString
|
|
17
|
+
})
|
|
18
|
+
out.contents
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let sourceRequestTotalName = "envio_source_request_total"
|
|
22
|
+
let sourceRequestTotalHelp = "The number of requests made to data sources."
|
|
23
|
+
let sourceRequestSecondsTotalName = "envio_source_request_seconds_total"
|
|
24
|
+
let sourceRequestSecondsTotalHelp = "Cumulative time spent on data source requests."
|
|
25
|
+
|
|
26
|
+
// Hand-rolls both HELP/TYPE blocks and their samples straight from
|
|
27
|
+
// SourceManager's aggregates — fully our own state, no prom-client registry
|
|
28
|
+
// involved. A leading "\n" on each block since, unlike prom-client's own
|
|
29
|
+
// metrics() output, renderGauge's output right before this doesn't end in one.
|
|
30
|
+
// Skips a method's seconds line when it has no timing (e.g. heightSubscription,
|
|
31
|
+
// which only ever records a count), and skips both blocks entirely when there's
|
|
32
|
+
// nothing to report at all.
|
|
33
|
+
let renderSourceRequests = (~samples: array<SourceManager.requestStatSample>) => {
|
|
34
|
+
if samples->Utils.Array.isEmpty {
|
|
35
|
+
""
|
|
36
|
+
} else {
|
|
37
|
+
let countOut = ref(
|
|
38
|
+
`\n# HELP ${sourceRequestTotalName} ${sourceRequestTotalHelp}\n# TYPE ${sourceRequestTotalName} counter`,
|
|
39
|
+
)
|
|
40
|
+
let secondsOut = ref(
|
|
41
|
+
`\n# HELP ${sourceRequestSecondsTotalName} ${sourceRequestSecondsTotalHelp}\n# TYPE ${sourceRequestSecondsTotalName} counter`,
|
|
42
|
+
)
|
|
43
|
+
samples->Array.forEach(sample => {
|
|
44
|
+
let labels = `{source="${sample.sourceName}",chainId="${sample.chainId->Int.toString}",method="${sample.method}"}`
|
|
45
|
+
countOut :=
|
|
46
|
+
countOut.contents ++ `\n${sourceRequestTotalName}${labels} ${sample.count->Int.toString}`
|
|
47
|
+
if sample.seconds !== 0. {
|
|
48
|
+
secondsOut :=
|
|
49
|
+
secondsOut.contents ++
|
|
50
|
+
`\n${sourceRequestSecondsTotalName}${labels} ${sample.seconds->Float.toFixed(~digits=3)}`
|
|
51
|
+
}
|
|
52
|
+
})
|
|
53
|
+
countOut.contents ++ secondsOut.contents
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
let collect = async (~state: option<IndexerState.t>) => {
|
|
58
|
+
let base = await PromClient.defaultRegister->PromClient.metrics
|
|
59
|
+
switch state {
|
|
60
|
+
| None => base
|
|
61
|
+
| Some(state) =>
|
|
62
|
+
let chainStates = state->IndexerState.chainStates
|
|
63
|
+
let sourceRequestSamples =
|
|
64
|
+
chainStates
|
|
65
|
+
->Dict.valuesToArray
|
|
66
|
+
->Array.flatMap(cs => cs->ChainState.sourceManager->SourceManager.getRequestStatSamples)
|
|
67
|
+
`${base}${renderGauge(
|
|
68
|
+
~name=indexingAddressesName,
|
|
69
|
+
~help=indexingAddressesHelp,
|
|
70
|
+
~chains=chainStates,
|
|
71
|
+
~value=cs => (cs->ChainState.toChainData).numAddresses,
|
|
72
|
+
)}${renderSourceRequests(~samples=sourceRequestSamples)}\n`
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Utils from "./Utils.res.mjs";
|
|
4
|
+
import * as ChainState from "./ChainState.res.mjs";
|
|
5
|
+
import * as PromClient from "prom-client";
|
|
6
|
+
import * as IndexerState from "./IndexerState.res.mjs";
|
|
7
|
+
import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
8
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
9
|
+
|
|
10
|
+
let indexingAddressesName = "envio_indexing_addresses";
|
|
11
|
+
|
|
12
|
+
let indexingAddressesHelp = "The number of addresses indexed on chain. Includes both static and dynamic addresses.";
|
|
13
|
+
|
|
14
|
+
function renderGauge(name, help, chains, value) {
|
|
15
|
+
let out = {
|
|
16
|
+
contents: `# HELP ` + name + ` ` + help + `\n# TYPE ` + name + ` gauge`
|
|
17
|
+
};
|
|
18
|
+
let prefix = `\n` + name + `{chainId="`;
|
|
19
|
+
Utils.Dict.forEachWithKey(chains, (chain, chainId) => {
|
|
20
|
+
out.contents = out.contents + prefix + chainId + `"} ` + value(chain).toString();
|
|
21
|
+
});
|
|
22
|
+
return out.contents;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let sourceRequestTotalName = "envio_source_request_total";
|
|
26
|
+
|
|
27
|
+
let sourceRequestTotalHelp = "The number of requests made to data sources.";
|
|
28
|
+
|
|
29
|
+
let sourceRequestSecondsTotalName = "envio_source_request_seconds_total";
|
|
30
|
+
|
|
31
|
+
let sourceRequestSecondsTotalHelp = "Cumulative time spent on data source requests.";
|
|
32
|
+
|
|
33
|
+
function renderSourceRequests(samples) {
|
|
34
|
+
if (Utils.$$Array.isEmpty(samples)) {
|
|
35
|
+
return "";
|
|
36
|
+
}
|
|
37
|
+
let countOut = {
|
|
38
|
+
contents: `\n# HELP ` + sourceRequestTotalName + ` ` + sourceRequestTotalHelp + `\n# TYPE ` + sourceRequestTotalName + ` counter`
|
|
39
|
+
};
|
|
40
|
+
let secondsOut = {
|
|
41
|
+
contents: `\n# HELP ` + sourceRequestSecondsTotalName + ` ` + sourceRequestSecondsTotalHelp + `\n# TYPE ` + sourceRequestSecondsTotalName + ` counter`
|
|
42
|
+
};
|
|
43
|
+
samples.forEach(sample => {
|
|
44
|
+
let labels = `{source="` + sample.sourceName + `",chainId="` + sample.chainId.toString() + `",method="` + sample.method + `"}`;
|
|
45
|
+
countOut.contents = countOut.contents + (`\n` + sourceRequestTotalName + labels + ` ` + sample.count.toString());
|
|
46
|
+
if (sample.seconds !== 0) {
|
|
47
|
+
secondsOut.contents = secondsOut.contents + (`\n` + sourceRequestSecondsTotalName + labels + ` ` + sample.seconds.toFixed(3));
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
return countOut.contents + secondsOut.contents;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function collect(state) {
|
|
55
|
+
let base = await PromClient.register.metrics();
|
|
56
|
+
if (state === undefined) {
|
|
57
|
+
return base;
|
|
58
|
+
}
|
|
59
|
+
let chainStates = IndexerState.chainStates(Primitive_option.valFromOption(state));
|
|
60
|
+
let sourceRequestSamples = Object.values(chainStates).flatMap(cs => SourceManager.getRequestStatSamples(ChainState.sourceManager(cs)));
|
|
61
|
+
return base + renderGauge(indexingAddressesName, indexingAddressesHelp, chainStates, cs => ChainState.toChainData(cs).numAddresses) + renderSourceRequests(sourceRequestSamples) + `\n`;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export {
|
|
65
|
+
indexingAddressesName,
|
|
66
|
+
indexingAddressesHelp,
|
|
67
|
+
renderGauge,
|
|
68
|
+
sourceRequestTotalName,
|
|
69
|
+
sourceRequestTotalHelp,
|
|
70
|
+
sourceRequestSecondsTotalName,
|
|
71
|
+
sourceRequestSecondsTotalHelp,
|
|
72
|
+
renderSourceRequests,
|
|
73
|
+
collect,
|
|
74
|
+
}
|
|
75
|
+
/* Utils Not a pure module */
|
package/src/PgStorage.res
CHANGED
|
@@ -1707,13 +1707,13 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1707
1707
|
// Initialize sink if configured
|
|
1708
1708
|
let sinkPromise = switch sink {
|
|
1709
1709
|
| Some(sink) => {
|
|
1710
|
-
let timerRef =
|
|
1710
|
+
let timerRef = Performance.now()
|
|
1711
1711
|
Some(
|
|
1712
1712
|
sink.writeBatch(~batch, ~updatedEntities=chUpdates)
|
|
1713
1713
|
->Promise.thenResolve(_ => {
|
|
1714
1714
|
Prometheus.StorageWrite.increment(
|
|
1715
1715
|
~storage=sink.name,
|
|
1716
|
-
~timeSeconds=timerRef->
|
|
1716
|
+
~timeSeconds=timerRef->Performance.secondsSince,
|
|
1717
1717
|
)
|
|
1718
1718
|
None
|
|
1719
1719
|
})
|
|
@@ -1724,7 +1724,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1724
1724
|
| None => None
|
|
1725
1725
|
}
|
|
1726
1726
|
|
|
1727
|
-
let primaryTimerRef =
|
|
1727
|
+
let primaryTimerRef = Performance.now()
|
|
1728
1728
|
await writeBatch(
|
|
1729
1729
|
sql,
|
|
1730
1730
|
~batch,
|
|
@@ -1741,7 +1741,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1741
1741
|
)
|
|
1742
1742
|
Prometheus.StorageWrite.increment(
|
|
1743
1743
|
~storage="postgres",
|
|
1744
|
-
~timeSeconds=primaryTimerRef->
|
|
1744
|
+
~timeSeconds=primaryTimerRef->Performance.secondsSince,
|
|
1745
1745
|
)
|
|
1746
1746
|
}
|
|
1747
1747
|
|
package/src/PgStorage.res.mjs
CHANGED
|
@@ -8,13 +8,13 @@ import * as Table from "./db/Table.res.mjs";
|
|
|
8
8
|
import * as Utils from "./Utils.res.mjs";
|
|
9
9
|
import * as Config from "./Config.res.mjs";
|
|
10
10
|
import * as Hasura from "./Hasura.res.mjs";
|
|
11
|
-
import * as Hrtime from "./bindings/Hrtime.res.mjs";
|
|
12
11
|
import * as Schema from "./db/Schema.res.mjs";
|
|
13
12
|
import * as Logging from "./Logging.res.mjs";
|
|
14
13
|
import * as Internal from "./Internal.res.mjs";
|
|
15
14
|
import Postgres from "postgres";
|
|
16
15
|
import * as ChainState from "./ChainState.res.mjs";
|
|
17
16
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
17
|
+
import * as Performance from "./bindings/Performance.res.mjs";
|
|
18
18
|
import * as Persistence from "./Persistence.res.mjs";
|
|
19
19
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
20
20
|
import * as EntityHistory from "./db/EntityHistory.res.mjs";
|
|
@@ -1116,16 +1116,16 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1116
1116
|
}
|
|
1117
1117
|
let sinkPromise;
|
|
1118
1118
|
if (sink !== undefined) {
|
|
1119
|
-
let timerRef =
|
|
1119
|
+
let timerRef = Performance.now();
|
|
1120
1120
|
sinkPromise = sink.writeBatch(batch, chUpdates).then(() => {
|
|
1121
|
-
Prometheus.StorageWrite.increment(sink.name,
|
|
1121
|
+
Prometheus.StorageWrite.increment(sink.name, Performance.secondsSince(timerRef));
|
|
1122
1122
|
}).catch(exn => exn);
|
|
1123
1123
|
} else {
|
|
1124
1124
|
sinkPromise = undefined;
|
|
1125
1125
|
}
|
|
1126
|
-
let primaryTimerRef =
|
|
1126
|
+
let primaryTimerRef = Performance.now();
|
|
1127
1127
|
await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, pgUpdates, sinkPromise, chainMetaData, undefined);
|
|
1128
|
-
return Prometheus.StorageWrite.increment("postgres",
|
|
1128
|
+
return Prometheus.StorageWrite.increment("postgres", Performance.secondsSince(primaryTimerRef));
|
|
1129
1129
|
};
|
|
1130
1130
|
let close = () => sql.end();
|
|
1131
1131
|
return {
|