envio 3.9.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -0
- package/fuel.schema.json +26 -0
- package/index.d.ts +35 -3
- package/package.json +6 -7
- package/src/Batch.res +9 -0
- package/src/ChainState.res +35 -39
- package/src/ChainState.res.mjs +14 -11
- package/src/ChainState.resi +15 -7
- package/src/Config.res +52 -44
- package/src/Config.res.mjs +51 -17
- package/src/Core.res +4 -0
- package/src/CrossChainState.res +15 -23
- package/src/CrossChainState.res.mjs +10 -18
- package/src/CrossChainState.resi +4 -1
- package/src/Ecosystem.res +7 -2
- package/src/Envio.res +6 -4
- package/src/EventConfigBuilder.res +169 -26
- package/src/EventConfigBuilder.res.mjs +124 -15
- package/src/HandlerRegister.res +21 -4
- package/src/HandlerRegister.res.mjs +24 -5
- package/src/InMemoryStore.res +9 -7
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/IndexerState.res +43 -6
- package/src/IndexerState.res.mjs +36 -7
- package/src/IndexerState.resi +4 -4
- package/src/Internal.res +16 -17
- package/src/Metrics.res +72 -6
- package/src/Metrics.res.mjs +30 -2
- package/src/Persistence.res +36 -30
- package/src/Persistence.res.mjs +1 -20
- package/src/PgStorage.res +104 -56
- package/src/PgStorage.res.mjs +57 -35
- package/src/PruneStaleHistory.res +85 -52
- package/src/PruneStaleHistory.res.mjs +63 -37
- package/src/Rollback.res +68 -45
- package/src/Rollback.res.mjs +38 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SafeCheckpointTracking.res +11 -11
- package/src/SafeCheckpointTracking.res.mjs +7 -6
- package/src/SimulateItems.res +33 -18
- package/src/SimulateItems.res.mjs +26 -22
- package/src/Sink.res +38 -26
- package/src/Sink.res.mjs +21 -16
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Utils.res +170 -0
- package/src/Utils.res.mjs +153 -0
- package/src/Writing.res +2 -2
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +366 -698
- package/src/bindings/ClickHouse.res.mjs +371 -421
- package/src/bindings/ClickHouseSink.res +337 -0
- package/src/bindings/ClickHouseSink.res.mjs +246 -0
- package/src/bindings/EventSource.res +8 -2
- package/src/bindings/NodeJs.res +6 -0
- package/src/bindings/Vitest.res +23 -0
- package/src/bindings/Vitest.res.mjs +3 -0
- package/src/bindings/WebSocket.res +9 -10
- package/src/db/CheckpointBounds.res +53 -0
- package/src/db/CheckpointBounds.res.mjs +44 -0
- package/src/db/EntityFilter.res +36 -17
- package/src/db/EntityFilter.res.mjs +25 -14
- package/src/db/EntityHistory.res +47 -22
- package/src/db/EntityHistory.res.mjs +19 -12
- package/src/db/InternalTable.res +84 -35
- package/src/db/InternalTable.res.mjs +56 -23
- package/src/db/RollbackFloors.res +55 -0
- package/src/db/RollbackFloors.res.mjs +92 -0
- package/src/db/Table.res +22 -4
- package/src/db/Table.res.mjs +30 -9
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/EvmHyperSyncSource.res +3 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcWs.res +124 -0
- package/src/sources/EvmRpcWs.res.mjs +117 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/FuelHyperSyncSource.res +1 -1
- package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
- package/src/sources/HeightFeed.res +568 -0
- package/src/sources/HeightFeed.res.mjs +452 -0
- package/src/sources/HeightStream.res +257 -0
- package/src/sources/HeightStream.res.mjs +194 -0
- package/src/sources/HyperSync.res +5 -0
- package/src/sources/HyperSync.res.mjs +3 -0
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncSSE.res +49 -0
- package/src/sources/HyperSyncSSE.res.mjs +79 -0
- package/src/sources/RpcSource.res +1 -1
- package/src/sources/RpcSource.res.mjs +2 -2
- package/src/sources/Source.res +54 -1
- package/src/sources/Source.res.mjs +25 -0
- package/src/sources/SourceManager.res +220 -227
- package/src/sources/SourceManager.res.mjs +139 -164
- package/src/sources/SourceManager.resi +8 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +70 -127
- package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
- package/src/sources/SvmHyperSyncSource.res +18 -15
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/svm.schema.json +1 -75
- package/src/MemoryStorage.res +0 -729
- package/src/MemoryStorage.res.mjs +0 -586
- package/src/sources/HyperSyncHeightStream.res +0 -129
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
- package/src/sources/RpcWebSocketHeightStream.res +0 -175
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/src/HandlerRegister.res
CHANGED
|
@@ -154,6 +154,7 @@ let buildOnEventRegistrationWith = (
|
|
|
154
154
|
~isWildcard,
|
|
155
155
|
~handler,
|
|
156
156
|
~contractRegister,
|
|
157
|
+
~where,
|
|
157
158
|
~fieldSelection?,
|
|
158
159
|
~startBlock?,
|
|
159
160
|
) :> Internal.onEventRegistration)
|
|
@@ -320,7 +321,7 @@ let addOnEventRegistration = (
|
|
|
320
321
|
)
|
|
321
322
|
| Fuel =>
|
|
322
323
|
JsError.throwWithMessage(
|
|
323
|
-
`The fields option of the "${eventName}" event registration on contract "${contractName}" is
|
|
324
|
+
`The fields option of the "${eventName}" event registration on contract "${contractName}" is not supported on Fuel. Select the fields in your config instead.`,
|
|
324
325
|
)
|
|
325
326
|
}
|
|
326
327
|
}
|
|
@@ -335,6 +336,21 @@ let addOnEventRegistration = (
|
|
|
335
336
|
| None => ()
|
|
336
337
|
| Some(eventConfig) =>
|
|
337
338
|
matched := true
|
|
339
|
+
switch (registration.config.ecosystem.name, fieldSelection) {
|
|
340
|
+
| (Svm, Some(fieldSelection)) if fieldSelection.instructionFields->Utils.Set.has("args") =>
|
|
341
|
+
let svmEventConfig =
|
|
342
|
+
eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig)
|
|
343
|
+
let declaresArgs = switch svmEventConfig.args {
|
|
344
|
+
| JSON.Array(args) => args->Array.length > 0
|
|
345
|
+
| _ => false
|
|
346
|
+
}
|
|
347
|
+
if !declaresArgs {
|
|
348
|
+
JsError.throwWithMessage(
|
|
349
|
+
`Invalid "args" field in the fields.instruction option of the "${eventName}" instruction on program "${contractName}". The instruction declares no args in config.yaml, so there is nothing to decode. Remove "args" from the selection, or declare the instruction's args.`,
|
|
350
|
+
)
|
|
351
|
+
}
|
|
352
|
+
| _ => ()
|
|
353
|
+
}
|
|
338
354
|
let reg = buildOnEventRegistrationWith(
|
|
339
355
|
~config=registration.config,
|
|
340
356
|
~chainId=chainConfig.id,
|
|
@@ -359,17 +375,18 @@ let addOnEventRegistration = (
|
|
|
359
375
|
if !matched.contents {
|
|
360
376
|
let (contractNames, eventNames) = registration->describeConfigured(~contractName)
|
|
361
377
|
let listOr = (names, empty) =>
|
|
362
|
-
names->Utils.Array.isEmpty ? empty : names->Array.
|
|
378
|
+
names->Utils.Array.isEmpty ? empty : names->Utils.Array.quotedJoin
|
|
379
|
+
let {contractNoun, eventNoun} = registration.config.ecosystem
|
|
363
380
|
if eventNames->Utils.Array.isEmpty {
|
|
364
381
|
JsError.throwWithMessage(
|
|
365
|
-
|
|
382
|
+
`${contractNoun->Utils.String.capitalize} "${contractName}" is not configured on any chain, so its handler for "${eventName}" would never run. Add it to your config, or remove the registration. Configured ${contractNoun}s: ${listOr(
|
|
366
383
|
contractNames,
|
|
367
384
|
"none",
|
|
368
385
|
)}.`,
|
|
369
386
|
)
|
|
370
387
|
} else {
|
|
371
388
|
JsError.throwWithMessage(
|
|
372
|
-
|
|
389
|
+
`${eventNoun->Utils.String.capitalize} "${eventName}" is not configured on ${contractNoun} "${contractName}", so its handler would never run. Add it to your config, or remove the registration. Configured ${eventNoun}s on "${contractName}": ${listOr(
|
|
373
390
|
eventNames,
|
|
374
391
|
"none",
|
|
375
392
|
)}.`,
|
|
@@ -102,7 +102,7 @@ function buildOnEventRegistrationWith(config, chainId, eventConfig, isWildcard,
|
|
|
102
102
|
case "fuel" :
|
|
103
103
|
return EventConfigBuilder.buildFuelOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, startBlock);
|
|
104
104
|
case "svm" :
|
|
105
|
-
return EventConfigBuilder.buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, fieldSelection, startBlock);
|
|
105
|
+
return EventConfigBuilder.buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, where, fieldSelection, startBlock);
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
108
|
|
|
@@ -209,7 +209,7 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
|
|
|
209
209
|
fieldSelection = EventConfigBuilder.resolveInlineFieldSelection(fields$1, contractName, eventName, registration.config.enableRawEvents);
|
|
210
210
|
break;
|
|
211
211
|
case "fuel" :
|
|
212
|
-
fieldSelection = Stdlib_JsError.throwWithMessage(`The fields option of the "` + eventName + `" event registration on contract "` + contractName + `" is
|
|
212
|
+
fieldSelection = Stdlib_JsError.throwWithMessage(`The fields option of the "` + eventName + `" event registration on contract "` + contractName + `" is not supported on Fuel. Select the fields in your config instead.`);
|
|
213
213
|
break;
|
|
214
214
|
case "svm" :
|
|
215
215
|
fieldSelection = EventConfigBuilder.resolveSvmInlineFieldSelection(fields$1, contractName, eventName);
|
|
@@ -231,6 +231,22 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
|
|
|
231
231
|
return;
|
|
232
232
|
}
|
|
233
233
|
matched.contents = true;
|
|
234
|
+
let match = registration.config.ecosystem.name;
|
|
235
|
+
switch (match) {
|
|
236
|
+
case "evm" :
|
|
237
|
+
case "fuel" :
|
|
238
|
+
break;
|
|
239
|
+
case "svm" :
|
|
240
|
+
if (fieldSelection !== undefined && fieldSelection.instructionFields.has("args")) {
|
|
241
|
+
let args = eventConfig.args;
|
|
242
|
+
let declaresArgs;
|
|
243
|
+
declaresArgs = Array.isArray(args) ? args.length !== 0 : false;
|
|
244
|
+
if (!declaresArgs) {
|
|
245
|
+
Stdlib_JsError.throwWithMessage(`Invalid "args" field in the fields.instruction option of the "` + eventName + `" instruction on program "` + contractName + `". The instruction declares no args in config.yaml, so there is nothing to decode. Remove "args" from the selection, or declare the instruction's args.`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
234
250
|
let reg = buildOnEventRegistrationWith(registration.config, chainConfig.id, eventConfig, isWildcard, handler, contractRegister, where, fieldSelection, contract.startBlock);
|
|
235
251
|
getChainRegistrations(registration, chainConfig.id).onEventRegistrations.push(reg);
|
|
236
252
|
});
|
|
@@ -243,13 +259,16 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
|
|
|
243
259
|
if (Utils.$$Array.isEmpty(names)) {
|
|
244
260
|
return empty;
|
|
245
261
|
} else {
|
|
246
|
-
return
|
|
262
|
+
return Utils.$$Array.quotedJoin(names);
|
|
247
263
|
}
|
|
248
264
|
};
|
|
265
|
+
let match$2 = registration.config.ecosystem;
|
|
266
|
+
let eventNoun = match$2.eventNoun;
|
|
267
|
+
let contractNoun = match$2.contractNoun;
|
|
249
268
|
if (Utils.$$Array.isEmpty(eventNames)) {
|
|
250
|
-
return Stdlib_JsError.throwWithMessage(`
|
|
269
|
+
return Stdlib_JsError.throwWithMessage(Utils.$$String.capitalize(contractNoun) + ` "` + contractName + `" is not configured on any chain, so its handler for "` + eventName + `" would never run. Add it to your config, or remove the registration. Configured ` + contractNoun + `s: ` + listOr(match$1[0], "none") + `.`);
|
|
251
270
|
} else {
|
|
252
|
-
return Stdlib_JsError.throwWithMessage(`
|
|
271
|
+
return Stdlib_JsError.throwWithMessage(Utils.$$String.capitalize(eventNoun) + ` "` + eventName + `" is not configured on ` + contractNoun + ` "` + contractName + `", so its handler would never run. Add it to your config, or remove the registration. Configured ` + eventNoun + `s on "` + contractName + `": ` + listOr(eventNames, "none") + `.`);
|
|
253
272
|
}
|
|
254
273
|
}
|
|
255
274
|
|
package/src/InMemoryStore.res
CHANGED
|
@@ -127,15 +127,15 @@ let dropCommittedEffects = (
|
|
|
127
127
|
|
|
128
128
|
let prepareRollbackDiff = async (
|
|
129
129
|
state: IndexerState.t,
|
|
130
|
-
~
|
|
130
|
+
~floors: RollbackFloors.t,
|
|
131
131
|
~rollbackDiffCheckpointId,
|
|
132
|
-
~
|
|
132
|
+
~progressedChains,
|
|
133
133
|
~rolledBackAddresses,
|
|
134
134
|
) => {
|
|
135
135
|
state->IndexerState.beginRollbackDiff(
|
|
136
|
-
~targetCheckpointId=rollbackTargetCheckpointId,
|
|
137
136
|
~diffCheckpointId=rollbackDiffCheckpointId,
|
|
138
|
-
~
|
|
137
|
+
~floors,
|
|
138
|
+
~progressedChains,
|
|
139
139
|
~rolledBackAddresses,
|
|
140
140
|
)
|
|
141
141
|
let persistence = state->IndexerState.persistence
|
|
@@ -152,7 +152,7 @@ let prepareRollbackDiff = async (
|
|
|
152
152
|
->Array.map(async entityConfig => {
|
|
153
153
|
let (removals, restoredEntities) = await persistence.storage.getRollbackData(
|
|
154
154
|
~entityConfig,
|
|
155
|
-
~
|
|
155
|
+
~floors,
|
|
156
156
|
)
|
|
157
157
|
|
|
158
158
|
removals->Array.forEach(({entityId, scope}: Persistence.rollbackRemoval) => {
|
|
@@ -215,7 +215,8 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
|
|
|
215
215
|
}
|
|
216
216
|
}
|
|
217
217
|
|
|
218
|
-
batch.registeredAddresses
|
|
218
|
+
batch.registeredAddresses
|
|
219
|
+
->Array.pushMany(
|
|
219
220
|
chainState
|
|
220
221
|
->ChainState.drainAddressesForWrite(
|
|
221
222
|
~toBlockInclusive=progressedChain.progressBlockNumber,
|
|
@@ -230,7 +231,8 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
|
|
|
230
231
|
},
|
|
231
232
|
checkpointId: checkpointIds->Array.getUnsafe(dc.checkpointIdx),
|
|
232
233
|
}),
|
|
233
|
-
)
|
|
234
|
+
)
|
|
235
|
+
->ignore
|
|
234
236
|
}
|
|
235
237
|
})
|
|
236
238
|
}
|
|
@@ -106,14 +106,14 @@ function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromD
|
|
|
106
106
|
inMemTable.changesCount = inMemTable.changesCount - keysToDelete.length;
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
-
async function prepareRollbackDiff(state,
|
|
110
|
-
IndexerState.beginRollbackDiff(state,
|
|
109
|
+
async function prepareRollbackDiff(state, floors, rollbackDiffCheckpointId, progressedChains, rolledBackAddresses) {
|
|
110
|
+
IndexerState.beginRollbackDiff(state, rollbackDiffCheckpointId, floors, progressedChains, rolledBackAddresses);
|
|
111
111
|
let persistence = IndexerState.persistence(state);
|
|
112
112
|
let committedCheckpointId = IndexerState.committedCheckpointId(state);
|
|
113
113
|
let deletedEntities = {};
|
|
114
114
|
let setEntities = {};
|
|
115
115
|
await Promise.all(persistence.allEntities.filter(entityConfig => entityConfig.storage.postgres).map(async entityConfig => {
|
|
116
|
-
let match = await persistence.storage.getRollbackData(entityConfig,
|
|
116
|
+
let match = await persistence.storage.getRollbackData(entityConfig, floors);
|
|
117
117
|
match[0].forEach(param => {
|
|
118
118
|
let entityId = param.entityId;
|
|
119
119
|
Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
|
package/src/IndexerState.res
CHANGED
|
@@ -355,7 +355,10 @@ let getChainState = (state: t, ~chainId: ChainId.t): ChainState.t =>
|
|
|
355
355
|
)
|
|
356
356
|
}
|
|
357
357
|
|
|
358
|
-
let
|
|
358
|
+
let getSafeCheckpointIdByChain = (state: t) =>
|
|
359
|
+
state.crossChainState->CrossChainState.getSafeCheckpointIdByChain(
|
|
360
|
+
~committedCheckpointId=state.committedCheckpointId,
|
|
361
|
+
)
|
|
359
362
|
|
|
360
363
|
let createBatch = (
|
|
361
364
|
state: t,
|
|
@@ -452,7 +455,14 @@ let entities = (state: t) => state.entities
|
|
|
452
455
|
// Every in-memory entity table across all scopes, cross-chain first. The size,
|
|
453
456
|
// drop and flush passes walk the whole store this way instead of assuming a
|
|
454
457
|
// single partition.
|
|
455
|
-
let eachEntityTable = (
|
|
458
|
+
let eachEntityTable = (
|
|
459
|
+
state: t,
|
|
460
|
+
fn: (
|
|
461
|
+
~entityConfig: Internal.entityConfig,
|
|
462
|
+
~scope: Internal.chainScope,
|
|
463
|
+
~table: InMemoryTable.Entity.t,
|
|
464
|
+
) => unit,
|
|
465
|
+
) => {
|
|
456
466
|
let chainStates = state.crossChainState->CrossChainState.chainStates
|
|
457
467
|
state.allEntities->Array.forEach(entityConfig =>
|
|
458
468
|
if entityConfig.crossChain {
|
|
@@ -516,6 +526,7 @@ let toMetrics = (state: t): Metrics.t => {
|
|
|
516
526
|
let chainStates = state.crossChainState->CrossChainState.chainStates
|
|
517
527
|
let sourceRequests = []
|
|
518
528
|
let sourceHeights = []
|
|
529
|
+
let sourceHeightStreams = []
|
|
519
530
|
chainStates->Utils.Dict.forEach(cs => {
|
|
520
531
|
let sourceManager = cs->ChainState.sourceManager
|
|
521
532
|
sourceManager
|
|
@@ -538,6 +549,16 @@ let toMetrics = (state: t): Metrics.t => {
|
|
|
538
549
|
height: s.height,
|
|
539
550
|
})
|
|
540
551
|
)
|
|
552
|
+
sourceManager
|
|
553
|
+
->SourceManager.getHeightStreamSamples
|
|
554
|
+
->Array.forEach(s =>
|
|
555
|
+
sourceHeightStreams->Array.push({
|
|
556
|
+
Metrics.source: s.sourceName,
|
|
557
|
+
chainId: s.chainId,
|
|
558
|
+
connectCount: s.stream.connectCount,
|
|
559
|
+
disconnectsByReason: s.stream.disconnectsByReason,
|
|
560
|
+
})
|
|
561
|
+
)
|
|
541
562
|
})
|
|
542
563
|
let historyPrunes = []
|
|
543
564
|
state.historyPruneStats->Utils.Dict.forEachWithKey((s, entityName) =>
|
|
@@ -600,6 +621,7 @@ let toMetrics = (state: t): Metrics.t => {
|
|
|
600
621
|
historyPrunes,
|
|
601
622
|
sourceRequests,
|
|
602
623
|
sourceHeights,
|
|
624
|
+
sourceHeightStreams,
|
|
603
625
|
}
|
|
604
626
|
}
|
|
605
627
|
|
|
@@ -793,6 +815,8 @@ let drainBatchRun = (state: t): Batch.t => {
|
|
|
793
815
|
}
|
|
794
816
|
}
|
|
795
817
|
|
|
818
|
+
let pendingRollback = (state: t): option<Persistence.rollback> => state.rollback
|
|
819
|
+
|
|
796
820
|
// Take the pending rollback diff to write, clearing it from the store.
|
|
797
821
|
let takeRollback = (state: t): option<Persistence.rollback> => {
|
|
798
822
|
let rollback = state.rollback
|
|
@@ -812,9 +836,9 @@ let markCommitted = (state: t, ~upToCheckpointId) => {
|
|
|
812
836
|
// Reset the in-memory tables and arm the rollback diff that the next write commits.
|
|
813
837
|
let beginRollbackDiff = (
|
|
814
838
|
state: t,
|
|
815
|
-
~targetCheckpointId,
|
|
816
839
|
~diffCheckpointId,
|
|
817
|
-
~
|
|
840
|
+
~floors,
|
|
841
|
+
~progressedChains: array<InternalTable.Chains.progressedChain>,
|
|
818
842
|
~rolledBackAddresses,
|
|
819
843
|
) => {
|
|
820
844
|
let perChainEntities = state.allEntities->EntityTables.perChain
|
|
@@ -829,10 +853,23 @@ let beginRollbackDiff = (
|
|
|
829
853
|
| Some({rolledBackAddresses: pending}) => pending->Array.concat(rolledBackAddresses)
|
|
830
854
|
| None => rolledBackAddresses
|
|
831
855
|
}
|
|
856
|
+
// Same for the chains: this rollback recomputes progress from the checkpoints,
|
|
857
|
+
// so a chain the pending diff already moved can land on exactly the block it
|
|
858
|
+
// is now at and go unreported here. Its stored progress still needs
|
|
859
|
+
// correcting, so the pending rows carry over — this rollback's row for a
|
|
860
|
+
// chain wins, being the later reading of the same chain state.
|
|
861
|
+
let progressedChains = switch state.rollback {
|
|
862
|
+
| Some({progressedChains: pending}) =>
|
|
863
|
+
let byChainId = Dict.make()
|
|
864
|
+
pending->Array.forEach(chain => byChainId->ChainId.Dict.set(chain.chainId, chain))
|
|
865
|
+
progressedChains->Array.forEach(chain => byChainId->ChainId.Dict.set(chain.chainId, chain))
|
|
866
|
+
byChainId->Dict.valuesToArray
|
|
867
|
+
| None => progressedChains
|
|
868
|
+
}
|
|
832
869
|
state.rollback = Some({
|
|
833
|
-
targetCheckpointId,
|
|
834
870
|
diffCheckpointId,
|
|
835
|
-
|
|
871
|
+
floors,
|
|
872
|
+
progressedChains,
|
|
836
873
|
rolledBackAddresses,
|
|
837
874
|
})
|
|
838
875
|
}
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -159,8 +159,8 @@ function getChainState(state, chainId) {
|
|
|
159
159
|
}
|
|
160
160
|
}
|
|
161
161
|
|
|
162
|
-
function
|
|
163
|
-
return CrossChainState.
|
|
162
|
+
function getSafeCheckpointIdByChain(state) {
|
|
163
|
+
return CrossChainState.getSafeCheckpointIdByChain(state.crossChainState, state.committedCheckpointId);
|
|
164
164
|
}
|
|
165
165
|
|
|
166
166
|
function createBatch(state, processedCheckpointId, batchSizeTarget, isRollback) {
|
|
@@ -392,6 +392,7 @@ function toMetrics(state) {
|
|
|
392
392
|
let chainStates = CrossChainState.chainStates(state.crossChainState);
|
|
393
393
|
let sourceRequests = [];
|
|
394
394
|
let sourceHeights = [];
|
|
395
|
+
let sourceHeightStreams = [];
|
|
395
396
|
Utils.Dict.forEach(chainStates, cs => {
|
|
396
397
|
let sourceManager = ChainState.sourceManager(cs);
|
|
397
398
|
SourceManager.getRequestStatSamples(sourceManager).forEach(s => {
|
|
@@ -410,6 +411,14 @@ function toMetrics(state) {
|
|
|
410
411
|
height: s.height
|
|
411
412
|
});
|
|
412
413
|
});
|
|
414
|
+
SourceManager.getHeightStreamSamples(sourceManager).forEach(s => {
|
|
415
|
+
sourceHeightStreams.push({
|
|
416
|
+
source: s.sourceName,
|
|
417
|
+
chainId: s.chainId,
|
|
418
|
+
connectCount: s.stream.connectCount,
|
|
419
|
+
disconnectsByReason: s.stream.disconnectsByReason
|
|
420
|
+
});
|
|
421
|
+
});
|
|
413
422
|
});
|
|
414
423
|
let historyPrunes = [];
|
|
415
424
|
Utils.Dict.forEachWithKey(state.historyPruneStats, (s, entityName) => {
|
|
@@ -464,7 +473,8 @@ function toMetrics(state) {
|
|
|
464
473
|
})),
|
|
465
474
|
historyPrunes: historyPrunes,
|
|
466
475
|
sourceRequests: sourceRequests,
|
|
467
|
-
sourceHeights: sourceHeights
|
|
476
|
+
sourceHeights: sourceHeights,
|
|
477
|
+
sourceHeightStreams: sourceHeightStreams
|
|
468
478
|
};
|
|
469
479
|
}
|
|
470
480
|
|
|
@@ -658,6 +668,10 @@ function drainBatchRun(state) {
|
|
|
658
668
|
};
|
|
659
669
|
}
|
|
660
670
|
|
|
671
|
+
function pendingRollback(state) {
|
|
672
|
+
return state.rollback;
|
|
673
|
+
}
|
|
674
|
+
|
|
661
675
|
function takeRollback(state) {
|
|
662
676
|
let rollback = state.rollback;
|
|
663
677
|
state.rollback = undefined;
|
|
@@ -668,18 +682,32 @@ function markCommitted(state, upToCheckpointId) {
|
|
|
668
682
|
state.committedCheckpointId = upToCheckpointId;
|
|
669
683
|
}
|
|
670
684
|
|
|
671
|
-
function beginRollbackDiff(state,
|
|
685
|
+
function beginRollbackDiff(state, diffCheckpointId, floors, progressedChains, rolledBackAddresses) {
|
|
672
686
|
let perChainEntities = EntityTables.perChain(state.allEntities);
|
|
673
687
|
state.entities = EntityTables.make(EntityTables.crossChain(state.allEntities));
|
|
674
688
|
Utils.Dict.forEach(CrossChainState.chainStates(state.crossChainState), cs => ChainState.resetEntities(cs, perChainEntities));
|
|
675
689
|
EffectState.resetForRollback(state.effectState);
|
|
676
690
|
let match = state.rollback;
|
|
677
691
|
let rolledBackAddresses$1 = match !== undefined ? match.rolledBackAddresses.concat(rolledBackAddresses) : rolledBackAddresses;
|
|
692
|
+
let match$1 = state.rollback;
|
|
693
|
+
let progressedChains$1;
|
|
694
|
+
if (match$1 !== undefined) {
|
|
695
|
+
let byChainId = {};
|
|
696
|
+
match$1.progressedChains.forEach(chain => {
|
|
697
|
+
byChainId[chain.chainId] = chain;
|
|
698
|
+
});
|
|
699
|
+
progressedChains.forEach(chain => {
|
|
700
|
+
byChainId[chain.chainId] = chain;
|
|
701
|
+
});
|
|
702
|
+
progressedChains$1 = Object.values(byChainId);
|
|
703
|
+
} else {
|
|
704
|
+
progressedChains$1 = progressedChains;
|
|
705
|
+
}
|
|
678
706
|
state.rollback = {
|
|
679
|
-
targetCheckpointId: targetCheckpointId,
|
|
680
707
|
diffCheckpointId: diffCheckpointId,
|
|
708
|
+
floors: floors,
|
|
681
709
|
rolledBackAddresses: rolledBackAddresses$1,
|
|
682
|
-
|
|
710
|
+
progressedChains: progressedChains$1
|
|
683
711
|
};
|
|
684
712
|
}
|
|
685
713
|
|
|
@@ -766,7 +794,7 @@ export {
|
|
|
766
794
|
markProcessingStalledOnFetch,
|
|
767
795
|
recordProcessedBatch,
|
|
768
796
|
createBatch,
|
|
769
|
-
|
|
797
|
+
getSafeCheckpointIdByChain,
|
|
770
798
|
config,
|
|
771
799
|
persistence,
|
|
772
800
|
allEntities,
|
|
@@ -812,6 +840,7 @@ export {
|
|
|
812
840
|
recordRollbackSuccess,
|
|
813
841
|
queueProcessedBatch,
|
|
814
842
|
drainBatchRun,
|
|
843
|
+
pendingRollback,
|
|
815
844
|
takeRollback,
|
|
816
845
|
markCommitted,
|
|
817
846
|
beginRollbackDiff,
|
package/src/IndexerState.resi
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
// The indexer state. `t` is opaque: other modules read it through the accessors
|
|
2
2
|
// and change it only through the transitions and setters exposed here.
|
|
3
3
|
|
|
4
|
-
|
|
5
4
|
type rollbackState =
|
|
6
5
|
| NoRollback
|
|
7
6
|
| ReorgDetected({chainId: ChainId.t, blockNumber: int})
|
|
@@ -67,7 +66,7 @@ let createBatch: (
|
|
|
67
66
|
~batchSizeTarget: int,
|
|
68
67
|
~isRollback: bool,
|
|
69
68
|
) => Batch.t
|
|
70
|
-
let
|
|
69
|
+
let getSafeCheckpointIdByChain: t => array<(ChainId.t, option<Internal.checkpointId>)>
|
|
71
70
|
|
|
72
71
|
// Accessors.
|
|
73
72
|
let config: t => Config.t
|
|
@@ -136,13 +135,14 @@ let recordRollbackSuccess: (t, ~timeSeconds: float, ~rollbackedProcessedEvents:
|
|
|
136
135
|
// Store domain operations.
|
|
137
136
|
let queueProcessedBatch: (t, ~batch: Batch.t) => unit
|
|
138
137
|
let drainBatchRun: t => Batch.t
|
|
138
|
+
let pendingRollback: t => option<Persistence.rollback>
|
|
139
139
|
let takeRollback: t => option<Persistence.rollback>
|
|
140
140
|
let markCommitted: (t, ~upToCheckpointId: Internal.checkpointId) => unit
|
|
141
141
|
let beginRollbackDiff: (
|
|
142
142
|
t,
|
|
143
|
-
~targetCheckpointId: Internal.checkpointId,
|
|
144
143
|
~diffCheckpointId: Internal.checkpointId,
|
|
145
|
-
~
|
|
144
|
+
~floors: RollbackFloors.t,
|
|
145
|
+
~progressedChains: array<InternalTable.Chains.progressedChain>,
|
|
146
146
|
~rolledBackAddresses: array<AddressRows.key>,
|
|
147
147
|
) => unit
|
|
148
148
|
let recordWriteFailure: (t, exn) => unit
|
package/src/Internal.res
CHANGED
|
@@ -573,15 +573,6 @@ type svmInstructionEventConfig = {
|
|
|
573
573
|
programId: SvmTypes.Pubkey.t,
|
|
574
574
|
/** Hex-encoded discriminator. `None` matches every instruction in the program. */
|
|
575
575
|
discriminator: option<string>,
|
|
576
|
-
/** Length of the discriminator in bytes (0 / 1 / 2 / 4 / 8). Drives the
|
|
577
|
-
`dN` selector at query time and the dispatch-key precomputation in the
|
|
578
|
-
router. */
|
|
579
|
-
discriminatorByteLen: int,
|
|
580
|
-
/** Disjunctive normal form: outer array is OR of AND-groups, inner array is
|
|
581
|
-
AND across positions. Empty outer array means "no account filter". */
|
|
582
|
-
accountFilters: array<svmAccountFilterGroup>,
|
|
583
|
-
/** `None` matches both outer and inner (CPI-invoked) instructions. */
|
|
584
|
-
isInner: option<bool>,
|
|
585
576
|
/** Positional account names from the Borsh schema, in declared order.
|
|
586
577
|
`[]` means no schema is attached for this instruction. */
|
|
587
578
|
accounts: array<string>,
|
|
@@ -597,10 +588,11 @@ type svmInstructionEventConfig = {
|
|
|
597
588
|
// Per-(event, chain) registration produced when user handler code registers an
|
|
598
589
|
// event (`onEvent`) or a dynamic contract registers. References its definition
|
|
599
590
|
// by value as `.eventConfig` and adds the handler binding plus the
|
|
600
|
-
// registration/`where`-derived fetch state. Not `private`: Fuel
|
|
601
|
-
//
|
|
602
|
-
//
|
|
603
|
-
//
|
|
591
|
+
// registration/`where`-derived fetch state. Not `private`: Fuel registrations
|
|
592
|
+
// add no ecosystem-specific fields (so the alias must stay directly
|
|
593
|
+
// constructable), and the ecosystem→base casts in sources are sound by
|
|
594
|
+
// ecosystem homogeneity — an EVM chain only ever holds
|
|
595
|
+
// `evmOnEventRegistration`s.
|
|
604
596
|
type onEventRegistration = {
|
|
605
597
|
// Chain-scoped sequential index — the registration's position in the
|
|
606
598
|
// chain's onEventRegistrations array, assigned when registration finishes
|
|
@@ -639,11 +631,18 @@ type evmOnEventRegistration = {
|
|
|
639
631
|
resolvedWhere: resolvedWhere,
|
|
640
632
|
}
|
|
641
633
|
|
|
642
|
-
// Fuel
|
|
643
|
-
//
|
|
644
|
-
// aliases of the base registration.
|
|
634
|
+
// Fuel registrations add no ecosystem-specific fetch state, so it's a bare
|
|
635
|
+
// alias of the base registration.
|
|
645
636
|
type fuelOnEventRegistration = onEventRegistration
|
|
646
|
-
|
|
637
|
+
|
|
638
|
+
type svmOnEventRegistration = {
|
|
639
|
+
...onEventRegistration,
|
|
640
|
+
/** Disjunctive normal form: outer array is OR of AND-groups, inner array is
|
|
641
|
+
AND across positions. Empty outer array means "no account filter". */
|
|
642
|
+
accountFilters: array<svmAccountFilterGroup>,
|
|
643
|
+
/** `None` matches both outer and inner (CPI-invoked) instructions. */
|
|
644
|
+
isInner: option<bool>,
|
|
645
|
+
}
|
|
647
646
|
|
|
648
647
|
type svmProgramConfig = {
|
|
649
648
|
name: string,
|
package/src/Metrics.res
CHANGED
|
@@ -98,6 +98,13 @@ type sourceHeightMetrics = {
|
|
|
98
98
|
height: int,
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
+
type sourceHeightStreamMetrics = {
|
|
102
|
+
source: string,
|
|
103
|
+
chainId: ChainId.t,
|
|
104
|
+
connectCount: int,
|
|
105
|
+
disconnectsByReason: array<(string, int)>,
|
|
106
|
+
}
|
|
107
|
+
|
|
101
108
|
type t = {
|
|
102
109
|
startTime: Date.t,
|
|
103
110
|
// Wall clock when this snapshot was built, so a scrape can be dated.
|
|
@@ -124,6 +131,7 @@ type t = {
|
|
|
124
131
|
historyPrunes: array<historyPruneMetrics>,
|
|
125
132
|
sourceRequests: array<sourceRequestMetrics>,
|
|
126
133
|
sourceHeights: array<sourceHeightMetrics>,
|
|
134
|
+
sourceHeightStreams: array<sourceHeightStreamMetrics>,
|
|
127
135
|
}
|
|
128
136
|
|
|
129
137
|
// Prometheus floats keep at most 3 decimals; integral values render without a
|
|
@@ -220,13 +228,23 @@ let renderMetrics = (b: builder, metrics: t) => {
|
|
|
220
228
|
metrics.storageWrites->Array.map(s => (`{storage="${s.storage->escapeLabelValue}"}`, s))
|
|
221
229
|
let historyPrunes =
|
|
222
230
|
metrics.historyPrunes->Array.map(s => (`{entity="${s.entity->escapeLabelValue}"}`, s))
|
|
231
|
+
// Every per-source series shares this label set: a scrape whose labels
|
|
232
|
+
// disagree between two of them is one nothing can join on.
|
|
233
|
+
let sourceLabels = (~source, ~chainId, ~extra: option<(string, string)>=?) => {
|
|
234
|
+
let pair = `{source="${source->escapeLabelValue}",chainId="${chainId->ChainId.toString}"`
|
|
235
|
+
switch extra {
|
|
236
|
+
| Some((name, value)) => `${pair},${name}="${value->escapeLabelValue}"}`
|
|
237
|
+
| None => pair ++ "}"
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
223
241
|
// Two sources can share a name (e.g. primary and fallback RPC urls on the
|
|
224
242
|
// same host), so aggregate by label set — duplicate samples would make
|
|
225
243
|
// Prometheus reject the scrape.
|
|
226
244
|
let sourceRequests = {
|
|
227
245
|
let byLabels: dict<sourceRequestMetrics> = Dict.make()
|
|
228
246
|
metrics.sourceRequests->Array.forEach(s => {
|
|
229
|
-
let labels =
|
|
247
|
+
let labels = sourceLabels(~source=s.source, ~chainId=s.chainId, ~extra=("method", s.method))
|
|
230
248
|
switch byLabels->Utils.Dict.dangerouslyGetNonOption(labels) {
|
|
231
249
|
| Some(existing) =>
|
|
232
250
|
byLabels->Dict.set(
|
|
@@ -241,7 +259,7 @@ let renderMetrics = (b: builder, metrics: t) => {
|
|
|
241
259
|
let sources = {
|
|
242
260
|
let byLabels: dict<int> = Dict.make()
|
|
243
261
|
metrics.sourceHeights->Array.forEach(s => {
|
|
244
|
-
let labels =
|
|
262
|
+
let labels = sourceLabels(~source=s.source, ~chainId=s.chainId)
|
|
245
263
|
switch byLabels->Utils.Dict.dangerouslyGetNonOption(labels) {
|
|
246
264
|
| Some(existing) if existing >= s.height => ()
|
|
247
265
|
| _ => byLabels->Dict.set(labels, s.height)
|
|
@@ -250,6 +268,34 @@ let renderMetrics = (b: builder, metrics: t) => {
|
|
|
250
268
|
byLabels->Dict.toArray
|
|
251
269
|
}
|
|
252
270
|
|
|
271
|
+
// Zero connects is the one count worth rendering flat: it is a stream that
|
|
272
|
+
// has never come up, which the disconnects say nothing about — retries at a
|
|
273
|
+
// connection that never opened are not disconnections — and which is
|
|
274
|
+
// otherwise indistinguishable from a chain that was never configured to
|
|
275
|
+
// stream at all.
|
|
276
|
+
let heightStreamConnects = {
|
|
277
|
+
let byLabels: dict<int> = Dict.make()
|
|
278
|
+
metrics.sourceHeightStreams->Array.forEach(s =>
|
|
279
|
+
byLabels->Utils.Dict.incrementBy(
|
|
280
|
+
sourceLabels(~source=s.source, ~chainId=s.chainId),
|
|
281
|
+
s.connectCount,
|
|
282
|
+
)
|
|
283
|
+
)
|
|
284
|
+
byLabels->Dict.toArray
|
|
285
|
+
}
|
|
286
|
+
let heightStreamDisconnects = {
|
|
287
|
+
let byLabels: dict<int> = Dict.make()
|
|
288
|
+
metrics.sourceHeightStreams->Array.forEach(s =>
|
|
289
|
+
s.disconnectsByReason->Array.forEach(((reason, count)) =>
|
|
290
|
+
byLabels->Utils.Dict.incrementBy(
|
|
291
|
+
sourceLabels(~source=s.source, ~chainId=s.chainId, ~extra=("reason", reason)),
|
|
292
|
+
count,
|
|
293
|
+
)
|
|
294
|
+
)
|
|
295
|
+
)
|
|
296
|
+
byLabels->Dict.toArray
|
|
297
|
+
}
|
|
298
|
+
|
|
253
299
|
b->single(
|
|
254
300
|
~name="envio_process_start_time_seconds",
|
|
255
301
|
~help="Start time of the process since unix epoch in seconds.",
|
|
@@ -463,6 +509,24 @@ let renderMetrics = (b: builder, metrics: t) => {
|
|
|
463
509
|
~entries=sourceRequests,
|
|
464
510
|
~value=s => s.seconds !== 0. ? Some(s.seconds) : None,
|
|
465
511
|
)
|
|
512
|
+
if heightStreamConnects->Array.length > 0 {
|
|
513
|
+
b->series(
|
|
514
|
+
~name="envio_source_height_stream_connects_total",
|
|
515
|
+
~help="The number of times a source's height subscription connected. Compare against the disconnects total, which is absent until the first disconnect and counts as zero while it is: one more connect than disconnects means the stream is up, and equal counts mean it is down and the indexer is polling instead. Zero connects means the stream has not come up, which is the normal reading for a chain that is still backfilling: subscriptions are only opened once a chain reaches the head.",
|
|
516
|
+
~kind="counter",
|
|
517
|
+
~entries=heightStreamConnects,
|
|
518
|
+
~value=count => count->Int.toFloat,
|
|
519
|
+
)
|
|
520
|
+
}
|
|
521
|
+
if heightStreamDisconnects->Array.length > 0 {
|
|
522
|
+
b->series(
|
|
523
|
+
~name="envio_source_height_stream_disconnects_total",
|
|
524
|
+
~help="The number of times a source's height subscription lost a connection, by reason. Failed retries are not counted, so this is outages rather than their length. A rotated disconnect is a connection that served its time; unsubscribed is the source being benched; every other reason ended a connection early.",
|
|
525
|
+
~kind="counter",
|
|
526
|
+
~entries=heightStreamDisconnects,
|
|
527
|
+
~value=count => count->Int.toFloat,
|
|
528
|
+
)
|
|
529
|
+
}
|
|
466
530
|
b->series(
|
|
467
531
|
~name="envio_source_known_height",
|
|
468
532
|
~help="The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.",
|
|
@@ -706,10 +770,12 @@ let renderMetrics = (b: builder, metrics: t) => {
|
|
|
706
770
|
~help="The number of address registrations per contract on chain, static and dynamic. An address shared by N contracts counts N times.",
|
|
707
771
|
~kind="gauge",
|
|
708
772
|
~entries=metrics.chains->Array.flatMap(m =>
|
|
709
|
-
m.addressesByContract->Array.map(
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
773
|
+
m.addressesByContract->Array.map(
|
|
774
|
+
((contract, count)) => (
|
|
775
|
+
`{chainId="${m.chainId->ChainId.toString}",contract="${contract->escapeLabelValue}"}`,
|
|
776
|
+
count,
|
|
777
|
+
),
|
|
778
|
+
)
|
|
713
779
|
),
|
|
714
780
|
~value=count => count->Int.toFloat,
|
|
715
781
|
)
|