envio 3.3.0-alpha.1 → 3.3.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +23 -21
- package/package.json +6 -6
- package/src/Batch.res +0 -6
- package/src/Batch.res.mjs +0 -11
- package/src/BatchProcessing.res +2 -12
- package/src/BatchProcessing.res.mjs +2 -7
- package/src/ChainFetching.res +42 -6
- package/src/ChainFetching.res.mjs +30 -11
- package/src/ChainMetadata.res +1 -11
- package/src/ChainMetadata.res.mjs +1 -10
- package/src/ChainState.res +150 -1
- package/src/ChainState.res.mjs +138 -9
- package/src/ChainState.resi +51 -1
- package/src/Config.res +3 -6
- package/src/Config.res.mjs +2 -3
- package/src/Core.res +7 -0
- package/src/CrossChainState.res +24 -26
- package/src/CrossChainState.res.mjs +19 -16
- package/src/Ecosystem.res +6 -3
- package/src/Envio.res +3 -2
- package/src/EventConfigBuilder.res +61 -23
- package/src/EventConfigBuilder.res.mjs +27 -13
- package/src/EventProcessing.res +63 -21
- package/src/EventProcessing.res.mjs +47 -22
- package/src/FetchState.res +53 -43
- package/src/FetchState.res.mjs +71 -54
- package/src/HandlerLoader.res +10 -1
- package/src/HandlerLoader.res.mjs +15 -8
- package/src/Internal.res +46 -7
- package/src/Internal.res.mjs +19 -1
- package/src/LoadLayer.res +5 -5
- package/src/LoadLayer.res.mjs +6 -6
- package/src/Main.res +3 -50
- package/src/Main.res.mjs +2 -23
- package/src/PgStorage.res +4 -4
- package/src/PgStorage.res.mjs +5 -5
- package/src/Prometheus.res +10 -10
- package/src/Prometheus.res.mjs +9 -9
- package/src/PruneStaleHistory.res +2 -2
- package/src/PruneStaleHistory.res.mjs +3 -3
- package/src/Rollback.res +3 -3
- package/src/Rollback.res.mjs +4 -4
- package/src/SimulateItems.res +80 -15
- package/src/SimulateItems.res.mjs +65 -21
- package/src/TestIndexer.res +48 -36
- package/src/TestIndexer.res.mjs +35 -28
- package/src/bindings/Performance.res +7 -0
- package/src/bindings/Performance.res.mjs +21 -0
- package/src/bindings/Performance.resi +7 -0
- package/src/sources/EventRouter.res +17 -21
- package/src/sources/EventRouter.res.mjs +6 -9
- package/src/sources/Evm.res +47 -67
- package/src/sources/Evm.res.mjs +42 -65
- package/src/sources/Fuel.res +3 -3
- package/src/sources/Fuel.res.mjs +1 -6
- package/src/sources/HyperFuelSource.res +15 -11
- package/src/sources/HyperFuelSource.res.mjs +13 -11
- package/src/sources/HyperSync.res +7 -1
- package/src/sources/HyperSync.res.mjs +6 -3
- package/src/sources/HyperSync.resi +2 -0
- package/src/sources/HyperSyncClient.res +14 -79
- package/src/sources/HyperSyncClient.res.mjs +1 -22
- package/src/sources/HyperSyncSource.res +44 -27
- package/src/sources/HyperSyncSource.res.mjs +32 -27
- package/src/sources/RpcSource.res +14 -10
- package/src/sources/RpcSource.res.mjs +11 -9
- package/src/sources/SimulateSource.res +4 -2
- package/src/sources/SimulateSource.res.mjs +1 -0
- package/src/sources/Source.res +5 -1
- package/src/sources/SourceManager.res +17 -16
- package/src/sources/SourceManager.res.mjs +10 -15
- package/src/sources/Svm.res +23 -6
- package/src/sources/Svm.res.mjs +24 -6
- package/src/sources/SvmHyperSyncClient.res +3 -29
- package/src/sources/SvmHyperSyncSource.res +78 -96
- package/src/sources/SvmHyperSyncSource.res.mjs +79 -91
- package/src/sources/TransactionStore.res +128 -0
- package/src/sources/TransactionStore.res.mjs +97 -0
- package/src/tui/Tui.res +13 -16
- package/src/tui/Tui.res.mjs +12 -14
- package/svm.schema.json +30 -37
- package/src/bindings/Hrtime.res +0 -58
- package/src/bindings/Hrtime.res.mjs +0 -90
- package/src/bindings/Hrtime.resi +0 -30
package/src/CrossChainState.res
CHANGED
|
@@ -57,7 +57,7 @@ let totalReadyCount = (crossChainState: t) => {
|
|
|
57
57
|
let total = ref(0)
|
|
58
58
|
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
59
59
|
let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
|
|
60
|
-
total := total.contents + cs->ChainState.
|
|
60
|
+
total := total.contents + cs->ChainState.bufferReadyCount
|
|
61
61
|
}
|
|
62
62
|
total.contents
|
|
63
63
|
}
|
|
@@ -65,9 +65,7 @@ let totalReadyCount = (crossChainState: t) => {
|
|
|
65
65
|
// --- Derived (pure). ---
|
|
66
66
|
|
|
67
67
|
let nextItemIsNone = (crossChainState: t): bool =>
|
|
68
|
-
!
|
|
69
|
-
crossChainState.chainStates->Dict.valuesToArray->Array.map(ChainState.fetchState),
|
|
70
|
-
)
|
|
68
|
+
!(crossChainState.chainStates->Dict.valuesToArray->Array.some(ChainState.hasReadyItem))
|
|
71
69
|
|
|
72
70
|
let getSafeCheckpointId = (crossChainState: t) => {
|
|
73
71
|
let result: ref<option<bigint>> = ref(None)
|
|
@@ -79,7 +77,7 @@ let getSafeCheckpointId = (crossChainState: t) => {
|
|
|
79
77
|
| Some(safeCheckpointTracking) =>
|
|
80
78
|
let safeCheckpointId =
|
|
81
79
|
safeCheckpointTracking->SafeCheckpointTracking.getSafeCheckpointId(
|
|
82
|
-
~sourceBlockNumber=
|
|
80
|
+
~sourceBlockNumber=cs->ChainState.knownHeight,
|
|
83
81
|
)
|
|
84
82
|
switch result.contents {
|
|
85
83
|
| None => result := Some(safeCheckpointId)
|
|
@@ -111,16 +109,9 @@ let createBatch = (
|
|
|
111
109
|
// in an append-only ClickHouse insert.
|
|
112
110
|
isRollback ? 1n : 0n,
|
|
113
111
|
),
|
|
114
|
-
~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
|
|
115
|
-
|
|
116
|
-
)
|
|
117
|
-
fetchState: cs->ChainState.fetchState,
|
|
118
|
-
progressBlockNumber: cs->ChainState.committedProgressBlockNumber,
|
|
119
|
-
totalEventsProcessed: cs->ChainState.numEventsProcessed,
|
|
120
|
-
sourceBlockNumber: (cs->ChainState.fetchState).knownHeight,
|
|
121
|
-
reorgDetection: cs->ChainState.reorgDetection,
|
|
122
|
-
chainConfig: cs->ChainState.chainConfig,
|
|
123
|
-
}),
|
|
112
|
+
~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
|
|
113
|
+
ChainState.toChainBeforeBatch,
|
|
114
|
+
),
|
|
124
115
|
~batchSizeTarget,
|
|
125
116
|
)
|
|
126
117
|
}
|
|
@@ -184,10 +175,7 @@ let priorityOrder = (crossChainState: t) =>
|
|
|
184
175
|
crossChainState.chainStates
|
|
185
176
|
->Dict.valuesToArray
|
|
186
177
|
->Array.toSorted((a, b) =>
|
|
187
|
-
Float.compare(
|
|
188
|
-
a->ChainState.fetchState->FetchState.getProgressPercentage,
|
|
189
|
-
b->ChainState.fetchState->FetchState.getProgressPercentage,
|
|
190
|
-
)
|
|
178
|
+
Float.compare(a->ChainState.getProgressPercentage, b->ChainState.getProgressPercentage)
|
|
191
179
|
)
|
|
192
180
|
|
|
193
181
|
// In-flight estimated items across every chain — the live draw against
|
|
@@ -235,11 +223,7 @@ let checkAndFetch = async (
|
|
|
235
223
|
for i in 0 to chainIds->Array.length - 1 {
|
|
236
224
|
let chainId = chainIds->Array.getUnsafe(i)
|
|
237
225
|
let cs = crossChainState->getChainState(chainId)
|
|
238
|
-
|
|
239
|
-
switch fetchState->FetchState.getNextQuery(
|
|
240
|
-
~budget=remaining,
|
|
241
|
-
~chainPendingBudget=cs->ChainState.pendingBudget,
|
|
242
|
-
) {
|
|
226
|
+
switch cs->ChainState.getNextQuery(~budget=remaining) {
|
|
243
227
|
| (WaitingForNewBlock | NothingToQuery) as action =>
|
|
244
228
|
actionByChain->Utils.Dict.setByInt(chainId, action)
|
|
245
229
|
| Ready(queries) =>
|
|
@@ -247,8 +231,7 @@ let checkAndFetch = async (
|
|
|
247
231
|
actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
|
|
248
232
|
queries->Array.forEach(query => {
|
|
249
233
|
query.chainId = chainId
|
|
250
|
-
query.progress =
|
|
251
|
-
fetchState->FetchState.getProgressPercentageAt(~blockNumber=query.fromBlock)
|
|
234
|
+
query.progress = cs->ChainState.getProgressPercentageAt(~blockNumber=query.fromBlock)
|
|
252
235
|
candidates->Array.push(query)
|
|
253
236
|
})
|
|
254
237
|
}
|
|
@@ -271,6 +254,21 @@ let checkAndFetch = async (
|
|
|
271
254
|
idx := idx.contents + 1
|
|
272
255
|
}
|
|
273
256
|
admittedByChain->Dict.forEachWithKey((queries, chainId) => {
|
|
257
|
+
let partitions = Dict.make()
|
|
258
|
+
queries->Array.forEach((query: FetchState.query) =>
|
|
259
|
+
partitions->Dict.set(
|
|
260
|
+
query.partitionId,
|
|
261
|
+
{
|
|
262
|
+
"fromBlock": query.fromBlock,
|
|
263
|
+
"targetBlock": query.toBlock,
|
|
264
|
+
},
|
|
265
|
+
)
|
|
266
|
+
)
|
|
267
|
+
Logging.trace({
|
|
268
|
+
"msg": "Started querying",
|
|
269
|
+
"chainId": chainId->Int.fromString->Option.getUnsafe,
|
|
270
|
+
"partitions": partitions,
|
|
271
|
+
})
|
|
274
272
|
actionByChain->Dict.set(chainId, FetchState.Ready(queries))
|
|
275
273
|
// Mark the admitted queries in flight and reserve their size against the
|
|
276
274
|
// shared budget; released as each response lands in handleQueryResult.
|
|
@@ -6,7 +6,6 @@ import * as Utils from "./Utils.res.mjs";
|
|
|
6
6
|
import * as Logging from "./Logging.res.mjs";
|
|
7
7
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
8
|
import * as ChainState from "./ChainState.res.mjs";
|
|
9
|
-
import * as FetchState from "./FetchState.res.mjs";
|
|
10
9
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
11
10
|
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
12
11
|
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
@@ -53,13 +52,13 @@ function totalReadyCount(crossChainState) {
|
|
|
53
52
|
let total = 0;
|
|
54
53
|
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
55
54
|
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
56
|
-
total = total +
|
|
55
|
+
total = total + ChainState.bufferReadyCount(cs) | 0;
|
|
57
56
|
}
|
|
58
57
|
return total;
|
|
59
58
|
}
|
|
60
59
|
|
|
61
60
|
function nextItemIsNone(crossChainState) {
|
|
62
|
-
return !
|
|
61
|
+
return !Object.values(crossChainState.chainStates).some(ChainState.hasReadyItem);
|
|
63
62
|
}
|
|
64
63
|
|
|
65
64
|
function getSafeCheckpointId(crossChainState) {
|
|
@@ -68,7 +67,7 @@ function getSafeCheckpointId(crossChainState) {
|
|
|
68
67
|
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
69
68
|
let safeCheckpointTracking = ChainState.safeCheckpointTracking(cs);
|
|
70
69
|
if (safeCheckpointTracking !== undefined) {
|
|
71
|
-
let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.
|
|
70
|
+
let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.knownHeight(cs));
|
|
72
71
|
let current = result;
|
|
73
72
|
if (!(current !== undefined && safeCheckpointId >= current)) {
|
|
74
73
|
result = safeCheckpointId;
|
|
@@ -84,14 +83,7 @@ function getSafeCheckpointId(crossChainState) {
|
|
|
84
83
|
function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, isRollback) {
|
|
85
84
|
return Batch.make(processedCheckpointId + (
|
|
86
85
|
isRollback ? 1n : 0n
|
|
87
|
-
), Utils.Dict.mapValues(crossChainState.chainStates,
|
|
88
|
-
fetchState: ChainState.fetchState(cs),
|
|
89
|
-
reorgDetection: ChainState.reorgDetection(cs),
|
|
90
|
-
progressBlockNumber: ChainState.committedProgressBlockNumber(cs),
|
|
91
|
-
sourceBlockNumber: ChainState.fetchState(cs).knownHeight,
|
|
92
|
-
totalEventsProcessed: ChainState.numEventsProcessed(cs),
|
|
93
|
-
chainConfig: ChainState.chainConfig(cs)
|
|
94
|
-
})), batchSizeTarget, crossChainState.isInReorgThreshold);
|
|
86
|
+
), Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, crossChainState.isInReorgThreshold);
|
|
95
87
|
}
|
|
96
88
|
|
|
97
89
|
function enterReorgThreshold(crossChainState) {
|
|
@@ -131,7 +123,7 @@ function applyBatchProgress(crossChainState, batch) {
|
|
|
131
123
|
}
|
|
132
124
|
|
|
133
125
|
function priorityOrder(crossChainState) {
|
|
134
|
-
return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(
|
|
126
|
+
return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.getProgressPercentage(a), ChainState.getProgressPercentage(b)));
|
|
135
127
|
}
|
|
136
128
|
|
|
137
129
|
function totalReservedSize(crossChainState) {
|
|
@@ -155,15 +147,14 @@ async function checkAndFetch(crossChainState, dispatchChain) {
|
|
|
155
147
|
for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
|
|
156
148
|
let chainId = chainIds[i];
|
|
157
149
|
let cs = crossChainState.chainStates[chainId];
|
|
158
|
-
let
|
|
159
|
-
let action = FetchState.getNextQuery(fetchState, remaining, ChainState.pendingBudget(cs));
|
|
150
|
+
let action = ChainState.getNextQuery(cs, remaining);
|
|
160
151
|
if (typeof action !== "object") {
|
|
161
152
|
actionByChain[chainId] = action;
|
|
162
153
|
} else {
|
|
163
154
|
actionByChain[chainId] = "NothingToQuery";
|
|
164
155
|
action._0.forEach(query => {
|
|
165
156
|
query.chainId = chainId;
|
|
166
|
-
query.progress =
|
|
157
|
+
query.progress = ChainState.getProgressPercentageAt(cs, query.fromBlock);
|
|
167
158
|
candidates.push(query);
|
|
168
159
|
});
|
|
169
160
|
}
|
|
@@ -179,6 +170,18 @@ async function checkAndFetch(crossChainState, dispatchChain) {
|
|
|
179
170
|
idx = idx + 1 | 0;
|
|
180
171
|
};
|
|
181
172
|
Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
|
|
173
|
+
let partitions = {};
|
|
174
|
+
queries.forEach(query => {
|
|
175
|
+
partitions[query.partitionId] = {
|
|
176
|
+
fromBlock: query.fromBlock,
|
|
177
|
+
targetBlock: query.toBlock
|
|
178
|
+
};
|
|
179
|
+
});
|
|
180
|
+
Logging.trace({
|
|
181
|
+
msg: "Started querying",
|
|
182
|
+
chainId: Stdlib_Int.fromString(chainId, undefined),
|
|
183
|
+
partitions: partitions
|
|
184
|
+
});
|
|
182
185
|
actionByChain[chainId] = {
|
|
183
186
|
TAG: "Ready",
|
|
184
187
|
_0: queries
|
package/src/Ecosystem.res
CHANGED
|
@@ -2,8 +2,6 @@ type name = | @as("evm") Evm | @as("fuel") Fuel | @as("svm") Svm
|
|
|
2
2
|
|
|
3
3
|
type t = {
|
|
4
4
|
name: name,
|
|
5
|
-
blockFields: array<string>,
|
|
6
|
-
transactionFields: array<string>,
|
|
7
5
|
blockNumberName: string,
|
|
8
6
|
blockTimestampName: string,
|
|
9
7
|
blockHashName: string,
|
|
@@ -33,8 +31,13 @@ type t = {
|
|
|
33
31
|
loggers (see `getItemLogger`). */
|
|
34
32
|
logger: Pino.t,
|
|
35
33
|
/** Materialise the user-facing event handed to handlers and contract
|
|
36
|
-
registration from an item's opaque payload.
|
|
34
|
+
registration from an item's opaque payload. `event.transaction` is written
|
|
35
|
+
onto the payload at batch prep (HyperSync) or inline (RPC/simulate). */
|
|
37
36
|
toEvent: Internal.eventItem => Internal.event,
|
|
37
|
+
/** Bitmask (as a float) of the transaction fields selected across the chain's
|
|
38
|
+
events — the set the store materialises at batch prep. `0.` when the
|
|
39
|
+
ecosystem carries transactions inline. */
|
|
40
|
+
transactionFieldMask: array<Internal.eventConfig> => float,
|
|
38
41
|
/** Build the per-item child logger for an event item, with
|
|
39
42
|
ecosystem-specific log fields (EVM/Fuel: contract/event/address; SVM:
|
|
40
43
|
program/instruction/programId). Closes over the injected logger. */
|
package/src/Envio.res
CHANGED
|
@@ -48,6 +48,7 @@ type svmTokenBalance = {
|
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
type svmTransaction = {
|
|
51
|
+
transactionIndex?: int,
|
|
51
52
|
signatures: array<string>,
|
|
52
53
|
feePayer?: SvmTypes.Pubkey.t,
|
|
53
54
|
success?: bool,
|
|
@@ -104,8 +105,8 @@ type svmInstruction = {
|
|
|
104
105
|
d8?: string,
|
|
105
106
|
/** Borsh-decoded params view. See [[svmInstructionParams]]. */
|
|
106
107
|
params?: svmInstructionParams,
|
|
107
|
-
/** Parent transaction.
|
|
108
|
-
`
|
|
108
|
+
/** Parent transaction. Carries only the fields selected via
|
|
109
|
+
`field_selection.transaction_fields`; absent when none are selected. */
|
|
109
110
|
transaction?: svmTransaction,
|
|
110
111
|
/** Program log entries scoped to this instruction. Absent when the
|
|
111
112
|
per-instruction `include_logs` flag is `false`. */
|
|
@@ -306,7 +306,14 @@ let resolveFieldSelection = (
|
|
|
306
306
|
| Some(fields) => Utils.Set.fromArray(fields)
|
|
307
307
|
| None => globalTransactionFieldsSet
|
|
308
308
|
}
|
|
309
|
-
(
|
|
309
|
+
// The base eventConfig stores these as a string set (field names match the
|
|
310
|
+
// typed variants at runtime).
|
|
311
|
+
(
|
|
312
|
+
selectedBlockFields,
|
|
313
|
+
selectedTransactionFields->(
|
|
314
|
+
Utils.magic: Utils.Set.t<Internal.evmTransactionField> => Utils.Set.t<string>
|
|
315
|
+
),
|
|
316
|
+
)
|
|
310
317
|
}
|
|
311
318
|
|
|
312
319
|
// ============== Client-side address filter ==============
|
|
@@ -319,28 +326,50 @@ let compileAddressFilter: string => (
|
|
|
319
326
|
return new Function("event", "blockNumber", "indexingAddresses", body);
|
|
320
327
|
}`)
|
|
321
328
|
|
|
322
|
-
// Body of the client-side address filter
|
|
323
|
-
//
|
|
324
|
-
// registered
|
|
325
|
-
//
|
|
326
|
-
//
|
|
327
|
-
|
|
328
|
-
|
|
329
|
+
// Body of the client-side address filter. Two analogous registered-at-or-before
|
|
330
|
+
// checks, ANDed: (1) for non-wildcard events, the log's srcAddress must itself be
|
|
331
|
+
// registered (ownership is resolved structurally by partition, but the temporal
|
|
332
|
+
// `effectiveStartBlock` gate lives here now); (2) a DNF of address-filtered param
|
|
333
|
+
// names (OR of AND-groups) for events that filter an indexed address param. The
|
|
334
|
+
// DNF is fixed here, so it's unrolled into one boolean expression — no per-event
|
|
335
|
+
// closure, loop, or array. `None` only for wildcard events without a param
|
|
336
|
+
// filter. Exposed for snapshotting.
|
|
337
|
+
// `srcAddressExpr` is the JS expression for the event's owning address: EVM and
|
|
338
|
+
// Fuel events expose `event.srcAddress`; SVM instructions expose `event.programId`.
|
|
339
|
+
let buildAddressFilterBody = (
|
|
340
|
+
groups: array<array<string>>,
|
|
341
|
+
~isWildcard: bool,
|
|
342
|
+
~srcAddressExpr: string="event.srcAddress",
|
|
343
|
+
): option<string> => {
|
|
344
|
+
let paramLeaf = name =>
|
|
345
|
+
`(ic = indexingAddresses[p[${JSON.stringify(
|
|
346
|
+
JSON.String(name),
|
|
347
|
+
)}]]) !== undefined && ic.effectiveStartBlock <= blockNumber`
|
|
348
|
+
let paramDnf = switch groups {
|
|
329
349
|
| [] => None
|
|
330
350
|
| _ =>
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
351
|
+
Some(
|
|
352
|
+
groups
|
|
353
|
+
->Array.map(group => "(" ++ group->Array.map(paramLeaf)->Array.join(" && ") ++ ")")
|
|
354
|
+
->Array.join(" || "),
|
|
355
|
+
)
|
|
356
|
+
}
|
|
357
|
+
let srcLeaf = `(ic = indexingAddresses[${srcAddressExpr}]) !== undefined && ic.effectiveStartBlock <= blockNumber`
|
|
358
|
+
switch (isWildcard, paramDnf) {
|
|
359
|
+
| (true, None) => None
|
|
360
|
+
| (true, Some(dnf)) => Some("var p = event.params, ic; return " ++ dnf ++ ";")
|
|
361
|
+
| (false, None) => Some("var ic; return " ++ srcLeaf ++ ";")
|
|
362
|
+
| (false, Some(dnf)) =>
|
|
363
|
+
Some("var p = event.params, ic; return " ++ srcLeaf ++ " && (" ++ dnf ++ ");")
|
|
338
364
|
}
|
|
339
365
|
}
|
|
340
366
|
|
|
341
|
-
let buildAddressFilter = (
|
|
342
|
-
|
|
343
|
-
|
|
367
|
+
let buildAddressFilter = (
|
|
368
|
+
groups: array<array<string>>,
|
|
369
|
+
~isWildcard: bool,
|
|
370
|
+
~srcAddressExpr: string="event.srcAddress",
|
|
371
|
+
): option<(Internal.eventPayload, int, dict<Internal.indexingContract>) => bool> =>
|
|
372
|
+
buildAddressFilterBody(groups, ~isWildcard, ~srcAddressExpr)->Option.map(compileAddressFilter)
|
|
344
373
|
|
|
345
374
|
// ============== Build complete EVM event config ==============
|
|
346
375
|
|
|
@@ -408,7 +437,7 @@ let buildEvmEventConfig = (
|
|
|
408
437
|
simulateParamsSchema: buildSimulateParamsSchema(params),
|
|
409
438
|
getEventFiltersOrThrow,
|
|
410
439
|
filterByAddresses,
|
|
411
|
-
clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups),
|
|
440
|
+
clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups, ~isWildcard),
|
|
412
441
|
dependsOnAddresses: !isWildcard || filterByAddresses,
|
|
413
442
|
startBlock: resolvedStartBlock,
|
|
414
443
|
selectedBlockFields,
|
|
@@ -427,9 +456,8 @@ let buildSvmInstructionEventConfig = (
|
|
|
427
456
|
~programId: SvmTypes.Pubkey.t,
|
|
428
457
|
~discriminator: option<string>,
|
|
429
458
|
~discriminatorByteLen: int,
|
|
430
|
-
~includeTransaction: bool,
|
|
431
459
|
~includeLogs: bool,
|
|
432
|
-
~
|
|
460
|
+
~transactionFields: array<Internal.svmTransactionField>=[],
|
|
433
461
|
~accountFilters: array<Internal.svmAccountFilterGroup>,
|
|
434
462
|
~isInner: option<bool>,
|
|
435
463
|
~isWildcard: bool,
|
|
@@ -444,6 +472,13 @@ let buildSvmInstructionEventConfig = (
|
|
|
444
472
|
S.json(~validate=false)
|
|
445
473
|
->Utils.Schema.coerceToJsonPgType
|
|
446
474
|
->(Utils.magic: S.t<JSON.t> => S.t<Internal.eventParams>)
|
|
475
|
+
|
|
476
|
+
// The base eventConfig stores these as a string set (field names match the
|
|
477
|
+
// typed variants at runtime).
|
|
478
|
+
let selectedTransactionFields =
|
|
479
|
+
Utils.Set.fromArray(transactionFields)->(
|
|
480
|
+
Utils.magic: Utils.Set.t<Internal.svmTransactionField> => Utils.Set.t<string>
|
|
481
|
+
)
|
|
447
482
|
{
|
|
448
483
|
id: switch discriminator {
|
|
449
484
|
| Some(d) => d
|
|
@@ -458,13 +493,13 @@ let buildSvmInstructionEventConfig = (
|
|
|
458
493
|
simulateParamsSchema: paramsSchema,
|
|
459
494
|
filterByAddresses: false,
|
|
460
495
|
dependsOnAddresses: !isWildcard,
|
|
496
|
+
clientAddressFilter: ?buildAddressFilter([], ~isWildcard, ~srcAddressExpr="event.programId"),
|
|
461
497
|
startBlock,
|
|
462
498
|
programId,
|
|
463
499
|
discriminator,
|
|
464
500
|
discriminatorByteLen,
|
|
465
|
-
includeTransaction,
|
|
466
501
|
includeLogs,
|
|
467
|
-
|
|
502
|
+
selectedTransactionFields,
|
|
468
503
|
accountFilters,
|
|
469
504
|
isInner,
|
|
470
505
|
accounts,
|
|
@@ -529,7 +564,10 @@ let buildFuelEventConfig = (
|
|
|
529
564
|
simulateParamsSchema: paramsSchema,
|
|
530
565
|
filterByAddresses: false,
|
|
531
566
|
dependsOnAddresses: !isWildcard,
|
|
567
|
+
clientAddressFilter: ?buildAddressFilter([], ~isWildcard),
|
|
532
568
|
startBlock,
|
|
569
|
+
// Fuel keeps the transaction inline on the payload; nothing to materialise.
|
|
570
|
+
selectedTransactionFields: Utils.Set.make(),
|
|
533
571
|
kind: fuelKind,
|
|
534
572
|
}
|
|
535
573
|
}
|
|
@@ -255,17 +255,27 @@ let compileAddressFilter = (function (body) {
|
|
|
255
255
|
return new Function("event", "blockNumber", "indexingAddresses", body);
|
|
256
256
|
});
|
|
257
257
|
|
|
258
|
-
function buildAddressFilterBody(groups) {
|
|
259
|
-
|
|
260
|
-
|
|
258
|
+
function buildAddressFilterBody(groups, isWildcard, srcAddressExprOpt) {
|
|
259
|
+
let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
|
|
260
|
+
let paramLeaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
|
|
261
|
+
let paramDnf = groups.length !== 0 ? groups.map(group => "(" + group.map(paramLeaf).join(" && ") + ")").join(" || ") : undefined;
|
|
262
|
+
let srcLeaf = `(ic = indexingAddresses[` + srcAddressExpr + `]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
|
|
263
|
+
if (isWildcard) {
|
|
264
|
+
if (paramDnf !== undefined) {
|
|
265
|
+
return "var p = event.params, ic; return " + paramDnf + ";";
|
|
266
|
+
} else {
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
} else if (paramDnf !== undefined) {
|
|
270
|
+
return "var p = event.params, ic; return " + srcLeaf + " && (" + paramDnf + ");";
|
|
271
|
+
} else {
|
|
272
|
+
return "var ic; return " + srcLeaf + ";";
|
|
261
273
|
}
|
|
262
|
-
let leaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
|
|
263
|
-
let groupExprs = groups.map(group => "(" + group.map(leaf).join(" && ") + ")");
|
|
264
|
-
return "var p = event.params, ic; return " + groupExprs.join(" || ") + ";";
|
|
265
274
|
}
|
|
266
275
|
|
|
267
|
-
function buildAddressFilter(groups) {
|
|
268
|
-
|
|
276
|
+
function buildAddressFilter(groups, isWildcard, srcAddressExprOpt) {
|
|
277
|
+
let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
|
|
278
|
+
return Stdlib_Option.map(buildAddressFilterBody(groups, isWildcard, srcAddressExpr), compileAddressFilter);
|
|
269
279
|
}
|
|
270
280
|
|
|
271
281
|
function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcard, handler, contractRegister, eventFilters, probeChainId, onEventBlockFilterSchema, blockFields, transactionFields, startBlock, globalBlockFieldsSetOpt, globalTransactionFieldsSetOpt) {
|
|
@@ -291,26 +301,28 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
|
|
|
291
301
|
isWildcard: isWildcard,
|
|
292
302
|
filterByAddresses: filterByAddresses,
|
|
293
303
|
dependsOnAddresses: !isWildcard || filterByAddresses,
|
|
294
|
-
clientAddressFilter:
|
|
304
|
+
clientAddressFilter: buildAddressFilter(match.addressFilterParamGroups, isWildcard, undefined),
|
|
295
305
|
handler: handler,
|
|
296
306
|
contractRegister: contractRegister,
|
|
297
307
|
paramsRawEventSchema: buildParamsSchema(params),
|
|
298
308
|
simulateParamsSchema: buildSimulateParamsSchema(params),
|
|
299
309
|
startBlock: resolvedStartBlock,
|
|
310
|
+
selectedTransactionFields: match$1[1],
|
|
300
311
|
getEventFiltersOrThrow: match.getEventFiltersOrThrow,
|
|
301
312
|
selectedBlockFields: match$1[0],
|
|
302
|
-
selectedTransactionFields: match$1[1],
|
|
303
313
|
sighash: sighash,
|
|
304
314
|
topicCount: topicCount,
|
|
305
315
|
paramsMetadata: params
|
|
306
316
|
};
|
|
307
317
|
}
|
|
308
318
|
|
|
309
|
-
function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen,
|
|
319
|
+
function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen, includeLogs, transactionFieldsOpt, accountFilters, isInner, isWildcard, handler, contractRegister, accountsOpt, argsOpt, definedTypesOpt, startBlock) {
|
|
320
|
+
let transactionFields = transactionFieldsOpt !== undefined ? transactionFieldsOpt : [];
|
|
310
321
|
let accounts = accountsOpt !== undefined ? accountsOpt : [];
|
|
311
322
|
let args = argsOpt !== undefined ? argsOpt : null;
|
|
312
323
|
let definedTypes = definedTypesOpt !== undefined ? definedTypesOpt : null;
|
|
313
324
|
let paramsSchema = Utils.Schema.coerceToJsonPgType(S$RescriptSchema.json(false));
|
|
325
|
+
let selectedTransactionFields = new Set(transactionFields);
|
|
314
326
|
return {
|
|
315
327
|
id: discriminator !== undefined ? discriminator : "none",
|
|
316
328
|
name: instructionName,
|
|
@@ -318,17 +330,17 @@ function buildSvmInstructionEventConfig(contractName, instructionName, programId
|
|
|
318
330
|
isWildcard: isWildcard,
|
|
319
331
|
filterByAddresses: false,
|
|
320
332
|
dependsOnAddresses: !isWildcard,
|
|
333
|
+
clientAddressFilter: buildAddressFilter([], isWildcard, "event.programId"),
|
|
321
334
|
handler: handler,
|
|
322
335
|
contractRegister: contractRegister,
|
|
323
336
|
paramsRawEventSchema: paramsSchema,
|
|
324
337
|
simulateParamsSchema: paramsSchema,
|
|
325
338
|
startBlock: startBlock,
|
|
339
|
+
selectedTransactionFields: selectedTransactionFields,
|
|
326
340
|
programId: programId,
|
|
327
341
|
discriminator: discriminator,
|
|
328
342
|
discriminatorByteLen: discriminatorByteLen,
|
|
329
|
-
includeTransaction: includeTransaction,
|
|
330
343
|
includeLogs: includeLogs,
|
|
331
|
-
includeTokenBalances: includeTokenBalances,
|
|
332
344
|
accountFilters: accountFilters,
|
|
333
345
|
isInner: isInner,
|
|
334
346
|
accounts: accounts,
|
|
@@ -387,11 +399,13 @@ function buildFuelEventConfig(contractName, eventName, kind, sighash, rawAbi, is
|
|
|
387
399
|
isWildcard: isWildcard,
|
|
388
400
|
filterByAddresses: false,
|
|
389
401
|
dependsOnAddresses: !isWildcard,
|
|
402
|
+
clientAddressFilter: buildAddressFilter([], isWildcard, undefined),
|
|
390
403
|
handler: handler,
|
|
391
404
|
contractRegister: contractRegister,
|
|
392
405
|
paramsRawEventSchema: paramsSchema,
|
|
393
406
|
simulateParamsSchema: paramsSchema,
|
|
394
407
|
startBlock: startBlock,
|
|
408
|
+
selectedTransactionFields: new Set(),
|
|
395
409
|
kind: fuelKind
|
|
396
410
|
};
|
|
397
411
|
}
|
package/src/EventProcessing.res
CHANGED
|
@@ -39,7 +39,7 @@ let runEventHandlerOrThrow = async (
|
|
|
39
39
|
let eventItem = item->Internal.castUnsafeEventItem
|
|
40
40
|
|
|
41
41
|
//Include the load in time before handler
|
|
42
|
-
let timeBeforeHandler =
|
|
42
|
+
let timeBeforeHandler = Performance.now()
|
|
43
43
|
|
|
44
44
|
try {
|
|
45
45
|
let contextParams: UserContext.contextParams = {
|
|
@@ -72,7 +72,7 @@ let runEventHandlerOrThrow = async (
|
|
|
72
72
|
}),
|
|
73
73
|
)
|
|
74
74
|
}
|
|
75
|
-
let handlerDuration = timeBeforeHandler->
|
|
75
|
+
let handlerDuration = timeBeforeHandler->Performance.secondsSince
|
|
76
76
|
Prometheus.ProcessingHandler.increment(
|
|
77
77
|
~contract=eventItem.eventConfig.contractName,
|
|
78
78
|
~event=eventItem.eventConfig.name,
|
|
@@ -264,11 +264,19 @@ let runBatchHandlersOrThrow = async (
|
|
|
264
264
|
}
|
|
265
265
|
}
|
|
266
266
|
|
|
267
|
-
let registerProcessEventBatchMetrics = (
|
|
268
|
-
logger
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
267
|
+
let registerProcessEventBatchMetrics = (
|
|
268
|
+
~logger,
|
|
269
|
+
~batch: Batch.t,
|
|
270
|
+
~loadDuration,
|
|
271
|
+
~handlerDuration,
|
|
272
|
+
) => {
|
|
273
|
+
batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
|
|
274
|
+
logger->Logging.childTrace({
|
|
275
|
+
"msg": "Finished processing",
|
|
276
|
+
"chainId": chainId->Int.fromString->Option.getUnsafe,
|
|
277
|
+
"batchSize": chainAfterBatch.batchSize,
|
|
278
|
+
"progress": chainAfterBatch.progressBlockNumber,
|
|
279
|
+
})
|
|
272
280
|
})
|
|
273
281
|
|
|
274
282
|
Prometheus.ProcessingBatch.registerMetrics(~loadDuration, ~handlerDuration)
|
|
@@ -281,6 +289,35 @@ type logPartitionInfo = {
|
|
|
281
289
|
lastItemBlockNumber?: int,
|
|
282
290
|
}
|
|
283
291
|
|
|
292
|
+
// Off the hot path: bulk-materialise the selected transaction fields for the
|
|
293
|
+
// batch's store-backed (HyperSync) items and write them onto the payloads, so
|
|
294
|
+
// handlers read plain objects. A batch can span chains, each with its own store
|
|
295
|
+
// and field mask, so group items by chain before materialising.
|
|
296
|
+
let materializeBatchTransactions = async (batch: Batch.t, ~chainStates: dict<ChainState.t>) => {
|
|
297
|
+
switch chainStates->Dict.valuesToArray {
|
|
298
|
+
// Single-chain indexers (the common case): every item belongs to the one
|
|
299
|
+
// chain, so skip the per-chain grouping and its allocations.
|
|
300
|
+
| [cs] => await cs->ChainState.materializeBatchItems(~items=batch.items)
|
|
301
|
+
| _ =>
|
|
302
|
+
let itemsByChain: dict<array<Internal.item>> = Dict.make()
|
|
303
|
+
batch.items->Array.forEach(item => {
|
|
304
|
+
let chainId = item->Internal.getItemChainId->Int.toString
|
|
305
|
+
switch itemsByChain->Utils.Dict.dangerouslyGetNonOption(chainId) {
|
|
306
|
+
| Some(items) => items->Array.push(item)
|
|
307
|
+
| None => itemsByChain->Dict.set(chainId, [item])
|
|
308
|
+
}
|
|
309
|
+
})
|
|
310
|
+
|
|
311
|
+
let _ = await itemsByChain
|
|
312
|
+
->Dict.toArray
|
|
313
|
+
->Array.map(async ((chainId, items)) => {
|
|
314
|
+
let cs = chainStates->Dict.getUnsafe(chainId)
|
|
315
|
+
await cs->ChainState.materializeBatchItems(~items)
|
|
316
|
+
})
|
|
317
|
+
->Promise.all
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
284
321
|
let processEventBatch = async (
|
|
285
322
|
~batch: Batch.t,
|
|
286
323
|
~indexerState: IndexerState.t,
|
|
@@ -289,33 +326,33 @@ let processEventBatch = async (
|
|
|
289
326
|
~config: Config.t,
|
|
290
327
|
~chainStates: dict<ChainState.t>,
|
|
291
328
|
) => {
|
|
292
|
-
let totalBatchSize = batch.totalBatchSize
|
|
293
329
|
// Compute chains state for this batch
|
|
294
330
|
let chains: Internal.chains = chainStates->computeChainsState
|
|
295
331
|
|
|
296
332
|
let logger = Logging.getLogger()
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
}
|
|
305
|
-
}),
|
|
333
|
+
|
|
334
|
+
batch.progressedChainsById->Dict.forEachWithKey((chainAfterBatch, chainId) => {
|
|
335
|
+
logger->Logging.childTrace({
|
|
336
|
+
"msg": "Started processing",
|
|
337
|
+
"chainId": chainId->Int.fromString->Option.getUnsafe,
|
|
338
|
+
"batchSize": chainAfterBatch.batchSize,
|
|
339
|
+
})
|
|
306
340
|
})
|
|
307
341
|
|
|
308
342
|
try {
|
|
309
343
|
// Backpressure: keep processing within keepLatestChangesLimit of the cycle.
|
|
310
344
|
await indexerState->Writing.awaitCapacity
|
|
311
345
|
|
|
312
|
-
let timeRef =
|
|
346
|
+
let timeRef = Performance.now()
|
|
313
347
|
|
|
314
348
|
if batch.items->Utils.Array.notEmpty {
|
|
349
|
+
// Materialise store-backed transactions onto payloads before any handler
|
|
350
|
+
// (preload or execute) reads them.
|
|
351
|
+
await materializeBatchTransactions(batch, ~chainStates)
|
|
315
352
|
await batch->preloadBatchOrThrow(~loadManager, ~persistence, ~indexerState, ~chains, ~config)
|
|
316
353
|
}
|
|
317
354
|
|
|
318
|
-
let elapsedTimeAfterLoaders = timeRef->
|
|
355
|
+
let elapsedTimeAfterLoaders = timeRef->Performance.secondsSince
|
|
319
356
|
|
|
320
357
|
if batch.items->Utils.Array.notEmpty {
|
|
321
358
|
await batch->runBatchHandlersOrThrow(
|
|
@@ -327,13 +364,18 @@ let processEventBatch = async (
|
|
|
327
364
|
)
|
|
328
365
|
}
|
|
329
366
|
|
|
330
|
-
let elapsedTimeAfterProcessing = timeRef->
|
|
367
|
+
let elapsedTimeAfterProcessing = timeRef->Performance.secondsSince
|
|
331
368
|
|
|
332
369
|
indexerState->Writing.commitBatch(~batch)
|
|
333
370
|
|
|
334
371
|
let loaderDuration = elapsedTimeAfterLoaders
|
|
335
372
|
let handlerDuration = elapsedTimeAfterProcessing -. loaderDuration
|
|
336
|
-
registerProcessEventBatchMetrics(
|
|
373
|
+
registerProcessEventBatchMetrics(
|
|
374
|
+
~logger,
|
|
375
|
+
~batch,
|
|
376
|
+
~loadDuration=loaderDuration,
|
|
377
|
+
~handlerDuration,
|
|
378
|
+
)
|
|
337
379
|
Ok()
|
|
338
380
|
} catch {
|
|
339
381
|
| Persistence.StorageError({message, reason}) =>
|