envio 3.5.0 → 3.5.1-svm-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +7 -0
- package/fuel.schema.json +7 -0
- package/index.d.ts +33 -10
- package/package.json +6 -6
- package/src/ChainFetching.res +11 -21
- package/src/ChainFetching.res.mjs +12 -20
- package/src/ChainState.res +41 -4
- package/src/ChainState.res.mjs +39 -5
- package/src/ChainState.resi +10 -1
- package/src/Config.res +62 -3
- package/src/Config.res.mjs +49 -13
- package/src/EffectState.res +2 -2
- package/src/EffectState.res.mjs +2 -2
- package/src/EntityTables.res +32 -0
- package/src/EntityTables.res.mjs +44 -0
- package/src/Envio.res +7 -7
- package/src/Envio.res.mjs +5 -6
- package/src/FetchState.res +38 -75
- package/src/FetchState.res.mjs +20 -62
- package/src/Hasura.res +37 -3
- package/src/Hasura.res.mjs +23 -5
- package/src/InMemoryStore.res +95 -43
- package/src/InMemoryStore.res.mjs +69 -38
- package/src/IndexerState.res +30 -24
- package/src/IndexerState.res.mjs +20 -35
- package/src/IndexerState.resi +8 -6
- package/src/Internal.res +29 -18
- package/src/Internal.res.mjs +27 -10
- package/src/LoadLayer.res +39 -8
- package/src/LoadLayer.res.mjs +36 -9
- package/src/LoadLayer.resi +2 -0
- package/src/Persistence.res +12 -1
- package/src/PgStorage.res +183 -37
- package/src/PgStorage.res.mjs +149 -34
- package/src/PruneStaleHistory.res +1 -0
- package/src/PruneStaleHistory.res.mjs +2 -1
- package/src/Sink.res +3 -3
- package/src/Sink.res.mjs +2 -2
- package/src/TestIndexer.res +128 -17
- package/src/TestIndexer.res.mjs +75 -9
- package/src/UserContext.res +19 -4
- package/src/UserContext.res.mjs +15 -9
- package/src/Writing.res +8 -16
- package/src/Writing.res.mjs +10 -10
- package/src/bindings/ClickHouse.res +40 -4
- package/src/bindings/ClickHouse.res.mjs +37 -5
- package/src/bindings/Vitest.res +0 -3
- package/src/bindings/Vitest.res.mjs +2 -12
- package/src/db/EntityHistory.res +64 -21
- package/src/db/EntityHistory.res.mjs +43 -22
- package/src/db/InternalTable.res.mjs +31 -31
- package/src/db/Table.res +24 -1
- package/src/db/Table.res.mjs +26 -4
- package/src/sources/AddressStore.res +77 -26
- package/src/sources/AddressStore.res.mjs +21 -10
- package/src/sources/SvmHyperSyncClient.res +8 -7
- package/src/sources/SvmHyperSyncSource.res +38 -1
- package/src/sources/SvmHyperSyncSource.res.mjs +24 -0
- package/svm.schema.json +17 -4
package/evm.schema.json
CHANGED
|
@@ -49,6 +49,13 @@
|
|
|
49
49
|
}
|
|
50
50
|
]
|
|
51
51
|
},
|
|
52
|
+
"disable_default_cross_chain": {
|
|
53
|
+
"description": "Make entities and effect caches per-chain instead of shared across every chain (recommended). Sharing then becomes explicit — add `@crossChain` to an entity in schema.graphql or `crossChain: true` to an effect. (default: false)",
|
|
54
|
+
"type": [
|
|
55
|
+
"boolean",
|
|
56
|
+
"null"
|
|
57
|
+
]
|
|
58
|
+
},
|
|
52
59
|
"ecosystem": {
|
|
53
60
|
"description": "Ecosystem of the project.",
|
|
54
61
|
"anyOf": [
|
package/fuel.schema.json
CHANGED
|
@@ -49,6 +49,13 @@
|
|
|
49
49
|
}
|
|
50
50
|
]
|
|
51
51
|
},
|
|
52
|
+
"disable_default_cross_chain": {
|
|
53
|
+
"description": "Make entities and effect caches per-chain instead of shared across every chain (recommended). Sharing then becomes explicit — add `@crossChain` to an entity in schema.graphql or `crossChain: true` to an effect. (default: false)",
|
|
54
|
+
"type": [
|
|
55
|
+
"boolean",
|
|
56
|
+
"null"
|
|
57
|
+
]
|
|
58
|
+
},
|
|
52
59
|
"ecosystem": {
|
|
53
60
|
"description": "Ecosystem of the project.",
|
|
54
61
|
"$ref": "#/$defs/EcosystemTag"
|
package/index.d.ts
CHANGED
|
@@ -48,8 +48,7 @@ export type EffectCaller = <I, O>(
|
|
|
48
48
|
input: I extends undefined ? undefined : I
|
|
49
49
|
) => Promise<O>;
|
|
50
50
|
|
|
51
|
-
/** The chain an Effect was called on. Available only on chain-scoped effects
|
|
52
|
-
* (`crossChain: false`). */
|
|
51
|
+
/** The chain an Effect was called on. Available only on chain-scoped effects. */
|
|
53
52
|
export type EffectChain = {
|
|
54
53
|
/** The chain id the effect handler was called on. */
|
|
55
54
|
readonly id: number;
|
|
@@ -64,8 +63,8 @@ export type EffectContext = {
|
|
|
64
63
|
/** Whether to cache this call's result. Defaults to the effect's `cache`
|
|
65
64
|
* option; set to `false` to skip caching for this specific invocation. */
|
|
66
65
|
cache: boolean;
|
|
67
|
-
/** The chain the effect was called on. Only available on
|
|
68
|
-
*
|
|
66
|
+
/** The chain the effect was called on. Only available on chain-scoped
|
|
67
|
+
* effects; accessing it on a cross-chain effect throws. */
|
|
69
68
|
readonly chain: EffectChain;
|
|
70
69
|
};
|
|
71
70
|
|
|
@@ -93,9 +92,10 @@ export type EffectOptions<Input, Output> = {
|
|
|
93
92
|
/** Whether the effect should be cached. */
|
|
94
93
|
readonly cache?: boolean;
|
|
95
94
|
/** Whether the effect's cache is shared across all chains. Defaults to
|
|
96
|
-
* `true
|
|
97
|
-
*
|
|
98
|
-
*
|
|
95
|
+
* `true`, or to `false` when config.yaml sets
|
|
96
|
+
* `disable_default_cross_chain: true`. Set to `false` to isolate the cache
|
|
97
|
+
* and rate limiting per chain and enable `context.chain.id` inside the
|
|
98
|
+
* handler. Changing this changes the effect's cache identity. */
|
|
99
99
|
readonly crossChain?: boolean;
|
|
100
100
|
};
|
|
101
101
|
|
|
@@ -225,8 +225,10 @@ export function createEffect<
|
|
|
225
225
|
/** Whether the effect should be cached. */
|
|
226
226
|
readonly cache?: boolean;
|
|
227
227
|
/** Whether the effect's cache is shared across all chains. Defaults to
|
|
228
|
-
* `true
|
|
229
|
-
*
|
|
228
|
+
* `true`, or to `false` when config.yaml sets
|
|
229
|
+
* `disable_default_cross_chain: true`. Set to `false` to isolate the cache
|
|
230
|
+
* and rate limiting per chain and enable `context.chain.id` inside the
|
|
231
|
+
* handler. */
|
|
230
232
|
readonly crossChain?: boolean;
|
|
231
233
|
},
|
|
232
234
|
handler: (args: EffectArgs<I>) => Promise<R>
|
|
@@ -502,6 +504,8 @@ type IndexerConfigTypes = {
|
|
|
502
504
|
};
|
|
503
505
|
svm?: { chains: Record<string, { id: number }> };
|
|
504
506
|
entities?: Record<string, object>;
|
|
507
|
+
// Union of the entity names whose rows belong to a single chain, or `never`.
|
|
508
|
+
perChainEntities?: string;
|
|
505
509
|
enums?: Record<string, string>;
|
|
506
510
|
};
|
|
507
511
|
|
|
@@ -1529,6 +1533,21 @@ type AddressRegistration = {
|
|
|
1529
1533
|
type ConfigEntities<Config extends IndexerConfigTypes = GlobalConfig> =
|
|
1530
1534
|
Config["entities"] extends Record<string, object> ? Config["entities"] : {};
|
|
1531
1535
|
|
|
1536
|
+
/** Entity names whose rows belong to a single chain. */
|
|
1537
|
+
type PerChainEntityNames<Config extends IndexerConfigTypes = GlobalConfig> =
|
|
1538
|
+
Config extends { perChainEntities: infer Names extends string } ? Names : never;
|
|
1539
|
+
|
|
1540
|
+
/** The row shape the chain-agnostic test-indexer operations exchange. A
|
|
1541
|
+
* per-chain entity's row is only identified together with its chain, so the
|
|
1542
|
+
* chain id travels alongside the entity fields. */
|
|
1543
|
+
type TestIndexerEntityRow<
|
|
1544
|
+
Config extends IndexerConfigTypes,
|
|
1545
|
+
Name,
|
|
1546
|
+
Entity
|
|
1547
|
+
> = Name extends PerChainEntityNames<Config>
|
|
1548
|
+
? Entity & { readonly chainId: number }
|
|
1549
|
+
: Entity;
|
|
1550
|
+
|
|
1532
1551
|
/** Entity operations available on test indexer for direct entity manipulation. */
|
|
1533
1552
|
type TestIndexerEntityOperations<Entity> = {
|
|
1534
1553
|
/** Get an entity by ID. Returns undefined if not found. */
|
|
@@ -1537,6 +1556,10 @@ type TestIndexerEntityOperations<Entity> = {
|
|
|
1537
1556
|
readonly getOrThrow: (id: EntityId<Entity>, message?: string) => Promise<Entity>;
|
|
1538
1557
|
/** Get all entities. */
|
|
1539
1558
|
readonly getAll: () => Promise<Entity[]>;
|
|
1559
|
+
/** Get the entities matching a filter. For a per-chain entity the filter
|
|
1560
|
+
* accepts `chainId`, which is how an id present on several chains is
|
|
1561
|
+
* narrowed to one. */
|
|
1562
|
+
readonly getWhere: (filter: GetWhereFilter<Entity>) => Promise<Entity[]>;
|
|
1540
1563
|
/** Set (create or update) an entity. */
|
|
1541
1564
|
readonly set: (entity: Entity) => void;
|
|
1542
1565
|
};
|
|
@@ -1711,7 +1734,7 @@ export type TestIndexerFromConfig<Config extends IndexerConfigTypes = GlobalConf
|
|
|
1711
1734
|
} & SingleEcosystemTestChains<Config> & {
|
|
1712
1735
|
/** Entity operations for direct manipulation outside of handlers. */
|
|
1713
1736
|
readonly [K in keyof ConfigEntities<Config>]: TestIndexerEntityOperations<
|
|
1714
|
-
ConfigEntities<Config>[K]
|
|
1737
|
+
TestIndexerEntityRow<Config, K, ConfigEntities<Config>[K]>
|
|
1715
1738
|
>;
|
|
1716
1739
|
};
|
|
1717
1740
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.5.
|
|
3
|
+
"version": "3.5.1-svm-alpha.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -69,10 +69,10 @@
|
|
|
69
69
|
"tsx": "4.21.0"
|
|
70
70
|
},
|
|
71
71
|
"optionalDependencies": {
|
|
72
|
-
"envio-linux-x64": "3.5.
|
|
73
|
-
"envio-linux-x64-musl": "3.5.
|
|
74
|
-
"envio-linux-arm64": "3.5.
|
|
75
|
-
"envio-darwin-x64": "3.5.
|
|
76
|
-
"envio-darwin-arm64": "3.5.
|
|
72
|
+
"envio-linux-x64": "3.5.1-svm-alpha.2",
|
|
73
|
+
"envio-linux-x64-musl": "3.5.1-svm-alpha.2",
|
|
74
|
+
"envio-linux-arm64": "3.5.1-svm-alpha.2",
|
|
75
|
+
"envio-darwin-x64": "3.5.1-svm-alpha.2",
|
|
76
|
+
"envio-darwin-arm64": "3.5.1-svm-alpha.2"
|
|
77
77
|
}
|
|
78
78
|
}
|
package/src/ChainFetching.res
CHANGED
|
@@ -24,25 +24,15 @@ let runContractRegistersOrThrow = async (
|
|
|
24
24
|
~ecosystem=config.ecosystem.name,
|
|
25
25
|
)
|
|
26
26
|
|
|
27
|
-
let
|
|
27
|
+
let registrations: array<AddressStore.registration> = []
|
|
28
28
|
|
|
29
29
|
let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
|
|
30
30
|
let eventItem = item->Internal.castUnsafeEventItem
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
let dc: Internal.indexingAddress = {
|
|
31
|
+
registrations->Array.push({
|
|
34
32
|
address: contractAddress,
|
|
35
33
|
contractName,
|
|
36
|
-
registrationBlock: blockNumber,
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
switch item->Internal.getItemDcs {
|
|
40
|
-
| None => {
|
|
41
|
-
item->Internal.setItemDcs([dc])
|
|
42
|
-
itemsWithDcs->Array.push(item)
|
|
43
|
-
}
|
|
44
|
-
| Some(dcs) => dcs->Array.push(dc)
|
|
45
|
-
}
|
|
34
|
+
registrationBlock: eventItem.blockNumber,
|
|
35
|
+
})
|
|
46
36
|
}
|
|
47
37
|
|
|
48
38
|
let promises = []
|
|
@@ -101,7 +91,7 @@ let runContractRegistersOrThrow = async (
|
|
|
101
91
|
let _ = await Promise.all(promises)
|
|
102
92
|
}
|
|
103
93
|
|
|
104
|
-
|
|
94
|
+
registrations
|
|
105
95
|
}
|
|
106
96
|
|
|
107
97
|
let rec onQueryResponse = async (
|
|
@@ -226,13 +216,13 @@ let rec onQueryResponse = async (
|
|
|
226
216
|
|
|
227
217
|
// Re-check staleness: contract registration is async, so the chain state
|
|
228
218
|
// may have rolled back by the time we apply the fetched items.
|
|
229
|
-
let proceed = (~
|
|
219
|
+
let proceed = (~newRegistrations) =>
|
|
230
220
|
if !(state->IndexerState.isStale(~stateId)) {
|
|
231
221
|
applyQueryResponse(
|
|
232
222
|
state,
|
|
233
223
|
~chainId,
|
|
234
224
|
~newItems,
|
|
235
|
-
~
|
|
225
|
+
~newRegistrations,
|
|
236
226
|
~knownHeight,
|
|
237
227
|
~latestFetchedBlock={
|
|
238
228
|
FetchState.blockNumber: latestFetchedBlockNumber,
|
|
@@ -248,7 +238,7 @@ let rec onQueryResponse = async (
|
|
|
248
238
|
}
|
|
249
239
|
|
|
250
240
|
switch itemsWithContractRegister {
|
|
251
|
-
| [] => proceed(~
|
|
241
|
+
| [] => proceed(~newRegistrations=[])
|
|
252
242
|
| _ =>
|
|
253
243
|
switch await runContractRegistersOrThrow(
|
|
254
244
|
~itemsWithContractRegister,
|
|
@@ -257,7 +247,7 @@ let rec onQueryResponse = async (
|
|
|
257
247
|
~blockStore,
|
|
258
248
|
) {
|
|
259
249
|
| exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
|
|
260
|
-
|
|
|
250
|
+
| newRegistrations => proceed(~newRegistrations)
|
|
261
251
|
}
|
|
262
252
|
}
|
|
263
253
|
}
|
|
@@ -267,7 +257,7 @@ and applyQueryResponse = (
|
|
|
267
257
|
state: IndexerState.t,
|
|
268
258
|
~chainId,
|
|
269
259
|
~newItems,
|
|
270
|
-
~
|
|
260
|
+
~newRegistrations,
|
|
271
261
|
~knownHeight,
|
|
272
262
|
~latestFetchedBlock,
|
|
273
263
|
~query,
|
|
@@ -281,7 +271,7 @@ and applyQueryResponse = (
|
|
|
281
271
|
~query,
|
|
282
272
|
~latestFetchedBlock,
|
|
283
273
|
~newItems,
|
|
284
|
-
~
|
|
274
|
+
~newRegistrations,
|
|
285
275
|
~knownHeight,
|
|
286
276
|
~transactionStore,
|
|
287
277
|
~blockStore,
|
|
@@ -18,21 +18,13 @@ import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
|
|
|
18
18
|
|
|
19
19
|
async function runContractRegistersOrThrow(itemsWithContractRegister, config, transactionStore, blockStore) {
|
|
20
20
|
await ChainState.materializePageItems(itemsWithContractRegister, transactionStore, blockStore, config.ecosystem.name);
|
|
21
|
-
let
|
|
21
|
+
let registrations = [];
|
|
22
22
|
let onRegister = (item, contractAddress, contractName) => {
|
|
23
|
-
|
|
24
|
-
let dc = {
|
|
23
|
+
registrations.push({
|
|
25
24
|
address: contractAddress,
|
|
26
25
|
contractName: contractName,
|
|
27
|
-
registrationBlock:
|
|
28
|
-
};
|
|
29
|
-
let dcs = item.dcs;
|
|
30
|
-
if (dcs !== undefined) {
|
|
31
|
-
dcs.push(dc);
|
|
32
|
-
} else {
|
|
33
|
-
item.dcs = [dc];
|
|
34
|
-
itemsWithDcs.push(item);
|
|
35
|
-
}
|
|
26
|
+
registrationBlock: item.blockNumber
|
|
27
|
+
});
|
|
36
28
|
};
|
|
37
29
|
let promises = [];
|
|
38
30
|
for (let idx = 0, idx_finish = itemsWithContractRegister.length; idx < idx_finish; ++idx) {
|
|
@@ -68,13 +60,13 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config, tr
|
|
|
68
60
|
if (Utils.$$Array.notEmpty(promises)) {
|
|
69
61
|
await Promise.all(promises);
|
|
70
62
|
}
|
|
71
|
-
return
|
|
63
|
+
return registrations;
|
|
72
64
|
}
|
|
73
65
|
|
|
74
|
-
function applyQueryResponse(state, chainId, newItems,
|
|
66
|
+
function applyQueryResponse(state, chainId, newItems, newRegistrations, knownHeight, latestFetchedBlock, query, transactionStore, blockStore) {
|
|
75
67
|
let chainState = IndexerState.getChainState(state, chainId);
|
|
76
68
|
let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
|
|
77
|
-
ChainState.handleQueryResult(chainState, query, newItems,
|
|
69
|
+
ChainState.handleQueryResult(chainState, query, newItems, newRegistrations, latestFetchedBlock, knownHeight, transactionStore, blockStore);
|
|
78
70
|
if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
|
|
79
71
|
ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
|
|
80
72
|
}
|
|
@@ -152,9 +144,9 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
152
144
|
itemsWithContractRegister.push(item);
|
|
153
145
|
}
|
|
154
146
|
}
|
|
155
|
-
let proceed =
|
|
147
|
+
let proceed = newRegistrations => {
|
|
156
148
|
if (!IndexerState.isStale(state, stateId)) {
|
|
157
|
-
applyQueryResponse(state, chainId, parsedQueueItems,
|
|
149
|
+
applyQueryResponse(state, chainId, parsedQueueItems, newRegistrations, knownHeight, {
|
|
158
150
|
blockNumber: latestFetchedBlockNumber,
|
|
159
151
|
blockTimestamp: latestFetchedBlockTimestamp
|
|
160
152
|
}, query, transactionStore, blockStore);
|
|
@@ -166,14 +158,14 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
|
|
|
166
158
|
if (itemsWithContractRegister.length === 0) {
|
|
167
159
|
return proceed([]);
|
|
168
160
|
}
|
|
169
|
-
let
|
|
161
|
+
let newRegistrations;
|
|
170
162
|
try {
|
|
171
|
-
|
|
163
|
+
newRegistrations = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), transactionStore, blockStore);
|
|
172
164
|
} catch (raw_exn) {
|
|
173
165
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
174
166
|
return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
|
|
175
167
|
}
|
|
176
|
-
return proceed(
|
|
168
|
+
return proceed(newRegistrations);
|
|
177
169
|
}
|
|
178
170
|
|
|
179
171
|
function finishWaitingForNewBlock(state, chainId, knownHeight, stateId, scheduleFetch, scheduleProcessing) {
|
package/src/ChainState.res
CHANGED
|
@@ -31,6 +31,10 @@ type t = {
|
|
|
31
31
|
// then smoothed with an EMA on every batch (see applyBatchProgress). None
|
|
32
32
|
// until the chain has processed at least one event.
|
|
33
33
|
mutable chainDensity: option<float>,
|
|
34
|
+
// In-memory tables for the entities whose rows belong to a single chain.
|
|
35
|
+
// Empty in the default cross-chain mode, where every entity's table lives on
|
|
36
|
+
// the indexer instead.
|
|
37
|
+
mutable entities: EntityTables.t,
|
|
34
38
|
mutable reorgDetection: ReorgDetection.t,
|
|
35
39
|
mutable safeCheckpointTracking: option<SafeCheckpointTracking.t>,
|
|
36
40
|
// Holds this chain's transactions (kept in Rust) keyed by (blockNumber,
|
|
@@ -96,11 +100,13 @@ let make = (
|
|
|
96
100
|
~chainDensity=None,
|
|
97
101
|
~blockStore=BlockStore.make(~ecosystem=Ecosystem.Evm, ~shouldChecksum=false),
|
|
98
102
|
~reorgThresholdReadyTolerance=100,
|
|
103
|
+
~perChainEntities: array<Internal.entityConfig>=[],
|
|
99
104
|
~logger: Pino.t,
|
|
100
105
|
): t => {
|
|
101
106
|
validateOnEventRegistrations(~chainId=chainConfig.id, onEventRegistrations)
|
|
102
107
|
{
|
|
103
108
|
logger,
|
|
109
|
+
entities: EntityTables.make(perChainEntities),
|
|
104
110
|
onEventRegistrations,
|
|
105
111
|
fetchState,
|
|
106
112
|
addressStore,
|
|
@@ -130,6 +136,19 @@ let make = (
|
|
|
130
136
|
}
|
|
131
137
|
}
|
|
132
138
|
|
|
139
|
+
// Every chain's contracts, not just one chain's: `context.chain.X.add` accepts
|
|
140
|
+
// any config contract, whichever chain declared it, so every chain's address
|
|
141
|
+
// store has to hold the whole set.
|
|
142
|
+
let configContractNames = (config: Config.t) => {
|
|
143
|
+
let names = Utils.Set.make()
|
|
144
|
+
config.chainMap
|
|
145
|
+
->ChainMap.values
|
|
146
|
+
->Array.forEach(chain =>
|
|
147
|
+
chain.contracts->Array.forEach(contract => names->Utils.Set.add(contract.name)->ignore)
|
|
148
|
+
)
|
|
149
|
+
names->Utils.Set.toArray
|
|
150
|
+
}
|
|
151
|
+
|
|
133
152
|
let makeInternal = (
|
|
134
153
|
~chainConfig: Config.chain,
|
|
135
154
|
~indexingAddresses: array<Internal.indexingAddress>,
|
|
@@ -174,7 +193,10 @@ let makeInternal = (
|
|
|
174
193
|
let addressStore = AddressStore.make(
|
|
175
194
|
~ecosystem=config.ecosystem.name,
|
|
176
195
|
~shouldChecksum=!lowercaseAddresses,
|
|
177
|
-
~contracts=AddressStore.contractsOf(
|
|
196
|
+
~contracts=AddressStore.contractsOf(
|
|
197
|
+
~onEventRegistrations,
|
|
198
|
+
~configContractNames=config->configContractNames,
|
|
199
|
+
),
|
|
178
200
|
)
|
|
179
201
|
|
|
180
202
|
let fetchState = FetchState.make(
|
|
@@ -302,6 +324,7 @@ let makeInternal = (
|
|
|
302
324
|
~chainReorgCheckpoints,
|
|
303
325
|
),
|
|
304
326
|
~committedProgressBlockNumber=progressBlockNumber,
|
|
327
|
+
~perChainEntities=config.allEntities->EntityTables.perChain,
|
|
305
328
|
~timestampCaughtUpToHeadOrEndblock,
|
|
306
329
|
~numEventsProcessed,
|
|
307
330
|
~transactionStore=TransactionStore.make(
|
|
@@ -392,6 +415,11 @@ let makeFromDbState = (
|
|
|
392
415
|
// --- Read accessors. ---
|
|
393
416
|
|
|
394
417
|
let logger = (cs: t) => cs.logger
|
|
418
|
+
let entities = (cs: t) => cs.entities
|
|
419
|
+
|
|
420
|
+
// Rollback discards every uncommitted change, so this chain's partition is
|
|
421
|
+
// rebuilt from scratch alongside the indexer's cross-chain one.
|
|
422
|
+
let resetEntities = (cs: t, ~perChainEntities) => cs.entities = EntityTables.make(perChainEntities)
|
|
395
423
|
let sourceManager = (cs: t) => cs.sourceManager
|
|
396
424
|
let chainConfig = (cs: t) => cs.chainConfig
|
|
397
425
|
let reorgDetection = (cs: t) => cs.reorgDetection
|
|
@@ -430,6 +458,15 @@ let setRollbackTargetBlock = (cs: t, ~blockNumber) => cs.rollbackTargetBlock = S
|
|
|
430
458
|
let knownHeight = (cs: t) => cs.fetchState.knownHeight
|
|
431
459
|
let contractAddresses = (cs: t, ~contractName) =>
|
|
432
460
|
cs.addressStore->AddressStore.contractAddresses(contractName)
|
|
461
|
+
|
|
462
|
+
// Hands over the dynamically registered addresses the database hasn't seen yet,
|
|
463
|
+
// up to the block the caller is about to commit, each paired with the checkpoint
|
|
464
|
+
// that owns its row. Destructive: the caller must persist them or take the
|
|
465
|
+
// indexer down. Throws with nothing consumed when a registration's block has no
|
|
466
|
+
// checkpoint in the batch.
|
|
467
|
+
let drainAddressesForWrite = (cs: t, ~toBlockInclusive, ~checkpointBlockNumbers) =>
|
|
468
|
+
cs.addressStore->AddressStore.drainForWrite(toBlockInclusive, checkpointBlockNumbers)
|
|
469
|
+
let hasAddressesToWrite = (cs: t) => cs.addressStore->AddressStore.pendingCount > 0
|
|
433
470
|
let bufferSize = (cs: t) => cs.fetchState->FetchState.bufferSize
|
|
434
471
|
let bufferReadyCount = (cs: t) => cs.fetchState->FetchState.bufferReadyCount
|
|
435
472
|
let getProgressPercentage = (cs: t) => cs.fetchState->FetchState.getProgressPercentage
|
|
@@ -841,7 +878,7 @@ let handleQueryResult = (
|
|
|
841
878
|
cs: t,
|
|
842
879
|
~query: FetchState.query,
|
|
843
880
|
~newItems,
|
|
844
|
-
~
|
|
881
|
+
~newRegistrations,
|
|
845
882
|
~latestFetchedBlock: FetchState.blockNumberAndTimestamp,
|
|
846
883
|
~knownHeight,
|
|
847
884
|
~transactionStore as txPage: option<TransactionStore.t>,
|
|
@@ -858,7 +895,7 @@ let handleQueryResult = (
|
|
|
858
895
|
| None => ()
|
|
859
896
|
}
|
|
860
897
|
|
|
861
|
-
let fs = switch
|
|
898
|
+
let fs = switch newRegistrations {
|
|
862
899
|
| [] => cs.fetchState
|
|
863
900
|
| _ =>
|
|
864
901
|
cs.fetchState->FetchState.registerDynamicContracts(
|
|
@@ -868,7 +905,7 @@ let handleQueryResult = (
|
|
|
868
905
|
// catch-up range — and an unbounded query can reach past the height that
|
|
869
906
|
// was known when it went out.
|
|
870
907
|
~claimCeiling=Pervasives.max(knownHeight, latestFetchedBlock.blockNumber),
|
|
871
|
-
|
|
908
|
+
newRegistrations,
|
|
872
909
|
)
|
|
873
910
|
}
|
|
874
911
|
|
package/src/ChainState.res.mjs
CHANGED
|
@@ -6,12 +6,14 @@ import * as Batch from "./Batch.res.mjs";
|
|
|
6
6
|
import * as Utils from "./Utils.res.mjs";
|
|
7
7
|
import * as ChainId from "./ChainId.res.mjs";
|
|
8
8
|
import * as Logging from "./Logging.res.mjs";
|
|
9
|
+
import * as ChainMap from "./ChainMap.res.mjs";
|
|
9
10
|
import * as EvmChain from "./sources/EvmChain.res.mjs";
|
|
10
11
|
import * as FieldMask from "./sources/FieldMask.res.mjs";
|
|
11
12
|
import * as BlockStore from "./sources/BlockStore.res.mjs";
|
|
12
13
|
import * as FetchState from "./FetchState.res.mjs";
|
|
13
14
|
import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
|
|
14
15
|
import * as AddressStore from "./sources/AddressStore.res.mjs";
|
|
16
|
+
import * as EntityTables from "./EntityTables.res.mjs";
|
|
15
17
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
16
18
|
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
17
19
|
import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
@@ -48,7 +50,7 @@ function validateOnEventRegistrations(chainId, registrations) {
|
|
|
48
50
|
});
|
|
49
51
|
}
|
|
50
52
|
|
|
51
|
-
function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, logger) {
|
|
53
|
+
function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, sourceManager, reorgDetection, committedProgressBlockNumber, safeCheckpointTrackingOpt, numEventsProcessedOpt, timestampCaughtUpToHeadOrEndblockOpt, isProgressAtHeadOpt, transactionStoreOpt, chainDensityOpt, blockStoreOpt, reorgThresholdReadyToleranceOpt, perChainEntitiesOpt, logger) {
|
|
52
54
|
let onEventRegistrations = onEventRegistrationsOpt !== undefined ? onEventRegistrationsOpt : [];
|
|
53
55
|
let safeCheckpointTracking = safeCheckpointTrackingOpt !== undefined ? Primitive_option.valFromOption(safeCheckpointTrackingOpt) : undefined;
|
|
54
56
|
let numEventsProcessed = numEventsProcessedOpt !== undefined ? numEventsProcessedOpt : 0;
|
|
@@ -58,6 +60,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
|
|
|
58
60
|
let chainDensity = chainDensityOpt !== undefined ? Primitive_option.valFromOption(chainDensityOpt) : undefined;
|
|
59
61
|
let blockStore = blockStoreOpt !== undefined ? Primitive_option.valFromOption(blockStoreOpt) : BlockStore.make("evm", false);
|
|
60
62
|
let reorgThresholdReadyTolerance = reorgThresholdReadyToleranceOpt !== undefined ? reorgThresholdReadyToleranceOpt : 100;
|
|
63
|
+
let perChainEntities = perChainEntitiesOpt !== undefined ? perChainEntitiesOpt : [];
|
|
61
64
|
validateOnEventRegistrations(chainConfig.id, onEventRegistrations);
|
|
62
65
|
return {
|
|
63
66
|
logger: logger,
|
|
@@ -73,6 +76,7 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
|
|
|
73
76
|
numEventsProcessed: numEventsProcessed,
|
|
74
77
|
pendingBudget: 0,
|
|
75
78
|
chainDensity: chainDensity,
|
|
79
|
+
entities: EntityTables.make(perChainEntities),
|
|
76
80
|
reorgDetection: reorgDetection,
|
|
77
81
|
safeCheckpointTracking: safeCheckpointTracking,
|
|
78
82
|
transactionStore: transactionStore,
|
|
@@ -90,6 +94,16 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
|
|
|
90
94
|
};
|
|
91
95
|
}
|
|
92
96
|
|
|
97
|
+
function configContractNames(config) {
|
|
98
|
+
let names = new Set();
|
|
99
|
+
ChainMap.values(config.chainMap).forEach(chain => {
|
|
100
|
+
chain.contracts.forEach(contract => {
|
|
101
|
+
names.add(contract.name);
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
return Array.from(names);
|
|
105
|
+
}
|
|
106
|
+
|
|
93
107
|
function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrationsByChainId, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, isResumedOpt, reducedPollingInterval) {
|
|
94
108
|
let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
|
|
95
109
|
let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
|
|
@@ -106,7 +120,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
106
120
|
}
|
|
107
121
|
});
|
|
108
122
|
let lowercaseAddresses = config.lowercaseAddresses;
|
|
109
|
-
let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations));
|
|
123
|
+
let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations, configContractNames(config)));
|
|
110
124
|
let match$1 = config.ecosystem.name;
|
|
111
125
|
let tmp;
|
|
112
126
|
switch (match$1) {
|
|
@@ -175,7 +189,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
175
189
|
}
|
|
176
190
|
let firstEventBlock$1 = fetchState.firstEventBlock;
|
|
177
191
|
let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
|
|
178
|
-
return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), config.reorgThresholdReadyTolerance, logger);
|
|
192
|
+
return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(BlockStore.make(config.ecosystem.name, !lowercaseAddresses)), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.allEntities), logger);
|
|
179
193
|
}
|
|
180
194
|
|
|
181
195
|
function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
|
|
@@ -198,6 +212,14 @@ function logger(cs) {
|
|
|
198
212
|
return cs.logger;
|
|
199
213
|
}
|
|
200
214
|
|
|
215
|
+
function entities(cs) {
|
|
216
|
+
return cs.entities;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function resetEntities(cs, perChainEntities) {
|
|
220
|
+
cs.entities = EntityTables.make(perChainEntities);
|
|
221
|
+
}
|
|
222
|
+
|
|
201
223
|
function sourceManager(cs) {
|
|
202
224
|
return cs.sourceManager;
|
|
203
225
|
}
|
|
@@ -259,6 +281,14 @@ function contractAddresses(cs, contractName) {
|
|
|
259
281
|
return cs.addressStore.contractAddresses(contractName);
|
|
260
282
|
}
|
|
261
283
|
|
|
284
|
+
function drainAddressesForWrite(cs, toBlockInclusive, checkpointBlockNumbers) {
|
|
285
|
+
return cs.addressStore.drainForWrite(toBlockInclusive, checkpointBlockNumbers);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function hasAddressesToWrite(cs) {
|
|
289
|
+
return cs.addressStore.pendingCount() > 0;
|
|
290
|
+
}
|
|
291
|
+
|
|
262
292
|
function bufferSize(cs) {
|
|
263
293
|
return FetchState.bufferSize(cs.fetchState);
|
|
264
294
|
}
|
|
@@ -565,14 +595,14 @@ async function materializePageItems(items, transactionStore, blockStore, ecosyst
|
|
|
565
595
|
]);
|
|
566
596
|
}
|
|
567
597
|
|
|
568
|
-
function handleQueryResult(cs, query, newItems,
|
|
598
|
+
function handleQueryResult(cs, query, newItems, newRegistrations, latestFetchedBlock, knownHeight, txPage, blockPage) {
|
|
569
599
|
if (txPage !== undefined) {
|
|
570
600
|
cs.transactionStore.merge(Primitive_option.valFromOption(txPage));
|
|
571
601
|
}
|
|
572
602
|
if (blockPage !== undefined) {
|
|
573
603
|
cs.blockStore.merge(Primitive_option.valFromOption(blockPage));
|
|
574
604
|
}
|
|
575
|
-
let fs =
|
|
605
|
+
let fs = newRegistrations.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.addressStore, Primitive_int.max(knownHeight, latestFetchedBlock.blockNumber), newRegistrations) : cs.fetchState;
|
|
576
606
|
cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
|
|
577
607
|
cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
|
|
578
608
|
}
|
|
@@ -822,6 +852,8 @@ export {
|
|
|
822
852
|
makeFromConfig,
|
|
823
853
|
makeFromDbState,
|
|
824
854
|
logger,
|
|
855
|
+
entities,
|
|
856
|
+
resetEntities,
|
|
825
857
|
sourceManager,
|
|
826
858
|
chainConfig,
|
|
827
859
|
reorgDetection,
|
|
@@ -837,6 +869,8 @@ export {
|
|
|
837
869
|
setRollbackTargetBlock,
|
|
838
870
|
knownHeight,
|
|
839
871
|
contractAddresses,
|
|
872
|
+
drainAddressesForWrite,
|
|
873
|
+
hasAddressesToWrite,
|
|
840
874
|
bufferSize,
|
|
841
875
|
bufferReadyCount,
|
|
842
876
|
getProgressPercentage,
|
package/src/ChainState.resi
CHANGED
|
@@ -21,6 +21,7 @@ let make: (
|
|
|
21
21
|
~chainDensity: option<float>=?,
|
|
22
22
|
~blockStore: BlockStore.t=?,
|
|
23
23
|
~reorgThresholdReadyTolerance: int=?,
|
|
24
|
+
~perChainEntities: array<Internal.entityConfig>=?,
|
|
24
25
|
~logger: Pino.t,
|
|
25
26
|
) => t
|
|
26
27
|
|
|
@@ -44,6 +45,8 @@ let makeFromDbState: (
|
|
|
44
45
|
|
|
45
46
|
// Accessors.
|
|
46
47
|
let logger: t => Pino.t
|
|
48
|
+
let entities: t => EntityTables.t
|
|
49
|
+
let resetEntities: (t, ~perChainEntities: array<Internal.entityConfig>) => unit
|
|
47
50
|
let sourceManager: t => SourceManager.t
|
|
48
51
|
let chainConfig: t => Config.chain
|
|
49
52
|
let reorgDetection: t => ReorgDetection.t
|
|
@@ -69,6 +72,12 @@ let setRollbackTargetBlock: (t, ~blockNumber: int) => unit
|
|
|
69
72
|
// Fetch-frontier reads.
|
|
70
73
|
let knownHeight: t => int
|
|
71
74
|
let contractAddresses: (t, ~contractName: string) => array<Address.t>
|
|
75
|
+
let drainAddressesForWrite: (
|
|
76
|
+
t,
|
|
77
|
+
~toBlockInclusive: int,
|
|
78
|
+
~checkpointBlockNumbers: array<int>,
|
|
79
|
+
) => array<AddressStore.drainedAddress>
|
|
80
|
+
let hasAddressesToWrite: t => bool
|
|
72
81
|
let bufferSize: t => int
|
|
73
82
|
let bufferReadyCount: t => int
|
|
74
83
|
let getProgressPercentage: t => float
|
|
@@ -110,7 +119,7 @@ let handleQueryResult: (
|
|
|
110
119
|
t,
|
|
111
120
|
~query: FetchState.query,
|
|
112
121
|
~newItems: array<Internal.item>,
|
|
113
|
-
~
|
|
122
|
+
~newRegistrations: array<AddressStore.registration>,
|
|
114
123
|
~latestFetchedBlock: FetchState.blockNumberAndTimestamp,
|
|
115
124
|
~knownHeight: int,
|
|
116
125
|
~transactionStore: option<TransactionStore.t>,
|