envio 3.8.0 → 3.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/index.d.ts +158 -104
- package/package.json +6 -6
- package/src/AddressRows.res +121 -0
- package/src/AddressRows.res.mjs +143 -0
- package/src/Batch.res +2 -0
- package/src/Batch.res.mjs +2 -1
- package/src/ChainState.res +76 -70
- package/src/ChainState.res.mjs +76 -65
- package/src/ChainState.resi +14 -9
- package/src/Config.res +23 -82
- package/src/Config.res.mjs +19 -63
- package/src/ContractMapping.res +66 -0
- package/src/ContractMapping.res.mjs +73 -0
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +4 -0
- package/src/EventConfigBuilder.res +0 -2
- package/src/EventConfigBuilder.res.mjs +0 -2
- package/src/FetchState.res +59 -67
- package/src/FetchState.res.mjs +34 -44
- package/src/InMemoryStore.res +17 -30
- package/src/InMemoryStore.res.mjs +11 -23
- package/src/IndexerState.res +21 -1
- package/src/IndexerState.res.mjs +9 -3
- package/src/IndexerState.resi +1 -0
- package/src/Main.res +25 -15
- package/src/Main.res.mjs +19 -14
- package/src/MemoryStorage.res +39 -66
- package/src/MemoryStorage.res.mjs +28 -65
- package/src/Metrics.res +15 -1
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +30 -16
- package/src/Persistence.res.mjs +8 -5
- package/src/PgStorage.res +183 -65
- package/src/PgStorage.res.mjs +92 -58
- package/src/Rollback.res +14 -10
- package/src/Rollback.res.mjs +4 -2
- package/src/SimulateItems.res +5 -12
- package/src/SimulateItems.res.mjs +6 -10
- package/src/TestIndexer.res +93 -111
- package/src/TestIndexer.res.mjs +61 -88
- package/src/Utils.res +7 -0
- package/src/Utils.res.mjs +9 -2
- package/src/Writing.res +2 -0
- package/src/Writing.res.mjs +2 -1
- package/src/bindings/ClickHouse.res +6 -0
- package/src/bindings/ClickHouse.res.mjs +4 -0
- package/src/bindings/NodeJs.res +9 -0
- package/src/bindings/NodeJs.res.mjs +8 -1
- package/src/bindings/Postgres.res +9 -0
- package/src/bindings/Postgres.res.mjs +3 -0
- package/src/db/EntityHistory.res +12 -18
- package/src/db/EntityHistory.res.mjs +3 -14
- package/src/db/InternalTable.res +171 -65
- package/src/db/InternalTable.res.mjs +176 -73
- package/src/db/Table.res +24 -0
- package/src/db/Table.res.mjs +20 -1
- package/src/sources/AddressSet.res +0 -22
- package/src/sources/AddressStore.res +48 -88
- package/src/sources/AddressStore.res.mjs +11 -22
- package/src/sources/SvmHyperSyncClient.res +17 -21
- package/src/sources/SvmHyperSyncSource.res +4 -9
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -13
- package/svm.schema.json +10 -4
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// The contract name <-> id mapping every stored address row references: a
|
|
2
|
+
// name's position in `names` is the id `envio_contracts` holds and
|
|
3
|
+
// `envio_addresses.contract_id` points at. Ids must mean the same thing on
|
|
4
|
+
// every chain and across restarts, so a mapping is built once — from the whole
|
|
5
|
+
// config, never from a filtered subset — and read everywhere else.
|
|
6
|
+
type t = {
|
|
7
|
+
// Id order. Index i is the name of contract id i.
|
|
8
|
+
names: array<string>,
|
|
9
|
+
idByName: dict<int>,
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
let indexNames = (names: array<string>): t => {
|
|
13
|
+
let idByName = Dict.make()
|
|
14
|
+
names->Array.forEachWithIndex((name, id) => idByName->Dict.set(name, id))
|
|
15
|
+
{names, idByName}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// Ids are 0-based positions in a smallint column, so 32768 contracts fit.
|
|
19
|
+
let maxContracts = 32768
|
|
20
|
+
|
|
21
|
+
// Names in any order; the codec puts them in byte order so the ids never depend
|
|
22
|
+
// on the order contracts happen to be declared in.
|
|
23
|
+
let make = (~names: array<string>): t => {
|
|
24
|
+
let canonical = Core.getAddon().canonicalContractNames(names)
|
|
25
|
+
if canonical->Array.length > maxContracts {
|
|
26
|
+
JsError.throwWithMessage(
|
|
27
|
+
`The indexer declares ${canonical
|
|
28
|
+
->Array.length
|
|
29
|
+
->Int.toString} contracts, more than the ${maxContracts->Int.toString} a smallint contract id can hold.`,
|
|
30
|
+
)
|
|
31
|
+
}
|
|
32
|
+
indexNames(canonical)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// What a storage holds before it has been initialized: no contract has an id
|
|
36
|
+
// yet, so every lookup fails rather than resolving against a stale mapping.
|
|
37
|
+
let empty = indexNames([])
|
|
38
|
+
|
|
39
|
+
// A mapping read back from storage. Taken verbatim: the stored order *is* the
|
|
40
|
+
// id order the stored rows were written against, so re-canonicalizing it would
|
|
41
|
+
// paper over a mapping that no longer matches those rows.
|
|
42
|
+
let fromStoredNames = indexNames
|
|
43
|
+
|
|
44
|
+
let names = (mapping: t) => mapping.names
|
|
45
|
+
|
|
46
|
+
let idOfOrThrow = (mapping: t, name, ~context="") =>
|
|
47
|
+
switch mapping.idByName->Utils.Dict.dangerouslyGetNonOption(name) {
|
|
48
|
+
| Some(id) => id
|
|
49
|
+
| None =>
|
|
50
|
+
JsError.throwWithMessage(
|
|
51
|
+
`Contract "${name}"${context} is missing from the indexer's contract list.`,
|
|
52
|
+
)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
let nameOfOrThrow = (mapping: t, id) =>
|
|
56
|
+
switch mapping.names->Array.get(id) {
|
|
57
|
+
| Some(name) => name
|
|
58
|
+
| None =>
|
|
59
|
+
JsError.throwWithMessage(
|
|
60
|
+
`Contract id ${id->Int.toString} is outside the indexer's contract list.`,
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
let isEqual = (a: t, b: t) =>
|
|
65
|
+
a.names->Array.length === b.names->Array.length &&
|
|
66
|
+
a.names->Array.everyWithIndex((name, idx) => b.names->Array.getUnsafe(idx) === name)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Core from "./Core.res.mjs";
|
|
4
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
5
|
+
|
|
6
|
+
function indexNames(names) {
|
|
7
|
+
let idByName = {};
|
|
8
|
+
names.forEach((name, id) => {
|
|
9
|
+
idByName[name] = id;
|
|
10
|
+
});
|
|
11
|
+
return {
|
|
12
|
+
names: names,
|
|
13
|
+
idByName: idByName
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function make(names) {
|
|
18
|
+
let canonical = Core.getAddon().canonicalContractNames(names);
|
|
19
|
+
if (canonical.length > 32768) {
|
|
20
|
+
Stdlib_JsError.throwWithMessage(`The indexer declares ` + canonical.length.toString() + ` contracts, more than the ` + (32768).toString() + ` a smallint contract id can hold.`);
|
|
21
|
+
}
|
|
22
|
+
return indexNames(canonical);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let empty = indexNames([]);
|
|
26
|
+
|
|
27
|
+
function names(mapping) {
|
|
28
|
+
return mapping.names;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function idOfOrThrow(mapping, name, contextOpt) {
|
|
32
|
+
let context = contextOpt !== undefined ? contextOpt : "";
|
|
33
|
+
let id = mapping.idByName[name];
|
|
34
|
+
if (id !== undefined) {
|
|
35
|
+
return id;
|
|
36
|
+
} else {
|
|
37
|
+
return Stdlib_JsError.throwWithMessage(`Contract "` + name + `"` + context + ` is missing from the indexer's contract list.`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function nameOfOrThrow(mapping, id) {
|
|
42
|
+
let name = mapping.names[id];
|
|
43
|
+
if (name !== undefined) {
|
|
44
|
+
return name;
|
|
45
|
+
} else {
|
|
46
|
+
return Stdlib_JsError.throwWithMessage(`Contract id ` + id.toString() + ` is outside the indexer's contract list.`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function isEqual(a, b) {
|
|
51
|
+
if (a.names.length === b.names.length) {
|
|
52
|
+
return a.names.every((name, idx) => b.names[idx] === name);
|
|
53
|
+
} else {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
let maxContracts = 32768;
|
|
59
|
+
|
|
60
|
+
let fromStoredNames = indexNames;
|
|
61
|
+
|
|
62
|
+
export {
|
|
63
|
+
indexNames,
|
|
64
|
+
maxContracts,
|
|
65
|
+
make,
|
|
66
|
+
empty,
|
|
67
|
+
fromStoredNames,
|
|
68
|
+
names,
|
|
69
|
+
idOfOrThrow,
|
|
70
|
+
nameOfOrThrow,
|
|
71
|
+
isEqual,
|
|
72
|
+
}
|
|
73
|
+
/* empty Not a pure module */
|
package/src/Core.res
CHANGED
|
@@ -47,6 +47,22 @@ type addon = {
|
|
|
47
47
|
addressStore: addressStoreCtor,
|
|
48
48
|
@as("MockHyperSyncServer")
|
|
49
49
|
mockHyperSyncServer: mockHyperSyncServerCtor,
|
|
50
|
+
encodeAddresses: (~ecosystem: string, ~addresses: array<Address.t>) => array<NodeJs.Buffer.t>,
|
|
51
|
+
renderAddresses: (
|
|
52
|
+
~ecosystem: string,
|
|
53
|
+
~shouldChecksum: bool,
|
|
54
|
+
~bytes: NodeJs.Buffer.t,
|
|
55
|
+
~lengths: array<int>,
|
|
56
|
+
) => array<Address.t>,
|
|
57
|
+
renderContractAddresses: (
|
|
58
|
+
~ecosystem: string,
|
|
59
|
+
~shouldChecksum: bool,
|
|
60
|
+
~bytes: NodeJs.Buffer.t,
|
|
61
|
+
~lengths: array<int>,
|
|
62
|
+
~contractIds: array<int>,
|
|
63
|
+
~contractId: int,
|
|
64
|
+
) => array<Address.t>,
|
|
65
|
+
canonicalContractNames: array<string> => array<string>,
|
|
50
66
|
// Ordered transaction-field names exposed for the field-code contract test
|
|
51
67
|
// (the ReScript `transactionFields` arrays must match the Rust ordinals).
|
|
52
68
|
evmTransactionFieldNames: unit => array<string>,
|
|
@@ -84,6 +100,10 @@ let loadDevAddon: ({..}, string) => addon = %raw(`function(req, envioDir) {
|
|
|
84
100
|
var path = Nodepath;
|
|
85
101
|
var fs = Nodefs;
|
|
86
102
|
|
|
103
|
+
// Vitest test.env points workers at the addon globalSetup already built.
|
|
104
|
+
var preBuilt = process.env.ENVIO_DEV_ADDON;
|
|
105
|
+
if (preBuilt && fs.existsSync(preBuilt)) return req(preBuilt);
|
|
106
|
+
|
|
87
107
|
var repoRoot = null;
|
|
88
108
|
var dir = path.resolve(envioDir);
|
|
89
109
|
for (var i = 0; i < 10; i++) {
|
package/src/Core.res.mjs
CHANGED
|
@@ -26,6 +26,10 @@ let loadDevAddon = (function(req, envioDir) {
|
|
|
26
26
|
var path = Nodepath;
|
|
27
27
|
var fs = Nodefs;
|
|
28
28
|
|
|
29
|
+
// Vitest test.env points workers at the addon globalSetup already built.
|
|
30
|
+
var preBuilt = process.env.ENVIO_DEV_ADDON;
|
|
31
|
+
if (preBuilt && fs.existsSync(preBuilt)) return req(preBuilt);
|
|
32
|
+
|
|
29
33
|
var repoRoot = null;
|
|
30
34
|
var dir = path.resolve(envioDir);
|
|
31
35
|
for (var i = 0; i < 10; i++) {
|
|
@@ -554,10 +554,8 @@ let validSvmAccountActivityFields = Utils.Set.fromArray([
|
|
|
554
554
|
"transactionAccountIndex",
|
|
555
555
|
"isSigner",
|
|
556
556
|
"isWritable",
|
|
557
|
-
"lamports",
|
|
558
557
|
"lamports.pre",
|
|
559
558
|
"lamports.post",
|
|
560
|
-
"token",
|
|
561
559
|
"token.mint",
|
|
562
560
|
"token.owner",
|
|
563
561
|
"token.decimals",
|
package/src/FetchState.res
CHANGED
|
@@ -1469,7 +1469,6 @@ let warnAddressRegistration = (
|
|
|
1469
1469
|
|
|
1470
1470
|
// A rejected registration is simply absent from every partition, so without a
|
|
1471
1471
|
// warning the user sees a contract that never indexes and nothing saying why.
|
|
1472
|
-
// Shared by config-time registration in `make` and by dynamic registration.
|
|
1473
1472
|
let warnRejectedRegistration = (
|
|
1474
1473
|
verdict: AddressStore.verdict,
|
|
1475
1474
|
~chainId: ChainId.t,
|
|
@@ -1477,20 +1476,7 @@ let warnRejectedRegistration = (
|
|
|
1477
1476
|
~contractName: string,
|
|
1478
1477
|
) =>
|
|
1479
1478
|
switch verdict {
|
|
1480
|
-
| Conflict({existingContractName}) =>
|
|
1481
|
-
warnAddressRegistration(
|
|
1482
|
-
~chainId,
|
|
1483
|
-
~contractAddress,
|
|
1484
|
-
~params={
|
|
1485
|
-
"existingContractType": existingContractName,
|
|
1486
|
-
"newContractType": contractName,
|
|
1487
|
-
},
|
|
1488
|
-
`Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`,
|
|
1489
|
-
)
|
|
1490
1479
|
| Duplicate({effectiveStartBlock, existingEffectiveStartBlock}) =>
|
|
1491
|
-
// FIXME: Instead of filtering out duplicates, we should check the block
|
|
1492
|
-
// number first. If a new registration has an earlier block number we
|
|
1493
|
-
// should register it for the missing block range.
|
|
1494
1480
|
if existingEffectiveStartBlock > effectiveStartBlock {
|
|
1495
1481
|
warnAddressRegistration(
|
|
1496
1482
|
~chainId,
|
|
@@ -1499,7 +1485,7 @@ let warnRejectedRegistration = (
|
|
|
1499
1485
|
"existingBlockNumber": existingEffectiveStartBlock,
|
|
1500
1486
|
"newBlockNumber": effectiveStartBlock,
|
|
1501
1487
|
},
|
|
1502
|
-
`Skipping contract registration:
|
|
1488
|
+
`Skipping same-contract re-registration: the address is already registered for this contract. The start block does not move earlier.`,
|
|
1503
1489
|
)
|
|
1504
1490
|
}
|
|
1505
1491
|
| Invalid =>
|
|
@@ -1723,9 +1709,10 @@ let registerDynamicContracts = (
|
|
|
1723
1709
|
// exactly what this batch adds.
|
|
1724
1710
|
let idCursor = addressStore->AddressStore.nextId
|
|
1725
1711
|
// The store resolves each address against both what it already holds and the
|
|
1726
|
-
// batch's own earlier entries, so
|
|
1727
|
-
// single batch
|
|
1728
|
-
// additions this chain fetches for, since it's what
|
|
1712
|
+
// batch's own earlier entries, so the same address registered twice for one
|
|
1713
|
+
// contract inside a single batch is a duplicate just as it is across batches.
|
|
1714
|
+
// It also decides which additions this chain fetches for, since it's what
|
|
1715
|
+
// holds the contract list.
|
|
1729
1716
|
let verdicts = addressStore->AddressStore.registerBatch(registrations)
|
|
1730
1717
|
|
|
1731
1718
|
let registeringContractNames = []
|
|
@@ -1741,7 +1728,7 @@ let registerDynamicContracts = (
|
|
|
1741
1728
|
// no partition to build. The address is still stored and persisted, so a
|
|
1742
1729
|
// config that later adds address-dependent events picks it up on restart.
|
|
1743
1730
|
| Added({fetchable: false}) => ()
|
|
1744
|
-
|
|
|
1731
|
+
| Duplicate(_) | Invalid =>
|
|
1745
1732
|
verdict->warnRejectedRegistration(
|
|
1746
1733
|
~chainId=fetchState.chainId,
|
|
1747
1734
|
~contractAddress=registration.address,
|
|
@@ -2627,7 +2614,7 @@ let make = (
|
|
|
2627
2614
|
~endBlock,
|
|
2628
2615
|
~onEventRegistrations: array<Internal.onEventRegistration>,
|
|
2629
2616
|
~addressStore: AddressStore.t,
|
|
2630
|
-
~
|
|
2617
|
+
~addressRows: AddressRows.seedRows,
|
|
2631
2618
|
~maxAddrInPartition,
|
|
2632
2619
|
~chainId: ChainId.t,
|
|
2633
2620
|
~maxOnBlockBufferSize,
|
|
@@ -2681,27 +2668,24 @@ let make = (
|
|
|
2681
2668
|
)
|
|
2682
2669
|
|
|
2683
2670
|
// Every address the chain indexes goes into the store — including ones whose
|
|
2684
|
-
// contract has no address-dependent events, so
|
|
2685
|
-
//
|
|
2671
|
+
// contract has no address-dependent events, so the address is still persisted
|
|
2672
|
+
// and a config that later adds events picks it up.
|
|
2673
|
+
//
|
|
2674
|
+
// These rows come from the config or from a resume, so they're already stored
|
|
2675
|
+
// and must never be drained back into a write. Only the rows the store
|
|
2676
|
+
// refuses come back: a resume seeds millions of them.
|
|
2686
2677
|
addressStore
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
addresses->Array.map((contract): AddressStore.registration => {
|
|
2691
|
-
address: contract.address,
|
|
2692
|
-
contractName: contract.contractName,
|
|
2693
|
-
registrationBlock: contract.registrationBlock,
|
|
2694
|
-
}),
|
|
2695
|
-
)
|
|
2696
|
-
// Verdicts are in the batch's order, so they line up with `addresses`. A
|
|
2697
|
-
// config address the store rejects is dropped exactly like a dynamic one, and
|
|
2698
|
-
// needs the same warning — restored dynamic addresses come through here too.
|
|
2699
|
-
->Array.forEachWithIndex((verdict, idx) => {
|
|
2700
|
-
let contract = addresses->Array.getUnsafe(idx)
|
|
2701
|
-
verdict->warnRejectedRegistration(
|
|
2678
|
+
->AddressStore.seedRows(addressRows)
|
|
2679
|
+
->Array.forEach(rejected => {
|
|
2680
|
+
warnAddressRegistration(
|
|
2702
2681
|
~chainId,
|
|
2703
|
-
~contractAddress=
|
|
2704
|
-
~
|
|
2682
|
+
~contractAddress=rejected.address,
|
|
2683
|
+
~params={
|
|
2684
|
+
"contractName": rejected.contractName,
|
|
2685
|
+
"existingBlockNumber": rejected.existingEffectiveStartBlock,
|
|
2686
|
+
"newBlockNumber": rejected.effectiveStartBlock,
|
|
2687
|
+
},
|
|
2688
|
+
`Skipping a stored address: it is already registered for this contract.`,
|
|
2705
2689
|
)
|
|
2706
2690
|
})
|
|
2707
2691
|
|
|
@@ -2709,23 +2693,24 @@ let make = (
|
|
|
2709
2693
|
let clientFilteredContracts = Utils.Set.make()
|
|
2710
2694
|
let registeringSetsByContract = Dict.make()
|
|
2711
2695
|
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2696
|
+
// What each contract needs a partition for is read back off the store rather
|
|
2697
|
+
// than re-derived from the seeded columns: the store is what resolved the
|
|
2698
|
+
// rows, including the ones it refused.
|
|
2699
|
+
contractNamesWithNormalEvents
|
|
2700
|
+
->Utils.Set.toArray
|
|
2701
|
+
->Array.forEach(contractName => {
|
|
2702
|
+
if addressStore->AddressStore.contractCount(contractName) > 0 {
|
|
2703
|
+
registeringSetsByContract->Dict.set(
|
|
2704
|
+
contractName,
|
|
2705
|
+
addressStore->AddressStore.makeSet(~contractName),
|
|
2706
|
+
)
|
|
2707
|
+
}
|
|
2708
|
+
})
|
|
2709
|
+
addressStore
|
|
2710
|
+
->AddressStore.dynamicContractNames
|
|
2711
|
+
->Array.forEach(contractName => {
|
|
2717
2712
|
if contractNamesWithNormalEvents->Utils.Set.has(contractName) {
|
|
2718
|
-
|
|
2719
|
-
registeringSetsByContract->Dict.set(
|
|
2720
|
-
contractName,
|
|
2721
|
-
addressStore->AddressStore.makeSet(~contractName),
|
|
2722
|
-
)
|
|
2723
|
-
}
|
|
2724
|
-
|
|
2725
|
-
// Detect dynamic contracts by registrationBlock
|
|
2726
|
-
if contract.registrationBlock !== -1 {
|
|
2727
|
-
dynamicContracts->Utils.Set.add(contractName)->ignore
|
|
2728
|
-
}
|
|
2713
|
+
dynamicContracts->Utils.Set.add(contractName)->ignore
|
|
2729
2714
|
}
|
|
2730
2715
|
})
|
|
2731
2716
|
|
|
@@ -2768,7 +2753,7 @@ let make = (
|
|
|
2768
2753
|
) {
|
|
2769
2754
|
JsError.throwWithMessage(
|
|
2770
2755
|
`Invalid configuration: Nothing to fetch on chain ${chainId->ChainId.toString}. ` ++
|
|
2771
|
-
`addresses=${addresses->Array.length->Int.toString}, ` ++
|
|
2756
|
+
`addresses=${addressRows.addresses->Array.length->Int.toString}, ` ++
|
|
2772
2757
|
`onEventRegistrations=${onEventRegistrations->Array.length->Int.toString}, ` ++
|
|
2773
2758
|
`normalRegistrations=${normalRegistrations
|
|
2774
2759
|
->Array.length
|
|
@@ -2845,20 +2830,28 @@ let rollbackPendingQueries = (mutPendingQueries: array<pendingQuery>, ~targetBlo
|
|
|
2845
2830
|
adjusted
|
|
2846
2831
|
}
|
|
2847
2832
|
|
|
2833
|
+
type rollbackResult = {
|
|
2834
|
+
fetchState: t,
|
|
2835
|
+
// The registrations the prune dropped, for the storage that deletes their rows.
|
|
2836
|
+
rolledBackAddresses: array<AddressStore.rolledBackAddress>,
|
|
2837
|
+
}
|
|
2838
|
+
|
|
2848
2839
|
/**
|
|
2849
2840
|
Rolls back fetch state to the given valid block.
|
|
2841
|
+
Prunes the store first, then rebuilds partitions from it: an address survives iff
|
|
2842
|
+
`filterByRegistrationBlock` keeps it, so the partitions and the rows the caller
|
|
2843
|
+
goes on to delete can't disagree about which registrations died.
|
|
2850
2844
|
Always recreates optimized partitions to avoid duplicate addresses:
|
|
2851
2845
|
- Wildcard: only rollback latestFetchedBlock
|
|
2852
2846
|
- Non-wildcard with lfb <= target: keep, adjust pending queries and mergeBlock
|
|
2853
2847
|
- Non-wildcard with lfb > target: delete, track addresses for recreation
|
|
2854
2848
|
*/
|
|
2855
|
-
let rollback = (
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
// Step 2: Categorize partitions
|
|
2849
|
+
let rollback = (
|
|
2850
|
+
fetchState: t,
|
|
2851
|
+
~rolledBackAddressStore: AddressStore.t,
|
|
2852
|
+
~targetBlockNumber,
|
|
2853
|
+
): rollbackResult => {
|
|
2854
|
+
let rolledBackAddresses = rolledBackAddressStore->AddressStore.rollback(targetBlockNumber)
|
|
2862
2855
|
let keptPartitions = []
|
|
2863
2856
|
let nextKeptIdRef = ref(0)
|
|
2864
2857
|
let registeringSetsByContract: dict<AddressSet.t> = Dict.make()
|
|
@@ -2935,10 +2928,9 @@ let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber
|
|
|
2935
2928
|
}
|
|
2936
2929
|
}
|
|
2937
2930
|
|
|
2938
|
-
// Step 3: Recreate partitions from deleted partition addresses
|
|
2939
2931
|
let optimizedPartitions = createPartitions(
|
|
2940
2932
|
~registeringSetsByContract,
|
|
2941
|
-
~addressStore,
|
|
2933
|
+
~addressStore=rolledBackAddressStore,
|
|
2942
2934
|
~dynamicContracts=fetchState.optimizedPartitions.dynamicContracts,
|
|
2943
2935
|
~clientFilteredContracts=fetchState.optimizedPartitions.clientFilteredContracts,
|
|
2944
2936
|
~normalSelection=fetchState.normalSelection,
|
|
@@ -2949,8 +2941,7 @@ let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber
|
|
|
2949
2941
|
~knownHeight=fetchState.knownHeight,
|
|
2950
2942
|
)
|
|
2951
2943
|
|
|
2952
|
-
|
|
2953
|
-
{
|
|
2944
|
+
let rolledBack = {
|
|
2954
2945
|
...fetchState,
|
|
2955
2946
|
// TODO: Test this. Currently it's not tested.
|
|
2956
2947
|
latestOnBlockBlockNumber: Pervasives.min(
|
|
@@ -2969,6 +2960,7 @@ let rollback = (fetchState: t, ~addressStore: AddressStore.t, ~targetBlockNumber
|
|
|
2969
2960
|
targetBlockNumber
|
|
2970
2961
|
),
|
|
2971
2962
|
)
|
|
2963
|
+
{fetchState: rolledBack, rolledBackAddresses}
|
|
2972
2964
|
}
|
|
2973
2965
|
|
|
2974
2966
|
// Reset pending queries by removing in-flight queries (ones without fetchedBlock).
|
package/src/FetchState.res.mjs
CHANGED
|
@@ -1123,25 +1123,16 @@ function warnRejectedRegistration(verdict, chainId, contractAddress, contractNam
|
|
|
1123
1123
|
contractName: contractName
|
|
1124
1124
|
}, `Skipping contract registration: Not a valid address for this chain's ecosystem.`);
|
|
1125
1125
|
}
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
}, `Skipping contract registration: Contract address is already registered at a later block number. Currently registration of the same contract address is not supported by Envio. Reach out to us if it's a problem for you.`);
|
|
1137
|
-
} else {
|
|
1138
|
-
return;
|
|
1139
|
-
}
|
|
1140
|
-
case "Conflict" :
|
|
1141
|
-
return warnAddressRegistration(chainId, contractAddress, {
|
|
1142
|
-
existingContractType: verdict.existingContractName,
|
|
1143
|
-
newContractType: contractName
|
|
1144
|
-
}, `Skipping contract registration: Contract address is already registered for one contract and cannot be registered for another contract.`);
|
|
1126
|
+
if (verdict.TAG === "Added") {
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
let existingEffectiveStartBlock = verdict.existingEffectiveStartBlock;
|
|
1130
|
+
let effectiveStartBlock = verdict.effectiveStartBlock;
|
|
1131
|
+
if (existingEffectiveStartBlock > effectiveStartBlock) {
|
|
1132
|
+
return warnAddressRegistration(chainId, contractAddress, {
|
|
1133
|
+
existingBlockNumber: existingEffectiveStartBlock,
|
|
1134
|
+
newBlockNumber: effectiveStartBlock
|
|
1135
|
+
}, `Skipping same-contract re-registration: the address is already registered for this contract. The start block does not move earlier.`);
|
|
1145
1136
|
}
|
|
1146
1137
|
}
|
|
1147
1138
|
|
|
@@ -1727,7 +1718,7 @@ function getReadyItemsCount(fetchState, targetSize, fromItem) {
|
|
|
1727
1718
|
return acc;
|
|
1728
1719
|
}
|
|
1729
1720
|
|
|
1730
|
-
function make$1(startBlock, endBlock, onEventRegistrations, addressStore,
|
|
1721
|
+
function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, maxAddrInPartition, chainId, maxOnBlockBufferSize, knownHeight, progressBlockNumberOpt, onBlockRegistrationsOpt, blockLagOpt, firstEventBlockOpt, clientFilterAddressThresholdOpt, isResumedOpt) {
|
|
1731
1722
|
let progressBlockNumber = progressBlockNumberOpt !== undefined ? progressBlockNumberOpt : startBlock - 1 | 0;
|
|
1732
1723
|
let onBlockRegistrations = onBlockRegistrationsOpt !== undefined ? onBlockRegistrationsOpt : [];
|
|
1733
1724
|
let blockLag = blockLagOpt !== undefined ? blockLagOpt : 0;
|
|
@@ -1762,29 +1753,24 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
|
|
|
1762
1753
|
});
|
|
1763
1754
|
}
|
|
1764
1755
|
let normalSelection = makeSelection(normalRegistrations, true, undefined);
|
|
1765
|
-
AddressStore.
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
}
|
|
1770
|
-
let contract = addresses[idx];
|
|
1771
|
-
warnRejectedRegistration(verdict, chainId, contract.address, contract.contractName);
|
|
1772
|
-
});
|
|
1756
|
+
AddressStore.seedRows(addressStore, addressRows).forEach(rejected => warnAddressRegistration(chainId, rejected.address, {
|
|
1757
|
+
contractName: rejected.contractName,
|
|
1758
|
+
existingBlockNumber: rejected.existingEffectiveStartBlock,
|
|
1759
|
+
newBlockNumber: rejected.effectiveStartBlock
|
|
1760
|
+
}, `Skipping a stored address: it is already registered for this contract.`));
|
|
1773
1761
|
let dynamicContracts = new Set();
|
|
1774
1762
|
let clientFilteredContracts = new Set();
|
|
1775
1763
|
let registeringSetsByContract = {};
|
|
1776
|
-
|
|
1777
|
-
|
|
1764
|
+
Array.from(contractNamesWithNormalEvents).forEach(contractName => {
|
|
1765
|
+
if (addressStore.contractCount(contractName) > 0) {
|
|
1766
|
+
registeringSetsByContract[contractName] = AddressStore.makeSet(addressStore, contractName, undefined);
|
|
1767
|
+
return;
|
|
1768
|
+
}
|
|
1769
|
+
});
|
|
1770
|
+
addressStore.dynamicContractNames().forEach(contractName => {
|
|
1778
1771
|
if (contractNamesWithNormalEvents.has(contractName)) {
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
}
|
|
1782
|
-
if (contract.registrationBlock !== -1) {
|
|
1783
|
-
dynamicContracts.add(contractName);
|
|
1784
|
-
return;
|
|
1785
|
-
} else {
|
|
1786
|
-
return;
|
|
1787
|
-
}
|
|
1772
|
+
dynamicContracts.add(contractName);
|
|
1773
|
+
return;
|
|
1788
1774
|
}
|
|
1789
1775
|
});
|
|
1790
1776
|
if (clientFilterAddressThreshold !== undefined) {
|
|
@@ -1797,7 +1783,7 @@ function make$1(startBlock, endBlock, onEventRegistrations, addressStore, addres
|
|
|
1797
1783
|
}
|
|
1798
1784
|
let optimizedPartitions = createPartitions(registeringSetsByContract, addressStore, dynamicContracts, clientFilteredContracts, normalSelection, maxAddrInPartition, partitions.length, partitions, progressBlockNumber, knownHeight);
|
|
1799
1785
|
if (optimizedPartitions.idsInAscOrder.length === 0 && Utils.$$Array.isEmpty(onBlockRegistrations)) {
|
|
1800
|
-
Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + ChainId.toString(chainId) + `. ` + (`addresses=` + addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
|
|
1786
|
+
Stdlib_JsError.throwWithMessage(`Invalid configuration: Nothing to fetch on chain ` + ChainId.toString(chainId) + `. ` + (`addresses=` + addressRows.addresses.length.toString() + `, `) + (`onEventRegistrations=` + onEventRegistrations.length.toString() + `, `) + (`normalRegistrations=` + normalRegistrations.length.toString() + `. `) + `Make sure that you provided at least one contract address to index, or have events with Wildcard mode enabled, or have onBlock handlers.`);
|
|
1801
1787
|
}
|
|
1802
1788
|
let buffer = [];
|
|
1803
1789
|
let latestOnBlockBlockNumber;
|
|
@@ -1861,8 +1847,8 @@ function rollbackPendingQueries(mutPendingQueries, targetBlockNumber) {
|
|
|
1861
1847
|
return adjusted;
|
|
1862
1848
|
}
|
|
1863
1849
|
|
|
1864
|
-
function rollback(fetchState,
|
|
1865
|
-
|
|
1850
|
+
function rollback(fetchState, rolledBackAddressStore, targetBlockNumber) {
|
|
1851
|
+
let rolledBackAddresses = rolledBackAddressStore.rollback(targetBlockNumber);
|
|
1866
1852
|
let keptPartitions = [];
|
|
1867
1853
|
let nextKeptIdRef = 0;
|
|
1868
1854
|
let registeringSetsByContract = {};
|
|
@@ -1921,8 +1907,8 @@ function rollback(fetchState, addressStore, targetBlockNumber) {
|
|
|
1921
1907
|
});
|
|
1922
1908
|
}
|
|
1923
1909
|
}
|
|
1924
|
-
let optimizedPartitions = createPartitions(registeringSetsByContract,
|
|
1925
|
-
|
|
1910
|
+
let optimizedPartitions = createPartitions(registeringSetsByContract, rolledBackAddressStore, fetchState.optimizedPartitions.dynamicContracts, fetchState.optimizedPartitions.clientFilteredContracts, fetchState.normalSelection, fetchState.optimizedPartitions.maxAddrInPartition, nextKeptIdRef, keptPartitions, targetBlockNumber, fetchState.knownHeight);
|
|
1911
|
+
let rolledBack = updateInternal({
|
|
1926
1912
|
optimizedPartitions: fetchState.optimizedPartitions,
|
|
1927
1913
|
startBlock: fetchState.startBlock,
|
|
1928
1914
|
endBlock: fetchState.endBlock,
|
|
@@ -1941,6 +1927,10 @@ function rollback(fetchState, addressStore, targetBlockNumber) {
|
|
|
1941
1927
|
tmp = item.kind === 0 ? item.blockNumber : item.blockNumber;
|
|
1942
1928
|
return tmp <= targetBlockNumber;
|
|
1943
1929
|
}), true, undefined, undefined);
|
|
1930
|
+
return {
|
|
1931
|
+
fetchState: rolledBack,
|
|
1932
|
+
rolledBackAddresses: rolledBackAddresses
|
|
1933
|
+
};
|
|
1944
1934
|
}
|
|
1945
1935
|
|
|
1946
1936
|
function resetPendingQueries(fetchState) {
|
package/src/InMemoryStore.res
CHANGED
|
@@ -130,11 +130,13 @@ let prepareRollbackDiff = async (
|
|
|
130
130
|
~rollbackTargetCheckpointId,
|
|
131
131
|
~rollbackDiffCheckpointId,
|
|
132
132
|
~progressBlockNumberByChainId,
|
|
133
|
+
~rolledBackAddresses,
|
|
133
134
|
) => {
|
|
134
135
|
state->IndexerState.beginRollbackDiff(
|
|
135
136
|
~targetCheckpointId=rollbackTargetCheckpointId,
|
|
136
137
|
~diffCheckpointId=rollbackDiffCheckpointId,
|
|
137
138
|
~progressBlockNumberByChainId,
|
|
139
|
+
~rolledBackAddresses,
|
|
138
140
|
)
|
|
139
141
|
let persistence = state->IndexerState.persistence
|
|
140
142
|
let committedCheckpointId = state->IndexerState.committedCheckpointId
|
|
@@ -194,12 +196,6 @@ let prepareRollbackDiff = async (
|
|
|
194
196
|
// registered them: the store is where they already live, and it knows which
|
|
195
197
|
// ones the database hasn't seen yet.
|
|
196
198
|
let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
|
|
197
|
-
let inMemTable = state->getInMemTable(
|
|
198
|
-
~entityConfig=InternalTable.EnvioAddresses.entityConfig,
|
|
199
|
-
~scope=CrossChain,
|
|
200
|
-
)
|
|
201
|
-
let committedCheckpointId = state->IndexerState.committedCheckpointId
|
|
202
|
-
|
|
203
199
|
batch.progressedChainsById->Utils.Dict.forEach(progressedChain => {
|
|
204
200
|
let chainId = progressedChain.fetchState.chainId
|
|
205
201
|
let chainState = state->IndexerState.getChainState(~chainId)
|
|
@@ -219,31 +215,22 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
|
|
|
219
215
|
}
|
|
220
216
|
}
|
|
221
217
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
->Array.forEach(dc => {
|
|
228
|
-
let entity: InternalTable.EnvioAddresses.t = {
|
|
229
|
-
id: InternalTable.EnvioAddresses.makeId(~chainId, ~address=dc.address),
|
|
230
|
-
chainId,
|
|
231
|
-
contractName: dc.contractName,
|
|
232
|
-
registrationBlock: dc.registrationBlock,
|
|
233
|
-
// Only ever written, never read back. Kept on the table so the column
|
|
234
|
-
// doesn't need a migration.
|
|
235
|
-
registrationLogIndex: -1,
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
inMemTable->InMemoryTable.Entity.set(
|
|
239
|
-
~committedCheckpointId,
|
|
240
|
-
Set({
|
|
241
|
-
entityId: entity.id->EntityId.unsafeOfString,
|
|
242
|
-
checkpointId: checkpointIds->Array.getUnsafe(dc.checkpointIdx),
|
|
243
|
-
entity: entity->InternalTable.EnvioAddresses.castToInternal,
|
|
244
|
-
}),
|
|
218
|
+
batch.registeredAddresses->Array.pushMany(
|
|
219
|
+
chainState
|
|
220
|
+
->ChainState.drainAddressesForWrite(
|
|
221
|
+
~toBlockInclusive=progressedChain.progressBlockNumber,
|
|
222
|
+
~checkpointBlockNumbers,
|
|
245
223
|
)
|
|
246
|
-
|
|
224
|
+
->Array.map((dc): AddressRows.staged => {
|
|
225
|
+
row: {
|
|
226
|
+
chainId,
|
|
227
|
+
address: dc.address,
|
|
228
|
+
contractId: dc.contractId,
|
|
229
|
+
registrationBlock: dc.registrationBlock,
|
|
230
|
+
},
|
|
231
|
+
checkpointId: checkpointIds->Array.getUnsafe(dc.checkpointIdx),
|
|
232
|
+
}),
|
|
233
|
+
)->ignore
|
|
247
234
|
}
|
|
248
235
|
})
|
|
249
236
|
}
|