envio 3.9.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -0
- package/fuel.schema.json +26 -0
- package/index.d.ts +35 -3
- package/package.json +6 -7
- package/src/Batch.res +9 -0
- package/src/ChainState.res +35 -39
- package/src/ChainState.res.mjs +14 -11
- package/src/ChainState.resi +15 -7
- package/src/Config.res +52 -44
- package/src/Config.res.mjs +51 -17
- package/src/Core.res +4 -0
- package/src/CrossChainState.res +15 -23
- package/src/CrossChainState.res.mjs +10 -18
- package/src/CrossChainState.resi +4 -1
- package/src/Ecosystem.res +7 -2
- package/src/Envio.res +6 -4
- package/src/EventConfigBuilder.res +169 -26
- package/src/EventConfigBuilder.res.mjs +124 -15
- package/src/HandlerRegister.res +21 -4
- package/src/HandlerRegister.res.mjs +24 -5
- package/src/InMemoryStore.res +9 -7
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/IndexerState.res +43 -6
- package/src/IndexerState.res.mjs +36 -7
- package/src/IndexerState.resi +4 -4
- package/src/Internal.res +16 -17
- package/src/Metrics.res +72 -6
- package/src/Metrics.res.mjs +30 -2
- package/src/Persistence.res +36 -30
- package/src/Persistence.res.mjs +1 -20
- package/src/PgStorage.res +104 -56
- package/src/PgStorage.res.mjs +57 -35
- package/src/PruneStaleHistory.res +85 -52
- package/src/PruneStaleHistory.res.mjs +63 -37
- package/src/Rollback.res +68 -45
- package/src/Rollback.res.mjs +38 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SafeCheckpointTracking.res +11 -11
- package/src/SafeCheckpointTracking.res.mjs +7 -6
- package/src/SimulateItems.res +33 -18
- package/src/SimulateItems.res.mjs +26 -22
- package/src/Sink.res +38 -26
- package/src/Sink.res.mjs +21 -16
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Utils.res +170 -0
- package/src/Utils.res.mjs +153 -0
- package/src/Writing.res +2 -2
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +366 -698
- package/src/bindings/ClickHouse.res.mjs +371 -421
- package/src/bindings/ClickHouseSink.res +337 -0
- package/src/bindings/ClickHouseSink.res.mjs +246 -0
- package/src/bindings/EventSource.res +8 -2
- package/src/bindings/NodeJs.res +6 -0
- package/src/bindings/Vitest.res +23 -0
- package/src/bindings/Vitest.res.mjs +3 -0
- package/src/bindings/WebSocket.res +9 -10
- package/src/db/CheckpointBounds.res +53 -0
- package/src/db/CheckpointBounds.res.mjs +44 -0
- package/src/db/EntityFilter.res +36 -17
- package/src/db/EntityFilter.res.mjs +25 -14
- package/src/db/EntityHistory.res +47 -22
- package/src/db/EntityHistory.res.mjs +19 -12
- package/src/db/InternalTable.res +84 -35
- package/src/db/InternalTable.res.mjs +56 -23
- package/src/db/RollbackFloors.res +55 -0
- package/src/db/RollbackFloors.res.mjs +92 -0
- package/src/db/Table.res +22 -4
- package/src/db/Table.res.mjs +30 -9
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/EvmHyperSyncSource.res +3 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcWs.res +124 -0
- package/src/sources/EvmRpcWs.res.mjs +117 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/FuelHyperSyncSource.res +1 -1
- package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
- package/src/sources/HeightFeed.res +568 -0
- package/src/sources/HeightFeed.res.mjs +452 -0
- package/src/sources/HeightStream.res +257 -0
- package/src/sources/HeightStream.res.mjs +194 -0
- package/src/sources/HyperSync.res +5 -0
- package/src/sources/HyperSync.res.mjs +3 -0
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncSSE.res +49 -0
- package/src/sources/HyperSyncSSE.res.mjs +79 -0
- package/src/sources/RpcSource.res +1 -1
- package/src/sources/RpcSource.res.mjs +2 -2
- package/src/sources/Source.res +54 -1
- package/src/sources/Source.res.mjs +25 -0
- package/src/sources/SourceManager.res +220 -227
- package/src/sources/SourceManager.res.mjs +139 -164
- package/src/sources/SourceManager.resi +8 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +70 -127
- package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
- package/src/sources/SvmHyperSyncSource.res +18 -15
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/svm.schema.json +1 -75
- package/src/MemoryStorage.res +0 -729
- package/src/MemoryStorage.res.mjs +0 -586
- package/src/sources/HyperSyncHeightStream.res +0 -129
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
- package/src/sources/RpcWebSocketHeightStream.res +0 -175
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/evm.schema.json
CHANGED
|
@@ -126,6 +126,17 @@
|
|
|
126
126
|
"type": "null"
|
|
127
127
|
}
|
|
128
128
|
]
|
|
129
|
+
},
|
|
130
|
+
"bytes_type": {
|
|
131
|
+
"description": "How the `Bytes` scalar in schema.graphql is represented. `hex` keeps 0x-prefixed hex strings stored as text, `uint8array` exposes `Uint8Array` values in handlers and stores raw bytes (BYTEA in Postgres, String in ClickHouse). (default: hex)",
|
|
132
|
+
"anyOf": [
|
|
133
|
+
{
|
|
134
|
+
"$ref": "#/$defs/BytesType"
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"type": "null"
|
|
138
|
+
}
|
|
139
|
+
]
|
|
129
140
|
}
|
|
130
141
|
},
|
|
131
142
|
"additionalProperties": false,
|
|
@@ -758,6 +769,21 @@
|
|
|
758
769
|
"checksum",
|
|
759
770
|
"lowercase"
|
|
760
771
|
]
|
|
772
|
+
},
|
|
773
|
+
"BytesType": {
|
|
774
|
+
"description": "How the schema.graphql `Bytes` scalar reaches handlers and storage.",
|
|
775
|
+
"oneOf": [
|
|
776
|
+
{
|
|
777
|
+
"description": "`0x`-prefixed hex strings, stored as text.",
|
|
778
|
+
"type": "string",
|
|
779
|
+
"const": "hex"
|
|
780
|
+
},
|
|
781
|
+
{
|
|
782
|
+
"description": "`Uint8Array` values, stored as raw bytes (`BYTEA` in Postgres, `String` in ClickHouse).",
|
|
783
|
+
"type": "string",
|
|
784
|
+
"const": "uint8array"
|
|
785
|
+
}
|
|
786
|
+
]
|
|
761
787
|
}
|
|
762
788
|
}
|
|
763
789
|
}
|
package/fuel.schema.json
CHANGED
|
@@ -83,6 +83,17 @@
|
|
|
83
83
|
"boolean",
|
|
84
84
|
"null"
|
|
85
85
|
]
|
|
86
|
+
},
|
|
87
|
+
"bytes_type": {
|
|
88
|
+
"description": "How the `Bytes` scalar in schema.graphql is represented. `hex` keeps 0x-prefixed hex strings stored as text, `uint8array` exposes `Uint8Array` values in handlers and stores raw bytes (BYTEA in Postgres, String in ClickHouse). (default: hex)",
|
|
89
|
+
"anyOf": [
|
|
90
|
+
{
|
|
91
|
+
"$ref": "#/$defs/BytesType"
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"type": "null"
|
|
95
|
+
}
|
|
96
|
+
]
|
|
86
97
|
}
|
|
87
98
|
},
|
|
88
99
|
"additionalProperties": false,
|
|
@@ -464,6 +475,21 @@
|
|
|
464
475
|
}
|
|
465
476
|
}
|
|
466
477
|
]
|
|
478
|
+
},
|
|
479
|
+
"BytesType": {
|
|
480
|
+
"description": "How the schema.graphql `Bytes` scalar reaches handlers and storage.",
|
|
481
|
+
"oneOf": [
|
|
482
|
+
{
|
|
483
|
+
"description": "`0x`-prefixed hex strings, stored as text.",
|
|
484
|
+
"type": "string",
|
|
485
|
+
"const": "hex"
|
|
486
|
+
},
|
|
487
|
+
{
|
|
488
|
+
"description": "`Uint8Array` values, stored as raw bytes (`BYTEA` in Postgres, `String` in ClickHouse).",
|
|
489
|
+
"type": "string",
|
|
490
|
+
"const": "uint8array"
|
|
491
|
+
}
|
|
492
|
+
]
|
|
467
493
|
}
|
|
468
494
|
}
|
|
469
495
|
}
|
package/index.d.ts
CHANGED
|
@@ -1423,7 +1423,7 @@ type SvmSelectedInstruction<Fields extends SvmFieldsSelection, ProgInstr> = {
|
|
|
1423
1423
|
readonly instructionName: string;
|
|
1424
1424
|
readonly discriminator: string;
|
|
1425
1425
|
readonly programId: SvmInstrField<Fields, "programId", string>;
|
|
1426
|
-
readonly data: SvmInstrField<Fields, "data",
|
|
1426
|
+
readonly data: SvmInstrField<Fields, "data", Uint8Array>;
|
|
1427
1427
|
readonly path: SvmInstrField<Fields, "path", readonly number[]>;
|
|
1428
1428
|
readonly isInner: SvmInstrField<Fields, "isInner", boolean>;
|
|
1429
1429
|
readonly args: SvmInstrField<
|
|
@@ -1484,6 +1484,37 @@ export type SvmOnInstructionHandlerArgs<
|
|
|
1484
1484
|
Instruction extends keyof SvmProgramsT[Program] & string = never,
|
|
1485
1485
|
> = SvmOnInstructionArgsFor<SvmInstruction<Fields, Program, Instruction>, GlobalConfig>;
|
|
1486
1486
|
|
|
1487
|
+
/** The named accounts of a configured instruction. Falls back to an open
|
|
1488
|
+
* record when the instruction carries no resolved schema, so its filter keys
|
|
1489
|
+
* are unconstrained rather than unusable. */
|
|
1490
|
+
type SvmAccountsOf<Program extends string, Instruction extends string> =
|
|
1491
|
+
SvmConfiguredInstruction<Program, Instruction>["accounts"];
|
|
1492
|
+
|
|
1493
|
+
/** One AND-group of account narrowings: every named account must match one of
|
|
1494
|
+
* its listed pubkeys. Names are the instruction's own account names. */
|
|
1495
|
+
export type SvmAccountsFilter<Accounts> = {
|
|
1496
|
+
readonly [K in keyof Accounts]?: string | readonly string[];
|
|
1497
|
+
};
|
|
1498
|
+
|
|
1499
|
+
/** The `where` option value of `indexer.onInstruction`.
|
|
1500
|
+
*
|
|
1501
|
+
* `accounts` accepts either a single AND-group of account narrowings, or an
|
|
1502
|
+
* array of them (OR semantics). `block.slot._gte` promotes to the
|
|
1503
|
+
* registration's startBlock and overrides the chain-level `start_block` — use
|
|
1504
|
+
* it to restrict per-instruction processing without touching `config.yaml`.
|
|
1505
|
+
* Only `_gte` is supported here; use `indexer.onSlot` for `_lte` / `_every`. */
|
|
1506
|
+
export type SvmOnInstructionWhere<Accounts> = {
|
|
1507
|
+
/** Match only inner (CPI-invoked) or only outer instructions. Absent
|
|
1508
|
+
* matches both. */
|
|
1509
|
+
readonly isInner?: boolean;
|
|
1510
|
+
readonly accounts?: SvmAccountsFilter<Accounts> | readonly SvmAccountsFilter<Accounts>[];
|
|
1511
|
+
readonly block?: {
|
|
1512
|
+
readonly slot?: {
|
|
1513
|
+
readonly _gte?: number;
|
|
1514
|
+
};
|
|
1515
|
+
};
|
|
1516
|
+
};
|
|
1517
|
+
|
|
1487
1518
|
/** Options for an SVM `indexer.onInstruction` registration. */
|
|
1488
1519
|
export type SvmOnInstructionOptions<
|
|
1489
1520
|
P extends string = string,
|
|
@@ -1497,6 +1528,7 @@ export type SvmOnInstructionOptions<
|
|
|
1497
1528
|
* `chains[].programs[].instructions[].name` in `config.yaml`. */
|
|
1498
1529
|
readonly instruction: I;
|
|
1499
1530
|
readonly fields?: Fields & SvmFieldsLiteralCheck<Fields>;
|
|
1531
|
+
readonly where?: SvmOnInstructionWhere<SvmAccountsOf<P, I>>;
|
|
1500
1532
|
};
|
|
1501
1533
|
|
|
1502
1534
|
/** Handler function for an SVM `indexer.onInstruction` registration. Takes
|
|
@@ -1905,8 +1937,8 @@ type SvmSimulateItem<Config extends IndexerConfigTypes = GlobalConfig> =
|
|
|
1905
1937
|
path?: readonly number[];
|
|
1906
1938
|
/** Override the program id. Defaults to the configured `program_id`. */
|
|
1907
1939
|
programId?: string;
|
|
1908
|
-
/** Raw instruction data
|
|
1909
|
-
data?:
|
|
1940
|
+
/** Raw instruction data. Defaults to the configured discriminator bytes. */
|
|
1941
|
+
data?: Uint8Array;
|
|
1910
1942
|
/** Whether this is a CPI-invoked inner instruction. */
|
|
1911
1943
|
isInner?: boolean;
|
|
1912
1944
|
/** Decoded args. Keys match the instruction's arg names. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.10.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -41,7 +41,6 @@
|
|
|
41
41
|
"licenses"
|
|
42
42
|
],
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@clickhouse/client": "1.17.0",
|
|
45
44
|
"@elastic/ecs-pino-format": "1.4.0",
|
|
46
45
|
"@fuel-ts/crypto": "0.96.1",
|
|
47
46
|
"@fuel-ts/errors": "0.96.1",
|
|
@@ -69,10 +68,10 @@
|
|
|
69
68
|
"tsx": "4.21.0"
|
|
70
69
|
},
|
|
71
70
|
"optionalDependencies": {
|
|
72
|
-
"envio-linux-x64": "3.
|
|
73
|
-
"envio-linux-x64-musl": "3.
|
|
74
|
-
"envio-linux-arm64": "3.
|
|
75
|
-
"envio-darwin-x64": "3.
|
|
76
|
-
"envio-darwin-arm64": "3.
|
|
71
|
+
"envio-linux-x64": "3.10.0",
|
|
72
|
+
"envio-linux-x64-musl": "3.10.0",
|
|
73
|
+
"envio-linux-arm64": "3.10.0",
|
|
74
|
+
"envio-darwin-x64": "3.10.0",
|
|
75
|
+
"envio-darwin-arm64": "3.10.0"
|
|
77
76
|
}
|
|
78
77
|
}
|
package/src/Batch.res
CHANGED
|
@@ -189,6 +189,15 @@ let addReorgCheckpoints = (
|
|
|
189
189
|
}
|
|
190
190
|
}
|
|
191
191
|
|
|
192
|
+
// Checkpoint ids are handed out in ascending order as each chain's items are
|
|
193
|
+
// walked in block order, so within a chain a higher id always means a later
|
|
194
|
+
// block. Rollback preserves that: it deletes the chain's ids above its target
|
|
195
|
+
// before any higher one is allocated, so the ids the re-indexed blocks get are
|
|
196
|
+
// above everything the chain still holds. An isolated rollback leans on the
|
|
197
|
+
// invariant — it deletes by `chain_id = c AND id > target`, which is only the
|
|
198
|
+
// chain's stale suffix if ids and blocks agree on order within the chain. Ids
|
|
199
|
+
// interleave freely *across* chains, which is exactly why that predicate can't
|
|
200
|
+
// be the id bound alone.
|
|
192
201
|
let prepareBatch = (
|
|
193
202
|
~checkpointIdBeforeBatch,
|
|
194
203
|
~chainsBeforeBatch: dict<chainBeforeBatch>,
|
package/src/ChainState.res
CHANGED
|
@@ -310,7 +310,11 @@ let makeInternal = (
|
|
|
310
310
|
chainId,
|
|
311
311
|
endpointUrl: hypersyncUrl,
|
|
312
312
|
apiToken,
|
|
313
|
-
onEventRegistrations
|
|
313
|
+
onEventRegistrations: onEventRegistrations->(
|
|
314
|
+
Utils.magic: array<Internal.onEventRegistration> => array<
|
|
315
|
+
Internal.svmOnEventRegistration,
|
|
316
|
+
>
|
|
317
|
+
),
|
|
314
318
|
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
315
319
|
addressStore,
|
|
316
320
|
}),
|
|
@@ -1205,21 +1209,17 @@ let markReady = (cs: t, ~readyAt) =>
|
|
|
1205
1209
|
cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
|
|
1206
1210
|
}
|
|
1207
1211
|
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
//
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
~eventsProcessedDiff,
|
|
1220
|
-
~rollbackTargetBlockNumber,
|
|
1221
|
-
~isReorgChain,
|
|
1222
|
-
): array<AddressRows.key> => {
|
|
1212
|
+
type progressDiff = {blockNumber: int, eventsProcessed: float}
|
|
1213
|
+
|
|
1214
|
+
type rolledBackTo =
|
|
1215
|
+
| RecomputedProgress(progressDiff)
|
|
1216
|
+
| ForkBlock(int)
|
|
1217
|
+
| Untouched
|
|
1218
|
+
|
|
1219
|
+
// Returns the address registrations the storage has to delete along with the
|
|
1220
|
+
// chain — the store is what decides which ones died, so the two halves of a
|
|
1221
|
+
// rollback can't disagree.
|
|
1222
|
+
let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
|
|
1223
1223
|
let rollbackTo = targetBlockNumber => {
|
|
1224
1224
|
let {fetchState, rolledBackAddresses} =
|
|
1225
1225
|
cs.fetchState->FetchState.rollback(
|
|
@@ -1234,15 +1234,15 @@ let rollback = (
|
|
|
1234
1234
|
})
|
|
1235
1235
|
}
|
|
1236
1236
|
|
|
1237
|
-
switch
|
|
1238
|
-
|
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1237
|
+
switch rolledBackTo {
|
|
1238
|
+
| RecomputedProgress({blockNumber, eventsProcessed}) =>
|
|
1239
|
+
// A rollback only ever takes a chain back. The diff recomputes progress from
|
|
1240
|
+
// the checkpoints still in the database, so a chain that an unwritten
|
|
1241
|
+
// rollback already took below them — to a fork block only that rollback
|
|
1242
|
+
// knew — would be handed their MIN back and moved forward onto blocks it
|
|
1243
|
+
// never re-indexed.
|
|
1244
|
+
let newProgressBlockNumber = Pervasives.min(blockNumber, cs.committedProgressBlockNumber)
|
|
1245
|
+
let newTotalEventsProcessed = cs.numEventsProcessed -. eventsProcessed
|
|
1246
1246
|
|
|
1247
1247
|
switch cs.safeCheckpointTracking {
|
|
1248
1248
|
| Some(safeCheckpointTracking) =>
|
|
@@ -1260,19 +1260,15 @@ let rollback = (
|
|
|
1260
1260
|
cs.processingBlockNumber = newProgressBlockNumber
|
|
1261
1261
|
cs.numEventsProcessed = newTotalEventsProcessed
|
|
1262
1262
|
rolledBackAddresses
|
|
1263
|
-
|
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
rolledBackAddresses
|
|
1274
|
-
} else {
|
|
1275
|
-
[]
|
|
1276
|
-
}
|
|
1263
|
+
| ForkBlock(forkBlock) =>
|
|
1264
|
+
// Rewinds the stores too, not just progress: otherwise the stale block hash
|
|
1265
|
+
// survives and re-triggers the same reorg.
|
|
1266
|
+
let rolledBackAddresses = rollbackTo(forkBlock)
|
|
1267
|
+
cs.transactionStore->TransactionStore.rollback(forkBlock)
|
|
1268
|
+
cs.blockStore->BlockStore.rollback(forkBlock)
|
|
1269
|
+
cs.committedProgressBlockNumber = Pervasives.min(cs.committedProgressBlockNumber, forkBlock)
|
|
1270
|
+
cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, forkBlock)
|
|
1271
|
+
rolledBackAddresses
|
|
1272
|
+
| Untouched => []
|
|
1277
1273
|
}
|
|
1278
1274
|
}
|
package/src/ChainState.res.mjs
CHANGED
|
@@ -857,7 +857,7 @@ function markReady(cs, readyAt) {
|
|
|
857
857
|
}
|
|
858
858
|
}
|
|
859
859
|
|
|
860
|
-
function rollback(cs,
|
|
860
|
+
function rollback(cs, rolledBackTo) {
|
|
861
861
|
let rollbackTo = targetBlockNumber => {
|
|
862
862
|
let match = FetchState.rollback(cs.fetchState, cs.addressStore, targetBlockNumber);
|
|
863
863
|
cs.fetchState = match.fetchState;
|
|
@@ -867,8 +867,13 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
|
|
|
867
867
|
contractId: param.contractId
|
|
868
868
|
}));
|
|
869
869
|
};
|
|
870
|
-
if (
|
|
871
|
-
|
|
870
|
+
if (typeof rolledBackTo !== "object") {
|
|
871
|
+
return [];
|
|
872
|
+
}
|
|
873
|
+
if (rolledBackTo.TAG === "RecomputedProgress") {
|
|
874
|
+
let match = rolledBackTo._0;
|
|
875
|
+
let newProgressBlockNumber = Primitive_int.min(match.blockNumber, cs.committedProgressBlockNumber);
|
|
876
|
+
let newTotalEventsProcessed = cs.numEventsProcessed - match.eventsProcessed;
|
|
872
877
|
let safeCheckpointTracking = cs.safeCheckpointTracking;
|
|
873
878
|
if (safeCheckpointTracking !== undefined) {
|
|
874
879
|
cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
|
|
@@ -881,14 +886,12 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
|
|
|
881
886
|
cs.numEventsProcessed = newTotalEventsProcessed;
|
|
882
887
|
return rolledBackAddresses;
|
|
883
888
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
cs.
|
|
889
|
-
cs.
|
|
890
|
-
cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
|
|
891
|
-
cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, rollbackTargetBlockNumber);
|
|
889
|
+
let forkBlock = rolledBackTo._0;
|
|
890
|
+
let rolledBackAddresses$1 = rollbackTo(forkBlock);
|
|
891
|
+
cs.transactionStore.rollback(forkBlock);
|
|
892
|
+
cs.blockStore.rollback(forkBlock);
|
|
893
|
+
cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, forkBlock);
|
|
894
|
+
cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, forkBlock);
|
|
892
895
|
return rolledBackAddresses$1;
|
|
893
896
|
}
|
|
894
897
|
|
package/src/ChainState.resi
CHANGED
|
@@ -157,10 +157,18 @@ let enterReorgThreshold: t => unit
|
|
|
157
157
|
let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
|
|
158
158
|
let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
|
|
159
159
|
let markReady: (t, ~readyAt: Date.t) => unit
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
160
|
+
// What a rollback's progress diff says about one chain: the block its surviving
|
|
161
|
+
// checkpoints put it back to, and the events the deleted ones carried away. One
|
|
162
|
+
// record because one diff row yields both — they are never known separately.
|
|
163
|
+
type progressDiff = {blockNumber: int, eventsProcessed: float}
|
|
164
|
+
|
|
165
|
+
// Where a rollback leaves one chain.
|
|
166
|
+
type rolledBackTo =
|
|
167
|
+
| RecomputedProgress(progressDiff)
|
|
168
|
+
// The reorg chain with no checkpoint left at or below the target, so only the
|
|
169
|
+
// fork block says where it goes.
|
|
170
|
+
| ForkBlock(int)
|
|
171
|
+
// A chain this rollback doesn't reach.
|
|
172
|
+
| Untouched
|
|
173
|
+
|
|
174
|
+
let rollback: (t, ~rolledBackTo: rolledBackTo) => array<AddressRows.key>
|
package/src/Config.res
CHANGED
|
@@ -177,24 +177,6 @@ let publicConfigChainSchema = S.schema(s =>
|
|
|
177
177
|
let svmEventDescriptorSchema = S.schema(s =>
|
|
178
178
|
{
|
|
179
179
|
"discriminator": s.matches(S.option(S.string)),
|
|
180
|
-
"discriminatorByteLen": s.matches(S.int),
|
|
181
|
-
// An array of AND-groups OR-ed together: the CLI normalizes both the flat
|
|
182
|
-
// and `any_of` YAML shapes to `Vec<Vec<SvmAccountFilterJson>>`.
|
|
183
|
-
"accountFilters": s.matches(
|
|
184
|
-
S.option(
|
|
185
|
-
S.array(
|
|
186
|
-
S.array(
|
|
187
|
-
S.schema(s =>
|
|
188
|
-
{
|
|
189
|
-
"position": s.matches(S.int),
|
|
190
|
-
"values": s.matches(S.array(S.string)),
|
|
191
|
-
}
|
|
192
|
-
),
|
|
193
|
-
),
|
|
194
|
-
),
|
|
195
|
-
),
|
|
196
|
-
),
|
|
197
|
-
"isInner": s.matches(S.option(S.bool)),
|
|
198
180
|
"accounts": s.matches(S.option(S.array(S.string))),
|
|
199
181
|
"args": s.matches(S.option(S.json(~validate=false))),
|
|
200
182
|
}
|
|
@@ -341,6 +323,7 @@ let getFieldTypeAndSchema = (prop, ~enumConfigsByName: dict<Table.enumConfig<Tab
|
|
|
341
323
|
|
|
342
324
|
let (fieldType, baseSchema) = switch typ {
|
|
343
325
|
| "string" => (Table.String, S.string->S.toUnknown)
|
|
326
|
+
| "bytes" => (Table.Bytea, Utils.Schema.bytes->S.toUnknown)
|
|
344
327
|
| "boolean" => (Table.Boolean, S.bool->S.toUnknown)
|
|
345
328
|
| "int" => (Table.Int32, S.int->S.toUnknown)
|
|
346
329
|
| "bigint" => (Table.BigInt({precision: ?prop["precision"]}), Utils.BigInt.schema->S.toUnknown)
|
|
@@ -374,13 +357,13 @@ let getFieldTypeAndSchema = (prop, ~enumConfigsByName: dict<Table.enumConfig<Tab
|
|
|
374
357
|
| other => JsError.throwWithMessage("Unknown field type in entity config: " ++ other)
|
|
375
358
|
}
|
|
376
359
|
|
|
377
|
-
let fieldSchema =
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
360
|
+
let fieldSchema = switch (fieldType, isArray) {
|
|
361
|
+
| (Table.Bytea, true) => Utils.Schema.bytesArray->S.toUnknown
|
|
362
|
+
| (_, true) => S.array(baseSchema)->S.toUnknown
|
|
363
|
+
| (_, false) => baseSchema
|
|
381
364
|
}
|
|
382
365
|
let fieldSchema = if isNullable {
|
|
383
|
-
|
|
366
|
+
Utils.Schema.nullTolerant(fieldSchema)->S.toUnknown
|
|
384
367
|
} else {
|
|
385
368
|
fieldSchema
|
|
386
369
|
}
|
|
@@ -753,11 +736,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
753
736
|
Utils.magic: _ => {
|
|
754
737
|
"svm": option<{
|
|
755
738
|
"discriminator": option<string>,
|
|
756
|
-
"discriminatorByteLen": int,
|
|
757
|
-
"accountFilters": option<
|
|
758
|
-
array<array<{"position": int, "values": array<string>}>>,
|
|
759
|
-
>,
|
|
760
|
-
"isInner": option<bool>,
|
|
761
739
|
"accounts": option<array<string>>,
|
|
762
740
|
"args": option<JSON.t>,
|
|
763
741
|
}>,
|
|
@@ -770,25 +748,11 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
770
748
|
`SVM instruction ${contractName}.${eventName} is missing the "svm" descriptor in internal config`,
|
|
771
749
|
)
|
|
772
750
|
}
|
|
773
|
-
let accountFilters =
|
|
774
|
-
svm["accountFilters"]
|
|
775
|
-
->Option.getOr([])
|
|
776
|
-
->Array.map(group =>
|
|
777
|
-
group->Array.map(
|
|
778
|
-
af => {
|
|
779
|
-
Internal.position: af["position"],
|
|
780
|
-
values: af["values"]->SvmTypes.Pubkey.fromStringsUnsafe,
|
|
781
|
-
},
|
|
782
|
-
)
|
|
783
|
-
)
|
|
784
751
|
(EventConfigBuilder.buildSvmInstructionEventConfig(
|
|
785
752
|
~contractName,
|
|
786
753
|
~instructionName=eventName,
|
|
787
754
|
~programId,
|
|
788
755
|
~discriminator=svm["discriminator"],
|
|
789
|
-
~discriminatorByteLen=svm["discriminatorByteLen"],
|
|
790
|
-
~accountFilters,
|
|
791
|
-
~isInner=svm["isInner"],
|
|
792
756
|
~accounts=svm["accounts"]->Option.getOr([]),
|
|
793
757
|
~args=svm["args"]->Option.getOr(JSON.Null),
|
|
794
758
|
~definedTypes=svmDefinedTypes,
|
|
@@ -1070,6 +1034,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
1070
1034
|
}
|
|
1071
1035
|
}
|
|
1072
1036
|
|
|
1037
|
+
// With no cross-chain entity, a reorg on one chain can never have changed a row
|
|
1038
|
+
// another chain owns, so its rollback stays isolated to that chain instead of
|
|
1039
|
+
// dragging every sibling back with it. A single chain has no sibling to spare,
|
|
1040
|
+
// and narrowing its rollback would only buy it a predicate that always holds.
|
|
1041
|
+
let isIsolatedMultichain = (config: t) =>
|
|
1042
|
+
config.chainMap->ChainMap.keys->Array.length > 1 &&
|
|
1043
|
+
config.userEntities->Array.every(entityConfig => !entityConfig.crossChain)
|
|
1044
|
+
|
|
1073
1045
|
// Canonicalize a user-provided address to the configured casing so it matches
|
|
1074
1046
|
// addresses parsed from config.yaml during routing. HyperSync/RPC data arrives
|
|
1075
1047
|
// already canonical; only user-land input (simulate srcAddress, contractRegister
|
|
@@ -1141,8 +1113,17 @@ let getEventConfig = (config: t, ~contractName, ~eventName, ~chainId: option<Cha
|
|
|
1141
1113
|
})
|
|
1142
1114
|
}
|
|
1143
1115
|
|
|
1144
|
-
|
|
1145
|
-
|
|
1116
|
+
// A chain that can't be rolled back (maxReorgDepth = 0) has no history to keep,
|
|
1117
|
+
// unless a cross-chain entity lets another chain's rollback reach its rows.
|
|
1118
|
+
let shouldSaveHistory = (config, ~isInReorgThreshold, ~chainId: option<ChainId.t>=?) =>
|
|
1119
|
+
config.shouldSaveFullHistory ||
|
|
1120
|
+
(config.shouldRollbackOnReorg &&
|
|
1121
|
+
isInReorgThreshold &&
|
|
1122
|
+
switch chainId {
|
|
1123
|
+
| Some(chainId) if config->isIsolatedMultichain =>
|
|
1124
|
+
(config.chainMap->ChainMap.get(chainId)).maxReorgDepth > 0
|
|
1125
|
+
| _ => true
|
|
1126
|
+
})
|
|
1146
1127
|
|
|
1147
1128
|
let shouldPruneHistory = (config, ~isInReorgThreshold) =>
|
|
1148
1129
|
!config.shouldSaveFullHistory && (config.shouldRollbackOnReorg && isInReorgThreshold)
|
|
@@ -1362,6 +1343,33 @@ let throwIfIncompatible = (
|
|
|
1362
1343
|
}
|
|
1363
1344
|
}
|
|
1364
1345
|
|
|
1346
|
+
let throwIfResumeIncompatible = (
|
|
1347
|
+
~storedEnvioInfo: option<JSON.t>,
|
|
1348
|
+
~storedContractMapping: ContractMapping.t,
|
|
1349
|
+
~envioInfo: JSON.t,
|
|
1350
|
+
~contractMapping: ContractMapping.t,
|
|
1351
|
+
~resetCommand: string,
|
|
1352
|
+
~runCommand: option<string>,
|
|
1353
|
+
) => {
|
|
1354
|
+
let changedPaths = switch storedEnvioInfo {
|
|
1355
|
+
| None => ["storage was initialized by an older envio version"]
|
|
1356
|
+
| Some(stored) => diffPaths(~stored, ~current=envioInfo)
|
|
1357
|
+
}
|
|
1358
|
+
let changedPaths =
|
|
1359
|
+
storedContractMapping->ContractMapping.isEqual(contractMapping)
|
|
1360
|
+
? changedPaths
|
|
1361
|
+
: changedPaths->Array.concat(["contracts"])
|
|
1362
|
+
let hasClickhouse = switch envioInfo {
|
|
1363
|
+
| Object(d) =>
|
|
1364
|
+
switch d->Dict.get("storage") {
|
|
1365
|
+
| Some(Object(s)) => s->Dict.get("clickhouse") == Some(Boolean(true))
|
|
1366
|
+
| _ => false
|
|
1367
|
+
}
|
|
1368
|
+
| _ => false
|
|
1369
|
+
}
|
|
1370
|
+
throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1365
1373
|
// The returned value is a pure function of the JSON: it holds only event
|
|
1366
1374
|
// definitions, never handler/contractRegister/where state (that's layered on
|
|
1367
1375
|
// separately as `HandlerRegister.registrationsByChainId`). That purity is
|
package/src/Config.res.mjs
CHANGED
|
@@ -17,6 +17,7 @@ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
|
17
17
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
18
18
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
19
19
|
import * as ContractMapping from "./ContractMapping.res.mjs";
|
|
20
|
+
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
20
21
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
21
22
|
import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
|
|
22
23
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
@@ -73,12 +74,6 @@ let publicConfigChainSchema = S$RescriptSchema.schema(s => ({
|
|
|
73
74
|
|
|
74
75
|
let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
|
|
75
76
|
discriminator: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
|
|
76
|
-
discriminatorByteLen: s.m(S$RescriptSchema.int),
|
|
77
|
-
accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
|
|
78
|
-
position: s.m(S$RescriptSchema.int),
|
|
79
|
-
values: s.m(S$RescriptSchema.array(S$RescriptSchema.string))
|
|
80
|
-
})))))),
|
|
81
|
-
isInner: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
|
|
82
77
|
accounts: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.string))),
|
|
83
78
|
args: s.m(S$RescriptSchema.option(S$RescriptSchema.json(false)))
|
|
84
79
|
}));
|
|
@@ -226,6 +221,12 @@ function getFieldTypeAndSchema(prop, enumConfigsByName) {
|
|
|
226
221
|
S$RescriptSchema.bool
|
|
227
222
|
];
|
|
228
223
|
break;
|
|
224
|
+
case "bytes" :
|
|
225
|
+
match = [
|
|
226
|
+
"Bytea",
|
|
227
|
+
Utils.Schema.bytes
|
|
228
|
+
];
|
|
229
|
+
break;
|
|
229
230
|
case "date" :
|
|
230
231
|
match = [
|
|
231
232
|
"Date",
|
|
@@ -277,10 +278,20 @@ function getFieldTypeAndSchema(prop, enumConfigsByName) {
|
|
|
277
278
|
match = Stdlib_JsError.throwWithMessage("Unknown field type in entity config: " + typ);
|
|
278
279
|
}
|
|
279
280
|
let baseSchema = match[1];
|
|
280
|
-
let
|
|
281
|
-
let fieldSchema
|
|
281
|
+
let fieldType = match[0];
|
|
282
|
+
let fieldSchema;
|
|
283
|
+
let exit = 0;
|
|
284
|
+
if (typeof fieldType !== "object" && fieldType === "Bytea" && isArray) {
|
|
285
|
+
fieldSchema = Utils.Schema.bytesArray;
|
|
286
|
+
} else {
|
|
287
|
+
exit = 1;
|
|
288
|
+
}
|
|
289
|
+
if (exit === 1) {
|
|
290
|
+
fieldSchema = isArray ? S$RescriptSchema.array(baseSchema) : baseSchema;
|
|
291
|
+
}
|
|
292
|
+
let fieldSchema$1 = isNullable ? Utils.Schema.nullTolerant(fieldSchema) : fieldSchema;
|
|
282
293
|
return [
|
|
283
|
-
|
|
294
|
+
fieldType,
|
|
284
295
|
fieldSchema$1,
|
|
285
296
|
isNullable,
|
|
286
297
|
isArray,
|
|
@@ -527,11 +538,7 @@ function fromPublic(publicConfigJson) {
|
|
|
527
538
|
) : addresses[0];
|
|
528
539
|
let s = eventItem.svm;
|
|
529
540
|
let svm = s !== undefined ? Primitive_option.valFromOption(s) : Stdlib_JsError.throwWithMessage(`SVM instruction ` + contractName + `.` + eventName + ` is missing the "svm" descriptor in internal config`);
|
|
530
|
-
|
|
531
|
-
position: af.position,
|
|
532
|
-
values: af.values
|
|
533
|
-
})));
|
|
534
|
-
return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, accountFilters, svm.isInner, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
|
|
541
|
+
return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
|
|
535
542
|
}
|
|
536
543
|
});
|
|
537
544
|
} else {
|
|
@@ -726,6 +733,14 @@ function fromPublic(publicConfigJson) {
|
|
|
726
733
|
};
|
|
727
734
|
}
|
|
728
735
|
|
|
736
|
+
function isIsolatedMultichain(config) {
|
|
737
|
+
if (ChainMap.keys(config.chainMap).length > 1) {
|
|
738
|
+
return config.userEntities.every(entityConfig => !entityConfig.crossChain);
|
|
739
|
+
} else {
|
|
740
|
+
return false;
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
|
|
729
744
|
function normalizeUserAddress(config, address) {
|
|
730
745
|
let match = config.ecosystem.name;
|
|
731
746
|
switch (match) {
|
|
@@ -789,11 +804,15 @@ function getEventConfig(config, contractName, eventName, chainId) {
|
|
|
789
804
|
});
|
|
790
805
|
}
|
|
791
806
|
|
|
792
|
-
function shouldSaveHistory(config, isInReorgThreshold) {
|
|
807
|
+
function shouldSaveHistory(config, isInReorgThreshold, chainId) {
|
|
793
808
|
if (config.shouldSaveFullHistory) {
|
|
794
809
|
return true;
|
|
795
|
-
} else if (config.shouldRollbackOnReorg) {
|
|
796
|
-
|
|
810
|
+
} else if (config.shouldRollbackOnReorg && isInReorgThreshold) {
|
|
811
|
+
if (chainId !== undefined && isIsolatedMultichain(config)) {
|
|
812
|
+
return ChainMap.get(config.chainMap, Primitive_option.valFromOption(chainId)).maxReorgDepth > 0;
|
|
813
|
+
} else {
|
|
814
|
+
return true;
|
|
815
|
+
}
|
|
797
816
|
} else {
|
|
798
817
|
return false;
|
|
799
818
|
}
|
|
@@ -1031,6 +1050,19 @@ function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhou
|
|
|
1031
1050
|
Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n 1. ` + padTo(option1, col) + `# resume indexing where it left off\n 2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
|
|
1032
1051
|
}
|
|
1033
1052
|
|
|
1053
|
+
function throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand) {
|
|
1054
|
+
let changedPaths = storedEnvioInfo !== undefined ? diffPaths(storedEnvioInfo, envioInfo) : ["storage was initialized by an older envio version"];
|
|
1055
|
+
let changedPaths$1 = ContractMapping.isEqual(storedContractMapping, contractMapping) ? changedPaths : changedPaths.concat(["contracts"]);
|
|
1056
|
+
let hasClickhouse;
|
|
1057
|
+
if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
|
|
1058
|
+
let match = envioInfo["storage"];
|
|
1059
|
+
hasClickhouse = typeof match === "object" && match !== null && !Array.isArray(match) ? Primitive_object.equal(match["clickhouse"], true) : false;
|
|
1060
|
+
} else {
|
|
1061
|
+
hasClickhouse = false;
|
|
1062
|
+
}
|
|
1063
|
+
throwIfIncompatible(changedPaths$1, resetCommand, runCommand, hasClickhouse);
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1034
1066
|
function load() {
|
|
1035
1067
|
let c = cached.contents;
|
|
1036
1068
|
if (c !== undefined) {
|
|
@@ -1075,6 +1107,7 @@ export {
|
|
|
1075
1107
|
publicConfigSchema,
|
|
1076
1108
|
contractMappingOf,
|
|
1077
1109
|
fromPublic,
|
|
1110
|
+
isIsolatedMultichain,
|
|
1078
1111
|
normalizeUserAddress,
|
|
1079
1112
|
normalizeSimulateAddress,
|
|
1080
1113
|
getEventConfig,
|
|
@@ -1087,6 +1120,7 @@ export {
|
|
|
1087
1120
|
canonicalJson,
|
|
1088
1121
|
diffPaths,
|
|
1089
1122
|
throwIfIncompatible,
|
|
1123
|
+
throwIfResumeIncompatible,
|
|
1090
1124
|
load,
|
|
1091
1125
|
getPgUserEntities,
|
|
1092
1126
|
}
|