envio 3.9.0 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/evm.schema.json +26 -0
  2. package/fuel.schema.json +26 -0
  3. package/index.d.ts +35 -3
  4. package/package.json +6 -7
  5. package/src/Batch.res +9 -0
  6. package/src/ChainState.res +35 -39
  7. package/src/ChainState.res.mjs +14 -11
  8. package/src/ChainState.resi +15 -7
  9. package/src/Config.res +52 -44
  10. package/src/Config.res.mjs +51 -17
  11. package/src/Core.res +4 -0
  12. package/src/CrossChainState.res +15 -23
  13. package/src/CrossChainState.res.mjs +10 -18
  14. package/src/CrossChainState.resi +4 -1
  15. package/src/Ecosystem.res +7 -2
  16. package/src/Envio.res +6 -4
  17. package/src/EventConfigBuilder.res +169 -26
  18. package/src/EventConfigBuilder.res.mjs +124 -15
  19. package/src/HandlerRegister.res +21 -4
  20. package/src/HandlerRegister.res.mjs +24 -5
  21. package/src/InMemoryStore.res +9 -7
  22. package/src/InMemoryStore.res.mjs +3 -3
  23. package/src/IndexerState.res +43 -6
  24. package/src/IndexerState.res.mjs +36 -7
  25. package/src/IndexerState.resi +4 -4
  26. package/src/Internal.res +16 -17
  27. package/src/Metrics.res +72 -6
  28. package/src/Metrics.res.mjs +30 -2
  29. package/src/Persistence.res +36 -30
  30. package/src/Persistence.res.mjs +1 -20
  31. package/src/PgStorage.res +104 -56
  32. package/src/PgStorage.res.mjs +57 -35
  33. package/src/PruneStaleHistory.res +85 -52
  34. package/src/PruneStaleHistory.res.mjs +63 -37
  35. package/src/Rollback.res +68 -45
  36. package/src/Rollback.res.mjs +38 -19
  37. package/src/RollbackCommit.res +7 -9
  38. package/src/RollbackCommit.res.mjs +5 -6
  39. package/src/SafeCheckpointTracking.res +11 -11
  40. package/src/SafeCheckpointTracking.res.mjs +7 -6
  41. package/src/SimulateItems.res +33 -18
  42. package/src/SimulateItems.res.mjs +26 -22
  43. package/src/Sink.res +38 -26
  44. package/src/Sink.res.mjs +21 -16
  45. package/src/TestIndexer.res +5 -5
  46. package/src/TestIndexer.res.mjs +1 -1
  47. package/src/Utils.res +170 -0
  48. package/src/Utils.res.mjs +153 -0
  49. package/src/Writing.res +2 -2
  50. package/src/Writing.res.mjs +1 -1
  51. package/src/bindings/ClickHouse.res +366 -698
  52. package/src/bindings/ClickHouse.res.mjs +371 -421
  53. package/src/bindings/ClickHouseSink.res +337 -0
  54. package/src/bindings/ClickHouseSink.res.mjs +246 -0
  55. package/src/bindings/EventSource.res +8 -2
  56. package/src/bindings/NodeJs.res +6 -0
  57. package/src/bindings/Vitest.res +23 -0
  58. package/src/bindings/Vitest.res.mjs +3 -0
  59. package/src/bindings/WebSocket.res +9 -10
  60. package/src/db/CheckpointBounds.res +53 -0
  61. package/src/db/CheckpointBounds.res.mjs +44 -0
  62. package/src/db/EntityFilter.res +36 -17
  63. package/src/db/EntityFilter.res.mjs +25 -14
  64. package/src/db/EntityHistory.res +47 -22
  65. package/src/db/EntityHistory.res.mjs +19 -12
  66. package/src/db/InternalTable.res +84 -35
  67. package/src/db/InternalTable.res.mjs +56 -23
  68. package/src/db/RollbackFloors.res +55 -0
  69. package/src/db/RollbackFloors.res.mjs +92 -0
  70. package/src/db/Table.res +22 -4
  71. package/src/db/Table.res.mjs +30 -9
  72. package/src/sources/Evm.res +2 -0
  73. package/src/sources/Evm.res.mjs +2 -0
  74. package/src/sources/EvmHyperSyncSource.res +3 -3
  75. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  76. package/src/sources/EvmRpcWs.res +124 -0
  77. package/src/sources/EvmRpcWs.res.mjs +117 -0
  78. package/src/sources/Fuel.res +2 -0
  79. package/src/sources/Fuel.res.mjs +2 -0
  80. package/src/sources/FuelHyperSyncSource.res +1 -1
  81. package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
  82. package/src/sources/HeightFeed.res +568 -0
  83. package/src/sources/HeightFeed.res.mjs +452 -0
  84. package/src/sources/HeightStream.res +257 -0
  85. package/src/sources/HeightStream.res.mjs +194 -0
  86. package/src/sources/HyperSync.res +5 -0
  87. package/src/sources/HyperSync.res.mjs +3 -0
  88. package/src/sources/HyperSync.resi +4 -0
  89. package/src/sources/HyperSyncSSE.res +49 -0
  90. package/src/sources/HyperSyncSSE.res.mjs +79 -0
  91. package/src/sources/RpcSource.res +1 -1
  92. package/src/sources/RpcSource.res.mjs +2 -2
  93. package/src/sources/Source.res +54 -1
  94. package/src/sources/Source.res.mjs +25 -0
  95. package/src/sources/SourceManager.res +220 -227
  96. package/src/sources/SourceManager.res.mjs +139 -164
  97. package/src/sources/SourceManager.resi +8 -0
  98. package/src/sources/Svm.res +2 -0
  99. package/src/sources/Svm.res.mjs +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +70 -127
  101. package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
  102. package/src/sources/SvmHyperSyncSource.res +18 -15
  103. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  104. package/svm.schema.json +1 -75
  105. package/src/MemoryStorage.res +0 -729
  106. package/src/MemoryStorage.res.mjs +0 -586
  107. package/src/sources/HyperSyncHeightStream.res +0 -129
  108. package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
  109. package/src/sources/RpcWebSocketHeightStream.res +0 -175
  110. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/evm.schema.json CHANGED
@@ -126,6 +126,17 @@
126
126
  "type": "null"
127
127
  }
128
128
  ]
129
+ },
130
+ "bytes_type": {
131
+ "description": "How the `Bytes` scalar in schema.graphql is represented. `hex` keeps 0x-prefixed hex strings stored as text, `uint8array` exposes `Uint8Array` values in handlers and stores raw bytes (BYTEA in Postgres, String in ClickHouse). (default: hex)",
132
+ "anyOf": [
133
+ {
134
+ "$ref": "#/$defs/BytesType"
135
+ },
136
+ {
137
+ "type": "null"
138
+ }
139
+ ]
129
140
  }
130
141
  },
131
142
  "additionalProperties": false,
@@ -758,6 +769,21 @@
758
769
  "checksum",
759
770
  "lowercase"
760
771
  ]
772
+ },
773
+ "BytesType": {
774
+ "description": "How the schema.graphql `Bytes` scalar reaches handlers and storage.",
775
+ "oneOf": [
776
+ {
777
+ "description": "`0x`-prefixed hex strings, stored as text.",
778
+ "type": "string",
779
+ "const": "hex"
780
+ },
781
+ {
782
+ "description": "`Uint8Array` values, stored as raw bytes (`BYTEA` in Postgres, `String` in ClickHouse).",
783
+ "type": "string",
784
+ "const": "uint8array"
785
+ }
786
+ ]
761
787
  }
762
788
  }
763
789
  }
package/fuel.schema.json CHANGED
@@ -83,6 +83,17 @@
83
83
  "boolean",
84
84
  "null"
85
85
  ]
86
+ },
87
+ "bytes_type": {
88
+ "description": "How the `Bytes` scalar in schema.graphql is represented. `hex` keeps 0x-prefixed hex strings stored as text, `uint8array` exposes `Uint8Array` values in handlers and stores raw bytes (BYTEA in Postgres, String in ClickHouse). (default: hex)",
89
+ "anyOf": [
90
+ {
91
+ "$ref": "#/$defs/BytesType"
92
+ },
93
+ {
94
+ "type": "null"
95
+ }
96
+ ]
86
97
  }
87
98
  },
88
99
  "additionalProperties": false,
@@ -464,6 +475,21 @@
464
475
  }
465
476
  }
466
477
  ]
478
+ },
479
+ "BytesType": {
480
+ "description": "How the schema.graphql `Bytes` scalar reaches handlers and storage.",
481
+ "oneOf": [
482
+ {
483
+ "description": "`0x`-prefixed hex strings, stored as text.",
484
+ "type": "string",
485
+ "const": "hex"
486
+ },
487
+ {
488
+ "description": "`Uint8Array` values, stored as raw bytes (`BYTEA` in Postgres, `String` in ClickHouse).",
489
+ "type": "string",
490
+ "const": "uint8array"
491
+ }
492
+ ]
467
493
  }
468
494
  }
469
495
  }
package/index.d.ts CHANGED
@@ -1423,7 +1423,7 @@ type SvmSelectedInstruction<Fields extends SvmFieldsSelection, ProgInstr> = {
1423
1423
  readonly instructionName: string;
1424
1424
  readonly discriminator: string;
1425
1425
  readonly programId: SvmInstrField<Fields, "programId", string>;
1426
- readonly data: SvmInstrField<Fields, "data", string>;
1426
+ readonly data: SvmInstrField<Fields, "data", Uint8Array>;
1427
1427
  readonly path: SvmInstrField<Fields, "path", readonly number[]>;
1428
1428
  readonly isInner: SvmInstrField<Fields, "isInner", boolean>;
1429
1429
  readonly args: SvmInstrField<
@@ -1484,6 +1484,37 @@ export type SvmOnInstructionHandlerArgs<
1484
1484
  Instruction extends keyof SvmProgramsT[Program] & string = never,
1485
1485
  > = SvmOnInstructionArgsFor<SvmInstruction<Fields, Program, Instruction>, GlobalConfig>;
1486
1486
 
1487
+ /** The named accounts of a configured instruction. Falls back to an open
1488
+ * record when the instruction carries no resolved schema, so its filter keys
1489
+ * are unconstrained rather than unusable. */
1490
+ type SvmAccountsOf<Program extends string, Instruction extends string> =
1491
+ SvmConfiguredInstruction<Program, Instruction>["accounts"];
1492
+
1493
+ /** One AND-group of account narrowings: every named account must match one of
1494
+ * its listed pubkeys. Names are the instruction's own account names. */
1495
+ export type SvmAccountsFilter<Accounts> = {
1496
+ readonly [K in keyof Accounts]?: string | readonly string[];
1497
+ };
1498
+
1499
+ /** The `where` option value of `indexer.onInstruction`.
1500
+ *
1501
+ * `accounts` accepts either a single AND-group of account narrowings, or an
1502
+ * array of them (OR semantics). `block.slot._gte` promotes to the
1503
+ * registration's startBlock and overrides the chain-level `start_block` — use
1504
+ * it to restrict per-instruction processing without touching `config.yaml`.
1505
+ * Only `_gte` is supported here; use `indexer.onSlot` for `_lte` / `_every`. */
1506
+ export type SvmOnInstructionWhere<Accounts> = {
1507
+ /** Match only inner (CPI-invoked) or only outer instructions. Absent
1508
+ * matches both. */
1509
+ readonly isInner?: boolean;
1510
+ readonly accounts?: SvmAccountsFilter<Accounts> | readonly SvmAccountsFilter<Accounts>[];
1511
+ readonly block?: {
1512
+ readonly slot?: {
1513
+ readonly _gte?: number;
1514
+ };
1515
+ };
1516
+ };
1517
+
1487
1518
  /** Options for an SVM `indexer.onInstruction` registration. */
1488
1519
  export type SvmOnInstructionOptions<
1489
1520
  P extends string = string,
@@ -1497,6 +1528,7 @@ export type SvmOnInstructionOptions<
1497
1528
  * `chains[].programs[].instructions[].name` in `config.yaml`. */
1498
1529
  readonly instruction: I;
1499
1530
  readonly fields?: Fields & SvmFieldsLiteralCheck<Fields>;
1531
+ readonly where?: SvmOnInstructionWhere<SvmAccountsOf<P, I>>;
1500
1532
  };
1501
1533
 
1502
1534
  /** Handler function for an SVM `indexer.onInstruction` registration. Takes
@@ -1905,8 +1937,8 @@ type SvmSimulateItem<Config extends IndexerConfigTypes = GlobalConfig> =
1905
1937
  path?: readonly number[];
1906
1938
  /** Override the program id. Defaults to the configured `program_id`. */
1907
1939
  programId?: string;
1908
- /** Raw instruction data, `0x`-prefixed hex. */
1909
- data?: string;
1940
+ /** Raw instruction data. Defaults to the configured discriminator bytes. */
1941
+ data?: Uint8Array;
1910
1942
  /** Whether this is a CPI-invoked inner instruction. */
1911
1943
  isInner?: boolean;
1912
1944
  /** Decoded args. Keys match the instruction's arg names. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "envio",
3
- "version": "3.9.0",
3
+ "version": "3.10.0",
4
4
  "type": "module",
5
5
  "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
6
6
  "bin": "./bin.mjs",
@@ -41,7 +41,6 @@
41
41
  "licenses"
42
42
  ],
43
43
  "dependencies": {
44
- "@clickhouse/client": "1.17.0",
45
44
  "@elastic/ecs-pino-format": "1.4.0",
46
45
  "@fuel-ts/crypto": "0.96.1",
47
46
  "@fuel-ts/errors": "0.96.1",
@@ -69,10 +68,10 @@
69
68
  "tsx": "4.21.0"
70
69
  },
71
70
  "optionalDependencies": {
72
- "envio-linux-x64": "3.9.0",
73
- "envio-linux-x64-musl": "3.9.0",
74
- "envio-linux-arm64": "3.9.0",
75
- "envio-darwin-x64": "3.9.0",
76
- "envio-darwin-arm64": "3.9.0"
71
+ "envio-linux-x64": "3.10.0",
72
+ "envio-linux-x64-musl": "3.10.0",
73
+ "envio-linux-arm64": "3.10.0",
74
+ "envio-darwin-x64": "3.10.0",
75
+ "envio-darwin-arm64": "3.10.0"
77
76
  }
78
77
  }
package/src/Batch.res CHANGED
@@ -189,6 +189,15 @@ let addReorgCheckpoints = (
189
189
  }
190
190
  }
191
191
 
192
+ // Checkpoint ids are handed out in ascending order as each chain's items are
193
+ // walked in block order, so within a chain a higher id always means a later
194
+ // block. Rollback preserves that: it deletes the chain's ids above its target
195
+ // before any higher one is allocated, so the ids the re-indexed blocks get are
196
+ // above everything the chain still holds. An isolated rollback leans on the
197
+ // invariant — it deletes by `chain_id = c AND id > target`, which is only the
198
+ // chain's stale suffix if ids and blocks agree on order within the chain. Ids
199
+ // interleave freely *across* chains, which is exactly why that predicate can't
200
+ // be the id bound alone.
192
201
  let prepareBatch = (
193
202
  ~checkpointIdBeforeBatch,
194
203
  ~chainsBeforeBatch: dict<chainBeforeBatch>,
@@ -310,7 +310,11 @@ let makeInternal = (
310
310
  chainId,
311
311
  endpointUrl: hypersyncUrl,
312
312
  apiToken,
313
- onEventRegistrations,
313
+ onEventRegistrations: onEventRegistrations->(
314
+ Utils.magic: array<Internal.onEventRegistration> => array<
315
+ Internal.svmOnEventRegistration,
316
+ >
317
+ ),
314
318
  clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
315
319
  addressStore,
316
320
  }),
@@ -1205,21 +1209,17 @@ let markReady = (cs: t, ~readyAt) =>
1205
1209
  cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
1206
1210
  }
1207
1211
 
1208
- // Roll a chain back to a reorg target. With a progress diff, restore fetch/
1209
- // safe-checkpoint/progress state to `newProgressBlockNumber`. A reorg chain
1210
- // with no diff entry still rewinds fetch state + stores to the target -
1211
- // otherwise the stale block hash stays in the store and re-triggers the same
1212
- // reorg.
1213
- // Returns the address registrations the storage has to delete along with it —
1214
- // the store is what decides which ones died, so the two halves of a rollback
1215
- // can't disagree.
1216
- let rollback = (
1217
- cs: t,
1218
- ~newProgressBlockNumber,
1219
- ~eventsProcessedDiff,
1220
- ~rollbackTargetBlockNumber,
1221
- ~isReorgChain,
1222
- ): array<AddressRows.key> => {
1212
+ type progressDiff = {blockNumber: int, eventsProcessed: float}
1213
+
1214
+ type rolledBackTo =
1215
+ | RecomputedProgress(progressDiff)
1216
+ | ForkBlock(int)
1217
+ | Untouched
1218
+
1219
+ // Returns the address registrations the storage has to delete along with the
1220
+ // chain the store is what decides which ones died, so the two halves of a
1221
+ // rollback can't disagree.
1222
+ let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
1223
1223
  let rollbackTo = targetBlockNumber => {
1224
1224
  let {fetchState, rolledBackAddresses} =
1225
1225
  cs.fetchState->FetchState.rollback(
@@ -1234,15 +1234,15 @@ let rollback = (
1234
1234
  })
1235
1235
  }
1236
1236
 
1237
- switch newProgressBlockNumber {
1238
- | Some(newProgressBlockNumber) =>
1239
- let newTotalEventsProcessed =
1240
- cs.numEventsProcessed -.
1241
- // Both dicts are populated together per progress-diff row, so a chain with
1242
- // a progress diff always has an events-processed diff too.
1243
- eventsProcessedDiff->Option.getOrThrow(
1244
- ~message="Missing events-processed diff for rolled-back chain",
1245
- )
1237
+ switch rolledBackTo {
1238
+ | RecomputedProgress({blockNumber, eventsProcessed}) =>
1239
+ // A rollback only ever takes a chain back. The diff recomputes progress from
1240
+ // the checkpoints still in the database, so a chain that an unwritten
1241
+ // rollback already took below them to a fork block only that rollback
1242
+ // knew would be handed their MIN back and moved forward onto blocks it
1243
+ // never re-indexed.
1244
+ let newProgressBlockNumber = Pervasives.min(blockNumber, cs.committedProgressBlockNumber)
1245
+ let newTotalEventsProcessed = cs.numEventsProcessed -. eventsProcessed
1246
1246
 
1247
1247
  switch cs.safeCheckpointTracking {
1248
1248
  | Some(safeCheckpointTracking) =>
@@ -1260,19 +1260,15 @@ let rollback = (
1260
1260
  cs.processingBlockNumber = newProgressBlockNumber
1261
1261
  cs.numEventsProcessed = newTotalEventsProcessed
1262
1262
  rolledBackAddresses
1263
- | None =>
1264
- if isReorgChain {
1265
- let rolledBackAddresses = rollbackTo(rollbackTargetBlockNumber)
1266
- cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)
1267
- cs.blockStore->BlockStore.rollback(rollbackTargetBlockNumber)
1268
- cs.committedProgressBlockNumber = Pervasives.min(
1269
- cs.committedProgressBlockNumber,
1270
- rollbackTargetBlockNumber,
1271
- )
1272
- cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, rollbackTargetBlockNumber)
1273
- rolledBackAddresses
1274
- } else {
1275
- []
1276
- }
1263
+ | ForkBlock(forkBlock) =>
1264
+ // Rewinds the stores too, not just progress: otherwise the stale block hash
1265
+ // survives and re-triggers the same reorg.
1266
+ let rolledBackAddresses = rollbackTo(forkBlock)
1267
+ cs.transactionStore->TransactionStore.rollback(forkBlock)
1268
+ cs.blockStore->BlockStore.rollback(forkBlock)
1269
+ cs.committedProgressBlockNumber = Pervasives.min(cs.committedProgressBlockNumber, forkBlock)
1270
+ cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, forkBlock)
1271
+ rolledBackAddresses
1272
+ | Untouched => []
1277
1273
  }
1278
1274
  }
@@ -857,7 +857,7 @@ function markReady(cs, readyAt) {
857
857
  }
858
858
  }
859
859
 
860
- function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTargetBlockNumber, isReorgChain) {
860
+ function rollback(cs, rolledBackTo) {
861
861
  let rollbackTo = targetBlockNumber => {
862
862
  let match = FetchState.rollback(cs.fetchState, cs.addressStore, targetBlockNumber);
863
863
  cs.fetchState = match.fetchState;
@@ -867,8 +867,13 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
867
867
  contractId: param.contractId
868
868
  }));
869
869
  };
870
- if (newProgressBlockNumber !== undefined) {
871
- let newTotalEventsProcessed = cs.numEventsProcessed - Stdlib_Option.getOrThrow(eventsProcessedDiff, "Missing events-processed diff for rolled-back chain");
870
+ if (typeof rolledBackTo !== "object") {
871
+ return [];
872
+ }
873
+ if (rolledBackTo.TAG === "RecomputedProgress") {
874
+ let match = rolledBackTo._0;
875
+ let newProgressBlockNumber = Primitive_int.min(match.blockNumber, cs.committedProgressBlockNumber);
876
+ let newTotalEventsProcessed = cs.numEventsProcessed - match.eventsProcessed;
872
877
  let safeCheckpointTracking = cs.safeCheckpointTracking;
873
878
  if (safeCheckpointTracking !== undefined) {
874
879
  cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
@@ -881,14 +886,12 @@ function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTarge
881
886
  cs.numEventsProcessed = newTotalEventsProcessed;
882
887
  return rolledBackAddresses;
883
888
  }
884
- if (!isReorgChain) {
885
- return [];
886
- }
887
- let rolledBackAddresses$1 = rollbackTo(rollbackTargetBlockNumber);
888
- cs.transactionStore.rollback(rollbackTargetBlockNumber);
889
- cs.blockStore.rollback(rollbackTargetBlockNumber);
890
- cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
891
- cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, rollbackTargetBlockNumber);
889
+ let forkBlock = rolledBackTo._0;
890
+ let rolledBackAddresses$1 = rollbackTo(forkBlock);
891
+ cs.transactionStore.rollback(forkBlock);
892
+ cs.blockStore.rollback(forkBlock);
893
+ cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, forkBlock);
894
+ cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, forkBlock);
892
895
  return rolledBackAddresses$1;
893
896
  }
894
897
 
@@ -157,10 +157,18 @@ let enterReorgThreshold: t => unit
157
157
  let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
158
158
  let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
159
159
  let markReady: (t, ~readyAt: Date.t) => unit
160
- let rollback: (
161
- t,
162
- ~newProgressBlockNumber: option<int>,
163
- ~eventsProcessedDiff: option<float>,
164
- ~rollbackTargetBlockNumber: int,
165
- ~isReorgChain: bool,
166
- ) => array<AddressRows.key>
160
+ // What a rollback's progress diff says about one chain: the block its surviving
161
+ // checkpoints put it back to, and the events the deleted ones carried away. One
162
+ // record because one diff row yields both — they are never known separately.
163
+ type progressDiff = {blockNumber: int, eventsProcessed: float}
164
+
165
+ // Where a rollback leaves one chain.
166
+ type rolledBackTo =
167
+ | RecomputedProgress(progressDiff)
168
+ // The reorg chain with no checkpoint left at or below the target, so only the
169
+ // fork block says where it goes.
170
+ | ForkBlock(int)
171
+ // A chain this rollback doesn't reach.
172
+ | Untouched
173
+
174
+ let rollback: (t, ~rolledBackTo: rolledBackTo) => array<AddressRows.key>
package/src/Config.res CHANGED
@@ -177,24 +177,6 @@ let publicConfigChainSchema = S.schema(s =>
177
177
  let svmEventDescriptorSchema = S.schema(s =>
178
178
  {
179
179
  "discriminator": s.matches(S.option(S.string)),
180
- "discriminatorByteLen": s.matches(S.int),
181
- // An array of AND-groups OR-ed together: the CLI normalizes both the flat
182
- // and `any_of` YAML shapes to `Vec<Vec<SvmAccountFilterJson>>`.
183
- "accountFilters": s.matches(
184
- S.option(
185
- S.array(
186
- S.array(
187
- S.schema(s =>
188
- {
189
- "position": s.matches(S.int),
190
- "values": s.matches(S.array(S.string)),
191
- }
192
- ),
193
- ),
194
- ),
195
- ),
196
- ),
197
- "isInner": s.matches(S.option(S.bool)),
198
180
  "accounts": s.matches(S.option(S.array(S.string))),
199
181
  "args": s.matches(S.option(S.json(~validate=false))),
200
182
  }
@@ -341,6 +323,7 @@ let getFieldTypeAndSchema = (prop, ~enumConfigsByName: dict<Table.enumConfig<Tab
341
323
 
342
324
  let (fieldType, baseSchema) = switch typ {
343
325
  | "string" => (Table.String, S.string->S.toUnknown)
326
+ | "bytes" => (Table.Bytea, Utils.Schema.bytes->S.toUnknown)
344
327
  | "boolean" => (Table.Boolean, S.bool->S.toUnknown)
345
328
  | "int" => (Table.Int32, S.int->S.toUnknown)
346
329
  | "bigint" => (Table.BigInt({precision: ?prop["precision"]}), Utils.BigInt.schema->S.toUnknown)
@@ -374,13 +357,13 @@ let getFieldTypeAndSchema = (prop, ~enumConfigsByName: dict<Table.enumConfig<Tab
374
357
  | other => JsError.throwWithMessage("Unknown field type in entity config: " ++ other)
375
358
  }
376
359
 
377
- let fieldSchema = if isArray {
378
- S.array(baseSchema)->S.toUnknown
379
- } else {
380
- baseSchema
360
+ let fieldSchema = switch (fieldType, isArray) {
361
+ | (Table.Bytea, true) => Utils.Schema.bytesArray->S.toUnknown
362
+ | (_, true) => S.array(baseSchema)->S.toUnknown
363
+ | (_, false) => baseSchema
381
364
  }
382
365
  let fieldSchema = if isNullable {
383
- S.null(fieldSchema)->S.toUnknown
366
+ Utils.Schema.nullTolerant(fieldSchema)->S.toUnknown
384
367
  } else {
385
368
  fieldSchema
386
369
  }
@@ -753,11 +736,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
753
736
  Utils.magic: _ => {
754
737
  "svm": option<{
755
738
  "discriminator": option<string>,
756
- "discriminatorByteLen": int,
757
- "accountFilters": option<
758
- array<array<{"position": int, "values": array<string>}>>,
759
- >,
760
- "isInner": option<bool>,
761
739
  "accounts": option<array<string>>,
762
740
  "args": option<JSON.t>,
763
741
  }>,
@@ -770,25 +748,11 @@ let fromPublic = (publicConfigJson: JSON.t) => {
770
748
  `SVM instruction ${contractName}.${eventName} is missing the "svm" descriptor in internal config`,
771
749
  )
772
750
  }
773
- let accountFilters =
774
- svm["accountFilters"]
775
- ->Option.getOr([])
776
- ->Array.map(group =>
777
- group->Array.map(
778
- af => {
779
- Internal.position: af["position"],
780
- values: af["values"]->SvmTypes.Pubkey.fromStringsUnsafe,
781
- },
782
- )
783
- )
784
751
  (EventConfigBuilder.buildSvmInstructionEventConfig(
785
752
  ~contractName,
786
753
  ~instructionName=eventName,
787
754
  ~programId,
788
755
  ~discriminator=svm["discriminator"],
789
- ~discriminatorByteLen=svm["discriminatorByteLen"],
790
- ~accountFilters,
791
- ~isInner=svm["isInner"],
792
756
  ~accounts=svm["accounts"]->Option.getOr([]),
793
757
  ~args=svm["args"]->Option.getOr(JSON.Null),
794
758
  ~definedTypes=svmDefinedTypes,
@@ -1070,6 +1034,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
1070
1034
  }
1071
1035
  }
1072
1036
 
1037
+ // With no cross-chain entity, a reorg on one chain can never have changed a row
1038
+ // another chain owns, so its rollback stays isolated to that chain instead of
1039
+ // dragging every sibling back with it. A single chain has no sibling to spare,
1040
+ // and narrowing its rollback would only buy it a predicate that always holds.
1041
+ let isIsolatedMultichain = (config: t) =>
1042
+ config.chainMap->ChainMap.keys->Array.length > 1 &&
1043
+ config.userEntities->Array.every(entityConfig => !entityConfig.crossChain)
1044
+
1073
1045
  // Canonicalize a user-provided address to the configured casing so it matches
1074
1046
  // addresses parsed from config.yaml during routing. HyperSync/RPC data arrives
1075
1047
  // already canonical; only user-land input (simulate srcAddress, contractRegister
@@ -1141,8 +1113,17 @@ let getEventConfig = (config: t, ~contractName, ~eventName, ~chainId: option<Cha
1141
1113
  })
1142
1114
  }
1143
1115
 
1144
- let shouldSaveHistory = (config, ~isInReorgThreshold) =>
1145
- config.shouldSaveFullHistory || (config.shouldRollbackOnReorg && isInReorgThreshold)
1116
+ // A chain that can't be rolled back (maxReorgDepth = 0) has no history to keep,
1117
+ // unless a cross-chain entity lets another chain's rollback reach its rows.
1118
+ let shouldSaveHistory = (config, ~isInReorgThreshold, ~chainId: option<ChainId.t>=?) =>
1119
+ config.shouldSaveFullHistory ||
1120
+ (config.shouldRollbackOnReorg &&
1121
+ isInReorgThreshold &&
1122
+ switch chainId {
1123
+ | Some(chainId) if config->isIsolatedMultichain =>
1124
+ (config.chainMap->ChainMap.get(chainId)).maxReorgDepth > 0
1125
+ | _ => true
1126
+ })
1146
1127
 
1147
1128
  let shouldPruneHistory = (config, ~isInReorgThreshold) =>
1148
1129
  !config.shouldSaveFullHistory && (config.shouldRollbackOnReorg && isInReorgThreshold)
@@ -1362,6 +1343,33 @@ let throwIfIncompatible = (
1362
1343
  }
1363
1344
  }
1364
1345
 
1346
+ let throwIfResumeIncompatible = (
1347
+ ~storedEnvioInfo: option<JSON.t>,
1348
+ ~storedContractMapping: ContractMapping.t,
1349
+ ~envioInfo: JSON.t,
1350
+ ~contractMapping: ContractMapping.t,
1351
+ ~resetCommand: string,
1352
+ ~runCommand: option<string>,
1353
+ ) => {
1354
+ let changedPaths = switch storedEnvioInfo {
1355
+ | None => ["storage was initialized by an older envio version"]
1356
+ | Some(stored) => diffPaths(~stored, ~current=envioInfo)
1357
+ }
1358
+ let changedPaths =
1359
+ storedContractMapping->ContractMapping.isEqual(contractMapping)
1360
+ ? changedPaths
1361
+ : changedPaths->Array.concat(["contracts"])
1362
+ let hasClickhouse = switch envioInfo {
1363
+ | Object(d) =>
1364
+ switch d->Dict.get("storage") {
1365
+ | Some(Object(s)) => s->Dict.get("clickhouse") == Some(Boolean(true))
1366
+ | _ => false
1367
+ }
1368
+ | _ => false
1369
+ }
1370
+ throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
1371
+ }
1372
+
1365
1373
  // The returned value is a pure function of the JSON: it holds only event
1366
1374
  // definitions, never handler/contractRegister/where state (that's layered on
1367
1375
  // separately as `HandlerRegister.registrationsByChainId`). That purity is
@@ -17,6 +17,7 @@ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
17
17
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
18
18
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
19
19
  import * as ContractMapping from "./ContractMapping.res.mjs";
20
+ import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
20
21
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
21
22
  import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
22
23
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -73,12 +74,6 @@ let publicConfigChainSchema = S$RescriptSchema.schema(s => ({
73
74
 
74
75
  let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
75
76
  discriminator: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
76
- discriminatorByteLen: s.m(S$RescriptSchema.int),
77
- accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
78
- position: s.m(S$RescriptSchema.int),
79
- values: s.m(S$RescriptSchema.array(S$RescriptSchema.string))
80
- })))))),
81
- isInner: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
82
77
  accounts: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.string))),
83
78
  args: s.m(S$RescriptSchema.option(S$RescriptSchema.json(false)))
84
79
  }));
@@ -226,6 +221,12 @@ function getFieldTypeAndSchema(prop, enumConfigsByName) {
226
221
  S$RescriptSchema.bool
227
222
  ];
228
223
  break;
224
+ case "bytes" :
225
+ match = [
226
+ "Bytea",
227
+ Utils.Schema.bytes
228
+ ];
229
+ break;
229
230
  case "date" :
230
231
  match = [
231
232
  "Date",
@@ -277,10 +278,20 @@ function getFieldTypeAndSchema(prop, enumConfigsByName) {
277
278
  match = Stdlib_JsError.throwWithMessage("Unknown field type in entity config: " + typ);
278
279
  }
279
280
  let baseSchema = match[1];
280
- let fieldSchema = isArray ? S$RescriptSchema.array(baseSchema) : baseSchema;
281
- let fieldSchema$1 = isNullable ? S$RescriptSchema.$$null(fieldSchema) : fieldSchema;
281
+ let fieldType = match[0];
282
+ let fieldSchema;
283
+ let exit = 0;
284
+ if (typeof fieldType !== "object" && fieldType === "Bytea" && isArray) {
285
+ fieldSchema = Utils.Schema.bytesArray;
286
+ } else {
287
+ exit = 1;
288
+ }
289
+ if (exit === 1) {
290
+ fieldSchema = isArray ? S$RescriptSchema.array(baseSchema) : baseSchema;
291
+ }
292
+ let fieldSchema$1 = isNullable ? Utils.Schema.nullTolerant(fieldSchema) : fieldSchema;
282
293
  return [
283
- match[0],
294
+ fieldType,
284
295
  fieldSchema$1,
285
296
  isNullable,
286
297
  isArray,
@@ -527,11 +538,7 @@ function fromPublic(publicConfigJson) {
527
538
  ) : addresses[0];
528
539
  let s = eventItem.svm;
529
540
  let svm = s !== undefined ? Primitive_option.valFromOption(s) : Stdlib_JsError.throwWithMessage(`SVM instruction ` + contractName + `.` + eventName + ` is missing the "svm" descriptor in internal config`);
530
- let accountFilters = Stdlib_Option.getOr(svm.accountFilters, []).map(group => group.map(af => ({
531
- position: af.position,
532
- values: af.values
533
- })));
534
- return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, accountFilters, svm.isInner, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
541
+ return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
535
542
  }
536
543
  });
537
544
  } else {
@@ -726,6 +733,14 @@ function fromPublic(publicConfigJson) {
726
733
  };
727
734
  }
728
735
 
736
+ function isIsolatedMultichain(config) {
737
+ if (ChainMap.keys(config.chainMap).length > 1) {
738
+ return config.userEntities.every(entityConfig => !entityConfig.crossChain);
739
+ } else {
740
+ return false;
741
+ }
742
+ }
743
+
729
744
  function normalizeUserAddress(config, address) {
730
745
  let match = config.ecosystem.name;
731
746
  switch (match) {
@@ -789,11 +804,15 @@ function getEventConfig(config, contractName, eventName, chainId) {
789
804
  });
790
805
  }
791
806
 
792
- function shouldSaveHistory(config, isInReorgThreshold) {
807
+ function shouldSaveHistory(config, isInReorgThreshold, chainId) {
793
808
  if (config.shouldSaveFullHistory) {
794
809
  return true;
795
- } else if (config.shouldRollbackOnReorg) {
796
- return isInReorgThreshold;
810
+ } else if (config.shouldRollbackOnReorg && isInReorgThreshold) {
811
+ if (chainId !== undefined && isIsolatedMultichain(config)) {
812
+ return ChainMap.get(config.chainMap, Primitive_option.valFromOption(chainId)).maxReorgDepth > 0;
813
+ } else {
814
+ return true;
815
+ }
797
816
  } else {
798
817
  return false;
799
818
  }
@@ -1031,6 +1050,19 @@ function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhou
1031
1050
  Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n 1. ` + padTo(option1, col) + `# resume indexing where it left off\n 2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
1032
1051
  }
1033
1052
 
1053
+ function throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand) {
1054
+ let changedPaths = storedEnvioInfo !== undefined ? diffPaths(storedEnvioInfo, envioInfo) : ["storage was initialized by an older envio version"];
1055
+ let changedPaths$1 = ContractMapping.isEqual(storedContractMapping, contractMapping) ? changedPaths : changedPaths.concat(["contracts"]);
1056
+ let hasClickhouse;
1057
+ if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
1058
+ let match = envioInfo["storage"];
1059
+ hasClickhouse = typeof match === "object" && match !== null && !Array.isArray(match) ? Primitive_object.equal(match["clickhouse"], true) : false;
1060
+ } else {
1061
+ hasClickhouse = false;
1062
+ }
1063
+ throwIfIncompatible(changedPaths$1, resetCommand, runCommand, hasClickhouse);
1064
+ }
1065
+
1034
1066
  function load() {
1035
1067
  let c = cached.contents;
1036
1068
  if (c !== undefined) {
@@ -1075,6 +1107,7 @@ export {
1075
1107
  publicConfigSchema,
1076
1108
  contractMappingOf,
1077
1109
  fromPublic,
1110
+ isIsolatedMultichain,
1078
1111
  normalizeUserAddress,
1079
1112
  normalizeSimulateAddress,
1080
1113
  getEventConfig,
@@ -1087,6 +1120,7 @@ export {
1087
1120
  canonicalJson,
1088
1121
  diffPaths,
1089
1122
  throwIfIncompatible,
1123
+ throwIfResumeIncompatible,
1090
1124
  load,
1091
1125
  getPgUserEntities,
1092
1126
  }