envio 3.9.0 → 3.10.0-subgraph

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/evm.schema.json +26 -0
  2. package/fuel.schema.json +26 -0
  3. package/index.d.ts +41 -8
  4. package/package.json +6 -7
  5. package/src/Batch.res +9 -0
  6. package/src/ChainState.res +35 -39
  7. package/src/ChainState.res.mjs +14 -11
  8. package/src/ChainState.resi +15 -7
  9. package/src/Config.res +61 -44
  10. package/src/Config.res.mjs +56 -19
  11. package/src/Core.res +19 -0
  12. package/src/Core.res.mjs +11 -0
  13. package/src/CrossChainState.res +15 -23
  14. package/src/CrossChainState.res.mjs +10 -18
  15. package/src/CrossChainState.resi +4 -1
  16. package/src/Ecosystem.res +7 -2
  17. package/src/Envio.res +6 -4
  18. package/src/EventConfigBuilder.res +169 -26
  19. package/src/EventConfigBuilder.res.mjs +124 -15
  20. package/src/EventProcessing.res +6 -6
  21. package/src/EventProcessing.res.mjs +8 -6
  22. package/src/HandlerLoader.res +20 -8
  23. package/src/HandlerLoader.res.mjs +11 -2
  24. package/src/HandlerRegister.res +21 -4
  25. package/src/HandlerRegister.res.mjs +24 -5
  26. package/src/InMemoryStore.res +9 -7
  27. package/src/InMemoryStore.res.mjs +3 -3
  28. package/src/IndexerState.res +43 -6
  29. package/src/IndexerState.res.mjs +36 -7
  30. package/src/IndexerState.resi +4 -4
  31. package/src/Internal.res +16 -17
  32. package/src/Metrics.res +72 -6
  33. package/src/Metrics.res.mjs +30 -2
  34. package/src/Persistence.res +36 -30
  35. package/src/Persistence.res.mjs +1 -20
  36. package/src/PgStorage.res +104 -56
  37. package/src/PgStorage.res.mjs +57 -35
  38. package/src/PruneStaleHistory.res +85 -52
  39. package/src/PruneStaleHistory.res.mjs +63 -37
  40. package/src/Rollback.res +68 -45
  41. package/src/Rollback.res.mjs +38 -19
  42. package/src/RollbackCommit.res +7 -9
  43. package/src/RollbackCommit.res.mjs +5 -6
  44. package/src/SafeCheckpointTracking.res +11 -11
  45. package/src/SafeCheckpointTracking.res.mjs +7 -6
  46. package/src/SimulateItems.res +33 -18
  47. package/src/SimulateItems.res.mjs +26 -22
  48. package/src/Sink.res +38 -26
  49. package/src/Sink.res.mjs +21 -16
  50. package/src/TestIndexer.res +5 -5
  51. package/src/TestIndexer.res.mjs +1 -1
  52. package/src/UserContext.res +358 -51
  53. package/src/UserContext.res.mjs +271 -35
  54. package/src/Utils.res +170 -0
  55. package/src/Utils.res.mjs +153 -0
  56. package/src/Writing.res +2 -2
  57. package/src/Writing.res.mjs +1 -1
  58. package/src/bindings/ClickHouse.res +366 -698
  59. package/src/bindings/ClickHouse.res.mjs +371 -421
  60. package/src/bindings/ClickHouseSink.res +337 -0
  61. package/src/bindings/ClickHouseSink.res.mjs +246 -0
  62. package/src/bindings/EventSource.res +8 -2
  63. package/src/bindings/NodeJs.res +6 -0
  64. package/src/bindings/Vitest.res +23 -0
  65. package/src/bindings/Vitest.res.mjs +3 -0
  66. package/src/bindings/WebSocket.res +9 -10
  67. package/src/db/CheckpointBounds.res +53 -0
  68. package/src/db/CheckpointBounds.res.mjs +44 -0
  69. package/src/db/EntityFilter.res +36 -17
  70. package/src/db/EntityFilter.res.mjs +25 -14
  71. package/src/db/EntityHistory.res +47 -22
  72. package/src/db/EntityHistory.res.mjs +19 -12
  73. package/src/db/InternalTable.res +84 -35
  74. package/src/db/InternalTable.res.mjs +56 -23
  75. package/src/db/RollbackFloors.res +55 -0
  76. package/src/db/RollbackFloors.res.mjs +92 -0
  77. package/src/db/Table.res +22 -4
  78. package/src/db/Table.res.mjs +30 -9
  79. package/src/sources/Evm.res +2 -0
  80. package/src/sources/Evm.res.mjs +2 -0
  81. package/src/sources/EvmHyperSyncSource.res +3 -3
  82. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  83. package/src/sources/EvmRpcWs.res +124 -0
  84. package/src/sources/EvmRpcWs.res.mjs +117 -0
  85. package/src/sources/Fuel.res +2 -0
  86. package/src/sources/Fuel.res.mjs +2 -0
  87. package/src/sources/FuelHyperSyncSource.res +1 -1
  88. package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
  89. package/src/sources/HeightFeed.res +568 -0
  90. package/src/sources/HeightFeed.res.mjs +452 -0
  91. package/src/sources/HeightStream.res +257 -0
  92. package/src/sources/HeightStream.res.mjs +194 -0
  93. package/src/sources/HyperSync.res +5 -0
  94. package/src/sources/HyperSync.res.mjs +3 -0
  95. package/src/sources/HyperSync.resi +4 -0
  96. package/src/sources/HyperSyncSSE.res +49 -0
  97. package/src/sources/HyperSyncSSE.res.mjs +79 -0
  98. package/src/sources/RpcSource.res +1 -1
  99. package/src/sources/RpcSource.res.mjs +2 -2
  100. package/src/sources/Source.res +54 -1
  101. package/src/sources/Source.res.mjs +25 -0
  102. package/src/sources/SourceManager.res +220 -227
  103. package/src/sources/SourceManager.res.mjs +139 -164
  104. package/src/sources/SourceManager.resi +8 -0
  105. package/src/sources/Svm.res +2 -0
  106. package/src/sources/Svm.res.mjs +2 -0
  107. package/src/sources/SvmHyperSyncClient.res +70 -127
  108. package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
  109. package/src/sources/SvmHyperSyncSource.res +18 -15
  110. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  111. package/src/subgraph/blocks.ts +176 -0
  112. package/src/subgraph/calls.ts +217 -0
  113. package/src/subgraph/conformance.ts +102 -0
  114. package/src/subgraph/division.ts +133 -0
  115. package/src/subgraph/errors.ts +90 -0
  116. package/src/subgraph/graph-ts-types/VERSION +2 -0
  117. package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
  118. package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
  119. package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
  120. package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
  121. package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
  122. package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
  123. package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
  124. package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
  125. package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
  126. package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
  127. package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
  128. package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
  129. package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
  130. package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
  131. package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
  132. package/src/subgraph/graph-ts-types/index.d.ts +102 -0
  133. package/src/subgraph/graph-ts.ts +1948 -0
  134. package/src/subgraph/hosts.ts +148 -0
  135. package/src/subgraph/runtime.ts +863 -0
  136. package/src/subgraph/scope.ts +66 -0
  137. package/svm.schema.json +6 -81
  138. package/src/MemoryStorage.res +0 -729
  139. package/src/MemoryStorage.res.mjs +0 -586
  140. package/src/sources/HyperSyncHeightStream.res +0 -129
  141. package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
  142. package/src/sources/RpcWebSocketHeightStream.res +0 -175
  143. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/src/PgStorage.res CHANGED
@@ -377,7 +377,7 @@ let makeLoadQuery = (~pgSchema, ~tableName, ~condition) => {
377
377
  let rec makeFilterCondition = (
378
378
  ~filter: EntityFilter.t,
379
379
  ~table: Table.table,
380
- ~params: array<JSON.t>,
380
+ ~params: array<unknown>,
381
381
  ) => {
382
382
  // Filters reference fields by API name, while the SQL references columns
383
383
  // by their possibly renamed db names.
@@ -398,7 +398,7 @@ let rec makeFilterCondition = (
398
398
  ~fieldValue: unknown,
399
399
  ~isArray,
400
400
  ) => {
401
- let param = try fieldValue->S.reverseConvertToJsonOrThrow(
401
+ let param = try fieldValue->S.reverseConvertOrThrow(
402
402
  isArray ? queryField.arrayFieldSchema : queryField.fieldSchema,
403
403
  ) catch {
404
404
  | exn =>
@@ -438,8 +438,8 @@ let rec makeFilterCondition = (
438
438
  // `LoadLayer.scopeFilter` is what puts this filter here, and the value is
439
439
  // range-checked to a non-negative safe integer, so it can carry nothing but
440
440
  // digits.
441
- | Eq({fieldName, fieldValue}) if (getQueryFieldOrThrow(fieldName)).isChainId =>
442
- `"${(getQueryFieldOrThrow(fieldName)).pgDbFieldName}" = ${fieldValue
441
+ | Eq({fieldName, fieldValue}) if getQueryFieldOrThrow(fieldName).isChainId =>
442
+ `"${getQueryFieldOrThrow(fieldName).pgDbFieldName}" = ${fieldValue
443
443
  ->ChainId.normalizeOrThrow
444
444
  ->ChainId.toString}`
445
445
  | Eq({fieldName, fieldValue}) => scalarCondition(~fieldName, ~fieldValue, ~op="=")
@@ -587,7 +587,7 @@ let makeTableBatchSetQuery = (
587
587
  ~itemSchema: S.t<'item>,
588
588
  ~chainIdMode: ChainId.mode=Int32,
589
589
  ) => {
590
- let {dbSchema, hasArrayField} =
590
+ let {dbSchema, hasArrayField, byteaColumnIndexes} =
591
591
  table->Table.toSqlParams(~schema=itemSchema, ~pgSchema, ~chainIdMode)
592
592
 
593
593
  // Should move this to a better place
@@ -614,7 +614,22 @@ let makeTableBatchSetQuery = (
614
614
  {
615
615
  "query": makeInsertUnnestSetQuery(~pgSchema, ~table, ~itemSchema, ~isRawEvents, ~chainIdMode),
616
616
  "convertOrThrow": S.compile(
617
- S.unnest(dbSchema),
617
+ S.unnest(dbSchema)->S.preprocess(_ => {
618
+ serializer: columns => {
619
+ let columns = columns->(Utils.magic: unknown => array<unknown>)
620
+ byteaColumnIndexes->Array.forEach(index =>
621
+ columns->Array.setUnsafe(
622
+ index,
623
+ columns
624
+ ->Array.getUnsafe(index)
625
+ ->(Utils.magic: unknown => array<unknown>)
626
+ ->Utils.Bytes.toPgArrayLiteral
627
+ ->(Utils.magic: string => unknown),
628
+ )
629
+ )
630
+ columns->(Utils.magic: array<unknown> => unknown)
631
+ },
632
+ }),
618
633
  ~input=Value,
619
634
  ~output=Unknown,
620
635
  ~mode=Sync,
@@ -1106,6 +1121,8 @@ let rec writeBatch = async (
1106
1121
  | Internal.CrossChain => None
1107
1122
  | Chain(chainId) => Some(chainId)
1108
1123
  }
1124
+ let shouldSaveHistory =
1125
+ config->Config.shouldSaveHistory(~isInReorgThreshold, ~chainId=?scopeChainId)
1109
1126
  let changes = switch (entityConfig.table->Table.getChainIdField, scopeChainId) {
1110
1127
  | (Some(field), Some(chainId)) =>
1111
1128
  changes->Array.map(change =>
@@ -1296,7 +1313,7 @@ let rec writeBatch = async (
1296
1313
  //valid event identifier, where all rows created after this eventIdentifier should
1297
1314
  //be deleted
1298
1315
  let rollbackTables = switch rollback {
1299
- | Some({targetCheckpointId: rollbackTargetCheckpointId, rolledBackAddresses}) =>
1316
+ | Some({floors, rolledBackAddresses, progressedChains}) =>
1300
1317
  Some(
1301
1318
  sql => {
1302
1319
  // Postgres owns history tables only for Postgres-backed entities;
@@ -1309,15 +1326,24 @@ let rec writeBatch = async (
1309
1326
  ~pgSchema,
1310
1327
  ~entityName=entityConfig.name,
1311
1328
  ~entityIndex=entityConfig.index,
1312
- ~rollbackTargetCheckpointId,
1329
+ ~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
1330
+ ~floors,
1313
1331
  )
1314
1332
  })
1315
1333
  promises
1316
- ->Array.push(
1317
- sql->InternalTable.Checkpoints.rollback(~pgSchema, ~rollbackTargetCheckpointId),
1318
- )
1334
+ ->Array.push(sql->InternalTable.Checkpoints.rollback(~pgSchema, ~floors))
1319
1335
  ->ignore
1320
1336
 
1337
+ // Runs before the batch's own progress write below, so a chain the
1338
+ // batch also progressed keeps the batch's later value.
1339
+ if progressedChains->Utils.Array.notEmpty {
1340
+ promises
1341
+ ->Array.push(
1342
+ sql->InternalTable.Chains.setProgressedChains(~pgSchema, ~progressedChains),
1343
+ )
1344
+ ->ignore
1345
+ }
1346
+
1321
1347
  // Addresses are insert-only, so undoing their registrations is a
1322
1348
  // delete rather than a history replay. It runs before the batch's own
1323
1349
  // inserts in the same transaction, so a re-registered address lands
@@ -1476,7 +1502,11 @@ let rollbackKeyColumns = (entityConfig: Internal.entityConfig) =>
1476
1502
  | None => [Table.idFieldName]
1477
1503
  }
1478
1504
 
1479
- let makeGetRollbackPreTargetRowsQuery = (~entityConfig: Internal.entityConfig, ~pgSchema) => {
1505
+ let makeGetRollbackPreTargetRowsQuery = (
1506
+ ~entityConfig: Internal.entityConfig,
1507
+ ~pgSchema,
1508
+ ~floors: RollbackFloors.t,
1509
+ ) => {
1480
1510
  let dataFieldNames = entityConfig.table.fields->Array.filterMap(fieldOrDerived =>
1481
1511
  switch fieldOrDerived {
1482
1512
  | Field(field) => field->Table.getPgDbFieldName->Some
@@ -1484,56 +1514,70 @@ let makeGetRollbackPreTargetRowsQuery = (~entityConfig: Internal.entityConfig, ~
1484
1514
  }
1485
1515
  )
1486
1516
 
1487
- let dataFieldsCommaSeparated =
1488
- dataFieldNames->Array.map(name => `"${name}"`)->Array.joinUnsafe(", ")
1489
-
1490
1517
  let historyTableName = EntityHistory.historyTableName(
1491
1518
  ~entityName=entityConfig.name,
1492
1519
  ~entityIndex=entityConfig.index,
1493
1520
  )
1521
+ // Every column is qualified: the per-chain bounds relation joined in has a
1522
+ // `chain_id` of its own, which a snake_case chain column shares.
1523
+ let tableRef = `"${historyTableName}"`
1524
+ let dataFieldsCommaSeparated =
1525
+ dataFieldNames->Array.map(name => `${tableRef}."${name}"`)->Array.joinUnsafe(", ")
1494
1526
 
1495
1527
  // A per-chain entity's rows are only comparable within a chain, so the row's
1496
1528
  // identity here is (id, chain id) rather than the id alone.
1497
1529
  let keyColumns = rollbackKeyColumns(entityConfig)
1498
- let keyColumnsCommaSeparated = keyColumns->Array.map(c => `"${c}"`)->Array.joinUnsafe(", ")
1530
+ let keyColumnsCommaSeparated =
1531
+ keyColumns->Array.map(c => `${tableRef}."${c}"`)->Array.joinUnsafe(", ")
1499
1532
  let keyMatch =
1500
- keyColumns
1501
- ->Array.map(c => `h."${c}" = "${historyTableName}"."${c}"`)
1502
- ->Array.joinUnsafe(" AND ")
1533
+ keyColumns->Array.map(c => `h."${c}" = ${tableRef}."${c}"`)->Array.joinUnsafe(" AND ")
1534
+ let bounds =
1535
+ floors.floors->CheckpointBounds.sql(
1536
+ ~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
1537
+ ~tableRef,
1538
+ )
1503
1539
 
1504
- `SELECT DISTINCT ON (${keyColumnsCommaSeparated}) ${dataFieldsCommaSeparated}, "${EntityHistory.changeFieldName}"
1505
- FROM "${pgSchema}"."${historyTableName}"
1506
- WHERE "${EntityHistory.checkpointIdFieldName}" <= $1
1540
+ `SELECT DISTINCT ON (${keyColumnsCommaSeparated}) ${dataFieldsCommaSeparated}, ${tableRef}."${EntityHistory.changeFieldName}"
1541
+ FROM "${pgSchema}"."${historyTableName}"${bounds.join}
1542
+ WHERE ${tableRef}."${EntityHistory.checkpointIdFieldName}" <= ${bounds.checkpointId}
1507
1543
  AND EXISTS (
1508
1544
  SELECT 1
1509
1545
  FROM "${pgSchema}"."${historyTableName}" h
1510
1546
  WHERE ${keyMatch}
1511
- AND h."${EntityHistory.checkpointIdFieldName}" > $1
1547
+ AND h."${EntityHistory.checkpointIdFieldName}" > ${bounds.checkpointId}
1512
1548
  )
1513
- ORDER BY ${keyColumnsCommaSeparated}, "${EntityHistory.checkpointIdFieldName}" DESC`
1549
+ ORDER BY ${keyColumnsCommaSeparated}, ${tableRef}."${EntityHistory.checkpointIdFieldName}" DESC`
1514
1550
  }
1515
1551
 
1516
1552
  // Returns entity IDs that were created after the rollback target and have no history before it.
1517
1553
  // DELETE rows at or before the target are returned by the restore query and classified in ReScript.
1518
- let makeGetRollbackRemovedIdsQuery = (~entityConfig: Internal.entityConfig, ~pgSchema) => {
1554
+ let makeGetRollbackRemovedIdsQuery = (
1555
+ ~entityConfig: Internal.entityConfig,
1556
+ ~pgSchema,
1557
+ ~floors: RollbackFloors.t,
1558
+ ) => {
1519
1559
  let historyTableName = EntityHistory.historyTableName(
1520
1560
  ~entityName=entityConfig.name,
1521
1561
  ~entityIndex=entityConfig.index,
1522
1562
  )
1563
+ let tableRef = `"${historyTableName}"`
1523
1564
  let keyColumns = rollbackKeyColumns(entityConfig)
1524
1565
  let keyMatch =
1525
- keyColumns
1526
- ->Array.map(c => `h."${c}" = "${historyTableName}"."${c}"`)
1527
- ->Array.joinUnsafe(" AND ")
1566
+ keyColumns->Array.map(c => `h."${c}" = ${tableRef}."${c}"`)->Array.joinUnsafe(" AND ")
1567
+ let bounds =
1568
+ floors.floors->CheckpointBounds.sql(
1569
+ ~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
1570
+ ~tableRef,
1571
+ )
1528
1572
 
1529
- `SELECT DISTINCT ${keyColumns->Array.map(c => `"${c}"`)->Array.joinUnsafe(", ")}
1530
- FROM "${pgSchema}"."${historyTableName}"
1531
- WHERE "${EntityHistory.checkpointIdFieldName}" > $1
1573
+ `SELECT DISTINCT ${keyColumns->Array.map(c => `${tableRef}."${c}"`)->Array.joinUnsafe(", ")}
1574
+ FROM "${pgSchema}"."${historyTableName}"${bounds.join}
1575
+ WHERE ${tableRef}."${EntityHistory.checkpointIdFieldName}" > ${bounds.checkpointId}
1532
1576
  AND NOT EXISTS (
1533
1577
  SELECT 1
1534
1578
  FROM "${pgSchema}"."${historyTableName}" h
1535
1579
  WHERE ${keyMatch}
1536
- AND h."${EntityHistory.checkpointIdFieldName}" <= $1
1580
+ AND h."${EntityHistory.checkpointIdFieldName}" <= ${bounds.checkpointId}
1537
1581
  )`
1538
1582
  }
1539
1583
 
@@ -1774,11 +1818,9 @@ let make = (
1774
1818
  ~contractMapping,
1775
1819
  ~envioInfo,
1776
1820
  ): Persistence.initialState => {
1777
- // Per-entity storage routing: PG owns tables only for entities that
1778
- // opted into Postgres; the sink mirrors only those that opted into
1779
- // ClickHouse.
1821
+ // PG owns tables only for entities that opted into Postgres; the sink
1822
+ // picks its own out of the full list.
1780
1823
  let pgEntities = entities->Array.filter((e: Internal.entityConfig) => e.storage.postgres)
1781
- let chEntities = entities->Array.filter((e: Internal.entityConfig) => e.storage.clickhouse)
1782
1824
 
1783
1825
  let schemaTableNames: array<schemaTableName> = await sql->Postgres.unsafe(
1784
1826
  makeSchemaTableNamesQuery(~pgSchema),
@@ -1807,7 +1849,7 @@ let make = (
1807
1849
 
1808
1850
  // Call sink.initialize before executing PG queries
1809
1851
  switch sink {
1810
- | Some(sink) => await sink.initialize(~chainConfigs, ~entities=chEntities, ~enums)
1852
+ | Some(sink) => await sink.initialize(~entities)
1811
1853
  | None => ()
1812
1854
  }
1813
1855
 
@@ -2271,8 +2313,14 @@ let make = (
2271
2313
  }
2272
2314
  }
2273
2315
 
2274
- let resumeInitialState = async (): Persistence.initialState => {
2275
- let (cache, chains, checkpointIdResult, reorgCheckpoints, (envioInfo, contractMapping)) = await Promise.all5((
2316
+ let resumeInitialState = async (~entities, ~throwIfIncompatible): Persistence.initialState => {
2317
+ let (
2318
+ cache,
2319
+ chains,
2320
+ checkpointIdResult,
2321
+ reorgCheckpoints,
2322
+ (storedEnvioInfo, storedContractMapping),
2323
+ ) = await Promise.all5((
2276
2324
  restoreEffectCache(~withUpload=false),
2277
2325
  InternalTable.Chains.getInitialState(
2278
2326
  sql,
@@ -2320,6 +2368,8 @@ let make = (
2320
2368
  ),
2321
2369
  ))
2322
2370
 
2371
+ throwIfIncompatible(~storedEnvioInfo, ~storedContractMapping)
2372
+
2323
2373
  await reloadIndexCatalog()
2324
2374
 
2325
2375
  let checkpointId = (checkpointIdResult->Array.getUnsafe(0))["id"]->BigInt.fromStringOrThrow
@@ -2334,7 +2384,7 @@ let make = (
2334
2384
 
2335
2385
  // Resume sink if present - needed to rollback any reorg changes
2336
2386
  switch sink {
2337
- | Some(sink) => await sink.resume(~checkpointId)
2387
+ | Some(sink) => await sink.resume(~checkpointId, ~chains, ~entities)
2338
2388
  | None => ()
2339
2389
  }
2340
2390
 
@@ -2344,8 +2394,8 @@ let make = (
2344
2394
  cache,
2345
2395
  chains,
2346
2396
  checkpointId,
2347
- contractMapping,
2348
- envioInfo,
2397
+ contractMapping: storedContractMapping,
2398
+ envioInfo: storedEnvioInfo,
2349
2399
  }
2350
2400
  }
2351
2401
 
@@ -2359,17 +2409,17 @@ let make = (
2359
2409
  %raw(`undefined`)
2360
2410
  )
2361
2411
 
2362
- let pruneStaleCheckpoints = (~safeCheckpointId) =>
2363
- InternalTable.Checkpoints.pruneStaleCheckpoints(sql, ~pgSchema, ~safeCheckpointId)
2412
+ let pruneStaleCheckpoints = (~safeCheckpoints) =>
2413
+ InternalTable.Checkpoints.pruneStaleCheckpoints(sql, ~pgSchema, ~safeCheckpoints)
2364
2414
 
2365
- let pruneStaleEntityHistory = (~entityName, ~entityIndex, ~chainIdColumn, ~safeCheckpointId) =>
2415
+ let pruneStaleEntityHistory = (~entityName, ~entityIndex, ~chainIdColumn, ~safeCheckpoints) =>
2366
2416
  EntityHistory.pruneStaleEntityHistory(
2367
2417
  sql,
2368
2418
  ~pgSchema,
2369
2419
  ~entityName,
2370
2420
  ~entityIndex,
2371
2421
  ~chainIdColumn,
2372
- ~safeCheckpointId,
2422
+ ~safeCheckpoints,
2373
2423
  )
2374
2424
 
2375
2425
  let getRollbackTargetCheckpoint = (~reorgChainId, ~lastKnownValidBlockNumber) =>
@@ -2380,26 +2430,24 @@ let make = (
2380
2430
  ~lastKnownValidBlockNumber,
2381
2431
  )
2382
2432
 
2383
- let getRollbackProgressDiff = (~rollbackTargetCheckpointId) =>
2384
- InternalTable.Checkpoints.getRollbackProgressDiff(sql, ~pgSchema, ~rollbackTargetCheckpointId)
2433
+ let getRollbackProgressDiff = (~floors) =>
2434
+ InternalTable.Checkpoints.getRollbackProgressDiff(sql, ~pgSchema, ~floors)
2385
2435
 
2386
- let getRollbackData = async (
2387
- ~entityConfig: Internal.entityConfig,
2388
- ~rollbackTargetCheckpointId,
2389
- ) => {
2436
+ let getRollbackData = async (~entityConfig: Internal.entityConfig, ~floors: RollbackFloors.t) => {
2437
+ let params = floors.floors->CheckpointBounds.params
2390
2438
  let (removedIdRows, rollbackRows) = await Promise.all2((
2391
2439
  // Get IDs of entities that should be deleted (created after rollback target with no prior history)
2392
2440
  sql
2393
2441
  ->Postgres.preparedUnsafe(
2394
- makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema),
2395
- [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
2442
+ makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema, ~floors),
2443
+ params,
2396
2444
  )
2397
2445
  ->(Utils.magic: promise<unknown> => promise<array<unknown>>),
2398
2446
  // Get the latest pre-target row, including its SET or DELETE action.
2399
2447
  sql
2400
2448
  ->Postgres.preparedUnsafe(
2401
- makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema),
2402
- [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
2449
+ makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema, ~floors),
2450
+ params,
2403
2451
  )
2404
2452
  ->(Utils.magic: promise<unknown> => promise<array<unknown>>),
2405
2453
  ))
@@ -29,6 +29,7 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
29
29
  import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
30
30
  import * as ContractMapping from "./ContractMapping.res.mjs";
31
31
  import * as IndexDefinition from "./db/IndexDefinition.res.mjs";
32
+ import * as CheckpointBounds from "./db/CheckpointBounds.res.mjs";
32
33
  import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
33
34
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
34
35
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -207,7 +208,7 @@ function getEntityHistory(entityConfig) {
207
208
  checkpointIdField,
208
209
  actionField
209
210
  ]), undefined);
210
- let setChangeSchema = EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), getRowSchema(entityConfig));
211
+ let setChangeSchema = EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), undefined, getRowSchema(entityConfig));
211
212
  let cache_setChangeSchemaRows = S$RescriptSchema.array(setChangeSchema);
212
213
  let cache$1 = {
213
214
  table: table,
@@ -296,7 +297,7 @@ function makeFilterCondition(filter, table, params) {
296
297
  let serializeParamOrThrow = (queryField, fieldName, fieldValue, isArray) => {
297
298
  let param;
298
299
  try {
299
- param = S$RescriptSchema.reverseConvertToJsonOrThrow(fieldValue, isArray ? queryField.arrayFieldSchema : queryField.fieldSchema);
300
+ param = S$RescriptSchema.reverseConvertOrThrow(fieldValue, isArray ? queryField.arrayFieldSchema : queryField.fieldSchema);
300
301
  } catch (raw_exn) {
301
302
  let exn = Primitive_exceptions.internalToException(raw_exn);
302
303
  throw {
@@ -410,12 +411,20 @@ VALUES` + placeholders + (
410
411
  function makeTableBatchSetQuery(pgSchema, table, itemSchema, chainIdModeOpt) {
411
412
  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
412
413
  let match = Table.toSqlParams(table, itemSchema, pgSchema, chainIdMode);
414
+ let byteaColumnIndexes = match.byteaColumnIndexes;
413
415
  let isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName;
414
416
  let isHistoryUpdate = table.tableName.startsWith(EntityHistory.historyTablePrefix);
415
417
  if ((isRawEvents || !match.hasArrayField) && !isHistoryUpdate) {
416
418
  return {
417
419
  query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents, chainIdMode),
418
- convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
420
+ convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(match.dbSchema), param => ({
421
+ s: columns => {
422
+ byteaColumnIndexes.forEach(index => {
423
+ columns[index] = Utils.Bytes.toPgArrayLiteral(columns[index]);
424
+ });
425
+ return columns;
426
+ }
427
+ })), "Output", "Input", "Sync", false),
419
428
  isInsertValues: false
420
429
  };
421
430
  } else {
@@ -681,7 +690,7 @@ function executeSet(sql, items, dbFunction) {
681
690
  async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, escapeTables) {
682
691
  try {
683
692
  let chainIdMode = config.chainIdMode;
684
- let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
693
+ let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold, undefined);
685
694
  let specificError = {
686
695
  contents: undefined
687
696
  };
@@ -723,6 +732,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
723
732
  let idsToDelete = [];
724
733
  let scopeChainId;
725
734
  scopeChainId = scope === "crossChain" ? undefined : Primitive_option.some(scope);
735
+ let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold, scopeChainId);
726
736
  let match = Table.getChainIdField(entityConfig.table);
727
737
  let changes$1;
728
738
  if (match !== undefined && scopeChainId !== undefined) {
@@ -835,11 +845,15 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
835
845
  });
836
846
  let rollbackTables;
837
847
  if (rollback !== undefined) {
848
+ let progressedChains = rollback.progressedChains;
838
849
  let rolledBackAddresses = rollback.rolledBackAddresses;
839
- let rollbackTargetCheckpointId = rollback.targetCheckpointId;
850
+ let floors = rollback.floors;
840
851
  rollbackTables = sql => {
841
- let promises = allEntities.filter(entityConfig => entityConfig.storage.postgres).map(entityConfig => EntityHistory.rollback(sql, pgSchema, entityConfig.name, entityConfig.index, rollbackTargetCheckpointId));
842
- promises.push(InternalTable.Checkpoints.rollback(sql, pgSchema, rollbackTargetCheckpointId));
852
+ let promises = allEntities.filter(entityConfig => entityConfig.storage.postgres).map(entityConfig => EntityHistory.rollback(sql, pgSchema, entityConfig.name, entityConfig.index, Table.getPgChainIdColumn(entityConfig.table), floors));
853
+ promises.push(InternalTable.Checkpoints.rollback(sql, pgSchema, floors));
854
+ if (Utils.$$Array.notEmpty(progressedChains)) {
855
+ promises.push(InternalTable.Chains.setProgressedChains(sql, pgSchema, progressedChains));
856
+ }
843
857
  if (Utils.$$Array.notEmpty(rolledBackAddresses)) {
844
858
  promises.push(InternalTable.EnvioAddresses.$$delete(sql, pgSchema, rolledBackAddresses, chainIdMode));
845
859
  }
@@ -917,41 +931,45 @@ function rollbackKeyColumns(entityConfig) {
917
931
  }
918
932
  }
919
933
 
920
- function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema) {
934
+ function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema, floors) {
921
935
  let dataFieldNames = Stdlib_Array.filterMap(entityConfig.table.fields, fieldOrDerived => {
922
936
  if (fieldOrDerived.TAG === "Field") {
923
937
  return Table.getPgDbFieldName(fieldOrDerived._0);
924
938
  }
925
939
  });
926
- let dataFieldsCommaSeparated = dataFieldNames.map(name => `"` + name + `"`).join(", ");
927
940
  let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
941
+ let tableRef = `"` + historyTableName + `"`;
942
+ let dataFieldsCommaSeparated = dataFieldNames.map(name => tableRef + `."` + name + `"`).join(", ");
928
943
  let keyColumns = rollbackKeyColumns(entityConfig);
929
- let keyColumnsCommaSeparated = keyColumns.map(c => `"` + c + `"`).join(", ");
930
- let keyMatch = keyColumns.map(c => `h."` + c + `" = "` + historyTableName + `"."` + c + `"`).join(" AND ");
931
- return `SELECT DISTINCT ON (` + keyColumnsCommaSeparated + `) ` + dataFieldsCommaSeparated + `, "` + EntityHistory.changeFieldName + `"
932
- FROM "` + pgSchema + `"."` + historyTableName + `"
933
- WHERE "` + EntityHistory.checkpointIdFieldName + `" <= $1
944
+ let keyColumnsCommaSeparated = keyColumns.map(c => tableRef + `."` + c + `"`).join(", ");
945
+ let keyMatch = keyColumns.map(c => `h."` + c + `" = ` + tableRef + `."` + c + `"`).join(" AND ");
946
+ let bounds = CheckpointBounds.sql(floors.floors, Table.getPgChainIdColumn(entityConfig.table), tableRef);
947
+ return `SELECT DISTINCT ON (` + keyColumnsCommaSeparated + `) ` + dataFieldsCommaSeparated + `, ` + tableRef + `."` + EntityHistory.changeFieldName + `"
948
+ FROM "` + pgSchema + `"."` + historyTableName + `"` + bounds.join + `
949
+ WHERE ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" <= ` + bounds.checkpointId + `
934
950
  AND EXISTS (
935
951
  SELECT 1
936
952
  FROM "` + pgSchema + `"."` + historyTableName + `" h
937
953
  WHERE ` + keyMatch + `
938
- AND h."` + EntityHistory.checkpointIdFieldName + `" > $1
954
+ AND h."` + EntityHistory.checkpointIdFieldName + `" > ` + bounds.checkpointId + `
939
955
  )
940
- ORDER BY ` + keyColumnsCommaSeparated + `, "` + EntityHistory.checkpointIdFieldName + `" DESC`;
956
+ ORDER BY ` + keyColumnsCommaSeparated + `, ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" DESC`;
941
957
  }
942
958
 
943
- function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
959
+ function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema, floors) {
944
960
  let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
961
+ let tableRef = `"` + historyTableName + `"`;
945
962
  let keyColumns = rollbackKeyColumns(entityConfig);
946
- let keyMatch = keyColumns.map(c => `h."` + c + `" = "` + historyTableName + `"."` + c + `"`).join(" AND ");
947
- return `SELECT DISTINCT ` + keyColumns.map(c => `"` + c + `"`).join(", ") + `
948
- FROM "` + pgSchema + `"."` + historyTableName + `"
949
- WHERE "` + EntityHistory.checkpointIdFieldName + `" > $1
963
+ let keyMatch = keyColumns.map(c => `h."` + c + `" = ` + tableRef + `."` + c + `"`).join(" AND ");
964
+ let bounds = CheckpointBounds.sql(floors.floors, Table.getPgChainIdColumn(entityConfig.table), tableRef);
965
+ return `SELECT DISTINCT ` + keyColumns.map(c => tableRef + `."` + c + `"`).join(", ") + `
966
+ FROM "` + pgSchema + `"."` + historyTableName + `"` + bounds.join + `
967
+ WHERE ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" > ` + bounds.checkpointId + `
950
968
  AND NOT EXISTS (
951
969
  SELECT 1
952
970
  FROM "` + pgSchema + `"."` + historyTableName + `" h
953
971
  WHERE ` + keyMatch + `
954
- AND h."` + EntityHistory.checkpointIdFieldName + `" <= $1
972
+ AND h."` + EntityHistory.checkpointIdFieldName + `" <= ` + bounds.checkpointId + `
955
973
  )`;
956
974
  }
957
975
 
@@ -1113,13 +1131,12 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1113
1131
  let entities = entitiesOpt !== undefined ? entitiesOpt : [];
1114
1132
  let enums = enumsOpt !== undefined ? enumsOpt : [];
1115
1133
  let pgEntities = entities.filter(e => e.storage.postgres);
1116
- let chEntities = entities.filter(e => e.storage.clickhouse);
1117
1134
  let schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
1118
1135
  if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(table => table.table_name === InternalTable.Chains.table.tableName ? true : table.table_name === "event_sync_state")) {
1119
1136
  Stdlib_JsError.throwWithMessage(`Cannot run Envio migrations on PostgreSQL schema "` + pgSchema + `" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: "pnpm envio local db-migrate down"\n2. Or specify a different schema name by setting the "ENVIO_PG_SCHEMA" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.`);
1120
1137
  }
1121
1138
  if (sink !== undefined) {
1122
- await sink.initialize(chainConfigs, chEntities, enums);
1139
+ await sink.initialize(entities);
1123
1140
  }
1124
1141
  let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true, chainIdMode);
1125
1142
  let rowsByChain = chainConfigs.map(chainConfig => ChainState.configStorageRows(chainConfig, ecosystem, contractMapping));
@@ -1398,7 +1415,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1398
1415
  return Logging.errorWithExn(Utils.prettifyExn(exn$1), `Failed to dump cache.`);
1399
1416
  }
1400
1417
  };
1401
- let resumeInitialState = async () => {
1418
+ let resumeInitialState = async (entities, throwIfIncompatible) => {
1402
1419
  let match = await Promise.all([
1403
1420
  restoreEffectCache(false),
1404
1421
  InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => rawInitialStates.map(rawInitialState => ({
@@ -1435,6 +1452,10 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1435
1452
  })
1436
1453
  ]);
1437
1454
  let match$1 = match[4];
1455
+ let storedContractMapping = match$1[1];
1456
+ let storedEnvioInfo = match$1[0];
1457
+ let chains = match[1];
1458
+ throwIfIncompatible(storedEnvioInfo, storedContractMapping);
1438
1459
  await reloadIndexCatalog();
1439
1460
  let checkpointId = BigInt(match[2][0].id);
1440
1461
  let reorgCheckpoints = match[3].map(raw => ({
@@ -1444,14 +1465,14 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1444
1465
  block_hash: raw.block_hash
1445
1466
  }));
1446
1467
  if (sink !== undefined) {
1447
- await sink.resume(checkpointId);
1468
+ await sink.resume(checkpointId, chains, entities);
1448
1469
  }
1449
1470
  return {
1450
1471
  cleanRun: false,
1451
- contractMapping: match$1[1],
1452
- envioInfo: match$1[0],
1472
+ contractMapping: storedContractMapping,
1473
+ envioInfo: storedEnvioInfo,
1453
1474
  cache: match[0],
1454
- chains: match[1],
1475
+ chains: chains,
1455
1476
  checkpointId: checkpointId,
1456
1477
  reorgCheckpoints: reorgCheckpoints
1457
1478
  };
@@ -1461,14 +1482,15 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
1461
1482
  return await sql.unsafe(query);
1462
1483
  };
1463
1484
  let setChainMeta = chainsData => InternalTable.Chains.setMeta(sql, pgSchema, chainsData).then(param => (undefined));
1464
- let pruneStaleCheckpoints = safeCheckpointId => InternalTable.Checkpoints.pruneStaleCheckpoints(sql, pgSchema, safeCheckpointId);
1465
- let pruneStaleEntityHistory = (entityName, entityIndex, chainIdColumn, safeCheckpointId) => EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpointId);
1485
+ let pruneStaleCheckpoints = safeCheckpoints => InternalTable.Checkpoints.pruneStaleCheckpoints(sql, pgSchema, safeCheckpoints);
1486
+ let pruneStaleEntityHistory = (entityName, entityIndex, chainIdColumn, safeCheckpoints) => EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints);
1466
1487
  let getRollbackTargetCheckpoint = (reorgChainId, lastKnownValidBlockNumber) => InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
1467
- let getRollbackProgressDiff = rollbackTargetCheckpointId => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId);
1468
- let getRollbackData = async (entityConfig, rollbackTargetCheckpointId) => {
1488
+ let getRollbackProgressDiff = floors => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, floors);
1489
+ let getRollbackData = async (entityConfig, floors) => {
1490
+ let params = CheckpointBounds.params(floors.floors);
1469
1491
  let match = await Promise.all([
1470
- sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
1471
- sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
1492
+ sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema, floors), params, {prepare: true}),
1493
+ sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema, floors), params, {prepare: true})
1472
1494
  ]);
1473
1495
  let chainIdSchema = rollbackChainIdSchema(entityConfig.table);
1474
1496
  let scopeOf = row => {