envio 3.9.0 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/evm.schema.json +26 -0
  2. package/fuel.schema.json +26 -0
  3. package/index.d.ts +35 -3
  4. package/package.json +6 -7
  5. package/src/Batch.res +9 -0
  6. package/src/ChainState.res +35 -39
  7. package/src/ChainState.res.mjs +14 -11
  8. package/src/ChainState.resi +15 -7
  9. package/src/Config.res +52 -44
  10. package/src/Config.res.mjs +51 -17
  11. package/src/Core.res +4 -0
  12. package/src/CrossChainState.res +15 -23
  13. package/src/CrossChainState.res.mjs +10 -18
  14. package/src/CrossChainState.resi +4 -1
  15. package/src/Ecosystem.res +7 -2
  16. package/src/Envio.res +6 -4
  17. package/src/EventConfigBuilder.res +169 -26
  18. package/src/EventConfigBuilder.res.mjs +124 -15
  19. package/src/HandlerRegister.res +21 -4
  20. package/src/HandlerRegister.res.mjs +24 -5
  21. package/src/InMemoryStore.res +9 -7
  22. package/src/InMemoryStore.res.mjs +3 -3
  23. package/src/IndexerState.res +43 -6
  24. package/src/IndexerState.res.mjs +36 -7
  25. package/src/IndexerState.resi +4 -4
  26. package/src/Internal.res +16 -17
  27. package/src/Metrics.res +72 -6
  28. package/src/Metrics.res.mjs +30 -2
  29. package/src/Persistence.res +36 -30
  30. package/src/Persistence.res.mjs +1 -20
  31. package/src/PgStorage.res +104 -56
  32. package/src/PgStorage.res.mjs +57 -35
  33. package/src/PruneStaleHistory.res +85 -52
  34. package/src/PruneStaleHistory.res.mjs +63 -37
  35. package/src/Rollback.res +68 -45
  36. package/src/Rollback.res.mjs +38 -19
  37. package/src/RollbackCommit.res +7 -9
  38. package/src/RollbackCommit.res.mjs +5 -6
  39. package/src/SafeCheckpointTracking.res +11 -11
  40. package/src/SafeCheckpointTracking.res.mjs +7 -6
  41. package/src/SimulateItems.res +33 -18
  42. package/src/SimulateItems.res.mjs +26 -22
  43. package/src/Sink.res +38 -26
  44. package/src/Sink.res.mjs +21 -16
  45. package/src/TestIndexer.res +5 -5
  46. package/src/TestIndexer.res.mjs +1 -1
  47. package/src/Utils.res +170 -0
  48. package/src/Utils.res.mjs +153 -0
  49. package/src/Writing.res +2 -2
  50. package/src/Writing.res.mjs +1 -1
  51. package/src/bindings/ClickHouse.res +366 -698
  52. package/src/bindings/ClickHouse.res.mjs +371 -421
  53. package/src/bindings/ClickHouseSink.res +337 -0
  54. package/src/bindings/ClickHouseSink.res.mjs +246 -0
  55. package/src/bindings/EventSource.res +8 -2
  56. package/src/bindings/NodeJs.res +6 -0
  57. package/src/bindings/Vitest.res +23 -0
  58. package/src/bindings/Vitest.res.mjs +3 -0
  59. package/src/bindings/WebSocket.res +9 -10
  60. package/src/db/CheckpointBounds.res +53 -0
  61. package/src/db/CheckpointBounds.res.mjs +44 -0
  62. package/src/db/EntityFilter.res +36 -17
  63. package/src/db/EntityFilter.res.mjs +25 -14
  64. package/src/db/EntityHistory.res +47 -22
  65. package/src/db/EntityHistory.res.mjs +19 -12
  66. package/src/db/InternalTable.res +84 -35
  67. package/src/db/InternalTable.res.mjs +56 -23
  68. package/src/db/RollbackFloors.res +55 -0
  69. package/src/db/RollbackFloors.res.mjs +92 -0
  70. package/src/db/Table.res +22 -4
  71. package/src/db/Table.res.mjs +30 -9
  72. package/src/sources/Evm.res +2 -0
  73. package/src/sources/Evm.res.mjs +2 -0
  74. package/src/sources/EvmHyperSyncSource.res +3 -3
  75. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  76. package/src/sources/EvmRpcWs.res +124 -0
  77. package/src/sources/EvmRpcWs.res.mjs +117 -0
  78. package/src/sources/Fuel.res +2 -0
  79. package/src/sources/Fuel.res.mjs +2 -0
  80. package/src/sources/FuelHyperSyncSource.res +1 -1
  81. package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
  82. package/src/sources/HeightFeed.res +568 -0
  83. package/src/sources/HeightFeed.res.mjs +452 -0
  84. package/src/sources/HeightStream.res +257 -0
  85. package/src/sources/HeightStream.res.mjs +194 -0
  86. package/src/sources/HyperSync.res +5 -0
  87. package/src/sources/HyperSync.res.mjs +3 -0
  88. package/src/sources/HyperSync.resi +4 -0
  89. package/src/sources/HyperSyncSSE.res +49 -0
  90. package/src/sources/HyperSyncSSE.res.mjs +79 -0
  91. package/src/sources/RpcSource.res +1 -1
  92. package/src/sources/RpcSource.res.mjs +2 -2
  93. package/src/sources/Source.res +54 -1
  94. package/src/sources/Source.res.mjs +25 -0
  95. package/src/sources/SourceManager.res +220 -227
  96. package/src/sources/SourceManager.res.mjs +139 -164
  97. package/src/sources/SourceManager.resi +8 -0
  98. package/src/sources/Svm.res +2 -0
  99. package/src/sources/Svm.res.mjs +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +70 -127
  101. package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
  102. package/src/sources/SvmHyperSyncSource.res +18 -15
  103. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  104. package/svm.schema.json +1 -75
  105. package/src/MemoryStorage.res +0 -729
  106. package/src/MemoryStorage.res.mjs +0 -586
  107. package/src/sources/HyperSyncHeightStream.res +0 -129
  108. package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
  109. package/src/sources/RpcWebSocketHeightStream.res +0 -175
  110. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
@@ -80,9 +80,20 @@ function renderMetrics(b, metrics) {
80
80
  `{entity="` + escapeLabelValue(s.entity) + `"}`,
81
81
  s
82
82
  ]);
83
+ let sourceLabels = (source, chainId, extra) => {
84
+ let pair = `{source="` + escapeLabelValue(source) + `",chainId="` + ChainId.toString(chainId) + `"`;
85
+ if (extra !== undefined) {
86
+ return pair + `,` + extra[0] + `="` + escapeLabelValue(extra[1]) + `"}`;
87
+ } else {
88
+ return pair + "}";
89
+ }
90
+ };
83
91
  let byLabels = {};
84
92
  metrics.sourceRequests.forEach(s => {
85
- let labels = `{source="` + escapeLabelValue(s.source) + `",chainId="` + ChainId.toString(s.chainId) + `",method="` + escapeLabelValue(s.method) + `"}`;
93
+ let labels = sourceLabels(s.source, s.chainId, [
94
+ "method",
95
+ s.method
96
+ ]);
86
97
  let existing = byLabels[labels];
87
98
  if (existing !== undefined) {
88
99
  byLabels[labels] = {
@@ -99,7 +110,7 @@ function renderMetrics(b, metrics) {
99
110
  let sourceRequests = Object.entries(byLabels);
100
111
  let byLabels$1 = {};
101
112
  metrics.sourceHeights.forEach(s => {
102
- let labels = `{source="` + escapeLabelValue(s.source) + `",chainId="` + ChainId.toString(s.chainId) + `"}`;
113
+ let labels = sourceLabels(s.source, s.chainId, undefined);
103
114
  let existing = byLabels$1[labels];
104
115
  if (existing !== undefined && existing >= s.height) {
105
116
  return;
@@ -109,6 +120,17 @@ function renderMetrics(b, metrics) {
109
120
  }
110
121
  });
111
122
  let sources = Object.entries(byLabels$1);
123
+ let byLabels$2 = {};
124
+ metrics.sourceHeightStreams.forEach(s => Utils.Dict.incrementBy(byLabels$2, sourceLabels(s.source, s.chainId, undefined), s.connectCount));
125
+ let heightStreamConnects = Object.entries(byLabels$2);
126
+ let byLabels$3 = {};
127
+ metrics.sourceHeightStreams.forEach(s => {
128
+ s.disconnectsByReason.forEach(param => Utils.Dict.incrementBy(byLabels$3, sourceLabels(s.source, s.chainId, [
129
+ "reason",
130
+ param[0]
131
+ ]), param[1]));
132
+ });
133
+ let heightStreamDisconnects = Object.entries(byLabels$3);
112
134
  single(b, "envio_process_start_time_seconds", "Start time of the process since unix epoch in seconds.", "gauge", metrics.startTime.getTime() / 1000);
113
135
  single(b, "envio_process_metric_time_seconds", "The time these metrics were collected. Use it to tell how fresh a snapshot is, or to measure rates between two snapshots.", "gauge", metrics.metricTime.getTime() / 1000);
114
136
  single(b, "envio_process_elapsed_seconds", "How long the indexer has been running. Divide a cumulative seconds metric by this to get the share of the run it took, eg envio_processing_seconds for time spent in event handlers.", "gauge", metrics.elapsedSeconds);
@@ -154,6 +176,12 @@ function renderMetrics(b, metrics) {
154
176
  return s.seconds;
155
177
  }
156
178
  });
179
+ if (heightStreamConnects.length !== 0) {
180
+ series(b, "envio_source_height_stream_connects_total", "The number of times a source's height subscription connected. Compare against the disconnects total, which is absent until the first disconnect and counts as zero while it is: one more connect than disconnects means the stream is up, and equal counts mean it is down and the indexer is polling instead. Zero connects means the stream has not come up, which is the normal reading for a chain that is still backfilling: subscriptions are only opened once a chain reaches the head.", "counter", heightStreamConnects, count => count);
181
+ }
182
+ if (heightStreamDisconnects.length !== 0) {
183
+ series(b, "envio_source_height_stream_disconnects_total", "The number of times a source's height subscription lost a connection, by reason. Failed retries are not counted, so this is outages rather than their length. A rotated disconnect is a connection that served its time; unsubscribed is the source being benched; every other reason ended a connection early.", "counter", heightStreamDisconnects, count => count);
184
+ }
157
185
  series(b, "envio_source_known_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", "gauge", sources, height => height);
158
186
  seriesOpt(b, "envio_reorg_detected_total", "Total number of reorgs detected", "counter", chains, m => {
159
187
  if (m.reorgCount > 0) {
@@ -59,14 +59,20 @@ type updatedEffectCache = {
59
59
  }
60
60
 
61
61
  type rollback = {
62
- targetCheckpointId: Internal.checkpointId,
63
62
  diffCheckpointId: Internal.checkpointId,
63
+ // How far back the deletes reach on each chain, travelling with the diff so
64
+ // the write leaves an untouched sibling's rows alone.
65
+ floors: RollbackFloors.t,
64
66
  // The address registrations the rollback dropped, as the chains' address
65
67
  // stores resolved them. Deleted by primary key in the same transaction.
66
68
  rolledBackAddresses: array<AddressRows.key>,
67
- // Last valid block per chain affected by the rollback. Read by
68
- // `RollbackCommit.fire` once the diff is durably written.
69
- progressBlockNumberByChainId: dict<int>,
69
+ // Where the rollback left every chain it moved. Written with the diff rather
70
+ // than waiting for a batch of those chains' own: the batch that carries the
71
+ // diff can belong to a chain the rollback never touched, and a chain whose
72
+ // stored progress outlived the checkpoints backing it would resume past
73
+ // blocks it never re-indexed. Also what `RollbackCommit.fire` reports once
74
+ // the diff is durably written.
75
+ progressedChains: array<InternalTable.Chains.progressedChain>,
70
76
  }
71
77
 
72
78
  // One flush group: the changes an entity accumulated within a single chain
@@ -101,7 +107,16 @@ type storage = {
101
107
  ~contractMapping: ContractMapping.t,
102
108
  ~envioInfo: JSON.t,
103
109
  ) => promise<initialState>,
104
- resumeInitialState: unit => promise<initialState>,
110
+ // `throwIfIncompatible` gets what the storage holds before any sink is
111
+ // resumed, so a config the stored one rules out is reported as such rather
112
+ // than as the sink tripping over tables it never created.
113
+ resumeInitialState: (
114
+ ~entities: array<Internal.entityConfig>,
115
+ ~throwIfIncompatible: (
116
+ ~storedEnvioInfo: option<JSON.t>,
117
+ ~storedContractMapping: ContractMapping.t,
118
+ ) => unit,
119
+ ) => promise<initialState>,
105
120
  // Returns rows matching the filter.
106
121
  // Field values are serialized and rows parsed with the table's field schemas.
107
122
  @raises("StorageError")
@@ -131,13 +146,13 @@ type storage = {
131
146
  // Update chain metadata
132
147
  setChainMeta: dict<InternalTable.Chains.metaFields> => promise<unknown>,
133
148
  // Prune old checkpoints
134
- pruneStaleCheckpoints: (~safeCheckpointId: Internal.checkpointId) => promise<unit>,
149
+ pruneStaleCheckpoints: (~safeCheckpoints: CheckpointBounds.t) => promise<unit>,
135
150
  // Prune stale entity history
136
151
  pruneStaleEntityHistory: (
137
152
  ~entityName: string,
138
153
  ~entityIndex: int,
139
154
  ~chainIdColumn: option<string>,
140
- ~safeCheckpointId: Internal.checkpointId,
155
+ ~safeCheckpoints: CheckpointBounds.t,
141
156
  ) => promise<unit>,
142
157
  // Get rollback target checkpoint
143
158
  getRollbackTargetCheckpoint: (
@@ -146,7 +161,7 @@ type storage = {
146
161
  ) => promise<option<Internal.checkpointId>>,
147
162
  // Get rollback progress diff
148
163
  getRollbackProgressDiff: (
149
- ~rollbackTargetCheckpointId: Internal.checkpointId,
164
+ ~floors: RollbackFloors.t,
150
165
  ) => promise<
151
166
  array<{
152
167
  "chain_id": ChainId.t,
@@ -159,7 +174,7 @@ type storage = {
159
174
  // before handing them back.
160
175
  getRollbackData: (
161
176
  ~entityConfig: Internal.entityConfig,
162
- ~rollbackTargetCheckpointId: Internal.checkpointId,
177
+ ~floors: RollbackFloors.t,
163
178
  ) => promise<(array<rollbackRemoval>, array<Internal.entity>)>,
164
179
  // Write batch to storage
165
180
  writeBatch: (
@@ -262,27 +277,18 @@ let init = {
262
277
  }
263
278
  ) {
264
279
  Logging.info(`Found existing indexer storage. Resuming indexing state...`)
265
- let initialState = await persistence.storage.resumeInitialState()
266
- let changedPaths = switch initialState.envioInfo {
267
- | None => ["storage was initialized by an older envio version"]
268
- | Some(stored) => Config.diffPaths(~stored, ~current=envioInfo)
269
- }
270
- let hasClickhouse = switch envioInfo {
271
- | Object(d) =>
272
- switch d->Dict.get("storage") {
273
- | Some(Object(s)) =>
274
- switch s->Dict.get("clickhouse") {
275
- | Some(Boolean(true)) => true
276
- | _ => false
277
- }
278
- | _ => false
279
- }
280
- | _ => false
281
- }
282
- Config.throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
283
- if !(initialState.contractMapping->ContractMapping.isEqual(contractMapping)) {
284
- Config.throwIfIncompatible(["contracts"], ~resetCommand, ~runCommand, ~hasClickhouse)
285
- }
280
+ let initialState = await persistence.storage.resumeInitialState(
281
+ ~entities=persistence.allEntities,
282
+ ~throwIfIncompatible=(~storedEnvioInfo, ~storedContractMapping) =>
283
+ Config.throwIfResumeIncompatible(
284
+ ~storedEnvioInfo,
285
+ ~storedContractMapping,
286
+ ~envioInfo,
287
+ ~contractMapping,
288
+ ~resetCommand,
289
+ ~runCommand,
290
+ ),
291
+ )
286
292
  persistence.storageStatus = Ready(initialState)
287
293
  let progress = Dict.make()
288
294
  initialState.chains->Array.forEach(c => {
@@ -5,7 +5,6 @@ import * as Logging from "./Logging.res.mjs";
5
5
  import * as EntityHistory from "./db/EntityHistory.res.mjs";
6
6
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
7
7
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
8
- import * as ContractMapping from "./ContractMapping.res.mjs";
9
8
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
10
9
 
11
10
  let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
@@ -61,25 +60,7 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
61
60
  tmp = typeof match !== "object" ? false : match.TAG === "Initializing";
62
61
  if (tmp) {
63
62
  Logging.info(`Found existing indexer storage. Resuming indexing state...`);
64
- let initialState$1 = await persistence.storage.resumeInitialState();
65
- let stored = initialState$1.envioInfo;
66
- let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["storage was initialized by an older envio version"];
67
- let hasClickhouse;
68
- if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
69
- let match$1 = envioInfo["storage"];
70
- if (typeof match$1 === "object" && match$1 !== null && !Array.isArray(match$1)) {
71
- let match$2 = match$1["clickhouse"];
72
- hasClickhouse = match$2 === true;
73
- } else {
74
- hasClickhouse = false;
75
- }
76
- } else {
77
- hasClickhouse = false;
78
- }
79
- Config.throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhouse);
80
- if (!ContractMapping.isEqual(initialState$1.contractMapping, contractMapping)) {
81
- Config.throwIfIncompatible(["contracts"], resetCommand, runCommand, hasClickhouse);
82
- }
63
+ let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, (storedEnvioInfo, storedContractMapping) => Config.throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand));
83
64
  persistence.storageStatus = {
84
65
  TAG: "Ready",
85
66
  _0: initialState$1
package/src/PgStorage.res CHANGED
@@ -377,7 +377,7 @@ let makeLoadQuery = (~pgSchema, ~tableName, ~condition) => {
377
377
  let rec makeFilterCondition = (
378
378
  ~filter: EntityFilter.t,
379
379
  ~table: Table.table,
380
- ~params: array<JSON.t>,
380
+ ~params: array<unknown>,
381
381
  ) => {
382
382
  // Filters reference fields by API name, while the SQL references columns
383
383
  // by their possibly renamed db names.
@@ -398,7 +398,7 @@ let rec makeFilterCondition = (
398
398
  ~fieldValue: unknown,
399
399
  ~isArray,
400
400
  ) => {
401
- let param = try fieldValue->S.reverseConvertToJsonOrThrow(
401
+ let param = try fieldValue->S.reverseConvertOrThrow(
402
402
  isArray ? queryField.arrayFieldSchema : queryField.fieldSchema,
403
403
  ) catch {
404
404
  | exn =>
@@ -438,8 +438,8 @@ let rec makeFilterCondition = (
438
438
  // `LoadLayer.scopeFilter` is what puts this filter here, and the value is
439
439
  // range-checked to a non-negative safe integer, so it can carry nothing but
440
440
  // digits.
441
- | Eq({fieldName, fieldValue}) if (getQueryFieldOrThrow(fieldName)).isChainId =>
442
- `"${(getQueryFieldOrThrow(fieldName)).pgDbFieldName}" = ${fieldValue
441
+ | Eq({fieldName, fieldValue}) if getQueryFieldOrThrow(fieldName).isChainId =>
442
+ `"${getQueryFieldOrThrow(fieldName).pgDbFieldName}" = ${fieldValue
443
443
  ->ChainId.normalizeOrThrow
444
444
  ->ChainId.toString}`
445
445
  | Eq({fieldName, fieldValue}) => scalarCondition(~fieldName, ~fieldValue, ~op="=")
@@ -587,7 +587,7 @@ let makeTableBatchSetQuery = (
587
587
  ~itemSchema: S.t<'item>,
588
588
  ~chainIdMode: ChainId.mode=Int32,
589
589
  ) => {
590
- let {dbSchema, hasArrayField} =
590
+ let {dbSchema, hasArrayField, byteaColumnIndexes} =
591
591
  table->Table.toSqlParams(~schema=itemSchema, ~pgSchema, ~chainIdMode)
592
592
 
593
593
  // Should move this to a better place
@@ -614,7 +614,22 @@ let makeTableBatchSetQuery = (
614
614
  {
615
615
  "query": makeInsertUnnestSetQuery(~pgSchema, ~table, ~itemSchema, ~isRawEvents, ~chainIdMode),
616
616
  "convertOrThrow": S.compile(
617
- S.unnest(dbSchema),
617
+ S.unnest(dbSchema)->S.preprocess(_ => {
618
+ serializer: columns => {
619
+ let columns = columns->(Utils.magic: unknown => array<unknown>)
620
+ byteaColumnIndexes->Array.forEach(index =>
621
+ columns->Array.setUnsafe(
622
+ index,
623
+ columns
624
+ ->Array.getUnsafe(index)
625
+ ->(Utils.magic: unknown => array<unknown>)
626
+ ->Utils.Bytes.toPgArrayLiteral
627
+ ->(Utils.magic: string => unknown),
628
+ )
629
+ )
630
+ columns->(Utils.magic: array<unknown> => unknown)
631
+ },
632
+ }),
618
633
  ~input=Value,
619
634
  ~output=Unknown,
620
635
  ~mode=Sync,
@@ -1106,6 +1121,8 @@ let rec writeBatch = async (
1106
1121
  | Internal.CrossChain => None
1107
1122
  | Chain(chainId) => Some(chainId)
1108
1123
  }
1124
+ let shouldSaveHistory =
1125
+ config->Config.shouldSaveHistory(~isInReorgThreshold, ~chainId=?scopeChainId)
1109
1126
  let changes = switch (entityConfig.table->Table.getChainIdField, scopeChainId) {
1110
1127
  | (Some(field), Some(chainId)) =>
1111
1128
  changes->Array.map(change =>
@@ -1296,7 +1313,7 @@ let rec writeBatch = async (
1296
1313
  //valid event identifier, where all rows created after this eventIdentifier should
1297
1314
  //be deleted
1298
1315
  let rollbackTables = switch rollback {
1299
- | Some({targetCheckpointId: rollbackTargetCheckpointId, rolledBackAddresses}) =>
1316
+ | Some({floors, rolledBackAddresses, progressedChains}) =>
1300
1317
  Some(
1301
1318
  sql => {
1302
1319
  // Postgres owns history tables only for Postgres-backed entities;
@@ -1309,15 +1326,24 @@ let rec writeBatch = async (
1309
1326
  ~pgSchema,
1310
1327
  ~entityName=entityConfig.name,
1311
1328
  ~entityIndex=entityConfig.index,
1312
- ~rollbackTargetCheckpointId,
1329
+ ~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
1330
+ ~floors,
1313
1331
  )
1314
1332
  })
1315
1333
  promises
1316
- ->Array.push(
1317
- sql->InternalTable.Checkpoints.rollback(~pgSchema, ~rollbackTargetCheckpointId),
1318
- )
1334
+ ->Array.push(sql->InternalTable.Checkpoints.rollback(~pgSchema, ~floors))
1319
1335
  ->ignore
1320
1336
 
1337
+ // Runs before the batch's own progress write below, so a chain the
1338
+ // batch also progressed keeps the batch's later value.
1339
+ if progressedChains->Utils.Array.notEmpty {
1340
+ promises
1341
+ ->Array.push(
1342
+ sql->InternalTable.Chains.setProgressedChains(~pgSchema, ~progressedChains),
1343
+ )
1344
+ ->ignore
1345
+ }
1346
+
1321
1347
  // Addresses are insert-only, so undoing their registrations is a
1322
1348
  // delete rather than a history replay. It runs before the batch's own
1323
1349
  // inserts in the same transaction, so a re-registered address lands
@@ -1476,7 +1502,11 @@ let rollbackKeyColumns = (entityConfig: Internal.entityConfig) =>
1476
1502
  | None => [Table.idFieldName]
1477
1503
  }
1478
1504
 
1479
- let makeGetRollbackPreTargetRowsQuery = (~entityConfig: Internal.entityConfig, ~pgSchema) => {
1505
+ let makeGetRollbackPreTargetRowsQuery = (
1506
+ ~entityConfig: Internal.entityConfig,
1507
+ ~pgSchema,
1508
+ ~floors: RollbackFloors.t,
1509
+ ) => {
1480
1510
  let dataFieldNames = entityConfig.table.fields->Array.filterMap(fieldOrDerived =>
1481
1511
  switch fieldOrDerived {
1482
1512
  | Field(field) => field->Table.getPgDbFieldName->Some
@@ -1484,56 +1514,70 @@ let makeGetRollbackPreTargetRowsQuery = (~entityConfig: Internal.entityConfig, ~
1484
1514
  }
1485
1515
  )
1486
1516
 
1487
- let dataFieldsCommaSeparated =
1488
- dataFieldNames->Array.map(name => `"${name}"`)->Array.joinUnsafe(", ")
1489
-
1490
1517
  let historyTableName = EntityHistory.historyTableName(
1491
1518
  ~entityName=entityConfig.name,
1492
1519
  ~entityIndex=entityConfig.index,
1493
1520
  )
1521
+ // Every column is qualified: the per-chain bounds relation joined in has a
1522
+ // `chain_id` of its own, which a snake_case chain column shares.
1523
+ let tableRef = `"${historyTableName}"`
1524
+ let dataFieldsCommaSeparated =
1525
+ dataFieldNames->Array.map(name => `${tableRef}."${name}"`)->Array.joinUnsafe(", ")
1494
1526
 
1495
1527
  // A per-chain entity's rows are only comparable within a chain, so the row's
1496
1528
  // identity here is (id, chain id) rather than the id alone.
1497
1529
  let keyColumns = rollbackKeyColumns(entityConfig)
1498
- let keyColumnsCommaSeparated = keyColumns->Array.map(c => `"${c}"`)->Array.joinUnsafe(", ")
1530
+ let keyColumnsCommaSeparated =
1531
+ keyColumns->Array.map(c => `${tableRef}."${c}"`)->Array.joinUnsafe(", ")
1499
1532
  let keyMatch =
1500
- keyColumns
1501
- ->Array.map(c => `h."${c}" = "${historyTableName}"."${c}"`)
1502
- ->Array.joinUnsafe(" AND ")
1533
+ keyColumns->Array.map(c => `h."${c}" = ${tableRef}."${c}"`)->Array.joinUnsafe(" AND ")
1534
+ let bounds =
1535
+ floors.floors->CheckpointBounds.sql(
1536
+ ~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
1537
+ ~tableRef,
1538
+ )
1503
1539
 
1504
- `SELECT DISTINCT ON (${keyColumnsCommaSeparated}) ${dataFieldsCommaSeparated}, "${EntityHistory.changeFieldName}"
1505
- FROM "${pgSchema}"."${historyTableName}"
1506
- WHERE "${EntityHistory.checkpointIdFieldName}" <= $1
1540
+ `SELECT DISTINCT ON (${keyColumnsCommaSeparated}) ${dataFieldsCommaSeparated}, ${tableRef}."${EntityHistory.changeFieldName}"
1541
+ FROM "${pgSchema}"."${historyTableName}"${bounds.join}
1542
+ WHERE ${tableRef}."${EntityHistory.checkpointIdFieldName}" <= ${bounds.checkpointId}
1507
1543
  AND EXISTS (
1508
1544
  SELECT 1
1509
1545
  FROM "${pgSchema}"."${historyTableName}" h
1510
1546
  WHERE ${keyMatch}
1511
- AND h."${EntityHistory.checkpointIdFieldName}" > $1
1547
+ AND h."${EntityHistory.checkpointIdFieldName}" > ${bounds.checkpointId}
1512
1548
  )
1513
- ORDER BY ${keyColumnsCommaSeparated}, "${EntityHistory.checkpointIdFieldName}" DESC`
1549
+ ORDER BY ${keyColumnsCommaSeparated}, ${tableRef}."${EntityHistory.checkpointIdFieldName}" DESC`
1514
1550
  }
1515
1551
 
1516
1552
  // Returns entity IDs that were created after the rollback target and have no history before it.
1517
1553
  // DELETE rows at or before the target are returned by the restore query and classified in ReScript.
1518
- let makeGetRollbackRemovedIdsQuery = (~entityConfig: Internal.entityConfig, ~pgSchema) => {
1554
+ let makeGetRollbackRemovedIdsQuery = (
1555
+ ~entityConfig: Internal.entityConfig,
1556
+ ~pgSchema,
1557
+ ~floors: RollbackFloors.t,
1558
+ ) => {
1519
1559
  let historyTableName = EntityHistory.historyTableName(
1520
1560
  ~entityName=entityConfig.name,
1521
1561
  ~entityIndex=entityConfig.index,
1522
1562
  )
1563
+ let tableRef = `"${historyTableName}"`
1523
1564
  let keyColumns = rollbackKeyColumns(entityConfig)
1524
1565
  let keyMatch =
1525
- keyColumns
1526
- ->Array.map(c => `h."${c}" = "${historyTableName}"."${c}"`)
1527
- ->Array.joinUnsafe(" AND ")
1566
+ keyColumns->Array.map(c => `h."${c}" = ${tableRef}."${c}"`)->Array.joinUnsafe(" AND ")
1567
+ let bounds =
1568
+ floors.floors->CheckpointBounds.sql(
1569
+ ~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
1570
+ ~tableRef,
1571
+ )
1528
1572
 
1529
- `SELECT DISTINCT ${keyColumns->Array.map(c => `"${c}"`)->Array.joinUnsafe(", ")}
1530
- FROM "${pgSchema}"."${historyTableName}"
1531
- WHERE "${EntityHistory.checkpointIdFieldName}" > $1
1573
+ `SELECT DISTINCT ${keyColumns->Array.map(c => `${tableRef}."${c}"`)->Array.joinUnsafe(", ")}
1574
+ FROM "${pgSchema}"."${historyTableName}"${bounds.join}
1575
+ WHERE ${tableRef}."${EntityHistory.checkpointIdFieldName}" > ${bounds.checkpointId}
1532
1576
  AND NOT EXISTS (
1533
1577
  SELECT 1
1534
1578
  FROM "${pgSchema}"."${historyTableName}" h
1535
1579
  WHERE ${keyMatch}
1536
- AND h."${EntityHistory.checkpointIdFieldName}" <= $1
1580
+ AND h."${EntityHistory.checkpointIdFieldName}" <= ${bounds.checkpointId}
1537
1581
  )`
1538
1582
  }
1539
1583
 
@@ -1774,11 +1818,9 @@ let make = (
1774
1818
  ~contractMapping,
1775
1819
  ~envioInfo,
1776
1820
  ): Persistence.initialState => {
1777
- // Per-entity storage routing: PG owns tables only for entities that
1778
- // opted into Postgres; the sink mirrors only those that opted into
1779
- // ClickHouse.
1821
+ // PG owns tables only for entities that opted into Postgres; the sink
1822
+ // picks its own out of the full list.
1780
1823
  let pgEntities = entities->Array.filter((e: Internal.entityConfig) => e.storage.postgres)
1781
- let chEntities = entities->Array.filter((e: Internal.entityConfig) => e.storage.clickhouse)
1782
1824
 
1783
1825
  let schemaTableNames: array<schemaTableName> = await sql->Postgres.unsafe(
1784
1826
  makeSchemaTableNamesQuery(~pgSchema),
@@ -1807,7 +1849,7 @@ let make = (
1807
1849
 
1808
1850
  // Call sink.initialize before executing PG queries
1809
1851
  switch sink {
1810
- | Some(sink) => await sink.initialize(~chainConfigs, ~entities=chEntities, ~enums)
1852
+ | Some(sink) => await sink.initialize(~entities)
1811
1853
  | None => ()
1812
1854
  }
1813
1855
 
@@ -2271,8 +2313,14 @@ let make = (
2271
2313
  }
2272
2314
  }
2273
2315
 
2274
- let resumeInitialState = async (): Persistence.initialState => {
2275
- let (cache, chains, checkpointIdResult, reorgCheckpoints, (envioInfo, contractMapping)) = await Promise.all5((
2316
+ let resumeInitialState = async (~entities, ~throwIfIncompatible): Persistence.initialState => {
2317
+ let (
2318
+ cache,
2319
+ chains,
2320
+ checkpointIdResult,
2321
+ reorgCheckpoints,
2322
+ (storedEnvioInfo, storedContractMapping),
2323
+ ) = await Promise.all5((
2276
2324
  restoreEffectCache(~withUpload=false),
2277
2325
  InternalTable.Chains.getInitialState(
2278
2326
  sql,
@@ -2320,6 +2368,8 @@ let make = (
2320
2368
  ),
2321
2369
  ))
2322
2370
 
2371
+ throwIfIncompatible(~storedEnvioInfo, ~storedContractMapping)
2372
+
2323
2373
  await reloadIndexCatalog()
2324
2374
 
2325
2375
  let checkpointId = (checkpointIdResult->Array.getUnsafe(0))["id"]->BigInt.fromStringOrThrow
@@ -2334,7 +2384,7 @@ let make = (
2334
2384
 
2335
2385
  // Resume sink if present - needed to rollback any reorg changes
2336
2386
  switch sink {
2337
- | Some(sink) => await sink.resume(~checkpointId)
2387
+ | Some(sink) => await sink.resume(~checkpointId, ~chains, ~entities)
2338
2388
  | None => ()
2339
2389
  }
2340
2390
 
@@ -2344,8 +2394,8 @@ let make = (
2344
2394
  cache,
2345
2395
  chains,
2346
2396
  checkpointId,
2347
- contractMapping,
2348
- envioInfo,
2397
+ contractMapping: storedContractMapping,
2398
+ envioInfo: storedEnvioInfo,
2349
2399
  }
2350
2400
  }
2351
2401
 
@@ -2359,17 +2409,17 @@ let make = (
2359
2409
  %raw(`undefined`)
2360
2410
  )
2361
2411
 
2362
- let pruneStaleCheckpoints = (~safeCheckpointId) =>
2363
- InternalTable.Checkpoints.pruneStaleCheckpoints(sql, ~pgSchema, ~safeCheckpointId)
2412
+ let pruneStaleCheckpoints = (~safeCheckpoints) =>
2413
+ InternalTable.Checkpoints.pruneStaleCheckpoints(sql, ~pgSchema, ~safeCheckpoints)
2364
2414
 
2365
- let pruneStaleEntityHistory = (~entityName, ~entityIndex, ~chainIdColumn, ~safeCheckpointId) =>
2415
+ let pruneStaleEntityHistory = (~entityName, ~entityIndex, ~chainIdColumn, ~safeCheckpoints) =>
2366
2416
  EntityHistory.pruneStaleEntityHistory(
2367
2417
  sql,
2368
2418
  ~pgSchema,
2369
2419
  ~entityName,
2370
2420
  ~entityIndex,
2371
2421
  ~chainIdColumn,
2372
- ~safeCheckpointId,
2422
+ ~safeCheckpoints,
2373
2423
  )
2374
2424
 
2375
2425
  let getRollbackTargetCheckpoint = (~reorgChainId, ~lastKnownValidBlockNumber) =>
@@ -2380,26 +2430,24 @@ let make = (
2380
2430
  ~lastKnownValidBlockNumber,
2381
2431
  )
2382
2432
 
2383
- let getRollbackProgressDiff = (~rollbackTargetCheckpointId) =>
2384
- InternalTable.Checkpoints.getRollbackProgressDiff(sql, ~pgSchema, ~rollbackTargetCheckpointId)
2433
+ let getRollbackProgressDiff = (~floors) =>
2434
+ InternalTable.Checkpoints.getRollbackProgressDiff(sql, ~pgSchema, ~floors)
2385
2435
 
2386
- let getRollbackData = async (
2387
- ~entityConfig: Internal.entityConfig,
2388
- ~rollbackTargetCheckpointId,
2389
- ) => {
2436
+ let getRollbackData = async (~entityConfig: Internal.entityConfig, ~floors: RollbackFloors.t) => {
2437
+ let params = floors.floors->CheckpointBounds.params
2390
2438
  let (removedIdRows, rollbackRows) = await Promise.all2((
2391
2439
  // Get IDs of entities that should be deleted (created after rollback target with no prior history)
2392
2440
  sql
2393
2441
  ->Postgres.preparedUnsafe(
2394
- makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema),
2395
- [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
2442
+ makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema, ~floors),
2443
+ params,
2396
2444
  )
2397
2445
  ->(Utils.magic: promise<unknown> => promise<array<unknown>>),
2398
2446
  // Get the latest pre-target row, including its SET or DELETE action.
2399
2447
  sql
2400
2448
  ->Postgres.preparedUnsafe(
2401
- makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema),
2402
- [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
2449
+ makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema, ~floors),
2450
+ params,
2403
2451
  )
2404
2452
  ->(Utils.magic: promise<unknown> => promise<array<unknown>>),
2405
2453
  ))