envio 3.9.0 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/evm.schema.json +26 -0
  2. package/fuel.schema.json +26 -0
  3. package/index.d.ts +35 -3
  4. package/package.json +6 -7
  5. package/src/Batch.res +9 -0
  6. package/src/ChainState.res +35 -39
  7. package/src/ChainState.res.mjs +14 -11
  8. package/src/ChainState.resi +15 -7
  9. package/src/Config.res +52 -44
  10. package/src/Config.res.mjs +51 -17
  11. package/src/Core.res +4 -0
  12. package/src/CrossChainState.res +15 -23
  13. package/src/CrossChainState.res.mjs +10 -18
  14. package/src/CrossChainState.resi +4 -1
  15. package/src/Ecosystem.res +7 -2
  16. package/src/Envio.res +6 -4
  17. package/src/EventConfigBuilder.res +169 -26
  18. package/src/EventConfigBuilder.res.mjs +124 -15
  19. package/src/HandlerRegister.res +21 -4
  20. package/src/HandlerRegister.res.mjs +24 -5
  21. package/src/InMemoryStore.res +9 -7
  22. package/src/InMemoryStore.res.mjs +3 -3
  23. package/src/IndexerState.res +43 -6
  24. package/src/IndexerState.res.mjs +36 -7
  25. package/src/IndexerState.resi +4 -4
  26. package/src/Internal.res +16 -17
  27. package/src/Metrics.res +72 -6
  28. package/src/Metrics.res.mjs +30 -2
  29. package/src/Persistence.res +36 -30
  30. package/src/Persistence.res.mjs +1 -20
  31. package/src/PgStorage.res +104 -56
  32. package/src/PgStorage.res.mjs +57 -35
  33. package/src/PruneStaleHistory.res +85 -52
  34. package/src/PruneStaleHistory.res.mjs +63 -37
  35. package/src/Rollback.res +68 -45
  36. package/src/Rollback.res.mjs +38 -19
  37. package/src/RollbackCommit.res +7 -9
  38. package/src/RollbackCommit.res.mjs +5 -6
  39. package/src/SafeCheckpointTracking.res +11 -11
  40. package/src/SafeCheckpointTracking.res.mjs +7 -6
  41. package/src/SimulateItems.res +33 -18
  42. package/src/SimulateItems.res.mjs +26 -22
  43. package/src/Sink.res +38 -26
  44. package/src/Sink.res.mjs +21 -16
  45. package/src/TestIndexer.res +5 -5
  46. package/src/TestIndexer.res.mjs +1 -1
  47. package/src/Utils.res +170 -0
  48. package/src/Utils.res.mjs +153 -0
  49. package/src/Writing.res +2 -2
  50. package/src/Writing.res.mjs +1 -1
  51. package/src/bindings/ClickHouse.res +366 -698
  52. package/src/bindings/ClickHouse.res.mjs +371 -421
  53. package/src/bindings/ClickHouseSink.res +337 -0
  54. package/src/bindings/ClickHouseSink.res.mjs +246 -0
  55. package/src/bindings/EventSource.res +8 -2
  56. package/src/bindings/NodeJs.res +6 -0
  57. package/src/bindings/Vitest.res +23 -0
  58. package/src/bindings/Vitest.res.mjs +3 -0
  59. package/src/bindings/WebSocket.res +9 -10
  60. package/src/db/CheckpointBounds.res +53 -0
  61. package/src/db/CheckpointBounds.res.mjs +44 -0
  62. package/src/db/EntityFilter.res +36 -17
  63. package/src/db/EntityFilter.res.mjs +25 -14
  64. package/src/db/EntityHistory.res +47 -22
  65. package/src/db/EntityHistory.res.mjs +19 -12
  66. package/src/db/InternalTable.res +84 -35
  67. package/src/db/InternalTable.res.mjs +56 -23
  68. package/src/db/RollbackFloors.res +55 -0
  69. package/src/db/RollbackFloors.res.mjs +92 -0
  70. package/src/db/Table.res +22 -4
  71. package/src/db/Table.res.mjs +30 -9
  72. package/src/sources/Evm.res +2 -0
  73. package/src/sources/Evm.res.mjs +2 -0
  74. package/src/sources/EvmHyperSyncSource.res +3 -3
  75. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  76. package/src/sources/EvmRpcWs.res +124 -0
  77. package/src/sources/EvmRpcWs.res.mjs +117 -0
  78. package/src/sources/Fuel.res +2 -0
  79. package/src/sources/Fuel.res.mjs +2 -0
  80. package/src/sources/FuelHyperSyncSource.res +1 -1
  81. package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
  82. package/src/sources/HeightFeed.res +568 -0
  83. package/src/sources/HeightFeed.res.mjs +452 -0
  84. package/src/sources/HeightStream.res +257 -0
  85. package/src/sources/HeightStream.res.mjs +194 -0
  86. package/src/sources/HyperSync.res +5 -0
  87. package/src/sources/HyperSync.res.mjs +3 -0
  88. package/src/sources/HyperSync.resi +4 -0
  89. package/src/sources/HyperSyncSSE.res +49 -0
  90. package/src/sources/HyperSyncSSE.res.mjs +79 -0
  91. package/src/sources/RpcSource.res +1 -1
  92. package/src/sources/RpcSource.res.mjs +2 -2
  93. package/src/sources/Source.res +54 -1
  94. package/src/sources/Source.res.mjs +25 -0
  95. package/src/sources/SourceManager.res +220 -227
  96. package/src/sources/SourceManager.res.mjs +139 -164
  97. package/src/sources/SourceManager.resi +8 -0
  98. package/src/sources/Svm.res +2 -0
  99. package/src/sources/Svm.res.mjs +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +70 -127
  101. package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
  102. package/src/sources/SvmHyperSyncSource.res +18 -15
  103. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  104. package/svm.schema.json +1 -75
  105. package/src/MemoryStorage.res +0 -729
  106. package/src/MemoryStorage.res.mjs +0 -586
  107. package/src/sources/HyperSyncHeightStream.res +0 -129
  108. package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
  109. package/src/sources/RpcWebSocketHeightStream.res +0 -175
  110. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
@@ -0,0 +1,53 @@
1
+ // A checkpoint id per chain that a query compares each chain's rows against. A
2
+ // schema with a cross-chain entity gets one bound for every chain, since a reorg
3
+ // on any chain reaches every chain's rows; otherwise each chain has its own, and
4
+ // a chain absent from them is left alone.
5
+ type t =
6
+ | EveryChain(Internal.checkpointId)
7
+ | PerChain(array<(ChainId.t, Internal.checkpointId)>)
8
+
9
+ // The pieces a query splices in to read its bound as `checkpointId`: a SELECT
10
+ // joins the bounds next to its table with `join`, a DELETE names them with
11
+ // `using` and matches the chain with `usingMatch` in its WHERE.
12
+ type sql = {
13
+ join: string,
14
+ using: string,
15
+ usingMatch: string,
16
+ checkpointId: string,
17
+ }
18
+
19
+ // The chain ids are cast to the widest integer type rather than the column's
20
+ // own: nothing indexes the join, so the values only have to compare.
21
+ let relation = `unnest($1::${(Postgres.BigInt :> string)}[],$2::${(Postgres.BigInt :> string)}[]) AS envio_bounds(chain_id, checkpoint_id)`
22
+
23
+ let sql = (bounds: t, ~chainIdColumn: option<string>, ~tableRef: string): sql =>
24
+ switch (bounds, chainIdColumn) {
25
+ | (EveryChain(_), _) => {join: "", using: "", usingMatch: "", checkpointId: "$1"}
26
+ | (PerChain(_), Some(column)) =>
27
+ let chainMatch = `envio_bounds.chain_id = ${tableRef}."${column}"`
28
+ {
29
+ join: ` JOIN ${relation} ON ${chainMatch}`,
30
+ using: ` USING ${relation}`,
31
+ usingMatch: ` AND ${chainMatch}`,
32
+ checkpointId: "envio_bounds.checkpoint_id",
33
+ }
34
+ | (PerChain(_), None) =>
35
+ JsError.throwWithMessage(
36
+ "Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.",
37
+ )
38
+ }
39
+
40
+ // Bound in the order `sql` reads them: the one bound as `$1`, or the chains and
41
+ // their bounds as the two parallel arrays the relation unnests.
42
+ let params = (bounds: t): unknown =>
43
+ switch bounds {
44
+ | EveryChain(checkpointId) =>
45
+ [checkpointId->BigInt.toString]->(Utils.magic: array<string> => unknown)
46
+ | PerChain(byChain) =>
47
+ [
48
+ byChain->Array.map(((chainId, _)) => chainId)->(Utils.magic: array<ChainId.t> => unknown),
49
+ byChain
50
+ ->Array.map(((_, checkpointId)) => checkpointId->BigInt.toString)
51
+ ->(Utils.magic: array<string> => unknown),
52
+ ]->(Utils.magic: array<unknown> => unknown)
53
+ }
@@ -0,0 +1,44 @@
1
+ // Generated by ReScript, PLEASE EDIT WITH CARE
2
+
3
+ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
4
+
5
+ let relation = `unnest($1::` + "BIGINT" + `[],$2::` + "BIGINT" + `[]) AS envio_bounds(chain_id, checkpoint_id)`;
6
+
7
+ function sql(bounds, chainIdColumn, tableRef) {
8
+ if (bounds.TAG === "EveryChain") {
9
+ return {
10
+ join: "",
11
+ using: "",
12
+ usingMatch: "",
13
+ checkpointId: "$1"
14
+ };
15
+ }
16
+ if (chainIdColumn === undefined) {
17
+ return Stdlib_JsError.throwWithMessage("Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.");
18
+ }
19
+ let chainMatch = `envio_bounds.chain_id = ` + tableRef + `."` + chainIdColumn + `"`;
20
+ return {
21
+ join: ` JOIN ` + relation + ` ON ` + chainMatch,
22
+ using: ` USING ` + relation,
23
+ usingMatch: ` AND ` + chainMatch,
24
+ checkpointId: "envio_bounds.checkpoint_id"
25
+ };
26
+ }
27
+
28
+ function params(bounds) {
29
+ if (bounds.TAG === "EveryChain") {
30
+ return [bounds._0.toString()];
31
+ }
32
+ let byChain = bounds._0;
33
+ return [
34
+ byChain.map(param => param[0]),
35
+ byChain.map(param => param[1].toString())
36
+ ];
37
+ }
38
+
39
+ export {
40
+ relation,
41
+ sql,
42
+ params,
43
+ }
44
+ /* No side effect */
@@ -23,34 +23,53 @@ module FieldValue = {
23
23
  //and serialize the types without parsing/wrapping them
24
24
  type t = option<tNonOptional>
25
25
 
26
+ // A Uint8Array is an object like a BigDecimal, so the unboxed variant can't
27
+ // tell them apart on its own.
28
+ let asBytes = (value: t) => value->(Utils.magic: t => unknown)->Utils.Bytes.asUint8Array
29
+
26
30
  let toString = (value: t) =>
27
- switch value {
28
- | Some(v) => v->toString
29
- | None => "undefined"
31
+ switch (value, value->asBytes) {
32
+ | (_, Some(bytes)) => bytes->Utils.Bytes.toHex
33
+ | (Some(v), None) => v->toString
34
+ | (None, None) => "undefined"
30
35
  }
31
36
 
32
37
  external castFrom: 'a => t = "%identity"
33
38
 
34
- let eq = (a, b) =>
35
- switch (a, b) {
39
+ let compareWith = (a, b, ~bigDecimal, ~bytes, ~fallback) =>
40
+ switch (a, b, a->asBytes, b->asBytes) {
41
+ | (_, _, Some(bytesA), Some(bytesB)) => bytes(Utils.Bytes.compare(bytesA, bytesB))
36
42
  //For big decimal use custom equals operator otherwise let Caml_obj.equal do its magic
37
- | (Some(BigDecimal(bdA)), Some(BigDecimal(bdB))) => BigDecimal.equals(bdA, bdB)
38
- | (a, b) => a == b
43
+ | (Some(BigDecimal(bdA)), Some(BigDecimal(bdB)), _, _) => bigDecimal(bdA, bdB)
44
+ | (a, b, _, _) => fallback(a, b)
39
45
  }
40
46
 
47
+ let eq = (a, b) =>
48
+ compareWith(
49
+ a,
50
+ b,
51
+ ~bigDecimal=BigDecimal.equals,
52
+ ~bytes=order => order === 0.,
53
+ ~fallback=(a, b) => a == b,
54
+ )
55
+
41
56
  let gt = (a, b) =>
42
- switch (a, b) {
43
- //For big decimal use custom equals operator otherwise let Caml_obj.equal do its magic
44
- | (Some(BigDecimal(bdA)), Some(BigDecimal(bdB))) => BigDecimal.gt(bdA, bdB)
45
- | (a, b) => a > b
46
- }
57
+ compareWith(
58
+ a,
59
+ b,
60
+ ~bigDecimal=BigDecimal.gt,
61
+ ~bytes=order => order > 0.,
62
+ ~fallback=(a, b) => a > b,
63
+ )
47
64
 
48
65
  let lt = (a, b) =>
49
- switch (a, b) {
50
- //For big decimal use custom equals operator otherwise let Caml_obj.equal do its magic
51
- | (Some(BigDecimal(bdA)), Some(BigDecimal(bdB))) => BigDecimal.lt(bdA, bdB)
52
- | (a, b) => a < b
53
- }
66
+ compareWith(
67
+ a,
68
+ b,
69
+ ~bigDecimal=BigDecimal.lt,
70
+ ~bytes=order => order < 0.,
71
+ ~fallback=(a, b) => a < b,
72
+ )
54
73
  }
55
74
 
56
75
  // The And case requires at least one nested filter (storage throws otherwise),
@@ -1,6 +1,7 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as Table from "./Table.res.mjs";
4
+ import * as Utils from "../Utils.res.mjs";
4
5
  import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
5
6
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
6
7
  import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
@@ -26,40 +27,50 @@ function toString(tNonOptional) {
26
27
  }
27
28
  }
28
29
 
30
+ function asBytes(value) {
31
+ return Utils.Bytes.asUint8Array(value);
32
+ }
33
+
29
34
  function toString$1(value) {
30
- if (value !== undefined) {
35
+ let match = Utils.Bytes.asUint8Array(value);
36
+ if (match !== undefined) {
37
+ return Utils.Bytes.toHex(match);
38
+ } else if (value !== undefined) {
31
39
  return toString(value);
32
40
  } else {
33
41
  return "undefined";
34
42
  }
35
43
  }
36
44
 
37
- function eq(a, b) {
45
+ function compareWith(a, b, bigDecimal, bytes, fallback) {
46
+ let match = Utils.Bytes.asUint8Array(a);
47
+ let match$1 = Utils.Bytes.asUint8Array(b);
48
+ if (match !== undefined && match$1 !== undefined) {
49
+ return bytes(Utils.Bytes.compare(match, match$1));
50
+ }
38
51
  if (typeof a === "object" && !Array.isArray(a) && typeof b === "object" && !Array.isArray(b)) {
39
- return a.isEqualTo(b);
52
+ return bigDecimal(a, b);
40
53
  } else {
41
- return Primitive_object.equal(a, b);
54
+ return fallback(a, b);
42
55
  }
43
56
  }
44
57
 
58
+ function eq(a, b) {
59
+ return compareWith(a, b, (prim0, prim1) => prim0.isEqualTo(prim1), order => order === 0, Primitive_object.equal);
60
+ }
61
+
45
62
  function gt(a, b) {
46
- if (typeof a === "object" && !Array.isArray(a) && typeof b === "object" && !Array.isArray(b)) {
47
- return a.isGreaterThan(b);
48
- } else {
49
- return Primitive_object.greaterthan(a, b);
50
- }
63
+ return compareWith(a, b, (prim0, prim1) => prim0.isGreaterThan(prim1), order => order > 0, Primitive_object.greaterthan);
51
64
  }
52
65
 
53
66
  function lt(a, b) {
54
- if (typeof a === "object" && !Array.isArray(a) && typeof b === "object" && !Array.isArray(b)) {
55
- return a.isLessThan(b);
56
- } else {
57
- return Primitive_object.lessthan(a, b);
58
- }
67
+ return compareWith(a, b, (prim0, prim1) => prim0.isLessThan(prim1), order => order < 0, Primitive_object.lessthan);
59
68
  }
60
69
 
61
70
  let FieldValue = {
71
+ asBytes: asBytes,
62
72
  toString: toString$1,
73
+ compareWith: compareWith,
63
74
  eq: eq,
64
75
  gt: gt,
65
76
  lt: lt
@@ -30,11 +30,20 @@ let unsafeCheckpointIdSchema =
30
30
  serializer: bigint => bigint->BigInt.toString,
31
31
  })
32
32
 
33
- let makeSetUpdateSchema = (~idSchema: S.t<EntityId.t>, entitySchema: S.t<'entity>): S.t<
34
- Change.t<'entity>,
35
- > => {
33
+ // `chainIdTag` carries the (column, chain id) of a per-chain entity whose rows
34
+ // all belong to one chain: the column is then a constant of the schema rather
35
+ // than a field read off every entity.
36
+ let makeSetUpdateSchema = (
37
+ ~idSchema: S.t<EntityId.t>,
38
+ ~chainIdTag: option<(string, ChainId.t)>=?,
39
+ entitySchema: S.t<'entity>,
40
+ ): S.t<Change.t<'entity>> => {
36
41
  S.object(s => {
37
42
  s.tag(changeFieldName, RowAction.SET)
43
+ switch chainIdTag {
44
+ | Some((column, chainId)) => s.tag(column, chainId)
45
+ | None => ()
46
+ }
38
47
  Change.Set({
39
48
  checkpointId: s.field(checkpointIdFieldName, unsafeCheckpointIdSchema),
40
49
  entityId: s.field(Table.idFieldName, idSchema),
@@ -91,27 +100,38 @@ let makeKeyMatch = (~chainIdColumn, ~left, ~right) =>
91
100
  ->Array.map(column => `${left}.${column} = ${right}.${column}`)
92
101
  ->Array.joinUnsafe(" AND ")
93
102
 
94
- let makePruneStaleEntityHistoryQuery = (~entityName, ~entityIndex, ~pgSchema, ~chainIdColumn) => {
103
+ let makePruneStaleEntityHistoryQuery = (
104
+ ~entityName,
105
+ ~entityIndex,
106
+ ~pgSchema,
107
+ ~chainIdColumn,
108
+ ~safeCheckpoints: CheckpointBounds.t,
109
+ ) => {
95
110
  let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
96
111
  let keyColumns = makeKeyColumns(~chainIdColumn)
97
112
  let anchorKeys = keyColumns->Array.map(column => `t.${column}`)->Array.joinUnsafe(", ")
113
+ let bounds = safeCheckpoints->CheckpointBounds.sql(~chainIdColumn, ~tableRef="t")
98
114
 
99
115
  // Whether a key still has a row above the safe checkpoint is an aggregate
100
116
  // over the same groups as the anchor, so it's computed in the one pass
101
- // rather than as a per-row correlated lookup.
102
- // `d.envio_checkpoint_id <= $1` is implied by the rest of the predicate — it
117
+ // rather than as a per-row correlated lookup. The DELETE's `<=` on the row
103
118
  // is there to keep the rows above the safe checkpoint out of the join.
119
+ //
120
+ // Per-chain bounds are joined in rather than run as a statement each: the
121
+ // anchors aggregate the whole table however narrow the bound is, and history
122
+ // carries no index to narrow the scan with.
104
123
  `WITH anchors AS (
105
124
  SELECT ${anchorKeys},
106
- MAX(t.${checkpointIdFieldName}) FILTER (WHERE t.${checkpointIdFieldName} <= $1) AS keep_checkpoint_id,
107
- bool_or(t.${checkpointIdFieldName} > $1) AS has_above
108
- FROM ${historyTableRef} t
125
+ MAX(t.${checkpointIdFieldName}) FILTER (WHERE t.${checkpointIdFieldName} <= ${bounds.checkpointId}) AS keep_checkpoint_id,
126
+ bool_or(t.${checkpointIdFieldName} > ${bounds.checkpointId}) AS has_above,
127
+ MIN(${bounds.checkpointId}) AS safe_checkpoint_id
128
+ FROM ${historyTableRef} t${bounds.join}
109
129
  GROUP BY ${anchorKeys}
110
130
  )
111
131
  DELETE FROM ${historyTableRef} d
112
132
  USING anchors a
113
133
  WHERE ${makeKeyMatch(~chainIdColumn, ~left="d", ~right="a")}
114
- AND d.${checkpointIdFieldName} <= $1
134
+ AND d.${checkpointIdFieldName} <= a.safe_checkpoint_id
115
135
  AND (d.${checkpointIdFieldName} < a.keep_checkpoint_id OR NOT a.has_above);`
116
136
  }
117
137
 
@@ -121,13 +141,18 @@ let pruneStaleEntityHistory = (
121
141
  ~entityIndex,
122
142
  ~pgSchema,
123
143
  ~chainIdColumn,
124
- ~safeCheckpointId,
125
- ): promise<unit> => {
144
+ ~safeCheckpoints,
145
+ ): promise<unit> =>
126
146
  sql->Postgres.preparedUnsafe(
127
- makePruneStaleEntityHistoryQuery(~entityName, ~entityIndex, ~pgSchema, ~chainIdColumn),
128
- [safeCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
147
+ makePruneStaleEntityHistoryQuery(
148
+ ~entityName,
149
+ ~entityIndex,
150
+ ~pgSchema,
151
+ ~chainIdColumn,
152
+ ~safeCheckpoints,
153
+ ),
154
+ safeCheckpoints->CheckpointBounds.params,
129
155
  )
130
- }
131
156
 
132
157
  // If an entity doesn't have a history before the update
133
158
  // we create it automatically with envio_checkpoint_id 0
@@ -173,7 +198,7 @@ let backfillHistory = (
173
198
  ~ids: array<EntityId.t>,
174
199
  ) => {
175
200
  let idPgType = table->Table.getIdPgFieldType(~pgSchema)
176
- let chainIdColumn = table->Table.getChainIdField->Option.map(Table.getPgDbFieldName)
201
+ let chainIdColumn = table->Table.getPgChainIdColumn
177
202
  let params = [table->Table.encodeIdsToJson(ids)->(Utils.magic: JSON.t => unknown)]
178
203
  sql
179
204
  ->Postgres.preparedUnsafe(
@@ -195,15 +220,15 @@ let rollback = (
195
220
  ~pgSchema,
196
221
  ~entityName,
197
222
  ~entityIndex,
198
- ~rollbackTargetCheckpointId: Internal.checkpointId,
223
+ ~chainIdColumn,
224
+ ~floors: RollbackFloors.t,
199
225
  ) => {
226
+ let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
227
+ let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef=historyTableRef)
200
228
  sql
201
229
  ->Postgres.preparedUnsafe(
202
- `DELETE FROM "${pgSchema}"."${historyTableName(
203
- ~entityName,
204
- ~entityIndex,
205
- )}" WHERE "${checkpointIdFieldName}" > $1;`,
206
- [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
230
+ `DELETE FROM ${historyTableRef}${bounds.using} WHERE "${checkpointIdFieldName}" > ${bounds.checkpointId}${bounds.usingMatch};`,
231
+ floors.floors->CheckpointBounds.params,
207
232
  )
208
233
  ->Utils.Promise.ignoreValue
209
234
  }
@@ -3,7 +3,7 @@
3
3
  import * as Table from "./Table.res.mjs";
4
4
  import * as ChainId from "../ChainId.res.mjs";
5
5
  import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
6
- import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
6
+ import * as CheckpointBounds from "./CheckpointBounds.res.mjs";
7
7
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
8
8
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
9
9
 
@@ -50,9 +50,12 @@ let unsafeCheckpointIdSchema = S$RescriptSchema.transform(S$RescriptSchema.setNa
50
50
  s: bigint => bigint.toString()
51
51
  }));
52
52
 
53
- function makeSetUpdateSchema(idSchema, entitySchema) {
53
+ function makeSetUpdateSchema(idSchema, chainIdTag, entitySchema) {
54
54
  return S$RescriptSchema.object(s => {
55
55
  s.tag(changeFieldName, "SET");
56
+ if (chainIdTag !== undefined) {
57
+ s.tag(chainIdTag[0], chainIdTag[1]);
58
+ }
56
59
  return {
57
60
  type: "SET",
58
61
  entityId: s.f(Table.idFieldName, idSchema),
@@ -83,26 +86,28 @@ function makeKeyMatch(chainIdColumn, left, right) {
83
86
  return makeKeyColumns(chainIdColumn).map(column => left + `.` + column + ` = ` + right + `.` + column).join(" AND ");
84
87
  }
85
88
 
86
- function makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn) {
89
+ function makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints) {
87
90
  let historyTableRef = `"` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `"`;
88
91
  let keyColumns = makeKeyColumns(chainIdColumn);
89
92
  let anchorKeys = keyColumns.map(column => `t.` + column).join(", ");
93
+ let bounds = CheckpointBounds.sql(safeCheckpoints, chainIdColumn, "t");
90
94
  return `WITH anchors AS (
91
95
  SELECT ` + anchorKeys + `,
92
- MAX(t.` + checkpointIdFieldName + `) FILTER (WHERE t.` + checkpointIdFieldName + ` <= $1) AS keep_checkpoint_id,
93
- bool_or(t.` + checkpointIdFieldName + ` > $1) AS has_above
94
- FROM ` + historyTableRef + ` t
96
+ MAX(t.` + checkpointIdFieldName + `) FILTER (WHERE t.` + checkpointIdFieldName + ` <= ` + bounds.checkpointId + `) AS keep_checkpoint_id,
97
+ bool_or(t.` + checkpointIdFieldName + ` > ` + bounds.checkpointId + `) AS has_above,
98
+ MIN(` + bounds.checkpointId + `) AS safe_checkpoint_id
99
+ FROM ` + historyTableRef + ` t` + bounds.join + `
95
100
  GROUP BY ` + anchorKeys + `
96
101
  )
97
102
  DELETE FROM ` + historyTableRef + ` d
98
103
  USING anchors a
99
104
  WHERE ` + makeKeyMatch(chainIdColumn, "d", "a") + `
100
- AND d.` + checkpointIdFieldName + ` <= $1
105
+ AND d.` + checkpointIdFieldName + ` <= a.safe_checkpoint_id
101
106
  AND (d.` + checkpointIdFieldName + ` < a.keep_checkpoint_id OR NOT a.has_above);`;
102
107
  }
103
108
 
104
- function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpointId) {
105
- return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn), [safeCheckpointId.toString()], {prepare: true});
109
+ function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints) {
110
+ return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints), CheckpointBounds.params(safeCheckpoints), {prepare: true});
106
111
  }
107
112
 
108
113
  function makeBackfillHistoryQuery(pgSchema, entityName, entityIndex, idPgType, chainIdColumn, chainId) {
@@ -125,13 +130,15 @@ FROM missing_history;`;
125
130
 
126
131
  function backfillHistory(sql, pgSchema, table, entityIndex, chainId, ids) {
127
132
  let idPgType = Table.getIdPgFieldType(table, pgSchema);
128
- let chainIdColumn = Stdlib_Option.map(Table.getChainIdField(table), Table.getPgDbFieldName);
133
+ let chainIdColumn = Table.getPgChainIdColumn(table);
129
134
  let params = [Table.encodeIdsToJson(table, ids)];
130
135
  return sql.unsafe(makeBackfillHistoryQuery(pgSchema, table.tableName, entityIndex, idPgType, chainIdColumn, chainId), params, {prepare: true});
131
136
  }
132
137
 
133
- function rollback(sql, pgSchema, entityName, entityIndex, rollbackTargetCheckpointId) {
134
- return sql.unsafe(`DELETE FROM "` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `" WHERE "` + checkpointIdFieldName + `" > $1;`, [rollbackTargetCheckpointId.toString()], {prepare: true});
138
+ function rollback(sql, pgSchema, entityName, entityIndex, chainIdColumn, floors) {
139
+ let historyTableRef = `"` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `"`;
140
+ let bounds = CheckpointBounds.sql(floors.floors, chainIdColumn, historyTableRef);
141
+ return sql.unsafe(`DELETE FROM ` + historyTableRef + bounds.using + ` WHERE "` + checkpointIdFieldName + `" > ` + bounds.checkpointId + bounds.usingMatch + `;`, CheckpointBounds.params(floors.floors), {prepare: true});
135
142
  }
136
143
 
137
144
  let checkpointIdFieldType = "UInt64";
@@ -556,18 +556,64 @@ module Checkpoints = {
556
556
 
557
557
  let initialCheckpointId = 0n
558
558
 
559
- let table = mkTable(
560
- "envio_checkpoints",
561
- ~fields=[
562
- mkField((#id: field :> string), UInt64, ~fieldSchema=S.bigint, ~isPrimaryKey),
563
- mkField((#chain_id: field :> string), ChainId, ~fieldSchema=ChainId.schema),
564
- mkField((#block_number: field :> string), Int32, ~fieldSchema=S.int),
565
- mkField((#block_hash: field :> string), String, ~fieldSchema=S.null(S.string), ~isNullable),
566
- mkField((#events_processed: field :> string), Int32, ~fieldSchema=S.int),
567
- ],
568
- )
559
+ // One definition per column, carrying what each storage needs: the field
560
+ // itself, the type ClickHouse gives it where that differs from Postgres, and
561
+ // where a batch keeps the column's values.
562
+ type column = {
563
+ field: fieldOrDerived,
564
+ clickHouseFieldType: fieldType,
565
+ valuesOf: Batch.t => array<unknown>,
566
+ }
567
+
568
+ let columns: array<column> = [
569
+ {
570
+ field: mkField((#id: field :> string), UInt64, ~fieldSchema=S.bigint, ~isPrimaryKey),
571
+ clickHouseFieldType: UInt64,
572
+ valuesOf: batch => batch.checkpointIds->(Utils.magic: array<bigint> => array<unknown>),
573
+ },
574
+ {
575
+ field: mkField((#chain_id: field :> string), ChainId, ~fieldSchema=ChainId.schema),
576
+ clickHouseFieldType: ChainId,
577
+ valuesOf: batch =>
578
+ batch.checkpointChainIds->(Utils.magic: array<ChainId.t> => array<unknown>),
579
+ },
580
+ {
581
+ field: mkField((#block_number: field :> string), Int32, ~fieldSchema=S.int),
582
+ clickHouseFieldType: Int32,
583
+ valuesOf: batch => batch.checkpointBlockNumbers->(Utils.magic: array<int> => array<unknown>),
584
+ },
585
+ {
586
+ field: mkField(
587
+ (#block_hash: field :> string),
588
+ String,
589
+ ~fieldSchema=S.null(S.string),
590
+ ~isNullable,
591
+ ),
592
+ clickHouseFieldType: String,
593
+ valuesOf: batch =>
594
+ batch.checkpointBlockHashes->(Utils.magic: array<Null.t<string>> => array<unknown>),
595
+ },
596
+ {
597
+ field: mkField((#events_processed: field :> string), Int32, ~fieldSchema=S.int),
598
+ // A count of every event a chain has processed outgrows an Int32 where
599
+ // Postgres keeps one, and ClickHouse has the id's width to spare.
600
+ clickHouseFieldType: UInt64,
601
+ valuesOf: batch =>
602
+ batch.checkpointEventsProcessed->(Utils.magic: array<int> => array<unknown>),
603
+ },
604
+ ]
605
+
606
+ let table = mkTable("envio_checkpoints", ~fields=columns->Array.map(({field}) => field))
569
607
 
570
608
  let makeGetReorgCheckpointsQuery = (~pgSchema): string => {
609
+ // The safe_block checkpoint itself is included, so it can be used for safe
610
+ // checkpoint tracking.
611
+ //
612
+ // Ordered by id, which within a chain is block order: both consumers of
613
+ // these rows — the safe-checkpoint scan and the block-store seed — read them
614
+ // as ascending. Physical row order can't stand in for that, since a rollback
615
+ // frees space that later checkpoints are written back into.
616
+ //
571
617
  // Use CTE to pre-filter chains and compute safe_block once per chain
572
618
  // This is faster because:
573
619
  // 1. Chains table is small, so filtering it first is cheap
@@ -590,7 +636,8 @@ FROM "${pgSchema}"."${table.tableName}" cp
590
636
  INNER JOIN reorg_chains rc
591
637
  ON cp."${(#chain_id: field :> string)}" = rc.id
592
638
  WHERE cp."${(#block_hash: field :> string)}" IS NOT NULL
593
- AND cp."${(#block_number: field :> string)}" >= rc.safe_block;` // Include safe_block checkpoint to use it for safe checkpoint tracking
639
+ AND cp."${(#block_number: field :> string)}" >= rc.safe_block
640
+ ORDER BY cp."${(#id: field :> string)}";`
594
641
  }
595
642
 
596
643
  let makeCommitedCheckpointIdQuery = (~pgSchema) => {
@@ -635,27 +682,33 @@ SELECT * FROM unnest($1::${(BigInt: Postgres.columnType :> string)}[],$2::${chai
635
682
  ->Utils.Promise.ignoreValue
636
683
  }
637
684
 
638
- let rollback = (sql, ~pgSchema, ~rollbackTargetCheckpointId: Internal.checkpointId) => {
685
+ // Optional to match the entity tables', where a cross-chain entity has none.
686
+ let chainIdColumn = Some((#chain_id: field :> string))
687
+
688
+ let rollback = (sql, ~pgSchema, ~floors: RollbackFloors.t) => {
689
+ let tableRef = `"${pgSchema}"."${table.tableName}"`
690
+ let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef)
639
691
  sql
640
692
  ->Postgres.preparedUnsafe(
641
- `DELETE FROM "${pgSchema}"."${table.tableName}" WHERE "${(#id: field :> string)}" > $1;`,
642
- [rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
693
+ `DELETE FROM ${tableRef}${bounds.using} WHERE "${(#id: field :> string)}" > ${bounds.checkpointId}${bounds.usingMatch};`,
694
+ floors.floors->CheckpointBounds.params,
643
695
  )
644
696
  ->Utils.Promise.ignoreValue
645
697
  }
646
698
 
647
- let makePruneStaleCheckpointsQuery = (~pgSchema) => {
648
- `DELETE FROM "${pgSchema}"."${table.tableName}" WHERE "${(#id: field :> string)}" < $1;`
699
+ let makePruneStaleCheckpointsQuery = (~pgSchema, ~safeCheckpoints: CheckpointBounds.t) => {
700
+ let tableRef = `"${pgSchema}"."${table.tableName}"`
701
+ let bounds = safeCheckpoints->CheckpointBounds.sql(~chainIdColumn, ~tableRef)
702
+ `DELETE FROM ${tableRef}${bounds.using} WHERE "${(#id: field :> string)}" < ${bounds.checkpointId}${bounds.usingMatch};`
649
703
  }
650
704
 
651
- let pruneStaleCheckpoints = (sql, ~pgSchema, ~safeCheckpointId: bigint) => {
705
+ let pruneStaleCheckpoints = (sql, ~pgSchema, ~safeCheckpoints) =>
652
706
  sql
653
707
  ->Postgres.preparedUnsafe(
654
- makePruneStaleCheckpointsQuery(~pgSchema),
655
- [safeCheckpointId->BigInt.toString]->Obj.magic,
708
+ makePruneStaleCheckpointsQuery(~pgSchema, ~safeCheckpoints),
709
+ safeCheckpoints->CheckpointBounds.params,
656
710
  )
657
711
  ->Utils.Promise.ignoreValue
658
- }
659
712
 
660
713
  let makeGetRollbackTargetCheckpointQuery = (~pgSchema) => {
661
714
  `SELECT "${(#id: field :> string)}" FROM "${pgSchema}"."${table.tableName}"
@@ -684,25 +737,22 @@ LIMIT 1;`
684
737
  })
685
738
  }
686
739
 
687
- let makeGetRollbackProgressDiffQuery = (~pgSchema) => {
740
+ let makeGetRollbackProgressDiffQuery = (~pgSchema, ~floors: RollbackFloors.t) => {
741
+ let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef="t")
688
742
  `SELECT
689
- "${(#chain_id: field :> string)}"::float8 as "${(#chain_id: field :> string)}",
690
- SUM("${(#events_processed: field :> string)}") as events_processed_diff,
691
- MIN("${(#block_number: field :> string)}") - 1 as new_progress_block_number
692
- FROM "${pgSchema}"."${table.tableName}"
693
- WHERE "${(#id: field :> string)}" > $1
694
- GROUP BY "${(#chain_id: field :> string)}";`
743
+ t."${(#chain_id: field :> string)}"::float8 as "${(#chain_id: field :> string)}",
744
+ SUM(t."${(#events_processed: field :> string)}") as events_processed_diff,
745
+ MIN(t."${(#block_number: field :> string)}") - 1 as new_progress_block_number
746
+ FROM "${pgSchema}"."${table.tableName}" t${bounds.join}
747
+ WHERE t."${(#id: field :> string)}" > ${bounds.checkpointId}
748
+ GROUP BY t."${(#chain_id: field :> string)}";`
695
749
  }
696
750
 
697
- let getRollbackProgressDiff = (
698
- sql,
699
- ~pgSchema,
700
- ~rollbackTargetCheckpointId: Internal.checkpointId,
701
- ) => {
751
+ let getRollbackProgressDiff = (sql, ~pgSchema, ~floors: RollbackFloors.t) =>
702
752
  sql
703
753
  ->Postgres.preparedUnsafe(
704
- makeGetRollbackProgressDiffQuery(~pgSchema),
705
- [rollbackTargetCheckpointId->BigInt.toString]->Obj.magic,
754
+ makeGetRollbackProgressDiffQuery(~pgSchema, ~floors),
755
+ floors.floors->CheckpointBounds.params,
706
756
  )
707
757
  ->(
708
758
  Utils.magic: promise<unknown> => promise<
@@ -713,7 +763,6 @@ GROUP BY "${(#chain_id: field :> string)}";`
713
763
  }>,
714
764
  >
715
765
  )
716
- }
717
766
  }
718
767
 
719
768
  module RawEvents = {