envio 3.9.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -0
- package/fuel.schema.json +26 -0
- package/index.d.ts +35 -3
- package/package.json +6 -7
- package/src/Batch.res +9 -0
- package/src/ChainState.res +35 -39
- package/src/ChainState.res.mjs +14 -11
- package/src/ChainState.resi +15 -7
- package/src/Config.res +52 -44
- package/src/Config.res.mjs +51 -17
- package/src/Core.res +4 -0
- package/src/CrossChainState.res +15 -23
- package/src/CrossChainState.res.mjs +10 -18
- package/src/CrossChainState.resi +4 -1
- package/src/Ecosystem.res +7 -2
- package/src/Envio.res +6 -4
- package/src/EventConfigBuilder.res +169 -26
- package/src/EventConfigBuilder.res.mjs +124 -15
- package/src/HandlerRegister.res +21 -4
- package/src/HandlerRegister.res.mjs +24 -5
- package/src/InMemoryStore.res +9 -7
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/IndexerState.res +43 -6
- package/src/IndexerState.res.mjs +36 -7
- package/src/IndexerState.resi +4 -4
- package/src/Internal.res +16 -17
- package/src/Metrics.res +72 -6
- package/src/Metrics.res.mjs +30 -2
- package/src/Persistence.res +36 -30
- package/src/Persistence.res.mjs +1 -20
- package/src/PgStorage.res +104 -56
- package/src/PgStorage.res.mjs +57 -35
- package/src/PruneStaleHistory.res +85 -52
- package/src/PruneStaleHistory.res.mjs +63 -37
- package/src/Rollback.res +68 -45
- package/src/Rollback.res.mjs +38 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SafeCheckpointTracking.res +11 -11
- package/src/SafeCheckpointTracking.res.mjs +7 -6
- package/src/SimulateItems.res +33 -18
- package/src/SimulateItems.res.mjs +26 -22
- package/src/Sink.res +38 -26
- package/src/Sink.res.mjs +21 -16
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Utils.res +170 -0
- package/src/Utils.res.mjs +153 -0
- package/src/Writing.res +2 -2
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +366 -698
- package/src/bindings/ClickHouse.res.mjs +371 -421
- package/src/bindings/ClickHouseSink.res +337 -0
- package/src/bindings/ClickHouseSink.res.mjs +246 -0
- package/src/bindings/EventSource.res +8 -2
- package/src/bindings/NodeJs.res +6 -0
- package/src/bindings/Vitest.res +23 -0
- package/src/bindings/Vitest.res.mjs +3 -0
- package/src/bindings/WebSocket.res +9 -10
- package/src/db/CheckpointBounds.res +53 -0
- package/src/db/CheckpointBounds.res.mjs +44 -0
- package/src/db/EntityFilter.res +36 -17
- package/src/db/EntityFilter.res.mjs +25 -14
- package/src/db/EntityHistory.res +47 -22
- package/src/db/EntityHistory.res.mjs +19 -12
- package/src/db/InternalTable.res +84 -35
- package/src/db/InternalTable.res.mjs +56 -23
- package/src/db/RollbackFloors.res +55 -0
- package/src/db/RollbackFloors.res.mjs +92 -0
- package/src/db/Table.res +22 -4
- package/src/db/Table.res.mjs +30 -9
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/EvmHyperSyncSource.res +3 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcWs.res +124 -0
- package/src/sources/EvmRpcWs.res.mjs +117 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/FuelHyperSyncSource.res +1 -1
- package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
- package/src/sources/HeightFeed.res +568 -0
- package/src/sources/HeightFeed.res.mjs +452 -0
- package/src/sources/HeightStream.res +257 -0
- package/src/sources/HeightStream.res.mjs +194 -0
- package/src/sources/HyperSync.res +5 -0
- package/src/sources/HyperSync.res.mjs +3 -0
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncSSE.res +49 -0
- package/src/sources/HyperSyncSSE.res.mjs +79 -0
- package/src/sources/RpcSource.res +1 -1
- package/src/sources/RpcSource.res.mjs +2 -2
- package/src/sources/Source.res +54 -1
- package/src/sources/Source.res.mjs +25 -0
- package/src/sources/SourceManager.res +220 -227
- package/src/sources/SourceManager.res.mjs +139 -164
- package/src/sources/SourceManager.resi +8 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +70 -127
- package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
- package/src/sources/SvmHyperSyncSource.res +18 -15
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/svm.schema.json +1 -75
- package/src/MemoryStorage.res +0 -729
- package/src/MemoryStorage.res.mjs +0 -586
- package/src/sources/HyperSyncHeightStream.res +0 -129
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
- package/src/sources/RpcWebSocketHeightStream.res +0 -175
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// A checkpoint id per chain that a query compares each chain's rows against. A
|
|
2
|
+
// schema with a cross-chain entity gets one bound for every chain, since a reorg
|
|
3
|
+
// on any chain reaches every chain's rows; otherwise each chain has its own, and
|
|
4
|
+
// a chain absent from them is left alone.
|
|
5
|
+
type t =
|
|
6
|
+
| EveryChain(Internal.checkpointId)
|
|
7
|
+
| PerChain(array<(ChainId.t, Internal.checkpointId)>)
|
|
8
|
+
|
|
9
|
+
// The pieces a query splices in to read its bound as `checkpointId`: a SELECT
|
|
10
|
+
// joins the bounds next to its table with `join`, a DELETE names them with
|
|
11
|
+
// `using` and matches the chain with `usingMatch` in its WHERE.
|
|
12
|
+
type sql = {
|
|
13
|
+
join: string,
|
|
14
|
+
using: string,
|
|
15
|
+
usingMatch: string,
|
|
16
|
+
checkpointId: string,
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// The chain ids are cast to the widest integer type rather than the column's
|
|
20
|
+
// own: nothing indexes the join, so the values only have to compare.
|
|
21
|
+
let relation = `unnest($1::${(Postgres.BigInt :> string)}[],$2::${(Postgres.BigInt :> string)}[]) AS envio_bounds(chain_id, checkpoint_id)`
|
|
22
|
+
|
|
23
|
+
let sql = (bounds: t, ~chainIdColumn: option<string>, ~tableRef: string): sql =>
|
|
24
|
+
switch (bounds, chainIdColumn) {
|
|
25
|
+
| (EveryChain(_), _) => {join: "", using: "", usingMatch: "", checkpointId: "$1"}
|
|
26
|
+
| (PerChain(_), Some(column)) =>
|
|
27
|
+
let chainMatch = `envio_bounds.chain_id = ${tableRef}."${column}"`
|
|
28
|
+
{
|
|
29
|
+
join: ` JOIN ${relation} ON ${chainMatch}`,
|
|
30
|
+
using: ` USING ${relation}`,
|
|
31
|
+
usingMatch: ` AND ${chainMatch}`,
|
|
32
|
+
checkpointId: "envio_bounds.checkpoint_id",
|
|
33
|
+
}
|
|
34
|
+
| (PerChain(_), None) =>
|
|
35
|
+
JsError.throwWithMessage(
|
|
36
|
+
"Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.",
|
|
37
|
+
)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Bound in the order `sql` reads them: the one bound as `$1`, or the chains and
|
|
41
|
+
// their bounds as the two parallel arrays the relation unnests.
|
|
42
|
+
let params = (bounds: t): unknown =>
|
|
43
|
+
switch bounds {
|
|
44
|
+
| EveryChain(checkpointId) =>
|
|
45
|
+
[checkpointId->BigInt.toString]->(Utils.magic: array<string> => unknown)
|
|
46
|
+
| PerChain(byChain) =>
|
|
47
|
+
[
|
|
48
|
+
byChain->Array.map(((chainId, _)) => chainId)->(Utils.magic: array<ChainId.t> => unknown),
|
|
49
|
+
byChain
|
|
50
|
+
->Array.map(((_, checkpointId)) => checkpointId->BigInt.toString)
|
|
51
|
+
->(Utils.magic: array<string> => unknown),
|
|
52
|
+
]->(Utils.magic: array<unknown> => unknown)
|
|
53
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
4
|
+
|
|
5
|
+
let relation = `unnest($1::` + "BIGINT" + `[],$2::` + "BIGINT" + `[]) AS envio_bounds(chain_id, checkpoint_id)`;
|
|
6
|
+
|
|
7
|
+
function sql(bounds, chainIdColumn, tableRef) {
|
|
8
|
+
if (bounds.TAG === "EveryChain") {
|
|
9
|
+
return {
|
|
10
|
+
join: "",
|
|
11
|
+
using: "",
|
|
12
|
+
usingMatch: "",
|
|
13
|
+
checkpointId: "$1"
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
if (chainIdColumn === undefined) {
|
|
17
|
+
return Stdlib_JsError.throwWithMessage("Internal error: per-chain checkpoint bounds can't bound a table with no chain-id column. Only a schema whose entities are all per-chain has them.");
|
|
18
|
+
}
|
|
19
|
+
let chainMatch = `envio_bounds.chain_id = ` + tableRef + `."` + chainIdColumn + `"`;
|
|
20
|
+
return {
|
|
21
|
+
join: ` JOIN ` + relation + ` ON ` + chainMatch,
|
|
22
|
+
using: ` USING ` + relation,
|
|
23
|
+
usingMatch: ` AND ` + chainMatch,
|
|
24
|
+
checkpointId: "envio_bounds.checkpoint_id"
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function params(bounds) {
|
|
29
|
+
if (bounds.TAG === "EveryChain") {
|
|
30
|
+
return [bounds._0.toString()];
|
|
31
|
+
}
|
|
32
|
+
let byChain = bounds._0;
|
|
33
|
+
return [
|
|
34
|
+
byChain.map(param => param[0]),
|
|
35
|
+
byChain.map(param => param[1].toString())
|
|
36
|
+
];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export {
|
|
40
|
+
relation,
|
|
41
|
+
sql,
|
|
42
|
+
params,
|
|
43
|
+
}
|
|
44
|
+
/* No side effect */
|
package/src/db/EntityFilter.res
CHANGED
|
@@ -23,34 +23,53 @@ module FieldValue = {
|
|
|
23
23
|
//and serialize the types without parsing/wrapping them
|
|
24
24
|
type t = option<tNonOptional>
|
|
25
25
|
|
|
26
|
+
// A Uint8Array is an object like a BigDecimal, so the unboxed variant can't
|
|
27
|
+
// tell them apart on its own.
|
|
28
|
+
let asBytes = (value: t) => value->(Utils.magic: t => unknown)->Utils.Bytes.asUint8Array
|
|
29
|
+
|
|
26
30
|
let toString = (value: t) =>
|
|
27
|
-
switch value {
|
|
28
|
-
| Some(
|
|
29
|
-
| None =>
|
|
31
|
+
switch (value, value->asBytes) {
|
|
32
|
+
| (_, Some(bytes)) => bytes->Utils.Bytes.toHex
|
|
33
|
+
| (Some(v), None) => v->toString
|
|
34
|
+
| (None, None) => "undefined"
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
external castFrom: 'a => t = "%identity"
|
|
33
38
|
|
|
34
|
-
let
|
|
35
|
-
switch (a, b) {
|
|
39
|
+
let compareWith = (a, b, ~bigDecimal, ~bytes, ~fallback) =>
|
|
40
|
+
switch (a, b, a->asBytes, b->asBytes) {
|
|
41
|
+
| (_, _, Some(bytesA), Some(bytesB)) => bytes(Utils.Bytes.compare(bytesA, bytesB))
|
|
36
42
|
//For big decimal use custom equals operator otherwise let Caml_obj.equal do its magic
|
|
37
|
-
| (Some(BigDecimal(bdA)), Some(BigDecimal(bdB))) =>
|
|
38
|
-
| (a, b) => a
|
|
43
|
+
| (Some(BigDecimal(bdA)), Some(BigDecimal(bdB)), _, _) => bigDecimal(bdA, bdB)
|
|
44
|
+
| (a, b, _, _) => fallback(a, b)
|
|
39
45
|
}
|
|
40
46
|
|
|
47
|
+
let eq = (a, b) =>
|
|
48
|
+
compareWith(
|
|
49
|
+
a,
|
|
50
|
+
b,
|
|
51
|
+
~bigDecimal=BigDecimal.equals,
|
|
52
|
+
~bytes=order => order === 0.,
|
|
53
|
+
~fallback=(a, b) => a == b,
|
|
54
|
+
)
|
|
55
|
+
|
|
41
56
|
let gt = (a, b) =>
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
57
|
+
compareWith(
|
|
58
|
+
a,
|
|
59
|
+
b,
|
|
60
|
+
~bigDecimal=BigDecimal.gt,
|
|
61
|
+
~bytes=order => order > 0.,
|
|
62
|
+
~fallback=(a, b) => a > b,
|
|
63
|
+
)
|
|
47
64
|
|
|
48
65
|
let lt = (a, b) =>
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
66
|
+
compareWith(
|
|
67
|
+
a,
|
|
68
|
+
b,
|
|
69
|
+
~bigDecimal=BigDecimal.lt,
|
|
70
|
+
~bytes=order => order < 0.,
|
|
71
|
+
~fallback=(a, b) => a < b,
|
|
72
|
+
)
|
|
54
73
|
}
|
|
55
74
|
|
|
56
75
|
// The And case requires at least one nested filter (storage throws otherwise),
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Table from "./Table.res.mjs";
|
|
4
|
+
import * as Utils from "../Utils.res.mjs";
|
|
4
5
|
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
5
6
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
6
7
|
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
@@ -26,40 +27,50 @@ function toString(tNonOptional) {
|
|
|
26
27
|
}
|
|
27
28
|
}
|
|
28
29
|
|
|
30
|
+
function asBytes(value) {
|
|
31
|
+
return Utils.Bytes.asUint8Array(value);
|
|
32
|
+
}
|
|
33
|
+
|
|
29
34
|
function toString$1(value) {
|
|
30
|
-
|
|
35
|
+
let match = Utils.Bytes.asUint8Array(value);
|
|
36
|
+
if (match !== undefined) {
|
|
37
|
+
return Utils.Bytes.toHex(match);
|
|
38
|
+
} else if (value !== undefined) {
|
|
31
39
|
return toString(value);
|
|
32
40
|
} else {
|
|
33
41
|
return "undefined";
|
|
34
42
|
}
|
|
35
43
|
}
|
|
36
44
|
|
|
37
|
-
function
|
|
45
|
+
function compareWith(a, b, bigDecimal, bytes, fallback) {
|
|
46
|
+
let match = Utils.Bytes.asUint8Array(a);
|
|
47
|
+
let match$1 = Utils.Bytes.asUint8Array(b);
|
|
48
|
+
if (match !== undefined && match$1 !== undefined) {
|
|
49
|
+
return bytes(Utils.Bytes.compare(match, match$1));
|
|
50
|
+
}
|
|
38
51
|
if (typeof a === "object" && !Array.isArray(a) && typeof b === "object" && !Array.isArray(b)) {
|
|
39
|
-
return a
|
|
52
|
+
return bigDecimal(a, b);
|
|
40
53
|
} else {
|
|
41
|
-
return
|
|
54
|
+
return fallback(a, b);
|
|
42
55
|
}
|
|
43
56
|
}
|
|
44
57
|
|
|
58
|
+
function eq(a, b) {
|
|
59
|
+
return compareWith(a, b, (prim0, prim1) => prim0.isEqualTo(prim1), order => order === 0, Primitive_object.equal);
|
|
60
|
+
}
|
|
61
|
+
|
|
45
62
|
function gt(a, b) {
|
|
46
|
-
|
|
47
|
-
return a.isGreaterThan(b);
|
|
48
|
-
} else {
|
|
49
|
-
return Primitive_object.greaterthan(a, b);
|
|
50
|
-
}
|
|
63
|
+
return compareWith(a, b, (prim0, prim1) => prim0.isGreaterThan(prim1), order => order > 0, Primitive_object.greaterthan);
|
|
51
64
|
}
|
|
52
65
|
|
|
53
66
|
function lt(a, b) {
|
|
54
|
-
|
|
55
|
-
return a.isLessThan(b);
|
|
56
|
-
} else {
|
|
57
|
-
return Primitive_object.lessthan(a, b);
|
|
58
|
-
}
|
|
67
|
+
return compareWith(a, b, (prim0, prim1) => prim0.isLessThan(prim1), order => order < 0, Primitive_object.lessthan);
|
|
59
68
|
}
|
|
60
69
|
|
|
61
70
|
let FieldValue = {
|
|
71
|
+
asBytes: asBytes,
|
|
62
72
|
toString: toString$1,
|
|
73
|
+
compareWith: compareWith,
|
|
63
74
|
eq: eq,
|
|
64
75
|
gt: gt,
|
|
65
76
|
lt: lt
|
package/src/db/EntityHistory.res
CHANGED
|
@@ -30,11 +30,20 @@ let unsafeCheckpointIdSchema =
|
|
|
30
30
|
serializer: bigint => bigint->BigInt.toString,
|
|
31
31
|
})
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
// `chainIdTag` carries the (column, chain id) of a per-chain entity whose rows
|
|
34
|
+
// all belong to one chain: the column is then a constant of the schema rather
|
|
35
|
+
// than a field read off every entity.
|
|
36
|
+
let makeSetUpdateSchema = (
|
|
37
|
+
~idSchema: S.t<EntityId.t>,
|
|
38
|
+
~chainIdTag: option<(string, ChainId.t)>=?,
|
|
39
|
+
entitySchema: S.t<'entity>,
|
|
40
|
+
): S.t<Change.t<'entity>> => {
|
|
36
41
|
S.object(s => {
|
|
37
42
|
s.tag(changeFieldName, RowAction.SET)
|
|
43
|
+
switch chainIdTag {
|
|
44
|
+
| Some((column, chainId)) => s.tag(column, chainId)
|
|
45
|
+
| None => ()
|
|
46
|
+
}
|
|
38
47
|
Change.Set({
|
|
39
48
|
checkpointId: s.field(checkpointIdFieldName, unsafeCheckpointIdSchema),
|
|
40
49
|
entityId: s.field(Table.idFieldName, idSchema),
|
|
@@ -91,27 +100,38 @@ let makeKeyMatch = (~chainIdColumn, ~left, ~right) =>
|
|
|
91
100
|
->Array.map(column => `${left}.${column} = ${right}.${column}`)
|
|
92
101
|
->Array.joinUnsafe(" AND ")
|
|
93
102
|
|
|
94
|
-
let makePruneStaleEntityHistoryQuery = (
|
|
103
|
+
let makePruneStaleEntityHistoryQuery = (
|
|
104
|
+
~entityName,
|
|
105
|
+
~entityIndex,
|
|
106
|
+
~pgSchema,
|
|
107
|
+
~chainIdColumn,
|
|
108
|
+
~safeCheckpoints: CheckpointBounds.t,
|
|
109
|
+
) => {
|
|
95
110
|
let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
|
|
96
111
|
let keyColumns = makeKeyColumns(~chainIdColumn)
|
|
97
112
|
let anchorKeys = keyColumns->Array.map(column => `t.${column}`)->Array.joinUnsafe(", ")
|
|
113
|
+
let bounds = safeCheckpoints->CheckpointBounds.sql(~chainIdColumn, ~tableRef="t")
|
|
98
114
|
|
|
99
115
|
// Whether a key still has a row above the safe checkpoint is an aggregate
|
|
100
116
|
// over the same groups as the anchor, so it's computed in the one pass
|
|
101
|
-
// rather than as a per-row correlated lookup.
|
|
102
|
-
// `d.envio_checkpoint_id <= $1` is implied by the rest of the predicate — it
|
|
117
|
+
// rather than as a per-row correlated lookup. The DELETE's `<=` on the row
|
|
103
118
|
// is there to keep the rows above the safe checkpoint out of the join.
|
|
119
|
+
//
|
|
120
|
+
// Per-chain bounds are joined in rather than run as a statement each: the
|
|
121
|
+
// anchors aggregate the whole table however narrow the bound is, and history
|
|
122
|
+
// carries no index to narrow the scan with.
|
|
104
123
|
`WITH anchors AS (
|
|
105
124
|
SELECT ${anchorKeys},
|
|
106
|
-
MAX(t.${checkpointIdFieldName}) FILTER (WHERE t.${checkpointIdFieldName} <= $
|
|
107
|
-
bool_or(t.${checkpointIdFieldName} > $
|
|
108
|
-
|
|
125
|
+
MAX(t.${checkpointIdFieldName}) FILTER (WHERE t.${checkpointIdFieldName} <= ${bounds.checkpointId}) AS keep_checkpoint_id,
|
|
126
|
+
bool_or(t.${checkpointIdFieldName} > ${bounds.checkpointId}) AS has_above,
|
|
127
|
+
MIN(${bounds.checkpointId}) AS safe_checkpoint_id
|
|
128
|
+
FROM ${historyTableRef} t${bounds.join}
|
|
109
129
|
GROUP BY ${anchorKeys}
|
|
110
130
|
)
|
|
111
131
|
DELETE FROM ${historyTableRef} d
|
|
112
132
|
USING anchors a
|
|
113
133
|
WHERE ${makeKeyMatch(~chainIdColumn, ~left="d", ~right="a")}
|
|
114
|
-
AND d.${checkpointIdFieldName} <=
|
|
134
|
+
AND d.${checkpointIdFieldName} <= a.safe_checkpoint_id
|
|
115
135
|
AND (d.${checkpointIdFieldName} < a.keep_checkpoint_id OR NOT a.has_above);`
|
|
116
136
|
}
|
|
117
137
|
|
|
@@ -121,13 +141,18 @@ let pruneStaleEntityHistory = (
|
|
|
121
141
|
~entityIndex,
|
|
122
142
|
~pgSchema,
|
|
123
143
|
~chainIdColumn,
|
|
124
|
-
~
|
|
125
|
-
): promise<unit> =>
|
|
144
|
+
~safeCheckpoints,
|
|
145
|
+
): promise<unit> =>
|
|
126
146
|
sql->Postgres.preparedUnsafe(
|
|
127
|
-
makePruneStaleEntityHistoryQuery(
|
|
128
|
-
|
|
147
|
+
makePruneStaleEntityHistoryQuery(
|
|
148
|
+
~entityName,
|
|
149
|
+
~entityIndex,
|
|
150
|
+
~pgSchema,
|
|
151
|
+
~chainIdColumn,
|
|
152
|
+
~safeCheckpoints,
|
|
153
|
+
),
|
|
154
|
+
safeCheckpoints->CheckpointBounds.params,
|
|
129
155
|
)
|
|
130
|
-
}
|
|
131
156
|
|
|
132
157
|
// If an entity doesn't have a history before the update
|
|
133
158
|
// we create it automatically with envio_checkpoint_id 0
|
|
@@ -173,7 +198,7 @@ let backfillHistory = (
|
|
|
173
198
|
~ids: array<EntityId.t>,
|
|
174
199
|
) => {
|
|
175
200
|
let idPgType = table->Table.getIdPgFieldType(~pgSchema)
|
|
176
|
-
let chainIdColumn = table->Table.
|
|
201
|
+
let chainIdColumn = table->Table.getPgChainIdColumn
|
|
177
202
|
let params = [table->Table.encodeIdsToJson(ids)->(Utils.magic: JSON.t => unknown)]
|
|
178
203
|
sql
|
|
179
204
|
->Postgres.preparedUnsafe(
|
|
@@ -195,15 +220,15 @@ let rollback = (
|
|
|
195
220
|
~pgSchema,
|
|
196
221
|
~entityName,
|
|
197
222
|
~entityIndex,
|
|
198
|
-
~
|
|
223
|
+
~chainIdColumn,
|
|
224
|
+
~floors: RollbackFloors.t,
|
|
199
225
|
) => {
|
|
226
|
+
let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
|
|
227
|
+
let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef=historyTableRef)
|
|
200
228
|
sql
|
|
201
229
|
->Postgres.preparedUnsafe(
|
|
202
|
-
`DELETE FROM "${
|
|
203
|
-
|
|
204
|
-
~entityIndex,
|
|
205
|
-
)}" WHERE "${checkpointIdFieldName}" > $1;`,
|
|
206
|
-
[rollbackTargetCheckpointId->BigInt.toString]->(Utils.magic: array<string> => unknown),
|
|
230
|
+
`DELETE FROM ${historyTableRef}${bounds.using} WHERE "${checkpointIdFieldName}" > ${bounds.checkpointId}${bounds.usingMatch};`,
|
|
231
|
+
floors.floors->CheckpointBounds.params,
|
|
207
232
|
)
|
|
208
233
|
->Utils.Promise.ignoreValue
|
|
209
234
|
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import * as Table from "./Table.res.mjs";
|
|
4
4
|
import * as ChainId from "../ChainId.res.mjs";
|
|
5
5
|
import * as Stdlib_BigInt from "@rescript/runtime/lib/es6/Stdlib_BigInt.js";
|
|
6
|
-
import * as
|
|
6
|
+
import * as CheckpointBounds from "./CheckpointBounds.res.mjs";
|
|
7
7
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
8
8
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
9
9
|
|
|
@@ -50,9 +50,12 @@ let unsafeCheckpointIdSchema = S$RescriptSchema.transform(S$RescriptSchema.setNa
|
|
|
50
50
|
s: bigint => bigint.toString()
|
|
51
51
|
}));
|
|
52
52
|
|
|
53
|
-
function makeSetUpdateSchema(idSchema, entitySchema) {
|
|
53
|
+
function makeSetUpdateSchema(idSchema, chainIdTag, entitySchema) {
|
|
54
54
|
return S$RescriptSchema.object(s => {
|
|
55
55
|
s.tag(changeFieldName, "SET");
|
|
56
|
+
if (chainIdTag !== undefined) {
|
|
57
|
+
s.tag(chainIdTag[0], chainIdTag[1]);
|
|
58
|
+
}
|
|
56
59
|
return {
|
|
57
60
|
type: "SET",
|
|
58
61
|
entityId: s.f(Table.idFieldName, idSchema),
|
|
@@ -83,26 +86,28 @@ function makeKeyMatch(chainIdColumn, left, right) {
|
|
|
83
86
|
return makeKeyColumns(chainIdColumn).map(column => left + `.` + column + ` = ` + right + `.` + column).join(" AND ");
|
|
84
87
|
}
|
|
85
88
|
|
|
86
|
-
function makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn) {
|
|
89
|
+
function makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints) {
|
|
87
90
|
let historyTableRef = `"` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `"`;
|
|
88
91
|
let keyColumns = makeKeyColumns(chainIdColumn);
|
|
89
92
|
let anchorKeys = keyColumns.map(column => `t.` + column).join(", ");
|
|
93
|
+
let bounds = CheckpointBounds.sql(safeCheckpoints, chainIdColumn, "t");
|
|
90
94
|
return `WITH anchors AS (
|
|
91
95
|
SELECT ` + anchorKeys + `,
|
|
92
|
-
MAX(t.` + checkpointIdFieldName + `) FILTER (WHERE t.` + checkpointIdFieldName + ` <=
|
|
93
|
-
bool_or(t.` + checkpointIdFieldName + ` >
|
|
94
|
-
|
|
96
|
+
MAX(t.` + checkpointIdFieldName + `) FILTER (WHERE t.` + checkpointIdFieldName + ` <= ` + bounds.checkpointId + `) AS keep_checkpoint_id,
|
|
97
|
+
bool_or(t.` + checkpointIdFieldName + ` > ` + bounds.checkpointId + `) AS has_above,
|
|
98
|
+
MIN(` + bounds.checkpointId + `) AS safe_checkpoint_id
|
|
99
|
+
FROM ` + historyTableRef + ` t` + bounds.join + `
|
|
95
100
|
GROUP BY ` + anchorKeys + `
|
|
96
101
|
)
|
|
97
102
|
DELETE FROM ` + historyTableRef + ` d
|
|
98
103
|
USING anchors a
|
|
99
104
|
WHERE ` + makeKeyMatch(chainIdColumn, "d", "a") + `
|
|
100
|
-
AND d.` + checkpointIdFieldName + ` <=
|
|
105
|
+
AND d.` + checkpointIdFieldName + ` <= a.safe_checkpoint_id
|
|
101
106
|
AND (d.` + checkpointIdFieldName + ` < a.keep_checkpoint_id OR NOT a.has_above);`;
|
|
102
107
|
}
|
|
103
108
|
|
|
104
|
-
function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn,
|
|
105
|
-
return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn),
|
|
109
|
+
function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints) {
|
|
110
|
+
return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpoints), CheckpointBounds.params(safeCheckpoints), {prepare: true});
|
|
106
111
|
}
|
|
107
112
|
|
|
108
113
|
function makeBackfillHistoryQuery(pgSchema, entityName, entityIndex, idPgType, chainIdColumn, chainId) {
|
|
@@ -125,13 +130,15 @@ FROM missing_history;`;
|
|
|
125
130
|
|
|
126
131
|
function backfillHistory(sql, pgSchema, table, entityIndex, chainId, ids) {
|
|
127
132
|
let idPgType = Table.getIdPgFieldType(table, pgSchema);
|
|
128
|
-
let chainIdColumn =
|
|
133
|
+
let chainIdColumn = Table.getPgChainIdColumn(table);
|
|
129
134
|
let params = [Table.encodeIdsToJson(table, ids)];
|
|
130
135
|
return sql.unsafe(makeBackfillHistoryQuery(pgSchema, table.tableName, entityIndex, idPgType, chainIdColumn, chainId), params, {prepare: true});
|
|
131
136
|
}
|
|
132
137
|
|
|
133
|
-
function rollback(sql, pgSchema, entityName, entityIndex,
|
|
134
|
-
|
|
138
|
+
function rollback(sql, pgSchema, entityName, entityIndex, chainIdColumn, floors) {
|
|
139
|
+
let historyTableRef = `"` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `"`;
|
|
140
|
+
let bounds = CheckpointBounds.sql(floors.floors, chainIdColumn, historyTableRef);
|
|
141
|
+
return sql.unsafe(`DELETE FROM ` + historyTableRef + bounds.using + ` WHERE "` + checkpointIdFieldName + `" > ` + bounds.checkpointId + bounds.usingMatch + `;`, CheckpointBounds.params(floors.floors), {prepare: true});
|
|
135
142
|
}
|
|
136
143
|
|
|
137
144
|
let checkpointIdFieldType = "UInt64";
|
package/src/db/InternalTable.res
CHANGED
|
@@ -556,18 +556,64 @@ module Checkpoints = {
|
|
|
556
556
|
|
|
557
557
|
let initialCheckpointId = 0n
|
|
558
558
|
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
559
|
+
// One definition per column, carrying what each storage needs: the field
|
|
560
|
+
// itself, the type ClickHouse gives it where that differs from Postgres, and
|
|
561
|
+
// where a batch keeps the column's values.
|
|
562
|
+
type column = {
|
|
563
|
+
field: fieldOrDerived,
|
|
564
|
+
clickHouseFieldType: fieldType,
|
|
565
|
+
valuesOf: Batch.t => array<unknown>,
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
let columns: array<column> = [
|
|
569
|
+
{
|
|
570
|
+
field: mkField((#id: field :> string), UInt64, ~fieldSchema=S.bigint, ~isPrimaryKey),
|
|
571
|
+
clickHouseFieldType: UInt64,
|
|
572
|
+
valuesOf: batch => batch.checkpointIds->(Utils.magic: array<bigint> => array<unknown>),
|
|
573
|
+
},
|
|
574
|
+
{
|
|
575
|
+
field: mkField((#chain_id: field :> string), ChainId, ~fieldSchema=ChainId.schema),
|
|
576
|
+
clickHouseFieldType: ChainId,
|
|
577
|
+
valuesOf: batch =>
|
|
578
|
+
batch.checkpointChainIds->(Utils.magic: array<ChainId.t> => array<unknown>),
|
|
579
|
+
},
|
|
580
|
+
{
|
|
581
|
+
field: mkField((#block_number: field :> string), Int32, ~fieldSchema=S.int),
|
|
582
|
+
clickHouseFieldType: Int32,
|
|
583
|
+
valuesOf: batch => batch.checkpointBlockNumbers->(Utils.magic: array<int> => array<unknown>),
|
|
584
|
+
},
|
|
585
|
+
{
|
|
586
|
+
field: mkField(
|
|
587
|
+
(#block_hash: field :> string),
|
|
588
|
+
String,
|
|
589
|
+
~fieldSchema=S.null(S.string),
|
|
590
|
+
~isNullable,
|
|
591
|
+
),
|
|
592
|
+
clickHouseFieldType: String,
|
|
593
|
+
valuesOf: batch =>
|
|
594
|
+
batch.checkpointBlockHashes->(Utils.magic: array<Null.t<string>> => array<unknown>),
|
|
595
|
+
},
|
|
596
|
+
{
|
|
597
|
+
field: mkField((#events_processed: field :> string), Int32, ~fieldSchema=S.int),
|
|
598
|
+
// A count of every event a chain has processed outgrows an Int32 where
|
|
599
|
+
// Postgres keeps one, and ClickHouse has the id's width to spare.
|
|
600
|
+
clickHouseFieldType: UInt64,
|
|
601
|
+
valuesOf: batch =>
|
|
602
|
+
batch.checkpointEventsProcessed->(Utils.magic: array<int> => array<unknown>),
|
|
603
|
+
},
|
|
604
|
+
]
|
|
605
|
+
|
|
606
|
+
let table = mkTable("envio_checkpoints", ~fields=columns->Array.map(({field}) => field))
|
|
569
607
|
|
|
570
608
|
let makeGetReorgCheckpointsQuery = (~pgSchema): string => {
|
|
609
|
+
// The safe_block checkpoint itself is included, so it can be used for safe
|
|
610
|
+
// checkpoint tracking.
|
|
611
|
+
//
|
|
612
|
+
// Ordered by id, which within a chain is block order: both consumers of
|
|
613
|
+
// these rows — the safe-checkpoint scan and the block-store seed — read them
|
|
614
|
+
// as ascending. Physical row order can't stand in for that, since a rollback
|
|
615
|
+
// frees space that later checkpoints are written back into.
|
|
616
|
+
//
|
|
571
617
|
// Use CTE to pre-filter chains and compute safe_block once per chain
|
|
572
618
|
// This is faster because:
|
|
573
619
|
// 1. Chains table is small, so filtering it first is cheap
|
|
@@ -590,7 +636,8 @@ FROM "${pgSchema}"."${table.tableName}" cp
|
|
|
590
636
|
INNER JOIN reorg_chains rc
|
|
591
637
|
ON cp."${(#chain_id: field :> string)}" = rc.id
|
|
592
638
|
WHERE cp."${(#block_hash: field :> string)}" IS NOT NULL
|
|
593
|
-
AND cp."${(#block_number: field :> string)}" >= rc.safe_block
|
|
639
|
+
AND cp."${(#block_number: field :> string)}" >= rc.safe_block
|
|
640
|
+
ORDER BY cp."${(#id: field :> string)}";`
|
|
594
641
|
}
|
|
595
642
|
|
|
596
643
|
let makeCommitedCheckpointIdQuery = (~pgSchema) => {
|
|
@@ -635,27 +682,33 @@ SELECT * FROM unnest($1::${(BigInt: Postgres.columnType :> string)}[],$2::${chai
|
|
|
635
682
|
->Utils.Promise.ignoreValue
|
|
636
683
|
}
|
|
637
684
|
|
|
638
|
-
|
|
685
|
+
// Optional to match the entity tables', where a cross-chain entity has none.
|
|
686
|
+
let chainIdColumn = Some((#chain_id: field :> string))
|
|
687
|
+
|
|
688
|
+
let rollback = (sql, ~pgSchema, ~floors: RollbackFloors.t) => {
|
|
689
|
+
let tableRef = `"${pgSchema}"."${table.tableName}"`
|
|
690
|
+
let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef)
|
|
639
691
|
sql
|
|
640
692
|
->Postgres.preparedUnsafe(
|
|
641
|
-
`DELETE FROM
|
|
642
|
-
|
|
693
|
+
`DELETE FROM ${tableRef}${bounds.using} WHERE "${(#id: field :> string)}" > ${bounds.checkpointId}${bounds.usingMatch};`,
|
|
694
|
+
floors.floors->CheckpointBounds.params,
|
|
643
695
|
)
|
|
644
696
|
->Utils.Promise.ignoreValue
|
|
645
697
|
}
|
|
646
698
|
|
|
647
|
-
let makePruneStaleCheckpointsQuery = (~pgSchema) => {
|
|
648
|
-
|
|
699
|
+
let makePruneStaleCheckpointsQuery = (~pgSchema, ~safeCheckpoints: CheckpointBounds.t) => {
|
|
700
|
+
let tableRef = `"${pgSchema}"."${table.tableName}"`
|
|
701
|
+
let bounds = safeCheckpoints->CheckpointBounds.sql(~chainIdColumn, ~tableRef)
|
|
702
|
+
`DELETE FROM ${tableRef}${bounds.using} WHERE "${(#id: field :> string)}" < ${bounds.checkpointId}${bounds.usingMatch};`
|
|
649
703
|
}
|
|
650
704
|
|
|
651
|
-
let pruneStaleCheckpoints = (sql, ~pgSchema, ~
|
|
705
|
+
let pruneStaleCheckpoints = (sql, ~pgSchema, ~safeCheckpoints) =>
|
|
652
706
|
sql
|
|
653
707
|
->Postgres.preparedUnsafe(
|
|
654
|
-
makePruneStaleCheckpointsQuery(~pgSchema),
|
|
655
|
-
|
|
708
|
+
makePruneStaleCheckpointsQuery(~pgSchema, ~safeCheckpoints),
|
|
709
|
+
safeCheckpoints->CheckpointBounds.params,
|
|
656
710
|
)
|
|
657
711
|
->Utils.Promise.ignoreValue
|
|
658
|
-
}
|
|
659
712
|
|
|
660
713
|
let makeGetRollbackTargetCheckpointQuery = (~pgSchema) => {
|
|
661
714
|
`SELECT "${(#id: field :> string)}" FROM "${pgSchema}"."${table.tableName}"
|
|
@@ -684,25 +737,22 @@ LIMIT 1;`
|
|
|
684
737
|
})
|
|
685
738
|
}
|
|
686
739
|
|
|
687
|
-
let makeGetRollbackProgressDiffQuery = (~pgSchema) => {
|
|
740
|
+
let makeGetRollbackProgressDiffQuery = (~pgSchema, ~floors: RollbackFloors.t) => {
|
|
741
|
+
let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef="t")
|
|
688
742
|
`SELECT
|
|
689
|
-
"${(#chain_id: field :> string)}"::float8 as "${(#chain_id: field :> string)}",
|
|
690
|
-
SUM("${(#events_processed: field :> string)}") as events_processed_diff,
|
|
691
|
-
MIN("${(#block_number: field :> string)}") - 1 as new_progress_block_number
|
|
692
|
-
FROM "${pgSchema}"."${table.tableName}"
|
|
693
|
-
WHERE "${(#id: field :> string)}" > $
|
|
694
|
-
GROUP BY "${(#chain_id: field :> string)}";`
|
|
743
|
+
t."${(#chain_id: field :> string)}"::float8 as "${(#chain_id: field :> string)}",
|
|
744
|
+
SUM(t."${(#events_processed: field :> string)}") as events_processed_diff,
|
|
745
|
+
MIN(t."${(#block_number: field :> string)}") - 1 as new_progress_block_number
|
|
746
|
+
FROM "${pgSchema}"."${table.tableName}" t${bounds.join}
|
|
747
|
+
WHERE t."${(#id: field :> string)}" > ${bounds.checkpointId}
|
|
748
|
+
GROUP BY t."${(#chain_id: field :> string)}";`
|
|
695
749
|
}
|
|
696
750
|
|
|
697
|
-
let getRollbackProgressDiff = (
|
|
698
|
-
sql,
|
|
699
|
-
~pgSchema,
|
|
700
|
-
~rollbackTargetCheckpointId: Internal.checkpointId,
|
|
701
|
-
) => {
|
|
751
|
+
let getRollbackProgressDiff = (sql, ~pgSchema, ~floors: RollbackFloors.t) =>
|
|
702
752
|
sql
|
|
703
753
|
->Postgres.preparedUnsafe(
|
|
704
|
-
makeGetRollbackProgressDiffQuery(~pgSchema),
|
|
705
|
-
|
|
754
|
+
makeGetRollbackProgressDiffQuery(~pgSchema, ~floors),
|
|
755
|
+
floors.floors->CheckpointBounds.params,
|
|
706
756
|
)
|
|
707
757
|
->(
|
|
708
758
|
Utils.magic: promise<unknown> => promise<
|
|
@@ -713,7 +763,6 @@ GROUP BY "${(#chain_id: field :> string)}";`
|
|
|
713
763
|
}>,
|
|
714
764
|
>
|
|
715
765
|
)
|
|
716
|
-
}
|
|
717
766
|
}
|
|
718
767
|
|
|
719
768
|
module RawEvents = {
|