envio 3.9.0 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +26 -0
- package/fuel.schema.json +26 -0
- package/index.d.ts +35 -3
- package/package.json +6 -7
- package/src/Batch.res +9 -0
- package/src/ChainState.res +35 -39
- package/src/ChainState.res.mjs +14 -11
- package/src/ChainState.resi +15 -7
- package/src/Config.res +52 -44
- package/src/Config.res.mjs +51 -17
- package/src/Core.res +4 -0
- package/src/CrossChainState.res +15 -23
- package/src/CrossChainState.res.mjs +10 -18
- package/src/CrossChainState.resi +4 -1
- package/src/Ecosystem.res +7 -2
- package/src/Envio.res +6 -4
- package/src/EventConfigBuilder.res +169 -26
- package/src/EventConfigBuilder.res.mjs +124 -15
- package/src/HandlerRegister.res +21 -4
- package/src/HandlerRegister.res.mjs +24 -5
- package/src/InMemoryStore.res +9 -7
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/IndexerState.res +43 -6
- package/src/IndexerState.res.mjs +36 -7
- package/src/IndexerState.resi +4 -4
- package/src/Internal.res +16 -17
- package/src/Metrics.res +72 -6
- package/src/Metrics.res.mjs +30 -2
- package/src/Persistence.res +36 -30
- package/src/Persistence.res.mjs +1 -20
- package/src/PgStorage.res +104 -56
- package/src/PgStorage.res.mjs +57 -35
- package/src/PruneStaleHistory.res +85 -52
- package/src/PruneStaleHistory.res.mjs +63 -37
- package/src/Rollback.res +68 -45
- package/src/Rollback.res.mjs +38 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SafeCheckpointTracking.res +11 -11
- package/src/SafeCheckpointTracking.res.mjs +7 -6
- package/src/SimulateItems.res +33 -18
- package/src/SimulateItems.res.mjs +26 -22
- package/src/Sink.res +38 -26
- package/src/Sink.res.mjs +21 -16
- package/src/TestIndexer.res +5 -5
- package/src/TestIndexer.res.mjs +1 -1
- package/src/Utils.res +170 -0
- package/src/Utils.res.mjs +153 -0
- package/src/Writing.res +2 -2
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +366 -698
- package/src/bindings/ClickHouse.res.mjs +371 -421
- package/src/bindings/ClickHouseSink.res +337 -0
- package/src/bindings/ClickHouseSink.res.mjs +246 -0
- package/src/bindings/EventSource.res +8 -2
- package/src/bindings/NodeJs.res +6 -0
- package/src/bindings/Vitest.res +23 -0
- package/src/bindings/Vitest.res.mjs +3 -0
- package/src/bindings/WebSocket.res +9 -10
- package/src/db/CheckpointBounds.res +53 -0
- package/src/db/CheckpointBounds.res.mjs +44 -0
- package/src/db/EntityFilter.res +36 -17
- package/src/db/EntityFilter.res.mjs +25 -14
- package/src/db/EntityHistory.res +47 -22
- package/src/db/EntityHistory.res.mjs +19 -12
- package/src/db/InternalTable.res +84 -35
- package/src/db/InternalTable.res.mjs +56 -23
- package/src/db/RollbackFloors.res +55 -0
- package/src/db/RollbackFloors.res.mjs +92 -0
- package/src/db/Table.res +22 -4
- package/src/db/Table.res.mjs +30 -9
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/EvmHyperSyncSource.res +3 -3
- package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
- package/src/sources/EvmRpcWs.res +124 -0
- package/src/sources/EvmRpcWs.res.mjs +117 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/FuelHyperSyncSource.res +1 -1
- package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
- package/src/sources/HeightFeed.res +568 -0
- package/src/sources/HeightFeed.res.mjs +452 -0
- package/src/sources/HeightStream.res +257 -0
- package/src/sources/HeightStream.res.mjs +194 -0
- package/src/sources/HyperSync.res +5 -0
- package/src/sources/HyperSync.res.mjs +3 -0
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncSSE.res +49 -0
- package/src/sources/HyperSyncSSE.res.mjs +79 -0
- package/src/sources/RpcSource.res +1 -1
- package/src/sources/RpcSource.res.mjs +2 -2
- package/src/sources/Source.res +54 -1
- package/src/sources/Source.res.mjs +25 -0
- package/src/sources/SourceManager.res +220 -227
- package/src/sources/SourceManager.res.mjs +139 -164
- package/src/sources/SourceManager.resi +8 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +70 -127
- package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
- package/src/sources/SvmHyperSyncSource.res +18 -15
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
- package/svm.schema.json +1 -75
- package/src/MemoryStorage.res +0 -729
- package/src/MemoryStorage.res.mjs +0 -586
- package/src/sources/HyperSyncHeightStream.res +0 -129
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
- package/src/sources/RpcWebSocketHeightStream.res +0 -175
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
package/src/Metrics.res.mjs
CHANGED
|
@@ -80,9 +80,20 @@ function renderMetrics(b, metrics) {
|
|
|
80
80
|
`{entity="` + escapeLabelValue(s.entity) + `"}`,
|
|
81
81
|
s
|
|
82
82
|
]);
|
|
83
|
+
let sourceLabels = (source, chainId, extra) => {
|
|
84
|
+
let pair = `{source="` + escapeLabelValue(source) + `",chainId="` + ChainId.toString(chainId) + `"`;
|
|
85
|
+
if (extra !== undefined) {
|
|
86
|
+
return pair + `,` + extra[0] + `="` + escapeLabelValue(extra[1]) + `"}`;
|
|
87
|
+
} else {
|
|
88
|
+
return pair + "}";
|
|
89
|
+
}
|
|
90
|
+
};
|
|
83
91
|
let byLabels = {};
|
|
84
92
|
metrics.sourceRequests.forEach(s => {
|
|
85
|
-
let labels =
|
|
93
|
+
let labels = sourceLabels(s.source, s.chainId, [
|
|
94
|
+
"method",
|
|
95
|
+
s.method
|
|
96
|
+
]);
|
|
86
97
|
let existing = byLabels[labels];
|
|
87
98
|
if (existing !== undefined) {
|
|
88
99
|
byLabels[labels] = {
|
|
@@ -99,7 +110,7 @@ function renderMetrics(b, metrics) {
|
|
|
99
110
|
let sourceRequests = Object.entries(byLabels);
|
|
100
111
|
let byLabels$1 = {};
|
|
101
112
|
metrics.sourceHeights.forEach(s => {
|
|
102
|
-
let labels =
|
|
113
|
+
let labels = sourceLabels(s.source, s.chainId, undefined);
|
|
103
114
|
let existing = byLabels$1[labels];
|
|
104
115
|
if (existing !== undefined && existing >= s.height) {
|
|
105
116
|
return;
|
|
@@ -109,6 +120,17 @@ function renderMetrics(b, metrics) {
|
|
|
109
120
|
}
|
|
110
121
|
});
|
|
111
122
|
let sources = Object.entries(byLabels$1);
|
|
123
|
+
let byLabels$2 = {};
|
|
124
|
+
metrics.sourceHeightStreams.forEach(s => Utils.Dict.incrementBy(byLabels$2, sourceLabels(s.source, s.chainId, undefined), s.connectCount));
|
|
125
|
+
let heightStreamConnects = Object.entries(byLabels$2);
|
|
126
|
+
let byLabels$3 = {};
|
|
127
|
+
metrics.sourceHeightStreams.forEach(s => {
|
|
128
|
+
s.disconnectsByReason.forEach(param => Utils.Dict.incrementBy(byLabels$3, sourceLabels(s.source, s.chainId, [
|
|
129
|
+
"reason",
|
|
130
|
+
param[0]
|
|
131
|
+
]), param[1]));
|
|
132
|
+
});
|
|
133
|
+
let heightStreamDisconnects = Object.entries(byLabels$3);
|
|
112
134
|
single(b, "envio_process_start_time_seconds", "Start time of the process since unix epoch in seconds.", "gauge", metrics.startTime.getTime() / 1000);
|
|
113
135
|
single(b, "envio_process_metric_time_seconds", "The time these metrics were collected. Use it to tell how fresh a snapshot is, or to measure rates between two snapshots.", "gauge", metrics.metricTime.getTime() / 1000);
|
|
114
136
|
single(b, "envio_process_elapsed_seconds", "How long the indexer has been running. Divide a cumulative seconds metric by this to get the share of the run it took, eg envio_processing_seconds for time spent in event handlers.", "gauge", metrics.elapsedSeconds);
|
|
@@ -154,6 +176,12 @@ function renderMetrics(b, metrics) {
|
|
|
154
176
|
return s.seconds;
|
|
155
177
|
}
|
|
156
178
|
});
|
|
179
|
+
if (heightStreamConnects.length !== 0) {
|
|
180
|
+
series(b, "envio_source_height_stream_connects_total", "The number of times a source's height subscription connected. Compare against the disconnects total, which is absent until the first disconnect and counts as zero while it is: one more connect than disconnects means the stream is up, and equal counts mean it is down and the indexer is polling instead. Zero connects means the stream has not come up, which is the normal reading for a chain that is still backfilling: subscriptions are only opened once a chain reaches the head.", "counter", heightStreamConnects, count => count);
|
|
181
|
+
}
|
|
182
|
+
if (heightStreamDisconnects.length !== 0) {
|
|
183
|
+
series(b, "envio_source_height_stream_disconnects_total", "The number of times a source's height subscription lost a connection, by reason. Failed retries are not counted, so this is outages rather than their length. A rotated disconnect is a connection that served its time; unsubscribed is the source being benched; every other reason ended a connection early.", "counter", heightStreamDisconnects, count => count);
|
|
184
|
+
}
|
|
157
185
|
series(b, "envio_source_known_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", "gauge", sources, height => height);
|
|
158
186
|
seriesOpt(b, "envio_reorg_detected_total", "Total number of reorgs detected", "counter", chains, m => {
|
|
159
187
|
if (m.reorgCount > 0) {
|
package/src/Persistence.res
CHANGED
|
@@ -59,14 +59,20 @@ type updatedEffectCache = {
|
|
|
59
59
|
}
|
|
60
60
|
|
|
61
61
|
type rollback = {
|
|
62
|
-
targetCheckpointId: Internal.checkpointId,
|
|
63
62
|
diffCheckpointId: Internal.checkpointId,
|
|
63
|
+
// How far back the deletes reach on each chain, travelling with the diff so
|
|
64
|
+
// the write leaves an untouched sibling's rows alone.
|
|
65
|
+
floors: RollbackFloors.t,
|
|
64
66
|
// The address registrations the rollback dropped, as the chains' address
|
|
65
67
|
// stores resolved them. Deleted by primary key in the same transaction.
|
|
66
68
|
rolledBackAddresses: array<AddressRows.key>,
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
|
|
69
|
+
// Where the rollback left every chain it moved. Written with the diff rather
|
|
70
|
+
// than waiting for a batch of those chains' own: the batch that carries the
|
|
71
|
+
// diff can belong to a chain the rollback never touched, and a chain whose
|
|
72
|
+
// stored progress outlived the checkpoints backing it would resume past
|
|
73
|
+
// blocks it never re-indexed. Also what `RollbackCommit.fire` reports once
|
|
74
|
+
// the diff is durably written.
|
|
75
|
+
progressedChains: array<InternalTable.Chains.progressedChain>,
|
|
70
76
|
}
|
|
71
77
|
|
|
72
78
|
// One flush group: the changes an entity accumulated within a single chain
|
|
@@ -101,7 +107,16 @@ type storage = {
|
|
|
101
107
|
~contractMapping: ContractMapping.t,
|
|
102
108
|
~envioInfo: JSON.t,
|
|
103
109
|
) => promise<initialState>,
|
|
104
|
-
|
|
110
|
+
// `throwIfIncompatible` gets what the storage holds before any sink is
|
|
111
|
+
// resumed, so a config the stored one rules out is reported as such rather
|
|
112
|
+
// than as the sink tripping over tables it never created.
|
|
113
|
+
resumeInitialState: (
|
|
114
|
+
~entities: array<Internal.entityConfig>,
|
|
115
|
+
~throwIfIncompatible: (
|
|
116
|
+
~storedEnvioInfo: option<JSON.t>,
|
|
117
|
+
~storedContractMapping: ContractMapping.t,
|
|
118
|
+
) => unit,
|
|
119
|
+
) => promise<initialState>,
|
|
105
120
|
// Returns rows matching the filter.
|
|
106
121
|
// Field values are serialized and rows parsed with the table's field schemas.
|
|
107
122
|
@raises("StorageError")
|
|
@@ -131,13 +146,13 @@ type storage = {
|
|
|
131
146
|
// Update chain metadata
|
|
132
147
|
setChainMeta: dict<InternalTable.Chains.metaFields> => promise<unknown>,
|
|
133
148
|
// Prune old checkpoints
|
|
134
|
-
pruneStaleCheckpoints: (~
|
|
149
|
+
pruneStaleCheckpoints: (~safeCheckpoints: CheckpointBounds.t) => promise<unit>,
|
|
135
150
|
// Prune stale entity history
|
|
136
151
|
pruneStaleEntityHistory: (
|
|
137
152
|
~entityName: string,
|
|
138
153
|
~entityIndex: int,
|
|
139
154
|
~chainIdColumn: option<string>,
|
|
140
|
-
~
|
|
155
|
+
~safeCheckpoints: CheckpointBounds.t,
|
|
141
156
|
) => promise<unit>,
|
|
142
157
|
// Get rollback target checkpoint
|
|
143
158
|
getRollbackTargetCheckpoint: (
|
|
@@ -146,7 +161,7 @@ type storage = {
|
|
|
146
161
|
) => promise<option<Internal.checkpointId>>,
|
|
147
162
|
// Get rollback progress diff
|
|
148
163
|
getRollbackProgressDiff: (
|
|
149
|
-
~
|
|
164
|
+
~floors: RollbackFloors.t,
|
|
150
165
|
) => promise<
|
|
151
166
|
array<{
|
|
152
167
|
"chain_id": ChainId.t,
|
|
@@ -159,7 +174,7 @@ type storage = {
|
|
|
159
174
|
// before handing them back.
|
|
160
175
|
getRollbackData: (
|
|
161
176
|
~entityConfig: Internal.entityConfig,
|
|
162
|
-
~
|
|
177
|
+
~floors: RollbackFloors.t,
|
|
163
178
|
) => promise<(array<rollbackRemoval>, array<Internal.entity>)>,
|
|
164
179
|
// Write batch to storage
|
|
165
180
|
writeBatch: (
|
|
@@ -262,27 +277,18 @@ let init = {
|
|
|
262
277
|
}
|
|
263
278
|
) {
|
|
264
279
|
Logging.info(`Found existing indexer storage. Resuming indexing state...`)
|
|
265
|
-
let initialState = await persistence.storage.resumeInitialState(
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
}
|
|
278
|
-
| _ => false
|
|
279
|
-
}
|
|
280
|
-
| _ => false
|
|
281
|
-
}
|
|
282
|
-
Config.throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
|
|
283
|
-
if !(initialState.contractMapping->ContractMapping.isEqual(contractMapping)) {
|
|
284
|
-
Config.throwIfIncompatible(["contracts"], ~resetCommand, ~runCommand, ~hasClickhouse)
|
|
285
|
-
}
|
|
280
|
+
let initialState = await persistence.storage.resumeInitialState(
|
|
281
|
+
~entities=persistence.allEntities,
|
|
282
|
+
~throwIfIncompatible=(~storedEnvioInfo, ~storedContractMapping) =>
|
|
283
|
+
Config.throwIfResumeIncompatible(
|
|
284
|
+
~storedEnvioInfo,
|
|
285
|
+
~storedContractMapping,
|
|
286
|
+
~envioInfo,
|
|
287
|
+
~contractMapping,
|
|
288
|
+
~resetCommand,
|
|
289
|
+
~runCommand,
|
|
290
|
+
),
|
|
291
|
+
)
|
|
286
292
|
persistence.storageStatus = Ready(initialState)
|
|
287
293
|
let progress = Dict.make()
|
|
288
294
|
initialState.chains->Array.forEach(c => {
|
package/src/Persistence.res.mjs
CHANGED
|
@@ -5,7 +5,6 @@ import * as Logging from "./Logging.res.mjs";
|
|
|
5
5
|
import * as EntityHistory from "./db/EntityHistory.res.mjs";
|
|
6
6
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
7
7
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
8
|
-
import * as ContractMapping from "./ContractMapping.res.mjs";
|
|
9
8
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
10
9
|
|
|
11
10
|
let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
|
|
@@ -61,25 +60,7 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
|
|
|
61
60
|
tmp = typeof match !== "object" ? false : match.TAG === "Initializing";
|
|
62
61
|
if (tmp) {
|
|
63
62
|
Logging.info(`Found existing indexer storage. Resuming indexing state...`);
|
|
64
|
-
let initialState$1 = await persistence.storage.resumeInitialState();
|
|
65
|
-
let stored = initialState$1.envioInfo;
|
|
66
|
-
let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["storage was initialized by an older envio version"];
|
|
67
|
-
let hasClickhouse;
|
|
68
|
-
if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
|
|
69
|
-
let match$1 = envioInfo["storage"];
|
|
70
|
-
if (typeof match$1 === "object" && match$1 !== null && !Array.isArray(match$1)) {
|
|
71
|
-
let match$2 = match$1["clickhouse"];
|
|
72
|
-
hasClickhouse = match$2 === true;
|
|
73
|
-
} else {
|
|
74
|
-
hasClickhouse = false;
|
|
75
|
-
}
|
|
76
|
-
} else {
|
|
77
|
-
hasClickhouse = false;
|
|
78
|
-
}
|
|
79
|
-
Config.throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhouse);
|
|
80
|
-
if (!ContractMapping.isEqual(initialState$1.contractMapping, contractMapping)) {
|
|
81
|
-
Config.throwIfIncompatible(["contracts"], resetCommand, runCommand, hasClickhouse);
|
|
82
|
-
}
|
|
63
|
+
let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, (storedEnvioInfo, storedContractMapping) => Config.throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand));
|
|
83
64
|
persistence.storageStatus = {
|
|
84
65
|
TAG: "Ready",
|
|
85
66
|
_0: initialState$1
|
package/src/PgStorage.res
CHANGED
|
@@ -377,7 +377,7 @@ let makeLoadQuery = (~pgSchema, ~tableName, ~condition) => {
|
|
|
377
377
|
let rec makeFilterCondition = (
|
|
378
378
|
~filter: EntityFilter.t,
|
|
379
379
|
~table: Table.table,
|
|
380
|
-
~params: array<
|
|
380
|
+
~params: array<unknown>,
|
|
381
381
|
) => {
|
|
382
382
|
// Filters reference fields by API name, while the SQL references columns
|
|
383
383
|
// by their possibly renamed db names.
|
|
@@ -398,7 +398,7 @@ let rec makeFilterCondition = (
|
|
|
398
398
|
~fieldValue: unknown,
|
|
399
399
|
~isArray,
|
|
400
400
|
) => {
|
|
401
|
-
let param = try fieldValue->S.
|
|
401
|
+
let param = try fieldValue->S.reverseConvertOrThrow(
|
|
402
402
|
isArray ? queryField.arrayFieldSchema : queryField.fieldSchema,
|
|
403
403
|
) catch {
|
|
404
404
|
| exn =>
|
|
@@ -438,8 +438,8 @@ let rec makeFilterCondition = (
|
|
|
438
438
|
// `LoadLayer.scopeFilter` is what puts this filter here, and the value is
|
|
439
439
|
// range-checked to a non-negative safe integer, so it can carry nothing but
|
|
440
440
|
// digits.
|
|
441
|
-
| Eq({fieldName, fieldValue}) if
|
|
442
|
-
`"${
|
|
441
|
+
| Eq({fieldName, fieldValue}) if getQueryFieldOrThrow(fieldName).isChainId =>
|
|
442
|
+
`"${getQueryFieldOrThrow(fieldName).pgDbFieldName}" = ${fieldValue
|
|
443
443
|
->ChainId.normalizeOrThrow
|
|
444
444
|
->ChainId.toString}`
|
|
445
445
|
| Eq({fieldName, fieldValue}) => scalarCondition(~fieldName, ~fieldValue, ~op="=")
|
|
@@ -587,7 +587,7 @@ let makeTableBatchSetQuery = (
|
|
|
587
587
|
~itemSchema: S.t<'item>,
|
|
588
588
|
~chainIdMode: ChainId.mode=Int32,
|
|
589
589
|
) => {
|
|
590
|
-
let {dbSchema, hasArrayField} =
|
|
590
|
+
let {dbSchema, hasArrayField, byteaColumnIndexes} =
|
|
591
591
|
table->Table.toSqlParams(~schema=itemSchema, ~pgSchema, ~chainIdMode)
|
|
592
592
|
|
|
593
593
|
// Should move this to a better place
|
|
@@ -614,7 +614,22 @@ let makeTableBatchSetQuery = (
|
|
|
614
614
|
{
|
|
615
615
|
"query": makeInsertUnnestSetQuery(~pgSchema, ~table, ~itemSchema, ~isRawEvents, ~chainIdMode),
|
|
616
616
|
"convertOrThrow": S.compile(
|
|
617
|
-
S.unnest(dbSchema)
|
|
617
|
+
S.unnest(dbSchema)->S.preprocess(_ => {
|
|
618
|
+
serializer: columns => {
|
|
619
|
+
let columns = columns->(Utils.magic: unknown => array<unknown>)
|
|
620
|
+
byteaColumnIndexes->Array.forEach(index =>
|
|
621
|
+
columns->Array.setUnsafe(
|
|
622
|
+
index,
|
|
623
|
+
columns
|
|
624
|
+
->Array.getUnsafe(index)
|
|
625
|
+
->(Utils.magic: unknown => array<unknown>)
|
|
626
|
+
->Utils.Bytes.toPgArrayLiteral
|
|
627
|
+
->(Utils.magic: string => unknown),
|
|
628
|
+
)
|
|
629
|
+
)
|
|
630
|
+
columns->(Utils.magic: array<unknown> => unknown)
|
|
631
|
+
},
|
|
632
|
+
}),
|
|
618
633
|
~input=Value,
|
|
619
634
|
~output=Unknown,
|
|
620
635
|
~mode=Sync,
|
|
@@ -1106,6 +1121,8 @@ let rec writeBatch = async (
|
|
|
1106
1121
|
| Internal.CrossChain => None
|
|
1107
1122
|
| Chain(chainId) => Some(chainId)
|
|
1108
1123
|
}
|
|
1124
|
+
let shouldSaveHistory =
|
|
1125
|
+
config->Config.shouldSaveHistory(~isInReorgThreshold, ~chainId=?scopeChainId)
|
|
1109
1126
|
let changes = switch (entityConfig.table->Table.getChainIdField, scopeChainId) {
|
|
1110
1127
|
| (Some(field), Some(chainId)) =>
|
|
1111
1128
|
changes->Array.map(change =>
|
|
@@ -1296,7 +1313,7 @@ let rec writeBatch = async (
|
|
|
1296
1313
|
//valid event identifier, where all rows created after this eventIdentifier should
|
|
1297
1314
|
//be deleted
|
|
1298
1315
|
let rollbackTables = switch rollback {
|
|
1299
|
-
| Some({
|
|
1316
|
+
| Some({floors, rolledBackAddresses, progressedChains}) =>
|
|
1300
1317
|
Some(
|
|
1301
1318
|
sql => {
|
|
1302
1319
|
// Postgres owns history tables only for Postgres-backed entities;
|
|
@@ -1309,15 +1326,24 @@ let rec writeBatch = async (
|
|
|
1309
1326
|
~pgSchema,
|
|
1310
1327
|
~entityName=entityConfig.name,
|
|
1311
1328
|
~entityIndex=entityConfig.index,
|
|
1312
|
-
~
|
|
1329
|
+
~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
|
|
1330
|
+
~floors,
|
|
1313
1331
|
)
|
|
1314
1332
|
})
|
|
1315
1333
|
promises
|
|
1316
|
-
->Array.push(
|
|
1317
|
-
sql->InternalTable.Checkpoints.rollback(~pgSchema, ~rollbackTargetCheckpointId),
|
|
1318
|
-
)
|
|
1334
|
+
->Array.push(sql->InternalTable.Checkpoints.rollback(~pgSchema, ~floors))
|
|
1319
1335
|
->ignore
|
|
1320
1336
|
|
|
1337
|
+
// Runs before the batch's own progress write below, so a chain the
|
|
1338
|
+
// batch also progressed keeps the batch's later value.
|
|
1339
|
+
if progressedChains->Utils.Array.notEmpty {
|
|
1340
|
+
promises
|
|
1341
|
+
->Array.push(
|
|
1342
|
+
sql->InternalTable.Chains.setProgressedChains(~pgSchema, ~progressedChains),
|
|
1343
|
+
)
|
|
1344
|
+
->ignore
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1321
1347
|
// Addresses are insert-only, so undoing their registrations is a
|
|
1322
1348
|
// delete rather than a history replay. It runs before the batch's own
|
|
1323
1349
|
// inserts in the same transaction, so a re-registered address lands
|
|
@@ -1476,7 +1502,11 @@ let rollbackKeyColumns = (entityConfig: Internal.entityConfig) =>
|
|
|
1476
1502
|
| None => [Table.idFieldName]
|
|
1477
1503
|
}
|
|
1478
1504
|
|
|
1479
|
-
let makeGetRollbackPreTargetRowsQuery = (
|
|
1505
|
+
let makeGetRollbackPreTargetRowsQuery = (
|
|
1506
|
+
~entityConfig: Internal.entityConfig,
|
|
1507
|
+
~pgSchema,
|
|
1508
|
+
~floors: RollbackFloors.t,
|
|
1509
|
+
) => {
|
|
1480
1510
|
let dataFieldNames = entityConfig.table.fields->Array.filterMap(fieldOrDerived =>
|
|
1481
1511
|
switch fieldOrDerived {
|
|
1482
1512
|
| Field(field) => field->Table.getPgDbFieldName->Some
|
|
@@ -1484,56 +1514,70 @@ let makeGetRollbackPreTargetRowsQuery = (~entityConfig: Internal.entityConfig, ~
|
|
|
1484
1514
|
}
|
|
1485
1515
|
)
|
|
1486
1516
|
|
|
1487
|
-
let dataFieldsCommaSeparated =
|
|
1488
|
-
dataFieldNames->Array.map(name => `"${name}"`)->Array.joinUnsafe(", ")
|
|
1489
|
-
|
|
1490
1517
|
let historyTableName = EntityHistory.historyTableName(
|
|
1491
1518
|
~entityName=entityConfig.name,
|
|
1492
1519
|
~entityIndex=entityConfig.index,
|
|
1493
1520
|
)
|
|
1521
|
+
// Every column is qualified: the per-chain bounds relation joined in has a
|
|
1522
|
+
// `chain_id` of its own, which a snake_case chain column shares.
|
|
1523
|
+
let tableRef = `"${historyTableName}"`
|
|
1524
|
+
let dataFieldsCommaSeparated =
|
|
1525
|
+
dataFieldNames->Array.map(name => `${tableRef}."${name}"`)->Array.joinUnsafe(", ")
|
|
1494
1526
|
|
|
1495
1527
|
// A per-chain entity's rows are only comparable within a chain, so the row's
|
|
1496
1528
|
// identity here is (id, chain id) rather than the id alone.
|
|
1497
1529
|
let keyColumns = rollbackKeyColumns(entityConfig)
|
|
1498
|
-
let keyColumnsCommaSeparated =
|
|
1530
|
+
let keyColumnsCommaSeparated =
|
|
1531
|
+
keyColumns->Array.map(c => `${tableRef}."${c}"`)->Array.joinUnsafe(", ")
|
|
1499
1532
|
let keyMatch =
|
|
1500
|
-
keyColumns
|
|
1501
|
-
|
|
1502
|
-
->
|
|
1533
|
+
keyColumns->Array.map(c => `h."${c}" = ${tableRef}."${c}"`)->Array.joinUnsafe(" AND ")
|
|
1534
|
+
let bounds =
|
|
1535
|
+
floors.floors->CheckpointBounds.sql(
|
|
1536
|
+
~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
|
|
1537
|
+
~tableRef,
|
|
1538
|
+
)
|
|
1503
1539
|
|
|
1504
|
-
`SELECT DISTINCT ON (${keyColumnsCommaSeparated}) ${dataFieldsCommaSeparated}, "${EntityHistory.changeFieldName}"
|
|
1505
|
-
FROM "${pgSchema}"."${historyTableName}"
|
|
1506
|
-
WHERE "${EntityHistory.checkpointIdFieldName}" <= $
|
|
1540
|
+
`SELECT DISTINCT ON (${keyColumnsCommaSeparated}) ${dataFieldsCommaSeparated}, ${tableRef}."${EntityHistory.changeFieldName}"
|
|
1541
|
+
FROM "${pgSchema}"."${historyTableName}"${bounds.join}
|
|
1542
|
+
WHERE ${tableRef}."${EntityHistory.checkpointIdFieldName}" <= ${bounds.checkpointId}
|
|
1507
1543
|
AND EXISTS (
|
|
1508
1544
|
SELECT 1
|
|
1509
1545
|
FROM "${pgSchema}"."${historyTableName}" h
|
|
1510
1546
|
WHERE ${keyMatch}
|
|
1511
|
-
AND h."${EntityHistory.checkpointIdFieldName}" > $
|
|
1547
|
+
AND h."${EntityHistory.checkpointIdFieldName}" > ${bounds.checkpointId}
|
|
1512
1548
|
)
|
|
1513
|
-
ORDER BY ${keyColumnsCommaSeparated}, "${EntityHistory.checkpointIdFieldName}" DESC`
|
|
1549
|
+
ORDER BY ${keyColumnsCommaSeparated}, ${tableRef}."${EntityHistory.checkpointIdFieldName}" DESC`
|
|
1514
1550
|
}
|
|
1515
1551
|
|
|
1516
1552
|
// Returns entity IDs that were created after the rollback target and have no history before it.
|
|
1517
1553
|
// DELETE rows at or before the target are returned by the restore query and classified in ReScript.
|
|
1518
|
-
let makeGetRollbackRemovedIdsQuery = (
|
|
1554
|
+
let makeGetRollbackRemovedIdsQuery = (
|
|
1555
|
+
~entityConfig: Internal.entityConfig,
|
|
1556
|
+
~pgSchema,
|
|
1557
|
+
~floors: RollbackFloors.t,
|
|
1558
|
+
) => {
|
|
1519
1559
|
let historyTableName = EntityHistory.historyTableName(
|
|
1520
1560
|
~entityName=entityConfig.name,
|
|
1521
1561
|
~entityIndex=entityConfig.index,
|
|
1522
1562
|
)
|
|
1563
|
+
let tableRef = `"${historyTableName}"`
|
|
1523
1564
|
let keyColumns = rollbackKeyColumns(entityConfig)
|
|
1524
1565
|
let keyMatch =
|
|
1525
|
-
keyColumns
|
|
1526
|
-
|
|
1527
|
-
->
|
|
1566
|
+
keyColumns->Array.map(c => `h."${c}" = ${tableRef}."${c}"`)->Array.joinUnsafe(" AND ")
|
|
1567
|
+
let bounds =
|
|
1568
|
+
floors.floors->CheckpointBounds.sql(
|
|
1569
|
+
~chainIdColumn=entityConfig.table->Table.getPgChainIdColumn,
|
|
1570
|
+
~tableRef,
|
|
1571
|
+
)
|
|
1528
1572
|
|
|
1529
|
-
`SELECT DISTINCT ${keyColumns->Array.map(c =>
|
|
1530
|
-
FROM "${pgSchema}"."${historyTableName}"
|
|
1531
|
-
WHERE "${EntityHistory.checkpointIdFieldName}" > $
|
|
1573
|
+
`SELECT DISTINCT ${keyColumns->Array.map(c => `${tableRef}."${c}"`)->Array.joinUnsafe(", ")}
|
|
1574
|
+
FROM "${pgSchema}"."${historyTableName}"${bounds.join}
|
|
1575
|
+
WHERE ${tableRef}."${EntityHistory.checkpointIdFieldName}" > ${bounds.checkpointId}
|
|
1532
1576
|
AND NOT EXISTS (
|
|
1533
1577
|
SELECT 1
|
|
1534
1578
|
FROM "${pgSchema}"."${historyTableName}" h
|
|
1535
1579
|
WHERE ${keyMatch}
|
|
1536
|
-
AND h."${EntityHistory.checkpointIdFieldName}" <= $
|
|
1580
|
+
AND h."${EntityHistory.checkpointIdFieldName}" <= ${bounds.checkpointId}
|
|
1537
1581
|
)`
|
|
1538
1582
|
}
|
|
1539
1583
|
|
|
@@ -1774,11 +1818,9 @@ let make = (
|
|
|
1774
1818
|
~contractMapping,
|
|
1775
1819
|
~envioInfo,
|
|
1776
1820
|
): Persistence.initialState => {
|
|
1777
|
-
//
|
|
1778
|
-
//
|
|
1779
|
-
// ClickHouse.
|
|
1821
|
+
// PG owns tables only for entities that opted into Postgres; the sink
|
|
1822
|
+
// picks its own out of the full list.
|
|
1780
1823
|
let pgEntities = entities->Array.filter((e: Internal.entityConfig) => e.storage.postgres)
|
|
1781
|
-
let chEntities = entities->Array.filter((e: Internal.entityConfig) => e.storage.clickhouse)
|
|
1782
1824
|
|
|
1783
1825
|
let schemaTableNames: array<schemaTableName> = await sql->Postgres.unsafe(
|
|
1784
1826
|
makeSchemaTableNamesQuery(~pgSchema),
|
|
@@ -1807,7 +1849,7 @@ let make = (
|
|
|
1807
1849
|
|
|
1808
1850
|
// Call sink.initialize before executing PG queries
|
|
1809
1851
|
switch sink {
|
|
1810
|
-
| Some(sink) => await sink.initialize(~
|
|
1852
|
+
| Some(sink) => await sink.initialize(~entities)
|
|
1811
1853
|
| None => ()
|
|
1812
1854
|
}
|
|
1813
1855
|
|
|
@@ -2271,8 +2313,14 @@ let make = (
|
|
|
2271
2313
|
}
|
|
2272
2314
|
}
|
|
2273
2315
|
|
|
2274
|
-
let resumeInitialState = async (): Persistence.initialState => {
|
|
2275
|
-
let (
|
|
2316
|
+
let resumeInitialState = async (~entities, ~throwIfIncompatible): Persistence.initialState => {
|
|
2317
|
+
let (
|
|
2318
|
+
cache,
|
|
2319
|
+
chains,
|
|
2320
|
+
checkpointIdResult,
|
|
2321
|
+
reorgCheckpoints,
|
|
2322
|
+
(storedEnvioInfo, storedContractMapping),
|
|
2323
|
+
) = await Promise.all5((
|
|
2276
2324
|
restoreEffectCache(~withUpload=false),
|
|
2277
2325
|
InternalTable.Chains.getInitialState(
|
|
2278
2326
|
sql,
|
|
@@ -2320,6 +2368,8 @@ let make = (
|
|
|
2320
2368
|
),
|
|
2321
2369
|
))
|
|
2322
2370
|
|
|
2371
|
+
throwIfIncompatible(~storedEnvioInfo, ~storedContractMapping)
|
|
2372
|
+
|
|
2323
2373
|
await reloadIndexCatalog()
|
|
2324
2374
|
|
|
2325
2375
|
let checkpointId = (checkpointIdResult->Array.getUnsafe(0))["id"]->BigInt.fromStringOrThrow
|
|
@@ -2334,7 +2384,7 @@ let make = (
|
|
|
2334
2384
|
|
|
2335
2385
|
// Resume sink if present - needed to rollback any reorg changes
|
|
2336
2386
|
switch sink {
|
|
2337
|
-
| Some(sink) => await sink.resume(~checkpointId)
|
|
2387
|
+
| Some(sink) => await sink.resume(~checkpointId, ~chains, ~entities)
|
|
2338
2388
|
| None => ()
|
|
2339
2389
|
}
|
|
2340
2390
|
|
|
@@ -2344,8 +2394,8 @@ let make = (
|
|
|
2344
2394
|
cache,
|
|
2345
2395
|
chains,
|
|
2346
2396
|
checkpointId,
|
|
2347
|
-
contractMapping,
|
|
2348
|
-
envioInfo,
|
|
2397
|
+
contractMapping: storedContractMapping,
|
|
2398
|
+
envioInfo: storedEnvioInfo,
|
|
2349
2399
|
}
|
|
2350
2400
|
}
|
|
2351
2401
|
|
|
@@ -2359,17 +2409,17 @@ let make = (
|
|
|
2359
2409
|
%raw(`undefined`)
|
|
2360
2410
|
)
|
|
2361
2411
|
|
|
2362
|
-
let pruneStaleCheckpoints = (~
|
|
2363
|
-
InternalTable.Checkpoints.pruneStaleCheckpoints(sql, ~pgSchema, ~
|
|
2412
|
+
let pruneStaleCheckpoints = (~safeCheckpoints) =>
|
|
2413
|
+
InternalTable.Checkpoints.pruneStaleCheckpoints(sql, ~pgSchema, ~safeCheckpoints)
|
|
2364
2414
|
|
|
2365
|
-
let pruneStaleEntityHistory = (~entityName, ~entityIndex, ~chainIdColumn, ~
|
|
2415
|
+
let pruneStaleEntityHistory = (~entityName, ~entityIndex, ~chainIdColumn, ~safeCheckpoints) =>
|
|
2366
2416
|
EntityHistory.pruneStaleEntityHistory(
|
|
2367
2417
|
sql,
|
|
2368
2418
|
~pgSchema,
|
|
2369
2419
|
~entityName,
|
|
2370
2420
|
~entityIndex,
|
|
2371
2421
|
~chainIdColumn,
|
|
2372
|
-
~
|
|
2422
|
+
~safeCheckpoints,
|
|
2373
2423
|
)
|
|
2374
2424
|
|
|
2375
2425
|
let getRollbackTargetCheckpoint = (~reorgChainId, ~lastKnownValidBlockNumber) =>
|
|
@@ -2380,26 +2430,24 @@ let make = (
|
|
|
2380
2430
|
~lastKnownValidBlockNumber,
|
|
2381
2431
|
)
|
|
2382
2432
|
|
|
2383
|
-
let getRollbackProgressDiff = (~
|
|
2384
|
-
InternalTable.Checkpoints.getRollbackProgressDiff(sql, ~pgSchema, ~
|
|
2433
|
+
let getRollbackProgressDiff = (~floors) =>
|
|
2434
|
+
InternalTable.Checkpoints.getRollbackProgressDiff(sql, ~pgSchema, ~floors)
|
|
2385
2435
|
|
|
2386
|
-
let getRollbackData = async (
|
|
2387
|
-
|
|
2388
|
-
~rollbackTargetCheckpointId,
|
|
2389
|
-
) => {
|
|
2436
|
+
let getRollbackData = async (~entityConfig: Internal.entityConfig, ~floors: RollbackFloors.t) => {
|
|
2437
|
+
let params = floors.floors->CheckpointBounds.params
|
|
2390
2438
|
let (removedIdRows, rollbackRows) = await Promise.all2((
|
|
2391
2439
|
// Get IDs of entities that should be deleted (created after rollback target with no prior history)
|
|
2392
2440
|
sql
|
|
2393
2441
|
->Postgres.preparedUnsafe(
|
|
2394
|
-
makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema),
|
|
2395
|
-
|
|
2442
|
+
makeGetRollbackRemovedIdsQuery(~entityConfig, ~pgSchema, ~floors),
|
|
2443
|
+
params,
|
|
2396
2444
|
)
|
|
2397
2445
|
->(Utils.magic: promise<unknown> => promise<array<unknown>>),
|
|
2398
2446
|
// Get the latest pre-target row, including its SET or DELETE action.
|
|
2399
2447
|
sql
|
|
2400
2448
|
->Postgres.preparedUnsafe(
|
|
2401
|
-
makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema),
|
|
2402
|
-
|
|
2449
|
+
makeGetRollbackPreTargetRowsQuery(~entityConfig, ~pgSchema, ~floors),
|
|
2450
|
+
params,
|
|
2403
2451
|
)
|
|
2404
2452
|
->(Utils.magic: promise<unknown> => promise<array<unknown>>),
|
|
2405
2453
|
))
|