envio 3.0.0-alpha.21 → 3.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +2 -48
- package/evm.schema.json +67 -0
- package/fuel.schema.json +67 -0
- package/index.d.ts +822 -38
- package/index.js +5 -3
- package/package.json +10 -8
- package/rescript.json +5 -9
- package/src/Address.res +4 -5
- package/src/Address.res.mjs +9 -12
- package/src/Api.res +15 -0
- package/src/Api.res.mjs +20 -0
- package/src/Batch.res +32 -34
- package/src/Batch.res.mjs +172 -187
- package/src/Bin.res +89 -0
- package/src/Bin.res.mjs +97 -0
- package/src/ChainFetcher.res +33 -57
- package/src/ChainFetcher.res.mjs +197 -227
- package/src/ChainManager.res +6 -14
- package/src/ChainManager.res.mjs +74 -85
- package/src/ChainMap.res +14 -16
- package/src/ChainMap.res.mjs +38 -38
- package/src/Config.res +193 -135
- package/src/Config.res.mjs +566 -592
- package/src/Core.res +182 -0
- package/src/Core.res.mjs +207 -0
- package/src/Ecosystem.res +25 -4
- package/src/Ecosystem.res.mjs +12 -13
- package/src/Env.res +20 -13
- package/src/Env.res.mjs +124 -113
- package/src/EnvSafe.res +269 -0
- package/src/EnvSafe.res.mjs +296 -0
- package/src/EnvSafe.resi +18 -0
- package/src/Envio.res +37 -26
- package/src/Envio.res.mjs +59 -60
- package/src/ErrorHandling.res +2 -2
- package/src/ErrorHandling.res.mjs +15 -15
- package/src/EventConfigBuilder.res +219 -81
- package/src/EventConfigBuilder.res.mjs +259 -202
- package/src/EventProcessing.res +27 -38
- package/src/EventProcessing.res.mjs +165 -183
- package/src/EventUtils.res +11 -11
- package/src/EventUtils.res.mjs +21 -22
- package/src/EvmTypes.res +0 -1
- package/src/EvmTypes.res.mjs +5 -5
- package/src/FetchState.res +360 -256
- package/src/FetchState.res.mjs +958 -914
- package/src/GlobalState.res +365 -351
- package/src/GlobalState.res.mjs +958 -992
- package/src/GlobalStateManager.res +1 -2
- package/src/GlobalStateManager.res.mjs +36 -44
- package/src/HandlerLoader.res +107 -23
- package/src/HandlerLoader.res.mjs +128 -38
- package/src/HandlerRegister.res +127 -103
- package/src/HandlerRegister.res.mjs +164 -164
- package/src/HandlerRegister.resi +12 -4
- package/src/Hasura.res +35 -22
- package/src/Hasura.res.mjs +158 -167
- package/src/InMemoryStore.res +20 -27
- package/src/InMemoryStore.res.mjs +64 -80
- package/src/InMemoryTable.res +34 -39
- package/src/InMemoryTable.res.mjs +165 -170
- package/src/Internal.res +52 -33
- package/src/Internal.res.mjs +84 -81
- package/src/LazyLoader.res.mjs +55 -61
- package/src/LoadLayer.res +77 -78
- package/src/LoadLayer.res.mjs +160 -189
- package/src/LoadManager.res +16 -21
- package/src/LoadManager.res.mjs +79 -84
- package/src/LogSelection.res +236 -68
- package/src/LogSelection.res.mjs +211 -141
- package/src/Logging.res +13 -9
- package/src/Logging.res.mjs +130 -143
- package/src/Main.res +428 -51
- package/src/Main.res.mjs +528 -271
- package/src/Persistence.res +77 -84
- package/src/Persistence.res.mjs +131 -132
- package/src/PgStorage.res +291 -167
- package/src/PgStorage.res.mjs +797 -817
- package/src/Prometheus.res +50 -58
- package/src/Prometheus.res.mjs +345 -373
- package/src/ReorgDetection.res +22 -24
- package/src/ReorgDetection.res.mjs +100 -106
- package/src/SafeCheckpointTracking.res +7 -7
- package/src/SafeCheckpointTracking.res.mjs +40 -43
- package/src/SimulateItems.res +41 -49
- package/src/SimulateItems.res.mjs +257 -272
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +22 -26
- package/src/TableIndices.res +1 -2
- package/src/TableIndices.res.mjs +42 -48
- package/src/TestIndexer.res +196 -189
- package/src/TestIndexer.res.mjs +536 -536
- package/src/TestIndexerProxyStorage.res +15 -16
- package/src/TestIndexerProxyStorage.res.mjs +98 -122
- package/src/TestIndexerWorker.res +4 -0
- package/src/TestIndexerWorker.res.mjs +7 -0
- package/src/Throttler.res +3 -3
- package/src/Throttler.res.mjs +23 -24
- package/src/Time.res +1 -1
- package/src/Time.res.mjs +18 -21
- package/src/TopicFilter.res +3 -3
- package/src/TopicFilter.res.mjs +29 -30
- package/src/UserContext.res +93 -54
- package/src/UserContext.res.mjs +197 -182
- package/src/Utils.res +141 -86
- package/src/Utils.res.mjs +334 -295
- package/src/bindings/BigDecimal.res +0 -2
- package/src/bindings/BigDecimal.res.mjs +19 -23
- package/src/bindings/ClickHouse.res +28 -27
- package/src/bindings/ClickHouse.res.mjs +243 -240
- package/src/bindings/DateFns.res +11 -11
- package/src/bindings/DateFns.res.mjs +7 -7
- package/src/bindings/EventSource.res.mjs +2 -2
- package/src/bindings/Express.res +2 -5
- package/src/bindings/Hrtime.res +2 -2
- package/src/bindings/Hrtime.res.mjs +30 -32
- package/src/bindings/Lodash.res.mjs +1 -1
- package/src/bindings/NodeJs.res +14 -9
- package/src/bindings/NodeJs.res.mjs +20 -20
- package/src/bindings/Pino.res +8 -10
- package/src/bindings/Pino.res.mjs +40 -43
- package/src/bindings/Postgres.res +2 -5
- package/src/bindings/Postgres.res.mjs +9 -9
- package/src/bindings/PromClient.res +17 -2
- package/src/bindings/PromClient.res.mjs +30 -7
- package/src/bindings/SDSL.res.mjs +2 -2
- package/src/bindings/Viem.res +4 -4
- package/src/bindings/Viem.res.mjs +20 -22
- package/src/bindings/Vitest.res +1 -1
- package/src/bindings/Vitest.res.mjs +2 -2
- package/src/bindings/WebSocket.res +1 -1
- package/src/db/EntityHistory.res +9 -3
- package/src/db/EntityHistory.res.mjs +84 -59
- package/src/db/InternalTable.res +62 -60
- package/src/db/InternalTable.res.mjs +271 -203
- package/src/db/Schema.res +1 -2
- package/src/db/Schema.res.mjs +28 -32
- package/src/db/Table.res +28 -27
- package/src/db/Table.res.mjs +276 -292
- package/src/sources/EventRouter.res +21 -16
- package/src/sources/EventRouter.res.mjs +55 -57
- package/src/sources/Evm.res +17 -1
- package/src/sources/Evm.res.mjs +16 -8
- package/src/sources/EvmChain.res +15 -17
- package/src/sources/EvmChain.res.mjs +40 -42
- package/src/sources/Fuel.res +14 -1
- package/src/sources/Fuel.res.mjs +16 -8
- package/src/sources/FuelSDK.res +1 -1
- package/src/sources/FuelSDK.res.mjs +6 -8
- package/src/sources/HyperFuel.res +8 -10
- package/src/sources/HyperFuel.res.mjs +113 -123
- package/src/sources/HyperFuelClient.res.mjs +6 -7
- package/src/sources/HyperFuelSource.res +19 -20
- package/src/sources/HyperFuelSource.res.mjs +339 -356
- package/src/sources/HyperSync.res +11 -13
- package/src/sources/HyperSync.res.mjs +206 -220
- package/src/sources/HyperSyncClient.res +5 -7
- package/src/sources/HyperSyncClient.res.mjs +70 -75
- package/src/sources/HyperSyncHeightStream.res +8 -9
- package/src/sources/HyperSyncHeightStream.res.mjs +78 -86
- package/src/sources/HyperSyncJsonApi.res +18 -15
- package/src/sources/HyperSyncJsonApi.res.mjs +201 -231
- package/src/sources/HyperSyncSource.res +17 -21
- package/src/sources/HyperSyncSource.res.mjs +268 -290
- package/src/sources/Rpc.res +5 -5
- package/src/sources/Rpc.res.mjs +168 -192
- package/src/sources/RpcSource.res +166 -167
- package/src/sources/RpcSource.res.mjs +972 -1046
- package/src/sources/RpcWebSocketHeightStream.res +10 -11
- package/src/sources/RpcWebSocketHeightStream.res.mjs +131 -145
- package/src/sources/SimulateSource.res +1 -1
- package/src/sources/SimulateSource.res.mjs +35 -38
- package/src/sources/Source.res +1 -1
- package/src/sources/Source.res.mjs +3 -3
- package/src/sources/SourceManager.res +39 -20
- package/src/sources/SourceManager.res.mjs +340 -371
- package/src/sources/SourceManager.resi +2 -1
- package/src/sources/Svm.res +12 -5
- package/src/sources/Svm.res.mjs +44 -41
- package/src/tui/Tui.res +23 -12
- package/src/tui/Tui.res.mjs +292 -290
- package/src/tui/bindings/Ink.res +2 -4
- package/src/tui/bindings/Ink.res.mjs +35 -41
- package/src/tui/components/BufferedProgressBar.res +7 -7
- package/src/tui/components/BufferedProgressBar.res.mjs +46 -46
- package/src/tui/components/CustomHooks.res +1 -2
- package/src/tui/components/CustomHooks.res.mjs +102 -122
- package/src/tui/components/Messages.res +1 -2
- package/src/tui/components/Messages.res.mjs +38 -42
- package/src/tui/components/SyncETA.res +10 -11
- package/src/tui/components/SyncETA.res.mjs +178 -196
- package/src/tui/components/TuiData.res +1 -1
- package/src/tui/components/TuiData.res.mjs +7 -6
- package/src/vendored/Rest.res +52 -66
- package/src/vendored/Rest.res.mjs +324 -364
- package/svm.schema.json +67 -0
- package/src/Address.gen.ts +0 -8
- package/src/Config.gen.ts +0 -19
- package/src/Envio.gen.ts +0 -55
- package/src/EvmTypes.gen.ts +0 -6
- package/src/InMemoryStore.gen.ts +0 -6
- package/src/Internal.gen.ts +0 -64
- package/src/PgStorage.gen.ts +0 -10
- package/src/PgStorage.res.d.mts +0 -5
- package/src/Types.ts +0 -56
- package/src/bindings/BigDecimal.gen.ts +0 -14
- package/src/bindings/BigDecimal.res.d.mts +0 -5
- package/src/bindings/BigInt.gen.ts +0 -10
- package/src/bindings/BigInt.res +0 -70
- package/src/bindings/BigInt.res.d.mts +0 -5
- package/src/bindings/BigInt.res.mjs +0 -154
- package/src/bindings/Ethers.res.d.mts +0 -5
- package/src/bindings/Pino.gen.ts +0 -17
- package/src/bindings/Postgres.gen.ts +0 -8
- package/src/bindings/Postgres.res.d.mts +0 -5
- package/src/bindings/Promise.res +0 -67
- package/src/bindings/Promise.res.mjs +0 -26
- package/src/db/InternalTable.gen.ts +0 -36
- package/src/sources/HyperSyncClient.gen.ts +0 -19
|
@@ -2,168 +2,159 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Table from "../db/Table.res.mjs";
|
|
4
4
|
import * as Utils from "../Utils.res.mjs";
|
|
5
|
-
import * as Js_exn from "rescript/lib/es6/js_exn.js";
|
|
6
5
|
import * as Logging from "../Logging.res.mjs";
|
|
7
|
-
import * as Belt_Array from "rescript/lib/es6/
|
|
6
|
+
import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
|
|
8
7
|
import * as Persistence from "../Persistence.res.mjs";
|
|
9
8
|
import * as EntityHistory from "../db/EntityHistory.res.mjs";
|
|
10
9
|
import * as InternalTable from "../db/InternalTable.res.mjs";
|
|
10
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
11
11
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
12
|
-
import * as
|
|
12
|
+
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
13
13
|
|
|
14
14
|
function getClickHouseFieldType(fieldType, isNullable, isArray) {
|
|
15
|
-
|
|
15
|
+
let baseType;
|
|
16
16
|
if (typeof fieldType !== "object") {
|
|
17
17
|
switch (fieldType) {
|
|
18
18
|
case "Boolean" :
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
baseType = "Bool";
|
|
20
|
+
break;
|
|
21
21
|
case "Uint32" :
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
baseType = "UInt32";
|
|
23
|
+
break;
|
|
24
24
|
case "UInt52" :
|
|
25
25
|
case "UInt64" :
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
baseType = "UInt64";
|
|
27
|
+
break;
|
|
28
28
|
case "Number" :
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
baseType = "Float64";
|
|
30
|
+
break;
|
|
31
31
|
case "Int32" :
|
|
32
32
|
case "Serial" :
|
|
33
|
-
|
|
34
|
-
|
|
33
|
+
baseType = "Int32";
|
|
34
|
+
break;
|
|
35
35
|
case "BigSerial" :
|
|
36
|
-
|
|
37
|
-
|
|
36
|
+
baseType = "Int64";
|
|
37
|
+
break;
|
|
38
38
|
case "String" :
|
|
39
39
|
case "Json" :
|
|
40
|
-
|
|
41
|
-
|
|
40
|
+
baseType = "String";
|
|
41
|
+
break;
|
|
42
42
|
case "Date" :
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
baseType = "DateTime64(3, 'UTC')";
|
|
44
|
+
break;
|
|
46
45
|
}
|
|
47
46
|
} else {
|
|
48
47
|
switch (fieldType.type) {
|
|
49
48
|
case "BigInt" :
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
49
|
+
let precision = fieldType.precision;
|
|
50
|
+
baseType = precision !== undefined && precision <= 38 ? `Decimal(` + precision.toString() + `,0)` : "String";
|
|
51
|
+
break;
|
|
53
52
|
case "BigDecimal" :
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
53
|
+
let config = fieldType.config;
|
|
54
|
+
if (config !== undefined) {
|
|
55
|
+
let scale = config[1];
|
|
56
|
+
let precision$1 = config[0];
|
|
57
|
+
baseType = precision$1 > 38 || scale > precision$1 ? "String" : `Decimal(` + precision$1.toString() + `,` + scale.toString() + `)`;
|
|
58
|
+
} else {
|
|
59
|
+
baseType = "String";
|
|
60
|
+
}
|
|
61
|
+
break;
|
|
63
62
|
case "Enum" :
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
baseType = enumType + "(" + enumValues + ")";
|
|
71
|
-
break;
|
|
63
|
+
let config$1 = fieldType.config;
|
|
64
|
+
let variantsLength = config$1.variants.length;
|
|
65
|
+
let enumType = variantsLength <= 127 ? "Enum8" : "Enum16";
|
|
66
|
+
let enumValues = Belt_Array.map(config$1.variants, variant => `'` + variant + `'`).join(", ");
|
|
67
|
+
baseType = enumType + `(` + enumValues + `)`;
|
|
68
|
+
break;
|
|
72
69
|
case "Entity" :
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
70
|
+
baseType = "String";
|
|
71
|
+
break;
|
|
76
72
|
}
|
|
77
73
|
}
|
|
78
|
-
|
|
74
|
+
let baseType$1 = isArray ? `Array(` + baseType + `)` : baseType;
|
|
79
75
|
if (isNullable) {
|
|
80
|
-
return
|
|
76
|
+
return `Nullable(` + baseType$1 + `)`;
|
|
81
77
|
} else {
|
|
82
78
|
return baseType$1;
|
|
83
79
|
}
|
|
84
80
|
}
|
|
85
81
|
|
|
86
82
|
function makeClickHouseEntitySchema(table) {
|
|
87
|
-
return S$RescriptSchema.schema(
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
dict[fieldName] = s.m(fieldSchema);
|
|
124
|
-
}));
|
|
125
|
-
return dict;
|
|
126
|
-
});
|
|
83
|
+
return S$RescriptSchema.schema(s => {
|
|
84
|
+
let dict = {};
|
|
85
|
+
Belt_Array.forEach(table.fields, field => {
|
|
86
|
+
if (field.TAG !== "Field") {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
let f = field._0;
|
|
90
|
+
let fieldName = Table.getDbFieldName(f);
|
|
91
|
+
let match = f.fieldType;
|
|
92
|
+
let fieldSchema;
|
|
93
|
+
if (typeof match !== "object") {
|
|
94
|
+
switch (match) {
|
|
95
|
+
case "UInt52" :
|
|
96
|
+
let uint52Schema = S$RescriptSchema.preprocess(S$RescriptSchema.float, param => ({
|
|
97
|
+
p: unknown => parseFloat(unknown)
|
|
98
|
+
}));
|
|
99
|
+
fieldSchema = f.isNullable ? S$RescriptSchema.$$null(uint52Schema) : (
|
|
100
|
+
f.isArray ? S$RescriptSchema.array(uint52Schema) : uint52Schema
|
|
101
|
+
);
|
|
102
|
+
break;
|
|
103
|
+
case "Date" :
|
|
104
|
+
let dateSchema = Utils.Schema.clickHouseDate;
|
|
105
|
+
fieldSchema = f.isNullable ? S$RescriptSchema.$$null(dateSchema) : (
|
|
106
|
+
f.isArray ? S$RescriptSchema.array(dateSchema) : dateSchema
|
|
107
|
+
);
|
|
108
|
+
break;
|
|
109
|
+
default:
|
|
110
|
+
fieldSchema = f.fieldSchema;
|
|
111
|
+
}
|
|
112
|
+
} else {
|
|
113
|
+
fieldSchema = f.fieldSchema;
|
|
114
|
+
}
|
|
115
|
+
dict[fieldName] = s.m(fieldSchema);
|
|
116
|
+
});
|
|
117
|
+
return dict;
|
|
118
|
+
});
|
|
127
119
|
}
|
|
128
120
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
121
|
+
let logger = Logging.createChild({
|
|
122
|
+
context: "ClickHouse"
|
|
123
|
+
});
|
|
132
124
|
|
|
133
125
|
async function insertWithRetry(client, table, values, format, retriesOpt) {
|
|
134
|
-
|
|
126
|
+
let retries = retriesOpt !== undefined ? retriesOpt : 8;
|
|
135
127
|
try {
|
|
136
128
|
return await client.insert({
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
129
|
+
table: table,
|
|
130
|
+
values: values,
|
|
131
|
+
format: format
|
|
132
|
+
});
|
|
133
|
+
} catch (raw_exn) {
|
|
134
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
144
135
|
if (retries > 0) {
|
|
145
|
-
|
|
136
|
+
let delayMs = Math.min(1000, 100 + ((900 * (8 - retries | 0) | 0) / 7 | 0) | 0);
|
|
146
137
|
if (values.length > 1) {
|
|
147
138
|
Logging.childWarn(logger, {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
139
|
+
msg: "ClickHouse insert failed, splitting batch in half and retrying",
|
|
140
|
+
table: table,
|
|
141
|
+
batchSize: values.length,
|
|
142
|
+
retriesLeft: retries,
|
|
143
|
+
err: Utils.prettifyExn(exn)
|
|
144
|
+
});
|
|
154
145
|
await Utils.delay(delayMs);
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
146
|
+
let mid = (values.length >> 1);
|
|
147
|
+
let first = values.slice(0, mid);
|
|
148
|
+
let second = values.slice(mid);
|
|
158
149
|
await insertWithRetry(client, table, first, format, retries - 1 | 0);
|
|
159
150
|
return await insertWithRetry(client, table, second, format, retries - 1 | 0);
|
|
160
151
|
}
|
|
161
152
|
Logging.childWarn(logger, {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
153
|
+
msg: "ClickHouse insert failed, retrying after delay",
|
|
154
|
+
table: table,
|
|
155
|
+
retriesLeft: retries,
|
|
156
|
+
err: Utils.prettifyExn(exn)
|
|
157
|
+
});
|
|
167
158
|
await Utils.delay(delayMs);
|
|
168
159
|
return await insertWithRetry(client, table, values, format, retries - 1 | 0);
|
|
169
160
|
}
|
|
@@ -172,144 +163,158 @@ async function insertWithRetry(client, table, values, format, retriesOpt) {
|
|
|
172
163
|
}
|
|
173
164
|
|
|
174
165
|
async function setCheckpointsOrThrow(client, batch, database) {
|
|
175
|
-
|
|
166
|
+
let checkpointsCount = batch.checkpointIds.length;
|
|
176
167
|
if (checkpointsCount === 0) {
|
|
177
|
-
return
|
|
168
|
+
return;
|
|
178
169
|
}
|
|
179
|
-
|
|
180
|
-
for(
|
|
170
|
+
let checkpointRows = [];
|
|
171
|
+
for (let idx = 0; idx < checkpointsCount; ++idx) {
|
|
181
172
|
checkpointRows.push([
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
173
|
+
batch.checkpointIds[idx].toString(),
|
|
174
|
+
batch.checkpointChainIds[idx],
|
|
175
|
+
batch.checkpointBlockNumbers[idx],
|
|
176
|
+
batch.checkpointBlockHashes[idx],
|
|
177
|
+
batch.checkpointEventsProcessed[idx]
|
|
178
|
+
]);
|
|
188
179
|
}
|
|
189
180
|
try {
|
|
190
|
-
return await insertWithRetry(client, database +
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
181
|
+
return await insertWithRetry(client, database + `.\`` + InternalTable.Checkpoints.table.tableName + `\``, checkpointRows, "JSONCompactEachRow", undefined);
|
|
182
|
+
} catch (raw_exn) {
|
|
183
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
194
184
|
throw {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
185
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
186
|
+
message: `Failed to insert checkpoints into ClickHouse table "` + InternalTable.Checkpoints.table.tableName + `"`,
|
|
187
|
+
reason: Utils.prettifyExn(exn),
|
|
188
|
+
Error: new Error()
|
|
189
|
+
};
|
|
200
190
|
}
|
|
201
191
|
}
|
|
202
192
|
|
|
203
193
|
async function setUpdatesOrThrow(client, cache, updates, entityConfig, database) {
|
|
204
194
|
if (updates.length === 0) {
|
|
205
|
-
return
|
|
195
|
+
return;
|
|
206
196
|
}
|
|
207
|
-
|
|
208
|
-
|
|
197
|
+
let cached = cache.get(entityConfig);
|
|
198
|
+
let match;
|
|
209
199
|
if (cached !== undefined) {
|
|
210
200
|
match = cached;
|
|
211
201
|
} else {
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
202
|
+
let cached_tableName = database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\``;
|
|
203
|
+
let cached_convertOrThrow = S$RescriptSchema.compile(S$RescriptSchema.union([
|
|
204
|
+
EntityHistory.makeSetUpdateSchema(makeClickHouseEntitySchema(entityConfig.table)),
|
|
205
|
+
S$RescriptSchema.object(s => {
|
|
206
|
+
s.tag(EntityHistory.changeFieldName, "DELETE");
|
|
207
|
+
return {
|
|
208
|
+
type: "DELETE",
|
|
209
|
+
entityId: s.f(Table.idFieldName, S$RescriptSchema.string),
|
|
210
|
+
checkpointId: s.f(EntityHistory.checkpointIdFieldName, EntityHistory.unsafeCheckpointIdSchema)
|
|
211
|
+
};
|
|
212
|
+
})
|
|
213
|
+
]), "Output", "Json", "Sync", false);
|
|
214
|
+
let cached$1 = {
|
|
225
215
|
tableName: cached_tableName,
|
|
226
216
|
convertOrThrow: cached_convertOrThrow
|
|
227
217
|
};
|
|
228
218
|
cache.set(entityConfig, cached$1);
|
|
229
219
|
match = cached$1;
|
|
230
220
|
}
|
|
231
|
-
|
|
232
|
-
|
|
221
|
+
let convertOrThrow = match.convertOrThrow;
|
|
222
|
+
let tableName = match.tableName;
|
|
233
223
|
try {
|
|
234
|
-
|
|
235
|
-
return convertOrThrow(update.latestChange);
|
|
236
|
-
});
|
|
224
|
+
let values = updates.map(update => convertOrThrow(update.latestChange));
|
|
237
225
|
return await insertWithRetry(client, tableName, values, "JSONEachRow", undefined);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
226
|
+
} catch (raw_exn) {
|
|
227
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
241
228
|
throw {
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
229
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
230
|
+
message: `Failed to insert items into ClickHouse table "` + tableName + `"`,
|
|
231
|
+
reason: Utils.prettifyExn(exn),
|
|
232
|
+
Error: new Error()
|
|
233
|
+
};
|
|
247
234
|
}
|
|
248
235
|
}
|
|
249
236
|
|
|
250
237
|
function makeCreateHistoryTableQuery(entityConfig, database) {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
return
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
238
|
+
let fieldDefinitions = Belt_Array.keepMap(entityConfig.table.fields, field => {
|
|
239
|
+
if (field.TAG !== "Field") {
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
let field$1 = field._0;
|
|
243
|
+
let fieldName = Table.getDbFieldName(field$1);
|
|
244
|
+
let clickHouseType = getClickHouseFieldType(field$1.fieldType, field$1.isNullable, field$1.isArray);
|
|
245
|
+
return `\`` + fieldName + `\` ` + clickHouseType;
|
|
246
|
+
});
|
|
247
|
+
return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\` (
|
|
248
|
+
` + fieldDefinitions.join(",\n ") + `,
|
|
249
|
+
\`` + EntityHistory.checkpointIdFieldName + `\` ` + getClickHouseFieldType("UInt64", false, false) + `,
|
|
250
|
+
\`` + EntityHistory.changeFieldName + `\` ` + getClickHouseFieldType({
|
|
251
|
+
type: "Enum",
|
|
252
|
+
config: EntityHistory.RowAction.config
|
|
253
|
+
}, false, false) + `
|
|
254
|
+
)
|
|
255
|
+
ENGINE = MergeTree()
|
|
256
|
+
ORDER BY (` + Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName + `)`;
|
|
264
257
|
}
|
|
265
258
|
|
|
266
259
|
function makeCreateCheckpointsTableQuery(database) {
|
|
267
|
-
return
|
|
260
|
+
return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\` (
|
|
261
|
+
\`` + "id" + `\` ` + getClickHouseFieldType("UInt64", false, false) + `,
|
|
262
|
+
\`` + "chain_id" + `\` ` + getClickHouseFieldType("Int32", false, false) + `,
|
|
263
|
+
\`` + "block_number" + `\` ` + getClickHouseFieldType("Int32", false, false) + `,
|
|
264
|
+
\`` + "block_hash" + `\` ` + getClickHouseFieldType("String", true, false) + `,
|
|
265
|
+
\`` + "events_processed" + `\` ` + getClickHouseFieldType("UInt64", false, false) + `
|
|
266
|
+
)
|
|
267
|
+
ENGINE = MergeTree()
|
|
268
|
+
ORDER BY (` + "id" + `)`;
|
|
268
269
|
}
|
|
269
270
|
|
|
270
271
|
function makeCreateViewQuery(entityConfig, database) {
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
return
|
|
272
|
+
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
273
|
+
let checkpointsTableName = InternalTable.Checkpoints.table.tableName;
|
|
274
|
+
let entityFields = Belt_Array.keepMap(entityConfig.table.fields, field => {
|
|
275
|
+
if (field.TAG !== "Field") {
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
let fieldName = Table.getDbFieldName(field._0);
|
|
279
|
+
return `\`` + fieldName + `\``;
|
|
280
|
+
}).join(", ");
|
|
281
|
+
return `CREATE VIEW IF NOT EXISTS ` + database + `.\`` + entityConfig.name + `\` AS
|
|
282
|
+
SELECT ` + entityFields + `
|
|
283
|
+
FROM (
|
|
284
|
+
SELECT ` + entityFields + `, \`` + EntityHistory.changeFieldName + `\`
|
|
285
|
+
FROM ` + database + `.\`` + historyTableName + `\`
|
|
286
|
+
WHERE \`` + EntityHistory.checkpointIdFieldName + `\` <= (SELECT max(` + "id" + `) FROM ` + database + `.\`` + checkpointsTableName + `\`)
|
|
287
|
+
ORDER BY \`` + EntityHistory.checkpointIdFieldName + `\` DESC
|
|
288
|
+
LIMIT 1 BY \`` + Table.idFieldName + `\`
|
|
289
|
+
)
|
|
290
|
+
WHERE \`` + EntityHistory.changeFieldName + `\` = '` + "SET" + `'`;
|
|
281
291
|
}
|
|
282
292
|
|
|
283
293
|
async function initialize(client, database, entities, param) {
|
|
284
294
|
try {
|
|
285
295
|
await client.exec({
|
|
286
|
-
|
|
287
|
-
|
|
296
|
+
query: `TRUNCATE DATABASE IF EXISTS ` + database
|
|
297
|
+
});
|
|
288
298
|
await client.exec({
|
|
289
|
-
|
|
290
|
-
|
|
299
|
+
query: `CREATE DATABASE IF NOT EXISTS ` + database
|
|
300
|
+
});
|
|
291
301
|
await client.exec({
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
await Promise.all(Belt_Array.map(entities,
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
});
|
|
298
|
-
})));
|
|
302
|
+
query: `USE ` + database
|
|
303
|
+
});
|
|
304
|
+
await Promise.all(Belt_Array.map(entities, entityConfig => client.exec({
|
|
305
|
+
query: makeCreateHistoryTableQuery(entityConfig, database)
|
|
306
|
+
})));
|
|
299
307
|
await client.exec({
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
await Promise.all(Belt_Array.map(entities,
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
});
|
|
306
|
-
})));
|
|
308
|
+
query: makeCreateCheckpointsTableQuery(database)
|
|
309
|
+
});
|
|
310
|
+
await Promise.all(Belt_Array.map(entities, entityConfig => client.exec({
|
|
311
|
+
query: makeCreateViewQuery(entityConfig, database)
|
|
312
|
+
})));
|
|
307
313
|
return Logging.trace("ClickHouse sink initialization completed successfully");
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
314
|
+
} catch (raw_exn) {
|
|
315
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
311
316
|
Logging.errorWithExn(exn, "Failed to initialize ClickHouse sink");
|
|
312
|
-
return
|
|
317
|
+
return Stdlib_JsError.throwWithMessage("ClickHouse initialization failed");
|
|
313
318
|
}
|
|
314
319
|
}
|
|
315
320
|
|
|
@@ -317,49 +322,47 @@ async function resume(client, database, checkpointId) {
|
|
|
317
322
|
try {
|
|
318
323
|
try {
|
|
319
324
|
await client.exec({
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
Js_exn.raiseError("ClickHouse resume failed");
|
|
325
|
+
query: `USE ` + database
|
|
326
|
+
});
|
|
327
|
+
} catch (raw_exn) {
|
|
328
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
329
|
+
Logging.errorWithExn(exn, `ClickHouse sink database "` + database + `" not found. Please run 'envio start -r' to reinitialize the indexer (it'll also drop Postgres database).`);
|
|
330
|
+
Stdlib_JsError.throwWithMessage("ClickHouse resume failed");
|
|
327
331
|
}
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
await Promise.all(Belt_Array.map(tables,
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
332
|
+
let tablesResult = await client.query({
|
|
333
|
+
query: `SHOW TABLES FROM ` + database + ` LIKE '` + EntityHistory.historyTablePrefix + `%'`
|
|
334
|
+
});
|
|
335
|
+
let tables = await tablesResult.json();
|
|
336
|
+
await Promise.all(Belt_Array.map(tables, table => {
|
|
337
|
+
let tableName = table.name;
|
|
338
|
+
return client.exec({
|
|
339
|
+
query: `ALTER TABLE ` + database + `.\`` + tableName + `\` DELETE WHERE \`` + EntityHistory.checkpointIdFieldName + `\` > ` + checkpointId.toString()
|
|
340
|
+
});
|
|
341
|
+
}));
|
|
338
342
|
return await client.exec({
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
}
|
|
342
|
-
|
|
343
|
-
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
|
|
343
|
+
query: `DELETE FROM ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\` WHERE \`` + Table.idFieldName + `\` > ` + checkpointId.toString()
|
|
344
|
+
});
|
|
345
|
+
} catch (raw_exn$1) {
|
|
346
|
+
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
|
|
344
347
|
if (exn$1.RE_EXN_ID === Persistence.StorageError) {
|
|
345
348
|
throw exn$1;
|
|
346
349
|
}
|
|
347
350
|
Logging.errorWithExn(exn$1, "Failed to resume ClickHouse sink");
|
|
348
|
-
return
|
|
351
|
+
return Stdlib_JsError.throwWithMessage("ClickHouse resume failed");
|
|
349
352
|
}
|
|
350
353
|
}
|
|
351
354
|
|
|
352
355
|
export {
|
|
353
|
-
getClickHouseFieldType
|
|
354
|
-
makeClickHouseEntitySchema
|
|
355
|
-
logger
|
|
356
|
-
insertWithRetry
|
|
357
|
-
setCheckpointsOrThrow
|
|
358
|
-
setUpdatesOrThrow
|
|
359
|
-
makeCreateHistoryTableQuery
|
|
360
|
-
makeCreateCheckpointsTableQuery
|
|
361
|
-
makeCreateViewQuery
|
|
362
|
-
initialize
|
|
363
|
-
resume
|
|
356
|
+
getClickHouseFieldType,
|
|
357
|
+
makeClickHouseEntitySchema,
|
|
358
|
+
logger,
|
|
359
|
+
insertWithRetry,
|
|
360
|
+
setCheckpointsOrThrow,
|
|
361
|
+
setUpdatesOrThrow,
|
|
362
|
+
makeCreateHistoryTableQuery,
|
|
363
|
+
makeCreateCheckpointsTableQuery,
|
|
364
|
+
makeCreateViewQuery,
|
|
365
|
+
initialize,
|
|
366
|
+
resume,
|
|
364
367
|
}
|
|
365
368
|
/* logger Not a pure module */
|