envio 3.0.0-alpha.21 → 3.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin.mjs +2 -48
- package/evm.schema.json +67 -0
- package/fuel.schema.json +67 -0
- package/index.d.ts +822 -38
- package/index.js +5 -3
- package/package.json +10 -8
- package/rescript.json +5 -9
- package/src/Address.res +4 -5
- package/src/Address.res.mjs +9 -12
- package/src/Api.res +15 -0
- package/src/Api.res.mjs +20 -0
- package/src/Batch.res +32 -34
- package/src/Batch.res.mjs +172 -187
- package/src/Bin.res +89 -0
- package/src/Bin.res.mjs +97 -0
- package/src/ChainFetcher.res +33 -57
- package/src/ChainFetcher.res.mjs +197 -227
- package/src/ChainManager.res +6 -14
- package/src/ChainManager.res.mjs +74 -85
- package/src/ChainMap.res +14 -16
- package/src/ChainMap.res.mjs +38 -38
- package/src/Config.res +193 -135
- package/src/Config.res.mjs +566 -592
- package/src/Core.res +182 -0
- package/src/Core.res.mjs +207 -0
- package/src/Ecosystem.res +25 -4
- package/src/Ecosystem.res.mjs +12 -13
- package/src/Env.res +20 -13
- package/src/Env.res.mjs +124 -113
- package/src/EnvSafe.res +269 -0
- package/src/EnvSafe.res.mjs +296 -0
- package/src/EnvSafe.resi +18 -0
- package/src/Envio.res +37 -26
- package/src/Envio.res.mjs +59 -60
- package/src/ErrorHandling.res +2 -2
- package/src/ErrorHandling.res.mjs +15 -15
- package/src/EventConfigBuilder.res +219 -81
- package/src/EventConfigBuilder.res.mjs +259 -202
- package/src/EventProcessing.res +27 -38
- package/src/EventProcessing.res.mjs +165 -183
- package/src/EventUtils.res +11 -11
- package/src/EventUtils.res.mjs +21 -22
- package/src/EvmTypes.res +0 -1
- package/src/EvmTypes.res.mjs +5 -5
- package/src/FetchState.res +360 -256
- package/src/FetchState.res.mjs +958 -914
- package/src/GlobalState.res +365 -351
- package/src/GlobalState.res.mjs +958 -992
- package/src/GlobalStateManager.res +1 -2
- package/src/GlobalStateManager.res.mjs +36 -44
- package/src/HandlerLoader.res +107 -23
- package/src/HandlerLoader.res.mjs +128 -38
- package/src/HandlerRegister.res +127 -103
- package/src/HandlerRegister.res.mjs +164 -164
- package/src/HandlerRegister.resi +12 -4
- package/src/Hasura.res +35 -22
- package/src/Hasura.res.mjs +158 -167
- package/src/InMemoryStore.res +20 -27
- package/src/InMemoryStore.res.mjs +64 -80
- package/src/InMemoryTable.res +34 -39
- package/src/InMemoryTable.res.mjs +165 -170
- package/src/Internal.res +52 -33
- package/src/Internal.res.mjs +84 -81
- package/src/LazyLoader.res.mjs +55 -61
- package/src/LoadLayer.res +77 -78
- package/src/LoadLayer.res.mjs +160 -189
- package/src/LoadManager.res +16 -21
- package/src/LoadManager.res.mjs +79 -84
- package/src/LogSelection.res +236 -68
- package/src/LogSelection.res.mjs +211 -141
- package/src/Logging.res +13 -9
- package/src/Logging.res.mjs +130 -143
- package/src/Main.res +428 -51
- package/src/Main.res.mjs +528 -271
- package/src/Persistence.res +77 -84
- package/src/Persistence.res.mjs +131 -132
- package/src/PgStorage.res +291 -167
- package/src/PgStorage.res.mjs +797 -817
- package/src/Prometheus.res +50 -58
- package/src/Prometheus.res.mjs +345 -373
- package/src/ReorgDetection.res +22 -24
- package/src/ReorgDetection.res.mjs +100 -106
- package/src/SafeCheckpointTracking.res +7 -7
- package/src/SafeCheckpointTracking.res.mjs +40 -43
- package/src/SimulateItems.res +41 -49
- package/src/SimulateItems.res.mjs +257 -272
- package/src/Sink.res +2 -2
- package/src/Sink.res.mjs +22 -26
- package/src/TableIndices.res +1 -2
- package/src/TableIndices.res.mjs +42 -48
- package/src/TestIndexer.res +196 -189
- package/src/TestIndexer.res.mjs +536 -536
- package/src/TestIndexerProxyStorage.res +15 -16
- package/src/TestIndexerProxyStorage.res.mjs +98 -122
- package/src/TestIndexerWorker.res +4 -0
- package/src/TestIndexerWorker.res.mjs +7 -0
- package/src/Throttler.res +3 -3
- package/src/Throttler.res.mjs +23 -24
- package/src/Time.res +1 -1
- package/src/Time.res.mjs +18 -21
- package/src/TopicFilter.res +3 -3
- package/src/TopicFilter.res.mjs +29 -30
- package/src/UserContext.res +93 -54
- package/src/UserContext.res.mjs +197 -182
- package/src/Utils.res +141 -86
- package/src/Utils.res.mjs +334 -295
- package/src/bindings/BigDecimal.res +0 -2
- package/src/bindings/BigDecimal.res.mjs +19 -23
- package/src/bindings/ClickHouse.res +28 -27
- package/src/bindings/ClickHouse.res.mjs +243 -240
- package/src/bindings/DateFns.res +11 -11
- package/src/bindings/DateFns.res.mjs +7 -7
- package/src/bindings/EventSource.res.mjs +2 -2
- package/src/bindings/Express.res +2 -5
- package/src/bindings/Hrtime.res +2 -2
- package/src/bindings/Hrtime.res.mjs +30 -32
- package/src/bindings/Lodash.res.mjs +1 -1
- package/src/bindings/NodeJs.res +14 -9
- package/src/bindings/NodeJs.res.mjs +20 -20
- package/src/bindings/Pino.res +8 -10
- package/src/bindings/Pino.res.mjs +40 -43
- package/src/bindings/Postgres.res +2 -5
- package/src/bindings/Postgres.res.mjs +9 -9
- package/src/bindings/PromClient.res +17 -2
- package/src/bindings/PromClient.res.mjs +30 -7
- package/src/bindings/SDSL.res.mjs +2 -2
- package/src/bindings/Viem.res +4 -4
- package/src/bindings/Viem.res.mjs +20 -22
- package/src/bindings/Vitest.res +1 -1
- package/src/bindings/Vitest.res.mjs +2 -2
- package/src/bindings/WebSocket.res +1 -1
- package/src/db/EntityHistory.res +9 -3
- package/src/db/EntityHistory.res.mjs +84 -59
- package/src/db/InternalTable.res +62 -60
- package/src/db/InternalTable.res.mjs +271 -203
- package/src/db/Schema.res +1 -2
- package/src/db/Schema.res.mjs +28 -32
- package/src/db/Table.res +28 -27
- package/src/db/Table.res.mjs +276 -292
- package/src/sources/EventRouter.res +21 -16
- package/src/sources/EventRouter.res.mjs +55 -57
- package/src/sources/Evm.res +17 -1
- package/src/sources/Evm.res.mjs +16 -8
- package/src/sources/EvmChain.res +15 -17
- package/src/sources/EvmChain.res.mjs +40 -42
- package/src/sources/Fuel.res +14 -1
- package/src/sources/Fuel.res.mjs +16 -8
- package/src/sources/FuelSDK.res +1 -1
- package/src/sources/FuelSDK.res.mjs +6 -8
- package/src/sources/HyperFuel.res +8 -10
- package/src/sources/HyperFuel.res.mjs +113 -123
- package/src/sources/HyperFuelClient.res.mjs +6 -7
- package/src/sources/HyperFuelSource.res +19 -20
- package/src/sources/HyperFuelSource.res.mjs +339 -356
- package/src/sources/HyperSync.res +11 -13
- package/src/sources/HyperSync.res.mjs +206 -220
- package/src/sources/HyperSyncClient.res +5 -7
- package/src/sources/HyperSyncClient.res.mjs +70 -75
- package/src/sources/HyperSyncHeightStream.res +8 -9
- package/src/sources/HyperSyncHeightStream.res.mjs +78 -86
- package/src/sources/HyperSyncJsonApi.res +18 -15
- package/src/sources/HyperSyncJsonApi.res.mjs +201 -231
- package/src/sources/HyperSyncSource.res +17 -21
- package/src/sources/HyperSyncSource.res.mjs +268 -290
- package/src/sources/Rpc.res +5 -5
- package/src/sources/Rpc.res.mjs +168 -192
- package/src/sources/RpcSource.res +166 -167
- package/src/sources/RpcSource.res.mjs +972 -1046
- package/src/sources/RpcWebSocketHeightStream.res +10 -11
- package/src/sources/RpcWebSocketHeightStream.res.mjs +131 -145
- package/src/sources/SimulateSource.res +1 -1
- package/src/sources/SimulateSource.res.mjs +35 -38
- package/src/sources/Source.res +1 -1
- package/src/sources/Source.res.mjs +3 -3
- package/src/sources/SourceManager.res +39 -20
- package/src/sources/SourceManager.res.mjs +340 -371
- package/src/sources/SourceManager.resi +2 -1
- package/src/sources/Svm.res +12 -5
- package/src/sources/Svm.res.mjs +44 -41
- package/src/tui/Tui.res +23 -12
- package/src/tui/Tui.res.mjs +292 -290
- package/src/tui/bindings/Ink.res +2 -4
- package/src/tui/bindings/Ink.res.mjs +35 -41
- package/src/tui/components/BufferedProgressBar.res +7 -7
- package/src/tui/components/BufferedProgressBar.res.mjs +46 -46
- package/src/tui/components/CustomHooks.res +1 -2
- package/src/tui/components/CustomHooks.res.mjs +102 -122
- package/src/tui/components/Messages.res +1 -2
- package/src/tui/components/Messages.res.mjs +38 -42
- package/src/tui/components/SyncETA.res +10 -11
- package/src/tui/components/SyncETA.res.mjs +178 -196
- package/src/tui/components/TuiData.res +1 -1
- package/src/tui/components/TuiData.res.mjs +7 -6
- package/src/vendored/Rest.res +52 -66
- package/src/vendored/Rest.res.mjs +324 -364
- package/svm.schema.json +67 -0
- package/src/Address.gen.ts +0 -8
- package/src/Config.gen.ts +0 -19
- package/src/Envio.gen.ts +0 -55
- package/src/EvmTypes.gen.ts +0 -6
- package/src/InMemoryStore.gen.ts +0 -6
- package/src/Internal.gen.ts +0 -64
- package/src/PgStorage.gen.ts +0 -10
- package/src/PgStorage.res.d.mts +0 -5
- package/src/Types.ts +0 -56
- package/src/bindings/BigDecimal.gen.ts +0 -14
- package/src/bindings/BigDecimal.res.d.mts +0 -5
- package/src/bindings/BigInt.gen.ts +0 -10
- package/src/bindings/BigInt.res +0 -70
- package/src/bindings/BigInt.res.d.mts +0 -5
- package/src/bindings/BigInt.res.mjs +0 -154
- package/src/bindings/Ethers.res.d.mts +0 -5
- package/src/bindings/Pino.gen.ts +0 -17
- package/src/bindings/Postgres.gen.ts +0 -8
- package/src/bindings/Postgres.res.d.mts +0 -5
- package/src/bindings/Promise.res +0 -67
- package/src/bindings/Promise.res.mjs +0 -26
- package/src/db/InternalTable.gen.ts +0 -36
- package/src/sources/HyperSyncClient.gen.ts +0 -19
package/src/PgStorage.res.mjs
CHANGED
|
@@ -2,58 +2,53 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Fs from "fs";
|
|
4
4
|
import * as Env from "./Env.res.mjs";
|
|
5
|
+
import * as Sink from "./Sink.res.mjs";
|
|
5
6
|
import * as Path from "path";
|
|
6
|
-
import * as $$Array from "rescript/lib/es6/array.js";
|
|
7
7
|
import * as Table from "./db/Table.res.mjs";
|
|
8
8
|
import * as Utils from "./Utils.res.mjs";
|
|
9
|
-
import * as $$BigInt from "./bindings/BigInt.res.mjs";
|
|
10
9
|
import * as Config from "./Config.res.mjs";
|
|
10
|
+
import * as Hasura from "./Hasura.res.mjs";
|
|
11
11
|
import * as Hrtime from "./bindings/Hrtime.res.mjs";
|
|
12
|
-
import * as Js_exn from "rescript/lib/es6/js_exn.js";
|
|
13
12
|
import * as Schema from "./db/Schema.res.mjs";
|
|
14
|
-
import * as Js_dict from "rescript/lib/es6/js_dict.js";
|
|
15
13
|
import * as Logging from "./Logging.res.mjs";
|
|
16
|
-
import * as $$Promise from "./bindings/Promise.res.mjs";
|
|
17
|
-
import * as Caml_obj from "rescript/lib/es6/caml_obj.js";
|
|
18
14
|
import * as Internal from "./Internal.res.mjs";
|
|
19
15
|
import Postgres from "postgres";
|
|
20
|
-
import * as Belt_Array from "rescript/lib/es6/
|
|
16
|
+
import * as Belt_Array from "@rescript/runtime/lib/es6/Belt_Array.js";
|
|
21
17
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
22
|
-
import * as Caml_option from "rescript/lib/es6/caml_option.js";
|
|
23
18
|
import * as Persistence from "./Persistence.res.mjs";
|
|
19
|
+
import * as ChainFetcher from "./ChainFetcher.res.mjs";
|
|
24
20
|
import * as EntityHistory from "./db/EntityHistory.res.mjs";
|
|
25
21
|
import * as InternalTable from "./db/InternalTable.res.mjs";
|
|
26
22
|
import * as Child_process from "child_process";
|
|
27
|
-
import * as
|
|
23
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
24
|
+
import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
|
|
25
|
+
import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
|
|
26
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
28
27
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
29
|
-
import * as
|
|
28
|
+
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
30
29
|
|
|
31
|
-
|
|
30
|
+
let getCacheRowCountFnName = "get_cache_row_count";
|
|
32
31
|
|
|
33
32
|
function makeClient() {
|
|
34
33
|
return Postgres({
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
48
|
-
});
|
|
34
|
+
host: Env.Db.host,
|
|
35
|
+
port: Env.Db.port,
|
|
36
|
+
database: Env.Db.database,
|
|
37
|
+
username: Env.Db.user,
|
|
38
|
+
password: Env.Db.password,
|
|
39
|
+
ssl: Env.Db.ssl,
|
|
40
|
+
max: Env.Db.maxConnections,
|
|
41
|
+
onnotice: Primitive_object.equal(Env.userLogLevel, "warn") || Primitive_object.equal(Env.userLogLevel, "error") ? undefined : _str => {},
|
|
42
|
+
transform: {
|
|
43
|
+
undefined: null
|
|
44
|
+
}
|
|
45
|
+
});
|
|
49
46
|
}
|
|
50
47
|
|
|
51
48
|
function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
})).join(", ");
|
|
56
|
-
return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
|
|
49
|
+
let indexName = tableName + "_" + indexFields.join("_");
|
|
50
|
+
let index = Belt_Array.map(indexFields, idx => `"` + idx + `"`).join(", ");
|
|
51
|
+
return `CREATE INDEX IF NOT EXISTS "` + indexName + `" ON "` + pgSchema + `"."` + tableName + `"(` + index + `);`;
|
|
57
52
|
}
|
|
58
53
|
|
|
59
54
|
function directionToSql(direction) {
|
|
@@ -73,104 +68,94 @@ function directionToIndexName(direction) {
|
|
|
73
68
|
}
|
|
74
69
|
|
|
75
70
|
function makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema) {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
var index = Belt_Array.map(indexFields, (function (f) {
|
|
80
|
-
return "\"" + f.fieldName + "\"" + directionToSql(f.direction);
|
|
81
|
-
})).join(", ");
|
|
82
|
-
return "CREATE INDEX IF NOT EXISTS \"" + indexName + "\" ON \"" + pgSchema + "\".\"" + tableName + "\"(" + index + ");";
|
|
71
|
+
let indexName = tableName + "_" + indexFields.map(f => f.fieldName + directionToIndexName(f.direction)).join("_");
|
|
72
|
+
let index = Belt_Array.map(indexFields, f => `"` + f.fieldName + `"` + directionToSql(f.direction)).join(", ");
|
|
73
|
+
return `CREATE INDEX IF NOT EXISTS "` + indexName + `" ON "` + pgSchema + `"."` + tableName + `"(` + index + `);`;
|
|
83
74
|
}
|
|
84
75
|
|
|
85
76
|
function makeCreateTableIndicesQuery(table, pgSchema) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
};
|
|
93
|
-
var singleIndices = Table.getSingleIndices(table);
|
|
94
|
-
var compositeIndices = Table.getCompositeIndices(table);
|
|
95
|
-
return Belt_Array.map(singleIndices, createIndex).join("\n") + Belt_Array.map(compositeIndices, createCompositeIndex).join("\n");
|
|
77
|
+
let tableName = table.tableName;
|
|
78
|
+
let createIndex = indexField => makeCreateIndexQuery(tableName, [indexField], pgSchema);
|
|
79
|
+
let createCompositeIndex = indexFields => makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema);
|
|
80
|
+
let singleIndices = Table.getSingleIndices(table);
|
|
81
|
+
let compositeIndices = Table.getCompositeIndices(table);
|
|
82
|
+
return singleIndices.map(createIndex).join("\n") + compositeIndices.map(createCompositeIndex).join("\n");
|
|
96
83
|
}
|
|
97
84
|
|
|
98
85
|
function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
primaryKeyFieldNames.length !== 0 ? ", PRIMARY KEY(" + primaryKey + ")" : ""
|
|
115
|
-
) + ");";
|
|
86
|
+
let fieldsMapped = Table.getFields(table).map(field => {
|
|
87
|
+
let defaultValue = field.defaultValue;
|
|
88
|
+
let isNullable = field.isNullable;
|
|
89
|
+
let fieldName = Table.getDbFieldName(field);
|
|
90
|
+
return `"` + fieldName + `" ` + Table.getPgFieldType(field.fieldType, pgSchema, field.isArray, isNumericArrayAsText, isNullable) + (
|
|
91
|
+
defaultValue !== undefined ? ` DEFAULT ` + defaultValue : (
|
|
92
|
+
isNullable ? `` : ` NOT NULL`
|
|
93
|
+
)
|
|
94
|
+
);
|
|
95
|
+
}).join(", ");
|
|
96
|
+
let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
|
|
97
|
+
let primaryKey = primaryKeyFieldNames.map(field => `"` + field + `"`).join(", ");
|
|
98
|
+
return `CREATE TABLE IF NOT EXISTS "` + pgSchema + `"."` + table.tableName + `"(` + fieldsMapped + (
|
|
99
|
+
primaryKeyFieldNames.length !== 0 ? `, PRIMARY KEY(` + primaryKey + `)` : ""
|
|
100
|
+
) + `);`;
|
|
116
101
|
}
|
|
117
102
|
|
|
118
|
-
|
|
103
|
+
let entityHistoryCache = new WeakMap();
|
|
119
104
|
|
|
120
105
|
function getEntityHistory(entityConfig) {
|
|
121
|
-
|
|
106
|
+
let cache = entityHistoryCache.get(entityConfig);
|
|
122
107
|
if (cache !== undefined) {
|
|
123
108
|
return cache;
|
|
124
109
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
110
|
+
let dataFields = Belt_Array.keepMap(entityConfig.table.fields, field => {
|
|
111
|
+
if (field.TAG !== "Field") {
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
let field$1 = field._0;
|
|
115
|
+
let match = field$1.fieldName;
|
|
116
|
+
if (match === "id") {
|
|
117
|
+
return {
|
|
118
|
+
TAG: "Field",
|
|
119
|
+
_0: {
|
|
120
|
+
fieldName: "id",
|
|
121
|
+
fieldType: field$1.fieldType,
|
|
122
|
+
fieldSchema: field$1.fieldSchema,
|
|
123
|
+
isArray: field$1.isArray,
|
|
124
|
+
isNullable: field$1.isNullable,
|
|
125
|
+
isPrimaryKey: true,
|
|
126
|
+
isIndex: field$1.isIndex,
|
|
127
|
+
linkedEntity: field$1.linkedEntity,
|
|
128
|
+
defaultValue: field$1.defaultValue
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
} else {
|
|
132
|
+
return {
|
|
133
|
+
TAG: "Field",
|
|
134
|
+
_0: {
|
|
135
|
+
fieldName: field$1.fieldName,
|
|
136
|
+
fieldType: field$1.fieldType,
|
|
137
|
+
fieldSchema: field$1.fieldSchema,
|
|
138
|
+
isArray: field$1.isArray,
|
|
139
|
+
isNullable: true,
|
|
140
|
+
isPrimaryKey: field$1.isPrimaryKey,
|
|
141
|
+
isIndex: false,
|
|
142
|
+
linkedEntity: field$1.linkedEntity,
|
|
143
|
+
defaultValue: field$1.defaultValue
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
let actionField = Table.mkField(EntityHistory.changeFieldName, EntityHistory.changeFieldType, S$RescriptSchema.never, undefined, undefined, undefined, undefined, undefined, undefined);
|
|
149
|
+
let checkpointIdField = Table.mkField(EntityHistory.checkpointIdFieldName, EntityHistory.checkpointIdFieldType, EntityHistory.unsafeCheckpointIdSchema, undefined, undefined, undefined, true, undefined, undefined);
|
|
150
|
+
let entityTableName = entityConfig.table.tableName;
|
|
151
|
+
let historyTableName = EntityHistory.historyTableName(entityTableName, entityConfig.index);
|
|
152
|
+
let table = Table.mkTable(historyTableName, undefined, Belt_Array.concat(dataFields, [
|
|
153
|
+
checkpointIdField,
|
|
154
|
+
actionField
|
|
155
|
+
]));
|
|
156
|
+
let setChangeSchema = EntityHistory.makeSetUpdateSchema(entityConfig.schema);
|
|
157
|
+
let cache_setChangeSchemaRows = S$RescriptSchema.array(setChangeSchema);
|
|
158
|
+
let cache$1 = {
|
|
174
159
|
table: table,
|
|
175
160
|
setChangeSchema: setChangeSchema,
|
|
176
161
|
setChangeSchemaRows: cache_setChangeSchemaRows
|
|
@@ -180,166 +165,161 @@ function getEntityHistory(entityConfig) {
|
|
|
180
165
|
}
|
|
181
166
|
|
|
182
167
|
function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
168
|
+
let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
|
|
169
|
+
let entities = entitiesOpt !== undefined ? entitiesOpt : [];
|
|
170
|
+
let enums = enumsOpt !== undefined ? enumsOpt : [];
|
|
171
|
+
let isEmptyPgSchema = isEmptyPgSchemaOpt !== undefined ? isEmptyPgSchemaOpt : false;
|
|
172
|
+
let generalTables = [
|
|
188
173
|
InternalTable.Chains.table,
|
|
189
174
|
InternalTable.PersistedState.table,
|
|
190
175
|
InternalTable.Checkpoints.table,
|
|
191
176
|
InternalTable.RawEvents.table
|
|
192
177
|
];
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
entities.forEach(
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
178
|
+
let allTables = generalTables.slice();
|
|
179
|
+
let allEntityTables = [];
|
|
180
|
+
entities.forEach(entityConfig => {
|
|
181
|
+
allEntityTables.push(entityConfig.table);
|
|
182
|
+
allTables.push(entityConfig.table);
|
|
183
|
+
allTables.push(getEntityHistory(entityConfig).table);
|
|
184
|
+
});
|
|
185
|
+
let derivedSchema = Schema.make(allEntityTables);
|
|
186
|
+
let query = {
|
|
202
187
|
contents: (
|
|
203
|
-
isEmptyPgSchema && pgSchema === "public" ?
|
|
204
|
-
|
|
188
|
+
isEmptyPgSchema && pgSchema === "public" ? `CREATE SCHEMA IF NOT EXISTS "` + pgSchema + `";\n` : `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;
|
|
189
|
+
CREATE SCHEMA "` + pgSchema + `";\n`
|
|
190
|
+
) + (`GRANT ALL ON SCHEMA "` + pgSchema + `" TO "` + pgUser + `";
|
|
191
|
+
GRANT ALL ON SCHEMA "` + pgSchema + `" TO public;`)
|
|
205
192
|
};
|
|
206
|
-
enums.forEach(
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
|
|
227
|
-
});
|
|
228
|
-
});
|
|
193
|
+
enums.forEach(enumConfig => {
|
|
194
|
+
let enumCreateQuery = `CREATE TYPE "` + pgSchema + `".` + enumConfig.name + ` AS ENUM(` + enumConfig.variants.map(v => `'` + v + `'`).join(", ") + `);`;
|
|
195
|
+
query.contents = query.contents + "\n" + enumCreateQuery;
|
|
196
|
+
});
|
|
197
|
+
allTables.forEach(table => {
|
|
198
|
+
query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema, isHasuraEnabled);
|
|
199
|
+
});
|
|
200
|
+
allTables.forEach(table => {
|
|
201
|
+
let indices = makeCreateTableIndicesQuery(table, pgSchema);
|
|
202
|
+
if (indices !== "") {
|
|
203
|
+
query.contents = query.contents + "\n" + indices;
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
});
|
|
207
|
+
entities.forEach(entity => {
|
|
208
|
+
Table.getDerivedFromFields(entity.table).forEach(derivedFromField => {
|
|
209
|
+
let indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
|
|
210
|
+
query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
|
|
211
|
+
});
|
|
212
|
+
});
|
|
229
213
|
query.contents = query.contents + "\n" + InternalTable.Views.makeMetaViewQuery(pgSchema);
|
|
230
214
|
query.contents = query.contents + "\n" + InternalTable.Views.makeChainMetadataViewQuery(pgSchema);
|
|
231
|
-
|
|
215
|
+
let initialChainsValuesQuery = InternalTable.Chains.makeInitialValuesQuery(pgSchema, chainConfigs);
|
|
232
216
|
if (initialChainsValuesQuery !== undefined) {
|
|
233
217
|
query.contents = query.contents + "\n" + initialChainsValuesQuery;
|
|
234
218
|
}
|
|
235
219
|
return [
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
220
|
+
query.contents,
|
|
221
|
+
`CREATE OR REPLACE FUNCTION ` + getCacheRowCountFnName + `(table_name text)
|
|
222
|
+
RETURNS integer AS $$
|
|
223
|
+
DECLARE
|
|
224
|
+
result integer;
|
|
225
|
+
BEGIN
|
|
226
|
+
EXECUTE format('SELECT COUNT(*) FROM "` + pgSchema + `".%I', table_name) INTO result;
|
|
227
|
+
RETURN result;
|
|
228
|
+
END;
|
|
229
|
+
$$ LANGUAGE plpgsql;`
|
|
230
|
+
];
|
|
239
231
|
}
|
|
240
232
|
|
|
241
233
|
function makeLoadByIdQuery(pgSchema, tableName) {
|
|
242
|
-
return
|
|
234
|
+
return `SELECT * FROM "` + pgSchema + `"."` + tableName + `" WHERE id = $1 LIMIT 1;`;
|
|
243
235
|
}
|
|
244
236
|
|
|
245
237
|
function makeLoadByFieldQuery(pgSchema, tableName, fieldName, operator) {
|
|
246
|
-
return
|
|
238
|
+
return `SELECT * FROM "` + pgSchema + `"."` + tableName + `" WHERE "` + fieldName + `" ` + operator + ` $1;`;
|
|
247
239
|
}
|
|
248
240
|
|
|
249
241
|
function makeLoadByIdsQuery(pgSchema, tableName) {
|
|
250
|
-
return
|
|
242
|
+
return `SELECT * FROM "` + pgSchema + `"."` + tableName + `" WHERE id = ANY($1::text[]);`;
|
|
251
243
|
}
|
|
252
244
|
|
|
253
245
|
function makeDeleteByIdQuery(pgSchema, tableName) {
|
|
254
|
-
return
|
|
246
|
+
return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = $1;`;
|
|
255
247
|
}
|
|
256
248
|
|
|
257
249
|
function makeDeleteByIdsQuery(pgSchema, tableName) {
|
|
258
|
-
return
|
|
250
|
+
return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = ANY($1::text[]);`;
|
|
259
251
|
}
|
|
260
252
|
|
|
261
253
|
function makeLoadAllQuery(pgSchema, tableName) {
|
|
262
|
-
return
|
|
254
|
+
return `SELECT * FROM "` + pgSchema + `"."` + tableName + `";`;
|
|
263
255
|
}
|
|
264
256
|
|
|
265
257
|
function makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents) {
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
return
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? "NOTHING" : "UPDATE SET " + quotedNonPrimaryFieldNames.map(function (fieldName) {
|
|
276
|
-
return fieldName + " = EXCLUDED." + fieldName;
|
|
277
|
-
}).join(",")
|
|
278
|
-
)
|
|
279
|
-
) + ";";
|
|
258
|
+
let match = Table.toSqlParams(table, itemSchema, pgSchema);
|
|
259
|
+
let quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
|
|
260
|
+
let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
|
|
261
|
+
return `INSERT INTO "` + pgSchema + `"."` + table.tableName + `" (` + match.quotedFieldNames.join(", ") + `)
|
|
262
|
+
SELECT * FROM unnest(` + match.arrayFieldTypes.map((arrayFieldType, idx) => `$` + (idx + 1 | 0).toString() + `::` + arrayFieldType).join(",") + `)` + (
|
|
263
|
+
isRawEvents || primaryKeyFieldNames.length === 0 ? `` : `ON CONFLICT(` + primaryKeyFieldNames.map(fieldName => `"` + fieldName + `"`).join(",") + `) DO ` + (
|
|
264
|
+
Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? `NOTHING` : `UPDATE SET ` + quotedNonPrimaryFieldNames.map(fieldName => fieldName + ` = EXCLUDED.` + fieldName).join(",")
|
|
265
|
+
)
|
|
266
|
+
) + ";";
|
|
280
267
|
}
|
|
281
268
|
|
|
282
269
|
function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
for(
|
|
270
|
+
let match = Table.toSqlParams(table, itemSchema, pgSchema);
|
|
271
|
+
let quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
|
|
272
|
+
let quotedFieldNames = match.quotedFieldNames;
|
|
273
|
+
let primaryKeyFieldNames = Table.getPrimaryKeyFieldNames(table);
|
|
274
|
+
let fieldsCount = quotedFieldNames.length;
|
|
275
|
+
let placeholders = "";
|
|
276
|
+
for (let idx = 1; idx <= itemsCount; ++idx) {
|
|
290
277
|
if (idx > 1) {
|
|
291
278
|
placeholders = placeholders + ",";
|
|
292
279
|
}
|
|
293
280
|
placeholders = placeholders + "(";
|
|
294
|
-
for(
|
|
281
|
+
for (let fieldIdx = 0; fieldIdx < fieldsCount; ++fieldIdx) {
|
|
295
282
|
if (fieldIdx > 0) {
|
|
296
283
|
placeholders = placeholders + ",";
|
|
297
284
|
}
|
|
298
|
-
placeholders = placeholders + (
|
|
285
|
+
placeholders = placeholders + (`$` + ((fieldIdx * itemsCount | 0) + idx | 0).toString());
|
|
299
286
|
}
|
|
300
287
|
placeholders = placeholders + ")";
|
|
301
288
|
}
|
|
302
|
-
return
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
}).join(",")
|
|
309
|
-
) : ""
|
|
310
|
-
) + ";";
|
|
289
|
+
return `INSERT INTO "` + pgSchema + `"."` + table.tableName + `" (` + quotedFieldNames.join(", ") + `)
|
|
290
|
+
VALUES` + placeholders + (
|
|
291
|
+
primaryKeyFieldNames.length !== 0 ? `ON CONFLICT(` + primaryKeyFieldNames.map(fieldName => `"` + fieldName + `"`).join(",") + `) DO ` + (
|
|
292
|
+
Utils.$$Array.isEmpty(quotedNonPrimaryFieldNames) ? `NOTHING` : `UPDATE SET ` + quotedNonPrimaryFieldNames.map(fieldName => fieldName + ` = EXCLUDED.` + fieldName).join(",")
|
|
293
|
+
) : ``
|
|
294
|
+
) + ";";
|
|
311
295
|
}
|
|
312
296
|
|
|
313
297
|
function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
298
|
+
let match = Table.toSqlParams(table, itemSchema, pgSchema);
|
|
299
|
+
let isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName;
|
|
300
|
+
let isHistoryUpdate = table.tableName.startsWith(EntityHistory.historyTablePrefix);
|
|
317
301
|
if ((isRawEvents || !match.hasArrayField) && !isHistoryUpdate) {
|
|
318
302
|
return {
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
303
|
+
query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
|
|
304
|
+
convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
|
|
305
|
+
isInsertValues: false
|
|
306
|
+
};
|
|
323
307
|
} else {
|
|
324
308
|
return {
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
};
|
|
332
|
-
})), "Output", "Input", "Sync", false),
|
|
333
|
-
isInsertValues: true
|
|
334
|
-
};
|
|
309
|
+
query: makeInsertValuesSetQuery(pgSchema, table, itemSchema, 500),
|
|
310
|
+
convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), param => ({
|
|
311
|
+
s: prim => prim.flat(1)
|
|
312
|
+
})), "Output", "Input", "Sync", false),
|
|
313
|
+
isInsertValues: true
|
|
314
|
+
};
|
|
335
315
|
}
|
|
336
316
|
}
|
|
337
317
|
|
|
338
318
|
function chunkArray(arr, chunkSize) {
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
while(i < arr.length) {
|
|
342
|
-
|
|
319
|
+
let chunks = [];
|
|
320
|
+
let i = 0;
|
|
321
|
+
while (i < arr.length) {
|
|
322
|
+
let chunk = arr.slice(i, i + chunkSize | 0);
|
|
343
323
|
chunks.push(chunk);
|
|
344
324
|
i = i + chunkSize | 0;
|
|
345
325
|
};
|
|
@@ -347,35 +327,30 @@ function chunkArray(arr, chunkSize) {
|
|
|
347
327
|
}
|
|
348
328
|
|
|
349
329
|
function removeInvalidUtf8InPlace(entities) {
|
|
350
|
-
entities.forEach(
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
}));
|
|
358
|
-
});
|
|
330
|
+
entities.forEach(item => Utils.Dict.forEachWithKey(item, (value, key) => {
|
|
331
|
+
if (typeof value === "string") {
|
|
332
|
+
item[key] = value.replaceAll("\x00", "");
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
}));
|
|
359
336
|
}
|
|
360
337
|
|
|
361
|
-
|
|
362
|
-
return s.f("message", S$RescriptSchema.string);
|
|
363
|
-
});
|
|
338
|
+
let pgErrorMessageSchema = S$RescriptSchema.object(s => s.f("message", S$RescriptSchema.string));
|
|
364
339
|
|
|
365
|
-
|
|
340
|
+
let PgEncodingError = /* @__PURE__ */Primitive_exceptions.create("PgStorage.PgEncodingError");
|
|
366
341
|
|
|
367
|
-
|
|
342
|
+
let setQueryCache = new WeakMap();
|
|
368
343
|
|
|
369
344
|
async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
|
|
370
345
|
if (items.length === 0) {
|
|
371
|
-
return
|
|
346
|
+
return;
|
|
372
347
|
}
|
|
373
|
-
|
|
374
|
-
|
|
348
|
+
let cached = setQueryCache.get(table);
|
|
349
|
+
let data;
|
|
375
350
|
if (cached !== undefined) {
|
|
376
|
-
data =
|
|
351
|
+
data = Primitive_option.valFromOption(cached);
|
|
377
352
|
} else {
|
|
378
|
-
|
|
353
|
+
let newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
|
|
379
354
|
setQueryCache.set(table, newQuery);
|
|
380
355
|
data = newQuery;
|
|
381
356
|
}
|
|
@@ -383,53 +358,57 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
|
|
|
383
358
|
if (!data.isInsertValues) {
|
|
384
359
|
return await sql.unsafe(data.query, data.convertOrThrow(items), {prepare: true});
|
|
385
360
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
chunks.forEach(
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
361
|
+
let chunks = chunkArray(items, 500);
|
|
362
|
+
let responses = [];
|
|
363
|
+
chunks.forEach(chunk => {
|
|
364
|
+
let chunkSize = chunk.length;
|
|
365
|
+
let isFullChunk = chunkSize === 500;
|
|
366
|
+
let params = data.convertOrThrow(chunk);
|
|
367
|
+
let response = isFullChunk ? sql.unsafe(data.query, params, {prepare: true}) : sql.unsafe(makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), params);
|
|
368
|
+
responses.push(response);
|
|
369
|
+
});
|
|
395
370
|
await Promise.all(responses);
|
|
396
|
-
return
|
|
397
|
-
}
|
|
398
|
-
|
|
399
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
371
|
+
return;
|
|
372
|
+
} catch (raw_exn) {
|
|
373
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
400
374
|
if (exn.RE_EXN_ID === S$RescriptSchema.Raised) {
|
|
401
375
|
throw {
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
376
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
377
|
+
message: `Failed to convert items for table "` + table.tableName + `"`,
|
|
378
|
+
reason: exn,
|
|
379
|
+
Error: new Error()
|
|
380
|
+
};
|
|
407
381
|
}
|
|
408
382
|
throw {
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
383
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
384
|
+
message: `Failed to insert items into table "` + table.tableName + `"`,
|
|
385
|
+
reason: Utils.prettifyExn(exn),
|
|
386
|
+
Error: new Error()
|
|
387
|
+
};
|
|
414
388
|
}
|
|
415
389
|
}
|
|
416
390
|
|
|
417
391
|
function makeSchemaTableNamesQuery(pgSchema) {
|
|
418
|
-
return
|
|
392
|
+
return `SELECT table_name FROM information_schema.tables WHERE table_schema = '` + pgSchema + `';`;
|
|
419
393
|
}
|
|
420
394
|
|
|
421
|
-
|
|
395
|
+
let cacheTablePrefixLength = Internal.cacheTablePrefix.length;
|
|
422
396
|
|
|
423
397
|
function makeSchemaCacheTableInfoQuery(pgSchema) {
|
|
424
|
-
return
|
|
398
|
+
return `SELECT
|
|
399
|
+
t.table_name,
|
|
400
|
+
` + getCacheRowCountFnName + `(t.table_name) as count
|
|
401
|
+
FROM information_schema.tables t
|
|
402
|
+
WHERE t.table_schema = '` + pgSchema + `'
|
|
403
|
+
AND t.table_name LIKE '` + Internal.cacheTablePrefix + `%';`;
|
|
425
404
|
}
|
|
426
405
|
|
|
427
|
-
|
|
406
|
+
let psqlExecState = {
|
|
428
407
|
contents: "Unknown"
|
|
429
408
|
};
|
|
430
409
|
|
|
431
410
|
async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName) {
|
|
432
|
-
|
|
411
|
+
let promise = psqlExecState.contents;
|
|
433
412
|
if (typeof promise === "object") {
|
|
434
413
|
if (promise.TAG === "Pending") {
|
|
435
414
|
return await promise._0;
|
|
@@ -437,36 +416,36 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containe
|
|
|
437
416
|
return promise._0;
|
|
438
417
|
}
|
|
439
418
|
}
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
419
|
+
let promise$1 = new Promise((resolve, _reject) => {
|
|
420
|
+
let binary = "psql";
|
|
421
|
+
Child_process.exec(binary + ` --version`, (error, param, param$1) => {
|
|
422
|
+
if (error === null) {
|
|
423
|
+
return resolve({
|
|
424
|
+
TAG: "Ok",
|
|
425
|
+
_0: binary + ` -h ` + pgHost + ` -p ` + pgPort.toString() + ` -U ` + pgUser + ` -d ` + pgDatabase
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
let binary$1 = `docker exec -i -u ` + pgUser + ` ` + containerName + ` psql`;
|
|
429
|
+
Child_process.exec(binary$1 + ` --version`, (error, param, param$1) => {
|
|
430
|
+
if (error === null) {
|
|
431
|
+
return resolve({
|
|
432
|
+
TAG: "Ok",
|
|
433
|
+
_0: binary$1 + ` -h ` + pgHost + ` -p ` + (5432).toString() + ` -U ` + pgUser + ` -d ` + pgDatabase
|
|
434
|
+
});
|
|
435
|
+
} else {
|
|
436
|
+
return resolve({
|
|
437
|
+
TAG: "Error",
|
|
438
|
+
_0: `Please check if "psql" binary is installed or Docker container "` + containerName + `" is running.`
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
});
|
|
443
|
+
});
|
|
465
444
|
psqlExecState.contents = {
|
|
466
445
|
TAG: "Pending",
|
|
467
446
|
_0: promise$1
|
|
468
447
|
};
|
|
469
|
-
|
|
448
|
+
let result = await promise$1;
|
|
470
449
|
psqlExecState.contents = {
|
|
471
450
|
TAG: "Resolved",
|
|
472
451
|
_0: result
|
|
@@ -479,46 +458,44 @@ async function deleteByIdsOrThrow(sql, pgSchema, ids, table) {
|
|
|
479
458
|
await (
|
|
480
459
|
ids.length !== 1 ? sql.unsafe(makeDeleteByIdsQuery(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeDeleteByIdQuery(pgSchema, table.tableName), ids, {prepare: true})
|
|
481
460
|
);
|
|
482
|
-
return
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
461
|
+
return;
|
|
462
|
+
} catch (raw_exn) {
|
|
463
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
486
464
|
throw {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
465
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
466
|
+
message: `Failed deleting "` + table.tableName + `" from storage by ids`,
|
|
467
|
+
reason: exn,
|
|
468
|
+
Error: new Error()
|
|
469
|
+
};
|
|
492
470
|
}
|
|
493
471
|
}
|
|
494
472
|
|
|
495
473
|
function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
}));
|
|
474
|
+
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
475
|
+
let allHistoryFieldNames = Belt_Array.keepMap(entityConfig.table.fields, fieldOrDerived => {
|
|
476
|
+
if (fieldOrDerived.TAG === "Field") {
|
|
477
|
+
return Table.getDbFieldName(fieldOrDerived._0);
|
|
478
|
+
}
|
|
479
|
+
});
|
|
503
480
|
allHistoryFieldNames.push(EntityHistory.checkpointIdFieldName);
|
|
504
481
|
allHistoryFieldNames.push(EntityHistory.changeFieldName);
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
482
|
+
let allHistoryFieldNamesStr = Belt_Array.map(allHistoryFieldNames, name => `"` + name + `"`).join(", ");
|
|
483
|
+
let selectParts = Belt_Array.map(allHistoryFieldNames, fieldName => {
|
|
484
|
+
if (fieldName === Table.idFieldName) {
|
|
485
|
+
return `u.` + Table.idFieldName;
|
|
486
|
+
} else if (fieldName === EntityHistory.checkpointIdFieldName) {
|
|
487
|
+
return `u.` + EntityHistory.checkpointIdFieldName;
|
|
488
|
+
} else if (fieldName === EntityHistory.changeFieldName) {
|
|
489
|
+
return `'` + "DELETE" + `'`;
|
|
490
|
+
} else {
|
|
491
|
+
return "NULL";
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
let selectPartsStr = selectParts.join(", ");
|
|
495
|
+
let checkpointIdPgType = Table.getPgFieldType(EntityHistory.checkpointIdFieldType, pgSchema, false, false, false);
|
|
496
|
+
return `INSERT INTO "` + pgSchema + `"."` + historyTableName + `" (` + allHistoryFieldNamesStr + `)
|
|
497
|
+
SELECT ` + selectPartsStr + `
|
|
498
|
+
FROM UNNEST($1::text[], $2::` + checkpointIdPgType + `[]) AS u(` + Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName + `)`;
|
|
522
499
|
}
|
|
523
500
|
|
|
524
501
|
function executeSet(sql, items, dbFunction) {
|
|
@@ -531,274 +508,257 @@ function executeSet(sql, items, dbFunction) {
|
|
|
531
508
|
|
|
532
509
|
async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, escapeTables) {
|
|
533
510
|
try {
|
|
534
|
-
|
|
535
|
-
|
|
511
|
+
let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
|
|
512
|
+
let specificError = {
|
|
536
513
|
contents: undefined
|
|
537
514
|
};
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
var containsRollbackDiffChange = update.containsRollbackDiffChange;
|
|
567
|
-
update.history.forEach(function (change) {
|
|
568
|
-
if (!containsRollbackDiffChange) {
|
|
569
|
-
backfillHistoryIds.add(change.entityId);
|
|
570
|
-
}
|
|
571
|
-
if (change.type === "SET") {
|
|
572
|
-
batchSetUpdates.push(change);
|
|
573
|
-
return ;
|
|
574
|
-
}
|
|
575
|
-
batchDeleteEntityIds.push(change.entityId);
|
|
576
|
-
batchDeleteCheckpointIds.push(change.checkpointId);
|
|
577
|
-
});
|
|
578
|
-
});
|
|
579
|
-
if (backfillHistoryIds.size !== 0) {
|
|
580
|
-
await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.name, entityConfig.index, Array.from(backfillHistoryIds));
|
|
581
|
-
}
|
|
582
|
-
if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
|
|
583
|
-
promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema), [
|
|
584
|
-
batchDeleteEntityIds,
|
|
585
|
-
$$BigInt.arrayToStringArray(batchDeleteCheckpointIds)
|
|
586
|
-
], {prepare: true}));
|
|
587
|
-
}
|
|
588
|
-
if (Utils.$$Array.notEmpty(batchSetUpdates)) {
|
|
589
|
-
if (shouldRemoveInvalidUtf8) {
|
|
590
|
-
var entities = batchSetUpdates.map(function (batchSetUpdate) {
|
|
591
|
-
if (batchSetUpdate.type === "SET") {
|
|
592
|
-
return batchSetUpdate.entity;
|
|
593
|
-
} else {
|
|
594
|
-
return Js_exn.raiseError("Expected Set action");
|
|
595
|
-
}
|
|
596
|
-
});
|
|
597
|
-
removeInvalidUtf8InPlace(entities);
|
|
598
|
-
}
|
|
599
|
-
var entityHistory = getEntityHistory(entityConfig);
|
|
600
|
-
promises.push(setOrThrow(sql, batchSetUpdates, entityHistory.table, entityHistory.setChangeSchema, pgSchema));
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
}
|
|
604
|
-
if (Utils.$$Array.notEmpty(entitiesToSet)) {
|
|
605
|
-
if (shouldRemoveInvalidUtf8) {
|
|
606
|
-
removeInvalidUtf8InPlace(entitiesToSet);
|
|
607
|
-
}
|
|
608
|
-
promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, entityConfig.schema, pgSchema));
|
|
515
|
+
let setRawEvents = __x => executeSet(__x, rawEvents, (sql, items) => setOrThrow(sql, items, InternalTable.RawEvents.table, InternalTable.RawEvents.schema, pgSchema));
|
|
516
|
+
let setEntities = Belt_Array.map(updatedEntities, param => {
|
|
517
|
+
let updates = param.updates;
|
|
518
|
+
let entityConfig = param.entityConfig;
|
|
519
|
+
let entitiesToSet = [];
|
|
520
|
+
let idsToDelete = [];
|
|
521
|
+
updates.forEach(row => {
|
|
522
|
+
let match = row.latestChange;
|
|
523
|
+
if (match.type === "SET") {
|
|
524
|
+
entitiesToSet.push(match.entity);
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
idsToDelete.push(match.entityId);
|
|
528
|
+
});
|
|
529
|
+
let shouldRemoveInvalidUtf8 = escapeTables !== undefined ? Primitive_option.valFromOption(escapeTables).has(entityConfig.table) : false;
|
|
530
|
+
return async sql => {
|
|
531
|
+
try {
|
|
532
|
+
let promises = [];
|
|
533
|
+
if (shouldSaveHistory) {
|
|
534
|
+
let backfillHistoryIds = new Set();
|
|
535
|
+
let batchSetUpdates = [];
|
|
536
|
+
let batchDeleteCheckpointIds = [];
|
|
537
|
+
let batchDeleteEntityIds = [];
|
|
538
|
+
updates.forEach(update => {
|
|
539
|
+
let containsRollbackDiffChange = update.containsRollbackDiffChange;
|
|
540
|
+
update.history.forEach(change => {
|
|
541
|
+
if (!containsRollbackDiffChange) {
|
|
542
|
+
backfillHistoryIds.add(change.entityId);
|
|
609
543
|
}
|
|
610
|
-
if (
|
|
611
|
-
|
|
544
|
+
if (change.type === "SET") {
|
|
545
|
+
batchSetUpdates.push(change);
|
|
546
|
+
return;
|
|
612
547
|
}
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
}
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
table: entityConfig.table
|
|
634
|
-
};
|
|
635
|
-
return ;
|
|
636
|
-
default:
|
|
637
|
-
specificError.contents = Utils.prettifyExn(exn);
|
|
638
|
-
return ;
|
|
639
|
-
}
|
|
640
|
-
} else {
|
|
641
|
-
if (normalizedExn.RE_EXN_ID !== S$RescriptSchema.Raised) {
|
|
642
|
-
return ;
|
|
548
|
+
batchDeleteEntityIds.push(change.entityId);
|
|
549
|
+
batchDeleteCheckpointIds.push(change.checkpointId);
|
|
550
|
+
});
|
|
551
|
+
});
|
|
552
|
+
if (backfillHistoryIds.size !== 0) {
|
|
553
|
+
await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.name, entityConfig.index, Array.from(backfillHistoryIds));
|
|
554
|
+
}
|
|
555
|
+
if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
|
|
556
|
+
promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema), [
|
|
557
|
+
batchDeleteEntityIds,
|
|
558
|
+
Utils.$$BigInt.arrayToStringArray(batchDeleteCheckpointIds)
|
|
559
|
+
], {prepare: true}));
|
|
560
|
+
}
|
|
561
|
+
if (Utils.$$Array.notEmpty(batchSetUpdates)) {
|
|
562
|
+
if (shouldRemoveInvalidUtf8) {
|
|
563
|
+
removeInvalidUtf8InPlace(batchSetUpdates.map(batchSetUpdate => {
|
|
564
|
+
if (batchSetUpdate.type === "SET") {
|
|
565
|
+
return batchSetUpdate.entity;
|
|
566
|
+
} else {
|
|
567
|
+
return Stdlib_JsError.throwWithMessage("Expected Set action");
|
|
643
568
|
}
|
|
644
|
-
|
|
645
|
-
}
|
|
569
|
+
}));
|
|
646
570
|
}
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
571
|
+
let entityHistory = getEntityHistory(entityConfig);
|
|
572
|
+
promises.push(setOrThrow(sql, batchSetUpdates, entityHistory.table, entityHistory.setChangeSchema, pgSchema));
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
if (Utils.$$Array.notEmpty(entitiesToSet)) {
|
|
576
|
+
if (shouldRemoveInvalidUtf8) {
|
|
577
|
+
removeInvalidUtf8InPlace(entitiesToSet);
|
|
578
|
+
}
|
|
579
|
+
promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, entityConfig.schema, pgSchema));
|
|
580
|
+
}
|
|
581
|
+
if (Utils.$$Array.notEmpty(idsToDelete)) {
|
|
582
|
+
promises.push(deleteByIdsOrThrow(sql, pgSchema, idsToDelete, entityConfig.table));
|
|
583
|
+
}
|
|
584
|
+
await Promise.all(promises);
|
|
585
|
+
return;
|
|
586
|
+
} catch (raw_exn) {
|
|
587
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
588
|
+
let normalizedExn = Primitive_exceptions.internalToException(exn.RE_EXN_ID === "JsExn" || exn.RE_EXN_ID !== Persistence.StorageError ? exn : exn.reason);
|
|
589
|
+
if (normalizedExn.RE_EXN_ID === "JsExn") {
|
|
590
|
+
let val;
|
|
591
|
+
try {
|
|
592
|
+
val = S$RescriptSchema.parseOrThrow(normalizedExn._1, pgErrorMessageSchema);
|
|
593
|
+
} catch (exn$1) {
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
switch (val) {
|
|
597
|
+
case "current transaction is aborted, commands ignored until end of transaction block" :
|
|
598
|
+
return;
|
|
599
|
+
case "invalid byte sequence for encoding \"UTF8\": 0x00" :
|
|
600
|
+
specificError.contents = {
|
|
601
|
+
RE_EXN_ID: PgEncodingError,
|
|
602
|
+
table: entityConfig.table
|
|
603
|
+
};
|
|
604
|
+
return;
|
|
605
|
+
default:
|
|
606
|
+
specificError.contents = Utils.prettifyExn(exn);
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
} else {
|
|
610
|
+
if (normalizedExn.RE_EXN_ID !== S$RescriptSchema.Raised) {
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
throw normalizedExn;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
});
|
|
618
|
+
let rollbackTables = rollbackTargetCheckpointId !== undefined ? sql => {
|
|
619
|
+
let promises = allEntities.map(entityConfig => EntityHistory.rollback(sql, pgSchema, entityConfig.name, entityConfig.index, rollbackTargetCheckpointId));
|
|
620
|
+
promises.push(InternalTable.Checkpoints.rollback(sql, pgSchema, rollbackTargetCheckpointId));
|
|
621
|
+
return Promise.all(promises);
|
|
622
|
+
} : undefined;
|
|
656
623
|
try {
|
|
657
624
|
await Promise.all([
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
if (exn === undefined) {
|
|
688
|
-
return ;
|
|
689
|
-
}
|
|
690
|
-
throw exn;
|
|
691
|
-
}),
|
|
692
|
-
Promise.all(Belt_Array.map(updatedEffectsCache, (function (param) {
|
|
693
|
-
return setEffectCacheOrThrow(param.effect, param.items, param.shouldInitialize);
|
|
694
|
-
})))
|
|
695
|
-
]);
|
|
696
|
-
var specificError$1 = specificError.contents;
|
|
625
|
+
sql.begin(async sql => {
|
|
626
|
+
if (rollbackTables !== undefined) {
|
|
627
|
+
await rollbackTables(sql);
|
|
628
|
+
}
|
|
629
|
+
let setOperations = Belt_Array.concat([
|
|
630
|
+
sql => InternalTable.Chains.setProgressedChains(sql, pgSchema, Utils.Dict.mapValuesToArray(batch.progressedChainsById, chainAfterBatch => ({
|
|
631
|
+
chainId: chainAfterBatch.fetchState.chainId,
|
|
632
|
+
progressBlockNumber: chainAfterBatch.progressBlockNumber,
|
|
633
|
+
sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
|
|
634
|
+
totalEventsProcessed: chainAfterBatch.totalEventsProcessed
|
|
635
|
+
}))),
|
|
636
|
+
setRawEvents
|
|
637
|
+
], setEntities);
|
|
638
|
+
if (shouldSaveHistory) {
|
|
639
|
+
setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed));
|
|
640
|
+
}
|
|
641
|
+
await Promise.all(Belt_Array.map(setOperations, dbFunc => dbFunc(sql)));
|
|
642
|
+
if (sinkPromise === undefined) {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
let exn = await sinkPromise;
|
|
646
|
+
if (exn === undefined) {
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
throw exn;
|
|
650
|
+
}),
|
|
651
|
+
Promise.all(Belt_Array.map(updatedEffectsCache, param => setEffectCacheOrThrow(param.effect, param.items, param.shouldInitialize)))
|
|
652
|
+
]);
|
|
653
|
+
let specificError$1 = specificError.contents;
|
|
697
654
|
if (specificError$1 === undefined) {
|
|
698
|
-
return
|
|
655
|
+
return;
|
|
699
656
|
}
|
|
700
657
|
throw specificError$1;
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
var specificError$2 = specificError.contents;
|
|
658
|
+
} catch (raw_exn) {
|
|
659
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
660
|
+
let specificError$2 = specificError.contents;
|
|
705
661
|
throw specificError$2 !== undefined ? specificError$2 : exn;
|
|
706
662
|
}
|
|
707
|
-
}
|
|
708
|
-
|
|
709
|
-
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
|
|
663
|
+
} catch (raw_exn$1) {
|
|
664
|
+
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
|
|
710
665
|
if (exn$1.RE_EXN_ID === PgEncodingError) {
|
|
711
|
-
|
|
666
|
+
let escapeTables$1 = escapeTables !== undefined ? Primitive_option.valFromOption(escapeTables) : new Set();
|
|
712
667
|
escapeTables$1.add(exn$1.table);
|
|
713
|
-
return await writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise,
|
|
668
|
+
return await writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, Primitive_option.some(escapeTables$1));
|
|
714
669
|
}
|
|
715
670
|
throw exn$1;
|
|
716
671
|
}
|
|
717
672
|
}
|
|
718
673
|
|
|
719
674
|
function makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema) {
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
675
|
+
let dataFieldNames = Belt_Array.keepMap(entityConfig.table.fields, fieldOrDerived => {
|
|
676
|
+
if (fieldOrDerived.TAG === "Field") {
|
|
677
|
+
return Table.getDbFieldName(fieldOrDerived._0);
|
|
678
|
+
}
|
|
679
|
+
});
|
|
680
|
+
let dataFieldsCommaSeparated = Belt_Array.map(dataFieldNames, name => `"` + name + `"`).join(", ");
|
|
681
|
+
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
682
|
+
return `SELECT DISTINCT ON (` + Table.idFieldName + `) ` + dataFieldsCommaSeparated + `
|
|
683
|
+
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
684
|
+
WHERE "` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
685
|
+
AND EXISTS (
|
|
686
|
+
SELECT 1
|
|
687
|
+
FROM "` + pgSchema + `"."` + historyTableName + `" h
|
|
688
|
+
WHERE h.` + Table.idFieldName + ` = "` + historyTableName + `".` + Table.idFieldName + `
|
|
689
|
+
AND h."` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
690
|
+
)
|
|
691
|
+
ORDER BY ` + Table.idFieldName + `, "` + EntityHistory.checkpointIdFieldName + `" DESC`;
|
|
731
692
|
}
|
|
732
693
|
|
|
733
694
|
function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
|
|
734
|
-
|
|
735
|
-
return
|
|
695
|
+
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
696
|
+
return `SELECT DISTINCT ` + Table.idFieldName + `
|
|
697
|
+
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
698
|
+
WHERE "` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
699
|
+
AND NOT EXISTS (
|
|
700
|
+
SELECT 1
|
|
701
|
+
FROM "` + pgSchema + `"."` + historyTableName + `" h
|
|
702
|
+
WHERE h.` + Table.idFieldName + ` = "` + historyTableName + `".` + Table.idFieldName + `
|
|
703
|
+
AND h."` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
704
|
+
)`;
|
|
736
705
|
}
|
|
737
706
|
|
|
738
707
|
function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
708
|
+
let containerName = "envio-postgres";
|
|
709
|
+
let psqlExecOptions_env = Object.fromEntries([
|
|
710
|
+
[
|
|
711
|
+
"PGPASSWORD",
|
|
712
|
+
pgPassword
|
|
713
|
+
],
|
|
714
|
+
[
|
|
715
|
+
"PATH",
|
|
716
|
+
process.env.PATH
|
|
717
|
+
]
|
|
718
|
+
]);
|
|
719
|
+
let psqlExecOptions = {
|
|
751
720
|
env: psqlExecOptions_env
|
|
752
721
|
};
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
return Utils.$$Array.notEmpty(envioTables);
|
|
757
|
-
};
|
|
758
|
-
var restoreEffectCache = async function (withUpload) {
|
|
722
|
+
let cacheDirPath = Path.resolve(".envio", "cache");
|
|
723
|
+
let isInitialized = async () => Utils.$$Array.notEmpty(await sql.unsafe(`SELECT table_schema FROM information_schema.tables WHERE table_schema = '` + pgSchema + `' AND (table_name = '` + "event_sync_state" + `' OR table_name = '` + InternalTable.Chains.table.tableName + `');`));
|
|
724
|
+
let restoreEffectCache = async withUpload => {
|
|
759
725
|
if (withUpload) {
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
]);
|
|
775
|
-
var exit = 0;
|
|
776
|
-
var message;
|
|
777
|
-
var entries = match[0];
|
|
726
|
+
let nothingToUploadErrorMessage = "Nothing to upload.";
|
|
727
|
+
let match = await Promise.all([
|
|
728
|
+
Stdlib_Promise.$$catch(Fs.promises.readdir(cacheDirPath).then(e => ({
|
|
729
|
+
TAG: "Ok",
|
|
730
|
+
_0: e
|
|
731
|
+
})), param => Promise.resolve({
|
|
732
|
+
TAG: "Error",
|
|
733
|
+
_0: nothingToUploadErrorMessage
|
|
734
|
+
})),
|
|
735
|
+
getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName)
|
|
736
|
+
]);
|
|
737
|
+
let exit = 0;
|
|
738
|
+
let message;
|
|
739
|
+
let entries = match[0];
|
|
778
740
|
if (entries.TAG === "Ok") {
|
|
779
|
-
|
|
741
|
+
let psqlExec = match[1];
|
|
780
742
|
if (psqlExec.TAG === "Ok") {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
743
|
+
let psqlExec$1 = psqlExec._0;
|
|
744
|
+
let cacheFiles = entries._0.filter(entry => entry.endsWith(".tsv"));
|
|
745
|
+
await Promise.all(cacheFiles.map(entry => {
|
|
746
|
+
let effectName = entry.slice(0, -4);
|
|
747
|
+
let table = Internal.makeCacheTable(effectName);
|
|
748
|
+
return sql.unsafe(makeCreateTableQuery(table, pgSchema, false)).then(() => {
|
|
749
|
+
let inputFile = Path.join(cacheDirPath, entry);
|
|
750
|
+
let command = psqlExec$1 + ` -c 'COPY "` + pgSchema + `"."` + table.tableName + `" FROM STDIN WITH (FORMAT text, HEADER);' < ` + inputFile;
|
|
751
|
+
return new Promise((resolve, reject) => {
|
|
752
|
+
Child_process.exec(command, psqlExecOptions, (error, stdout, param) => {
|
|
753
|
+
if (error === null) {
|
|
754
|
+
return resolve(stdout);
|
|
755
|
+
} else {
|
|
756
|
+
return reject(error);
|
|
757
|
+
}
|
|
758
|
+
});
|
|
784
759
|
});
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
var table = Internal.makeCacheTable(effectName);
|
|
788
|
-
return sql.unsafe(makeCreateTableQuery(table, pgSchema, false)).then(function () {
|
|
789
|
-
var inputFile = Path.join(cacheDirPath, entry);
|
|
790
|
-
var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + table.tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
|
|
791
|
-
return new Promise((function (resolve, reject) {
|
|
792
|
-
Child_process.exec(command, psqlExecOptions, (function (error, stdout, param) {
|
|
793
|
-
if (error === null) {
|
|
794
|
-
return resolve(stdout);
|
|
795
|
-
} else {
|
|
796
|
-
return reject(error);
|
|
797
|
-
}
|
|
798
|
-
}));
|
|
799
|
-
}));
|
|
800
|
-
});
|
|
801
|
-
}));
|
|
760
|
+
});
|
|
761
|
+
}));
|
|
802
762
|
Logging.info("Successfully uploaded cache.");
|
|
803
763
|
} else {
|
|
804
764
|
message = match[1]._0;
|
|
@@ -812,343 +772,363 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
812
772
|
if (message === nothingToUploadErrorMessage) {
|
|
813
773
|
Logging.info("No cache found to upload.");
|
|
814
774
|
} else {
|
|
815
|
-
Logging.error(
|
|
775
|
+
Logging.error(`Failed to upload cache, continuing without it. ` + message);
|
|
816
776
|
}
|
|
817
777
|
}
|
|
818
|
-
|
|
819
778
|
}
|
|
820
|
-
|
|
779
|
+
let cacheTableInfo = await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema));
|
|
821
780
|
if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
|
|
822
|
-
await onNewTables(cacheTableInfo.map(
|
|
823
|
-
return info.table_name;
|
|
824
|
-
}));
|
|
781
|
+
await onNewTables(cacheTableInfo.map(info => info.table_name));
|
|
825
782
|
}
|
|
826
|
-
|
|
827
|
-
cacheTableInfo.forEach(
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
783
|
+
let cache = {};
|
|
784
|
+
cacheTableInfo.forEach(param => {
|
|
785
|
+
let effectName = param.table_name.slice(cacheTablePrefixLength);
|
|
786
|
+
cache[effectName] = {
|
|
787
|
+
effectName: effectName,
|
|
788
|
+
count: param.count
|
|
789
|
+
};
|
|
790
|
+
});
|
|
834
791
|
return cache;
|
|
835
792
|
};
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(
|
|
842
|
-
|
|
843
|
-
})) {
|
|
844
|
-
Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
|
|
793
|
+
let initialize = async (chainConfigsOpt, entitiesOpt, enumsOpt) => {
|
|
794
|
+
let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
|
|
795
|
+
let entities = entitiesOpt !== undefined ? entitiesOpt : [];
|
|
796
|
+
let enums = enumsOpt !== undefined ? enumsOpt : [];
|
|
797
|
+
let schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
|
|
798
|
+
if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(table => table.table_name === InternalTable.Chains.table.tableName ? true : table.table_name === "event_sync_state")) {
|
|
799
|
+
Stdlib_JsError.throwWithMessage(`Cannot run Envio migrations on PostgreSQL schema "` + pgSchema + `" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: "pnpm envio local db-migrate down"\n2. Or specify a different schema name by setting the "ENVIO_PG_SCHEMA" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.`);
|
|
845
800
|
}
|
|
846
801
|
if (sink !== undefined) {
|
|
847
802
|
await sink.initialize(chainConfigs, entities, enums);
|
|
848
803
|
}
|
|
849
|
-
|
|
850
|
-
await sql.begin(
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
804
|
+
let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, entities, enums, Utils.$$Array.isEmpty(schemaTableNames));
|
|
805
|
+
await sql.begin(sql => Promise.all(queries.map(query => sql.unsafe(query))));
|
|
806
|
+
let ids = [];
|
|
807
|
+
let addrChainIds = [];
|
|
808
|
+
let addrContractNames = [];
|
|
809
|
+
chainConfigs.forEach(chain => {
|
|
810
|
+
chain.contracts.forEach(contract => {
|
|
811
|
+
contract.addresses.forEach(address => {
|
|
812
|
+
ids.push(Config.EnvioAddresses.makeId(chain.id, address));
|
|
813
|
+
addrChainIds.push(chain.id);
|
|
814
|
+
addrContractNames.push(contract.name);
|
|
854
815
|
});
|
|
855
|
-
|
|
816
|
+
});
|
|
817
|
+
});
|
|
818
|
+
if (ids.length !== 0) {
|
|
819
|
+
await sql.unsafe(`INSERT INTO "` + pgSchema + `"."` + Config.EnvioAddresses.table.tableName + `" ("id", "chain_id", "registration_block", "registration_log_index", "contract_name")
|
|
820
|
+
SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::text[]) AS t(id, chain_id, contract_name);`, [
|
|
821
|
+
ids,
|
|
822
|
+
addrChainIds,
|
|
823
|
+
addrContractNames
|
|
824
|
+
]);
|
|
825
|
+
}
|
|
826
|
+
let cache = await restoreEffectCache(true);
|
|
856
827
|
if (onInitialize !== undefined) {
|
|
857
828
|
await onInitialize();
|
|
858
829
|
}
|
|
859
830
|
return {
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
reorgCheckpoints: []
|
|
878
|
-
};
|
|
831
|
+
cleanRun: true,
|
|
832
|
+
cache: cache,
|
|
833
|
+
chains: chainConfigs.map(chainConfig => ({
|
|
834
|
+
id: chainConfig.id,
|
|
835
|
+
startBlock: chainConfig.startBlock,
|
|
836
|
+
endBlock: chainConfig.endBlock,
|
|
837
|
+
maxReorgDepth: chainConfig.maxReorgDepth,
|
|
838
|
+
progressBlockNumber: -1,
|
|
839
|
+
numEventsProcessed: 0,
|
|
840
|
+
firstEventBlockNumber: undefined,
|
|
841
|
+
timestampCaughtUpToHeadOrEndblock: undefined,
|
|
842
|
+
indexingAddresses: ChainFetcher.configAddresses(chainConfig),
|
|
843
|
+
sourceBlockNumber: 0
|
|
844
|
+
})),
|
|
845
|
+
checkpointId: InternalTable.Checkpoints.initialCheckpointId,
|
|
846
|
+
reorgCheckpoints: []
|
|
847
|
+
};
|
|
879
848
|
};
|
|
880
|
-
|
|
881
|
-
|
|
849
|
+
let loadByIdsOrThrow = async (ids, table, rowsSchema) => {
|
|
850
|
+
let rows;
|
|
882
851
|
try {
|
|
883
852
|
rows = await (
|
|
884
853
|
ids.length !== 1 ? sql.unsafe(makeLoadByIdsQuery(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeLoadByIdQuery(pgSchema, table.tableName), ids, {prepare: true})
|
|
885
854
|
);
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
855
|
+
} catch (raw_exn) {
|
|
856
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
889
857
|
throw {
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
858
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
859
|
+
message: `Failed loading "` + table.tableName + `" from storage by ids`,
|
|
860
|
+
reason: exn,
|
|
861
|
+
Error: new Error()
|
|
862
|
+
};
|
|
895
863
|
}
|
|
896
864
|
try {
|
|
897
865
|
return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
|
|
866
|
+
} catch (raw_exn$1) {
|
|
867
|
+
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
|
|
901
868
|
throw {
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
869
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
870
|
+
message: `Failed to parse "` + table.tableName + `" loaded from storage by ids`,
|
|
871
|
+
reason: exn$1,
|
|
872
|
+
Error: new Error()
|
|
873
|
+
};
|
|
907
874
|
}
|
|
908
875
|
};
|
|
909
|
-
|
|
910
|
-
|
|
876
|
+
let loadByFieldOrThrow = async (fieldName, fieldSchema, fieldValue, operator, table, rowsSchema) => {
|
|
877
|
+
let params;
|
|
911
878
|
try {
|
|
912
879
|
params = [S$RescriptSchema.reverseConvertToJsonOrThrow(fieldValue, fieldSchema)];
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
|
|
880
|
+
} catch (raw_exn) {
|
|
881
|
+
let exn = Primitive_exceptions.internalToException(raw_exn);
|
|
916
882
|
throw {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
883
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
884
|
+
message: `Failed loading "` + table.tableName + `" from storage by field "` + fieldName + `". Couldn't serialize provided value.`,
|
|
885
|
+
reason: exn,
|
|
886
|
+
Error: new Error()
|
|
887
|
+
};
|
|
922
888
|
}
|
|
923
|
-
|
|
889
|
+
let rows;
|
|
924
890
|
try {
|
|
925
891
|
rows = await sql.unsafe(makeLoadByFieldQuery(pgSchema, table.tableName, fieldName, operator), params, {prepare: true});
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
|
|
892
|
+
} catch (raw_exn$1) {
|
|
893
|
+
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
|
|
929
894
|
throw {
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
895
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
896
|
+
message: `Failed loading "` + table.tableName + `" from storage by field "` + fieldName + `"`,
|
|
897
|
+
reason: exn$1,
|
|
898
|
+
Error: new Error()
|
|
899
|
+
};
|
|
935
900
|
}
|
|
936
901
|
try {
|
|
937
902
|
return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
var exn$2 = Caml_js_exceptions.internalToOCamlException(raw_exn$2);
|
|
903
|
+
} catch (raw_exn$2) {
|
|
904
|
+
let exn$2 = Primitive_exceptions.internalToException(raw_exn$2);
|
|
941
905
|
throw {
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
906
|
+
RE_EXN_ID: Persistence.StorageError,
|
|
907
|
+
message: `Failed to parse "` + table.tableName + `" loaded from storage by ids`,
|
|
908
|
+
reason: exn$2,
|
|
909
|
+
Error: new Error()
|
|
910
|
+
};
|
|
947
911
|
}
|
|
948
912
|
};
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
var match = effect.storageMeta;
|
|
954
|
-
var table = match.table;
|
|
913
|
+
let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema);
|
|
914
|
+
let setEffectCacheOrThrow = async (effect, items, initialize) => {
|
|
915
|
+
let match = effect.storageMeta;
|
|
916
|
+
let table = match.table;
|
|
955
917
|
if (initialize) {
|
|
956
918
|
await sql.unsafe(makeCreateTableQuery(table, pgSchema, false));
|
|
957
919
|
if (onNewTables !== undefined) {
|
|
958
920
|
await onNewTables([table.tableName]);
|
|
959
921
|
}
|
|
960
|
-
|
|
961
922
|
}
|
|
962
923
|
return await setOrThrow$1(items, table, match.itemSchema);
|
|
963
924
|
};
|
|
964
|
-
|
|
925
|
+
let dumpEffectCache = async () => {
|
|
965
926
|
try {
|
|
966
|
-
|
|
967
|
-
return i.count > 0;
|
|
968
|
-
});
|
|
927
|
+
let cacheTableInfo = (await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema))).filter(i => i.count > 0);
|
|
969
928
|
if (!Utils.$$Array.notEmpty(cacheTableInfo)) {
|
|
970
|
-
return
|
|
929
|
+
return;
|
|
971
930
|
}
|
|
972
931
|
try {
|
|
973
932
|
await Fs.promises.access(cacheDirPath);
|
|
974
|
-
}
|
|
975
|
-
catch (exn){
|
|
933
|
+
} catch (exn) {
|
|
976
934
|
await Fs.promises.mkdir(cacheDirPath, {
|
|
977
|
-
|
|
978
|
-
|
|
935
|
+
recursive: true
|
|
936
|
+
});
|
|
979
937
|
}
|
|
980
|
-
|
|
938
|
+
let psqlExec = await getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName);
|
|
981
939
|
if (psqlExec.TAG !== "Ok") {
|
|
982
|
-
return Logging.error(
|
|
940
|
+
return Logging.error(`Failed to dump cache. ` + psqlExec._0);
|
|
983
941
|
}
|
|
984
|
-
|
|
985
|
-
Logging.info(
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
return reject(error);
|
|
999
|
-
}
|
|
1000
|
-
}));
|
|
1001
|
-
}));
|
|
942
|
+
let psqlExec$1 = psqlExec._0;
|
|
943
|
+
Logging.info(`Dumping cache: ` + cacheTableInfo.map(param => param.table_name + " (" + String(param.count) + " rows)").join(", "));
|
|
944
|
+
let promises = cacheTableInfo.map(async param => {
|
|
945
|
+
let tableName = param.table_name;
|
|
946
|
+
let cacheName = tableName.slice(cacheTablePrefixLength);
|
|
947
|
+
let outputFile = Path.join(cacheDirPath, cacheName + ".tsv");
|
|
948
|
+
let command = psqlExec$1 + ` -c 'COPY "` + pgSchema + `"."` + tableName + `" TO STDOUT WITH (FORMAT text, HEADER);' > ` + outputFile;
|
|
949
|
+
return new Promise((resolve, reject) => {
|
|
950
|
+
Child_process.exec(command, psqlExecOptions, (error, stdout, param) => {
|
|
951
|
+
if (error === null) {
|
|
952
|
+
return resolve(stdout);
|
|
953
|
+
} else {
|
|
954
|
+
return reject(error);
|
|
955
|
+
}
|
|
1002
956
|
});
|
|
957
|
+
});
|
|
958
|
+
});
|
|
1003
959
|
await Promise.all(promises);
|
|
1004
|
-
return Logging.info(
|
|
1005
|
-
}
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
return Logging.errorWithExn(Utils.prettifyExn(exn$1), "Failed to dump cache.");
|
|
960
|
+
return Logging.info(`Successfully dumped cache to ` + cacheDirPath);
|
|
961
|
+
} catch (raw_exn) {
|
|
962
|
+
let exn$1 = Primitive_exceptions.internalToException(raw_exn);
|
|
963
|
+
return Logging.errorWithExn(Utils.prettifyExn(exn$1), `Failed to dump cache.`);
|
|
1009
964
|
}
|
|
1010
965
|
};
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
id: BigInt(raw.id),
|
|
1037
|
-
chain_id: raw.chain_id,
|
|
1038
|
-
block_number: raw.block_number,
|
|
1039
|
-
block_hash: raw.block_hash
|
|
1040
|
-
};
|
|
1041
|
-
}));
|
|
966
|
+
let resumeInitialState = async () => {
|
|
967
|
+
let match = await Promise.all([
|
|
968
|
+
restoreEffectCache(false),
|
|
969
|
+
InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => Belt_Array.map(rawInitialStates, rawInitialState => ({
|
|
970
|
+
id: rawInitialState.id,
|
|
971
|
+
startBlock: rawInitialState.startBlock,
|
|
972
|
+
endBlock: Primitive_option.fromNull(rawInitialState.endBlock),
|
|
973
|
+
maxReorgDepth: rawInitialState.maxReorgDepth,
|
|
974
|
+
progressBlockNumber: rawInitialState.progressBlockNumber,
|
|
975
|
+
numEventsProcessed: rawInitialState.numEventsProcessed,
|
|
976
|
+
firstEventBlockNumber: Primitive_option.fromNull(rawInitialState.firstEventBlockNumber),
|
|
977
|
+
timestampCaughtUpToHeadOrEndblock: Primitive_option.fromNull(rawInitialState.timestampCaughtUpToHeadOrEndblock),
|
|
978
|
+
indexingAddresses: rawInitialState.indexingAddresses,
|
|
979
|
+
sourceBlockNumber: rawInitialState.sourceBlockNumber
|
|
980
|
+
}))),
|
|
981
|
+
sql.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)),
|
|
982
|
+
sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema))
|
|
983
|
+
]);
|
|
984
|
+
let checkpointId = BigInt(match[2][0].id);
|
|
985
|
+
let reorgCheckpoints = Belt_Array.map(match[3], raw => ({
|
|
986
|
+
id: BigInt(raw.id),
|
|
987
|
+
chain_id: raw.chain_id,
|
|
988
|
+
block_number: raw.block_number,
|
|
989
|
+
block_hash: raw.block_hash
|
|
990
|
+
}));
|
|
1042
991
|
if (sink !== undefined) {
|
|
1043
992
|
await sink.resume(checkpointId);
|
|
1044
993
|
}
|
|
1045
994
|
return {
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
995
|
+
cleanRun: false,
|
|
996
|
+
cache: match[0],
|
|
997
|
+
chains: match[1],
|
|
998
|
+
checkpointId: checkpointId,
|
|
999
|
+
reorgCheckpoints: reorgCheckpoints
|
|
1000
|
+
};
|
|
1052
1001
|
};
|
|
1053
|
-
|
|
1054
|
-
|
|
1002
|
+
let reset = async () => {
|
|
1003
|
+
let query = `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;`;
|
|
1055
1004
|
return await sql.unsafe(query);
|
|
1056
1005
|
};
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
var getRollbackTargetCheckpoint = function (reorgChainId, lastKnownValidBlockNumber) {
|
|
1069
|
-
return InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
|
|
1070
|
-
};
|
|
1071
|
-
var getRollbackProgressDiff = function (rollbackTargetCheckpointId) {
|
|
1072
|
-
return InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId);
|
|
1073
|
-
};
|
|
1074
|
-
var getRollbackData = async function (entityConfig, rollbackTargetCheckpointId) {
|
|
1075
|
-
return await Promise.all([
|
|
1076
|
-
sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
|
|
1077
|
-
sql.unsafe(makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
|
|
1078
|
-
]);
|
|
1079
|
-
};
|
|
1080
|
-
var writeBatchMethod = async function (batch, rawEvents, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities) {
|
|
1081
|
-
var sinkPromise;
|
|
1006
|
+
let setChainMeta = chainsData => InternalTable.Chains.setMeta(sql, pgSchema, chainsData).then(param => (undefined));
|
|
1007
|
+
let pruneStaleCheckpoints = safeCheckpointId => InternalTable.Checkpoints.pruneStaleCheckpoints(sql, pgSchema, safeCheckpointId);
|
|
1008
|
+
let pruneStaleEntityHistory = (entityName, entityIndex, safeCheckpointId) => EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, safeCheckpointId);
|
|
1009
|
+
let getRollbackTargetCheckpoint = (reorgChainId, lastKnownValidBlockNumber) => InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
|
|
1010
|
+
let getRollbackProgressDiff = rollbackTargetCheckpointId => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId);
|
|
1011
|
+
let getRollbackData = async (entityConfig, rollbackTargetCheckpointId) => await Promise.all([
|
|
1012
|
+
sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
|
|
1013
|
+
sql.unsafe(makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
|
|
1014
|
+
]);
|
|
1015
|
+
let writeBatchMethod = async (batch, rawEvents, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities) => {
|
|
1016
|
+
let sinkPromise;
|
|
1082
1017
|
if (sink !== undefined) {
|
|
1083
|
-
|
|
1084
|
-
sinkPromise =
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
return exn;
|
|
1088
|
-
}));
|
|
1018
|
+
let timerRef = Hrtime.makeTimer();
|
|
1019
|
+
sinkPromise = sink.writeBatch(batch, updatedEntities).then(() => {
|
|
1020
|
+
Prometheus.StorageWrite.increment(sink.name, Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef)));
|
|
1021
|
+
}).catch(exn => exn);
|
|
1089
1022
|
} else {
|
|
1090
1023
|
sinkPromise = undefined;
|
|
1091
1024
|
}
|
|
1092
|
-
|
|
1025
|
+
let primaryTimerRef = Hrtime.makeTimer();
|
|
1026
|
+
await writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, undefined);
|
|
1027
|
+
return Prometheus.StorageWrite.increment("postgres", Hrtime.toSecondsFloat(Hrtime.timeSince(primaryTimerRef)));
|
|
1093
1028
|
};
|
|
1094
1029
|
return {
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1030
|
+
name: "postgres",
|
|
1031
|
+
isInitialized: isInitialized,
|
|
1032
|
+
initialize: initialize,
|
|
1033
|
+
resumeInitialState: resumeInitialState,
|
|
1034
|
+
loadByIdsOrThrow: loadByIdsOrThrow,
|
|
1035
|
+
loadByFieldOrThrow: loadByFieldOrThrow,
|
|
1036
|
+
dumpEffectCache: dumpEffectCache,
|
|
1037
|
+
reset: reset,
|
|
1038
|
+
setChainMeta: setChainMeta,
|
|
1039
|
+
pruneStaleCheckpoints: pruneStaleCheckpoints,
|
|
1040
|
+
pruneStaleEntityHistory: pruneStaleEntityHistory,
|
|
1041
|
+
getRollbackTargetCheckpoint: getRollbackTargetCheckpoint,
|
|
1042
|
+
getRollbackProgressDiff: getRollbackProgressDiff,
|
|
1043
|
+
getRollbackData: getRollbackData,
|
|
1044
|
+
writeBatch: writeBatchMethod
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
function makeStorageFromEnv(config, sqlOpt, pgSchemaOpt, isHasuraEnabledOpt) {
|
|
1049
|
+
let sql = sqlOpt !== undefined ? Primitive_option.valFromOption(sqlOpt) : makeClient();
|
|
1050
|
+
let pgSchema = pgSchemaOpt !== undefined ? pgSchemaOpt : Env.Db.publicSchema;
|
|
1051
|
+
let isHasuraEnabled = isHasuraEnabledOpt !== undefined ? isHasuraEnabledOpt : Env.Hasura.enabled;
|
|
1052
|
+
let tmp;
|
|
1053
|
+
if (config.storage.clickhouse) {
|
|
1054
|
+
let host = Env.ClickHouse.host();
|
|
1055
|
+
let username = Env.ClickHouse.username();
|
|
1056
|
+
let password = Env.ClickHouse.password();
|
|
1057
|
+
let missing = [];
|
|
1058
|
+
let checkEnv = (opt, name) => {
|
|
1059
|
+
if (opt !== undefined) {
|
|
1060
|
+
return;
|
|
1061
|
+
} else {
|
|
1062
|
+
missing.push(name);
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
checkEnv(host, "ENVIO_CLICKHOUSE_HOST");
|
|
1067
|
+
checkEnv(username, "ENVIO_CLICKHOUSE_USERNAME");
|
|
1068
|
+
checkEnv(password, "ENVIO_CLICKHOUSE_PASSWORD");
|
|
1069
|
+
if (missing.length !== 0) {
|
|
1070
|
+
Stdlib_JsError.throwWithMessage(`ClickHouse storage is enabled but required env vars are not set: ` + missing.join(", ") + `. Please set them, disable clickhouse in the \`storage\` config, or run \`envio dev\` for a pre-configured local ClickHouse.`);
|
|
1071
|
+
}
|
|
1072
|
+
tmp = Sink.makeClickHouse(host, Env.ClickHouse.database(), username, password);
|
|
1073
|
+
} else {
|
|
1074
|
+
tmp = undefined;
|
|
1075
|
+
}
|
|
1076
|
+
return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, tmp, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
|
|
1077
|
+
role: Env.Hasura.role,
|
|
1078
|
+
secret: Env.Hasura.secret
|
|
1079
|
+
}, pgSchema, config.userEntities, Env.Hasura.aggregateEntities, Env.Hasura.responseLimit, Schema.make(Belt_Array.map(config.allEntities, e => e.table))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking tables`))) : undefined, isHasuraEnabled ? tableNames => Stdlib_Promise.$$catch(Hasura.trackTables(Env.Hasura.graphqlEndpoint, {
|
|
1080
|
+
role: Env.Hasura.role,
|
|
1081
|
+
secret: Env.Hasura.secret
|
|
1082
|
+
}, pgSchema, tableNames), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking new tables`))) : undefined);
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
function makePersistenceFromConfig(config, storageOpt) {
|
|
1086
|
+
let storage = storageOpt !== undefined ? storageOpt : makeStorageFromEnv(config, undefined, undefined, undefined);
|
|
1087
|
+
return Persistence.make(config.userEntities, config.allEnums, storage);
|
|
1110
1088
|
}
|
|
1111
1089
|
|
|
1112
|
-
|
|
1090
|
+
let maxItemsPerQuery = 500;
|
|
1113
1091
|
|
|
1114
1092
|
export {
|
|
1115
|
-
getCacheRowCountFnName
|
|
1116
|
-
makeClient
|
|
1117
|
-
makeCreateIndexQuery
|
|
1118
|
-
directionToSql
|
|
1119
|
-
directionToIndexName
|
|
1120
|
-
makeCreateCompositeIndexQuery
|
|
1121
|
-
makeCreateTableIndicesQuery
|
|
1122
|
-
makeCreateTableQuery
|
|
1123
|
-
entityHistoryCache
|
|
1124
|
-
getEntityHistory
|
|
1125
|
-
makeInitializeTransaction
|
|
1126
|
-
makeLoadByIdQuery
|
|
1127
|
-
makeLoadByFieldQuery
|
|
1128
|
-
makeLoadByIdsQuery
|
|
1129
|
-
makeDeleteByIdQuery
|
|
1130
|
-
makeDeleteByIdsQuery
|
|
1131
|
-
makeLoadAllQuery
|
|
1132
|
-
makeInsertUnnestSetQuery
|
|
1133
|
-
makeInsertValuesSetQuery
|
|
1134
|
-
maxItemsPerQuery
|
|
1135
|
-
makeTableBatchSetQuery
|
|
1136
|
-
chunkArray
|
|
1137
|
-
removeInvalidUtf8InPlace
|
|
1138
|
-
pgErrorMessageSchema
|
|
1139
|
-
PgEncodingError
|
|
1140
|
-
setQueryCache
|
|
1141
|
-
setOrThrow
|
|
1142
|
-
makeSchemaTableNamesQuery
|
|
1143
|
-
cacheTablePrefixLength
|
|
1144
|
-
makeSchemaCacheTableInfoQuery
|
|
1145
|
-
getConnectedPsqlExec
|
|
1146
|
-
deleteByIdsOrThrow
|
|
1147
|
-
makeInsertDeleteUpdatesQuery
|
|
1148
|
-
executeSet
|
|
1149
|
-
writeBatch
|
|
1150
|
-
makeGetRollbackRestoredEntitiesQuery
|
|
1151
|
-
makeGetRollbackRemovedIdsQuery
|
|
1152
|
-
make
|
|
1093
|
+
getCacheRowCountFnName,
|
|
1094
|
+
makeClient,
|
|
1095
|
+
makeCreateIndexQuery,
|
|
1096
|
+
directionToSql,
|
|
1097
|
+
directionToIndexName,
|
|
1098
|
+
makeCreateCompositeIndexQuery,
|
|
1099
|
+
makeCreateTableIndicesQuery,
|
|
1100
|
+
makeCreateTableQuery,
|
|
1101
|
+
entityHistoryCache,
|
|
1102
|
+
getEntityHistory,
|
|
1103
|
+
makeInitializeTransaction,
|
|
1104
|
+
makeLoadByIdQuery,
|
|
1105
|
+
makeLoadByFieldQuery,
|
|
1106
|
+
makeLoadByIdsQuery,
|
|
1107
|
+
makeDeleteByIdQuery,
|
|
1108
|
+
makeDeleteByIdsQuery,
|
|
1109
|
+
makeLoadAllQuery,
|
|
1110
|
+
makeInsertUnnestSetQuery,
|
|
1111
|
+
makeInsertValuesSetQuery,
|
|
1112
|
+
maxItemsPerQuery,
|
|
1113
|
+
makeTableBatchSetQuery,
|
|
1114
|
+
chunkArray,
|
|
1115
|
+
removeInvalidUtf8InPlace,
|
|
1116
|
+
pgErrorMessageSchema,
|
|
1117
|
+
PgEncodingError,
|
|
1118
|
+
setQueryCache,
|
|
1119
|
+
setOrThrow,
|
|
1120
|
+
makeSchemaTableNamesQuery,
|
|
1121
|
+
cacheTablePrefixLength,
|
|
1122
|
+
makeSchemaCacheTableInfoQuery,
|
|
1123
|
+
getConnectedPsqlExec,
|
|
1124
|
+
deleteByIdsOrThrow,
|
|
1125
|
+
makeInsertDeleteUpdatesQuery,
|
|
1126
|
+
executeSet,
|
|
1127
|
+
writeBatch,
|
|
1128
|
+
makeGetRollbackRestoredEntitiesQuery,
|
|
1129
|
+
makeGetRollbackRemovedIdsQuery,
|
|
1130
|
+
make,
|
|
1131
|
+
makeStorageFromEnv,
|
|
1132
|
+
makePersistenceFromConfig,
|
|
1153
1133
|
}
|
|
1154
1134
|
/* entityHistoryCache Not a pure module */
|