envio 3.0.0-alpha.19 → 3.0.0-alpha.19-main-node-pg-client
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/Config.res +1 -0
- package/src/Config.res.mjs +2 -2
- package/src/Env.res +1 -1
- package/src/Env.res.mjs +2 -2
- package/src/Internal.res.mjs +1 -1
- package/src/PgStorage.res +192 -124
- package/src/PgStorage.res.mjs +219 -112
- package/src/bindings/ClickHouse.res +1 -1
- package/src/bindings/ClickHouse.res.mjs +1 -1
- package/src/bindings/Pg.res +146 -0
- package/src/bindings/{Postgres.res.d.mts → Pg.res.d.mts} +1 -1
- package/src/bindings/Pg.res.mjs +110 -0
- package/src/db/EntityHistory.res +21 -21
- package/src/db/EntityHistory.res.mjs +16 -3
- package/src/db/InternalTable.res +47 -49
- package/src/db/InternalTable.res.mjs +57 -23
- package/src/db/Table.res +45 -33
- package/src/db/Table.res.mjs +43 -41
- package/src/PgStorage.gen.ts +0 -10
- package/src/bindings/Postgres.gen.ts +0 -8
- package/src/bindings/Postgres.res +0 -120
- package/src/bindings/Postgres.res.mjs +0 -17
package/src/PgStorage.res.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
|
+
import * as Pg from "./bindings/Pg.res.mjs";
|
|
3
4
|
import * as Fs from "fs";
|
|
4
5
|
import * as Env from "./Env.res.mjs";
|
|
5
6
|
import * as Path from "path";
|
|
@@ -14,9 +15,7 @@ import * as Schema from "./db/Schema.res.mjs";
|
|
|
14
15
|
import * as Js_dict from "rescript/lib/es6/js_dict.js";
|
|
15
16
|
import * as Logging from "./Logging.res.mjs";
|
|
16
17
|
import * as $$Promise from "./bindings/Promise.res.mjs";
|
|
17
|
-
import * as Caml_obj from "rescript/lib/es6/caml_obj.js";
|
|
18
18
|
import * as Internal from "./Internal.res.mjs";
|
|
19
|
-
import Postgres from "postgres";
|
|
20
19
|
import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
|
|
21
20
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
22
21
|
import * as Caml_option from "rescript/lib/es6/caml_option.js";
|
|
@@ -31,20 +30,14 @@ import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
|
|
|
31
30
|
var getCacheRowCountFnName = "get_cache_row_count";
|
|
32
31
|
|
|
33
32
|
function makeClient() {
|
|
34
|
-
return
|
|
33
|
+
return Pg.makePool({
|
|
35
34
|
host: Env.Db.host,
|
|
36
35
|
port: Env.Db.port,
|
|
37
36
|
database: Env.Db.database,
|
|
38
37
|
username: Env.Db.user,
|
|
39
38
|
password: Env.Db.password,
|
|
40
39
|
ssl: Env.Db.ssl,
|
|
41
|
-
max: Env.Db.maxConnections
|
|
42
|
-
onnotice: Caml_obj.equal(Env.userLogLevel, "warn") || Caml_obj.equal(Env.userLogLevel, "error") ? undefined : (function (_str) {
|
|
43
|
-
|
|
44
|
-
}),
|
|
45
|
-
transform: {
|
|
46
|
-
undefined: null
|
|
47
|
-
}
|
|
40
|
+
max: Env.Db.maxConnections
|
|
48
41
|
});
|
|
49
42
|
}
|
|
50
43
|
|
|
@@ -164,7 +157,7 @@ function getEntityHistory(entityConfig) {
|
|
|
164
157
|
var checkpointIdField = Table.mkField(EntityHistory.checkpointIdFieldName, EntityHistory.checkpointIdFieldType, EntityHistory.unsafeCheckpointIdSchema, undefined, undefined, undefined, true, undefined, undefined);
|
|
165
158
|
var entityTableName = entityConfig.table.tableName;
|
|
166
159
|
var historyTableName = EntityHistory.historyTableName(entityTableName, entityConfig.index);
|
|
167
|
-
var table = Table.mkTable(historyTableName, undefined, Belt_Array.concat(dataFields, [
|
|
160
|
+
var table = Table.mkTable(historyTableName, "h" + String(entityConfig.index), undefined, Belt_Array.concat(dataFields, [
|
|
168
161
|
checkpointIdField,
|
|
169
162
|
actionField
|
|
170
163
|
]));
|
|
@@ -310,29 +303,66 @@ function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
|
|
|
310
303
|
) + ";";
|
|
311
304
|
}
|
|
312
305
|
|
|
306
|
+
function makeJsonStringifier(table, jsonFieldIndices) {
|
|
307
|
+
if (!Utils.$$Array.notEmpty(jsonFieldIndices)) {
|
|
308
|
+
return ;
|
|
309
|
+
}
|
|
310
|
+
var jsonFieldNames = Belt_Array.keepMap(Table.getFields(table), (function (field) {
|
|
311
|
+
var match = field.fieldType;
|
|
312
|
+
if (typeof match !== "object" && match === "Json") {
|
|
313
|
+
return Table.getDbFieldName(field);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
}));
|
|
317
|
+
return (function (items) {
|
|
318
|
+
return items.map(function (item) {
|
|
319
|
+
var dict = Js_dict.fromArray(Js_dict.entries(item));
|
|
320
|
+
jsonFieldNames.forEach(function (name) {
|
|
321
|
+
var v = Js_dict.get(dict, name);
|
|
322
|
+
if (v !== undefined) {
|
|
323
|
+
dict[name] = JSON.stringify(Caml_option.valFromOption(v));
|
|
324
|
+
return ;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
});
|
|
328
|
+
return dict;
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
|
|
313
333
|
function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
|
|
314
334
|
var match = Table.toSqlParams(table, itemSchema, pgSchema);
|
|
335
|
+
var dbSchema = match.dbSchema;
|
|
336
|
+
var jsonStringifier = makeJsonStringifier(table, match.jsonFieldIndices);
|
|
315
337
|
var isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName;
|
|
316
338
|
var isHistoryUpdate = table.tableName.startsWith(EntityHistory.historyTablePrefix);
|
|
317
|
-
if ((isRawEvents || !match.hasArrayField) && !isHistoryUpdate) {
|
|
318
|
-
return {
|
|
319
|
-
query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
|
|
320
|
-
convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
|
|
321
|
-
isInsertValues: false
|
|
322
|
-
};
|
|
323
|
-
} else {
|
|
339
|
+
if (!((isRawEvents || !match.hasArrayField) && !isHistoryUpdate)) {
|
|
324
340
|
return {
|
|
325
341
|
query: makeInsertValuesSetQuery(pgSchema, table, itemSchema, 500),
|
|
326
|
-
convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), (function (param) {
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
342
|
+
convertOrThrow: isHistoryUpdate ? S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), (function (param) {
|
|
343
|
+
return {
|
|
344
|
+
s: (function (prim) {
|
|
345
|
+
return prim.flat(1);
|
|
346
|
+
})
|
|
347
|
+
};
|
|
348
|
+
})), "Output", "Input", "Sync", false) : S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(dbSchema), (function (param) {
|
|
349
|
+
return {
|
|
350
|
+
s: (function (prim) {
|
|
351
|
+
return prim.flat(1);
|
|
352
|
+
})
|
|
353
|
+
};
|
|
354
|
+
})), "Output", "Input", "Sync", false),
|
|
355
|
+
isInsertValues: true,
|
|
356
|
+
jsonStringifier: jsonStringifier
|
|
334
357
|
};
|
|
335
358
|
}
|
|
359
|
+
var baseConvert = S$RescriptSchema.compile(S$RescriptSchema.unnest(dbSchema), "Output", "Input", "Sync", false);
|
|
360
|
+
return {
|
|
361
|
+
query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
|
|
362
|
+
convertOrThrow: baseConvert,
|
|
363
|
+
isInsertValues: false,
|
|
364
|
+
jsonStringifier: jsonStringifier
|
|
365
|
+
};
|
|
336
366
|
}
|
|
337
367
|
|
|
338
368
|
function chunkArray(arr, chunkSize) {
|
|
@@ -373,26 +403,42 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
|
|
|
373
403
|
var cached = setQueryCache.get(table);
|
|
374
404
|
var data;
|
|
375
405
|
if (cached !== undefined) {
|
|
376
|
-
data =
|
|
406
|
+
data = cached;
|
|
377
407
|
} else {
|
|
378
408
|
var newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
|
|
379
409
|
setQueryCache.set(table, newQuery);
|
|
380
410
|
data = newQuery;
|
|
381
411
|
}
|
|
412
|
+
var prepareItems = function (rawItems) {
|
|
413
|
+
var stringify = data.jsonStringifier;
|
|
414
|
+
if (stringify !== undefined) {
|
|
415
|
+
return stringify(rawItems);
|
|
416
|
+
} else {
|
|
417
|
+
return rawItems;
|
|
418
|
+
}
|
|
419
|
+
};
|
|
382
420
|
try {
|
|
383
|
-
if (
|
|
384
|
-
|
|
421
|
+
if (data.isInsertValues) {
|
|
422
|
+
var chunks = chunkArray(items, 500);
|
|
423
|
+
var responses = [];
|
|
424
|
+
chunks.forEach(function (chunk) {
|
|
425
|
+
var chunkSize = chunk.length;
|
|
426
|
+
var isFullChunk = chunkSize === 500;
|
|
427
|
+
var response = sql.query({
|
|
428
|
+
text: isFullChunk ? data.query : makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize),
|
|
429
|
+
values: data.convertOrThrow(prepareItems(chunk)),
|
|
430
|
+
name: isFullChunk ? "insert_" + table.stmtId : undefined
|
|
431
|
+
});
|
|
432
|
+
responses.push(response);
|
|
433
|
+
});
|
|
434
|
+
await Promise.all(responses);
|
|
435
|
+
return ;
|
|
385
436
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
var isFullChunk = chunkSize === 500;
|
|
391
|
-
var params = data.convertOrThrow(chunk);
|
|
392
|
-
var response = isFullChunk ? sql.unsafe(data.query, params, {prepare: true}) : sql.unsafe(makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), params);
|
|
393
|
-
responses.push(response);
|
|
437
|
+
await sql.query({
|
|
438
|
+
text: data.query,
|
|
439
|
+
values: data.convertOrThrow(prepareItems(items)),
|
|
440
|
+
name: "upsert_" + table.stmtId
|
|
394
441
|
});
|
|
395
|
-
await Promise.all(responses);
|
|
396
442
|
return ;
|
|
397
443
|
}
|
|
398
444
|
catch (raw_exn){
|
|
@@ -477,7 +523,15 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containe
|
|
|
477
523
|
async function deleteByIdsOrThrow(sql, pgSchema, ids, table) {
|
|
478
524
|
try {
|
|
479
525
|
await (
|
|
480
|
-
ids.length !== 1 ? sql.
|
|
526
|
+
ids.length !== 1 ? sql.query({
|
|
527
|
+
text: makeDeleteByIdsQuery(pgSchema, table.tableName),
|
|
528
|
+
values: [ids],
|
|
529
|
+
name: "delete_ids_" + table.stmtId
|
|
530
|
+
}) : sql.query({
|
|
531
|
+
text: makeDeleteByIdQuery(pgSchema, table.tableName),
|
|
532
|
+
values: ids,
|
|
533
|
+
name: "delete_id_" + table.stmtId
|
|
534
|
+
})
|
|
481
535
|
);
|
|
482
536
|
return ;
|
|
483
537
|
}
|
|
@@ -529,7 +583,7 @@ function executeSet(sql, items, dbFunction) {
|
|
|
529
583
|
}
|
|
530
584
|
}
|
|
531
585
|
|
|
532
|
-
async function writeBatch(
|
|
586
|
+
async function writeBatch(pool, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, escapeTables) {
|
|
533
587
|
try {
|
|
534
588
|
var shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
|
|
535
589
|
var specificError = {
|
|
@@ -580,10 +634,14 @@ async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpo
|
|
|
580
634
|
await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.name, entityConfig.index, Array.from(backfillHistoryIds));
|
|
581
635
|
}
|
|
582
636
|
if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
|
|
583
|
-
promises.push(sql.
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
637
|
+
promises.push(sql.query({
|
|
638
|
+
text: makeInsertDeleteUpdatesQuery(entityConfig, pgSchema),
|
|
639
|
+
values: [
|
|
640
|
+
batchDeleteEntityIds,
|
|
641
|
+
$$BigInt.arrayToStringArray(batchDeleteCheckpointIds)
|
|
642
|
+
],
|
|
643
|
+
name: "insert_deletes_" + String(entityConfig.index)
|
|
644
|
+
}));
|
|
587
645
|
}
|
|
588
646
|
if (Utils.$$Array.notEmpty(batchSetUpdates)) {
|
|
589
647
|
if (shouldRemoveInvalidUtf8) {
|
|
@@ -655,40 +713,40 @@ async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpo
|
|
|
655
713
|
}) : undefined;
|
|
656
714
|
try {
|
|
657
715
|
await Promise.all([
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
716
|
+
Pg.beginSql(pool, (async function (sql) {
|
|
717
|
+
if (rollbackTables !== undefined) {
|
|
718
|
+
await rollbackTables(sql);
|
|
719
|
+
}
|
|
720
|
+
var setOperations = Belt_Array.concat([
|
|
721
|
+
(function (sql) {
|
|
722
|
+
return InternalTable.Chains.setProgressedChains(sql, pgSchema, Utils.Dict.mapValuesToArray(batch.progressedChainsById, (function (chainAfterBatch) {
|
|
723
|
+
return {
|
|
724
|
+
chainId: chainAfterBatch.fetchState.chainId,
|
|
725
|
+
progressBlockNumber: chainAfterBatch.progressBlockNumber,
|
|
726
|
+
sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
|
|
727
|
+
totalEventsProcessed: chainAfterBatch.totalEventsProcessed
|
|
728
|
+
};
|
|
729
|
+
})));
|
|
730
|
+
}),
|
|
731
|
+
setRawEvents
|
|
732
|
+
], setEntities);
|
|
733
|
+
if (shouldSaveHistory) {
|
|
734
|
+
setOperations.push(function (sql) {
|
|
735
|
+
return InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed);
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
await Promise.all(Belt_Array.map(setOperations, (function (dbFunc) {
|
|
739
|
+
return dbFunc(sql);
|
|
740
|
+
})));
|
|
741
|
+
if (sinkPromise === undefined) {
|
|
742
|
+
return ;
|
|
743
|
+
}
|
|
744
|
+
var exn = await Caml_option.valFromOption(sinkPromise);
|
|
745
|
+
if (exn === undefined) {
|
|
746
|
+
return ;
|
|
747
|
+
}
|
|
748
|
+
throw exn;
|
|
749
|
+
})),
|
|
692
750
|
Promise.all(Belt_Array.map(updatedEffectsCache, (function (param) {
|
|
693
751
|
return setEffectCacheOrThrow(param.effect, param.items, param.shouldInitialize);
|
|
694
752
|
})))
|
|
@@ -710,7 +768,7 @@ async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpo
|
|
|
710
768
|
if (exn$1.RE_EXN_ID === PgEncodingError) {
|
|
711
769
|
var escapeTables$1 = escapeTables !== undefined ? Caml_option.valFromOption(escapeTables) : new Set();
|
|
712
770
|
escapeTables$1.add(exn$1.table);
|
|
713
|
-
return await writeBatch(
|
|
771
|
+
return await writeBatch(pool, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, Caml_option.some(escapeTables$1));
|
|
714
772
|
}
|
|
715
773
|
throw exn$1;
|
|
716
774
|
}
|
|
@@ -735,7 +793,7 @@ function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
|
|
|
735
793
|
return "SELECT DISTINCT " + Table.idFieldName + "\nFROM \"" + pgSchema + "\".\"" + historyTableName + "\"\nWHERE \"" + EntityHistory.checkpointIdFieldName + "\" > $1\nAND NOT EXISTS (\n SELECT 1\n FROM \"" + pgSchema + "\".\"" + historyTableName + "\" h\n WHERE h." + Table.idFieldName + " = \"" + historyTableName + "\"." + Table.idFieldName + "\n AND h.\"" + EntityHistory.checkpointIdFieldName + "\" <= $1\n)";
|
|
736
794
|
}
|
|
737
795
|
|
|
738
|
-
function make(
|
|
796
|
+
function make(pool, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
|
|
739
797
|
var containerName = "envio-postgres";
|
|
740
798
|
var psqlExecOptions_env = Js_dict.fromArray([
|
|
741
799
|
[
|
|
@@ -752,8 +810,10 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
752
810
|
};
|
|
753
811
|
var cacheDirPath = Path.resolve(".envio", "cache");
|
|
754
812
|
var isInitialized = async function () {
|
|
755
|
-
var
|
|
756
|
-
|
|
813
|
+
var match = await pool.query({
|
|
814
|
+
text: "SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND (table_name = 'event_sync_state' OR table_name = '" + InternalTable.Chains.table.tableName + "');"
|
|
815
|
+
});
|
|
816
|
+
return match.rows.length > 0;
|
|
757
817
|
};
|
|
758
818
|
var restoreEffectCache = async function (withUpload) {
|
|
759
819
|
if (withUpload) {
|
|
@@ -785,7 +845,9 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
785
845
|
await Promise.all(cacheFiles.map(function (entry) {
|
|
786
846
|
var effectName = entry.slice(0, -4);
|
|
787
847
|
var table = Internal.makeCacheTable(effectName);
|
|
788
|
-
return
|
|
848
|
+
return pool.query({
|
|
849
|
+
text: makeCreateTableQuery(table, pgSchema, false)
|
|
850
|
+
}).then(function (param) {
|
|
789
851
|
var inputFile = Path.join(cacheDirPath, entry);
|
|
790
852
|
var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + table.tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
|
|
791
853
|
return new Promise((function (resolve, reject) {
|
|
@@ -817,7 +879,9 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
817
879
|
}
|
|
818
880
|
|
|
819
881
|
}
|
|
820
|
-
var cacheTableInfo = await
|
|
882
|
+
var cacheTableInfo = (await pool.query({
|
|
883
|
+
text: makeSchemaCacheTableInfoQuery(pgSchema)
|
|
884
|
+
})).rows;
|
|
821
885
|
if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
|
|
822
886
|
await onNewTables(cacheTableInfo.map(function (info) {
|
|
823
887
|
return info.table_name;
|
|
@@ -837,7 +901,9 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
837
901
|
var chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
|
|
838
902
|
var entities = entitiesOpt !== undefined ? entitiesOpt : [];
|
|
839
903
|
var enums = enumsOpt !== undefined ? enumsOpt : [];
|
|
840
|
-
var schemaTableNames = await
|
|
904
|
+
var schemaTableNames = (await pool.query({
|
|
905
|
+
text: makeSchemaTableNamesQuery(pgSchema)
|
|
906
|
+
})).rows;
|
|
841
907
|
if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(function (table) {
|
|
842
908
|
return table.table_name === InternalTable.Chains.table.tableName ? true : table.table_name === "event_sync_state";
|
|
843
909
|
})) {
|
|
@@ -847,11 +913,13 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
847
913
|
await sink.initialize(chainConfigs, entities, enums);
|
|
848
914
|
}
|
|
849
915
|
var queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, entities, enums, Utils.$$Array.isEmpty(schemaTableNames));
|
|
850
|
-
await
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
916
|
+
await Pg.beginSql(pool, (function (sql) {
|
|
917
|
+
return Promise.all(queries.map(function (query) {
|
|
918
|
+
return sql.query({
|
|
919
|
+
text: query
|
|
920
|
+
});
|
|
921
|
+
}));
|
|
922
|
+
}));
|
|
855
923
|
var cache = await restoreEffectCache(true);
|
|
856
924
|
if (onInitialize !== undefined) {
|
|
857
925
|
await onInitialize();
|
|
@@ -878,10 +946,18 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
878
946
|
};
|
|
879
947
|
};
|
|
880
948
|
var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
|
|
881
|
-
var
|
|
949
|
+
var val;
|
|
882
950
|
try {
|
|
883
|
-
|
|
884
|
-
ids.length !== 1 ?
|
|
951
|
+
val = await (
|
|
952
|
+
ids.length !== 1 ? pool.query({
|
|
953
|
+
text: makeLoadByIdsQuery(pgSchema, table.tableName),
|
|
954
|
+
values: [ids],
|
|
955
|
+
name: "load_ids_" + table.stmtId
|
|
956
|
+
}) : pool.query({
|
|
957
|
+
text: makeLoadByIdQuery(pgSchema, table.tableName),
|
|
958
|
+
values: ids,
|
|
959
|
+
name: "load_id_" + table.stmtId
|
|
960
|
+
})
|
|
885
961
|
);
|
|
886
962
|
}
|
|
887
963
|
catch (raw_exn){
|
|
@@ -894,7 +970,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
894
970
|
};
|
|
895
971
|
}
|
|
896
972
|
try {
|
|
897
|
-
return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
|
|
973
|
+
return S$RescriptSchema.parseOrThrow(val.rows, rowsSchema);
|
|
898
974
|
}
|
|
899
975
|
catch (raw_exn$1){
|
|
900
976
|
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
|
|
@@ -920,9 +996,13 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
920
996
|
Error: new Error()
|
|
921
997
|
};
|
|
922
998
|
}
|
|
923
|
-
var
|
|
999
|
+
var val;
|
|
924
1000
|
try {
|
|
925
|
-
|
|
1001
|
+
val = await pool.query({
|
|
1002
|
+
text: makeLoadByFieldQuery(pgSchema, table.tableName, fieldName, operator),
|
|
1003
|
+
values: params,
|
|
1004
|
+
name: "load_field_" + table.tableName + "_" + fieldName + "_" + operator
|
|
1005
|
+
});
|
|
926
1006
|
}
|
|
927
1007
|
catch (raw_exn$1){
|
|
928
1008
|
var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
|
|
@@ -934,7 +1014,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
934
1014
|
};
|
|
935
1015
|
}
|
|
936
1016
|
try {
|
|
937
|
-
return S$RescriptSchema.parseOrThrow(rows, rowsSchema);
|
|
1017
|
+
return S$RescriptSchema.parseOrThrow(val.rows, rowsSchema);
|
|
938
1018
|
}
|
|
939
1019
|
catch (raw_exn$2){
|
|
940
1020
|
var exn$2 = Caml_js_exceptions.internalToOCamlException(raw_exn$2);
|
|
@@ -947,13 +1027,15 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
947
1027
|
}
|
|
948
1028
|
};
|
|
949
1029
|
var setOrThrow$1 = function (items, table, itemSchema) {
|
|
950
|
-
return setOrThrow(
|
|
1030
|
+
return setOrThrow(pool, items, table, itemSchema, pgSchema);
|
|
951
1031
|
};
|
|
952
1032
|
var setEffectCacheOrThrow = async function (effect, items, initialize) {
|
|
953
1033
|
var match = effect.storageMeta;
|
|
954
1034
|
var table = match.table;
|
|
955
1035
|
if (initialize) {
|
|
956
|
-
await
|
|
1036
|
+
await pool.query({
|
|
1037
|
+
text: makeCreateTableQuery(table, pgSchema, false)
|
|
1038
|
+
});
|
|
957
1039
|
if (onNewTables !== undefined) {
|
|
958
1040
|
await onNewTables([table.tableName]);
|
|
959
1041
|
}
|
|
@@ -963,7 +1045,9 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
963
1045
|
};
|
|
964
1046
|
var dumpEffectCache = async function () {
|
|
965
1047
|
try {
|
|
966
|
-
var cacheTableInfo = (await
|
|
1048
|
+
var cacheTableInfo = (await pool.query({
|
|
1049
|
+
text: makeSchemaCacheTableInfoQuery(pgSchema)
|
|
1050
|
+
})).rows.filter(function (i) {
|
|
967
1051
|
return i.count > 0;
|
|
968
1052
|
});
|
|
969
1053
|
if (!Utils.$$Array.notEmpty(cacheTableInfo)) {
|
|
@@ -1011,7 +1095,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1011
1095
|
var resumeInitialState = async function () {
|
|
1012
1096
|
var match = await Promise.all([
|
|
1013
1097
|
restoreEffectCache(false),
|
|
1014
|
-
InternalTable.Chains.getInitialState(
|
|
1098
|
+
InternalTable.Chains.getInitialState(pool, pgSchema).then(function (rawInitialStates) {
|
|
1015
1099
|
return Belt_Array.map(rawInitialStates, (function (rawInitialState) {
|
|
1016
1100
|
return {
|
|
1017
1101
|
id: rawInitialState.id,
|
|
@@ -1027,8 +1111,16 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1027
1111
|
};
|
|
1028
1112
|
}));
|
|
1029
1113
|
}),
|
|
1030
|
-
|
|
1031
|
-
|
|
1114
|
+
pool.query({
|
|
1115
|
+
text: InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)
|
|
1116
|
+
}).then(function (r) {
|
|
1117
|
+
return r.rows;
|
|
1118
|
+
}),
|
|
1119
|
+
pool.query({
|
|
1120
|
+
text: InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema)
|
|
1121
|
+
}).then(function (r) {
|
|
1122
|
+
return r.rows;
|
|
1123
|
+
})
|
|
1032
1124
|
]);
|
|
1033
1125
|
var checkpointId = BigInt(match[2][0].id);
|
|
1034
1126
|
var reorgCheckpoints = Belt_Array.map(match[3], (function (raw) {
|
|
@@ -1052,29 +1144,43 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1052
1144
|
};
|
|
1053
1145
|
var reset = async function () {
|
|
1054
1146
|
var query = "DROP SCHEMA IF EXISTS \"" + pgSchema + "\" CASCADE;";
|
|
1055
|
-
return await
|
|
1147
|
+
return await pool.query({
|
|
1148
|
+
text: query
|
|
1149
|
+
});
|
|
1056
1150
|
};
|
|
1057
1151
|
var setChainMeta = function (chainsData) {
|
|
1058
|
-
return InternalTable.Chains.setMeta(
|
|
1152
|
+
return InternalTable.Chains.setMeta(pool, pgSchema, chainsData).then(function (param) {
|
|
1059
1153
|
return undefined;
|
|
1060
1154
|
});
|
|
1061
1155
|
};
|
|
1062
1156
|
var pruneStaleCheckpoints = function (safeCheckpointId) {
|
|
1063
|
-
return InternalTable.Checkpoints.pruneStaleCheckpoints(
|
|
1157
|
+
return InternalTable.Checkpoints.pruneStaleCheckpoints(pool, pgSchema, safeCheckpointId);
|
|
1064
1158
|
};
|
|
1065
1159
|
var pruneStaleEntityHistory = function (entityName, entityIndex, safeCheckpointId) {
|
|
1066
|
-
return EntityHistory.pruneStaleEntityHistory(
|
|
1160
|
+
return EntityHistory.pruneStaleEntityHistory(pool, entityName, entityIndex, pgSchema, safeCheckpointId);
|
|
1067
1161
|
};
|
|
1068
1162
|
var getRollbackTargetCheckpoint = function (reorgChainId, lastKnownValidBlockNumber) {
|
|
1069
|
-
return InternalTable.Checkpoints.getRollbackTargetCheckpoint(
|
|
1163
|
+
return InternalTable.Checkpoints.getRollbackTargetCheckpoint(pool, pgSchema, reorgChainId, lastKnownValidBlockNumber);
|
|
1070
1164
|
};
|
|
1071
1165
|
var getRollbackProgressDiff = function (rollbackTargetCheckpointId) {
|
|
1072
|
-
return InternalTable.Checkpoints.getRollbackProgressDiff(
|
|
1166
|
+
return InternalTable.Checkpoints.getRollbackProgressDiff(pool, pgSchema, rollbackTargetCheckpointId);
|
|
1073
1167
|
};
|
|
1074
1168
|
var getRollbackData = async function (entityConfig, rollbackTargetCheckpointId) {
|
|
1075
1169
|
return await Promise.all([
|
|
1076
|
-
|
|
1077
|
-
|
|
1170
|
+
pool.query({
|
|
1171
|
+
text: makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema),
|
|
1172
|
+
values: [rollbackTargetCheckpointId.toString()],
|
|
1173
|
+
name: "rollback_removed_" + String(entityConfig.index)
|
|
1174
|
+
}).then(function (r) {
|
|
1175
|
+
return r.rows;
|
|
1176
|
+
}),
|
|
1177
|
+
pool.query({
|
|
1178
|
+
text: makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema),
|
|
1179
|
+
values: [rollbackTargetCheckpointId.toString()],
|
|
1180
|
+
name: "rollback_restored_" + String(entityConfig.index)
|
|
1181
|
+
}).then(function (r) {
|
|
1182
|
+
return r.rows;
|
|
1183
|
+
})
|
|
1078
1184
|
]);
|
|
1079
1185
|
};
|
|
1080
1186
|
var writeBatchMethod = async function (batch, rawEvents, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities) {
|
|
@@ -1089,7 +1195,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
1089
1195
|
} else {
|
|
1090
1196
|
sinkPromise = undefined;
|
|
1091
1197
|
}
|
|
1092
|
-
return await writeBatch(
|
|
1198
|
+
return await writeBatch(pool, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, undefined);
|
|
1093
1199
|
};
|
|
1094
1200
|
return {
|
|
1095
1201
|
isInitialized: isInitialized,
|
|
@@ -1132,6 +1238,7 @@ export {
|
|
|
1132
1238
|
makeInsertUnnestSetQuery ,
|
|
1133
1239
|
makeInsertValuesSetQuery ,
|
|
1134
1240
|
maxItemsPerQuery ,
|
|
1241
|
+
makeJsonStringifier ,
|
|
1135
1242
|
makeTableBatchSetQuery ,
|
|
1136
1243
|
chunkArray ,
|
|
1137
1244
|
removeInvalidUtf8InPlace ,
|
|
@@ -158,7 +158,7 @@ let setCheckpointsOrThrow = async (client, ~batch: Batch.t, ~database: string) =
|
|
|
158
158
|
for idx in 0 to checkpointsCount - 1 {
|
|
159
159
|
checkpointRows
|
|
160
160
|
->Js.Array2.push((
|
|
161
|
-
batch.checkpointIds->Belt.Array.getUnsafe(idx),
|
|
161
|
+
batch.checkpointIds->Belt.Array.getUnsafe(idx)->BigInt.toString,
|
|
162
162
|
batch.checkpointChainIds->Belt.Array.getUnsafe(idx),
|
|
163
163
|
batch.checkpointBlockNumbers->Belt.Array.getUnsafe(idx),
|
|
164
164
|
batch.checkpointBlockHashes->Belt.Array.getUnsafe(idx),
|
|
@@ -134,7 +134,7 @@ async function setCheckpointsOrThrow(client, batch, database) {
|
|
|
134
134
|
var checkpointRows = [];
|
|
135
135
|
for(var idx = 0; idx < checkpointsCount; ++idx){
|
|
136
136
|
checkpointRows.push([
|
|
137
|
-
batch.checkpointIds[idx],
|
|
137
|
+
batch.checkpointIds[idx].toString(),
|
|
138
138
|
batch.checkpointChainIds[idx],
|
|
139
139
|
batch.checkpointBlockNumbers[idx],
|
|
140
140
|
batch.checkpointBlockHashes[idx],
|