envio 3.5.1 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +7 -0
- package/fuel.schema.json +7 -0
- package/index.d.ts +33 -10
- package/package.json +6 -6
- package/src/ChainState.res +12 -0
- package/src/ChainState.res.mjs +15 -2
- package/src/ChainState.resi +3 -0
- package/src/Config.res +60 -2
- package/src/Config.res.mjs +49 -13
- package/src/EffectState.res +2 -2
- package/src/EffectState.res.mjs +2 -2
- package/src/EntityTables.res +32 -0
- package/src/EntityTables.res.mjs +44 -0
- package/src/Envio.res +7 -7
- package/src/Envio.res.mjs +5 -6
- package/src/Hasura.res +37 -3
- package/src/Hasura.res.mjs +23 -5
- package/src/InMemoryStore.res +48 -8
- package/src/InMemoryStore.res.mjs +37 -7
- package/src/IndexerState.res +30 -24
- package/src/IndexerState.res.mjs +20 -35
- package/src/IndexerState.resi +8 -6
- package/src/Internal.res +29 -11
- package/src/Internal.res.mjs +27 -10
- package/src/LoadLayer.res +39 -8
- package/src/LoadLayer.res.mjs +36 -9
- package/src/LoadLayer.resi +2 -0
- package/src/Persistence.res +12 -1
- package/src/PgStorage.res +183 -37
- package/src/PgStorage.res.mjs +149 -34
- package/src/PruneStaleHistory.res +1 -0
- package/src/PruneStaleHistory.res.mjs +2 -1
- package/src/Sink.res +3 -3
- package/src/Sink.res.mjs +2 -2
- package/src/TestIndexer.res +128 -17
- package/src/TestIndexer.res.mjs +75 -9
- package/src/UserContext.res +19 -4
- package/src/UserContext.res.mjs +15 -9
- package/src/Writing.res +8 -16
- package/src/Writing.res.mjs +10 -10
- package/src/bindings/ClickHouse.res +40 -4
- package/src/bindings/ClickHouse.res.mjs +37 -5
- package/src/db/EntityHistory.res +64 -21
- package/src/db/EntityHistory.res.mjs +43 -22
- package/src/db/InternalTable.res.mjs +31 -31
- package/src/db/Table.res +24 -1
- package/src/db/Table.res.mjs +26 -4
- package/svm.schema.json +7 -0
package/src/PgStorage.res.mjs
CHANGED
|
@@ -103,6 +103,35 @@ function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdMode
|
|
|
103
103
|
) + `);`;
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
let rowSchemaCache = new WeakMap();
|
|
107
|
+
|
|
108
|
+
function getRowSchema(entityConfig) {
|
|
109
|
+
let cached = rowSchemaCache.get(entityConfig);
|
|
110
|
+
if (cached !== undefined) {
|
|
111
|
+
return Primitive_option.valFromOption(cached);
|
|
112
|
+
}
|
|
113
|
+
let chainIdField = Table.getChainIdField(entityConfig.table);
|
|
114
|
+
let schema = chainIdField !== undefined ? S$RescriptSchema.schema(s => {
|
|
115
|
+
let dict = {};
|
|
116
|
+
let match = entityConfig.schema.t;
|
|
117
|
+
let exit = 0;
|
|
118
|
+
if (typeof match !== "object" || match.TAG !== "object") {
|
|
119
|
+
exit = 1;
|
|
120
|
+
} else {
|
|
121
|
+
match.items.forEach(param => {
|
|
122
|
+
dict[param.location] = s.m(param.schema);
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
if (exit === 1) {
|
|
126
|
+
Stdlib_JsError.throwWithMessage(`Unexpected non-object schema for entity "` + entityConfig.name + `".`);
|
|
127
|
+
}
|
|
128
|
+
dict[chainIdField.fieldName] = s.m(ChainId.schema);
|
|
129
|
+
return dict;
|
|
130
|
+
}) : entityConfig.schema;
|
|
131
|
+
rowSchemaCache.set(entityConfig, schema);
|
|
132
|
+
return schema;
|
|
133
|
+
}
|
|
134
|
+
|
|
106
135
|
let entityHistoryCache = new WeakMap();
|
|
107
136
|
|
|
108
137
|
function getEntityHistory(entityConfig) {
|
|
@@ -127,6 +156,7 @@ function getEntityHistory(entityConfig) {
|
|
|
127
156
|
isNullable: field$1.isNullable,
|
|
128
157
|
isPrimaryKey: true,
|
|
129
158
|
isIndex: field$1.isIndex,
|
|
159
|
+
isChainId: field$1.isChainId,
|
|
130
160
|
linkedEntity: field$1.linkedEntity,
|
|
131
161
|
defaultValue: field$1.defaultValue,
|
|
132
162
|
description: field$1.description,
|
|
@@ -134,6 +164,11 @@ function getEntityHistory(entityConfig) {
|
|
|
134
164
|
clickhouseDbName: field$1.clickhouseDbName
|
|
135
165
|
}
|
|
136
166
|
};
|
|
167
|
+
} else if (field$1.isChainId) {
|
|
168
|
+
return {
|
|
169
|
+
TAG: "Field",
|
|
170
|
+
_0: field$1
|
|
171
|
+
};
|
|
137
172
|
} else {
|
|
138
173
|
return {
|
|
139
174
|
TAG: "Field",
|
|
@@ -145,6 +180,7 @@ function getEntityHistory(entityConfig) {
|
|
|
145
180
|
isNullable: true,
|
|
146
181
|
isPrimaryKey: field$1.isPrimaryKey,
|
|
147
182
|
isIndex: false,
|
|
183
|
+
isChainId: field$1.isChainId,
|
|
148
184
|
linkedEntity: field$1.linkedEntity,
|
|
149
185
|
defaultValue: field$1.defaultValue,
|
|
150
186
|
description: field$1.description,
|
|
@@ -154,15 +190,15 @@ function getEntityHistory(entityConfig) {
|
|
|
154
190
|
};
|
|
155
191
|
}
|
|
156
192
|
});
|
|
157
|
-
let actionField = Table.mkField(EntityHistory.changeFieldName, EntityHistory.changeFieldType, S$RescriptSchema.never, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined);
|
|
158
|
-
let checkpointIdField = Table.mkField(EntityHistory.checkpointIdFieldName, EntityHistory.checkpointIdFieldType, EntityHistory.unsafeCheckpointIdSchema, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined);
|
|
193
|
+
let actionField = Table.mkField(EntityHistory.changeFieldName, EntityHistory.changeFieldType, S$RescriptSchema.never, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined);
|
|
194
|
+
let checkpointIdField = Table.mkField(EntityHistory.checkpointIdFieldName, EntityHistory.checkpointIdFieldType, EntityHistory.unsafeCheckpointIdSchema, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined);
|
|
159
195
|
let entityTableName = entityConfig.table.tableName;
|
|
160
196
|
let historyTableName = EntityHistory.historyTableName(entityTableName, entityConfig.index);
|
|
161
197
|
let table = Table.mkTable(historyTableName, undefined, dataFields.concat([
|
|
162
198
|
checkpointIdField,
|
|
163
199
|
actionField
|
|
164
200
|
]), undefined);
|
|
165
|
-
let setChangeSchema = EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), entityConfig
|
|
201
|
+
let setChangeSchema = EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), getRowSchema(entityConfig));
|
|
166
202
|
let cache_setChangeSchemaRows = S$RescriptSchema.array(setChangeSchema);
|
|
167
203
|
let cache$1 = {
|
|
168
204
|
table: table,
|
|
@@ -285,12 +321,21 @@ function makeFilterCondition(filter, table, params) {
|
|
|
285
321
|
}
|
|
286
322
|
}
|
|
287
323
|
|
|
288
|
-
function
|
|
289
|
-
|
|
324
|
+
function makeChainIdCondition(table, chainId) {
|
|
325
|
+
let match = Table.getChainIdField(table);
|
|
326
|
+
if (match !== undefined && chainId !== undefined) {
|
|
327
|
+
return ` AND "` + Table.getPgDbFieldName(match) + `" = $2`;
|
|
328
|
+
} else {
|
|
329
|
+
return "";
|
|
330
|
+
}
|
|
290
331
|
}
|
|
291
332
|
|
|
292
|
-
function
|
|
293
|
-
return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id =
|
|
333
|
+
function makeDeleteByIdQuery(pgSchema, tableName, chainIdCondition) {
|
|
334
|
+
return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = $1` + chainIdCondition + `;`;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function makeDeleteByIdsQuery(pgSchema, tableName, idPgType, chainIdCondition) {
|
|
338
|
+
return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = ANY($1::` + idPgType + `[])` + chainIdCondition + `;`;
|
|
294
339
|
}
|
|
295
340
|
|
|
296
341
|
function makeLoadAllQuery(pgSchema, tableName) {
|
|
@@ -547,11 +592,16 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containe
|
|
|
547
592
|
return result;
|
|
548
593
|
}
|
|
549
594
|
|
|
550
|
-
async function deleteByIdsOrThrow(sql, pgSchema, ids, table) {
|
|
595
|
+
async function deleteByIdsOrThrow(sql, pgSchema, ids, table, chainIdOpt) {
|
|
596
|
+
let chainId = chainIdOpt !== undefined ? Primitive_option.valFromOption(chainIdOpt) : undefined;
|
|
597
|
+
let chainIdCondition = makeChainIdCondition(table, chainId);
|
|
598
|
+
let chainIdParams = chainId !== undefined ? (
|
|
599
|
+
chainIdCondition !== "" ? [Primitive_option.valFromOption(chainId)] : []
|
|
600
|
+
) : [];
|
|
551
601
|
let idsJson = Table.encodeIdsToJson(table, ids);
|
|
552
602
|
try {
|
|
553
603
|
await (
|
|
554
|
-
ids.length !== 1 ? sql.unsafe(makeDeleteByIdsQuery(pgSchema, table.tableName, Table.getIdPgFieldType(table, pgSchema)), [idsJson], {prepare: true}) : sql.unsafe(makeDeleteByIdQuery(pgSchema, table.tableName), idsJson, {prepare: true})
|
|
604
|
+
ids.length !== 1 ? sql.unsafe(makeDeleteByIdsQuery(pgSchema, table.tableName, Table.getIdPgFieldType(table, pgSchema), chainIdCondition), [idsJson].concat(chainIdParams), {prepare: true}) : sql.unsafe(makeDeleteByIdQuery(pgSchema, table.tableName, chainIdCondition), idsJson.concat(chainIdParams), {prepare: true})
|
|
555
605
|
);
|
|
556
606
|
return;
|
|
557
607
|
} catch (raw_exn) {
|
|
@@ -565,7 +615,7 @@ async function deleteByIdsOrThrow(sql, pgSchema, ids, table) {
|
|
|
565
615
|
}
|
|
566
616
|
}
|
|
567
617
|
|
|
568
|
-
function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
|
|
618
|
+
function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema, chainId) {
|
|
569
619
|
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
570
620
|
let allHistoryFieldNames = Stdlib_Array.filterMap(entityConfig.table.fields, fieldOrDerived => {
|
|
571
621
|
if (fieldOrDerived.TAG === "Field") {
|
|
@@ -575,6 +625,8 @@ function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
|
|
|
575
625
|
allHistoryFieldNames.push(EntityHistory.checkpointIdFieldName);
|
|
576
626
|
allHistoryFieldNames.push(EntityHistory.changeFieldName);
|
|
577
627
|
let allHistoryFieldNamesStr = allHistoryFieldNames.map(name => `"` + name + `"`).join(", ");
|
|
628
|
+
let match = Table.getChainIdField(entityConfig.table);
|
|
629
|
+
let chainIdColumn = match !== undefined && chainId !== undefined ? Table.getPgDbFieldName(match) : "";
|
|
578
630
|
let selectParts = allHistoryFieldNames.map(fieldName => {
|
|
579
631
|
if (fieldName === Table.idFieldName) {
|
|
580
632
|
return `u.` + Table.idFieldName;
|
|
@@ -582,6 +634,8 @@ function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
|
|
|
582
634
|
return `u.` + EntityHistory.checkpointIdFieldName;
|
|
583
635
|
} else if (fieldName === EntityHistory.changeFieldName) {
|
|
584
636
|
return `'` + "DELETE" + `'`;
|
|
637
|
+
} else if (chainIdColumn !== "" && fieldName === chainIdColumn) {
|
|
638
|
+
return "$3";
|
|
585
639
|
} else {
|
|
586
640
|
return "NULL";
|
|
587
641
|
}
|
|
@@ -640,9 +694,36 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
640
694
|
}
|
|
641
695
|
};
|
|
642
696
|
let setEntities = updatedEntities.map(param => {
|
|
697
|
+
let changes = param.changes;
|
|
698
|
+
let scope = param.scope;
|
|
643
699
|
let entityConfig = param.entityConfig;
|
|
644
700
|
let entitiesToSet = [];
|
|
645
701
|
let idsToDelete = [];
|
|
702
|
+
let scopeChainId;
|
|
703
|
+
scopeChainId = scope === "crossChain" ? undefined : Primitive_option.some(scope);
|
|
704
|
+
let match = Table.getChainIdField(entityConfig.table);
|
|
705
|
+
let changes$1;
|
|
706
|
+
if (match !== undefined && scopeChainId !== undefined) {
|
|
707
|
+
let chainId = Primitive_option.valFromOption(scopeChainId);
|
|
708
|
+
changes$1 = changes.map(change => {
|
|
709
|
+
if (change.type === "SET") {
|
|
710
|
+
return {
|
|
711
|
+
type: "SET",
|
|
712
|
+
entityId: change.entityId,
|
|
713
|
+
entity: Internal.stampChainId(change.entity, match.fieldName, chainId),
|
|
714
|
+
checkpointId: change.checkpointId
|
|
715
|
+
};
|
|
716
|
+
} else {
|
|
717
|
+
return change;
|
|
718
|
+
}
|
|
719
|
+
});
|
|
720
|
+
} else {
|
|
721
|
+
changes$1 = changes;
|
|
722
|
+
}
|
|
723
|
+
let match$1 = Table.getChainIdField(entityConfig.table);
|
|
724
|
+
let chainIdParams = match$1 !== undefined ? (
|
|
725
|
+
scopeChainId !== undefined ? [Primitive_option.valFromOption(scopeChainId)] : []
|
|
726
|
+
) : [];
|
|
646
727
|
let diffCheckpointId = Stdlib_Option.map(rollback, r => r.diffCheckpointId);
|
|
647
728
|
let batchSetUpdates = [];
|
|
648
729
|
let batchDeleteEntityIds = [];
|
|
@@ -650,7 +731,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
650
731
|
let idsWithDiff = new Set();
|
|
651
732
|
let latestChangeById = {};
|
|
652
733
|
let orderedIds = [];
|
|
653
|
-
|
|
734
|
+
changes$1.forEach(change => {
|
|
654
735
|
let entityId = change.entityId;
|
|
655
736
|
let entityKey = EntityId.toKey(entityId);
|
|
656
737
|
if (Stdlib_Option.isNone(latestChangeById[entityKey])) {
|
|
@@ -691,13 +772,13 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
691
772
|
let promises = [];
|
|
692
773
|
if (shouldSaveHistory) {
|
|
693
774
|
if (backfillHistoryIds.size !== 0) {
|
|
694
|
-
await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.table, entityConfig.index, Array.from(backfillHistoryIds));
|
|
775
|
+
await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.table, entityConfig.index, scopeChainId, Array.from(backfillHistoryIds));
|
|
695
776
|
}
|
|
696
777
|
if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
|
|
697
|
-
promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema), [
|
|
778
|
+
promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema, scopeChainId), [
|
|
698
779
|
Table.encodeIdsToJson(entityConfig.table, batchDeleteEntityIds),
|
|
699
780
|
Utils.$$BigInt.arrayToStringArray(batchDeleteCheckpointIds)
|
|
700
|
-
], {prepare: true}));
|
|
781
|
+
].concat(chainIdParams), {prepare: true}));
|
|
701
782
|
}
|
|
702
783
|
if (Utils.$$Array.notEmpty(batchSetUpdates)) {
|
|
703
784
|
if (shouldRemoveInvalidUtf8) {
|
|
@@ -717,10 +798,10 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
717
798
|
if (shouldRemoveInvalidUtf8) {
|
|
718
799
|
removeInvalidUtf8InPlace(entitiesToSet);
|
|
719
800
|
}
|
|
720
|
-
promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, entityConfig
|
|
801
|
+
promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, getRowSchema(entityConfig), pgSchema, chainIdMode));
|
|
721
802
|
}
|
|
722
803
|
if (Utils.$$Array.notEmpty(idsToDelete)) {
|
|
723
|
-
promises.push(deleteByIdsOrThrow(sql, pgSchema, idsToDelete, entityConfig.table));
|
|
804
|
+
promises.push(deleteByIdsOrThrow(sql, pgSchema, idsToDelete, entityConfig.table, Primitive_option.some(scopeChainId)));
|
|
724
805
|
}
|
|
725
806
|
await Promise.all(promises);
|
|
726
807
|
return;
|
|
@@ -795,6 +876,18 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
795
876
|
}
|
|
796
877
|
}
|
|
797
878
|
|
|
879
|
+
function rollbackKeyColumns(entityConfig) {
|
|
880
|
+
let field = Table.getChainIdField(entityConfig.table);
|
|
881
|
+
if (field !== undefined) {
|
|
882
|
+
return [
|
|
883
|
+
Table.idFieldName,
|
|
884
|
+
Table.getPgDbFieldName(field)
|
|
885
|
+
];
|
|
886
|
+
} else {
|
|
887
|
+
return [Table.idFieldName];
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
|
|
798
891
|
function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema) {
|
|
799
892
|
let dataFieldNames = Stdlib_Array.filterMap(entityConfig.table.fields, fieldOrDerived => {
|
|
800
893
|
if (fieldOrDerived.TAG === "Field") {
|
|
@@ -803,27 +896,32 @@ function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema) {
|
|
|
803
896
|
});
|
|
804
897
|
let dataFieldsCommaSeparated = dataFieldNames.map(name => `"` + name + `"`).join(", ");
|
|
805
898
|
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
806
|
-
|
|
899
|
+
let keyColumns = rollbackKeyColumns(entityConfig);
|
|
900
|
+
let keyColumnsCommaSeparated = keyColumns.map(c => `"` + c + `"`).join(", ");
|
|
901
|
+
let keyMatch = keyColumns.map(c => `h."` + c + `" = "` + historyTableName + `"."` + c + `"`).join(" AND ");
|
|
902
|
+
return `SELECT DISTINCT ON (` + keyColumnsCommaSeparated + `) ` + dataFieldsCommaSeparated + `, "` + EntityHistory.changeFieldName + `"
|
|
807
903
|
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
808
904
|
WHERE "` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
809
905
|
AND EXISTS (
|
|
810
906
|
SELECT 1
|
|
811
907
|
FROM "` + pgSchema + `"."` + historyTableName + `" h
|
|
812
|
-
WHERE
|
|
908
|
+
WHERE ` + keyMatch + `
|
|
813
909
|
AND h."` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
814
910
|
)
|
|
815
|
-
ORDER BY
|
|
911
|
+
ORDER BY ` + keyColumnsCommaSeparated + `, "` + EntityHistory.checkpointIdFieldName + `" DESC`;
|
|
816
912
|
}
|
|
817
913
|
|
|
818
914
|
function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
|
|
819
915
|
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
820
|
-
|
|
916
|
+
let keyColumns = rollbackKeyColumns(entityConfig);
|
|
917
|
+
let keyMatch = keyColumns.map(c => `h."` + c + `" = "` + historyTableName + `"."` + c + `"`).join(" AND ");
|
|
918
|
+
return `SELECT DISTINCT ` + keyColumns.map(c => `"` + c + `"`).join(", ") + `
|
|
821
919
|
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
822
920
|
WHERE "` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
823
921
|
AND NOT EXISTS (
|
|
824
922
|
SELECT 1
|
|
825
923
|
FROM "` + pgSchema + `"."` + historyTableName + `" h
|
|
826
|
-
WHERE
|
|
924
|
+
WHERE ` + keyMatch + `
|
|
827
925
|
AND h."` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
828
926
|
)`;
|
|
829
927
|
}
|
|
@@ -833,7 +931,9 @@ let rollbackRowStateSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.o
|
|
|
833
931
|
s.f(EntityHistory.changeFieldName, EntityHistory.RowAction.schema)
|
|
834
932
|
]));
|
|
835
933
|
|
|
836
|
-
let
|
|
934
|
+
let rollbackChainIdSchema = Utils.$$WeakMap.memoize(table => Stdlib_Option.map(Table.getChainIdField(table), field => S$RescriptSchema.object(s => s.f(Table.getPgDbFieldName(field), ChainId.schema))));
|
|
935
|
+
|
|
936
|
+
let rollbackRemovedIdSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.object(s => s.f(Table.idFieldName, Table.getIdSchema(table))));
|
|
837
937
|
|
|
838
938
|
function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, chainIdModeOpt, sink, onInitialize, onNewTables) {
|
|
839
939
|
let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
|
|
@@ -863,16 +963,12 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
863
963
|
let catalog = await refreshIndexCatalog();
|
|
864
964
|
let invalidIndexNames = IndexCatalog.invalidNames(catalog);
|
|
865
965
|
if (invalidIndexNames.length !== 0) {
|
|
866
|
-
Logging.warn({
|
|
966
|
+
return Logging.warn({
|
|
867
967
|
storage: storageName,
|
|
868
968
|
msg: `Ignoring invalid PostgreSQL indexes in schema "` + pgSchema + `". They can't serve queries, so the indexer builds its own alongside them.`,
|
|
869
969
|
indexes: invalidIndexNames
|
|
870
970
|
});
|
|
871
971
|
}
|
|
872
|
-
return Logging.info({
|
|
873
|
-
storage: storageName,
|
|
874
|
-
msg: `Found ` + IndexCatalog.size(catalog).toString() + ` existing indexes in the indexer schema.`
|
|
875
|
-
});
|
|
876
972
|
};
|
|
877
973
|
let isInitialized = async () => Utils.$$Array.notEmpty(await sql.unsafe(`SELECT table_schema FROM information_schema.tables WHERE table_schema = '` + pgSchema + `' AND (table_name = '` + "event_sync_state" + `' OR table_name = '` + InternalTable.Chains.table.tableName + `');`));
|
|
878
974
|
let scanCacheDir = async () => {
|
|
@@ -1062,7 +1158,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::` + addrCh
|
|
|
1062
1158
|
};
|
|
1063
1159
|
}
|
|
1064
1160
|
try {
|
|
1065
|
-
return S$RescriptSchema.parseOrThrow(rows, Table.
|
|
1161
|
+
return S$RescriptSchema.parseOrThrow(rows, Table.pgEntityRowsSchema(table));
|
|
1066
1162
|
} catch (raw_exn$1) {
|
|
1067
1163
|
let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
|
|
1068
1164
|
throw {
|
|
@@ -1331,7 +1427,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::` + addrCh
|
|
|
1331
1427
|
};
|
|
1332
1428
|
let setChainMeta = chainsData => InternalTable.Chains.setMeta(sql, pgSchema, chainsData).then(param => (undefined));
|
|
1333
1429
|
let pruneStaleCheckpoints = safeCheckpointId => InternalTable.Checkpoints.pruneStaleCheckpoints(sql, pgSchema, safeCheckpointId);
|
|
1334
|
-
let pruneStaleEntityHistory = (entityName, entityIndex, safeCheckpointId) => EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, safeCheckpointId);
|
|
1430
|
+
let pruneStaleEntityHistory = (entityName, entityIndex, chainIdColumn, safeCheckpointId) => EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, chainIdColumn, safeCheckpointId);
|
|
1335
1431
|
let getRollbackTargetCheckpoint = (reorgChainId, lastKnownValidBlockNumber) => InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
|
|
1336
1432
|
let getRollbackProgressDiff = rollbackTargetCheckpointId => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId);
|
|
1337
1433
|
let getRollbackData = async (entityConfig, rollbackTargetCheckpointId) => {
|
|
@@ -1339,7 +1435,18 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::` + addrCh
|
|
|
1339
1435
|
sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
|
|
1340
1436
|
sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
|
|
1341
1437
|
]);
|
|
1342
|
-
let
|
|
1438
|
+
let chainIdSchema = rollbackChainIdSchema(entityConfig.table);
|
|
1439
|
+
let scopeOf = row => {
|
|
1440
|
+
if (chainIdSchema !== undefined) {
|
|
1441
|
+
return S$RescriptSchema.parseOrThrow(row, Primitive_option.valFromOption(chainIdSchema));
|
|
1442
|
+
} else {
|
|
1443
|
+
return "crossChain";
|
|
1444
|
+
}
|
|
1445
|
+
};
|
|
1446
|
+
let removals = match[0].map(row => ({
|
|
1447
|
+
entityId: S$RescriptSchema.parseOrThrow(row, rollbackRemovedIdSchema(entityConfig.table)),
|
|
1448
|
+
scope: scopeOf(row)
|
|
1449
|
+
}));
|
|
1343
1450
|
let restoredEntitiesResult = [];
|
|
1344
1451
|
match[1].forEach(row => {
|
|
1345
1452
|
let match = S$RescriptSchema.parseOrThrow(row, rollbackRowStateSchema(entityConfig.table));
|
|
@@ -1347,10 +1454,13 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::` + addrCh
|
|
|
1347
1454
|
restoredEntitiesResult.push(row);
|
|
1348
1455
|
return;
|
|
1349
1456
|
}
|
|
1350
|
-
|
|
1457
|
+
removals.push({
|
|
1458
|
+
entityId: match[0],
|
|
1459
|
+
scope: scopeOf(row)
|
|
1460
|
+
});
|
|
1351
1461
|
});
|
|
1352
1462
|
return [
|
|
1353
|
-
|
|
1463
|
+
removals,
|
|
1354
1464
|
restoredEntitiesResult
|
|
1355
1465
|
];
|
|
1356
1466
|
};
|
|
@@ -1459,11 +1569,14 @@ export {
|
|
|
1459
1569
|
slowOnLargeDatabaseNotice,
|
|
1460
1570
|
getSchemaIndexes,
|
|
1461
1571
|
makeCreateTableQuery,
|
|
1572
|
+
rowSchemaCache,
|
|
1573
|
+
getRowSchema,
|
|
1462
1574
|
entityHistoryCache,
|
|
1463
1575
|
getEntityHistory,
|
|
1464
1576
|
makeInitializeTransaction,
|
|
1465
1577
|
makeLoadQuery,
|
|
1466
1578
|
makeFilterCondition,
|
|
1579
|
+
makeChainIdCondition,
|
|
1467
1580
|
makeDeleteByIdQuery,
|
|
1468
1581
|
makeDeleteByIdsQuery,
|
|
1469
1582
|
makeLoadAllQuery,
|
|
@@ -1487,12 +1600,14 @@ export {
|
|
|
1487
1600
|
makeInsertDeleteUpdatesQuery,
|
|
1488
1601
|
executeSet,
|
|
1489
1602
|
writeBatch,
|
|
1603
|
+
rollbackKeyColumns,
|
|
1490
1604
|
makeGetRollbackPreTargetRowsQuery,
|
|
1491
1605
|
makeGetRollbackRemovedIdsQuery,
|
|
1492
1606
|
rollbackRowStateSchema,
|
|
1493
|
-
|
|
1607
|
+
rollbackChainIdSchema,
|
|
1608
|
+
rollbackRemovedIdSchema,
|
|
1494
1609
|
make,
|
|
1495
1610
|
makeStorageFromEnv,
|
|
1496
1611
|
makePersistenceFromConfig,
|
|
1497
1612
|
}
|
|
1498
|
-
/*
|
|
1613
|
+
/* rowSchemaCache Not a pure module */
|
|
@@ -103,6 +103,7 @@ let pruneEntities = async (state: IndexerState.t, ~entities, ~safeCheckpointId)
|
|
|
103
103
|
switch await persistence.storage.pruneStaleEntityHistory(
|
|
104
104
|
~entityName=entityConfig.name,
|
|
105
105
|
~entityIndex=entityConfig.index,
|
|
106
|
+
~chainIdColumn=entityConfig.table->Table.getChainIdField->Option.map(Table.getPgDbFieldName),
|
|
106
107
|
~safeCheckpointId,
|
|
107
108
|
) {
|
|
108
109
|
| () =>
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
2
|
|
|
3
3
|
import * as Env from "./Env.res.mjs";
|
|
4
|
+
import * as Table from "./db/Table.res.mjs";
|
|
4
5
|
import * as Utils from "./Utils.res.mjs";
|
|
5
6
|
import * as Config from "./Config.res.mjs";
|
|
6
7
|
import * as Logging from "./Logging.res.mjs";
|
|
@@ -69,7 +70,7 @@ async function pruneEntities(state, entities, safeCheckpointId) {
|
|
|
69
70
|
let exit = 0;
|
|
70
71
|
let val;
|
|
71
72
|
try {
|
|
72
|
-
val = await persistence.storage.pruneStaleEntityHistory(entityConfig.name, entityConfig.index, safeCheckpointId);
|
|
73
|
+
val = await persistence.storage.pruneStaleEntityHistory(entityConfig.name, entityConfig.index, Stdlib_Option.map(Table.getChainIdField(entityConfig.table), Table.getPgDbFieldName), safeCheckpointId);
|
|
73
74
|
exit = 1;
|
|
74
75
|
} catch (raw_exn) {
|
|
75
76
|
let exn = Primitive_exceptions.internalToException(raw_exn);
|
package/src/Sink.res
CHANGED
|
@@ -22,7 +22,7 @@ let makeClickHouse = (~host, ~database, ~username, ~password, ~chainIdMode: Chai
|
|
|
22
22
|
// Don't pass database to the client; it would fail if the database doesn't
|
|
23
23
|
// exist yet. Each query qualifies the name explicitly or runs USE first.
|
|
24
24
|
|
|
25
|
-
let cache =
|
|
25
|
+
let cache = Dict.make()
|
|
26
26
|
|
|
27
27
|
{
|
|
28
28
|
name: "clickhouse",
|
|
@@ -34,8 +34,8 @@ let makeClickHouse = (~host, ~database, ~username, ~password, ~chainIdMode: Chai
|
|
|
34
34
|
},
|
|
35
35
|
writeBatch: async (~batch, ~updatedEntities) => {
|
|
36
36
|
await Promise.all(
|
|
37
|
-
updatedEntities->Array.map(({entityConfig, changes}) => {
|
|
38
|
-
ClickHouse.setUpdatesOrThrow(client, ~cache, ~changes, ~entityConfig, ~database)
|
|
37
|
+
updatedEntities->Array.map(({entityConfig, scope, changes}) => {
|
|
38
|
+
ClickHouse.setUpdatesOrThrow(client, ~cache, ~changes, ~entityConfig, ~scope, ~database)
|
|
39
39
|
}),
|
|
40
40
|
)->Utils.Promise.ignoreValue
|
|
41
41
|
await ClickHouse.setCheckpointsOrThrow(client, ~batch, ~database)
|
package/src/Sink.res.mjs
CHANGED
|
@@ -10,7 +10,7 @@ function makeClickHouse(host, database, username, password, chainIdModeOpt) {
|
|
|
10
10
|
username: username,
|
|
11
11
|
password: password
|
|
12
12
|
});
|
|
13
|
-
let cache =
|
|
13
|
+
let cache = {};
|
|
14
14
|
return {
|
|
15
15
|
name: "clickhouse",
|
|
16
16
|
initialize: ($staropt$star, $staropt$star$1, $staropt$star$2) => {
|
|
@@ -21,7 +21,7 @@ function makeClickHouse(host, database, username, password, chainIdModeOpt) {
|
|
|
21
21
|
},
|
|
22
22
|
resume: checkpointId => ClickHouse.resume(client, database, checkpointId),
|
|
23
23
|
writeBatch: async (batch, updatedEntities) => {
|
|
24
|
-
await Promise.all(updatedEntities.map(param => ClickHouse.setUpdatesOrThrow(client, cache, param.changes, param.entityConfig, database)));
|
|
24
|
+
await Promise.all(updatedEntities.map(param => ClickHouse.setUpdatesOrThrow(client, cache, param.changes, param.entityConfig, param.scope, database)));
|
|
25
25
|
return await ClickHouse.setCheckpointsOrThrow(client, batch, database);
|
|
26
26
|
}
|
|
27
27
|
};
|