envio 3.3.0-alpha.11 → 3.3.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/package.json +6 -6
- package/src/ChainFetching.res +21 -12
- package/src/ChainFetching.res.mjs +20 -9
- package/src/ChainState.res +20 -8
- package/src/ChainState.res.mjs +13 -12
- package/src/ChainState.resi +1 -1
- package/src/CrossChainState.res +64 -41
- package/src/CrossChainState.res.mjs +18 -13
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/FetchState.res +115 -76
- package/src/FetchState.res.mjs +114 -104
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +10 -19
- package/src/IndexerState.res.mjs +7 -6
- package/src/IndexerState.resi +1 -9
- package/src/Internal.res +87 -12
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +28 -12
- package/src/Writing.res.mjs +15 -7
- package/src/bindings/NodeJs.res +5 -0
- package/src/sources/SvmHyperSyncSource.res +10 -0
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
package/src/PgStorage.res.mjs
CHANGED
|
@@ -493,15 +493,18 @@ function makeSchemaTableNamesQuery(pgSchema) {
|
|
|
493
493
|
return `SELECT table_name FROM information_schema.tables WHERE table_schema = '` + pgSchema + `';`;
|
|
494
494
|
}
|
|
495
495
|
|
|
496
|
-
let cacheTablePrefixLength = Internal.cacheTablePrefix.length;
|
|
497
|
-
|
|
498
496
|
function makeSchemaCacheTableInfoQuery(pgSchema) {
|
|
499
|
-
return `SELECT
|
|
497
|
+
return `SELECT
|
|
500
498
|
t.table_name,
|
|
501
499
|
` + getCacheRowCountFnName + `(t.table_name) as count
|
|
502
500
|
FROM information_schema.tables t
|
|
503
|
-
WHERE t.table_schema = '` + pgSchema + `'
|
|
504
|
-
AND t.table_name
|
|
501
|
+
WHERE t.table_schema = '` + pgSchema + `'
|
|
502
|
+
AND t.table_name ~ '^envio_([0-9]+_)?effect_.+'
|
|
503
|
+
AND (
|
|
504
|
+
SELECT array_agg(c.column_name::text ORDER BY c.column_name::text)
|
|
505
|
+
FROM information_schema.columns c
|
|
506
|
+
WHERE c.table_schema = t.table_schema AND c.table_name = t.table_name
|
|
507
|
+
) = ARRAY['id', 'output'];`;
|
|
505
508
|
}
|
|
506
509
|
|
|
507
510
|
let psqlExecState = {
|
|
@@ -766,7 +769,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
766
769
|
}
|
|
767
770
|
throw exn;
|
|
768
771
|
}),
|
|
769
|
-
Promise.all(updatedEffectsCache.map(param => setEffectCacheOrThrow(param.
|
|
772
|
+
Promise.all(updatedEffectsCache.map(param => setEffectCacheOrThrow(param.table, param.itemSchema, param.items, param.shouldInitialize)))
|
|
770
773
|
]);
|
|
771
774
|
let specificError$1 = specificError.contents;
|
|
772
775
|
if (specificError$1 === undefined) {
|
|
@@ -789,7 +792,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
|
|
|
789
792
|
}
|
|
790
793
|
}
|
|
791
794
|
|
|
792
|
-
function
|
|
795
|
+
function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema) {
|
|
793
796
|
let dataFieldNames = Stdlib_Array.filterMap(entityConfig.table.fields, fieldOrDerived => {
|
|
794
797
|
if (fieldOrDerived.TAG === "Field") {
|
|
795
798
|
return Table.getPgDbFieldName(fieldOrDerived._0);
|
|
@@ -797,31 +800,36 @@ function makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema) {
|
|
|
797
800
|
});
|
|
798
801
|
let dataFieldsCommaSeparated = dataFieldNames.map(name => `"` + name + `"`).join(", ");
|
|
799
802
|
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
800
|
-
return `SELECT DISTINCT ON (` + Table.idFieldName + `) ` + dataFieldsCommaSeparated + `
|
|
801
|
-
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
802
|
-
WHERE "` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
ORDER BY ` + Table.idFieldName +
|
|
803
|
+
return `SELECT DISTINCT ON ("` + Table.idFieldName + `") ` + dataFieldsCommaSeparated + `, "` + EntityHistory.changeFieldName + `"
|
|
804
|
+
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
805
|
+
WHERE "` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
806
|
+
AND EXISTS (
|
|
807
|
+
SELECT 1
|
|
808
|
+
FROM "` + pgSchema + `"."` + historyTableName + `" h
|
|
809
|
+
WHERE h."` + Table.idFieldName + `" = "` + historyTableName + `"."` + Table.idFieldName + `"
|
|
810
|
+
AND h."` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
811
|
+
)
|
|
812
|
+
ORDER BY "` + Table.idFieldName + `", "` + EntityHistory.checkpointIdFieldName + `" DESC`;
|
|
810
813
|
}
|
|
811
814
|
|
|
812
815
|
function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
|
|
813
816
|
let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
|
|
814
|
-
return `SELECT DISTINCT ` + Table.idFieldName + `
|
|
815
|
-
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
816
|
-
WHERE "` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
817
|
-
AND NOT EXISTS (
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
)`;
|
|
817
|
+
return `SELECT DISTINCT "` + Table.idFieldName + `"
|
|
818
|
+
FROM "` + pgSchema + `"."` + historyTableName + `"
|
|
819
|
+
WHERE "` + EntityHistory.checkpointIdFieldName + `" > $1
|
|
820
|
+
AND NOT EXISTS (
|
|
821
|
+
SELECT 1
|
|
822
|
+
FROM "` + pgSchema + `"."` + historyTableName + `" h
|
|
823
|
+
WHERE h."` + Table.idFieldName + `" = "` + historyTableName + `"."` + Table.idFieldName + `"
|
|
824
|
+
AND h."` + EntityHistory.checkpointIdFieldName + `" <= $1
|
|
825
|
+
)`;
|
|
823
826
|
}
|
|
824
827
|
|
|
828
|
+
let rollbackRowStateSchema = S$RescriptSchema.object(s => [
|
|
829
|
+
s.f(Table.idFieldName, S$RescriptSchema.string),
|
|
830
|
+
s.f(EntityHistory.changeFieldName, EntityHistory.RowAction.schema)
|
|
831
|
+
]);
|
|
832
|
+
|
|
825
833
|
function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
|
|
826
834
|
let containerName = "envio-postgres";
|
|
827
835
|
let psqlExecOptions_env = Object.fromEntries([
|
|
@@ -839,32 +847,60 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
839
847
|
};
|
|
840
848
|
let cacheDirPath = Path.resolve(".envio", "cache");
|
|
841
849
|
let isInitialized = async () => Utils.$$Array.notEmpty(await sql.unsafe(`SELECT table_schema FROM information_schema.tables WHERE table_schema = '` + pgSchema + `' AND (table_name = '` + "event_sync_state" + `' OR table_name = '` + InternalTable.Chains.table.tableName + `');`));
|
|
850
|
+
let scanCacheDir = async () => {
|
|
851
|
+
let topEntries;
|
|
852
|
+
try {
|
|
853
|
+
topEntries = await Fs.promises.readdir(cacheDirPath);
|
|
854
|
+
} catch (exn) {
|
|
855
|
+
topEntries = [];
|
|
856
|
+
}
|
|
857
|
+
let result = [];
|
|
858
|
+
await Promise.all(topEntries.map(async entry => {
|
|
859
|
+
let entryPath = Path.join(cacheDirPath, entry);
|
|
860
|
+
let isDir = (await Fs.promises.stat(entryPath)).isDirectory();
|
|
861
|
+
if (isDir) {
|
|
862
|
+
let subEntries = await Fs.promises.readdir(entryPath);
|
|
863
|
+
let tsvs = subEntries.filter(sub => sub.endsWith(".tsv"));
|
|
864
|
+
let chainId = Internal.EffectCache.parseChainId(entry);
|
|
865
|
+
if (chainId !== undefined) {
|
|
866
|
+
tsvs.forEach(sub => {
|
|
867
|
+
let effectName = sub.slice(0, -4);
|
|
868
|
+
let table = Internal.makeCacheTable(effectName, chainId);
|
|
869
|
+
result.push([
|
|
870
|
+
table,
|
|
871
|
+
Path.join(entryPath, sub)
|
|
872
|
+
]);
|
|
873
|
+
});
|
|
874
|
+
return;
|
|
875
|
+
} else if (Utils.$$Array.notEmpty(tsvs)) {
|
|
876
|
+
return Stdlib_JsError.throwWithMessage(`Invalid effect cache directory ".envio/cache/` + entry + `". Chain cache directories must be named by a numeric chain id (e.g. "1"). Found cache files: ` + tsvs.join(", ") + `.`);
|
|
877
|
+
} else {
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
if (!entry.endsWith(".tsv")) {
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
let effectName = entry.slice(0, -4);
|
|
885
|
+
let table = Internal.makeCacheTable(effectName, "crossChain");
|
|
886
|
+
result.push([
|
|
887
|
+
table,
|
|
888
|
+
entryPath
|
|
889
|
+
]);
|
|
890
|
+
}));
|
|
891
|
+
return result;
|
|
892
|
+
};
|
|
842
893
|
let restoreEffectCache = async withUpload => {
|
|
843
894
|
if (withUpload) {
|
|
844
|
-
let
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
TAG: "Ok",
|
|
848
|
-
_0: e
|
|
849
|
-
})), param => Promise.resolve({
|
|
850
|
-
TAG: "Error",
|
|
851
|
-
_0: nothingToUploadErrorMessage
|
|
852
|
-
})),
|
|
853
|
-
getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName)
|
|
854
|
-
]);
|
|
855
|
-
let exit = 0;
|
|
856
|
-
let message;
|
|
857
|
-
let entries = match[0];
|
|
858
|
-
if (entries.TAG === "Ok") {
|
|
859
|
-
let psqlExec = match[1];
|
|
895
|
+
let entries = await scanCacheDir();
|
|
896
|
+
if (entries.length !== 0) {
|
|
897
|
+
let psqlExec = await getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containerName);
|
|
860
898
|
if (psqlExec.TAG === "Ok") {
|
|
861
899
|
let psqlExec$1 = psqlExec._0;
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
let
|
|
865
|
-
let table = Internal.makeCacheTable(effectName);
|
|
900
|
+
await Promise.all(entries.map(param => {
|
|
901
|
+
let inputFile = param[1];
|
|
902
|
+
let table = param[0];
|
|
866
903
|
return sql.unsafe(makeCreateTableQuery(table, pgSchema, false)).then(() => {
|
|
867
|
-
let inputFile = Path.join(cacheDirPath, entry);
|
|
868
904
|
let command = psqlExec$1 + ` -c 'COPY "` + pgSchema + `"."` + table.tableName + `" FROM STDIN WITH (FORMAT text, HEADER);' < ` + inputFile;
|
|
869
905
|
return new Promise((resolve, reject) => {
|
|
870
906
|
Child_process.exec(command, psqlExecOptions, (error, stdout, param) => {
|
|
@@ -879,19 +915,10 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
879
915
|
}));
|
|
880
916
|
Logging.info("Successfully uploaded cache.");
|
|
881
917
|
} else {
|
|
882
|
-
|
|
883
|
-
exit = 1;
|
|
918
|
+
Logging.error(`Failed to upload cache, continuing without it. ` + psqlExec._0);
|
|
884
919
|
}
|
|
885
920
|
} else {
|
|
886
|
-
|
|
887
|
-
exit = 1;
|
|
888
|
-
}
|
|
889
|
-
if (exit === 1) {
|
|
890
|
-
if (message === nothingToUploadErrorMessage) {
|
|
891
|
-
Logging.info("No cache found to upload.");
|
|
892
|
-
} else {
|
|
893
|
-
Logging.error(`Failed to upload cache, continuing without it. ` + message);
|
|
894
|
-
}
|
|
921
|
+
Logging.info("No cache found to upload.");
|
|
895
922
|
}
|
|
896
923
|
}
|
|
897
924
|
let cacheTableInfo = await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema));
|
|
@@ -900,11 +927,17 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
|
|
|
900
927
|
}
|
|
901
928
|
let cache = {};
|
|
902
929
|
cacheTableInfo.forEach(param => {
|
|
903
|
-
let
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
930
|
+
let tableName = param.table_name;
|
|
931
|
+
let match = Internal.EffectCache.fromTableName(tableName);
|
|
932
|
+
if (match !== undefined) {
|
|
933
|
+
cache[tableName] = {
|
|
934
|
+
effectName: match[0],
|
|
935
|
+
scope: match[1],
|
|
936
|
+
tableName: tableName,
|
|
937
|
+
count: param.count
|
|
938
|
+
};
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
908
941
|
});
|
|
909
942
|
return cache;
|
|
910
943
|
};
|
|
@@ -998,16 +1031,14 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
998
1031
|
}
|
|
999
1032
|
};
|
|
1000
1033
|
let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema);
|
|
1001
|
-
let setEffectCacheOrThrow = async (
|
|
1002
|
-
let match = effect.storageMeta;
|
|
1003
|
-
let table = match.table;
|
|
1034
|
+
let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
|
|
1004
1035
|
if (initialize) {
|
|
1005
1036
|
await sql.unsafe(makeCreateTableQuery(table, pgSchema, false));
|
|
1006
1037
|
if (onNewTables !== undefined) {
|
|
1007
1038
|
await onNewTables([table.tableName]);
|
|
1008
1039
|
}
|
|
1009
1040
|
}
|
|
1010
|
-
return await setOrThrow$1(items, table,
|
|
1041
|
+
return await setOrThrow$1(items, table, itemSchema);
|
|
1011
1042
|
};
|
|
1012
1043
|
let dumpEffectCache = async () => {
|
|
1013
1044
|
try {
|
|
@@ -1030,10 +1061,16 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1030
1061
|
Logging.info(`Dumping cache: ` + cacheTableInfo.map(param => param.table_name + " (" + param.count.toString() + " rows)").join(", "));
|
|
1031
1062
|
let promises = cacheTableInfo.map(async param => {
|
|
1032
1063
|
let tableName = param.table_name;
|
|
1033
|
-
let
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1064
|
+
let match = Internal.EffectCache.fromTableName(tableName);
|
|
1065
|
+
if (match === undefined) {
|
|
1066
|
+
return "";
|
|
1067
|
+
}
|
|
1068
|
+
let outputPath = Path.join(cacheDirPath, Internal.EffectCache.toCachePath(match[0], match[1]));
|
|
1069
|
+
await Fs.promises.mkdir(Path.dirname(outputPath), {
|
|
1070
|
+
recursive: true
|
|
1071
|
+
});
|
|
1072
|
+
let command = psqlExec$1 + ` -c 'COPY "` + pgSchema + `"."` + tableName + `" TO STDOUT WITH (FORMAT text, HEADER);' > ` + outputPath;
|
|
1073
|
+
return await new Promise((resolve, reject) => {
|
|
1037
1074
|
Child_process.exec(command, psqlExecOptions, (error, stdout, param) => {
|
|
1038
1075
|
if (error === null) {
|
|
1039
1076
|
return resolve(stdout);
|
|
@@ -1097,10 +1134,26 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
|
|
|
1097
1134
|
let pruneStaleEntityHistory = (entityName, entityIndex, safeCheckpointId) => EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, safeCheckpointId);
|
|
1098
1135
|
let getRollbackTargetCheckpoint = (reorgChainId, lastKnownValidBlockNumber) => InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
|
|
1099
1136
|
let getRollbackProgressDiff = rollbackTargetCheckpointId => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId);
|
|
1100
|
-
let getRollbackData = async (entityConfig, rollbackTargetCheckpointId) =>
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1137
|
+
let getRollbackData = async (entityConfig, rollbackTargetCheckpointId) => {
|
|
1138
|
+
let match = await Promise.all([
|
|
1139
|
+
sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
|
|
1140
|
+
sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
|
|
1141
|
+
]);
|
|
1142
|
+
let removedIds = match[0].map(row => row.id);
|
|
1143
|
+
let restoredEntitiesResult = [];
|
|
1144
|
+
match[1].forEach(row => {
|
|
1145
|
+
let match = S$RescriptSchema.parseOrThrow(row, rollbackRowStateSchema);
|
|
1146
|
+
if (match[1] === "SET") {
|
|
1147
|
+
restoredEntitiesResult.push(row);
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
removedIds.push(match[0]);
|
|
1151
|
+
});
|
|
1152
|
+
return [
|
|
1153
|
+
removedIds,
|
|
1154
|
+
restoredEntitiesResult
|
|
1155
|
+
];
|
|
1156
|
+
};
|
|
1104
1157
|
let writeBatchMethod = async (batch, rollback, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities, chainMetaData) => {
|
|
1105
1158
|
let pgUpdates = [];
|
|
1106
1159
|
let chUpdates = [];
|
|
@@ -1227,15 +1280,15 @@ export {
|
|
|
1227
1280
|
setQueryCache,
|
|
1228
1281
|
setOrThrow,
|
|
1229
1282
|
makeSchemaTableNamesQuery,
|
|
1230
|
-
cacheTablePrefixLength,
|
|
1231
1283
|
makeSchemaCacheTableInfoQuery,
|
|
1232
1284
|
getConnectedPsqlExec,
|
|
1233
1285
|
deleteByIdsOrThrow,
|
|
1234
1286
|
makeInsertDeleteUpdatesQuery,
|
|
1235
1287
|
executeSet,
|
|
1236
1288
|
writeBatch,
|
|
1237
|
-
|
|
1289
|
+
makeGetRollbackPreTargetRowsQuery,
|
|
1238
1290
|
makeGetRollbackRemovedIdsQuery,
|
|
1291
|
+
rollbackRowStateSchema,
|
|
1239
1292
|
make,
|
|
1240
1293
|
makeStorageFromEnv,
|
|
1241
1294
|
makePersistenceFromConfig,
|
package/src/Prometheus.res
CHANGED
|
@@ -606,29 +606,39 @@ let effectLabelsSchema = S.object(s => {
|
|
|
606
606
|
s.field("effect", S.string)
|
|
607
607
|
})
|
|
608
608
|
|
|
609
|
+
// For metrics whose backing state lives per scope ("crossChain" or a chain
|
|
610
|
+
// id) — without the label, scopes of the same effect would clobber each
|
|
611
|
+
// other's gauge value.
|
|
612
|
+
let effectScopeLabelsSchema = S.schema(s =>
|
|
613
|
+
{
|
|
614
|
+
"effect": s.matches(S.string),
|
|
615
|
+
"scope": s.matches(S.string),
|
|
616
|
+
}
|
|
617
|
+
)
|
|
618
|
+
|
|
609
619
|
module EffectCalls = {
|
|
610
620
|
let timeCounter = SafeCounter.makeOrThrow(
|
|
611
621
|
~name="envio_effect_call_seconds",
|
|
612
622
|
~help="Processing time taken to call the Effect function.",
|
|
613
|
-
~labelSchema=
|
|
623
|
+
~labelSchema=effectScopeLabelsSchema,
|
|
614
624
|
)
|
|
615
625
|
|
|
616
626
|
let sumTimeCounter = SafeCounter.makeOrThrow(
|
|
617
627
|
~name="envio_effect_call_seconds_total",
|
|
618
628
|
~help="Cumulative time spent calling the Effect function during the indexing process.",
|
|
619
|
-
~labelSchema=
|
|
629
|
+
~labelSchema=effectScopeLabelsSchema,
|
|
620
630
|
)
|
|
621
631
|
|
|
622
632
|
let totalCallsCount = SafeCounter.makeOrThrow(
|
|
623
633
|
~name="envio_effect_call_total",
|
|
624
634
|
~help="Cumulative number of resolved Effect function calls during the indexing process.",
|
|
625
|
-
~labelSchema=
|
|
635
|
+
~labelSchema=effectScopeLabelsSchema,
|
|
626
636
|
)
|
|
627
637
|
|
|
628
638
|
let activeCallsCount = SafeGauge.makeOrThrow(
|
|
629
639
|
~name="envio_effect_active_calls",
|
|
630
640
|
~help="The number of Effect function calls that are currently running.",
|
|
631
|
-
~labelSchema=
|
|
641
|
+
~labelSchema=effectScopeLabelsSchema,
|
|
632
642
|
)
|
|
633
643
|
}
|
|
634
644
|
|
|
@@ -636,11 +646,11 @@ module EffectCacheCount = {
|
|
|
636
646
|
let gauge = SafeGauge.makeOrThrow(
|
|
637
647
|
~name="envio_effect_cache",
|
|
638
648
|
~help="The number of items in the effect cache.",
|
|
639
|
-
~labelSchema=
|
|
649
|
+
~labelSchema=effectScopeLabelsSchema,
|
|
640
650
|
)
|
|
641
651
|
|
|
642
|
-
let set = (~count, ~effectName) => {
|
|
643
|
-
gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
|
|
652
|
+
let set = (~count, ~effectName, ~scope) => {
|
|
653
|
+
gauge->SafeGauge.handleInt(~labels={"effect": effectName, "scope": scope}, ~value=count)
|
|
644
654
|
}
|
|
645
655
|
}
|
|
646
656
|
|
|
@@ -660,7 +670,7 @@ module EffectQueueCount = {
|
|
|
660
670
|
let gauge = SafeGauge.makeOrThrow(
|
|
661
671
|
~name="envio_effect_queue",
|
|
662
672
|
~help="The number of effect calls waiting in the rate limit queue.",
|
|
663
|
-
~labelSchema=
|
|
673
|
+
~labelSchema=effectScopeLabelsSchema,
|
|
664
674
|
)
|
|
665
675
|
|
|
666
676
|
let timeCounter = SafeCounter.makeOrThrow(
|
|
@@ -669,8 +679,8 @@ module EffectQueueCount = {
|
|
|
669
679
|
~labelSchema=effectLabelsSchema,
|
|
670
680
|
)
|
|
671
681
|
|
|
672
|
-
let set = (~count, ~effectName) => {
|
|
673
|
-
gauge->SafeGauge.handleInt(~labels=effectName, ~value=count)
|
|
682
|
+
let set = (~count, ~effectName, ~scope) => {
|
|
683
|
+
gauge->SafeGauge.handleInt(~labels={"effect": effectName, "scope": scope}, ~value=count)
|
|
674
684
|
}
|
|
675
685
|
}
|
|
676
686
|
|
package/src/Prometheus.res.mjs
CHANGED
|
@@ -651,13 +651,18 @@ let ProgressLatency = {
|
|
|
651
651
|
|
|
652
652
|
let effectLabelsSchema = S$RescriptSchema.object(s => s.f("effect", S$RescriptSchema.string));
|
|
653
653
|
|
|
654
|
-
let
|
|
654
|
+
let effectScopeLabelsSchema = S$RescriptSchema.schema(s => ({
|
|
655
|
+
effect: s.m(S$RescriptSchema.string),
|
|
656
|
+
scope: s.m(S$RescriptSchema.string)
|
|
657
|
+
}));
|
|
658
|
+
|
|
659
|
+
let timeCounter$5 = makeOrThrow("envio_effect_call_seconds", "Processing time taken to call the Effect function.", effectScopeLabelsSchema);
|
|
655
660
|
|
|
656
|
-
let sumTimeCounter$1 = makeOrThrow("envio_effect_call_seconds_total", "Cumulative time spent calling the Effect function during the indexing process.",
|
|
661
|
+
let sumTimeCounter$1 = makeOrThrow("envio_effect_call_seconds_total", "Cumulative time spent calling the Effect function during the indexing process.", effectScopeLabelsSchema);
|
|
657
662
|
|
|
658
|
-
let totalCallsCount = makeOrThrow("envio_effect_call_total", "Cumulative number of resolved Effect function calls during the indexing process.",
|
|
663
|
+
let totalCallsCount = makeOrThrow("envio_effect_call_total", "Cumulative number of resolved Effect function calls during the indexing process.", effectScopeLabelsSchema);
|
|
659
664
|
|
|
660
|
-
let activeCallsCount = makeOrThrow$1("envio_effect_active_calls", "The number of Effect function calls that are currently running.",
|
|
665
|
+
let activeCallsCount = makeOrThrow$1("envio_effect_active_calls", "The number of Effect function calls that are currently running.", effectScopeLabelsSchema);
|
|
661
666
|
|
|
662
667
|
let EffectCalls = {
|
|
663
668
|
timeCounter: timeCounter$5,
|
|
@@ -666,10 +671,13 @@ let EffectCalls = {
|
|
|
666
671
|
activeCallsCount: activeCallsCount
|
|
667
672
|
};
|
|
668
673
|
|
|
669
|
-
let gauge$19 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.",
|
|
674
|
+
let gauge$19 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectScopeLabelsSchema);
|
|
670
675
|
|
|
671
|
-
function set$19(count, effectName) {
|
|
672
|
-
handleInt$1(gauge$19,
|
|
676
|
+
function set$19(count, effectName, scope) {
|
|
677
|
+
handleInt$1(gauge$19, {
|
|
678
|
+
effect: effectName,
|
|
679
|
+
scope: scope
|
|
680
|
+
}, count);
|
|
673
681
|
}
|
|
674
682
|
|
|
675
683
|
let EffectCacheCount = {
|
|
@@ -688,12 +696,15 @@ let EffectCacheInvalidationsCount = {
|
|
|
688
696
|
increment: increment$7
|
|
689
697
|
};
|
|
690
698
|
|
|
691
|
-
let gauge$20 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.",
|
|
699
|
+
let gauge$20 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectScopeLabelsSchema);
|
|
692
700
|
|
|
693
701
|
let timeCounter$6 = makeOrThrow("envio_effect_queue_wait_seconds", "The time spent waiting in the rate limit queue.", effectLabelsSchema);
|
|
694
702
|
|
|
695
|
-
function set$20(count, effectName) {
|
|
696
|
-
handleInt$1(gauge$20,
|
|
703
|
+
function set$20(count, effectName, scope) {
|
|
704
|
+
handleInt$1(gauge$20, {
|
|
705
|
+
effect: effectName,
|
|
706
|
+
scope: scope
|
|
707
|
+
}, count);
|
|
697
708
|
}
|
|
698
709
|
|
|
699
710
|
let EffectQueueCount = {
|
|
@@ -825,6 +836,7 @@ export {
|
|
|
825
836
|
ProgressEventsCount,
|
|
826
837
|
ProgressLatency,
|
|
827
838
|
effectLabelsSchema,
|
|
839
|
+
effectScopeLabelsSchema,
|
|
828
840
|
EffectCalls,
|
|
829
841
|
EffectCacheCount,
|
|
830
842
|
EffectCacheInvalidationsCount,
|
package/src/Rollback.res
CHANGED
|
@@ -72,7 +72,7 @@ let rec rollback = async (
|
|
|
72
72
|
// found yet. Wait for the ReorgDetected branch above to find it and re-kick.
|
|
73
73
|
| FindingReorgDepth => ()
|
|
74
74
|
| FoundReorgDepth(_) if state->IndexerState.isProcessing =>
|
|
75
|
-
Logging.
|
|
75
|
+
Logging.trace("Waiting for batch to finish processing before executing rollback")
|
|
76
76
|
| FoundReorgDepth({chain: reorgChain, rollbackTargetBlockNumber}) =>
|
|
77
77
|
await executeRollback(
|
|
78
78
|
state,
|
|
@@ -96,10 +96,10 @@ and executeRollback = async (
|
|
|
96
96
|
) => {
|
|
97
97
|
let startTime = Performance.now()
|
|
98
98
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
99
|
+
// Not derived from the reorg chain's logger: that would bind its chainId onto
|
|
100
|
+
// every line, colliding with the per-chain chainId on the "Rollbacked" logs.
|
|
101
|
+
// The reorg chain is identified by the reorgChain param instead.
|
|
102
|
+
let logger = Logging.createChild(
|
|
103
103
|
~params={
|
|
104
104
|
"action": "Rollback",
|
|
105
105
|
"reorgChain": reorgChain,
|
|
@@ -161,10 +161,12 @@ and executeRollback = async (
|
|
|
161
161
|
}
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
let rolledBackChains = []
|
|
164
165
|
state
|
|
165
166
|
->IndexerState.chainStates
|
|
166
167
|
->Utils.Dict.forEach(cs => {
|
|
167
168
|
let chainId = (cs->ChainState.chainConfig).id
|
|
169
|
+
let fromBlock = cs->ChainState.committedProgressBlockNumber
|
|
168
170
|
cs->ChainState.rollback(
|
|
169
171
|
~newProgressBlockNumber=newProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
|
|
170
172
|
chainId,
|
|
@@ -175,6 +177,19 @@ and executeRollback = async (
|
|
|
175
177
|
~rollbackTargetBlockNumber,
|
|
176
178
|
~isReorgChain=chainId === reorgChainId,
|
|
177
179
|
)
|
|
180
|
+
let toBlock = cs->ChainState.committedProgressBlockNumber
|
|
181
|
+
if fromBlock !== toBlock {
|
|
182
|
+
rolledBackChains
|
|
183
|
+
->Array.push({
|
|
184
|
+
"chainId": chainId,
|
|
185
|
+
"fromBlock": fromBlock,
|
|
186
|
+
"toBlock": toBlock,
|
|
187
|
+
"rollbackedEvents": eventsProcessedDiffByChain
|
|
188
|
+
->Utils.Dict.dangerouslyGetByIntNonOption(chainId)
|
|
189
|
+
->Option.getOr(0.),
|
|
190
|
+
})
|
|
191
|
+
->ignore
|
|
192
|
+
}
|
|
178
193
|
})
|
|
179
194
|
|
|
180
195
|
let diff = await state->InMemoryStore.prepareRollbackDiff(
|
|
@@ -183,15 +198,19 @@ and executeRollback = async (
|
|
|
183
198
|
~progressBlockNumberByChainId=newProgressBlockNumberPerChain,
|
|
184
199
|
)
|
|
185
200
|
|
|
201
|
+
rolledBackChains->Array.forEach(chain => {
|
|
202
|
+
logger->Logging.childInfo({
|
|
203
|
+
"msg": "Rollbacked",
|
|
204
|
+
"chainId": chain["chainId"],
|
|
205
|
+
"fromBlock": chain["fromBlock"],
|
|
206
|
+
"toBlock": chain["toBlock"],
|
|
207
|
+
"rollbackedEvents": chain["rollbackedEvents"],
|
|
208
|
+
})
|
|
209
|
+
})
|
|
186
210
|
logger->Logging.childTrace({
|
|
187
|
-
"msg": "
|
|
188
|
-
"
|
|
189
|
-
|
|
190
|
-
"upserted": diff["setEntities"],
|
|
191
|
-
},
|
|
192
|
-
"rollbackedEvents": rollbackedProcessedEvents.contents,
|
|
193
|
-
"beforeCheckpointId": state->IndexerState.committedCheckpointId,
|
|
194
|
-
"targetCheckpointId": rollbackTargetCheckpointId,
|
|
211
|
+
"msg": "Rollback entity changes",
|
|
212
|
+
"deleted": diff["deletedEntities"],
|
|
213
|
+
"upserted": diff["setEntities"],
|
|
195
214
|
})
|
|
196
215
|
Prometheus.RollbackSuccess.increment(
|
|
197
216
|
~timeSeconds=Performance.secondsSince(startTime),
|
package/src/Rollback.res.mjs
CHANGED
|
@@ -33,8 +33,7 @@ async function getLastKnownValidBlock(chainState, reorgBlockNumber, isRealtime)
|
|
|
33
33
|
|
|
34
34
|
async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, scheduleFetch, scheduleProcessing) {
|
|
35
35
|
let startTime = Performance.now();
|
|
36
|
-
let
|
|
37
|
-
let logger = Logging.createChildFrom(ChainState.logger(chainState), {
|
|
36
|
+
let logger = Logging.createChild({
|
|
38
37
|
action: "Rollback",
|
|
39
38
|
reorgChain: reorgChain,
|
|
40
39
|
targetBlockNumber: rollbackTargetBlockNumber
|
|
@@ -55,20 +54,34 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
|
|
|
55
54
|
eventsProcessedDiffByChain[diff.chain_id] = eventsProcessedDiff;
|
|
56
55
|
newProgressBlockNumberPerChain[diff.chain_id] = rollbackTargetCheckpointId === 0n && diff.chain_id === reorgChain ? Primitive_int.min(diff.new_progress_block_number, rollbackTargetBlockNumber) : diff.new_progress_block_number;
|
|
57
56
|
}
|
|
57
|
+
let rolledBackChains = [];
|
|
58
58
|
Utils.Dict.forEach(IndexerState.chainStates(state), cs => {
|
|
59
59
|
let chainId = ChainState.chainConfig(cs).id;
|
|
60
|
+
let fromBlock = ChainState.committedProgressBlockNumber(cs);
|
|
60
61
|
ChainState.rollback(cs, newProgressBlockNumberPerChain[chainId], eventsProcessedDiffByChain[chainId], rollbackTargetBlockNumber, chainId === reorgChain);
|
|
62
|
+
let toBlock = ChainState.committedProgressBlockNumber(cs);
|
|
63
|
+
if (fromBlock !== toBlock) {
|
|
64
|
+
rolledBackChains.push({
|
|
65
|
+
chainId: chainId,
|
|
66
|
+
fromBlock: fromBlock,
|
|
67
|
+
toBlock: toBlock,
|
|
68
|
+
rollbackedEvents: Stdlib_Option.getOr(eventsProcessedDiffByChain[chainId], 0)
|
|
69
|
+
});
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
61
72
|
});
|
|
62
73
|
let diff$1 = await InMemoryStore.prepareRollbackDiff(state, rollbackTargetCheckpointId, IndexerState.committedCheckpointId(state) + 1n, newProgressBlockNumberPerChain);
|
|
74
|
+
rolledBackChains.forEach(chain => Logging.childInfo(logger, {
|
|
75
|
+
msg: "Rollbacked",
|
|
76
|
+
chainId: chain.chainId,
|
|
77
|
+
fromBlock: chain.fromBlock,
|
|
78
|
+
toBlock: chain.toBlock,
|
|
79
|
+
rollbackedEvents: chain.rollbackedEvents
|
|
80
|
+
}));
|
|
63
81
|
Logging.childTrace(logger, {
|
|
64
|
-
msg: "
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
upserted: diff$1.setEntities
|
|
68
|
-
},
|
|
69
|
-
rollbackedEvents: rollbackedProcessedEvents,
|
|
70
|
-
beforeCheckpointId: IndexerState.committedCheckpointId(state),
|
|
71
|
-
targetCheckpointId: rollbackTargetCheckpointId
|
|
82
|
+
msg: "Rollback entity changes",
|
|
83
|
+
deleted: diff$1.deletedEntities,
|
|
84
|
+
upserted: diff$1.setEntities
|
|
72
85
|
});
|
|
73
86
|
Prometheus.RollbackSuccess.increment(Performance.secondsSince(startTime), rollbackedProcessedEvents);
|
|
74
87
|
IndexerState.completeRollback(state, eventsProcessedDiffByChain);
|
|
@@ -97,7 +110,7 @@ async function rollback(state, scheduleFetch, scheduleProcessing, scheduleRollba
|
|
|
97
110
|
return scheduleRollback();
|
|
98
111
|
case "FoundReorgDepth" :
|
|
99
112
|
if (IndexerState.isProcessing(state)) {
|
|
100
|
-
return Logging.
|
|
113
|
+
return Logging.trace("Waiting for batch to finish processing before executing rollback");
|
|
101
114
|
} else {
|
|
102
115
|
return await executeRollback(state, match.chain, match.rollbackTargetBlockNumber, scheduleFetch, scheduleProcessing);
|
|
103
116
|
}
|