envio 3.3.0-alpha.12 → 3.3.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/package.json +6 -6
- package/src/ChainState.res +20 -8
- package/src/ChainState.res.mjs +13 -12
- package/src/ChainState.resi +1 -1
- package/src/CrossChainState.res +38 -33
- package/src/CrossChainState.res.mjs +9 -12
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/FetchState.res +15 -1
- package/src/FetchState.res.mjs +6 -1
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +10 -19
- package/src/IndexerState.res.mjs +7 -6
- package/src/IndexerState.resi +1 -9
- package/src/Internal.res +87 -12
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +28 -12
- package/src/Writing.res.mjs +15 -7
- package/src/bindings/NodeJs.res +5 -0
package/src/Envio.res.mjs
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Address from "./Address.res.mjs";
|
|
4
4
|
import * as Process from "process";
|
|
5
|
-
import * as Internal from "./Internal.res.mjs";
|
|
6
5
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
6
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
7
7
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
8
8
|
|
|
9
9
|
function durationToMs(duration) {
|
|
@@ -18,41 +18,36 @@ function durationToMs(duration) {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
+
let effectNameRe = /^[A-Za-z0-9_-][A-Za-z0-9_.-]*$/;
|
|
22
|
+
|
|
21
23
|
function createEffect(options, handler) {
|
|
24
|
+
if (!effectNameRe.test(options.name)) {
|
|
25
|
+
Stdlib_JsError.throwWithMessage(`Invalid effect name "` + options.name + `". Effect names may contain letters, numbers, underscores, hyphens and dots (but must not start with a dot or contain a path separator), because the name is used as the cache table name and cache file path.`);
|
|
26
|
+
}
|
|
22
27
|
let outputSchema = S$RescriptSchema.schema(param => options.output);
|
|
23
28
|
let itemSchema = S$RescriptSchema.schema(s => ({
|
|
24
29
|
id: s.m(S$RescriptSchema.string),
|
|
25
30
|
output: s.m(outputSchema)
|
|
26
31
|
}));
|
|
27
32
|
let match = options.cache;
|
|
28
|
-
let match$1 = options.
|
|
33
|
+
let match$1 = options.crossChain;
|
|
34
|
+
let match$2 = options.rateLimit;
|
|
29
35
|
let tmp;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
tmp = {
|
|
35
|
-
callsPerDuration: calls,
|
|
36
|
-
durationMs: durationToMs(match$1.per),
|
|
37
|
-
availableCalls: calls,
|
|
38
|
-
windowStartTime: Date.now(),
|
|
39
|
-
queueCount: 0,
|
|
40
|
-
nextWindowPromise: undefined
|
|
41
|
-
};
|
|
42
|
-
}
|
|
36
|
+
tmp = match$2 === false ? undefined : ({
|
|
37
|
+
callsPerDuration: match$2.calls,
|
|
38
|
+
durationMs: durationToMs(match$2.per)
|
|
39
|
+
});
|
|
43
40
|
return {
|
|
44
41
|
name: options.name,
|
|
45
42
|
handler: handler,
|
|
46
43
|
storageMeta: {
|
|
47
44
|
itemSchema: itemSchema,
|
|
48
|
-
outputSchema: outputSchema
|
|
49
|
-
table: Internal.makeCacheTable(options.name)
|
|
45
|
+
outputSchema: outputSchema
|
|
50
46
|
},
|
|
51
47
|
defaultShouldCache: match !== undefined ? match : false,
|
|
48
|
+
crossChain: match$1 !== undefined ? match$1 : true,
|
|
52
49
|
output: outputSchema,
|
|
53
50
|
input: S$RescriptSchema.schema(param => options.input),
|
|
54
|
-
activeCallsCount: 0,
|
|
55
|
-
prevCallStartTimerRef: null,
|
|
56
51
|
rateLimit: tmp
|
|
57
52
|
};
|
|
58
53
|
}
|
|
@@ -102,6 +97,7 @@ let TestHelpers = {
|
|
|
102
97
|
|
|
103
98
|
export {
|
|
104
99
|
durationToMs,
|
|
100
|
+
effectNameRe,
|
|
105
101
|
createEffect,
|
|
106
102
|
isNonInteractive,
|
|
107
103
|
TestHelpers,
|
package/src/FetchState.res
CHANGED
|
@@ -1465,6 +1465,11 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
|
|
|
1465
1465
|
// Most parallel in-flight chunk queries a single partition may have at once.
|
|
1466
1466
|
let maxPendingChunksPerPartition = 12
|
|
1467
1467
|
|
|
1468
|
+
// Most parallel in-flight queries a single chain may have at once, across all
|
|
1469
|
+
// its partitions. Bounds source load on chains with many partitions, where the
|
|
1470
|
+
// per-partition cap alone would admit thousands of concurrent queries.
|
|
1471
|
+
let maxChainConcurrency = 100
|
|
1472
|
+
|
|
1468
1473
|
// Generates candidate queries for a gap range (a hole left between
|
|
1469
1474
|
// completed/pending chunks, e.g. from an out-of-order partial response). Gaps
|
|
1470
1475
|
// carry the range's low fromBlock, so the acceptance pass takes them before
|
|
@@ -1939,12 +1944,21 @@ let getNextQuery = (
|
|
|
1939
1944
|
let streamCount = acceptanceStream->Array.length
|
|
1940
1945
|
let remainingBudget = ref(chainTargetItems)
|
|
1941
1946
|
let acceptIdx = ref(0)
|
|
1942
|
-
|
|
1947
|
+
// In-flight queries count against the chain's concurrency cap alongside the
|
|
1948
|
+
// ones accepted this tick; once the cap is reached no later candidate can be
|
|
1949
|
+
// accepted (they're only later in fromBlock order), so the walk stops.
|
|
1950
|
+
let usedConcurrency = ref(reservations->Array.length)
|
|
1951
|
+
while (
|
|
1952
|
+
remainingBudget.contents > 0. &&
|
|
1953
|
+
acceptIdx.contents < streamCount &&
|
|
1954
|
+
usedConcurrency.contents < maxChainConcurrency
|
|
1955
|
+
) {
|
|
1943
1956
|
let (_, itemsEst, maybeQuery) = acceptanceStream->Array.getUnsafe(acceptIdx.contents)
|
|
1944
1957
|
switch maybeQuery {
|
|
1945
1958
|
| Some(query) =>
|
|
1946
1959
|
let partitionIdx = partitionIndexById->Dict.getUnsafe(query.partitionId)
|
|
1947
1960
|
queriesByPartitionIndex->Array.getUnsafe(partitionIdx)->Array.push(query)->ignore
|
|
1961
|
+
usedConcurrency := usedConcurrency.contents + 1
|
|
1948
1962
|
| None => ()
|
|
1949
1963
|
}
|
|
1950
1964
|
remainingBudget := remainingBudget.contents -. itemsEst->Int.toFloat
|
package/src/FetchState.res.mjs
CHANGED
|
@@ -1331,12 +1331,14 @@ function getNextQuery(param, chainTargetBlock, chainTargetItems, $staropt$star)
|
|
|
1331
1331
|
let streamCount = acceptanceStream.length;
|
|
1332
1332
|
let remainingBudget = chainTargetItems;
|
|
1333
1333
|
let acceptIdx = 0;
|
|
1334
|
-
|
|
1334
|
+
let usedConcurrency = reservations.length;
|
|
1335
|
+
while (remainingBudget > 0 && acceptIdx < streamCount && usedConcurrency < 100) {
|
|
1335
1336
|
let match$1 = acceptanceStream[acceptIdx];
|
|
1336
1337
|
let maybeQuery = match$1[2];
|
|
1337
1338
|
if (maybeQuery !== undefined) {
|
|
1338
1339
|
let partitionIdx = partitionIndexById[maybeQuery.partitionId];
|
|
1339
1340
|
queriesByPartitionIndex[partitionIdx].push(maybeQuery);
|
|
1341
|
+
usedConcurrency = usedConcurrency + 1 | 0;
|
|
1340
1342
|
}
|
|
1341
1343
|
remainingBudget = remainingBudget - match$1[1];
|
|
1342
1344
|
acceptIdx = acceptIdx + 1 | 0;
|
|
@@ -1769,6 +1771,8 @@ let blockItemLogIndex = 16777216;
|
|
|
1769
1771
|
|
|
1770
1772
|
let maxPendingChunksPerPartition = 12;
|
|
1771
1773
|
|
|
1774
|
+
let maxChainConcurrency = 100;
|
|
1775
|
+
|
|
1772
1776
|
export {
|
|
1773
1777
|
deriveContractNameByAddress,
|
|
1774
1778
|
densityItemsTarget,
|
|
@@ -1794,6 +1798,7 @@ export {
|
|
|
1794
1798
|
handleQueryResult,
|
|
1795
1799
|
startFetchingQueries,
|
|
1796
1800
|
maxPendingChunksPerPartition,
|
|
1801
|
+
maxChainConcurrency,
|
|
1797
1802
|
pushGapFillQueries,
|
|
1798
1803
|
getNextQuery,
|
|
1799
1804
|
hasReadyItem,
|
package/src/InMemoryStore.res
CHANGED
|
@@ -8,25 +8,13 @@ let getInMemTable = (
|
|
|
8
8
|
): InMemoryTable.Entity.t =>
|
|
9
9
|
state->IndexerState.entities->IndexerState.EntityTables.get(~entityName=entityConfig.name)
|
|
10
10
|
|
|
11
|
-
let getEffectInMemTable = (
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
| None =>
|
|
17
|
-
let table: IndexerState.effectCacheInMemTable = {
|
|
18
|
-
idsToStore: [],
|
|
19
|
-
dict: Dict.make(),
|
|
20
|
-
changesCount: 0.,
|
|
21
|
-
invalidationsCount: 0,
|
|
22
|
-
effect,
|
|
23
|
-
}
|
|
24
|
-
effects->Dict.set(key, table)
|
|
25
|
-
table
|
|
26
|
-
}
|
|
27
|
-
}
|
|
11
|
+
let getEffectInMemTable = (
|
|
12
|
+
state: IndexerState.t,
|
|
13
|
+
~effect: Internal.effect,
|
|
14
|
+
~scope: Internal.chainScope,
|
|
15
|
+
) => state->IndexerState.effectState->EffectState.getTable(~effect, ~scope)
|
|
28
16
|
|
|
29
|
-
let hasEffectOutput = (inMemTable:
|
|
17
|
+
let hasEffectOutput = (inMemTable: EffectState.effectCacheInMemTable, key) =>
|
|
30
18
|
switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
31
19
|
| Some(Set(_)) => true
|
|
32
20
|
| Some(Delete(_)) | None => false
|
|
@@ -36,7 +24,7 @@ let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
|
|
|
36
24
|
// optional output, so it must never be wrapped in another option here: Some(None)
|
|
37
25
|
// is encoded as the nested-option sentinel and would leak to the handler.
|
|
38
26
|
let getEffectOutputUnsafe = (
|
|
39
|
-
inMemTable:
|
|
27
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
40
28
|
key,
|
|
41
29
|
): Internal.effectOutput =>
|
|
42
30
|
switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
@@ -47,7 +35,7 @@ let getEffectOutputUnsafe = (
|
|
|
47
35
|
// Records a handler output. Persisted on the next write only when shouldCache;
|
|
48
36
|
// otherwise kept in memory (re-run on a later miss) but never written to the db.
|
|
49
37
|
let setEffectOutput = (
|
|
50
|
-
inMemTable:
|
|
38
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
51
39
|
~checkpointId,
|
|
52
40
|
~cacheKey,
|
|
53
41
|
~output,
|
|
@@ -65,7 +53,7 @@ let setEffectOutput = (
|
|
|
65
53
|
|
|
66
54
|
// Seeds an entry from a db read. Stamped with loadedFromDbCheckpointId so it's
|
|
67
55
|
// always droppable (re-readable from the db) and never re-persisted.
|
|
68
|
-
let initEffectOutputFromDb = (inMemTable:
|
|
56
|
+
let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~cacheKey, ~output) =>
|
|
69
57
|
if inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(cacheKey)->Option.isNone {
|
|
70
58
|
inMemTable.changesCount = inMemTable.changesCount +. 1.
|
|
71
59
|
inMemTable.dict->Dict.set(
|
|
@@ -78,7 +66,7 @@ let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~c
|
|
|
78
66
|
// cache:false). Uncommitted entries stay warm. With keepLoadedFromDb, entries
|
|
79
67
|
// seeded from a db read are spared. Mirrors entity dropCommittedChanges.
|
|
80
68
|
let dropCommittedEffects = (
|
|
81
|
-
inMemTable:
|
|
69
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
82
70
|
~committedCheckpointId,
|
|
83
71
|
~keepLoadedFromDb,
|
|
84
72
|
) => {
|
|
@@ -117,17 +105,17 @@ let prepareRollbackDiff = async (
|
|
|
117
105
|
->Array.map(async entityConfig => {
|
|
118
106
|
let entityTable = state->getInMemTable(~entityConfig)
|
|
119
107
|
|
|
120
|
-
let (
|
|
108
|
+
let (removedIds, restoredEntitiesResult) = await persistence.storage.getRollbackData(
|
|
121
109
|
~entityConfig,
|
|
122
110
|
~rollbackTargetCheckpointId,
|
|
123
111
|
)
|
|
124
112
|
|
|
125
|
-
|
|
126
|
-
deletedEntities->Utils.Dict.push(entityConfig.name,
|
|
113
|
+
removedIds->Array.forEach(entityId => {
|
|
114
|
+
deletedEntities->Utils.Dict.push(entityConfig.name, entityId)
|
|
127
115
|
entityTable->InMemoryTable.Entity.set(
|
|
128
116
|
~committedCheckpointId,
|
|
129
117
|
Delete({
|
|
130
|
-
entityId
|
|
118
|
+
entityId,
|
|
131
119
|
checkpointId: rollbackDiffCheckpointId,
|
|
132
120
|
}),
|
|
133
121
|
)
|
|
@@ -4,6 +4,7 @@ import * as Table from "./db/Table.res.mjs";
|
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Config from "./Config.res.mjs";
|
|
6
6
|
import * as Internal from "./Internal.res.mjs";
|
|
7
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
7
8
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
8
9
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
9
10
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
@@ -13,22 +14,8 @@ function getInMemTable(state, entityConfig) {
|
|
|
13
14
|
return IndexerState.EntityTables.get(IndexerState.entities(state), entityConfig.name);
|
|
14
15
|
}
|
|
15
16
|
|
|
16
|
-
function getEffectInMemTable(state, effect) {
|
|
17
|
-
|
|
18
|
-
let effects = IndexerState.effects(state);
|
|
19
|
-
let table = effects[key];
|
|
20
|
-
if (table !== undefined) {
|
|
21
|
-
return table;
|
|
22
|
-
}
|
|
23
|
-
let table$1 = {
|
|
24
|
-
idsToStore: [],
|
|
25
|
-
invalidationsCount: 0,
|
|
26
|
-
dict: {},
|
|
27
|
-
changesCount: 0,
|
|
28
|
-
effect: effect
|
|
29
|
-
};
|
|
30
|
-
effects[key] = table$1;
|
|
31
|
-
return table$1;
|
|
17
|
+
function getEffectInMemTable(state, effect, scope) {
|
|
18
|
+
return EffectState.getTable(IndexerState.effectState(state), effect, scope);
|
|
32
19
|
}
|
|
33
20
|
|
|
34
21
|
function hasEffectOutput(inMemTable, key) {
|
|
@@ -102,11 +89,11 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
|
|
|
102
89
|
await Promise.all(persistence.allEntities.map(async entityConfig => {
|
|
103
90
|
let entityTable = getInMemTable(state, entityConfig);
|
|
104
91
|
let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
|
|
105
|
-
match[0].forEach(
|
|
106
|
-
Utils.Dict.push(deletedEntities, entityConfig.name,
|
|
92
|
+
match[0].forEach(entityId => {
|
|
93
|
+
Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
|
|
107
94
|
InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
|
|
108
95
|
type: "DELETE",
|
|
109
|
-
entityId:
|
|
96
|
+
entityId: entityId,
|
|
110
97
|
checkpointId: rollbackDiffCheckpointId
|
|
111
98
|
});
|
|
112
99
|
});
|
package/src/IndexerState.res
CHANGED
|
@@ -45,26 +45,13 @@ module EntityTables = {
|
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
type effectCacheInMemTable = {
|
|
49
|
-
// Cache keys whose handler output is persisted on the next write. Drained
|
|
50
|
-
// each write; eviction is driven by the per-entry checkpointId instead.
|
|
51
|
-
mutable idsToStore: array<string>,
|
|
52
|
-
mutable invalidationsCount: int,
|
|
53
|
-
// Each entry is stamped with the checkpoint that referenced it (or
|
|
54
|
-
// loadedFromDbCheckpointId for db reads), so committed entries can be
|
|
55
|
-
// dropped once persisted/re-derivable, mirroring entity changes.
|
|
56
|
-
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
57
|
-
mutable changesCount: float,
|
|
58
|
-
effect: Internal.effect,
|
|
59
|
-
}
|
|
60
|
-
|
|
61
48
|
type t = {
|
|
62
49
|
config: Config.t,
|
|
63
50
|
persistence: Persistence.t,
|
|
64
51
|
// --- In-memory store: entity/effect tables and the pending-write queue. ---
|
|
65
52
|
allEntities: array<Internal.entityConfig>,
|
|
66
53
|
mutable entities: EntityTables.t,
|
|
67
|
-
|
|
54
|
+
effectState: EffectState.t,
|
|
68
55
|
mutable rollback: option<Persistence.rollback>,
|
|
69
56
|
// Last checkpoint persisted to the db.
|
|
70
57
|
mutable committedCheckpointId: Internal.checkpointId,
|
|
@@ -143,7 +130,7 @@ let make = (
|
|
|
143
130
|
persistence,
|
|
144
131
|
allEntities: persistence.allEntities,
|
|
145
132
|
entities: EntityTables.make(persistence.allEntities),
|
|
146
|
-
|
|
133
|
+
effectState: EffectState.make(),
|
|
147
134
|
rollback: None,
|
|
148
135
|
committedCheckpointId,
|
|
149
136
|
processedCheckpointId: committedCheckpointId,
|
|
@@ -211,8 +198,12 @@ let makeFromDbState = (
|
|
|
211
198
|
|
|
212
199
|
Prometheus.ProcessingMaxBatchSize.set(~maxBatchSize=config.batchSize)
|
|
213
200
|
Prometheus.ReorgThreshold.set(~isInReorgThreshold)
|
|
214
|
-
initialState.cache->Utils.Dict.forEach(({effectName, count}) => {
|
|
215
|
-
Prometheus.EffectCacheCount.set(
|
|
201
|
+
initialState.cache->Utils.Dict.forEach(({effectName, count, scope}) => {
|
|
202
|
+
Prometheus.EffectCacheCount.set(
|
|
203
|
+
~count,
|
|
204
|
+
~effectName,
|
|
205
|
+
~scope=scope->Internal.EffectCache.scopeToString,
|
|
206
|
+
)
|
|
216
207
|
})
|
|
217
208
|
|
|
218
209
|
// updateSyncTimeOnRestart wipes the saved timestamp so a restart re-enters
|
|
@@ -385,7 +376,7 @@ let config = (state: t) => state.config
|
|
|
385
376
|
let persistence = (state: t) => state.persistence
|
|
386
377
|
let allEntities = (state: t) => state.allEntities
|
|
387
378
|
let entities = (state: t) => state.entities
|
|
388
|
-
let
|
|
379
|
+
let effectState = (state: t) => state.effectState
|
|
389
380
|
let committedCheckpointId = (state: t) => state.committedCheckpointId
|
|
390
381
|
let processedCheckpointId = (state: t) => state.processedCheckpointId
|
|
391
382
|
let processedBatches = (state: t) => state.processedBatches
|
|
@@ -485,7 +476,7 @@ let beginRollbackDiff = (
|
|
|
485
476
|
~progressBlockNumberByChainId,
|
|
486
477
|
) => {
|
|
487
478
|
state.entities = EntityTables.make(state.allEntities)
|
|
488
|
-
state.
|
|
479
|
+
state.effectState->EffectState.resetForRollback
|
|
489
480
|
state.rollback = Some({
|
|
490
481
|
targetCheckpointId,
|
|
491
482
|
diffCheckpointId,
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import * as Internal from "./Internal.res.mjs";
|
|
|
9
9
|
import * as Throttler from "./Throttler.res.mjs";
|
|
10
10
|
import * as ChainState from "./ChainState.res.mjs";
|
|
11
11
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
12
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
12
13
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
13
14
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
14
15
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
@@ -70,7 +71,7 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
70
71
|
persistence: persistence,
|
|
71
72
|
allEntities: persistence.allEntities,
|
|
72
73
|
entities: make$1(persistence.allEntities),
|
|
73
|
-
|
|
74
|
+
effectState: EffectState.make(),
|
|
74
75
|
rollback: undefined,
|
|
75
76
|
committedCheckpointId: committedCheckpointId,
|
|
76
77
|
processedCheckpointId: committedCheckpointId,
|
|
@@ -116,7 +117,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
116
117
|
Prometheus.ReorgThreshold.set(isInReorgThreshold);
|
|
117
118
|
Utils.Dict.forEach(initialState.cache, param => {
|
|
118
119
|
let count = param.count;
|
|
119
|
-
Prometheus.EffectCacheCount.set(count, param.effectName);
|
|
120
|
+
Prometheus.EffectCacheCount.set(count, param.effectName, Internal.EffectCache.scopeToString(param.scope));
|
|
120
121
|
});
|
|
121
122
|
let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
|
|
122
123
|
let chainStates = {};
|
|
@@ -266,8 +267,8 @@ function entities(state) {
|
|
|
266
267
|
return state.entities;
|
|
267
268
|
}
|
|
268
269
|
|
|
269
|
-
function
|
|
270
|
-
return state.
|
|
270
|
+
function effectState(state) {
|
|
271
|
+
return state.effectState;
|
|
271
272
|
}
|
|
272
273
|
|
|
273
274
|
function committedCheckpointId(state) {
|
|
@@ -419,7 +420,7 @@ function markCommitted(state, upToCheckpointId) {
|
|
|
419
420
|
|
|
420
421
|
function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
|
|
421
422
|
state.entities = make$1(state.allEntities);
|
|
422
|
-
state.
|
|
423
|
+
EffectState.resetForRollback(state.effectState);
|
|
423
424
|
state.rollback = {
|
|
424
425
|
targetCheckpointId: targetCheckpointId,
|
|
425
426
|
diffCheckpointId: diffCheckpointId,
|
|
@@ -512,7 +513,7 @@ export {
|
|
|
512
513
|
persistence,
|
|
513
514
|
allEntities,
|
|
514
515
|
entities,
|
|
515
|
-
|
|
516
|
+
effectState,
|
|
516
517
|
committedCheckpointId,
|
|
517
518
|
processedCheckpointId,
|
|
518
519
|
processedBatches,
|
package/src/IndexerState.resi
CHANGED
|
@@ -16,14 +16,6 @@ module EntityTables: {
|
|
|
16
16
|
let get: (t, ~entityName: string) => InMemoryTable.Entity.t
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
type effectCacheInMemTable = {
|
|
20
|
-
mutable idsToStore: array<string>,
|
|
21
|
-
mutable invalidationsCount: int,
|
|
22
|
-
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
23
|
-
mutable changesCount: float,
|
|
24
|
-
effect: Internal.effect,
|
|
25
|
-
}
|
|
26
|
-
|
|
27
19
|
type t
|
|
28
20
|
|
|
29
21
|
let make: (
|
|
@@ -86,7 +78,7 @@ let config: t => Config.t
|
|
|
86
78
|
let persistence: t => Persistence.t
|
|
87
79
|
let allEntities: t => array<Internal.entityConfig>
|
|
88
80
|
let entities: t => EntityTables.t
|
|
89
|
-
let
|
|
81
|
+
let effectState: t => EffectState.t
|
|
90
82
|
let committedCheckpointId: t => Internal.checkpointId
|
|
91
83
|
let processedCheckpointId: t => Internal.checkpointId
|
|
92
84
|
let processedBatches: t => array<Batch.t>
|
package/src/Internal.res
CHANGED
|
@@ -752,33 +752,108 @@ type effectCacheItem = {id: string, output: effectOutput}
|
|
|
752
752
|
type effectCacheStorageMeta = {
|
|
753
753
|
itemSchema: S.t<effectCacheItem>,
|
|
754
754
|
outputSchema: S.t<effectOutput>,
|
|
755
|
-
table: Table.table,
|
|
756
755
|
}
|
|
757
|
-
type
|
|
756
|
+
type rateLimitOptions = {
|
|
758
757
|
callsPerDuration: int,
|
|
759
758
|
durationMs: int,
|
|
760
|
-
mutable availableCalls: int,
|
|
761
|
-
mutable windowStartTime: float,
|
|
762
|
-
mutable queueCount: int,
|
|
763
|
-
mutable nextWindowPromise: option<promise<unit>>,
|
|
764
759
|
}
|
|
765
760
|
type effect = {
|
|
766
761
|
name: string,
|
|
767
762
|
handler: effectArgs => promise<effectOutput>,
|
|
768
763
|
storageMeta: effectCacheStorageMeta,
|
|
769
764
|
defaultShouldCache: bool,
|
|
765
|
+
// When true (the default) a single cache is shared across every chain and the
|
|
766
|
+
// handler must not read context.chain. When false the cache is isolated per
|
|
767
|
+
// chain and context.chain.id is available.
|
|
768
|
+
crossChain: bool,
|
|
770
769
|
output: S.t<effectOutput>,
|
|
771
770
|
input: S.t<effectInput>,
|
|
772
|
-
|
|
773
|
-
mutable activeCallsCount: int,
|
|
774
|
-
mutable prevCallStartTimerRef: Performance.timeRef,
|
|
775
|
-
rateLimit: option<rateLimitState>,
|
|
771
|
+
rateLimit: option<rateLimitOptions>,
|
|
776
772
|
}
|
|
773
|
+
|
|
774
|
+
// Whether some piece of data (currently an effect cache; entities in a future
|
|
775
|
+
// version) is shared across every chain or isolated to a single chain. Unboxed:
|
|
776
|
+
// `CrossChain` is the string "crossChain" and `Chain(id)` is the raw chain id,
|
|
777
|
+
// discriminated by runtime type.
|
|
778
|
+
@unboxed
|
|
779
|
+
type chainScope =
|
|
780
|
+
| @as("crossChain") CrossChain
|
|
781
|
+
| Chain(int)
|
|
782
|
+
|
|
777
783
|
let cacheTablePrefix = "envio_effect_"
|
|
784
|
+
|
|
785
|
+
// The single reversible mapping between an effect's (name, scope) and its
|
|
786
|
+
// canonical Postgres cache-table name and .envio/cache file path. Everything
|
|
787
|
+
// that needs a cache address goes through here instead of slicing prefixes.
|
|
788
|
+
// CrossChain -> envio_effect_<name> <name>.tsv
|
|
789
|
+
// Chain(1) -> envio_1_effect_<name> 1/<name>.tsv
|
|
790
|
+
// Chain(137) -> envio_137_effect_<name> 137/<name>.tsv
|
|
791
|
+
module EffectCache = {
|
|
792
|
+
let toTableName = (~effectName, ~scope) =>
|
|
793
|
+
switch scope {
|
|
794
|
+
| CrossChain => cacheTablePrefix ++ effectName
|
|
795
|
+
| Chain(chainId) => `envio_${chainId->Int.toString}_effect_${effectName}`
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
|
|
799
|
+
let scopeToString = scope =>
|
|
800
|
+
switch scope {
|
|
801
|
+
| CrossChain => "crossChain"
|
|
802
|
+
| Chain(chainId) => chainId->Int.toString
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
|
|
806
|
+
// Int.fromString alone follows parseInt semantics and accepts both.
|
|
807
|
+
let parseChainId = str =>
|
|
808
|
+
switch Int.fromString(str) {
|
|
809
|
+
| Some(chainId) if chainId >= 0 && chainId->Int.toString === str => Some(chainId)
|
|
810
|
+
| _ => None
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
let chainScopedRe = /^envio_([0-9]+)_effect_(.+)$/
|
|
814
|
+
let crossChainRe = /^envio_effect_(.+)$/
|
|
815
|
+
|
|
816
|
+
// Inverse of toTableName. Returns None for any table name that isn't a cache
|
|
817
|
+
// table. Chain-scoped is tried first: the `_effect_` separator keeps effect
|
|
818
|
+
// names that themselves start with digits unambiguous.
|
|
819
|
+
let fromTableName = (tableName): option<(string, chainScope)> =>
|
|
820
|
+
switch RegExp.exec(chainScopedRe, tableName) {
|
|
821
|
+
| Some(result) =>
|
|
822
|
+
switch (
|
|
823
|
+
RegExp.Result.matches(result)->Array.get(0),
|
|
824
|
+
RegExp.Result.matches(result)->Array.get(1),
|
|
825
|
+
) {
|
|
826
|
+
| (Some(Some(chainIdStr)), Some(Some(effectName))) =>
|
|
827
|
+
switch parseChainId(chainIdStr) {
|
|
828
|
+
| Some(chainId) => Some((effectName, Chain(chainId)))
|
|
829
|
+
| None => None
|
|
830
|
+
}
|
|
831
|
+
| _ => None
|
|
832
|
+
}
|
|
833
|
+
| None =>
|
|
834
|
+
switch RegExp.exec(crossChainRe, tableName) {
|
|
835
|
+
| Some(result) =>
|
|
836
|
+
switch RegExp.Result.matches(result)->Array.get(0) {
|
|
837
|
+
| Some(Some(effectName)) => Some((effectName, CrossChain))
|
|
838
|
+
| _ => None
|
|
839
|
+
}
|
|
840
|
+
| None => None
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
// Relative posix path within .envio/cache. Chain-scoped caches live one
|
|
845
|
+
// directory level deep, named by chain id.
|
|
846
|
+
let toCachePath = (~effectName, ~scope) =>
|
|
847
|
+
switch scope {
|
|
848
|
+
| CrossChain => effectName ++ ".tsv"
|
|
849
|
+
| Chain(chainId) => `${chainId->Int.toString}/${effectName}.tsv`
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
778
853
|
let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<JSON.t> => S.t<effectOutput>)
|
|
779
|
-
let makeCacheTable = (~effectName) => {
|
|
854
|
+
let makeCacheTable = (~effectName, ~scope) => {
|
|
780
855
|
Table.mkTable(
|
|
781
|
-
|
|
856
|
+
EffectCache.toTableName(~effectName, ~scope),
|
|
782
857
|
~fields=[
|
|
783
858
|
Table.mkField("id", String, ~fieldSchema=S.string, ~isPrimaryKey=true),
|
|
784
859
|
Table.mkField("output", Json, ~fieldSchema=cacheOutputSchema, ~isNullable=true),
|
package/src/Internal.res.mjs
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
import * as Table from "./db/Table.res.mjs";
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Address from "./Address.res.mjs";
|
|
6
|
+
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
7
|
+
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
6
8
|
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
7
9
|
|
|
8
10
|
let allEvmBlockFields = [
|
|
@@ -166,10 +168,101 @@ let fuelTransferParamsSchema = S$RescriptSchema.schema(s => ({
|
|
|
166
168
|
|
|
167
169
|
let cacheTablePrefix = "envio_effect_";
|
|
168
170
|
|
|
171
|
+
function toTableName(effectName, scope) {
|
|
172
|
+
if (scope === "crossChain") {
|
|
173
|
+
return cacheTablePrefix + effectName;
|
|
174
|
+
} else {
|
|
175
|
+
return `envio_` + scope.toString() + `_effect_` + effectName;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function scopeToString(scope) {
|
|
180
|
+
if (scope === "crossChain") {
|
|
181
|
+
return "crossChain";
|
|
182
|
+
} else {
|
|
183
|
+
return scope.toString();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function parseChainId(str) {
|
|
188
|
+
let chainId = Stdlib_Int.fromString(str, undefined);
|
|
189
|
+
if (chainId !== undefined && chainId >= 0 && chainId.toString() === str) {
|
|
190
|
+
return chainId;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let chainScopedRe = /^envio_([0-9]+)_effect_(.+)$/;
|
|
195
|
+
|
|
196
|
+
let crossChainRe = /^envio_effect_(.+)$/;
|
|
197
|
+
|
|
198
|
+
function fromTableName(tableName) {
|
|
199
|
+
let result = chainScopedRe.exec(tableName);
|
|
200
|
+
if (result == null) {
|
|
201
|
+
let result$1 = crossChainRe.exec(tableName);
|
|
202
|
+
if (result$1 == null) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
let match = result$1.slice(1)[0];
|
|
206
|
+
if (match === undefined) {
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
let effectName = Primitive_option.valFromOption(match);
|
|
210
|
+
if (effectName !== undefined) {
|
|
211
|
+
return [
|
|
212
|
+
effectName,
|
|
213
|
+
"crossChain"
|
|
214
|
+
];
|
|
215
|
+
} else {
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
let match$1 = result.slice(1)[0];
|
|
220
|
+
let match$2 = result.slice(1)[1];
|
|
221
|
+
if (match$1 === undefined) {
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
let chainIdStr = Primitive_option.valFromOption(match$1);
|
|
225
|
+
if (chainIdStr === undefined) {
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
if (match$2 === undefined) {
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
let effectName$1 = Primitive_option.valFromOption(match$2);
|
|
232
|
+
if (effectName$1 === undefined) {
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
let chainId = parseChainId(chainIdStr);
|
|
236
|
+
if (chainId !== undefined) {
|
|
237
|
+
return [
|
|
238
|
+
effectName$1,
|
|
239
|
+
chainId
|
|
240
|
+
];
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function toCachePath(effectName, scope) {
|
|
245
|
+
if (scope === "crossChain") {
|
|
246
|
+
return effectName + ".tsv";
|
|
247
|
+
} else {
|
|
248
|
+
return scope.toString() + `/` + effectName + `.tsv`;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
let EffectCache = {
|
|
253
|
+
toTableName: toTableName,
|
|
254
|
+
scopeToString: scopeToString,
|
|
255
|
+
parseChainId: parseChainId,
|
|
256
|
+
chainScopedRe: chainScopedRe,
|
|
257
|
+
crossChainRe: crossChainRe,
|
|
258
|
+
fromTableName: fromTableName,
|
|
259
|
+
toCachePath: toCachePath
|
|
260
|
+
};
|
|
261
|
+
|
|
169
262
|
let cacheOutputSchema = S$RescriptSchema.json(false);
|
|
170
263
|
|
|
171
|
-
function makeCacheTable(effectName) {
|
|
172
|
-
return Table.mkTable(
|
|
264
|
+
function makeCacheTable(effectName, scope) {
|
|
265
|
+
return Table.mkTable(toTableName(effectName, scope), undefined, [
|
|
173
266
|
Table.mkField("id", "String", S$RescriptSchema.string, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined),
|
|
174
267
|
Table.mkField("output", "Json", cacheOutputSchema, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined)
|
|
175
268
|
], undefined);
|
|
@@ -195,6 +288,7 @@ export {
|
|
|
195
288
|
fuelSupplyParamsSchema,
|
|
196
289
|
fuelTransferParamsSchema,
|
|
197
290
|
cacheTablePrefix,
|
|
291
|
+
EffectCache,
|
|
198
292
|
cacheOutputSchema,
|
|
199
293
|
makeCacheTable,
|
|
200
294
|
loadedFromDbCheckpointId,
|