envio 3.3.0-alpha.9 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +9 -8
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +0 -7
- package/src/BatchProcessing.res.mjs +1 -8
- package/src/ChainFetching.res +30 -18
- package/src/ChainFetching.res.mjs +23 -13
- package/src/ChainState.res +134 -55
- package/src/ChainState.res.mjs +78 -36
- package/src/ChainState.resi +14 -2
- package/src/Config.res +9 -5
- package/src/Config.res.mjs +2 -2
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +115 -34
- package/src/CrossChainState.res.mjs +37 -14
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/EventConfigBuilder.res +21 -46
- package/src/EventConfigBuilder.res.mjs +18 -25
- package/src/EventProcessing.res +8 -5
- package/src/EventProcessing.res.mjs +2 -1
- package/src/FetchState.res +814 -467
- package/src/FetchState.res.mjs +541 -369
- package/src/HandlerRegister.res +3 -1
- package/src/HandlerRegister.res.mjs +3 -2
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +15 -39
- package/src/IndexerState.res.mjs +18 -29
- package/src/IndexerState.resi +2 -10
- package/src/Internal.res +98 -14
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/LogSelection.res +0 -62
- package/src/LogSelection.res.mjs +0 -74
- package/src/Metrics.res +1 -1
- package/src/Metrics.res.mjs +1 -1
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/PruneStaleHistory.res +146 -35
- package/src/PruneStaleHistory.res.mjs +114 -20
- package/src/RawEvent.res +2 -2
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/SimulateDeadInputTracker.res +1 -1
- package/src/SimulateItems.res +38 -6
- package/src/SimulateItems.res.mjs +28 -9
- package/src/TestIndexer.res +2 -2
- package/src/TestIndexer.res.mjs +2 -2
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +60 -21
- package/src/Writing.res.mjs +27 -9
- package/src/bindings/NodeJs.res +5 -0
- package/src/bindings/Viem.res +0 -5
- package/src/sources/EventRouter.res +0 -21
- package/src/sources/EventRouter.res.mjs +0 -12
- package/src/sources/EvmChain.res +2 -27
- package/src/sources/EvmChain.res.mjs +2 -23
- package/src/sources/EvmRpcClient.res +58 -23
- package/src/sources/EvmRpcClient.res.mjs +11 -5
- package/src/sources/HyperSync.res +9 -38
- package/src/sources/HyperSync.res.mjs +16 -28
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +88 -11
- package/src/sources/HyperSyncClient.res.mjs +39 -6
- package/src/sources/HyperSyncSource.res +18 -199
- package/src/sources/HyperSyncSource.res.mjs +9 -128
- package/src/sources/Rpc.res +0 -32
- package/src/sources/Rpc.res.mjs +1 -46
- package/src/sources/RpcSource.res +129 -522
- package/src/sources/RpcSource.res.mjs +161 -366
- package/src/sources/SimulateSource.res +37 -19
- package/src/sources/SimulateSource.res.mjs +27 -10
- package/src/sources/SourceManager.res +3 -7
- package/src/sources/SourceManager.res.mjs +2 -7
- package/src/sources/SourceManager.resi +0 -2
- package/src/sources/SvmHyperSyncSource.res +13 -3
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
package/src/HandlerRegister.res
CHANGED
|
@@ -679,7 +679,9 @@ let finishRegistration = (~config: Config.t): registrationsByChainId => {
|
|
|
679
679
|
config.ecosystem.name === Evm &&
|
|
680
680
|
(registration->getResolvedWhere).topicSelections->Utils.Array.isEmpty
|
|
681
681
|
if !isDroppedByWhere {
|
|
682
|
-
onEventRegistrations
|
|
682
|
+
onEventRegistrations
|
|
683
|
+
->Array.push({...registration, index: onEventRegistrations->Array.length})
|
|
684
|
+
->ignore
|
|
683
685
|
}
|
|
684
686
|
| None => ()
|
|
685
687
|
}
|
|
@@ -388,10 +388,11 @@ function finishRegistration(config) {
|
|
|
388
388
|
return;
|
|
389
389
|
}
|
|
390
390
|
let isDroppedByWhere = config.ecosystem.name === "evm" && Utils.$$Array.isEmpty(registration$1.resolvedWhere.topicSelections);
|
|
391
|
-
if (
|
|
392
|
-
onEventRegistrations.push(registration$1);
|
|
391
|
+
if (isDroppedByWhere) {
|
|
393
392
|
return;
|
|
394
393
|
}
|
|
394
|
+
let newrecord = {...registration$1};
|
|
395
|
+
onEventRegistrations.push((newrecord.index = onEventRegistrations.length, newrecord));
|
|
395
396
|
});
|
|
396
397
|
});
|
|
397
398
|
registrationsByChainId[key] = {
|
package/src/InMemoryStore.res
CHANGED
|
@@ -8,25 +8,13 @@ let getInMemTable = (
|
|
|
8
8
|
): InMemoryTable.Entity.t =>
|
|
9
9
|
state->IndexerState.entities->IndexerState.EntityTables.get(~entityName=entityConfig.name)
|
|
10
10
|
|
|
11
|
-
let getEffectInMemTable = (
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
| None =>
|
|
17
|
-
let table: IndexerState.effectCacheInMemTable = {
|
|
18
|
-
idsToStore: [],
|
|
19
|
-
dict: Dict.make(),
|
|
20
|
-
changesCount: 0.,
|
|
21
|
-
invalidationsCount: 0,
|
|
22
|
-
effect,
|
|
23
|
-
}
|
|
24
|
-
effects->Dict.set(key, table)
|
|
25
|
-
table
|
|
26
|
-
}
|
|
27
|
-
}
|
|
11
|
+
let getEffectInMemTable = (
|
|
12
|
+
state: IndexerState.t,
|
|
13
|
+
~effect: Internal.effect,
|
|
14
|
+
~scope: Internal.chainScope,
|
|
15
|
+
) => state->IndexerState.effectState->EffectState.getTable(~effect, ~scope)
|
|
28
16
|
|
|
29
|
-
let hasEffectOutput = (inMemTable:
|
|
17
|
+
let hasEffectOutput = (inMemTable: EffectState.effectCacheInMemTable, key) =>
|
|
30
18
|
switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
31
19
|
| Some(Set(_)) => true
|
|
32
20
|
| Some(Delete(_)) | None => false
|
|
@@ -36,7 +24,7 @@ let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
|
|
|
36
24
|
// optional output, so it must never be wrapped in another option here: Some(None)
|
|
37
25
|
// is encoded as the nested-option sentinel and would leak to the handler.
|
|
38
26
|
let getEffectOutputUnsafe = (
|
|
39
|
-
inMemTable:
|
|
27
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
40
28
|
key,
|
|
41
29
|
): Internal.effectOutput =>
|
|
42
30
|
switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
@@ -47,7 +35,7 @@ let getEffectOutputUnsafe = (
|
|
|
47
35
|
// Records a handler output. Persisted on the next write only when shouldCache;
|
|
48
36
|
// otherwise kept in memory (re-run on a later miss) but never written to the db.
|
|
49
37
|
let setEffectOutput = (
|
|
50
|
-
inMemTable:
|
|
38
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
51
39
|
~checkpointId,
|
|
52
40
|
~cacheKey,
|
|
53
41
|
~output,
|
|
@@ -65,7 +53,7 @@ let setEffectOutput = (
|
|
|
65
53
|
|
|
66
54
|
// Seeds an entry from a db read. Stamped with loadedFromDbCheckpointId so it's
|
|
67
55
|
// always droppable (re-readable from the db) and never re-persisted.
|
|
68
|
-
let initEffectOutputFromDb = (inMemTable:
|
|
56
|
+
let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~cacheKey, ~output) =>
|
|
69
57
|
if inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(cacheKey)->Option.isNone {
|
|
70
58
|
inMemTable.changesCount = inMemTable.changesCount +. 1.
|
|
71
59
|
inMemTable.dict->Dict.set(
|
|
@@ -78,7 +66,7 @@ let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~c
|
|
|
78
66
|
// cache:false). Uncommitted entries stay warm. With keepLoadedFromDb, entries
|
|
79
67
|
// seeded from a db read are spared. Mirrors entity dropCommittedChanges.
|
|
80
68
|
let dropCommittedEffects = (
|
|
81
|
-
inMemTable:
|
|
69
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
82
70
|
~committedCheckpointId,
|
|
83
71
|
~keepLoadedFromDb,
|
|
84
72
|
) => {
|
|
@@ -117,17 +105,17 @@ let prepareRollbackDiff = async (
|
|
|
117
105
|
->Array.map(async entityConfig => {
|
|
118
106
|
let entityTable = state->getInMemTable(~entityConfig)
|
|
119
107
|
|
|
120
|
-
let (
|
|
108
|
+
let (removedIds, restoredEntitiesResult) = await persistence.storage.getRollbackData(
|
|
121
109
|
~entityConfig,
|
|
122
110
|
~rollbackTargetCheckpointId,
|
|
123
111
|
)
|
|
124
112
|
|
|
125
|
-
|
|
126
|
-
deletedEntities->Utils.Dict.push(entityConfig.name,
|
|
113
|
+
removedIds->Array.forEach(entityId => {
|
|
114
|
+
deletedEntities->Utils.Dict.push(entityConfig.name, entityId)
|
|
127
115
|
entityTable->InMemoryTable.Entity.set(
|
|
128
116
|
~committedCheckpointId,
|
|
129
117
|
Delete({
|
|
130
|
-
entityId
|
|
118
|
+
entityId,
|
|
131
119
|
checkpointId: rollbackDiffCheckpointId,
|
|
132
120
|
}),
|
|
133
121
|
)
|
|
@@ -4,6 +4,7 @@ import * as Table from "./db/Table.res.mjs";
|
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Config from "./Config.res.mjs";
|
|
6
6
|
import * as Internal from "./Internal.res.mjs";
|
|
7
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
7
8
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
8
9
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
9
10
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
@@ -13,22 +14,8 @@ function getInMemTable(state, entityConfig) {
|
|
|
13
14
|
return IndexerState.EntityTables.get(IndexerState.entities(state), entityConfig.name);
|
|
14
15
|
}
|
|
15
16
|
|
|
16
|
-
function getEffectInMemTable(state, effect) {
|
|
17
|
-
|
|
18
|
-
let effects = IndexerState.effects(state);
|
|
19
|
-
let table = effects[key];
|
|
20
|
-
if (table !== undefined) {
|
|
21
|
-
return table;
|
|
22
|
-
}
|
|
23
|
-
let table$1 = {
|
|
24
|
-
idsToStore: [],
|
|
25
|
-
invalidationsCount: 0,
|
|
26
|
-
dict: {},
|
|
27
|
-
changesCount: 0,
|
|
28
|
-
effect: effect
|
|
29
|
-
};
|
|
30
|
-
effects[key] = table$1;
|
|
31
|
-
return table$1;
|
|
17
|
+
function getEffectInMemTable(state, effect, scope) {
|
|
18
|
+
return EffectState.getTable(IndexerState.effectState(state), effect, scope);
|
|
32
19
|
}
|
|
33
20
|
|
|
34
21
|
function hasEffectOutput(inMemTable, key) {
|
|
@@ -102,11 +89,11 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
|
|
|
102
89
|
await Promise.all(persistence.allEntities.map(async entityConfig => {
|
|
103
90
|
let entityTable = getInMemTable(state, entityConfig);
|
|
104
91
|
let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
|
|
105
|
-
match[0].forEach(
|
|
106
|
-
Utils.Dict.push(deletedEntities, entityConfig.name,
|
|
92
|
+
match[0].forEach(entityId => {
|
|
93
|
+
Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
|
|
107
94
|
InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
|
|
108
95
|
type: "DELETE",
|
|
109
|
-
entityId:
|
|
96
|
+
entityId: entityId,
|
|
110
97
|
checkpointId: rollbackDiffCheckpointId
|
|
111
98
|
});
|
|
112
99
|
});
|
package/src/IndexerState.res
CHANGED
|
@@ -6,23 +6,6 @@ type rollbackState =
|
|
|
6
6
|
| FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
|
|
7
7
|
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
8
8
|
|
|
9
|
-
module WriteThrottlers = {
|
|
10
|
-
type t = {pruneStaleEntityHistory: Throttler.t}
|
|
11
|
-
let make = (): t => {
|
|
12
|
-
let pruneStaleEntityHistory = {
|
|
13
|
-
let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis
|
|
14
|
-
let logger = Logging.createChild(
|
|
15
|
-
~params={
|
|
16
|
-
"context": "Throttler for pruning stale entity history data",
|
|
17
|
-
"intervalMillis": intervalMillis,
|
|
18
|
-
},
|
|
19
|
-
)
|
|
20
|
-
Throttler.make(~intervalMillis, ~logger)
|
|
21
|
-
}
|
|
22
|
-
{pruneStaleEntityHistory: pruneStaleEntityHistory}
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
|
|
26
9
|
module EntityTables = {
|
|
27
10
|
type t = dict<InMemoryTable.Entity.t>
|
|
28
11
|
exception UndefinedEntity({entityName: string})
|
|
@@ -45,26 +28,13 @@ module EntityTables = {
|
|
|
45
28
|
}
|
|
46
29
|
}
|
|
47
30
|
|
|
48
|
-
type effectCacheInMemTable = {
|
|
49
|
-
// Cache keys whose handler output is persisted on the next write. Drained
|
|
50
|
-
// each write; eviction is driven by the per-entry checkpointId instead.
|
|
51
|
-
mutable idsToStore: array<string>,
|
|
52
|
-
mutable invalidationsCount: int,
|
|
53
|
-
// Each entry is stamped with the checkpoint that referenced it (or
|
|
54
|
-
// loadedFromDbCheckpointId for db reads), so committed entries can be
|
|
55
|
-
// dropped once persisted/re-derivable, mirroring entity changes.
|
|
56
|
-
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
57
|
-
mutable changesCount: float,
|
|
58
|
-
effect: Internal.effect,
|
|
59
|
-
}
|
|
60
|
-
|
|
61
31
|
type t = {
|
|
62
32
|
config: Config.t,
|
|
63
33
|
persistence: Persistence.t,
|
|
64
34
|
// --- In-memory store: entity/effect tables and the pending-write queue. ---
|
|
65
35
|
allEntities: array<Internal.entityConfig>,
|
|
66
36
|
mutable entities: EntityTables.t,
|
|
67
|
-
|
|
37
|
+
effectState: EffectState.t,
|
|
68
38
|
mutable rollback: option<Persistence.rollback>,
|
|
69
39
|
// Last checkpoint persisted to the db.
|
|
70
40
|
mutable committedCheckpointId: Internal.checkpointId,
|
|
@@ -94,7 +64,9 @@ type t = {
|
|
|
94
64
|
crossChainState: CrossChainState.t,
|
|
95
65
|
mutable rollbackState: rollbackState,
|
|
96
66
|
indexerStartTime: Date.t,
|
|
97
|
-
|
|
67
|
+
// When an entity's history was last pruned. No key = never pruned yet,
|
|
68
|
+
// which counts as overdue.
|
|
69
|
+
lastPrunedAtMillis: dict<float>,
|
|
98
70
|
loadManager: LoadManager.t,
|
|
99
71
|
keepProcessAlive: bool,
|
|
100
72
|
exitAfterFirstEventBlock: bool,
|
|
@@ -143,7 +115,7 @@ let make = (
|
|
|
143
115
|
persistence,
|
|
144
116
|
allEntities: persistence.allEntities,
|
|
145
117
|
entities: EntityTables.make(persistence.allEntities),
|
|
146
|
-
|
|
118
|
+
effectState: EffectState.make(),
|
|
147
119
|
rollback: None,
|
|
148
120
|
committedCheckpointId,
|
|
149
121
|
processedCheckpointId: committedCheckpointId,
|
|
@@ -164,7 +136,7 @@ let make = (
|
|
|
164
136
|
),
|
|
165
137
|
indexerStartTime: Date.make(),
|
|
166
138
|
rollbackState: NoRollback,
|
|
167
|
-
|
|
139
|
+
lastPrunedAtMillis: Dict.make(),
|
|
168
140
|
loadManager: LoadManager.make(),
|
|
169
141
|
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
170
142
|
exitAfterFirstEventBlock,
|
|
@@ -211,8 +183,12 @@ let makeFromDbState = (
|
|
|
211
183
|
|
|
212
184
|
Prometheus.ProcessingMaxBatchSize.set(~maxBatchSize=config.batchSize)
|
|
213
185
|
Prometheus.ReorgThreshold.set(~isInReorgThreshold)
|
|
214
|
-
initialState.cache->Utils.Dict.forEach(({effectName, count}) => {
|
|
215
|
-
Prometheus.EffectCacheCount.set(
|
|
186
|
+
initialState.cache->Utils.Dict.forEach(({effectName, count, scope}) => {
|
|
187
|
+
Prometheus.EffectCacheCount.set(
|
|
188
|
+
~count,
|
|
189
|
+
~effectName,
|
|
190
|
+
~scope=scope->Internal.EffectCache.scopeToString,
|
|
191
|
+
)
|
|
216
192
|
})
|
|
217
193
|
|
|
218
194
|
// updateSyncTimeOnRestart wipes the saved timestamp so a restart re-enters
|
|
@@ -385,7 +361,7 @@ let config = (state: t) => state.config
|
|
|
385
361
|
let persistence = (state: t) => state.persistence
|
|
386
362
|
let allEntities = (state: t) => state.allEntities
|
|
387
363
|
let entities = (state: t) => state.entities
|
|
388
|
-
let
|
|
364
|
+
let effectState = (state: t) => state.effectState
|
|
389
365
|
let committedCheckpointId = (state: t) => state.committedCheckpointId
|
|
390
366
|
let processedCheckpointId = (state: t) => state.processedCheckpointId
|
|
391
367
|
let processedBatches = (state: t) => state.processedBatches
|
|
@@ -405,7 +381,7 @@ let keepProcessAlive = (state: t) => state.keepProcessAlive
|
|
|
405
381
|
let exitAfterFirstEventBlock = (state: t) => state.exitAfterFirstEventBlock
|
|
406
382
|
let isStopped = (state: t) => state.isStopped
|
|
407
383
|
let epoch = (state: t) => state.epoch
|
|
408
|
-
let
|
|
384
|
+
let lastPrunedAtMillis = (state: t) => state.lastPrunedAtMillis
|
|
409
385
|
let simulateDeadInputTracker = (state: t) => state.simulateDeadInputTracker
|
|
410
386
|
|
|
411
387
|
// --- Store domain operations. ---
|
|
@@ -485,7 +461,7 @@ let beginRollbackDiff = (
|
|
|
485
461
|
~progressBlockNumberByChainId,
|
|
486
462
|
) => {
|
|
487
463
|
state.entities = EntityTables.make(state.allEntities)
|
|
488
|
-
state.
|
|
464
|
+
state.effectState->EffectState.resetForRollback
|
|
489
465
|
state.rollback = Some({
|
|
490
466
|
targetCheckpointId,
|
|
491
467
|
diffCheckpointId,
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import * as Internal from "./Internal.res.mjs";
|
|
|
9
9
|
import * as Throttler from "./Throttler.res.mjs";
|
|
10
10
|
import * as ChainState from "./ChainState.res.mjs";
|
|
11
11
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
12
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
12
13
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
13
14
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
14
15
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
@@ -20,21 +21,9 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
|
|
|
20
21
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
21
22
|
import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
|
|
22
23
|
|
|
23
|
-
function make() {
|
|
24
|
-
let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
|
|
25
|
-
let logger = Logging.createChild({
|
|
26
|
-
context: "Throttler for pruning stale entity history data",
|
|
27
|
-
intervalMillis: intervalMillis
|
|
28
|
-
});
|
|
29
|
-
let pruneStaleEntityHistory = Throttler.make(intervalMillis, logger);
|
|
30
|
-
return {
|
|
31
|
-
pruneStaleEntityHistory: pruneStaleEntityHistory
|
|
32
|
-
};
|
|
33
|
-
}
|
|
34
|
-
|
|
35
24
|
let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("IndexerState.EntityTables.UndefinedEntity");
|
|
36
25
|
|
|
37
|
-
function make
|
|
26
|
+
function make(entities) {
|
|
38
27
|
let init = {};
|
|
39
28
|
entities.forEach(entityConfig => {
|
|
40
29
|
init[entityConfig.name] = InMemoryTable.Entity.make();
|
|
@@ -54,7 +43,7 @@ function get(self, entityName) {
|
|
|
54
43
|
}
|
|
55
44
|
}
|
|
56
45
|
|
|
57
|
-
function make$
|
|
46
|
+
function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
|
|
58
47
|
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
|
|
59
48
|
let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
|
|
60
49
|
let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
|
|
@@ -69,8 +58,8 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
69
58
|
config: config,
|
|
70
59
|
persistence: persistence,
|
|
71
60
|
allEntities: persistence.allEntities,
|
|
72
|
-
entities: make
|
|
73
|
-
|
|
61
|
+
entities: make(persistence.allEntities),
|
|
62
|
+
effectState: EffectState.make(),
|
|
74
63
|
rollback: undefined,
|
|
75
64
|
committedCheckpointId: committedCheckpointId,
|
|
76
65
|
processedCheckpointId: committedCheckpointId,
|
|
@@ -86,7 +75,7 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
86
75
|
crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, targetBufferSize),
|
|
87
76
|
rollbackState: "NoRollback",
|
|
88
77
|
indexerStartTime: new Date(),
|
|
89
|
-
|
|
78
|
+
lastPrunedAtMillis: {},
|
|
90
79
|
loadManager: LoadManager.make(),
|
|
91
80
|
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
92
81
|
exitAfterFirstEventBlock: exitAfterFirstEventBlock,
|
|
@@ -116,7 +105,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
116
105
|
Prometheus.ReorgThreshold.set(isInReorgThreshold);
|
|
117
106
|
Utils.Dict.forEach(initialState.cache, param => {
|
|
118
107
|
let count = param.count;
|
|
119
|
-
Prometheus.EffectCacheCount.set(count, param.effectName);
|
|
108
|
+
Prometheus.EffectCacheCount.set(count, param.effectName, Internal.EffectCache.scopeToString(param.scope));
|
|
120
109
|
});
|
|
121
110
|
let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
|
|
122
111
|
let chainStates = {};
|
|
@@ -142,7 +131,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
142
131
|
if (allChainsReady.contents) {
|
|
143
132
|
Prometheus.ProgressReady.setAllReady();
|
|
144
133
|
}
|
|
145
|
-
return make$
|
|
134
|
+
return make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
|
|
146
135
|
}
|
|
147
136
|
|
|
148
137
|
function isStale(state, stateId) {
|
|
@@ -266,8 +255,8 @@ function entities(state) {
|
|
|
266
255
|
return state.entities;
|
|
267
256
|
}
|
|
268
257
|
|
|
269
|
-
function
|
|
270
|
-
return state.
|
|
258
|
+
function effectState(state) {
|
|
259
|
+
return state.effectState;
|
|
271
260
|
}
|
|
272
261
|
|
|
273
262
|
function committedCheckpointId(state) {
|
|
@@ -346,8 +335,8 @@ function epoch(state) {
|
|
|
346
335
|
return state.epoch;
|
|
347
336
|
}
|
|
348
337
|
|
|
349
|
-
function
|
|
350
|
-
return state.
|
|
338
|
+
function lastPrunedAtMillis(state) {
|
|
339
|
+
return state.lastPrunedAtMillis;
|
|
351
340
|
}
|
|
352
341
|
|
|
353
342
|
function simulateDeadInputTracker(state) {
|
|
@@ -418,8 +407,8 @@ function markCommitted(state, upToCheckpointId) {
|
|
|
418
407
|
}
|
|
419
408
|
|
|
420
409
|
function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
|
|
421
|
-
state.entities = make
|
|
422
|
-
state.
|
|
410
|
+
state.entities = make(state.allEntities);
|
|
411
|
+
EffectState.resetForRollback(state.effectState);
|
|
423
412
|
state.rollback = {
|
|
424
413
|
targetCheckpointId: targetCheckpointId,
|
|
425
414
|
diffCheckpointId: diffCheckpointId,
|
|
@@ -478,7 +467,7 @@ function takeChainMetaSnapshot(state) {
|
|
|
478
467
|
}
|
|
479
468
|
|
|
480
469
|
let EntityTables = {
|
|
481
|
-
make: make
|
|
470
|
+
make: make,
|
|
482
471
|
get: get
|
|
483
472
|
};
|
|
484
473
|
|
|
@@ -486,7 +475,7 @@ let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
|
|
|
486
475
|
|
|
487
476
|
export {
|
|
488
477
|
EntityTables,
|
|
489
|
-
make$
|
|
478
|
+
make$1 as make,
|
|
490
479
|
makeFromDbState,
|
|
491
480
|
unexpectedErrorMsg,
|
|
492
481
|
isStale,
|
|
@@ -512,7 +501,7 @@ export {
|
|
|
512
501
|
persistence,
|
|
513
502
|
allEntities,
|
|
514
503
|
entities,
|
|
515
|
-
|
|
504
|
+
effectState,
|
|
516
505
|
committedCheckpointId,
|
|
517
506
|
processedCheckpointId,
|
|
518
507
|
processedBatches,
|
|
@@ -532,7 +521,7 @@ export {
|
|
|
532
521
|
exitAfterFirstEventBlock,
|
|
533
522
|
isStopped,
|
|
534
523
|
epoch,
|
|
535
|
-
|
|
524
|
+
lastPrunedAtMillis,
|
|
536
525
|
simulateDeadInputTracker,
|
|
537
526
|
queueProcessedBatch,
|
|
538
527
|
drainBatchRun,
|
package/src/IndexerState.resi
CHANGED
|
@@ -16,14 +16,6 @@ module EntityTables: {
|
|
|
16
16
|
let get: (t, ~entityName: string) => InMemoryTable.Entity.t
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
type effectCacheInMemTable = {
|
|
20
|
-
mutable idsToStore: array<string>,
|
|
21
|
-
mutable invalidationsCount: int,
|
|
22
|
-
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
23
|
-
mutable changesCount: float,
|
|
24
|
-
effect: Internal.effect,
|
|
25
|
-
}
|
|
26
|
-
|
|
27
19
|
type t
|
|
28
20
|
|
|
29
21
|
let make: (
|
|
@@ -86,7 +78,7 @@ let config: t => Config.t
|
|
|
86
78
|
let persistence: t => Persistence.t
|
|
87
79
|
let allEntities: t => array<Internal.entityConfig>
|
|
88
80
|
let entities: t => EntityTables.t
|
|
89
|
-
let
|
|
81
|
+
let effectState: t => EffectState.t
|
|
90
82
|
let committedCheckpointId: t => Internal.checkpointId
|
|
91
83
|
let processedCheckpointId: t => Internal.checkpointId
|
|
92
84
|
let processedBatches: t => array<Batch.t>
|
|
@@ -106,7 +98,7 @@ let keepProcessAlive: t => bool
|
|
|
106
98
|
let exitAfterFirstEventBlock: t => bool
|
|
107
99
|
let isStopped: t => bool
|
|
108
100
|
let epoch: t => int
|
|
109
|
-
let
|
|
101
|
+
let lastPrunedAtMillis: t => dict<float>
|
|
110
102
|
let simulateDeadInputTracker: t => option<SimulateDeadInputTracker.t>
|
|
111
103
|
|
|
112
104
|
// Store domain operations.
|
package/src/Internal.res
CHANGED
|
@@ -330,6 +330,9 @@ type eventPayload
|
|
|
330
330
|
@get external getPayloadBlock: eventPayload => Nullable.t<eventBlock> = "block"
|
|
331
331
|
@set external setPayloadBlock: (eventPayload, eventBlock) => unit = "block"
|
|
332
332
|
|
|
333
|
+
// The log's emitting address (EVM/Fuel; the program id carries it for SVM).
|
|
334
|
+
@get external getPayloadSrcAddress: eventPayload => Address.t = "srcAddress"
|
|
335
|
+
|
|
333
336
|
type genericLoaderArgs<'event, 'context> = {
|
|
334
337
|
event: 'event,
|
|
335
338
|
context: 'context,
|
|
@@ -555,6 +558,11 @@ type svmInstructionEventConfig = {
|
|
|
555
558
|
// must stay directly constructable), and the evm→base cast in sources is sound
|
|
556
559
|
// by ecosystem homogeneity — an EVM chain only ever holds `evmOnEventRegistration`s.
|
|
557
560
|
type onEventRegistration = {
|
|
561
|
+
// Chain-scoped sequential index — the registration's position in the
|
|
562
|
+
// chain's onEventRegistrations array, assigned when registration finishes
|
|
563
|
+
// (-1 until then). Native-routed items reference their registration by this
|
|
564
|
+
// index across the napi boundary; sources resolve it before creating an item.
|
|
565
|
+
index: int,
|
|
558
566
|
eventConfig: eventConfig,
|
|
559
567
|
handler: option<handler>,
|
|
560
568
|
contractRegister: option<contractRegister>,
|
|
@@ -601,8 +609,9 @@ type indexingAddress = {
|
|
|
601
609
|
|
|
602
610
|
type dcs = array<indexingAddress>
|
|
603
611
|
|
|
604
|
-
// Duplicate the type from item
|
|
605
|
-
//
|
|
612
|
+
// Duplicate the type from item to keep item properly unboxed. Runtime event
|
|
613
|
+
// items carry the registration their source already resolved from the
|
|
614
|
+
// ChainState-owned registration array.
|
|
606
615
|
type eventItem = private {
|
|
607
616
|
kind: [#0],
|
|
608
617
|
onEventRegistration: onEventRegistration,
|
|
@@ -743,33 +752,108 @@ type effectCacheItem = {id: string, output: effectOutput}
|
|
|
743
752
|
type effectCacheStorageMeta = {
|
|
744
753
|
itemSchema: S.t<effectCacheItem>,
|
|
745
754
|
outputSchema: S.t<effectOutput>,
|
|
746
|
-
table: Table.table,
|
|
747
755
|
}
|
|
748
|
-
type
|
|
756
|
+
type rateLimitOptions = {
|
|
749
757
|
callsPerDuration: int,
|
|
750
758
|
durationMs: int,
|
|
751
|
-
mutable availableCalls: int,
|
|
752
|
-
mutable windowStartTime: float,
|
|
753
|
-
mutable queueCount: int,
|
|
754
|
-
mutable nextWindowPromise: option<promise<unit>>,
|
|
755
759
|
}
|
|
756
760
|
type effect = {
|
|
757
761
|
name: string,
|
|
758
762
|
handler: effectArgs => promise<effectOutput>,
|
|
759
763
|
storageMeta: effectCacheStorageMeta,
|
|
760
764
|
defaultShouldCache: bool,
|
|
765
|
+
// When true (the default) a single cache is shared across every chain and the
|
|
766
|
+
// handler must not read context.chain. When false the cache is isolated per
|
|
767
|
+
// chain and context.chain.id is available.
|
|
768
|
+
crossChain: bool,
|
|
761
769
|
output: S.t<effectOutput>,
|
|
762
770
|
input: S.t<effectInput>,
|
|
763
|
-
|
|
764
|
-
mutable activeCallsCount: int,
|
|
765
|
-
mutable prevCallStartTimerRef: Performance.timeRef,
|
|
766
|
-
rateLimit: option<rateLimitState>,
|
|
771
|
+
rateLimit: option<rateLimitOptions>,
|
|
767
772
|
}
|
|
773
|
+
|
|
774
|
+
// Whether some piece of data (currently an effect cache; entities in a future
|
|
775
|
+
// version) is shared across every chain or isolated to a single chain. Unboxed:
|
|
776
|
+
// `CrossChain` is the string "crossChain" and `Chain(id)` is the raw chain id,
|
|
777
|
+
// discriminated by runtime type.
|
|
778
|
+
@unboxed
|
|
779
|
+
type chainScope =
|
|
780
|
+
| @as("crossChain") CrossChain
|
|
781
|
+
| Chain(int)
|
|
782
|
+
|
|
768
783
|
let cacheTablePrefix = "envio_effect_"
|
|
784
|
+
|
|
785
|
+
// The single reversible mapping between an effect's (name, scope) and its
|
|
786
|
+
// canonical Postgres cache-table name and .envio/cache file path. Everything
|
|
787
|
+
// that needs a cache address goes through here instead of slicing prefixes.
|
|
788
|
+
// CrossChain -> envio_effect_<name> <name>.tsv
|
|
789
|
+
// Chain(1) -> envio_1_effect_<name> 1/<name>.tsv
|
|
790
|
+
// Chain(137) -> envio_137_effect_<name> 137/<name>.tsv
|
|
791
|
+
module EffectCache = {
|
|
792
|
+
let toTableName = (~effectName, ~scope) =>
|
|
793
|
+
switch scope {
|
|
794
|
+
| CrossChain => cacheTablePrefix ++ effectName
|
|
795
|
+
| Chain(chainId) => `envio_${chainId->Int.toString}_effect_${effectName}`
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
|
|
799
|
+
let scopeToString = scope =>
|
|
800
|
+
switch scope {
|
|
801
|
+
| CrossChain => "crossChain"
|
|
802
|
+
| Chain(chainId) => chainId->Int.toString
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
|
|
806
|
+
// Int.fromString alone follows parseInt semantics and accepts both.
|
|
807
|
+
let parseChainId = str =>
|
|
808
|
+
switch Int.fromString(str) {
|
|
809
|
+
| Some(chainId) if chainId >= 0 && chainId->Int.toString === str => Some(chainId)
|
|
810
|
+
| _ => None
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
let chainScopedRe = /^envio_([0-9]+)_effect_(.+)$/
|
|
814
|
+
let crossChainRe = /^envio_effect_(.+)$/
|
|
815
|
+
|
|
816
|
+
// Inverse of toTableName. Returns None for any table name that isn't a cache
|
|
817
|
+
// table. Chain-scoped is tried first: the `_effect_` separator keeps effect
|
|
818
|
+
// names that themselves start with digits unambiguous.
|
|
819
|
+
let fromTableName = (tableName): option<(string, chainScope)> =>
|
|
820
|
+
switch RegExp.exec(chainScopedRe, tableName) {
|
|
821
|
+
| Some(result) =>
|
|
822
|
+
switch (
|
|
823
|
+
RegExp.Result.matches(result)->Array.get(0),
|
|
824
|
+
RegExp.Result.matches(result)->Array.get(1),
|
|
825
|
+
) {
|
|
826
|
+
| (Some(Some(chainIdStr)), Some(Some(effectName))) =>
|
|
827
|
+
switch parseChainId(chainIdStr) {
|
|
828
|
+
| Some(chainId) => Some((effectName, Chain(chainId)))
|
|
829
|
+
| None => None
|
|
830
|
+
}
|
|
831
|
+
| _ => None
|
|
832
|
+
}
|
|
833
|
+
| None =>
|
|
834
|
+
switch RegExp.exec(crossChainRe, tableName) {
|
|
835
|
+
| Some(result) =>
|
|
836
|
+
switch RegExp.Result.matches(result)->Array.get(0) {
|
|
837
|
+
| Some(Some(effectName)) => Some((effectName, CrossChain))
|
|
838
|
+
| _ => None
|
|
839
|
+
}
|
|
840
|
+
| None => None
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
// Relative posix path within .envio/cache. Chain-scoped caches live one
|
|
845
|
+
// directory level deep, named by chain id.
|
|
846
|
+
let toCachePath = (~effectName, ~scope) =>
|
|
847
|
+
switch scope {
|
|
848
|
+
| CrossChain => effectName ++ ".tsv"
|
|
849
|
+
| Chain(chainId) => `${chainId->Int.toString}/${effectName}.tsv`
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
769
853
|
let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<JSON.t> => S.t<effectOutput>)
|
|
770
|
-
let makeCacheTable = (~effectName) => {
|
|
854
|
+
let makeCacheTable = (~effectName, ~scope) => {
|
|
771
855
|
Table.mkTable(
|
|
772
|
-
|
|
856
|
+
EffectCache.toTableName(~effectName, ~scope),
|
|
773
857
|
~fields=[
|
|
774
858
|
Table.mkField("id", String, ~fieldSchema=S.string, ~isPrimaryKey=true),
|
|
775
859
|
Table.mkField("output", Json, ~fieldSchema=cacheOutputSchema, ~isNullable=true),
|