envio 3.3.0-alpha.8 → 3.3.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +9 -8
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +0 -7
- package/src/BatchProcessing.res.mjs +1 -8
- package/src/ChainFetching.res +30 -18
- package/src/ChainFetching.res.mjs +23 -13
- package/src/ChainState.res +205 -59
- package/src/ChainState.res.mjs +130 -27
- package/src/ChainState.resi +16 -1
- package/src/Config.res +9 -5
- package/src/Config.res.mjs +2 -2
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +147 -39
- package/src/CrossChainState.res.mjs +42 -10
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +30 -56
- package/src/EventConfigBuilder.res.mjs +24 -30
- package/src/EventProcessing.res +8 -5
- package/src/EventProcessing.res.mjs +2 -1
- package/src/FetchState.res +842 -474
- package/src/FetchState.res.mjs +545 -356
- package/src/HandlerLoader.res +1 -1
- package/src/HandlerLoader.res.mjs +1 -1
- package/src/HandlerRegister.res +478 -305
- package/src/HandlerRegister.res.mjs +277 -209
- package/src/HandlerRegister.resi +11 -6
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +15 -39
- package/src/IndexerState.res.mjs +18 -29
- package/src/IndexerState.resi +2 -10
- package/src/Internal.res +121 -18
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/LogSelection.res +92 -217
- package/src/LogSelection.res.mjs +95 -184
- package/src/Main.res +29 -143
- package/src/Main.res.mjs +25 -88
- package/src/Metrics.res +1 -1
- package/src/Metrics.res.mjs +1 -1
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/PruneStaleHistory.res +146 -35
- package/src/PruneStaleHistory.res.mjs +114 -20
- package/src/RawEvent.res +2 -2
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateDeadInputTracker.res +1 -1
- package/src/SimulateItems.res +39 -7
- package/src/SimulateItems.res.mjs +28 -9
- package/src/TestIndexer.res +2 -2
- package/src/TestIndexer.res.mjs +2 -2
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +60 -21
- package/src/Writing.res.mjs +27 -9
- package/src/bindings/NodeJs.res +5 -0
- package/src/bindings/Viem.res +0 -5
- package/src/sources/EventRouter.res +0 -21
- package/src/sources/EventRouter.res.mjs +0 -12
- package/src/sources/Evm.res +4 -1
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/EvmChain.res +2 -27
- package/src/sources/EvmChain.res.mjs +2 -23
- package/src/sources/EvmRpcClient.res +58 -23
- package/src/sources/EvmRpcClient.res.mjs +11 -5
- package/src/sources/Fuel.res +3 -1
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/HyperSync.res +9 -38
- package/src/sources/HyperSync.res.mjs +16 -28
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +88 -11
- package/src/sources/HyperSyncClient.res.mjs +39 -6
- package/src/sources/HyperSyncSource.res +18 -210
- package/src/sources/HyperSyncSource.res.mjs +9 -137
- package/src/sources/Rpc.res +0 -32
- package/src/sources/Rpc.res.mjs +1 -46
- package/src/sources/RpcSource.res +129 -533
- package/src/sources/RpcSource.res.mjs +161 -379
- package/src/sources/SimulateSource.res +37 -19
- package/src/sources/SimulateSource.res.mjs +27 -10
- package/src/sources/SourceManager.res +4 -14
- package/src/sources/SourceManager.res.mjs +3 -9
- package/src/sources/SourceManager.resi +0 -2
- package/src/sources/SvmHyperSyncSource.res +13 -3
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
package/src/InMemoryStore.res
CHANGED
|
@@ -8,25 +8,13 @@ let getInMemTable = (
|
|
|
8
8
|
): InMemoryTable.Entity.t =>
|
|
9
9
|
state->IndexerState.entities->IndexerState.EntityTables.get(~entityName=entityConfig.name)
|
|
10
10
|
|
|
11
|
-
let getEffectInMemTable = (
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
| None =>
|
|
17
|
-
let table: IndexerState.effectCacheInMemTable = {
|
|
18
|
-
idsToStore: [],
|
|
19
|
-
dict: Dict.make(),
|
|
20
|
-
changesCount: 0.,
|
|
21
|
-
invalidationsCount: 0,
|
|
22
|
-
effect,
|
|
23
|
-
}
|
|
24
|
-
effects->Dict.set(key, table)
|
|
25
|
-
table
|
|
26
|
-
}
|
|
27
|
-
}
|
|
11
|
+
let getEffectInMemTable = (
|
|
12
|
+
state: IndexerState.t,
|
|
13
|
+
~effect: Internal.effect,
|
|
14
|
+
~scope: Internal.chainScope,
|
|
15
|
+
) => state->IndexerState.effectState->EffectState.getTable(~effect, ~scope)
|
|
28
16
|
|
|
29
|
-
let hasEffectOutput = (inMemTable:
|
|
17
|
+
let hasEffectOutput = (inMemTable: EffectState.effectCacheInMemTable, key) =>
|
|
30
18
|
switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
31
19
|
| Some(Set(_)) => true
|
|
32
20
|
| Some(Delete(_)) | None => false
|
|
@@ -36,7 +24,7 @@ let hasEffectOutput = (inMemTable: IndexerState.effectCacheInMemTable, key) =>
|
|
|
36
24
|
// optional output, so it must never be wrapped in another option here: Some(None)
|
|
37
25
|
// is encoded as the nested-option sentinel and would leak to the handler.
|
|
38
26
|
let getEffectOutputUnsafe = (
|
|
39
|
-
inMemTable:
|
|
27
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
40
28
|
key,
|
|
41
29
|
): Internal.effectOutput =>
|
|
42
30
|
switch inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(key) {
|
|
@@ -47,7 +35,7 @@ let getEffectOutputUnsafe = (
|
|
|
47
35
|
// Records a handler output. Persisted on the next write only when shouldCache;
|
|
48
36
|
// otherwise kept in memory (re-run on a later miss) but never written to the db.
|
|
49
37
|
let setEffectOutput = (
|
|
50
|
-
inMemTable:
|
|
38
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
51
39
|
~checkpointId,
|
|
52
40
|
~cacheKey,
|
|
53
41
|
~output,
|
|
@@ -65,7 +53,7 @@ let setEffectOutput = (
|
|
|
65
53
|
|
|
66
54
|
// Seeds an entry from a db read. Stamped with loadedFromDbCheckpointId so it's
|
|
67
55
|
// always droppable (re-readable from the db) and never re-persisted.
|
|
68
|
-
let initEffectOutputFromDb = (inMemTable:
|
|
56
|
+
let initEffectOutputFromDb = (inMemTable: EffectState.effectCacheInMemTable, ~cacheKey, ~output) =>
|
|
69
57
|
if inMemTable.dict->Utils.Dict.dangerouslyGetNonOption(cacheKey)->Option.isNone {
|
|
70
58
|
inMemTable.changesCount = inMemTable.changesCount +. 1.
|
|
71
59
|
inMemTable.dict->Dict.set(
|
|
@@ -78,7 +66,7 @@ let initEffectOutputFromDb = (inMemTable: IndexerState.effectCacheInMemTable, ~c
|
|
|
78
66
|
// cache:false). Uncommitted entries stay warm. With keepLoadedFromDb, entries
|
|
79
67
|
// seeded from a db read are spared. Mirrors entity dropCommittedChanges.
|
|
80
68
|
let dropCommittedEffects = (
|
|
81
|
-
inMemTable:
|
|
69
|
+
inMemTable: EffectState.effectCacheInMemTable,
|
|
82
70
|
~committedCheckpointId,
|
|
83
71
|
~keepLoadedFromDb,
|
|
84
72
|
) => {
|
|
@@ -117,17 +105,17 @@ let prepareRollbackDiff = async (
|
|
|
117
105
|
->Array.map(async entityConfig => {
|
|
118
106
|
let entityTable = state->getInMemTable(~entityConfig)
|
|
119
107
|
|
|
120
|
-
let (
|
|
108
|
+
let (removedIds, restoredEntitiesResult) = await persistence.storage.getRollbackData(
|
|
121
109
|
~entityConfig,
|
|
122
110
|
~rollbackTargetCheckpointId,
|
|
123
111
|
)
|
|
124
112
|
|
|
125
|
-
|
|
126
|
-
deletedEntities->Utils.Dict.push(entityConfig.name,
|
|
113
|
+
removedIds->Array.forEach(entityId => {
|
|
114
|
+
deletedEntities->Utils.Dict.push(entityConfig.name, entityId)
|
|
127
115
|
entityTable->InMemoryTable.Entity.set(
|
|
128
116
|
~committedCheckpointId,
|
|
129
117
|
Delete({
|
|
130
|
-
entityId
|
|
118
|
+
entityId,
|
|
131
119
|
checkpointId: rollbackDiffCheckpointId,
|
|
132
120
|
}),
|
|
133
121
|
)
|
|
@@ -4,6 +4,7 @@ import * as Table from "./db/Table.res.mjs";
|
|
|
4
4
|
import * as Utils from "./Utils.res.mjs";
|
|
5
5
|
import * as Config from "./Config.res.mjs";
|
|
6
6
|
import * as Internal from "./Internal.res.mjs";
|
|
7
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
7
8
|
import * as IndexerState from "./IndexerState.res.mjs";
|
|
8
9
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
9
10
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
@@ -13,22 +14,8 @@ function getInMemTable(state, entityConfig) {
|
|
|
13
14
|
return IndexerState.EntityTables.get(IndexerState.entities(state), entityConfig.name);
|
|
14
15
|
}
|
|
15
16
|
|
|
16
|
-
function getEffectInMemTable(state, effect) {
|
|
17
|
-
|
|
18
|
-
let effects = IndexerState.effects(state);
|
|
19
|
-
let table = effects[key];
|
|
20
|
-
if (table !== undefined) {
|
|
21
|
-
return table;
|
|
22
|
-
}
|
|
23
|
-
let table$1 = {
|
|
24
|
-
idsToStore: [],
|
|
25
|
-
invalidationsCount: 0,
|
|
26
|
-
dict: {},
|
|
27
|
-
changesCount: 0,
|
|
28
|
-
effect: effect
|
|
29
|
-
};
|
|
30
|
-
effects[key] = table$1;
|
|
31
|
-
return table$1;
|
|
17
|
+
function getEffectInMemTable(state, effect, scope) {
|
|
18
|
+
return EffectState.getTable(IndexerState.effectState(state), effect, scope);
|
|
32
19
|
}
|
|
33
20
|
|
|
34
21
|
function hasEffectOutput(inMemTable, key) {
|
|
@@ -102,11 +89,11 @@ async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDi
|
|
|
102
89
|
await Promise.all(persistence.allEntities.map(async entityConfig => {
|
|
103
90
|
let entityTable = getInMemTable(state, entityConfig);
|
|
104
91
|
let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
|
|
105
|
-
match[0].forEach(
|
|
106
|
-
Utils.Dict.push(deletedEntities, entityConfig.name,
|
|
92
|
+
match[0].forEach(entityId => {
|
|
93
|
+
Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
|
|
107
94
|
InMemoryTable.Entity.set(entityTable, committedCheckpointId, {
|
|
108
95
|
type: "DELETE",
|
|
109
|
-
entityId:
|
|
96
|
+
entityId: entityId,
|
|
110
97
|
checkpointId: rollbackDiffCheckpointId
|
|
111
98
|
});
|
|
112
99
|
});
|
package/src/IndexerState.res
CHANGED
|
@@ -6,23 +6,6 @@ type rollbackState =
|
|
|
6
6
|
| FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
|
|
7
7
|
| RollbackReady({eventsProcessedDiffByChain: dict<float>})
|
|
8
8
|
|
|
9
|
-
module WriteThrottlers = {
|
|
10
|
-
type t = {pruneStaleEntityHistory: Throttler.t}
|
|
11
|
-
let make = (): t => {
|
|
12
|
-
let pruneStaleEntityHistory = {
|
|
13
|
-
let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis
|
|
14
|
-
let logger = Logging.createChild(
|
|
15
|
-
~params={
|
|
16
|
-
"context": "Throttler for pruning stale entity history data",
|
|
17
|
-
"intervalMillis": intervalMillis,
|
|
18
|
-
},
|
|
19
|
-
)
|
|
20
|
-
Throttler.make(~intervalMillis, ~logger)
|
|
21
|
-
}
|
|
22
|
-
{pruneStaleEntityHistory: pruneStaleEntityHistory}
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
|
|
26
9
|
module EntityTables = {
|
|
27
10
|
type t = dict<InMemoryTable.Entity.t>
|
|
28
11
|
exception UndefinedEntity({entityName: string})
|
|
@@ -45,26 +28,13 @@ module EntityTables = {
|
|
|
45
28
|
}
|
|
46
29
|
}
|
|
47
30
|
|
|
48
|
-
type effectCacheInMemTable = {
|
|
49
|
-
// Cache keys whose handler output is persisted on the next write. Drained
|
|
50
|
-
// each write; eviction is driven by the per-entry checkpointId instead.
|
|
51
|
-
mutable idsToStore: array<string>,
|
|
52
|
-
mutable invalidationsCount: int,
|
|
53
|
-
// Each entry is stamped with the checkpoint that referenced it (or
|
|
54
|
-
// loadedFromDbCheckpointId for db reads), so committed entries can be
|
|
55
|
-
// dropped once persisted/re-derivable, mirroring entity changes.
|
|
56
|
-
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
57
|
-
mutable changesCount: float,
|
|
58
|
-
effect: Internal.effect,
|
|
59
|
-
}
|
|
60
|
-
|
|
61
31
|
type t = {
|
|
62
32
|
config: Config.t,
|
|
63
33
|
persistence: Persistence.t,
|
|
64
34
|
// --- In-memory store: entity/effect tables and the pending-write queue. ---
|
|
65
35
|
allEntities: array<Internal.entityConfig>,
|
|
66
36
|
mutable entities: EntityTables.t,
|
|
67
|
-
|
|
37
|
+
effectState: EffectState.t,
|
|
68
38
|
mutable rollback: option<Persistence.rollback>,
|
|
69
39
|
// Last checkpoint persisted to the db.
|
|
70
40
|
mutable committedCheckpointId: Internal.checkpointId,
|
|
@@ -94,7 +64,9 @@ type t = {
|
|
|
94
64
|
crossChainState: CrossChainState.t,
|
|
95
65
|
mutable rollbackState: rollbackState,
|
|
96
66
|
indexerStartTime: Date.t,
|
|
97
|
-
|
|
67
|
+
// When an entity's history was last pruned. No key = never pruned yet,
|
|
68
|
+
// which counts as overdue.
|
|
69
|
+
lastPrunedAtMillis: dict<float>,
|
|
98
70
|
loadManager: LoadManager.t,
|
|
99
71
|
keepProcessAlive: bool,
|
|
100
72
|
exitAfterFirstEventBlock: bool,
|
|
@@ -143,7 +115,7 @@ let make = (
|
|
|
143
115
|
persistence,
|
|
144
116
|
allEntities: persistence.allEntities,
|
|
145
117
|
entities: EntityTables.make(persistence.allEntities),
|
|
146
|
-
|
|
118
|
+
effectState: EffectState.make(),
|
|
147
119
|
rollback: None,
|
|
148
120
|
committedCheckpointId,
|
|
149
121
|
processedCheckpointId: committedCheckpointId,
|
|
@@ -164,7 +136,7 @@ let make = (
|
|
|
164
136
|
),
|
|
165
137
|
indexerStartTime: Date.make(),
|
|
166
138
|
rollbackState: NoRollback,
|
|
167
|
-
|
|
139
|
+
lastPrunedAtMillis: Dict.make(),
|
|
168
140
|
loadManager: LoadManager.make(),
|
|
169
141
|
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
170
142
|
exitAfterFirstEventBlock,
|
|
@@ -211,8 +183,12 @@ let makeFromDbState = (
|
|
|
211
183
|
|
|
212
184
|
Prometheus.ProcessingMaxBatchSize.set(~maxBatchSize=config.batchSize)
|
|
213
185
|
Prometheus.ReorgThreshold.set(~isInReorgThreshold)
|
|
214
|
-
initialState.cache->Utils.Dict.forEach(({effectName, count}) => {
|
|
215
|
-
Prometheus.EffectCacheCount.set(
|
|
186
|
+
initialState.cache->Utils.Dict.forEach(({effectName, count, scope}) => {
|
|
187
|
+
Prometheus.EffectCacheCount.set(
|
|
188
|
+
~count,
|
|
189
|
+
~effectName,
|
|
190
|
+
~scope=scope->Internal.EffectCache.scopeToString,
|
|
191
|
+
)
|
|
216
192
|
})
|
|
217
193
|
|
|
218
194
|
// updateSyncTimeOnRestart wipes the saved timestamp so a restart re-enters
|
|
@@ -385,7 +361,7 @@ let config = (state: t) => state.config
|
|
|
385
361
|
let persistence = (state: t) => state.persistence
|
|
386
362
|
let allEntities = (state: t) => state.allEntities
|
|
387
363
|
let entities = (state: t) => state.entities
|
|
388
|
-
let
|
|
364
|
+
let effectState = (state: t) => state.effectState
|
|
389
365
|
let committedCheckpointId = (state: t) => state.committedCheckpointId
|
|
390
366
|
let processedCheckpointId = (state: t) => state.processedCheckpointId
|
|
391
367
|
let processedBatches = (state: t) => state.processedBatches
|
|
@@ -405,7 +381,7 @@ let keepProcessAlive = (state: t) => state.keepProcessAlive
|
|
|
405
381
|
let exitAfterFirstEventBlock = (state: t) => state.exitAfterFirstEventBlock
|
|
406
382
|
let isStopped = (state: t) => state.isStopped
|
|
407
383
|
let epoch = (state: t) => state.epoch
|
|
408
|
-
let
|
|
384
|
+
let lastPrunedAtMillis = (state: t) => state.lastPrunedAtMillis
|
|
409
385
|
let simulateDeadInputTracker = (state: t) => state.simulateDeadInputTracker
|
|
410
386
|
|
|
411
387
|
// --- Store domain operations. ---
|
|
@@ -485,7 +461,7 @@ let beginRollbackDiff = (
|
|
|
485
461
|
~progressBlockNumberByChainId,
|
|
486
462
|
) => {
|
|
487
463
|
state.entities = EntityTables.make(state.allEntities)
|
|
488
|
-
state.
|
|
464
|
+
state.effectState->EffectState.resetForRollback
|
|
489
465
|
state.rollback = Some({
|
|
490
466
|
targetCheckpointId,
|
|
491
467
|
diffCheckpointId,
|
package/src/IndexerState.res.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import * as Internal from "./Internal.res.mjs";
|
|
|
9
9
|
import * as Throttler from "./Throttler.res.mjs";
|
|
10
10
|
import * as ChainState from "./ChainState.res.mjs";
|
|
11
11
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
12
|
+
import * as EffectState from "./EffectState.res.mjs";
|
|
12
13
|
import * as LoadManager from "./LoadManager.res.mjs";
|
|
13
14
|
import * as ErrorHandling from "./ErrorHandling.res.mjs";
|
|
14
15
|
import * as InMemoryTable from "./InMemoryTable.res.mjs";
|
|
@@ -20,21 +21,9 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
|
|
|
20
21
|
import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
|
|
21
22
|
import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
|
|
22
23
|
|
|
23
|
-
function make() {
|
|
24
|
-
let intervalMillis = Env.ThrottleWrites.pruneStaleDataIntervalMillis;
|
|
25
|
-
let logger = Logging.createChild({
|
|
26
|
-
context: "Throttler for pruning stale entity history data",
|
|
27
|
-
intervalMillis: intervalMillis
|
|
28
|
-
});
|
|
29
|
-
let pruneStaleEntityHistory = Throttler.make(intervalMillis, logger);
|
|
30
|
-
return {
|
|
31
|
-
pruneStaleEntityHistory: pruneStaleEntityHistory
|
|
32
|
-
};
|
|
33
|
-
}
|
|
34
|
-
|
|
35
24
|
let UndefinedEntity = /* @__PURE__ */Primitive_exceptions.create("IndexerState.EntityTables.UndefinedEntity");
|
|
36
25
|
|
|
37
|
-
function make
|
|
26
|
+
function make(entities) {
|
|
38
27
|
let init = {};
|
|
39
28
|
entities.forEach(entityConfig => {
|
|
40
29
|
init[entityConfig.name] = InMemoryTable.Entity.make();
|
|
@@ -54,7 +43,7 @@ function get(self, entityName) {
|
|
|
54
43
|
}
|
|
55
44
|
}
|
|
56
45
|
|
|
57
|
-
function make$
|
|
46
|
+
function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt, committedCheckpointIdOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError) {
|
|
58
47
|
let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
|
|
59
48
|
let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
|
|
60
49
|
let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
|
|
@@ -69,8 +58,8 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
69
58
|
config: config,
|
|
70
59
|
persistence: persistence,
|
|
71
60
|
allEntities: persistence.allEntities,
|
|
72
|
-
entities: make
|
|
73
|
-
|
|
61
|
+
entities: make(persistence.allEntities),
|
|
62
|
+
effectState: EffectState.make(),
|
|
74
63
|
rollback: undefined,
|
|
75
64
|
committedCheckpointId: committedCheckpointId,
|
|
76
65
|
processedCheckpointId: committedCheckpointId,
|
|
@@ -86,7 +75,7 @@ function make$2(config, persistence, chainStates, isInReorgThreshold, isRealtime
|
|
|
86
75
|
crossChainState: CrossChainState.make(chainStates, isInReorgThreshold, isRealtime, targetBufferSize),
|
|
87
76
|
rollbackState: "NoRollback",
|
|
88
77
|
indexerStartTime: new Date(),
|
|
89
|
-
|
|
78
|
+
lastPrunedAtMillis: {},
|
|
90
79
|
loadManager: LoadManager.make(),
|
|
91
80
|
keepProcessAlive: isDevelopmentMode || shouldUseTui,
|
|
92
81
|
exitAfterFirstEventBlock: exitAfterFirstEventBlock,
|
|
@@ -116,7 +105,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
116
105
|
Prometheus.ReorgThreshold.set(isInReorgThreshold);
|
|
117
106
|
Utils.Dict.forEach(initialState.cache, param => {
|
|
118
107
|
let count = param.count;
|
|
119
|
-
Prometheus.EffectCacheCount.set(count, param.effectName);
|
|
108
|
+
Prometheus.EffectCacheCount.set(count, param.effectName, Internal.EffectCache.scopeToString(param.scope));
|
|
120
109
|
});
|
|
121
110
|
let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
|
|
122
111
|
let chainStates = {};
|
|
@@ -142,7 +131,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
|
|
|
142
131
|
if (allChainsReady.contents) {
|
|
143
132
|
Prometheus.ProgressReady.setAllReady();
|
|
144
133
|
}
|
|
145
|
-
return make$
|
|
134
|
+
return make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError);
|
|
146
135
|
}
|
|
147
136
|
|
|
148
137
|
function isStale(state, stateId) {
|
|
@@ -266,8 +255,8 @@ function entities(state) {
|
|
|
266
255
|
return state.entities;
|
|
267
256
|
}
|
|
268
257
|
|
|
269
|
-
function
|
|
270
|
-
return state.
|
|
258
|
+
function effectState(state) {
|
|
259
|
+
return state.effectState;
|
|
271
260
|
}
|
|
272
261
|
|
|
273
262
|
function committedCheckpointId(state) {
|
|
@@ -346,8 +335,8 @@ function epoch(state) {
|
|
|
346
335
|
return state.epoch;
|
|
347
336
|
}
|
|
348
337
|
|
|
349
|
-
function
|
|
350
|
-
return state.
|
|
338
|
+
function lastPrunedAtMillis(state) {
|
|
339
|
+
return state.lastPrunedAtMillis;
|
|
351
340
|
}
|
|
352
341
|
|
|
353
342
|
function simulateDeadInputTracker(state) {
|
|
@@ -418,8 +407,8 @@ function markCommitted(state, upToCheckpointId) {
|
|
|
418
407
|
}
|
|
419
408
|
|
|
420
409
|
function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId) {
|
|
421
|
-
state.entities = make
|
|
422
|
-
state.
|
|
410
|
+
state.entities = make(state.allEntities);
|
|
411
|
+
EffectState.resetForRollback(state.effectState);
|
|
423
412
|
state.rollback = {
|
|
424
413
|
targetCheckpointId: targetCheckpointId,
|
|
425
414
|
diffCheckpointId: diffCheckpointId,
|
|
@@ -478,7 +467,7 @@ function takeChainMetaSnapshot(state) {
|
|
|
478
467
|
}
|
|
479
468
|
|
|
480
469
|
let EntityTables = {
|
|
481
|
-
make: make
|
|
470
|
+
make: make,
|
|
482
471
|
get: get
|
|
483
472
|
};
|
|
484
473
|
|
|
@@ -486,7 +475,7 @@ let unexpectedErrorMsg = "Indexer has failed with an unexpected error";
|
|
|
486
475
|
|
|
487
476
|
export {
|
|
488
477
|
EntityTables,
|
|
489
|
-
make$
|
|
478
|
+
make$1 as make,
|
|
490
479
|
makeFromDbState,
|
|
491
480
|
unexpectedErrorMsg,
|
|
492
481
|
isStale,
|
|
@@ -512,7 +501,7 @@ export {
|
|
|
512
501
|
persistence,
|
|
513
502
|
allEntities,
|
|
514
503
|
entities,
|
|
515
|
-
|
|
504
|
+
effectState,
|
|
516
505
|
committedCheckpointId,
|
|
517
506
|
processedCheckpointId,
|
|
518
507
|
processedBatches,
|
|
@@ -532,7 +521,7 @@ export {
|
|
|
532
521
|
exitAfterFirstEventBlock,
|
|
533
522
|
isStopped,
|
|
534
523
|
epoch,
|
|
535
|
-
|
|
524
|
+
lastPrunedAtMillis,
|
|
536
525
|
simulateDeadInputTracker,
|
|
537
526
|
queueProcessedBatch,
|
|
538
527
|
drainBatchRun,
|
package/src/IndexerState.resi
CHANGED
|
@@ -16,14 +16,6 @@ module EntityTables: {
|
|
|
16
16
|
let get: (t, ~entityName: string) => InMemoryTable.Entity.t
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
type effectCacheInMemTable = {
|
|
20
|
-
mutable idsToStore: array<string>,
|
|
21
|
-
mutable invalidationsCount: int,
|
|
22
|
-
mutable dict: dict<Change.t<Internal.effectOutput>>,
|
|
23
|
-
mutable changesCount: float,
|
|
24
|
-
effect: Internal.effect,
|
|
25
|
-
}
|
|
26
|
-
|
|
27
19
|
type t
|
|
28
20
|
|
|
29
21
|
let make: (
|
|
@@ -86,7 +78,7 @@ let config: t => Config.t
|
|
|
86
78
|
let persistence: t => Persistence.t
|
|
87
79
|
let allEntities: t => array<Internal.entityConfig>
|
|
88
80
|
let entities: t => EntityTables.t
|
|
89
|
-
let
|
|
81
|
+
let effectState: t => EffectState.t
|
|
90
82
|
let committedCheckpointId: t => Internal.checkpointId
|
|
91
83
|
let processedCheckpointId: t => Internal.checkpointId
|
|
92
84
|
let processedBatches: t => array<Batch.t>
|
|
@@ -106,7 +98,7 @@ let keepProcessAlive: t => bool
|
|
|
106
98
|
let exitAfterFirstEventBlock: t => bool
|
|
107
99
|
let isStopped: t => bool
|
|
108
100
|
let epoch: t => int
|
|
109
|
-
let
|
|
101
|
+
let lastPrunedAtMillis: t => dict<float>
|
|
110
102
|
let simulateDeadInputTracker: t => option<SimulateDeadInputTracker.t>
|
|
111
103
|
|
|
112
104
|
// Store domain operations.
|
package/src/Internal.res
CHANGED
|
@@ -330,6 +330,9 @@ type eventPayload
|
|
|
330
330
|
@get external getPayloadBlock: eventPayload => Nullable.t<eventBlock> = "block"
|
|
331
331
|
@set external setPayloadBlock: (eventPayload, eventBlock) => unit = "block"
|
|
332
332
|
|
|
333
|
+
// The log's emitting address (EVM/Fuel; the program id carries it for SVM).
|
|
334
|
+
@get external getPayloadSrcAddress: eventPayload => Address.t = "srcAddress"
|
|
335
|
+
|
|
333
336
|
type genericLoaderArgs<'event, 'context> = {
|
|
334
337
|
event: 'event,
|
|
335
338
|
context: 'context,
|
|
@@ -459,15 +462,34 @@ type topicSelection = {
|
|
|
459
462
|
topic3: array<EvmTypes.Hex.t>,
|
|
460
463
|
}
|
|
461
464
|
|
|
465
|
+
// A single topic position of a resolved `where`: either static pre-encoded
|
|
466
|
+
// values, or a marker for "the currently registered addresses of this
|
|
467
|
+
// contract", expanded to topic values when a source query is built.
|
|
468
|
+
type topicFilter =
|
|
469
|
+
| Values(array<EvmTypes.Hex.t>)
|
|
470
|
+
| ContractAddresses({contractName: string})
|
|
471
|
+
|
|
472
|
+
type resolvedTopicSelection = {
|
|
473
|
+
topic0: array<EvmTypes.Hex.t>,
|
|
474
|
+
topic1: topicFilter,
|
|
475
|
+
topic2: topicFilter,
|
|
476
|
+
topic3: topicFilter,
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// The registered `where` fully resolved at registration time for one chain.
|
|
480
|
+
// `topicSelections` is in disjunctive normal form (outer array is OR);
|
|
481
|
+
// an empty array means the `where` returned `false` for this chain.
|
|
482
|
+
type resolvedWhere = {
|
|
483
|
+
topicSelections: array<resolvedTopicSelection>,
|
|
484
|
+
startBlock: option<int>,
|
|
485
|
+
}
|
|
486
|
+
|
|
462
487
|
// Per-event, per-invocation arguments passed to a `where` callback. The
|
|
463
488
|
// concrete `chain` shape (which contract key it exposes) is generated per
|
|
464
489
|
// event in user-project codegen — here it's an open record so codegen'd
|
|
465
490
|
// types subtype-coerce into it cleanly.
|
|
466
491
|
type onEventWhereArgs<'chain> = {chain: 'chain}
|
|
467
492
|
|
|
468
|
-
type eventFilters =
|
|
469
|
-
Static(array<topicSelection>) | Dynamic(array<Address.t> => array<topicSelection>)
|
|
470
|
-
|
|
471
493
|
type evmEventConfig = {
|
|
472
494
|
...eventConfig,
|
|
473
495
|
selectedBlockFields: Utils.Set.t<evmBlockField>,
|
|
@@ -536,6 +558,11 @@ type svmInstructionEventConfig = {
|
|
|
536
558
|
// must stay directly constructable), and the evm→base cast in sources is sound
|
|
537
559
|
// by ecosystem homogeneity — an EVM chain only ever holds `evmOnEventRegistration`s.
|
|
538
560
|
type onEventRegistration = {
|
|
561
|
+
// Chain-scoped sequential index — the registration's position in the
|
|
562
|
+
// chain's onEventRegistrations array, assigned when registration finishes
|
|
563
|
+
// (-1 until then). Native-routed items reference their registration by this
|
|
564
|
+
// index across the napi boundary; sources resolve it before creating an item.
|
|
565
|
+
index: int,
|
|
539
566
|
eventConfig: eventConfig,
|
|
540
567
|
handler: option<handler>,
|
|
541
568
|
contractRegister: option<contractRegister>,
|
|
@@ -557,7 +584,7 @@ type onEventRegistration = {
|
|
|
557
584
|
|
|
558
585
|
type evmOnEventRegistration = {
|
|
559
586
|
...onEventRegistration,
|
|
560
|
-
|
|
587
|
+
resolvedWhere: resolvedWhere,
|
|
561
588
|
}
|
|
562
589
|
|
|
563
590
|
// Fuel and SVM registrations add no ecosystem-specific fetch state (their
|
|
@@ -582,8 +609,9 @@ type indexingAddress = {
|
|
|
582
609
|
|
|
583
610
|
type dcs = array<indexingAddress>
|
|
584
611
|
|
|
585
|
-
// Duplicate the type from item
|
|
586
|
-
//
|
|
612
|
+
// Duplicate the type from item to keep item properly unboxed. Runtime event
|
|
613
|
+
// items carry the registration their source already resolved from the
|
|
614
|
+
// ChainState-owned registration array.
|
|
587
615
|
type eventItem = private {
|
|
588
616
|
kind: [#0],
|
|
589
617
|
onEventRegistration: onEventRegistration,
|
|
@@ -724,33 +752,108 @@ type effectCacheItem = {id: string, output: effectOutput}
|
|
|
724
752
|
type effectCacheStorageMeta = {
|
|
725
753
|
itemSchema: S.t<effectCacheItem>,
|
|
726
754
|
outputSchema: S.t<effectOutput>,
|
|
727
|
-
table: Table.table,
|
|
728
755
|
}
|
|
729
|
-
type
|
|
756
|
+
type rateLimitOptions = {
|
|
730
757
|
callsPerDuration: int,
|
|
731
758
|
durationMs: int,
|
|
732
|
-
mutable availableCalls: int,
|
|
733
|
-
mutable windowStartTime: float,
|
|
734
|
-
mutable queueCount: int,
|
|
735
|
-
mutable nextWindowPromise: option<promise<unit>>,
|
|
736
759
|
}
|
|
737
760
|
type effect = {
|
|
738
761
|
name: string,
|
|
739
762
|
handler: effectArgs => promise<effectOutput>,
|
|
740
763
|
storageMeta: effectCacheStorageMeta,
|
|
741
764
|
defaultShouldCache: bool,
|
|
765
|
+
// When true (the default) a single cache is shared across every chain and the
|
|
766
|
+
// handler must not read context.chain. When false the cache is isolated per
|
|
767
|
+
// chain and context.chain.id is available.
|
|
768
|
+
crossChain: bool,
|
|
742
769
|
output: S.t<effectOutput>,
|
|
743
770
|
input: S.t<effectInput>,
|
|
744
|
-
|
|
745
|
-
mutable activeCallsCount: int,
|
|
746
|
-
mutable prevCallStartTimerRef: Performance.timeRef,
|
|
747
|
-
rateLimit: option<rateLimitState>,
|
|
771
|
+
rateLimit: option<rateLimitOptions>,
|
|
748
772
|
}
|
|
773
|
+
|
|
774
|
+
// Whether some piece of data (currently an effect cache; entities in a future
|
|
775
|
+
// version) is shared across every chain or isolated to a single chain. Unboxed:
|
|
776
|
+
// `CrossChain` is the string "crossChain" and `Chain(id)` is the raw chain id,
|
|
777
|
+
// discriminated by runtime type.
|
|
778
|
+
@unboxed
|
|
779
|
+
type chainScope =
|
|
780
|
+
| @as("crossChain") CrossChain
|
|
781
|
+
| Chain(int)
|
|
782
|
+
|
|
749
783
|
let cacheTablePrefix = "envio_effect_"
|
|
784
|
+
|
|
785
|
+
// The single reversible mapping between an effect's (name, scope) and its
|
|
786
|
+
// canonical Postgres cache-table name and .envio/cache file path. Everything
|
|
787
|
+
// that needs a cache address goes through here instead of slicing prefixes.
|
|
788
|
+
// CrossChain -> envio_effect_<name> <name>.tsv
|
|
789
|
+
// Chain(1) -> envio_1_effect_<name> 1/<name>.tsv
|
|
790
|
+
// Chain(137) -> envio_137_effect_<name> 137/<name>.tsv
|
|
791
|
+
module EffectCache = {
|
|
792
|
+
let toTableName = (~effectName, ~scope) =>
|
|
793
|
+
switch scope {
|
|
794
|
+
| CrossChain => cacheTablePrefix ++ effectName
|
|
795
|
+
| Chain(chainId) => `envio_${chainId->Int.toString}_effect_${effectName}`
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
|
|
799
|
+
let scopeToString = scope =>
|
|
800
|
+
switch scope {
|
|
801
|
+
| CrossChain => "crossChain"
|
|
802
|
+
| Chain(chainId) => chainId->Int.toString
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
|
|
806
|
+
// Int.fromString alone follows parseInt semantics and accepts both.
|
|
807
|
+
let parseChainId = str =>
|
|
808
|
+
switch Int.fromString(str) {
|
|
809
|
+
| Some(chainId) if chainId >= 0 && chainId->Int.toString === str => Some(chainId)
|
|
810
|
+
| _ => None
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
let chainScopedRe = /^envio_([0-9]+)_effect_(.+)$/
|
|
814
|
+
let crossChainRe = /^envio_effect_(.+)$/
|
|
815
|
+
|
|
816
|
+
// Inverse of toTableName. Returns None for any table name that isn't a cache
|
|
817
|
+
// table. Chain-scoped is tried first: the `_effect_` separator keeps effect
|
|
818
|
+
// names that themselves start with digits unambiguous.
|
|
819
|
+
let fromTableName = (tableName): option<(string, chainScope)> =>
|
|
820
|
+
switch RegExp.exec(chainScopedRe, tableName) {
|
|
821
|
+
| Some(result) =>
|
|
822
|
+
switch (
|
|
823
|
+
RegExp.Result.matches(result)->Array.get(0),
|
|
824
|
+
RegExp.Result.matches(result)->Array.get(1),
|
|
825
|
+
) {
|
|
826
|
+
| (Some(Some(chainIdStr)), Some(Some(effectName))) =>
|
|
827
|
+
switch parseChainId(chainIdStr) {
|
|
828
|
+
| Some(chainId) => Some((effectName, Chain(chainId)))
|
|
829
|
+
| None => None
|
|
830
|
+
}
|
|
831
|
+
| _ => None
|
|
832
|
+
}
|
|
833
|
+
| None =>
|
|
834
|
+
switch RegExp.exec(crossChainRe, tableName) {
|
|
835
|
+
| Some(result) =>
|
|
836
|
+
switch RegExp.Result.matches(result)->Array.get(0) {
|
|
837
|
+
| Some(Some(effectName)) => Some((effectName, CrossChain))
|
|
838
|
+
| _ => None
|
|
839
|
+
}
|
|
840
|
+
| None => None
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
// Relative posix path within .envio/cache. Chain-scoped caches live one
|
|
845
|
+
// directory level deep, named by chain id.
|
|
846
|
+
let toCachePath = (~effectName, ~scope) =>
|
|
847
|
+
switch scope {
|
|
848
|
+
| CrossChain => effectName ++ ".tsv"
|
|
849
|
+
| Chain(chainId) => `${chainId->Int.toString}/${effectName}.tsv`
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
750
853
|
let cacheOutputSchema = S.json(~validate=false)->(Utils.magic: S.t<JSON.t> => S.t<effectOutput>)
|
|
751
|
-
let makeCacheTable = (~effectName) => {
|
|
854
|
+
let makeCacheTable = (~effectName, ~scope) => {
|
|
752
855
|
Table.mkTable(
|
|
753
|
-
|
|
856
|
+
EffectCache.toTableName(~effectName, ~scope),
|
|
754
857
|
~fields=[
|
|
755
858
|
Table.mkField("id", String, ~fieldSchema=S.string, ~isPrimaryKey=true),
|
|
756
859
|
Table.mkField("output", Json, ~fieldSchema=cacheOutputSchema, ~isNullable=true),
|